Repository: OpenListTeam/OpenList Branch: main Commit: 9a2ba1dabe3a Files: 744 Total size: 3.2 MB Directory structure: gitextract_q94m8ixv/ ├── .air.toml ├── .github/ │ ├── FUNDING.yml │ ├── ISSUE_TEMPLATE/ │ │ ├── 00-bug_report_zh.yml │ │ ├── 01-bug_report_en.yml │ │ ├── 02-feature_request_zh.yml │ │ ├── 03-feature_request_en.yml │ │ └── config.yml │ ├── PULL_REQUEST_TEMPLATE.md │ └── workflows/ │ ├── beta_release.yml │ ├── build.yml │ ├── changelog.yml │ ├── issue_pr_comment.yml │ ├── release.yml │ ├── release_docker.yml │ ├── sync_repo.yml │ ├── test_docker.yml │ └── trigger-makefile-update.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── Dockerfile.ci ├── LICENSE ├── README.md ├── README_cn.md ├── README_ja.md ├── README_nl.md ├── SECURITY.md ├── build.sh ├── cmd/ │ ├── admin.go │ ├── cancel2FA.go │ ├── common.go │ ├── crypt.go │ ├── flags/ │ │ └── config.go │ ├── kill.go │ ├── lang.go │ ├── restart.go │ ├── root.go │ ├── server.go │ ├── start.go │ ├── stop_default.go │ ├── stop_windows.go │ ├── storage.go │ ├── user.go │ └── version.go ├── docker-compose.yml ├── drivers/ │ ├── 115/ │ │ ├── appver.go │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── 115_open/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ ├── upload.go │ │ └── util.go │ ├── 115_share/ │ │ ├── driver.go │ │ ├── meta.go │ │ └── utils.go │ ├── 123/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ ├── upload.go │ │ └── util.go │ ├── 123_link/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── parse.go │ │ ├── types.go │ │ └── util.go │ ├── 123_open/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── token.go │ │ ├── types.go │ │ ├── upload.go │ │ └── util.go │ ├── 123_share/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── 139/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── 189/ │ │ ├── driver.go │ │ ├── help.go │ │ ├── login.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── 189_tv/ │ │ ├── driver.go │ │ ├── help.go │ │ ├── meta.go │ │ ├── types.go │ │ └── utils.go │ ├── 189pc/ │ │ ├── driver.go │ │ ├── help.go │ │ ├── meta.go │ │ ├── types.go │ │ └── utils.go │ ├── alias/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── alist_v3/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── aliyundrive/ │ │ ├── driver.go │ │ ├── global.go │ │ ├── help.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── aliyundrive_open/ │ │ ├── driver.go │ │ ├── limiter.go │ │ ├── meta.go │ │ ├── types.go │ │ ├── upload.go │ │ └── util.go │ ├── aliyundrive_share/ │ │ ├── driver.go │ │ ├── limiter.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── all.go │ ├── autoindex/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ ├── util.go │ │ └── util_test.go │ ├── azure_blob/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── baidu_netdisk/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── baidu_photo/ │ │ ├── driver.go │ │ ├── help.go │ │ ├── meta.go │ │ ├── types.go │ │ └── utils.go │ ├── base/ │ │ ├── client.go │ │ ├── types.go │ │ ├── upload.go │ │ └── util.go │ ├── chaoxing/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── chunk/ │ │ ├── driver.go │ │ ├── meta.go │ │ └── obj.go │ ├── cloudreve/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── cloudreve_v4/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── cnb_releases/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── crypt/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── degoo/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ ├── upload.go │ │ └── util.go │ ├── doubao/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── doubao_share/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── dropbox/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── febbox/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── oauth2.go │ │ ├── types.go │ │ └── util.go │ ├── ftp/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── github/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── github_releases/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── models.go │ │ ├── types.go │ │ └── util.go │ ├── google_drive/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── google_photo/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── halalcloud/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── options.go │ │ ├── types.go │ │ └── util.go │ ├── halalcloud_open/ │ │ ├── common.go │ │ ├── driver.go │ │ ├── driver_curd_impl.go │ │ ├── driver_get_link.go │ │ ├── driver_init.go │ │ ├── driver_interface.go │ │ ├── halalcloud_upload.go │ │ ├── meta.go │ │ ├── obj_file.go │ │ └── utils.go │ ├── ilanzou/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── ipfs_api/ │ │ ├── driver.go │ │ └── meta.go │ ├── kodbox/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── lanzou/ │ │ ├── driver.go │ │ ├── help.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── lenovonas_share/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── local/ │ │ ├── benchmark_calculatedirsize_test.go │ │ ├── copy_namedpipes.go │ │ ├── copy_namedpipes_x.go │ │ ├── driver.go │ │ ├── meta.go │ │ ├── token_bucket.go │ │ ├── util.go │ │ ├── util_unix.go │ │ └── util_windows.go │ ├── mediafire/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── mediatrack/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── mega/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── misskey/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── mopan/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── netease_music/ │ │ ├── crypto.go │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ ├── upload.go │ │ └── util.go │ ├── onedrive/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── onedrive_app/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── onedrive_sharelink/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── openlist/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── openlist_share/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── pikpak/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── pikpak_share/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── proton_drive/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── quark_open/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── quark_uc/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── quark_uc_tv/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── s3/ │ │ ├── doge.go │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── seafile/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── sftp/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── smb/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── strm/ │ │ ├── driver.go │ │ ├── hook.go │ │ ├── meta.go │ │ └── util.go │ ├── teambition/ │ │ ├── driver.go │ │ ├── help.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── teldrive/ │ │ ├── copy.go │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ ├── upload.go │ │ └── util.go │ ├── template/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── terabox/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── thunder/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── thunder_browser/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── thunderx/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── url_tree/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ ├── urls_test.go │ │ └── util.go │ ├── uss/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── virtual/ │ │ ├── driver.go │ │ ├── meta.go │ │ └── util.go │ ├── webdav/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── odrvcookie/ │ │ │ ├── cookie.go │ │ │ └── fetch.go │ │ ├── types.go │ │ └── util.go │ ├── weiyun/ │ │ ├── driver.go │ │ ├── meta.go │ │ └── types.go │ ├── wopan/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ ├── wps/ │ │ ├── driver.go │ │ ├── meta.go │ │ ├── types.go │ │ └── util.go │ └── yandex_disk/ │ ├── driver.go │ ├── meta.go │ ├── types.go │ └── util.go ├── entrypoint.sh ├── go.mod ├── go.sum ├── internal/ │ ├── archive/ │ │ ├── all.go │ │ ├── archives/ │ │ │ ├── archives.go │ │ │ └── utils.go │ │ ├── iso9660/ │ │ │ ├── iso9660.go │ │ │ └── utils.go │ │ ├── rardecode/ │ │ │ ├── rardecode.go │ │ │ └── utils.go │ │ ├── sevenzip/ │ │ │ ├── sevenzip.go │ │ │ └── utils.go │ │ ├── tool/ │ │ │ ├── base.go │ │ │ ├── helper.go │ │ │ └── utils.go │ │ └── zip/ │ │ ├── utils.go │ │ └── zip.go │ ├── authn/ │ │ └── authn.go │ ├── bootstrap/ │ │ ├── config.go │ │ ├── data/ │ │ │ ├── data.go │ │ │ ├── dev.go │ │ │ ├── setting.go │ │ │ ├── task.go │ │ │ └── user.go │ │ ├── db.go │ │ ├── index.go │ │ ├── log.go │ │ ├── offline_download.go │ │ ├── patch/ │ │ │ ├── all.go │ │ │ ├── v3_24_0/ │ │ │ │ └── hash_password.go │ │ │ ├── v3_32_0/ │ │ │ │ └── update_authn.go │ │ │ ├── v3_41_0/ │ │ │ │ └── grant_permission.go │ │ │ ├── v4_1_8/ │ │ │ │ └── alias.go │ │ │ └── v4_1_9/ │ │ │ ├── skip_tls.go │ │ │ └── webdav.go │ │ ├── patch.go │ │ ├── run.go │ │ ├── storage.go │ │ ├── stream_limit.go │ │ └── task.go │ ├── cache/ │ │ ├── keyed_cache.go │ │ ├── type.go │ │ ├── typed_cache.go │ │ └── utils.go │ ├── conf/ │ │ ├── config.go │ │ ├── const.go │ │ └── var.go │ ├── db/ │ │ ├── db.go │ │ ├── meta.go │ │ ├── searchnode.go │ │ ├── settingitem.go │ │ ├── sharing.go │ │ ├── sshkey.go │ │ ├── storage.go │ │ ├── tasks.go │ │ ├── user.go │ │ └── util.go │ ├── driver/ │ │ ├── config.go │ │ ├── driver.go │ │ ├── item.go │ │ └── utils.go │ ├── errs/ │ │ ├── driver.go │ │ ├── errors.go │ │ ├── errors_test.go │ │ ├── object.go │ │ ├── operate.go │ │ ├── search.go │ │ ├── unwrap.go │ │ └── user.go │ ├── fs/ │ │ ├── archive.go │ │ ├── copy_move.go │ │ ├── fs.go │ │ ├── get.go │ │ ├── link.go │ │ ├── list.go │ │ ├── other.go │ │ ├── put.go │ │ └── walk.go │ ├── fuse/ │ │ ├── fs.go │ │ └── mount.go │ ├── message/ │ │ ├── http.go │ │ ├── message.go │ │ └── ws.go │ ├── model/ │ │ ├── archive.go │ │ ├── args.go │ │ ├── direct_upload.go │ │ ├── file.go │ │ ├── meta.go │ │ ├── obj.go │ │ ├── object.go │ │ ├── req.go │ │ ├── search.go │ │ ├── setting.go │ │ ├── sharing.go │ │ ├── sshkey.go │ │ ├── storage.go │ │ ├── task.go │ │ └── user.go │ ├── net/ │ │ ├── oss.go │ │ ├── oss_test.go │ │ ├── request.go │ │ ├── request_test.go │ │ ├── serve.go │ │ └── util.go │ ├── offline_download/ │ │ ├── 115/ │ │ │ └── client.go │ │ ├── 115_open/ │ │ │ └── client.go │ │ ├── 123/ │ │ │ └── client.go │ │ ├── 123_open/ │ │ │ └── client.go │ │ ├── all.go │ │ ├── aria2/ │ │ │ ├── aria2.go │ │ │ └── notify.go │ │ ├── http/ │ │ │ ├── client.go │ │ │ └── util.go │ │ ├── pikpak/ │ │ │ ├── pikpak.go │ │ │ └── util.go │ │ ├── qbit/ │ │ │ └── qbit.go │ │ ├── thunder/ │ │ │ ├── thunder.go │ │ │ └── util.go │ │ ├── thunder_browser/ │ │ │ ├── thunder_browser.go │ │ │ └── util.go │ │ ├── thunderx/ │ │ │ ├── thunderx.go │ │ │ └── utils.go │ │ ├── tool/ │ │ │ ├── add.go │ │ │ ├── base.go │ │ │ ├── download.go │ │ │ ├── tools.go │ │ │ └── transfer.go │ │ └── transmission/ │ │ └── client.go │ ├── op/ │ │ ├── archive.go │ │ ├── cache.go │ │ ├── const.go │ │ ├── driver.go │ │ ├── driver_test.go │ │ ├── fs.go │ │ ├── hook.go │ │ ├── meta.go │ │ ├── path.go │ │ ├── recursive_list.go │ │ ├── setting.go │ │ ├── sharing.go │ │ ├── sshkey.go │ │ ├── storage.go │ │ ├── storage_test.go │ │ └── user.go │ ├── search/ │ │ ├── bleve/ │ │ │ ├── init.go │ │ │ └── search.go │ │ ├── build.go │ │ ├── db/ │ │ │ ├── init.go │ │ │ └── search.go │ │ ├── db_non_full_text/ │ │ │ ├── init.go │ │ │ └── search.go │ │ ├── import.go │ │ ├── meilisearch/ │ │ │ ├── init.go │ │ │ ├── search.go │ │ │ ├── task_queue.go │ │ │ └── utils.go │ │ ├── search.go │ │ ├── searcher/ │ │ │ ├── manage.go │ │ │ └── searcher.go │ │ └── util.go │ ├── setting/ │ │ └── setting.go │ ├── sharing/ │ │ ├── archive.go │ │ ├── get.go │ │ ├── link.go │ │ ├── list.go │ │ └── sharing.go │ ├── sign/ │ │ ├── archive.go │ │ └── sign.go │ ├── stream/ │ │ ├── limit.go │ │ ├── stream.go │ │ ├── stream_test.go │ │ └── util.go │ ├── task/ │ │ ├── base.go │ │ └── manager.go │ └── task_group/ │ ├── group.go │ └── transfer.go ├── main.go ├── pkg/ │ ├── aria2/ │ │ └── rpc/ │ │ ├── README.md │ │ ├── call.go │ │ ├── call_test.go │ │ ├── client.go │ │ ├── client_test.go │ │ ├── const.go │ │ ├── json2.go │ │ ├── notification.go │ │ ├── proc.go │ │ ├── proto.go │ │ └── resp.go │ ├── buffer/ │ │ ├── bytes.go │ │ ├── bytes_test.go │ │ └── file.go │ ├── chanio/ │ │ └── chanio.go │ ├── cookie/ │ │ └── cookie.go │ ├── cron/ │ │ ├── cron.go │ │ └── cron_test.go │ ├── errgroup/ │ │ └── errgroup.go │ ├── generic/ │ │ └── queue.go │ ├── generic_sync/ │ │ ├── map.go │ │ └── map_test.go │ ├── http_range/ │ │ └── range.go │ ├── mq/ │ │ └── mq.go │ ├── pool/ │ │ └── pool.go │ ├── qbittorrent/ │ │ └── client.go │ ├── sign/ │ │ ├── hmac.go │ │ └── sign.go │ ├── singleflight/ │ │ ├── signleflight_test.go │ │ ├── singleflight.go │ │ └── var.go │ ├── task/ │ │ ├── errors.go │ │ ├── manager.go │ │ ├── task.go │ │ └── task_test.go │ └── utils/ │ ├── balance.go │ ├── bool.go │ ├── ctx.go │ ├── email.go │ ├── file.go │ ├── file_test.go │ ├── hash/ │ │ └── gcid.go │ ├── hash.go │ ├── hash_test.go │ ├── html.go │ ├── http.go │ ├── io.go │ ├── ip.go │ ├── json.go │ ├── log.go │ ├── map.go │ ├── oauth2.go │ ├── path.go │ ├── path_test.go │ ├── random/ │ │ └── random.go │ ├── slice.go │ ├── str.go │ ├── time.go │ └── url.go ├── public/ │ └── public.go ├── server/ │ ├── common/ │ │ ├── auth.go │ │ ├── base.go │ │ ├── check.go │ │ ├── check_test.go │ │ ├── common.go │ │ ├── hide_privacy_test.go │ │ ├── ldap.go │ │ ├── proxy.go │ │ ├── resp.go │ │ └── sign.go │ ├── debug.go │ ├── ftp/ │ │ ├── afero.go │ │ ├── fsmanage.go │ │ ├── fsread.go │ │ ├── fsup.go │ │ ├── site.go │ │ └── upload_stage.go │ ├── ftp.go │ ├── handles/ │ │ ├── archive.go │ │ ├── auth.go │ │ ├── const.go │ │ ├── direct_upload.go │ │ ├── down.go │ │ ├── driver.go │ │ ├── fsbatch.go │ │ ├── fsmanage.go │ │ ├── fsread.go │ │ ├── fsup.go │ │ ├── helper.go │ │ ├── index.go │ │ ├── ldap_login.go │ │ ├── meta.go │ │ ├── offline_download.go │ │ ├── scan.go │ │ ├── search.go │ │ ├── setting.go │ │ ├── sharing.go │ │ ├── sshkey.go │ │ ├── ssologin.go │ │ ├── storage.go │ │ ├── task.go │ │ ├── user.go │ │ └── webauthn.go │ ├── middlewares/ │ │ ├── auth.go │ │ ├── check.go │ │ ├── down.go │ │ ├── filtered_logger.go │ │ ├── fsup.go │ │ ├── https.go │ │ ├── limit.go │ │ ├── search.go │ │ └── sharing.go │ ├── router.go │ ├── s3/ │ │ ├── backend.go │ │ ├── ioutils.go │ │ ├── list.go │ │ ├── logger.go │ │ ├── pager.go │ │ ├── server.go │ │ └── utils.go │ ├── s3.go │ ├── sftp/ │ │ ├── const.go │ │ ├── hostkey.go │ │ └── sftp.go │ ├── sftp.go │ ├── static/ │ │ ├── config.go │ │ └── static.go │ ├── utils.go │ ├── webdav/ │ │ ├── buffered_response_writer.go │ │ ├── file.go │ │ ├── if.go │ │ ├── internal/ │ │ │ └── xml/ │ │ │ ├── README │ │ │ ├── atom_test.go │ │ │ ├── example_test.go │ │ │ ├── marshal.go │ │ │ ├── marshal_test.go │ │ │ ├── read.go │ │ │ ├── read_test.go │ │ │ ├── typeinfo.go │ │ │ ├── xml.go │ │ │ └── xml_test.go │ │ ├── litmus_test_server.go │ │ ├── lock.go │ │ ├── lock_test.go │ │ ├── prop.go │ │ ├── util.go │ │ ├── webdav.go │ │ ├── xml.go │ │ └── xml_test.go │ └── webdav.go └── wrapper/ ├── zcc-arm64 ├── zcc-win7 ├── zcc-win7-386 ├── zcxx-arm64 ├── zcxx-win7 └── zcxx-win7-386 ================================================ FILE CONTENTS ================================================ ================================================ FILE: .air.toml ================================================ root = "." testdata_dir = "testdata" tmp_dir = "tmp" [build] args_bin = ["server"] bin = "./tmp/main" cmd = "go build -o ./tmp/main ." delay = 0 exclude_dir = ["assets", "tmp", "vendor", "testdata"] exclude_file = [] exclude_regex = ["_test.go"] exclude_unchanged = false follow_symlink = false full_bin = "" include_dir = [] include_ext = ["go", "tpl", "tmpl", "html"] include_file = [] kill_delay = "0s" log = "build-errors.log" poll = false poll_interval = 0 rerun = false rerun_delay = 500 send_interrupt = false stop_on_error = false [color] app = "" build = "yellow" main = "magenta" runner = "green" watcher = "cyan" [log] main_only = false time = false [misc] clean_on_exit = false [screen] clear_on_rebuild = false keep_scroll = true ================================================ FILE: .github/FUNDING.yml ================================================ # These are supported funding model platforms github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username otechie: # Replace with a single Otechie username lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry custom: [] ================================================ FILE: .github/ISSUE_TEMPLATE/00-bug_report_zh.yml ================================================ name: "错误报告" description: 错误报告 / 问题 title: "[BUG] 请修改标题为您遇到的问题" labels: [bug] body: - type: markdown attributes: value: | 感谢您花时间填写此错误报告。 请**务必**确认您的问题无重复，且不是因为您的操作、网络或第三方软件问题。 - type: checkboxes attributes: label: 请确认以下事项 description: | 您必须阅读、检查、确认、同意以下内容，否则您的问题一定会被直接关闭。 或者您可以去[讨论区](https://github.com/OpenListTeam/OpenList/discussions)。 options: - label: | 我已确认阅读并同意 [AGPL-3.0 第15条](https://www.gnu.org/licenses/agpl-3.0.txt#:~:text=15.%20Disclaimer%20of%20Warranty.) 。 本程序不提供任何明示或暗示的担保，使用风险由您自行承担。 - label: | 我已确认阅读并同意 [AGPL-3.0 第16条](https://www.gnu.org/licenses/agpl-3.0.txt#:~:text=16.%20Limitation%20of%20Liability.) 。 无论何种情况，版权持有人或其他分发者均不对使用本程序所造成的任何损失承担责任。 - label: | 我确认我的描述清晰，语法礼貌，能帮助开发者快速定位问题，并符合社区规则。 - label: | 我已确认阅读了[OpenList文档](https://doc.oplist.org)。 - label: | 我已确认没有重复的问题或讨论。 - label: | 我已确认是`OpenList`的问题，而不是其他原因（例如 [网络](https://doc.oplist.org/faq/howto#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host-1) ，`依赖`或`操作`）。 - label: | 我认为此问题必须由`OpenList`处理，而非第三方。 - label: | 我已确认这个问题在最新版本中没有被修复。 - label: | 我没有阅读这个清单，只是闭眼选中了所有的复选框，请关闭这个 Issue 。 - type: input id: version attributes: label: OpenList 版本（必填） description: | 您使用的是哪个版本的软件？请不要使用`latest`或`master`作为答案。 placeholder: v4.xx.xx validations: required: true - type: input id: driver attributes: label: 使用的存储驱动（必填） description: | 您使用的是哪个存储驱动？ placeholder: "例如: OneDrive" validations: required: true - type: textarea id: bug-description attributes: label: 问题描述（必填） validations: required: true - type: textarea id: logs attributes: label: 日志（必填） description: | 请复制粘贴错误日志，或者截图。(可隐藏隐私字段) [查看方法](https://doc.oplist.org/faq/howto#%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%9A%E4%BD%8Dbug) validations: required: true - type: textarea id: config attributes: label: 配置文件内容（必填） description: | 请提供您的`OpenList`应用的配置文件，并截图相关存储配置。(可隐藏隐私字段) validations: required: true - type: textarea id: reproduction attributes: label: 复现链接（可选） description: | 请提供能复现此问题的链接。 ================================================ FILE: .github/ISSUE_TEMPLATE/01-bug_report_en.yml ================================================ name: "Bug Report" description: Bug Report / Issue title: "[BUG] Please modify the title to describe the issue you are facing" labels: [bug] body: - type: markdown attributes: value: | Thank you for taking the time to fill out this bug report. Please **make sure** your issue is not a duplicate and is not caused by your own operation, network, or third-party software. - type: checkboxes attributes: label: Please confirm the following description: | You must read, check, confirm, and agree to all the following, otherwise your issue will definitely be closed directly. Or you can go to the [discussions](https://github.com/OpenListTeam/OpenList/discussions). options: - label: | I have read and agree to [AGPL-3.0 Section 15](https://www.gnu.org/licenses/agpl-3.0.txt#:~:text=15.%20Disclaimer%20of%20Warranty.) . The program is provided "as is" without any warranties; you bear all risks of using it. - label: | I have read and agree to [AGPL-3.0 Section 16](https://www.gnu.org/licenses/agpl-3.0.txt#:~:text=16.%20Limitation%20of%20Liability.) . The copyright holders and distributors are not liable for any damages resulting from the use or inability to use the program. - label: | I confirm my description is clear, polite, helps developers quickly locate the issue, and complies with community rules. - label: | I have read the [OpenList documentation](https://doc.oplist.org). - label: | I confirm there are no duplicate issues or discussions. - label: | I confirm this is an `OpenList` issue, not caused by other reasons (such as [network](https://doc.oplist.org/faq/howto#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host-1), dependencies, or operation). - label: | I believe this issue must be handled by `OpenList` and not by a third party. - label: | I confirm this issue is not fixed in the latest version. - label: | I have not read these checkboxes and therefore I just ticked them all, Please close this issue. - type: input id: version attributes: label: OpenList Version (required) description: | What version of the software are you using? Please do not use `latest` or `master` as the answer. placeholder: v4.xx.xx validations: required: true - type: input id: driver attributes: label: Storage Driver Used (required) description: | Which storage driver are you using? placeholder: "e.g. OneDrive" validations: required: true - type: textarea id: bug-description attributes: label: Bug Description (required) validations: required: true - type: textarea id: logs attributes: label: Logs (required) description: | Please copy and paste any relevant log output or screenshots. (You may mask sensitive fields) [Guide](https://doc.oplist.org/faq/howto#how-to-quickly-locate-bugs) validations: required: true - type: textarea id: config attributes: label: Configuration File Content (required) description: | Please provide your `OpenList` application's configuration file and a screenshot of the relevant storage configuration. (You may mask sensitive fields) validations: required: true - type: textarea id: reproduction attributes: label: Reproduction Link (optional) description: | Please provide a link to a repo or page that can reproduce this issue. ================================================ FILE: .github/ISSUE_TEMPLATE/02-feature_request_zh.yml ================================================ name: "功能请求" description: 功能请求 / 增强 title: "[Feature] 请修改标题为您的功能名称" labels: [enhancement] body: - type: checkboxes attributes: label: 请确认以下事项 description: | 您必须阅读、检查、确认、同意以下内容，否则您的问题可能会被直接关闭。 或者您可以去[讨论区](https://github.com/OpenListTeam/OpenList/discussions)。 options: - label: | 我已确认阅读并同意 [AGPL-3.0 第15条](https://www.gnu.org/licenses/agpl-3.0.txt#:~:text=15.%20Disclaimer%20of%20Warranty.) 。 本程序不提供任何明示或暗示的担保，使用风险由您自行承担。 - label: | 我已确认阅读并同意 [AGPL-3.0 第16条](https://www.gnu.org/licenses/agpl-3.0.txt#:~:text=16.%20Limitation%20of%20Liability.) 。 无论何种情况，版权持有人或其他分发者均不对使用本程序所造成的任何损失承担责任。 - label: | 我确认我的描述清晰，语法礼貌，能帮助开发者快速定位问题，并符合社区规则。 - label: | 我已确认阅读了[OpenList文档](https://doc.oplist.org)。 - label: | 我已确认没有重复的问题或讨论。 - label: | 我认为此问题必须由`OpenList`处理，而非第三方。 - label: | 我已确认此功能尚未被实现。 - label: | 我已确认此功能是合理的，且有普遍需求，并非我个人需要。 - label: | 我没有阅读这个清单，只是闭眼选中了所有的复选框，请关闭这个 Issue 。 - type: textarea id: feature-description attributes: label: 需求描述 validations: required: true - type: textarea id: suggested-solution attributes: label: 实现思路 description: | 实现此需求的解决思路。 - type: textarea id: additional-context attributes: label: 附加信息 description: | 相关的任何其他上下文或截图，或者你觉得有帮助的信息 ================================================ FILE: .github/ISSUE_TEMPLATE/03-feature_request_en.yml ================================================ name: "Feature Request" description: Feature Request / Enhancement title: "[Feature] Please modify the title to your feature name" labels: [enhancement] body: - type: checkboxes attributes: label: Please confirm the following description: | You must read, check, confirm, and agree to all the following, otherwise your request may be closed directly. Or you can go to the [discussions](https://github.com/OpenListTeam/OpenList/discussions). options: - label: | I have read and agree to [AGPL-3.0 Section 15](https://www.gnu.org/licenses/agpl-3.0.txt#:~:text=15.%20Disclaimer%20of%20Warranty.). The program is provided "as is" without any warranties; you bear all risks of using it. - label: | I have read and agree to [AGPL-3.0 Section 16](https://www.gnu.org/licenses/agpl-3.0.txt#:~:text=16.%20Limitation%20of%20Liability.). The copyright holders and distributors are not liable for any damages resulting from the use or inability to use the program. - label: | I confirm my description is clear, polite, helps developers quickly locate the issue, and complies with community rules. - label: | I have read the [OpenList documentation](https://doc.oplist.org). - label: | I confirm there are no duplicate issues or discussions. - label: | I believe this issue must be handled by `OpenList` and not by a third party. - label: | I confirm this feature has not been implemented yet. - label: | I confirm this feature is reasonable and has general demand, not just my personal need. - label: | I have not read these checkboxes and therefore I just ticked them all, Please close this issue. - type: textarea id: feature-description attributes: label: Feature Description validations: required: true - type: textarea id: suggested-solution attributes: label: Suggested Solution description: | Solution or approach to achieve this feature. - type: textarea id: additional-context attributes: label: Additional Information description: | Any other context or screenshots related to this feature request, or information you find helpful. ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: true contact_links: - name: 问题和讨论 url: https://github.com/OpenListTeam/OpenList/discussions about: 讨论、问题、想法等 - name: Questions & Discussions url: https://github.com/OpenListTeam/OpenList/discussions about: Discuss issues, ideas, etc. - name: 即时聊天 url: https://t.me/OpenListTeam about: 与我们聊天 - name: Chat url: https://t.me/OpenListTeam about: Chat with us ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ ## Description / 描述 ## Motivation and Context / 背景 Closes #XXXX Relates to #XXXX ## How Has This Been Tested? / 测试 ## Checklist / 检查清单 - [ ] I have read the [CONTRIBUTING](https://github.com/OpenListTeam/OpenList/blob/main/CONTRIBUTING.md) document. 我已阅读 [CONTRIBUTING](https://github.com/OpenListTeam/OpenList/blob/main/CONTRIBUTING.md) 文档。 - [ ] I have formatted my code with `go fmt` or [prettier](https://prettier.io/). 我已使用 `go fmt` 或 [prettier](https://prettier.io/) 格式化提交的代码。 - [ ] I have added appropriate labels to this PR (or mentioned needed labels in the description if lacking permissions). 我已为此 PR 添加了适当的标签（如无权限或需要的标签不存在，请在描述中说明，管理员将后续处理）。 - [ ] I have requested review from relevant code authors using the "Request review" feature when applicable. 我已在适当情况下使用"Request review"功能请求相关代码作者进行审查。 - [ ] I have updated the repository accordingly (If it’s needed). 我已相应更新了相关仓库（若适用）。 - [ ] [OpenList-Frontend](https://github.com/OpenListTeam/OpenList-Frontend) #XXXX - [ ] [OpenList-Docs](https://github.com/OpenListTeam/OpenList-Docs) #XXXX ================================================ FILE: .github/workflows/beta_release.yml ================================================ name: Beta Release builds on: push: branches: ["main"] workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true permissions: contents: write jobs: changelog: name: Beta Release Changelog runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - name: Create or update ref id: create-or-update-ref uses: ovsds/create-or-update-ref-action@v1 with: ref: tags/beta sha: ${{ github.sha }} - name: Delete beta tag run: git tag -d beta continue-on-error: true - name: changelog # or changelogithub@0.12 if ensure the stable result id: changelog run: | git tag -l npx changelogithub --output CHANGELOG.md - name: Upload assets to beta release uses: softprops/action-gh-release@v2 with: body_path: CHANGELOG.md files: CHANGELOG.md prerelease: true tag_name: beta - name: Upload assets to github artifact uses: actions/upload-artifact@v4 with: name: beta changelog path: ${{ github.workspace }}/CHANGELOG.md compression-level: 0 if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` release: needs: - changelog strategy: matrix: include: - target: "!(*musl*|*windows-arm64*|*windows7-*|*android*|*freebsd*)" # xgo and loongarch hash: "md5" - target: "linux-!(arm*)-musl*" #musl-not-arm hash: "md5-linux-musl" - target: "linux-arm*-musl*" #musl-arm hash: "md5-linux-musl-arm" - target: "windows-arm64" #win-arm64 hash: "md5-windows-arm64" - target: "windows7-*" #win7 hash: "md5-windows7" - target: "android-*" #android hash: "md5-android" - target: "freebsd-*" #freebsd hash: "md5-freebsd" name: Beta Release runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - name: Setup Go uses: actions/setup-go@v5 with: go-version: "1.25.0" - name: Setup web run: bash build.sh dev web env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Build uses: OpenListTeam/cgo-actions@v1.2.2 with: targets: ${{ matrix.target }} musl-target-format: $os-$musl-$arch github-token: ${{ secrets.GITHUB_TOKEN }} out-dir: build output: openlist-$target$ext musl-base-url: "https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" x-flags: | github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$built_at github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling - name: Compress run: | bash build.sh zip ${{ matrix.hash }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # See above - name: Upload assets to beta release uses: softprops/action-gh-release@v2 with: files: build/compress/* prerelease: true tag_name: beta - name: Clean illegal characters from matrix.target id: clean_target_name run: | ILLEGAL_CHARS_REGEX='[":<>|*?\\/\r\n]' CLEANED_TARGET=$(echo "${{ matrix.target }}" | sed -E "s/$ILLEGAL_CHARS_REGEX//g") echo "Original target: ${{ matrix.target }}" echo "Cleaned target: $CLEANED_TARGET" echo "cleaned_target=$CLEANED_TARGET" >> $GITHUB_ENV - name: Upload assets to github artifact uses: actions/upload-artifact@v4 with: name: beta builds for ${{ env.cleaned_target }} path: ${{ github.workspace }}/build/compress/* compression-level: 0 if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` ================================================ FILE: .github/workflows/build.yml ================================================ name: Test Build on: pull_request: branches: ["main"] workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: build: strategy: matrix: target: - darwin-amd64 - darwin-arm64 - windows-amd64 - linux-arm64-musl - linux-amd64-musl - windows-arm64 - android-arm64 name: Build ${{ matrix.target }} runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - uses: benjlevesque/short-sha@v3.0 id: short-sha - name: Setup Go uses: actions/setup-go@v5 with: go-version: "1.25.0" - name: Setup web run: bash build.sh dev web env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Build uses: OpenListTeam/cgo-actions@v1.2.2 with: targets: ${{ matrix.target }} musl-target-format: $os-$musl-$arch github-token: ${{ secrets.GITHUB_TOKEN }} out-dir: build x-flags: | github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$built_at github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling output: openlist$ext - name: Upload artifact uses: actions/upload-artifact@v4 with: name: openlist_${{ steps.short-sha.outputs.sha }}_${{ matrix.target }} path: build/* ================================================ FILE: .github/workflows/changelog.yml ================================================ name: Release Automatic changelog on: push: tags: - 'v*' permissions: contents: write jobs: changelog: name: Create Release runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 - name: Delete beta tag run: git tag -d beta continue-on-error: true - run: npx changelogithub # or changelogithub@0.12 if ensure the stable result env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} ================================================ FILE: .github/workflows/issue_pr_comment.yml ================================================ name: Issue or PR Auto Reply on: issues: types: [opened] pull_request: types: [opened] permissions: issues: write pull-requests: write jobs: auto-reply: runs-on: ubuntu-latest if: github.event_name == 'issues' steps: - name: Check issue for unchecked tasks and reply uses: actions/github-script@v7 with: script: | let comment = ""; const issueTitle = context.payload.issue.title || ""; const titleNotEdited = /(请修改标题|Please modify the title)/i.test(issueTitle); if (titleNotEdited) { comment = "⚠️ 请修改标题以更好地描述您的问题或需求，并删除示例提示。当前 Issue 将被自动关闭。如需继续提交，请创建新的 Issue。\n"; comment += "⚠️ Please modify the title to better describe your issue or request, and remove the example prompt. This issue will be automatically closed. If you wish to proceed, please create a new issue.\n"; await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body: comment }); await github.rest.issues.update({ ...context.repo, issue_number: context.issue.number, state: 'closed', state_reason: 'not_planned', labels: ['invalid'] }); return; } const issueBody = context.payload.issue.body || ""; const confirmHasRead = /- \[ \] (?!我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody); const confirmNotRead = /- \[[xX]\] (?:我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody); if (confirmNotRead) { comment = "⚠️ 你的 Issue 不符合提交规则。请先阅读相关规范后再重新提交。当前 Issue 将被自动关闭。如需继续提交，请确认已了解规则后重新打开或创建新的 Issue。\n"; comment += "⚠️ Your issue does not comply with the submission rules. Please read the guidelines before submitting again. This issue will be automatically closed. If you wish to proceed, please confirm that you have reviewed the rules before reopening or creating a new issue.\n"; await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body: comment }); await github.rest.issues.update({ ...context.repo, issue_number: context.issue.number, state: 'closed', state_reason: 'not_planned', labels: ['invalid'] }); return; } if (confirmHasRead) { comment = "感谢您联系OpenList。我们会尽快回复您。\n"; comment += "Thanks for contacting OpenList. We will reply to you as soon as possible.\n\n"; comment += "由于您提出的 Issue 中包含部分未确认的项目，为了更好地管理项目，在人工审核后可能会直接关闭此问题。\n"; comment += "如果您能确认并补充相关未确认项目的信息，欢迎随时重新提交。我们会及时关注并处理。感谢您的理解与支持！\n"; comment += "Since your issue contains some unchecked tasks, it may be closed after manual review.\n"; comment += "If you can confirm and provide information for the unchecked tasks, feel free to resubmit.\n"; comment += "We will pay attention and handle it in a timely manner.\n\n"; comment += "感谢您的理解与支持！\n"; comment += "Thank you for your understanding and support!\n"; await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body: comment }); } pr-title-check: runs-on: ubuntu-latest if: github.event_name == 'pull_request' steps: - name: Check PR title for required prefix and comment uses: actions/github-script@v7 with: script: | const title = context.payload.pull_request.title || ""; const ok = /^(feat|docs|fix|style|refactor|chore)\(.+?\)!?: /i.test(title); if (!ok) { let comment = "⚠️ PR 标题需以 `feat(): `, `docs(): `, `fix(): `, `style(): `, `refactor(): `, `chore(): ` 其中之一开头，例如：`feat(component): 新增功能`。\n"; comment += "⚠️ The PR title must start with `feat(): `, `docs(): `, `fix(): `, `style(): `, or `refactor(): `, `chore(): `. For example: `feat(component): add new feature`.\n\n"; comment += "如果跨多个组件，请使用主要组件作为前缀，并在标题中枚举、描述中说明。\n"; comment += "If it spans multiple components, use the main component as the prefix and enumerate in the title, describe in the body.\n\n"; comment += "如果是破坏性变更，请在类型后添加 `!`，例如 `feat(component)!: 破坏性变更`。\n"; comment += "For breaking changes, add `!` after the type, e.g., `feat(component)!: breaking change`.\n\n"; await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body: comment }); } ================================================ FILE: .github/workflows/release.yml ================================================ name: Release builds on: release: types: [ published ] permissions: contents: write jobs: # Set release to prerelease first prerelease: name: Set Prerelease runs-on: ubuntu-latest steps: - name: Prerelease uses: irongut/EditRelease@v1.2.0 with: token: ${{ secrets.GITHUB_TOKEN }} id: ${{ github.event.release.id }} prerelease: true # Main release job for all platforms release: needs: prerelease strategy: matrix: build-type: [ 'standard', 'lite' ] target-platform: [ '', 'android', 'freebsd', 'linux_musl', 'linux_musl_arm' ] name: Release ${{ matrix.target-platform && format('{0} ', matrix.target-platform) || '' }}${{ matrix.build-type == 'lite' && 'Lite' || '' }} runs-on: ubuntu-latest steps: - name: Free Disk Space (Ubuntu) if: matrix.target-platform == '' uses: jlumbroso/free-disk-space@main with: tool-cache: false android: true dotnet: true haskell: true large-packages: true docker-images: true swap-storage: true - name: Setup Go uses: actions/setup-go@v5 with: go-version: '1.25.0' - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - name: Install dependencies if: matrix.target-platform == '' run: | sudo snap install zig --classic --beta docker pull crazymax/xgo:latest go install github.com/crazy-max/xgo@latest sudo apt install upx - name: Build run: | bash build.sh release ${{ matrix.build-type == 'lite' && 'lite' || '' }} ${{ matrix.target-platform }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload assets uses: softprops/action-gh-release@v2 with: files: build/compress/* prerelease: false tag_name: ${{ github.event.release.tag_name }} ================================================ FILE: .github/workflows/release_docker.yml ================================================ name: Release builds (Docker) on: workflow_dispatch: inputs: manual_tag: description: 'Tag name (like v0.1.0). Required if as_latest is true.' required: false type: string as_latest: description: 'Tag as latest?' required: true default: 'false' type: choice options: - 'true' - 'false' push: tags: - 'v*' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: DOCKERHUB_ORG_NAME: ${{ vars.DOCKERHUB_ORG_NAME || 'openlistteam' }} GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'openlistteam' }} IMAGE_NAME: openlist-git IMAGE_NAME_DOCKERHUB: openlist REGISTRY: ghcr.io ARTIFACT_NAME: 'binaries_docker_release' ARTIFACT_NAME_LITE: 'binaries_docker_release_lite' RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/ppc64le,linux/riscv64,linux/loong64' ### Temporarily disable Docker builds for linux/s390x architectures for unknown reasons. IMAGE_PUSH: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} permissions: packages: write jobs: build_binary: name: Build Binaries for Docker Release runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: '1.25.0' - name: Cache Musl id: cache-musl uses: actions/cache@v4 with: path: build/musl-libs key: docker-musl-libs-v2 - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' run: bash build.sh prepare docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build go binary (release) run: bash build.sh release docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: ${{ env.ARTIFACT_NAME }} overwrite: true path: | build/ !build/*.tgz !build/musl-libs/** build_binary_lite: name: Build Binaries for Docker Release (Lite) runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: '1.25.0' - name: Cache Musl id: cache-musl uses: actions/cache@v4 with: path: build/musl-libs key: docker-musl-libs-v2 - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' run: bash build.sh prepare lite docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build go binary (release) run: bash build.sh release lite docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: ${{ env.ARTIFACT_NAME_LITE }} overwrite: true path: | build/ !build/*.tgz !build/musl-libs/** release_docker: needs: build_binary name: Release Docker image runs-on: ubuntu-latest strategy: matrix: image: ["latest", "ffmpeg", "aria2", "aio"] include: - image: "latest" base_image_tag: "base" build_arg: "" tag_favor: "" - image: "ffmpeg" base_image_tag: "ffmpeg" build_arg: INSTALL_FFMPEG=true tag_favor: "suffix=-ffmpeg,onlatest=true" - image: "aria2" base_image_tag: "aria2" build_arg: INSTALL_ARIA2=true tag_favor: "suffix=-aria2,onlatest=true" - image: "aio" base_image_tag: "aio" build_arg: | INSTALL_FFMPEG=true INSTALL_ARIA2=true tag_favor: "suffix=-aio,onlatest=true" steps: - name: Checkout uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: name: ${{ env.ARTIFACT_NAME }} path: 'build/' - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to DockerHub Container Registry if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: | ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} ${{ env.DOCKERHUB_ORG_NAME }}/${{ env.IMAGE_NAME_DOCKERHUB }} tags: > ${{ github.event_name == 'workflow_dispatch' && format('type=raw,value={0}', github.event.inputs.manual_tag) || format('type=raw,value={0}', github.ref_name) }} flavor: | latest=${{ github.event_name == 'push' || github.event.inputs.as_latest == 'true' }} ${{ matrix.tag_favor }} - name: Build and push id: docker_build uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ci push: ${{ env.IMAGE_PUSH == 'true' }} build-args: | BASE_IMAGE_TAG=${{ matrix.base_image_tag }} ${{ matrix.build_arg }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: ${{ env.RELEASE_PLATFORMS }} release_docker_lite: needs: build_binary_lite name: Release Docker image (Lite) runs-on: ubuntu-latest strategy: matrix: image: ["latest", "ffmpeg", "aria2", "aio"] include: - image: "latest" base_image_tag: "base" build_arg: "" tag_favor: "suffix=-lite,onlatest=true" - image: "ffmpeg" base_image_tag: "ffmpeg" build_arg: INSTALL_FFMPEG=true tag_favor: "suffix=-lite-ffmpeg,onlatest=true" - image: "aria2" base_image_tag: "aria2" build_arg: INSTALL_ARIA2=true tag_favor: "suffix=-lite-aria2,onlatest=true" - image: "aio" base_image_tag: "aio" build_arg: | INSTALL_FFMPEG=true INSTALL_ARIA2=true tag_favor: "suffix=-lite-aio,onlatest=true" steps: - name: Checkout uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: name: ${{ env.ARTIFACT_NAME_LITE }} path: 'build/' - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to DockerHub Container Registry if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: | ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} ${{ env.DOCKERHUB_ORG_NAME }}/${{ env.IMAGE_NAME_DOCKERHUB }} tags: > ${{ github.event_name == 'workflow_dispatch' && format('type=raw,value={0}', github.event.inputs.manual_tag) || format('type=raw,value={0}', github.ref_name) }} flavor: | latest=${{ github.event_name == 'push' || github.event.inputs.as_latest == 'true' }} ${{ matrix.tag_favor }} - name: Build and push id: docker_build uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ci push: ${{ env.IMAGE_PUSH == 'true' }} build-args: | BASE_IMAGE_TAG=${{ matrix.base_image_tag }} ${{ matrix.build_arg }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: ${{ env.RELEASE_PLATFORMS }} ================================================ FILE: .github/workflows/sync_repo.yml ================================================ name: Sync to Gitee on: push: branches: - main workflow_dispatch: jobs: sync: runs-on: ubuntu-latest name: Sync GitHub to Gitee steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - name: Setup SSH run: | mkdir -p ~/.ssh echo "${{ secrets.GITEE_SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa ssh-keyscan gitee.com >> ~/.ssh/known_hosts - name: Create single commit and push run: | git config user.name "GitHub Actions" git config user.email "actions@github.com" # Create a new branch git checkout --orphan new-main git add . git commit -m "Sync from GitHub: $(date)" # Add Gitee remote and force push git remote add gitee ${{ vars.GITEE_REPO_URL }} git push --force gitee new-main:main ================================================ FILE: .github/workflows/test_docker.yml ================================================ name: Beta Release (Docker) on: workflow_dispatch: push: branches: - main pull_request: branches: - main concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: DOCKERHUB_ORG_NAME: ${{ vars.DOCKERHUB_ORG_NAME || 'openlistteam' }} GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'openlistteam' }} IMAGE_NAME: openlist-git IMAGE_NAME_DOCKERHUB: openlist REGISTRY: ghcr.io ARTIFACT_NAME: 'binaries_docker_release' RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/ppc64le,linux/riscv64,linux/loong64' ### Temporarily disable Docker builds for linux/s390x architectures for unknown reasons. IMAGE_PUSH: ${{ github.event_name == 'push' }} IMAGE_TAGS_BETA: | type=ref,event=pr type=raw,value=beta,enable={{is_default_branch}} jobs: build_binary: name: Build Binaries for Docker Release (Beta) runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: '1.25.0' - name: Cache Musl id: cache-musl uses: actions/cache@v4 with: path: build/musl-libs key: docker-musl-libs-v2 - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' run: bash build.sh prepare docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build go binary (beta) run: bash build.sh beta docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: ${{ env.ARTIFACT_NAME }} overwrite: true path: | build/ !build/*.tgz !build/musl-libs/** release_docker: needs: build_binary name: Release Docker image (Beta) runs-on: ubuntu-latest permissions: packages: write strategy: matrix: image: ["latest", "ffmpeg", "aria2", "aio"] include: - image: "latest" base_image_tag: "base" build_arg: "" tag_favor: "" - image: "ffmpeg" base_image_tag: "ffmpeg" build_arg: INSTALL_FFMPEG=true tag_favor: "suffix=-ffmpeg,onlatest=true" - image: "aria2" base_image_tag: "aria2" build_arg: INSTALL_ARIA2=true tag_favor: "suffix=-aria2,onlatest=true" - image: "aio" base_image_tag: "aio" build_arg: | INSTALL_FFMPEG=true INSTALL_ARIA2=true tag_favor: "suffix=-aio,onlatest=true" steps: - name: Checkout uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: name: ${{ env.ARTIFACT_NAME }} path: 'build/' - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Login to DockerHub Container Registry if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: | ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} ${{ env.DOCKERHUB_ORG_NAME }}/${{ env.IMAGE_NAME_DOCKERHUB }} tags: ${{ env.IMAGE_TAGS_BETA }} flavor: | ${{ matrix.tag_favor }} - name: Build and push id: docker_build uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ci push: ${{ env.IMAGE_PUSH == 'true' }} build-args: | BASE_IMAGE_TAG=${{ matrix.base_image_tag }} ${{ matrix.build_arg }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: ${{ env.RELEASE_PLATFORMS }} ================================================ FILE: .github/workflows/trigger-makefile-update.yml ================================================ name: Trigger OpenWRT Update on: push: tags: - 'v*' workflow_dispatch: inputs: tag: description: 'Release tag to trigger update for' required: true type: string jobs: trigger-makefile-update: runs-on: ubuntu-latest steps: - name: Trigger Makefile hash update uses: peter-evans/repository-dispatch@v3 with: token: ${{ secrets.EXTERNAL_REPO_TOKEN_LUCI_APP_OPENLIST }} repository: ${{ vars.HOOK_REPO || 'OpenListTeam/OpenList-OpenWRT' }} event-type: update-hashes client-payload: | { "source_repository": "${{ github.repository }}", "release_tag": "${{ inputs.tag || github.ref_name }}", "release_name": "${{ inputs.tag || github.ref_name }}", "release_url": "${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ inputs.tag || github.ref_name }}", "triggered_by": "${{ github.actor }}", "trigger_reason": "${{ github.event_name }}" } - name: Log trigger information run: | echo "🚀 Successfully triggered Makefile hash update" echo "📦 Target repository: OpenListTeam/luci-app-openlist" echo "🏷️ Tag: ${{ inputs.tag || github.ref_name }}" echo "👤 Triggered by: ${{ github.actor }}" echo "📅 Trigger time: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" ================================================ FILE: .gitignore ================================================ .idea/ .DS_Store output/ /dist/ # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib *.db *.bin # Test binary, built with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Dependency directories (remove the comment below to include it) # vendor/ /bin/* *.json /build /data/ /tmp/ /log/ /lang/ /daemon/ /public/dist/* /!public/dist/README.md .VSCodeCounter ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Contributor Covenant Code of Conduct ## Our Pledge We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. ## Our Standards Examples of behavior that contributes to a positive environment for our community include: * Demonstrating empathy and kindness toward other people * Being respectful of differing opinions, viewpoints, and experiences * Giving and gracefully accepting constructive feedback * Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience * Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: * The use of sexualized language or imagery, and sexual attention or advances of any kind * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or email address, without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. ## Scope This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at [Telegram Group](https://t.me/OpenListTeam). All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident. ## Enforcement Guidelines Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: ### 1. Correction **Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. **Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. ### 2. Warning **Community Impact**: A violation through a single incident or series of actions. **Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. ### 3. Temporary Ban **Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. **Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. ### 4. Permanent Ban **Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. **Consequence**: A permanent ban from any sort of public interaction within the community. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations. ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing ## Setup your machine `OpenList` is written in [Go](https://golang.org/) and [SolidJS](https://www.solidjs.com/). Prerequisites: - [git](https://git-scm.com) - [Go 1.24+](https://golang.org/doc/install) - [gcc](https://gcc.gnu.org/) - [nodejs](https://nodejs.org/) ## Cloning a fork Fork and clone `OpenList` and `OpenList-Frontend` anywhere: ```shell $ git clone https://github.com//OpenList.git $ git clone --recurse-submodules https://github.com//OpenList-Frontend.git ``` ## Creating a branch Create a new branch from the `main` branch, with an appropriate name. ```shell $ git checkout -b ``` ## Preview your change ### backend ```shell $ go run main.go ``` ### frontend ```shell $ pnpm dev ``` ## Add a new driver Copy `drivers/template` folder and rename it, and follow the comments in it. ## Create a commit Commit messages should be well formatted, and to make that "standardized". Submit your pull request. For PR titles, follow [Conventional Commits](https://www.conventionalcommits.org). https://github.com/OpenListTeam/OpenList/issues/376 It's suggested to sign your commits. See: [How to sign commits](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits) ## Submit a pull request Please make sure your code has been formatted with `go fmt` or [prettier](https://prettier.io/) before submitting. Push your branch to your `openlist` fork and open a pull request against the `main` branch. ## Merge your pull request Your pull request will be merged after review. Please wait for the maintainer to merge your pull request after review. At least 1 approving review is required by reviewers with write access. You can also request a review from maintainers. ## Delete your branch (Optional) After your pull request is merged, you can delete your branch. --- Thank you for your contribution! Let's make OpenList better together! ================================================ FILE: Dockerfile ================================================ ### Default image is base. You can add other support by modifying BASE_IMAGE_TAG. The following parameters are supported: base (default), aria2, ffmpeg, aio ARG BASE_IMAGE_TAG=base FROM alpine:edge AS builder LABEL stage=go-builder WORKDIR /app/ RUN apk add --no-cache bash curl jq gcc git go musl-dev COPY go.mod go.sum ./ RUN go mod download COPY ./ ./ RUN bash build.sh release docker FROM openlistteam/openlist-base-image:${BASE_IMAGE_TAG} LABEL MAINTAINER="OpenList" ARG INSTALL_FFMPEG=false ARG INSTALL_ARIA2=false ARG USER=openlist ARG UID=1001 ARG GID=1001 WORKDIR /opt/openlist/ RUN addgroup -g ${GID} ${USER} && \ adduser -D -u ${UID} -G ${USER} ${USER} && \ mkdir -p /opt/openlist/data COPY --from=builder --chmod=755 --chown=${UID}:${GID} /app/bin/openlist ./ COPY --chmod=755 --chown=${UID}:${GID} entrypoint.sh /entrypoint.sh USER ${USER} RUN /entrypoint.sh version ENV UMASK=022 RUN_ARIA2=${INSTALL_ARIA2} VOLUME /opt/openlist/data/ EXPOSE 5244 5245 CMD [ "/entrypoint.sh" ] ================================================ FILE: Dockerfile.ci ================================================ ARG BASE_IMAGE_TAG=base FROM ghcr.io/openlistteam/openlist-base-image:${BASE_IMAGE_TAG} LABEL MAINTAINER="OpenList" ARG TARGETPLATFORM ARG INSTALL_FFMPEG=false ARG INSTALL_ARIA2=false ARG USER=openlist ARG UID=1001 ARG GID=1001 WORKDIR /opt/openlist/ RUN addgroup -g ${GID} ${USER} && \ adduser -D -u ${UID} -G ${USER} ${USER} && \ mkdir -p /opt/openlist/data COPY --chmod=755 --chown=${UID}:${GID} /build/${TARGETPLATFORM}/openlist ./ COPY --chmod=755 --chown=${UID}:${GID} entrypoint.sh /entrypoint.sh USER ${USER} RUN /entrypoint.sh version ENV UMASK=022 RUN_ARIA2=${INSTALL_ARIA2} VOLUME /opt/openlist/data/ EXPOSE 5244 5245 CMD [ "/entrypoint.sh" ] ================================================ FILE: LICENSE ================================================ GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software. A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public. The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version. An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU Affero General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Remote Network Interaction; Use with the GNU General Public License. Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements. You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see . ================================================ FILE: README.md ================================================
logo

OpenList is a resilient, long-term governance, community-driven fork of AList — built to defend open source against trust-based attacks.

latest version License Build status latest version discussions Downloads
--- - English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Dutch](./README_nl.md) - [Contributing](./CONTRIBUTING.md) - [CODE OF CONDUCT](./CODE_OF_CONDUCT.md) - [LICENSE](./LICENSE) ## Disclaimer OpenList is an open-source project independently maintained by the OpenList Team, following the AGPL-3.0 license and committed to maintaining complete code openness and modification transparency. We have noticed the emergence of some third-party projects in the community with names similar to this project, such as OpenListApp/OpenListApp, as well as some paid proprietary software using the same or similar naming. To avoid user confusion, we hereby declare: - OpenList has no official association with any third-party derivative projects. - All software, code, and services of this project are maintained by the OpenList Team and are freely available on GitHub. - Project documentation and API services primarily rely on charitable resources provided by Cloudflare. There are currently no paid plans or commercial deployments, and the use of existing features does not involve any costs. We respect the community's rights to free use and derivative development, but we also strongly urge downstream projects: - Should not use the "OpenList" name for impersonation promotion or commercial gain; - Must not distribute OpenList-based code in a closed-source manner or violate AGPL license terms. To better maintain healthy ecosystem development, we recommend: - Clearly indicate the project source and choose appropriate open-source licenses in accordance with the open-source spirit; - If involving commercial use, please avoid using "OpenList" or any confusing naming as the project name; - If you need to use materials located under OpenListTeam/Logo, you may modify and use them under compliance with the agreement. Thank you for your support and understanding of the OpenList project. ## Features - [x] Multiple storages - [x] Local storage - [x] [Aliyundrive](https://www.alipan.com) - [x] OneDrive / Sharepoint ([Global](https://www.microsoft.com/en-us/microsoft-365/onedrive/online-cloud-storage), [CN](https://portal.partner.microsoftonline.cn), DE, US) - [x] [189cloud](https://cloud.189.cn) (Personal, Family) - [x] [GoogleDrive](https://drive.google.com) - [x] [123pan](https://www.123pan.com) - [x] [FTP / SFTP](https://en.wikipedia.org/wiki/File_Transfer_Protocol) - [x] [PikPak](https://www.mypikpak.com) - [x] [S3](https://aws.amazon.com/s3) - [x] [Seafile](https://seafile.com) - [x] [UPYUN Storage Service](https://www.upyun.com/products/file-storage) - [x] [WebDAV](https://en.wikipedia.org/wiki/WebDAV) - [x] Teambition([China](https://www.teambition.com), [International](https://us.teambition.com)) - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn) - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) - [x] [Terabox](https://www.terabox.com/main) - [x] [UC](https://drive.uc.cn) - [x] [Quark](https://pan.quark.cn) - [x] [Thunder](https://pan.xunlei.com) - [x] [Lanzou](https://www.lanzou.com) - [x] [ILanzou](https://www.ilanzou.com) - [x] [Google photo](https://photos.google.com) - [x] [Mega.nz](https://mega.nz) - [x] [Baidu photo](https://photo.baidu.com) - [x] [SMB](https://en.wikipedia.org/wiki/Server_Message_Block) - [x] [115](https://115.com) - [X] [Cloudreve](https://cloudreve.org) - [x] [Dropbox](https://www.dropbox.com) - [x] [FeijiPan](https://www.feijipan.com) - [x] [dogecloud](https://www.dogecloud.com/product/oss) - [x] [Azure Blob Storage](https://azure.microsoft.com/products/storage/blobs) - [x] [Chaoxing](https://www.chaoxing.com) - [x] [CNB](https://cnb.cool/) - [x] [Degoo](https://degoo.com) - [x] [Doubao](https://www.doubao.com) - [x] [Febbox](https://www.febbox.com) - [x] [GitHub](https://github.com) - [x] [OpenList](https://github.com/OpenListTeam/OpenList) - [x] [Teldrive](https://github.com/tgdrive/teldrive) - [x] [Weiyun](https://www.weiyun.com) - [x] Easy to deploy and out-of-the-box - [x] File preview (PDF, markdown, code, plain text, ...) - [x] Image preview in gallery mode - [x] Video and audio preview, support lyrics and subtitles - [x] Office documents preview (docx, pptx, xlsx, ...) - [x] `README.md` preview rendering - [x] File permalink copy and direct file download - [x] Dark mode - [x] I18n - [x] Protected routes (password protection and authentication) - [x] WebDAV - [x] Docker Deploy - [x] Cloudflare Workers proxy - [x] File/Folder package download - [x] Web upload(Can allow visitors to upload), delete, mkdir, rename, move and copy - [x] Offline download - [x] Copy files between two storage - [x] Multi-thread downloading acceleration for single-thread download/stream ## Document - 📘 [Global Site](https://doc.oplist.org) - 📚 [Backup Site](https://doc.openlist.team) - 🌏 [CN Site](https://doc.oplist.org.cn) ## Demo - 🌎 [Global Demo](https://demo.oplist.org) - 🇨🇳 [CN Demo](https://demo.oplist.org.cn) ## Discussion Please refer to [*Discussions*](https://github.com/OpenListTeam/OpenList/discussions) for raising general questions, ***Issues* is for bug reports and feature requests only.** ## Sponsor [![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") ## License The `OpenList` is open-source software licensed under the [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) license. ## Disclaimer - This project is a free and open-source software designed to facilitate file sharing via net disks, primarily intended to support the downloading and learning of the Go programming language. - Please comply with all applicable laws and regulations when using this software. Any form of misuse is strictly prohibited. - The software is based on official SDKs or APIs without any modification, disruption, or interference with their behavior. - It only performs HTTP 302 redirects or traffic forwarding, and does not intercept, store, or tamper with any user data. - This project is not affiliated with any official platform or service provider. - The software is provided "as is", without any warranties of any kind, either express or implied, including but not limited to warranties of merchantability or fitness for a particular purpose. - The maintainers are not liable for any direct or indirect damages arising from the use of, or inability to use, this software. - You are solely responsible for any risks associated with using this software, including but not limited to account bans or download speed limitations. - This project is licensed under the [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) License. Please see the [LICENSE](./LICENSE) file for details. ## Contact Us - [@GitHub](https://github.com/OpenListTeam) - [Telegram Group](https://t.me/OpenListTeam) - [Telegram Channel](https://t.me/OpenListOfficial) ## Contributors We sincerely thank the author [Xhofe](https://github.com/Xhofe) of the original project [AlistGo/alist](https://github.com/AlistGo/alist) and all other contributors. Thanks goes to these wonderful people: [![Contributors](https://contrib.rocks/image?repo=OpenListTeam/OpenList)](https://github.com/OpenListTeam/OpenList/graphs/contributors) ================================================ FILE: README_cn.md ================================================
logo

OpenList 是一个有韧性、长期治理、社区驱动的 AList 分支，旨在防御基于信任的开源攻击。

latest version License Build status latest version discussions Downloads
--- - [English](./README.md) | 中文 | [日本語](./README_ja.md) | [Dutch](./README_nl.md) - [贡献指南](./CONTRIBUTING.md) - [行为准则](./CODE_OF_CONDUCT.md) - [许可证](./LICENSE) ## 免责声明 OpenList 是一个由 OpenList 团队独立维护的开源项目，遵循 AGPL-3.0 许可证，致力于保持完整的代码开放性和修改透明性。 我们注意到社区中出现了一些与本项目名称相似的第三方项目，如 OpenListApp/OpenListApp，以及部分采用相同或近似命名的收费专有软件。为避免用户误解，现声明如下： - OpenList 与任何第三方衍生项目无官方关联。 - 本项目的全部软件、代码与服务由 OpenList 团队维护，可在 GitHub 免费获取。 - 项目文档与 API 服务均主要依托于 Cloudflare 提供的公益资源，目前无任何收费计划或商业部署，现有功能使用不涉及任何支出。 我们尊重社区的自由使用与衍生开发权利，但也强烈呼吁下游项目： - 不应以“OpenList”名义进行冒名宣传或获取商业利益； - 不得将基于 OpenList 的代码进行闭源分发或违反 AGPL 许可证条款。 为了更好地维护生态健康发展，我们建议： - 明确注明项目来源，并以符合开源精神的方式选择适当的开源许可证； - 如涉及商业用途，请避免使用“OpenList”或任何会产生混淆的方式作为项目名称； - 若需使用本项目位于 OpenListTeam/Logo 下的素材，可在遵守协议的前提下进行修改后使用。 感谢您对 OpenList 项目的支持与理解。 ## 功能 - [x] 多种存储 - [x] 本地存储 - [x] [阿里云盘](https://www.alipan.com) - [x] OneDrive / Sharepoint ([国际版](https://www.microsoft.com/en-us/microsoft-365/onedrive/online-cloud-storage), [中国](https://portal.partner.microsoftonline.cn), DE, US) - [x] [天翼云盘](https://cloud.189.cn)（个人、家庭） - [x] [GoogleDrive](https://drive.google.com) - [x] [123云盘](https://www.123pan.com) - [x] [FTP / SFTP](https://en.wikipedia.org/wiki/File_Transfer_Protocol) - [x] [PikPak](https://www.mypikpak.com) - [x] [S3](https://aws.amazon.com/s3) - [x] [Seafile](https://seafile.com) - [x] [又拍云对象存储](https://www.upyun.com/products/file-storage) - [x] [WebDAV](https://en.wikipedia.org/wiki/WebDAV) - [x] Teambition([中国](https://www.teambition.com), [国际](https://us.teambition.com)) - [x] [MediaFire](https://www.mediafire.com) - [x] [分秒帧](https://www.mediatrack.cn) - [x] [ProtonDrive](https://proton.me/drive) - [x] [和彩云](https://yun.139.com)（个人、家庭、群组） - [x] [YandexDisk](https://disk.yandex.com) - [x] [百度网盘](http://pan.baidu.com) - [x] [Terabox](https://www.terabox.com/main) - [x] [UC网盘](https://drive.uc.cn) - [x] [夸克网盘](https://pan.quark.cn) - [x] [迅雷网盘](https://pan.xunlei.com) - [x] [蓝奏云](https://www.lanzou.com) - [x] [蓝奏云优享版](https://www.ilanzou.com) - [x] [Google 相册](https://photos.google.com) - [x] [Mega.nz](https://mega.nz) - [x] [百度相册](https://photo.baidu.com) - [x] [SMB](https://en.wikipedia.org/wiki/Server_Message_Block) - [x] [115](https://115.com) - [x] [Cloudreve](https://cloudreve.org) - [x] [Dropbox](https://www.dropbox.com) - [x] [飞机盘](https://www.feijipan.com) - [x] [多吉云](https://www.dogecloud.com/product/oss) - [x] [Azure Blob Storage](https://azure.microsoft.com/products/storage/blobs) - [x] [超星](https://www.chaoxing.com) - [x] [CNB](https://cnb.cool/) - [x] [Degoo](https://degoo.com) - [x] [豆包](https://www.doubao.com) - [x] [Febbox](https://www.febbox.com) - [x] [GitHub](https://github.com) - [x] [OpenList](https://github.com/OpenListTeam/OpenList) - [x] [Teldrive](https://github.com/tgdrive/teldrive) - [x] [微云](https://www.weiyun.com) - [x] 部署方便，开箱即用 - [x] 文件预览（PDF、markdown、代码、纯文本等） - [x] 画廊模式下的图片预览 - [x] 视频和音频预览，支持歌词和字幕 - [x] Office 文档预览（docx、pptx、xlsx 等） - [x] `README.md` 预览渲染 - [x] 文件永久链接复制和直接文件下载 - [x] 黑暗模式 - [x] 国际化 - [x] 受保护的路由（密码保护和认证） - [x] WebDAV - [x] Docker 部署 - [x] Cloudflare Workers 代理 - [x] 文件/文件夹打包下载 - [x] 网页上传（可允许访客上传）、删除、新建文件夹、重命名、移动和复制 - [x] 离线下载 - [x] 跨存储复制文件 - [x] 单文件多线程下载/流式加速 ## 文档 - 🌏 [国内站点](https://doc.oplist.org.cn) - 📘 [海外站点](https://doc.oplist.org) - 📚 [备用站点](https://doc.openlist.team) ## 演示 - 🇨🇳 [国内演示站](https://demo.oplist.org.cn) - 🌎 [海外演示站](https://demo.oplist.org) ## 讨论 如有一般性问题请前往 [*Discussions*](https://github.com/OpenListTeam/OpenList/discussions) 讨论区，***Issues* 仅用于错误报告和功能请求。** ## 赞助者 [![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") ## 许可证 `OpenList` 是基于 [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) 许可证的开源软件。 ## 免责声明 - 本项目为免费开源软件，旨在通过网盘便捷分享文件，主要用于 Go 语言的下载与学习。 - 使用本软件时请遵守相关法律法规，严禁任何形式的滥用。 - 本软件基于官方 SDK 或 API 实现，未对其行为进行任何修改、破坏或干扰。 - 仅进行 HTTP 302 跳转或流量转发，不拦截、存储或篡改任何用户数据。 - 本项目与任何官方平台或服务提供商无关。 - 本软件按“原样”提供，不附带任何明示或暗示的担保，包括但不限于适销性或特定用途的适用性。 - 维护者不对因使用或无法使用本软件而导致的任何直接或间接损失负责。 - 您需自行承担使用本软件的所有风险，包括但不限于账号被封、下载限速等。 - 本项目遵循 [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) 许可证，详情请参见 [LICENSE](./LICENSE) 文件。 ## 联系我们 - [@GitHub](https://github.com/OpenListTeam) - [Telegram 交流群](https://t.me/OpenListTeam) - [Telegram 频道](https://t.me/OpenListOfficial) ## 贡献者 我们衷心感谢原项目 [AlistGo/alist](https://github.com/AlistGo/alist) 的作者 [Xhofe](https://github.com/Xhofe) 及所有其他贡献者。 感谢这些优秀的人： [![Contributors](https://contrib.rocks/image?repo=OpenListTeam/OpenList)](https://github.com/OpenListTeam/OpenList/graphs/contributors) ================================================ FILE: README_ja.md ================================================
logo

OpenList は、信頼ベースの攻撃からオープンソースを守るために構築された、レジリエントで長期ガバナンス、コミュニティ主導の AList フォークです。

latest version License Build status latest version discussions Downloads
--- - [English](./README.md) | [中文](./README_cn.md) | 日本語 | [Dutch](./README_nl.md) - [コントリビュート](./CONTRIBUTING.md) - [行動規範](./CODE_OF_CONDUCT.md) - [ライセンス](./LICENSE) ## 免責事項 OpenListは、OpenListチームが独立して維持するオープンソースプロジェクトであり、AGPL-3.0ライセンスに従い、完全なコードの開放性と変更の透明性を維持することに専念しています。 コミュニティ内で、OpenListApp/OpenListAppなど、本プロジェクトと類似した名称を持つサードパーティプロジェクトや、同一または類似した命名を採用する有料専有ソフトウェアが出現していることを確認しています。ユーザーの誤解を避けるため、以下のように宣言いたします： - OpenListは、いかなるサードパーティ派生プロジェクトとも公式な関連性はありません。 - 本プロジェクトのすべてのソフトウェア、コード、サービスはOpenListチームによって維持され、GitHubで無料で取得できます。 - プロジェクトドキュメントとAPIサービスは主にCloudflareが提供する公益リソースに依存しており、現在有料プランや商業展開はなく、既存機能の使用に費用は発生しません。 私たちはコミュニティの自由な使用と派生開発の権利を尊重しますが、下流プロジェクトに強く呼びかけます： - 「OpenList」の名前で偽装宣伝や商業利益を得るべきではありません； - OpenListベースのコードをクローズドソースで配布したり、AGPLライセンス条項に違反してはいけません。 エコシステムの健全な発展をより良く維持するため、以下を推奨します： - プロジェクトの出典を明確に示し、オープンソース精神に合致する適切なオープンソースライセンスを選択する； - 商業用途が関わる場合は、「OpenList」や混乱を招く可能性のある名前をプロジェクト名として使用することを避ける； - OpenListTeam/Logo下の素材を使用する必要がある場合は、協定を遵守した上で修正して使用できます。 OpenListプロジェクトへのご支援とご理解をありがとうございます。 ## 特徴 - [x] 複数ストレージ - [x] ローカルストレージ - [x] [Aliyundrive](https://www.alipan.com) - [x] OneDrive / Sharepoint ([グローバル](https://www.microsoft.com/en-us/microsoft-365/onedrive/online-cloud-storage), [中国](https://portal.partner.microsoftonline.cn), DE, US) - [x] [189cloud](https://cloud.189.cn)（個人、家族） - [x] [GoogleDrive](https://drive.google.com) - [x] [123pan](https://www.123pan.com) - [x] [FTP / SFTP](https://en.wikipedia.org/wiki/File_Transfer_Protocol) - [x] [PikPak](https://www.mypikpak.com) - [x] [S3](https://aws.amazon.com/s3) - [x] [Seafile](https://seafile.com) - [x] [UPYUN Storage Service](https://www.upyun.com/products/file-storage) - [x] [WebDAV](https://en.wikipedia.org/wiki/WebDAV) - [x] Teambition([中国](https://www.teambition.com), [国際](https://us.teambition.com)) - [x] [Mediatrack](https://www.mediatrack.cn) - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com)（個人、家族、グループ） - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) - [x] [Terabox](https://www.terabox.com/main) - [x] [UC](https://drive.uc.cn) - [x] [Quark](https://pan.quark.cn) - [x] [Thunder](https://pan.xunlei.com) - [x] [Lanzou](https://www.lanzou.com) - [x] [ILanzou](https://www.ilanzou.com) - [x] [Google photo](https://photos.google.com) - [x] [Mega.nz](https://mega.nz) - [x] [Baidu photo](https://photo.baidu.com) - [x] [SMB](https://en.wikipedia.org/wiki/Server_Message_Block) - [x] [115](https://115.com) - [x] [Cloudreve](https://cloudreve.org) - [x] [Dropbox](https://www.dropbox.com) - [x] [FeijiPan](https://www.feijipan.com) - [x] [dogecloud](https://www.dogecloud.com/product/oss) - [x] [Azure Blob Storage](https://azure.microsoft.com/products/storage/blobs) - [x] [Chaoxing](https://www.chaoxing.com) - [x] [CNB](https://cnb.cool/) - [x] [Degoo](https://degoo.com) - [x] [Doubao](https://www.doubao.com) - [x] [Febbox](https://www.febbox.com) - [x] [GitHub](https://github.com) - [x] [OpenList](https://github.com/OpenListTeam/OpenList) - [x] [Teldrive](https://github.com/tgdrive/teldrive) - [x] [Weiyun](https://www.weiyun.com) - [x] [MediaFire](https://www.mediafire.com) - [x] 簡単にデプロイでき、すぐに使える - [x] ファイルプレビュー（PDF、markdown、コード、テキストなど） - [x] ギャラリーモードでの画像プレビュー - [x] ビデオ・オーディオプレビュー、歌詞・字幕対応 - [x] Officeドキュメントプレビュー（docx、pptx、xlsxなど） - [x] `README.md` プレビュー表示 - [x] ファイルのパーマリンクコピーと直接ダウンロード - [x] ダークモード - [x] 国際化対応 - [x] 保護されたルート（パスワード保護と認証） - [x] WebDAV - [x] Dockerデプロイ - [x] Cloudflare Workersプロキシ - [x] ファイル/フォルダのパッケージダウンロード - [x] Webアップロード（訪問者のアップロード許可可）、削除、フォルダ作成、リネーム、移動、コピー - [x] オフラインダウンロード - [x] ストレージ間のファイルコピー - [x] 単一ファイルのマルチスレッドダウンロード/ストリーム加速 ## ドキュメント - 📘 [グローバルサイト](https://doc.oplist.org) - 📚 [バックアップサイト](https://doc.openlist.team) - 🌏 [CNサイト](https://doc.oplist.org.cn) ## デモ - 🌎 [グローバルデモ](https://demo.oplist.org) - 🇨🇳 [CNデモ](https://demo.oplist.org.cn) ## ディスカッション 一般的な質問は [*Discussions*](https://github.com/OpenListTeam/OpenList/discussions) をご利用ください。***Issues* はバグ報告と機能リクエスト専用です。** ## スポンサー [![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") ## ライセンス 「OpenList」は [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) ライセンスの下で公開されているオープンソースソフトウェアです。 ## 免責事項 - 本プロジェクトは無料のオープンソースソフトウェアであり、ネットワークディスクを通じたファイル共有を容易にすることを目的とし、主に Go 言語のダウンロードと学習をサポートします。 - 本ソフトウェアの利用にあたっては、関連する法令を遵守し、不正利用を固く禁じます。 - 本ソフトウェアは公式 SDK または API に基づいており、その動作を一切改変・破壊・妨害しません。 - 302 リダイレクトまたはトラフィック転送のみを行い、ユーザーデータの傍受・保存・改ざんは一切行いません。 - 本プロジェクトは、いかなる公式プラットフォームやサービスプロバイダーとも関係ありません。 - 本ソフトウェアは「現状有姿」で提供されており、商品性や特定目的への適合性を含むいかなる保証もありません。 - 本ソフトウェアの使用または使用不能によるいかなる直接的・間接的損害についても、メンテナは責任を負いません。 - 本ソフトウェアの利用に伴うすべてのリスク（アカウントの凍結やダウンロード速度制限などを含む）は、利用者自身が負うものとします。 - 本プロジェクトは [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) ライセンスに従います。詳細は [LICENSE](./LICENSE) ファイルをご覧ください。 ## お問い合わせ - [@GitHub](https://github.com/OpenListTeam) - [Telegram グループ](https://t.me/OpenListTeam) - [Telegram チャンネル](https://t.me/OpenListOfficial) ## コントリビューター オリジナルプロジェクト [AlistGo/alist](https://github.com/AlistGo/alist) の作者 [Xhofe](https://github.com/Xhofe) およびその他すべての貢献者に心より感謝いたします。 素晴らしい皆様に感謝します： [![Contributors](https://contrib.rocks/image?repo=OpenListTeam/OpenList)](https://github.com/OpenListTeam/OpenList/graphs/contributors) ================================================ FILE: README_nl.md ================================================
logo

OpenList is een veerkrachtige, langetermijn, door de gemeenschap geleide fork van AList — gebouwd om open source te beschermen tegen op vertrouwen gebaseerde aanvallen.

latest version License Build status latest version discussions Downloads
--- - [English](./README.md) | [中文](./README_cn.md) | [日本語](./README_ja.md) | Dutch - [Bijdragen](./CONTRIBUTING.md) - [Gedragscode](./CODE_OF_CONDUCT.md) - [Licentie](./LICENSE) ## Disclaimer OpenList is een open-source project dat onafhankelijk wordt onderhouden door het OpenList Team, volgend op de AGPL-3.0 licentie en toegewijd aan het behouden van volledige code openheid en transparantie van wijzigingen. We hebben gemerkt dat er in de gemeenschap enkele derde partij projecten zijn verschenen met namen vergelijkbaar met dit project, zoals OpenListApp/OpenListApp, evenals enkele betaalde eigendomssoftware die dezelfde of soortgelijke naamgeving gebruikt. Om verwarring bij gebruikers te voorkomen, verklaren we hierbij: - OpenList heeft geen officiële associatie met enige derde partij afgeleide projecten. - Alle software, code en diensten van dit project worden onderhouden door het OpenList Team en zijn gratis beschikbaar op GitHub. - Projectdocumentatie en API diensten zijn voornamelijk afhankelijk van liefdadigheidsbronnen verstrekt door Cloudflare. Er zijn momenteel geen betaalplannen of commerciële implementaties, en het gebruik van bestaande functies brengt geen kosten met zich mee. We respecteren de rechten van de gemeenschap voor vrij gebruik en afgeleide ontwikkeling, maar we roepen downstream projecten ook ten zeerste op: - Mogen niet de "OpenList" naam gebruiken voor namaakpromotie of commercieel gewin; - Mogen OpenList-gebaseerde code niet distribueren op een closed-source manier of AGPL licentievoorwaarden schenden. Om een gezonde ecosysteemontwikkeling beter te onderhouden, bevelen we aan: - Duidelijk de projectbron aangeven en passende open-source licenties kiezen in overeenstemming met de open-source geest; - Bij commercieel gebruik, vermijd het gebruik van "OpenList" of enige verwarrende naamgeving als projectnaam; - Als u materialen onder OpenListTeam/Logo moet gebruiken, kunt u deze wijzigen en gebruiken onder naleving van de overeenkomst. Dank u voor uw ondersteuning en begrip ## Functies - [x] Meerdere opslagmogelijkheden - [x] Lokale opslag - [x] [Aliyundrive](https://www.alipan.com) - [x] OneDrive / Sharepoint ([Global](https://www.microsoft.com/en-us/microsoft-365/onedrive/online-cloud-storage), [CN](https://portal.partner.microsoftonline.cn), DE, US) - [x] [189cloud](https://cloud.189.cn) (Persoonlijk, Familie) - [x] [GoogleDrive](https://drive.google.com) - [x] [123pan](https://www.123pan.com) - [x] [FTP / SFTP](https://en.wikipedia.org/wiki/File_Transfer_Protocol) - [x] [PikPak](https://www.mypikpak.com) - [x] [S3](https://aws.amazon.com/s3) - [x] [Seafile](https://seafile.com) - [x] [UPYUN Storage Service](https://www.upyun.com/products/file-storage) - [x] [WebDAV](https://en.wikipedia.org/wiki/WebDAV) - [x] Teambition([China](https://www.teambition.com), [Internationaal](https://us.teambition.com)) - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn) - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com) (Persoonlijk, Familie, Groep) - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) - [x] [Terabox](https://www.terabox.com/main) - [x] [UC](https://drive.uc.cn) - [x] [Quark](https://pan.quark.cn) - [x] [Thunder](https://pan.xunlei.com) - [x] [Lanzou](https://www.lanzou.com) - [x] [ILanzou](https://www.ilanzou.com) - [x] [Google photo](https://photos.google.com) - [x] [Mega.nz](https://mega.nz) - [x] [Baidu photo](https://photo.baidu.com) - [x] [SMB](https://en.wikipedia.org/wiki/Server_Message_Block) - [x] [115](https://115.com) - [x] [Cloudreve](https://cloudreve.org) - [x] [Dropbox](https://www.dropbox.com) - [x] [FeijiPan](https://www.feijipan.com) - [x] [dogecloud](https://www.dogecloud.com/product/oss) - [x] [Azure Blob Storage](https://azure.microsoft.com/products/storage/blobs) - [x] [Chaoxing](https://www.chaoxing.com) - [x] [CNB](https://cnb.cool/) - [x] [Degoo](https://degoo.com) - [x] [Doubao](https://www.doubao.com) - [x] [Febbox](https://www.febbox.com) - [x] [GitHub](https://github.com) - [x] [OpenList](https://github.com/OpenListTeam/OpenList) - [x] [Teldrive](https://github.com/tgdrive/teldrive) - [x] [Weiyun](https://www.weiyun.com) - [x] Eenvoudig te implementeren en direct te gebruiken - [x] Bestandsvoorbeeld (PDF, markdown, code, platte tekst, ...) - [x] Afbeeldingsvoorbeeld in galerijweergave - [x] Video- en audiovoorbeeld, ondersteuning voor songteksten en ondertitels - [x] Office-documenten voorbeeld (docx, pptx, xlsx, ...) - [x] `README.md` voorbeeldweergave - [x] Permalink kopiëren en direct downloaden van bestanden - [x] Donkere modus - [x] I18n - [x] Beschermde routes (wachtwoordbeveiliging en authenticatie) - [x] WebDAV - [x] Docker implementatie - [x] Cloudflare Workers proxy - [x] Bestands-/map-pakket download - [x] Webupload (bezoekers kunnen uploaden toestaan), verwijderen, map aanmaken, hernoemen, verplaatsen en kopiëren - [x] Offline download - [x] Bestanden kopiëren tussen twee opslaglocaties - [x] Multi-thread downloadversnelling voor enkelvoudige download/stream ## Documentatie - 📘 [Global Site](https://doc.oplist.org) - 📚 [Backup Site](https://doc.openlist.team) - 🌏 [CN Site](https://doc.oplist.org.cn) ## Demo - 🌎 [Global Demo](https://demo.oplist.org) - 🇨🇳 [CN Demo](https://demo.oplist.org.cn) ## Discussie Stel algemene vragen in [*Discussions*](https://github.com/OpenListTeam/OpenList/discussions), ***Issues* zijn alleen voor bugmeldingen en feature requests.** ## Sponsoren [![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") ## Licentie `OpenList` is open-source software onder de [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) licentie. ## Disclaimer - Dit project is gratis en open-source software, ontworpen om het delen van bestanden via netdisks te vergemakkelijken, voornamelijk bedoeld ter ondersteuning van het downloaden en leren van de programmeertaal Go. - Houd u bij het gebruik van deze software aan alle toepasselijke wetten en voorschriften. Elk misbruik is ten strengste verboden. - De software is gebaseerd op officiële SDK's of API's zonder enige wijziging, verstoring of beïnvloeding van hun gedrag. - Het voert alleen HTTP 302-omleidingen of verkeersdoorsturing uit en onderschept, slaat of wijzigt geen gebruikersgegevens. - Dit project is niet gelieerd aan enig officieel platform of dienstverlener. - De software wordt geleverd "zoals deze is", zonder enige vorm van garantie, expliciet of impliciet, inclusief maar niet beperkt tot garanties van verkoopbaarheid of geschiktheid voor een bepaald doel. - De beheerders zijn niet aansprakelijk voor enige directe of indirecte schade die voortvloeit uit het gebruik van of het onvermogen om deze software te gebruiken. - U bent zelf verantwoordelijk voor alle risico's die gepaard gaan met het gebruik van deze software, inclusief maar niet beperkt tot accountblokkades of downloadbeperkingen. - Dit project valt onder de [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) licentie. Zie het [LICENSE](./LICENSE) bestand voor details. ## Contact - [@GitHub](https://github.com/OpenListTeam) - [Telegram Groep](https://t.me/OpenListTeam) - [Telegram Kanaal](https://t.me/OpenListOfficial) ## Bijdragers Wij danken de auteur [Xhofe](https://github.com/Xhofe) van het originele project [AlistGo/alist](https://github.com/AlistGo/alist) en alle andere bijdragers. Dank aan deze geweldige mensen: [![Contributors](https://contrib.rocks/image?repo=OpenListTeam/OpenList)](https://github.com/OpenListTeam/OpenList/graphs/contributors) ================================================ FILE: SECURITY.md ================================================ # Security Policy ## Supported Versions Only the latest stable release receives security patches. We strongly recommend always keeping OpenList up to date. | Version | Supported | | -------------------- | ------------------ | | Latest stable (v4.x) | :white_check_mark: | | Older versions | :x: | ## Reporting a Vulnerability **Please do NOT report security vulnerabilities through public GitHub Issues.** If you discover a security vulnerability in OpenList, please report it responsibly by using one of the following channels: - **GitHub Private Security Advisory** (preferred): [Submit here](https://github.com/OpenListTeam/OpenList/security/advisories/new) - **Telegram**: Contact a maintainer privately via [@OpenListTeam](https://t.me/OpenListTeam) When reporting, please include as much of the following as possible: - A description of the vulnerability and its potential impact - The affected version(s) - Step-by-step instructions to reproduce the issue - Any proof-of-concept code or screenshots (if applicable) - Suggested mitigation or fix (optional but appreciated) ## Security Best Practices for Users To keep your OpenList instance secure: - Always update to the latest release. - Use a strong, unique admin password and change it after first login. - Enable HTTPS (TLS) for your deployment — do **not** expose OpenList over plain HTTP on the public internet. - Limit exposed ports using a reverse proxy (e.g., Nginx, Caddy). - Set up access controls and avoid enabling guest access unless necessary. - Regularly review mounted storage permissions and revoke unused API tokens. - When using Docker, avoid running the container as root if possible. ## Acknowledgments We sincerely thank all security researchers and community members who responsibly disclose vulnerabilities and help make OpenList safer for everyone. --- # 安全政策 ## 支持的版本 我们仅对最新稳定版本提供安全补丁。强烈建议始终保持 OpenList 为最新版本。 | 版本 | 是否支持 | | ------------------ | ------------------ | | 最新稳定版（v4.x） | :white_check_mark: | | 旧版本 | :x: | ## 报告漏洞 **请勿通过公开的 GitHub Issues 报告安全漏洞。** 如果您在 OpenList 中发现安全漏洞，请通过以下渠道之一负责任地进行报告： - **GitHub 私密安全公告**（推荐）：[点击提交](https://github.com/OpenListTeam/OpenList/security/advisories/new) - **Telegram**：通过 [@OpenListTeam](https://t.me/OpenListTeam) 私信联系维护者 报告时，请尽量提供以下信息： - 漏洞描述及其潜在影响 - 受影响的版本 - 复现问题的详细步骤 - 概念验证代码或截图（如有） - 建议的缓解措施或修复方案（可选，但非常欢迎） ## 用户安全最佳实践 为保障您的 OpenList 实例安全： - 始终更新至最新版本。 - 使用强且唯一的管理员密码，并在首次登录后立即修改。 - 为您的部署启用 HTTPS（TLS）—— **请勿**在公网上以明文 HTTP 方式暴露 OpenList。 - 使用反向代理（如 Nginx、Caddy）限制对外暴露的端口。 - 配置访问控制，非必要情况下不要开启访客访问。 - 定期检查已挂载存储的权限，并撤销未使用的 API 令牌。 - 使用 Docker 部署时，尽可能避免以 root 用户运行容器。 ## 致谢 我们衷心感谢所有负责任地披露漏洞、帮助 OpenList 变得更加安全的安全研究人员和社区成员。 ================================================ FILE: build.sh ================================================ set -e appName="openlist" builtAt="$(date +'%F %T %z')" gitAuthor="The OpenList Projects Contributors " gitCommit=$(git log --pretty=format:"%h" -1) # Set frontend repository, default to OpenListTeam/OpenList-Frontend frontendRepo="${FRONTEND_REPO:-OpenListTeam/OpenList-Frontend}" githubAuthArgs="" if [ -n "$GITHUB_TOKEN" ]; then githubAuthArgs="--header \"Authorization: Bearer $GITHUB_TOKEN\"" fi # Check for lite parameter useLite=false if [[ "$*" == *"lite"* ]]; then useLite=true fi if [ "$1" = "dev" ]; then version="dev" webVersion="rolling" elif [ "$1" = "beta" ]; then version="beta" webVersion="rolling" else git tag -d beta || true # Always true if there's no tag version=$(git describe --abbrev=0 --tags 2>/dev/null || echo "v0.0.0") webVersion=$(eval "curl -fsSL --max-time 2 $githubAuthArgs \"https://api.github.com/repos/$frontendRepo/releases/latest\"" | grep "tag_name" | head -n 1 | awk -F ":" '{print $2}' | sed 's/\"//g;s/,//g;s/ //g') fi echo "backend version: $version" echo "frontend version: $webVersion" if [ "$useLite" = true ]; then echo "using lite frontend" else echo "using standard frontend" fi ldflags="\ -w -s \ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$builtAt' \ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=$gitAuthor' \ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$gitCommit' \ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$version' \ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=$webVersion' \ " FetchWebRolling() { pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/tags/rolling\"") pre_release_assets=$(echo "$pre_release_json" | jq -r '.assets[].browser_download_url') # There is no lite for rolling pre_release_tar_url=$(echo "$pre_release_assets" | grep "openlist-frontend-dist" | grep -v "lite" | grep "\.tar\.gz$") curl -fsSL "$pre_release_tar_url" -o dist.tar.gz rm -rf public/dist && mkdir -p public/dist tar -zxvf dist.tar.gz -C public/dist rm -rf dist.tar.gz } FetchWebRelease() { release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/latest\"") release_assets=$(echo "$release_json" | jq -r '.assets[].browser_download_url') if [ "$useLite" = true ]; then release_tar_url=$(echo "$release_assets" | grep "openlist-frontend-dist-lite" | grep "\.tar\.gz$") else release_tar_url=$(echo "$release_assets" | grep "openlist-frontend-dist" | grep -v "lite" | grep "\.tar\.gz$") fi curl -fsSL "$release_tar_url" -o dist.tar.gz rm -rf public/dist && mkdir -p public/dist tar -zxvf dist.tar.gz -C public/dist rm -rf dist.tar.gz } BuildWinArm64() { echo building for windows-arm64 chmod +x ./wrapper/zcc-arm64 chmod +x ./wrapper/zcxx-arm64 export GOOS=windows export GOARCH=arm64 export CC=$(pwd)/wrapper/zcc-arm64 export CXX=$(pwd)/wrapper/zcxx-arm64 export CGO_ENABLED=1 go build -o "$1" -ldflags="$ldflags" -tags=jsoniter . } BuildWin7() { # Setup Win7 Go compiler (patched version that supports Windows 7) go_version=$(go version | grep -o 'go[0-9]\+\.[0-9]\+\.[0-9]\+' | sed 's/go//') echo "Detected Go version: $go_version" curl -fsSL --retry 3 -o go-win7.zip -H "Authorization: Bearer $GITHUB_TOKEN" \ "https://github.com/XTLS/go-win7/releases/download/patched-${go_version}/go-for-win7-linux-amd64.zip" rm -rf go-win7 unzip go-win7.zip -d go-win7 rm go-win7.zip # Set permissions for all wrapper files chmod +x ./wrapper/zcc-win7 chmod +x ./wrapper/zcxx-win7 chmod +x ./wrapper/zcc-win7-386 chmod +x ./wrapper/zcxx-win7-386 # Build for both 386 and amd64 architectures for arch in "386" "amd64"; do echo "building for windows7-${arch}" export GOOS=windows export GOARCH=${arch} export CGO_ENABLED=1 # Use architecture-specific wrapper files if [ "$arch" = "386" ]; then export CC=$(pwd)/wrapper/zcc-win7-386 export CXX=$(pwd)/wrapper/zcxx-win7-386 else export CC=$(pwd)/wrapper/zcc-win7 export CXX=$(pwd)/wrapper/zcxx-win7 fi # Use the patched Go compiler for Win7 compatibility $(pwd)/go-win7/bin/go build -o "${1}-${arch}.exe" -ldflags="$ldflags" -tags=jsoniter . done } BuildDev() { rm -rf .git/ mkdir -p "dist" muslflags="--extldflags '-static -fpic' $ldflags" BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross) for i in "${FILES[@]}"; do url="${BASE}${i}.tgz" curl -fsSL -o "${i}.tgz" "${url}" sudo tar xf "${i}.tgz" --strip-components 1 -C /usr/local done OS_ARCHES=(linux-musl-amd64 linux-musl-arm64) CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc) for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} echo building for ${os_arch} export GOOS=${os_arch%%-*} export GOARCH=${os_arch##*-} export CC=${cgo_cc} export CGO_ENABLED=1 go build -o ./dist/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . done xgo -targets=windows/amd64,darwin/amd64,darwin/arm64 -out "$appName" -ldflags="$ldflags" -tags=jsoniter . mv "$appName"-* dist cd dist # cp ./"$appName"-windows-amd64.exe ./"$appName"-windows-amd64-upx.exe # upx -9 ./"$appName"-windows-amd64-upx.exe find . -type f -print0 | xargs -0 md5sum >md5.txt cat md5.txt } BuildDocker() { go build -o ./bin/"$appName" -ldflags="$ldflags" -tags=jsoniter . } PrepareBuildDockerMusl() { mkdir -p build/musl-libs BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross i486-linux-musl-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross riscv64-linux-musl-cross powerpc64le-linux-musl-cross loongarch64-linux-musl-cross) ## Disable s390x-linux-musl-cross builds for i in "${FILES[@]}"; do url="${BASE}${i}.tgz" lib_tgz="build/${i}.tgz" curl -fsSL -o "${lib_tgz}" "${url}" tar xf "${lib_tgz}" --strip-components 1 -C build/musl-libs rm -f "${lib_tgz}" done } BuildDockerMultiplatform() { go mod download # run PrepareBuildDockerMusl before build export PATH=$PATH:$PWD/build/musl-libs/bin docker_lflags="--extldflags '-static -fpic' $ldflags" export CGO_ENABLED=1 OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-riscv64 linux-ppc64le linux-loong64) ## Disable linux-s390x builds CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc loongarch64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} os=${os_arch%%-*} arch=${os_arch##*-} export GOOS=$os export GOARCH=$arch export CC=${cgo_cc} echo "building for $os_arch" go build -o build/$os/$arch/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . done DOCKER_ARM_ARCHES=(linux-arm/v6 linux-arm/v7) CGO_ARGS=(armv6-linux-musleabihf-gcc armv7l-linux-musleabihf-gcc) GO_ARM=(6 7) export GOOS=linux export GOARCH=arm for i in "${!DOCKER_ARM_ARCHES[@]}"; do docker_arch=${DOCKER_ARM_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} export GOARM=${GO_ARM[$i]} export CC=${cgo_cc} echo "building for $docker_arch" go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . done } BuildRelease() { rm -rf .git/ mkdir -p "build" BuildWinArm64 ./build/"$appName"-windows-arm64.exe BuildWin7 ./build/"$appName"-windows7 xgo -out "$appName" -ldflags="$ldflags" -tags=jsoniter . # why? Because some target platforms seem to have issues with upx compression # upx -9 ./"$appName"-linux-amd64 # cp ./"$appName"-windows-amd64.exe ./"$appName"-windows-amd64-upx.exe # upx -9 ./"$appName"-windows-amd64-upx.exe mv "$appName"-* build # Build LoongArch with glibc (both old world abi1.0 and new world abi2.0) # Separate from musl builds to avoid cache conflicts BuildLoongGLIBC ./build/$appName-linux-loong64-abi1.0 abi1.0 BuildLoongGLIBC ./build/$appName-linux-loong64 abi2.0 } BuildLoongGLIBC() { local target_abi="$2" local output_file="$1" local oldWorldGoVersion="1.25.0" if [ "$target_abi" = "abi1.0" ]; then echo building for linux-loong64-abi1.0 else echo building for linux-loong64-abi2.0 target_abi="abi2.0" # Default to abi2.0 if not specified fi # Note: No longer need global cache cleanup since ABI1.0 uses isolated cache directory echo "Using optimized cache strategy: ABI1.0 has isolated cache, ABI2.0 uses standard cache" if [ "$target_abi" = "abi1.0" ]; then # Setup abi1.0 toolchain and patched Go compiler similar to cgo-action implementation echo "Setting up Loongson old-world ABI1.0 toolchain and patched Go compiler..." # Download and setup patched Go compiler for old-world if ! curl -fsSL --retry 3 -H "Authorization: Bearer $GITHUB_TOKEN" \ "https://github.com/loong64/loong64-abi1.0-toolchains/releases/download/20250821/go${oldWorldGoVersion}.linux-amd64.tar.gz" \ -o go-loong64-abi1.0.tar.gz; then echo "Error: Failed to download patched Go compiler for old-world ABI1.0" if [ -n "$GITHUB_TOKEN" ]; then echo "Error output from curl:" curl -fsSL --retry 3 -H "Authorization: Bearer $GITHUB_TOKEN" \ "https://github.com/loong64/loong64-abi1.0-toolchains/releases/download/20250821/go${oldWorldGoVersion}.linux-amd64.tar.gz" \ -o go-loong64-abi1.0.tar.gz || true fi return 1 fi rm -rf go-loong64-abi1.0 mkdir go-loong64-abi1.0 if ! tar -xzf go-loong64-abi1.0.tar.gz -C go-loong64-abi1.0 --strip-components=1; then echo "Error: Failed to extract patched Go compiler" return 1 fi rm go-loong64-abi1.0.tar.gz # Download and setup GCC toolchain for old-world if ! curl -fsSL --retry 3 -H "Authorization: Bearer $GITHUB_TOKEN" \ "https://github.com/loong64/loong64-abi1.0-toolchains/releases/download/20250722/loongson-gnu-toolchain-8.3.novec-x86_64-loongarch64-linux-gnu-rc1.1.tar.xz" \ -o gcc8-loong64-abi1.0.tar.xz; then echo "Error: Failed to download GCC toolchain for old-world ABI1.0" if [ -n "$GITHUB_TOKEN" ]; then echo "Error output from curl:" curl -fsSL --retry 3 -H "Authorization: Bearer $GITHUB_TOKEN" \ "https://github.com/loong64/loong64-abi1.0-toolchains/releases/download/20250722/loongson-gnu-toolchain-8.3.novec-x86_64-loongarch64-linux-gnu-rc1.1.tar.xz" \ -o gcc8-loong64-abi1.0.tar.xz || true fi return 1 fi rm -rf gcc8-loong64-abi1.0 mkdir gcc8-loong64-abi1.0 if ! tar -Jxf gcc8-loong64-abi1.0.tar.xz -C gcc8-loong64-abi1.0 --strip-components=1; then echo "Error: Failed to extract GCC toolchain" return 1 fi rm gcc8-loong64-abi1.0.tar.xz # Setup separate cache directory for ABI1.0 to avoid cache pollution abi1_cache_dir="$(pwd)/go-loong64-abi1.0-cache" mkdir -p "$abi1_cache_dir" echo "Using separate cache directory for ABI1.0: $abi1_cache_dir" # Use patched Go compiler for old-world build (critical for ABI1.0 compatibility) echo "Building with patched Go compiler for old-world ABI1.0..." echo "Using isolated cache directory: $abi1_cache_dir" # Use env command to set environment variables locally without affecting global environment if ! env GOOS=linux GOARCH=loong64 \ CC="$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-gcc" \ CXX="$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-g++" \ CGO_ENABLED=1 \ GOCACHE="$abi1_cache_dir" \ $(pwd)/go-loong64-abi1.0/bin/go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then echo "Error: Build failed with patched Go compiler" echo "Attempting retry with cache cleanup..." env GOCACHE="$abi1_cache_dir" $(pwd)/go-loong64-abi1.0/bin/go clean -cache if ! env GOOS=linux GOARCH=loong64 \ CC="$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-gcc" \ CXX="$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-g++" \ CGO_ENABLED=1 \ GOCACHE="$abi1_cache_dir" \ $(pwd)/go-loong64-abi1.0/bin/go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then echo "Error: Build failed again after cache cleanup" echo "Build environment details:" echo "GOOS=linux" echo "GOARCH=loong64" echo "CC=$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-gcc" echo "CXX=$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-g++" echo "CGO_ENABLED=1" echo "GOCACHE=$abi1_cache_dir" echo "Go version: $($(pwd)/go-loong64-abi1.0/bin/go version)" echo "GCC version: $($(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-gcc --version | head -1)" return 1 fi fi else # Setup abi2.0 toolchain for new world glibc build echo "Setting up new-world ABI2.0 toolchain..." if ! curl -fsSL --retry 3 -H "Authorization: Bearer $GITHUB_TOKEN" \ "https://github.com/loong64/cross-tools/releases/download/20250507/x86_64-cross-tools-loongarch64-unknown-linux-gnu-legacy.tar.xz" \ -o gcc12-loong64-abi2.0.tar.xz; then echo "Error: Failed to download GCC toolchain for new-world ABI2.0" if [ -n "$GITHUB_TOKEN" ]; then echo "Error output from curl:" curl -fsSL --retry 3 -H "Authorization: Bearer $GITHUB_TOKEN" \ "https://github.com/loong64/cross-tools/releases/download/20250507/x86_64-cross-tools-loongarch64-unknown-linux-gnu-legacy.tar.xz" \ -o gcc12-loong64-abi2.0.tar.xz || true fi return 1 fi rm -rf gcc12-loong64-abi2.0 mkdir gcc12-loong64-abi2.0 if ! tar -Jxf gcc12-loong64-abi2.0.tar.xz -C gcc12-loong64-abi2.0 --strip-components=1; then echo "Error: Failed to extract GCC toolchain" return 1 fi rm gcc12-loong64-abi2.0.tar.xz export GOOS=linux export GOARCH=loong64 export CC=$(pwd)/gcc12-loong64-abi2.0/bin/loongarch64-unknown-linux-gnu-gcc export CXX=$(pwd)/gcc12-loong64-abi2.0/bin/loongarch64-unknown-linux-gnu-g++ export CGO_ENABLED=1 # Use standard Go compiler for new-world build echo "Building with standard Go compiler for new-world ABI2.0..." if ! go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then echo "Error: Build failed with standard Go compiler" echo "Attempting retry with cache cleanup..." go clean -cache if ! go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then echo "Error: Build failed again after cache cleanup" echo "Build environment details:" echo "GOOS=$GOOS" echo "GOARCH=$GOARCH" echo "CC=$CC" echo "CXX=$CXX" echo "CGO_ENABLED=$CGO_ENABLED" echo "Go version: $(go version)" echo "GCC version: $($CC --version | head -1)" return 1 fi fi fi } BuildReleaseLinuxMusl() { rm -rf .git/ mkdir -p "build" muslflags="--extldflags '-static -fpic' $ldflags" BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross mips-linux-musl-cross mips64-linux-musl-cross mips64el-linux-musl-cross mipsel-linux-musl-cross powerpc64le-linux-musl-cross s390x-linux-musl-cross loongarch64-linux-musl-cross) for i in "${FILES[@]}"; do url="${BASE}${i}.tgz" curl -fsSL -o "${i}.tgz" "${url}" sudo tar xf "${i}.tgz" --strip-components 1 -C /usr/local rm -f "${i}.tgz" done OS_ARCHES=(linux-musl-amd64 linux-musl-arm64 linux-musl-mips linux-musl-mips64 linux-musl-mips64le linux-musl-mipsle linux-musl-ppc64le linux-musl-s390x linux-musl-loong64) CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc mips-linux-musl-gcc mips64-linux-musl-gcc mips64el-linux-musl-gcc mipsel-linux-musl-gcc powerpc64le-linux-musl-gcc s390x-linux-musl-gcc loongarch64-linux-musl-gcc) for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} echo building for ${os_arch} export GOOS=${os_arch%%-*} export GOARCH=${os_arch##*-} export CC=${cgo_cc} export CGO_ENABLED=1 go build -o ./build/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . done } BuildReleaseLinuxMuslArm() { rm -rf .git/ mkdir -p "build" muslflags="--extldflags '-static -fpic' $ldflags" BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" FILES=(arm-linux-musleabi-cross arm-linux-musleabihf-cross armel-linux-musleabi-cross armel-linux-musleabihf-cross armv5l-linux-musleabi-cross armv5l-linux-musleabihf-cross armv6-linux-musleabi-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross armv7m-linux-musleabi-cross armv7r-linux-musleabihf-cross) for i in "${FILES[@]}"; do url="${BASE}${i}.tgz" curl -fsSL -o "${i}.tgz" "${url}" sudo tar xf "${i}.tgz" --strip-components 1 -C /usr/local rm -f "${i}.tgz" done OS_ARCHES=(linux-musleabi-arm linux-musleabihf-arm linux-musleabi-armel linux-musleabihf-armel linux-musleabi-armv5l linux-musleabihf-armv5l linux-musleabi-armv6 linux-musleabihf-armv6 linux-musleabihf-armv7l linux-musleabi-armv7m linux-musleabihf-armv7r) CGO_ARGS=(arm-linux-musleabi-gcc arm-linux-musleabihf-gcc armel-linux-musleabi-gcc armel-linux-musleabihf-gcc armv5l-linux-musleabi-gcc armv5l-linux-musleabihf-gcc armv6-linux-musleabi-gcc armv6-linux-musleabihf-gcc armv7l-linux-musleabihf-gcc armv7m-linux-musleabi-gcc armv7r-linux-musleabihf-gcc) GOARMS=('' '' '' '' '5' '5' '6' '6' '7' '7' '7') for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} arm=${GOARMS[$i]} echo building for ${os_arch} export GOOS=linux export GOARCH=arm export CC=${cgo_cc} export CGO_ENABLED=1 export GOARM=${arm} go build -o ./build/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . done } BuildReleaseAndroid() { rm -rf .git/ mkdir -p "build" wget https://dl.google.com/android/repository/android-ndk-r26b-linux.zip unzip android-ndk-r26b-linux.zip rm android-ndk-r26b-linux.zip OS_ARCHES=(amd64 arm64 386 arm) CGO_ARGS=(x86_64-linux-android24-clang aarch64-linux-android24-clang i686-linux-android24-clang armv7a-linux-androideabi24-clang) for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=$(realpath android-ndk-r26b/toolchains/llvm/prebuilt/linux-x86_64/bin/${CGO_ARGS[$i]}) echo building for android-${os_arch} export GOOS=android export GOARCH=${os_arch##*-} export CC=${cgo_cc} export CGO_ENABLED=1 go build -o ./build/$appName-android-$os_arch -ldflags="$ldflags" -tags=jsoniter . android-ndk-r26b/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip ./build/$appName-android-$os_arch done } BuildReleaseFreeBSD() { rm -rf .git/ mkdir -p "build/freebsd" # Get latest FreeBSD 14.x release version from GitHub freebsd_version=$(eval "curl -fsSL --max-time 2 $githubAuthArgs \"https://api.github.com/repos/freebsd/freebsd-src/tags\"" | \ jq -r '.[].name' | \ grep '^release/14\.' | \ grep -v -- '-p[0-9]*$' | \ sort -V | \ tail -1 | \ sed 's/release\///' | \ sed 's/\.0$//') if [ -z "$freebsd_version" ]; then echo "Failed to get FreeBSD version, falling back to 14.3" freebsd_version="14.3" fi echo "Using FreeBSD version: $freebsd_version" OS_ARCHES=(amd64 arm64 i386) GO_ARCHES=(amd64 arm64 386) CGO_ARGS=(x86_64-unknown-freebsd${freebsd_version} aarch64-unknown-freebsd${freebsd_version} i386-unknown-freebsd${freebsd_version}) for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc="clang --target=${CGO_ARGS[$i]} --sysroot=/opt/freebsd/${os_arch}" echo building for freebsd-${os_arch} sudo mkdir -p "/opt/freebsd/${os_arch}" wget -q https://download.freebsd.org/releases/${os_arch}/${freebsd_version}-RELEASE/base.txz sudo tar -xf ./base.txz -C /opt/freebsd/${os_arch} rm base.txz export GOOS=freebsd export GOARCH=${GO_ARCHES[$i]} export CC=${cgo_cc} export CGO_ENABLED=1 export CGO_LDFLAGS="-fuse-ld=lld" go build -o ./build/$appName-freebsd-$os_arch -ldflags="$ldflags" -tags=jsoniter . done } MakeRelease() { cd build if [ -d compress ]; then rm -rv compress fi mkdir compress # Add -lite suffix if useLite is true liteSuffix="" if [ "$useLite" = true ]; then liteSuffix="-lite" fi for i in $(find . -type f -name "$appName-linux-*"); do cp "$i" "$appName" tar -czvf compress/"$i$liteSuffix".tar.gz "$appName" rm -f "$appName" done for i in $(find . -type f -name "$appName-android-*"); do cp "$i" "$appName" tar -czvf compress/"$i$liteSuffix".tar.gz "$appName" rm -f "$appName" done for i in $(find . -type f -name "$appName-darwin-*"); do cp "$i" "$appName" tar -czvf compress/"$i$liteSuffix".tar.gz "$appName" rm -f "$appName" done for i in $(find . -type f -name "$appName-freebsd-*"); do cp "$i" "$appName" tar -czvf compress/"$i$liteSuffix".tar.gz "$appName" rm -f "$appName" done for i in $(find . -type f \( -name "$appName-windows-*" -o -name "$appName-windows7-*" \)); do cp "$i" "$appName".exe zip compress/$(echo $i | sed 's/\.[^.]*$//')$liteSuffix.zip "$appName".exe rm -f "$appName".exe done cd compress # Handle MD5 filename - add -lite suffix only if not already present md5FileName="$1" if [ "$useLite" = true ] && [[ "$1" != *"-lite.txt" ]]; then md5FileName=$(echo "$1" | sed 's/\.txt$/-lite.txt/') fi find . -type f -print0 | xargs -0 md5sum >"$md5FileName" cat "$md5FileName" cd ../.. } # Parse parameters to handle lite parameter position flexibility buildType="" dockerType="" otherParam="" for arg in "$@"; do case $arg in dev|beta|release|zip|prepare) if [ -z "$buildType" ]; then buildType="$arg" fi ;; docker|docker-multiplatform|linux_musl_arm|linux_musl|android|freebsd|web) if [ -z "$dockerType" ]; then dockerType="$arg" fi ;; lite) # lite parameter is already handled above ;; *) if [ -z "$otherParam" ]; then otherParam="$arg" fi ;; esac done if [ "$buildType" = "dev" ]; then FetchWebRolling if [ "$dockerType" = "docker" ]; then BuildDocker elif [ "$dockerType" = "docker-multiplatform" ]; then BuildDockerMultiplatform elif [ "$dockerType" = "web" ]; then echo "web only" else BuildDev fi elif [ "$buildType" = "release" -o "$buildType" = "beta" ]; then if [ "$buildType" = "beta" ]; then FetchWebRolling else FetchWebRelease fi if [ "$dockerType" = "docker" ]; then BuildDocker elif [ "$dockerType" = "docker-multiplatform" ]; then BuildDockerMultiplatform elif [ "$dockerType" = "linux_musl_arm" ]; then BuildReleaseLinuxMuslArm if [ "$useLite" = true ]; then MakeRelease "md5-linux-musl-arm-lite.txt" else MakeRelease "md5-linux-musl-arm.txt" fi elif [ "$dockerType" = "linux_musl" ]; then BuildReleaseLinuxMusl if [ "$useLite" = true ]; then MakeRelease "md5-linux-musl-lite.txt" else MakeRelease "md5-linux-musl.txt" fi elif [ "$dockerType" = "android" ]; then BuildReleaseAndroid if [ "$useLite" = true ]; then MakeRelease "md5-android-lite.txt" else MakeRelease "md5-android.txt" fi elif [ "$dockerType" = "freebsd" ]; then BuildReleaseFreeBSD if [ "$useLite" = true ]; then MakeRelease "md5-freebsd-lite.txt" else MakeRelease "md5-freebsd.txt" fi elif [ "$dockerType" = "web" ]; then echo "web only" else BuildRelease if [ "$useLite" = true ]; then MakeRelease "md5-lite.txt" else MakeRelease "md5.txt" fi fi elif [ "$buildType" = "prepare" ]; then if [ "$dockerType" = "docker-multiplatform" ]; then PrepareBuildDockerMusl fi elif [ "$buildType" = "zip" ]; then if [ -n "$otherParam" ]; then if [ "$useLite" = true ]; then MakeRelease "$otherParam-lite.txt" else MakeRelease "$otherParam.txt" fi elif [ -n "$dockerType" ]; then if [ "$useLite" = true ]; then MakeRelease "$dockerType-lite.txt" else MakeRelease "$dockerType.txt" fi else if [ "$useLite" = true ]; then MakeRelease "md5-lite.txt" else MakeRelease "md5.txt" fi fi else echo -e "Parameter error" echo -e "Usage: $0 {dev|beta|release|zip|prepare} [docker|docker-multiplatform|linux_musl_arm|linux_musl|android|freebsd|web] [lite] [other_params]" echo -e "Examples:" echo -e " $0 dev" echo -e " $0 dev lite" echo -e " $0 dev docker" echo -e " $0 dev docker lite" echo -e " $0 release" echo -e " $0 release lite" echo -e " $0 release docker lite" echo -e " $0 release linux_musl" fi ================================================ FILE: cmd/admin.go ================================================ /* Copyright © 2022 NAME HERE */ package cmd import ( "fmt" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/spf13/cobra" ) // AdminCmd represents the password command var AdminCmd = &cobra.Command{ Use: "admin", Aliases: []string{"password"}, Short: "Show admin user's info and some operations about admin user's password", Run: func(cmd *cobra.Command, args []string) { bootstrap.Init() defer bootstrap.Release() admin, err := op.GetAdmin() if err != nil { utils.Log.Errorf("failed get admin user: %+v", err) } else { utils.Log.Infof("get admin user from CLI") fmt.Println("Admin user's username:", admin.Username) fmt.Println("The password can only be output at the first startup, and then stored as a hash value, which cannot be reversed") fmt.Println("You can reset the password with a random string by running [openlist admin random]") fmt.Println("You can also set a new password by running [openlist admin set NEW_PASSWORD]") } }, } var RandomPasswordCmd = &cobra.Command{ Use: "random", Short: "Reset admin user's password to a random string", Run: func(cmd *cobra.Command, args []string) { utils.Log.Infof("reset admin user's password to a random string from CLI") newPwd := random.String(8) setAdminPassword(newPwd) }, } var SetPasswordCmd = &cobra.Command{ Use: "set", Short: "Set admin user's password", RunE: func(cmd *cobra.Command, args []string) error { if len(args) == 0 { return fmt.Errorf("Please enter the new password") } setAdminPassword(args[0]) return nil }, } var ShowTokenCmd = &cobra.Command{ Use: "token", Short: "Show admin token", Run: func(cmd *cobra.Command, args []string) { bootstrap.Init() defer bootstrap.Release() token := setting.GetStr(conf.Token) utils.Log.Infof("show admin token from CLI") fmt.Println("Admin token:", token) }, } func setAdminPassword(pwd string) { bootstrap.Init() defer bootstrap.Release() admin, err := op.GetAdmin() if err != nil { utils.Log.Errorf("failed get admin user: %+v", err) return } admin.SetPassword(pwd) if err := op.UpdateUser(admin); err != nil { utils.Log.Errorf("failed update admin user: %+v", err) return } utils.Log.Infof("admin user has been update from CLI") fmt.Println("admin user has been updated:") fmt.Println("username:", admin.Username) fmt.Println("password:", pwd) DelAdminCacheOnline() } func init() { RootCmd.AddCommand(AdminCmd) AdminCmd.AddCommand(RandomPasswordCmd) AdminCmd.AddCommand(SetPasswordCmd) AdminCmd.AddCommand(ShowTokenCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // passwordCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: // passwordCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } ================================================ FILE: cmd/cancel2FA.go ================================================ /* Copyright © 2022 NAME HERE */ package cmd import ( "fmt" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/spf13/cobra" ) // Cancel2FACmd represents the delete2fa command var Cancel2FACmd = &cobra.Command{ Use: "cancel2fa", Short: "Delete 2FA of admin user", Run: func(cmd *cobra.Command, args []string) { bootstrap.Init() defer bootstrap.Release() admin, err := op.GetAdmin() if err != nil { utils.Log.Errorf("failed to get admin user: %+v", err) } else { err := op.Cancel2FAByUser(admin) if err != nil { utils.Log.Errorf("failed to cancel 2FA: %+v", err) } else { utils.Log.Infof("2FA is canceled from CLI") fmt.Println("2FA canceled") DelAdminCacheOnline() } } }, } func init() { RootCmd.AddCommand(Cancel2FACmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // cancel2FACmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: // cancel2FACmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } ================================================ FILE: cmd/common.go ================================================ package cmd import ( "os" "path/filepath" "strconv" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" ) func Init() { bootstrap.Init() } func Release() { bootstrap.Release() } var pid = -1 var pidFile string func initDaemon() { ex, err := os.Executable() if err != nil { log.Fatal(err) } exPath := filepath.Dir(ex) _ = os.MkdirAll(filepath.Join(exPath, "daemon"), 0700) pidFile = filepath.Join(exPath, "daemon/pid") if utils.Exists(pidFile) { bytes, err := os.ReadFile(pidFile) if err != nil { log.Fatal("failed to read pid file", err) } id, err := strconv.Atoi(string(bytes)) if err != nil { log.Fatal("failed to parse pid data", err) } pid = id } } ================================================ FILE: cmd/crypt.go ================================================ package cmd import ( "io" "os" "path" "path/filepath" "strings" rcCrypt "github.com/rclone/rclone/backend/crypt" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/obscure" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) // encryption and decryption command format for Crypt driver type options struct { op string //decrypt or encrypt src string //source dir or file dst string //out destination pwd string //de/encrypt password salt string filenameEncryption string //reference drivers\crypt\meta.go Addtion dirnameEncryption string filenameEncode string suffix string } var opt options // CryptCmd represents the crypt command var CryptCmd = &cobra.Command{ Use: "crypt", Short: "Encrypt or decrypt local file or dir", Example: `openlist crypt -s ./src/encrypt/ --op=de --pwd=123456 --salt=345678`, Run: func(cmd *cobra.Command, args []string) { opt.validate() opt.cryptFileDir() }, } func init() { RootCmd.AddCommand(CryptCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // versionCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: CryptCmd.Flags().StringVarP(&opt.src, "src", "s", "", "src file or dir to encrypt/decrypt") CryptCmd.Flags().StringVarP(&opt.dst, "dst", "d", "", "dst dir to output,if not set,output to src dir") CryptCmd.Flags().StringVar(&opt.op, "op", "", "de or en which stands for decrypt or encrypt") CryptCmd.Flags().StringVar(&opt.pwd, "pwd", "", "password used to encrypt/decrypt,if not contain ___Obfuscated___ prefix,will be obfuscated before used") CryptCmd.Flags().StringVar(&opt.salt, "salt", "", "salt used to encrypt/decrypt,if not contain ___Obfuscated___ prefix,will be obfuscated before used") CryptCmd.Flags().StringVar(&opt.filenameEncryption, "filename-encrypt", "off", "filename encryption mode: off,standard,obfuscate") CryptCmd.Flags().StringVar(&opt.dirnameEncryption, "dirname-encrypt", "false", "is dirname encryption enabled:true,false") CryptCmd.Flags().StringVar(&opt.filenameEncode, "filename-encode", "base64", "filename encoding mode: base64,base32,base32768") CryptCmd.Flags().StringVar(&opt.suffix, "suffix", ".bin", "suffix for encrypted file,default is .bin") } func (o *options) validate() { if o.src == "" { log.Fatal("src can not be empty") } if o.op != "encrypt" && o.op != "decrypt" && o.op != "en" && o.op != "de" { log.Fatal("op must be encrypt or decrypt") } if o.filenameEncryption != "off" && o.filenameEncryption != "standard" && o.filenameEncryption != "obfuscate" { log.Fatal("filename_encryption must be off,standard,obfuscate") } if o.filenameEncode != "base64" && o.filenameEncode != "base32" && o.filenameEncode != "base32768" { log.Fatal("filename_encode must be base64,base32,base32768") } } func (o *options) cryptFileDir() { src, _ := filepath.Abs(o.src) log.Infof("src abs is %v", src) fileInfo, err := os.Stat(src) if err != nil { log.Fatalf("reading file/dir %v failed,err:%v", src, err) } pwd := updateObfusParm(o.pwd) salt := updateObfusParm(o.salt) //create cipher config := configmap.Simple{ "password": pwd, "password2": salt, "filename_encryption": o.filenameEncryption, "directory_name_encryption": o.dirnameEncryption, "filename_encoding": o.filenameEncode, "suffix": o.suffix, "pass_bad_blocks": "", } log.Infof("config:%v", config) cipher, err := rcCrypt.NewCipher(config) if err != nil { log.Fatalf("create cipher failed,err:%v", err) } dst := "" //check and create dst dir if o.dst != "" { dst, _ = filepath.Abs(o.dst) checkCreateDir(dst) } // src is file if !fileInfo.IsDir() { //file if dst == "" { dst = filepath.Dir(src) } o.cryptFile(cipher, src, dst) return } // src is dir if dst == "" { //if src is dir and not set dst dir ,create ${src}_crypt dir as dst dir dst = path.Join(filepath.Dir(src), fileInfo.Name()+"_crypt") } log.Infof("dst : %v", dst) dirnameMap := make(map[string]string) pathSeparator := string(os.PathSeparator) filepath.Walk(src, func(p string, info os.FileInfo, err error) error { if err != nil { log.Errorf("get file %v info failed, err:%v", p, err) return err } if p == src { return nil } log.Infof("current path %v", p) // relative path rp := strings.ReplaceAll(p, src, "") log.Infof("relative path %v", rp) rpds := strings.Split(rp, pathSeparator) if info.IsDir() { // absolute dst dir for current path dd := "" if o.dirnameEncryption == "true" { if o.op == "encrypt" || o.op == "en" { for i := range rpds { oname := rpds[i] if _, ok := dirnameMap[rpds[i]]; ok { rpds[i] = dirnameMap[rpds[i]] } else { rpds[i] = cipher.EncryptDirName(rpds[i]) dirnameMap[oname] = rpds[i] } } dd = path.Join(dst, strings.Join(rpds, pathSeparator)) } else { for i := range rpds { oname := rpds[i] if _, ok := dirnameMap[rpds[i]]; ok { rpds[i] = dirnameMap[rpds[i]] } else { dnn, err := cipher.DecryptDirName(rpds[i]) if err != nil { log.Fatalf("decrypt dir name %v failed,err:%v", rpds[i], err) } rpds[i] = dnn dirnameMap[oname] = dnn } } dd = path.Join(dst, strings.Join(rpds, pathSeparator)) } } else { dd = path.Join(dst, rp) } log.Infof("create output dir %v", dd) checkCreateDir(dd) return nil } // file dst dir fdd := dst if o.dirnameEncryption == "true" { for i := range rpds { if i == len(rpds)-1 { break } fdd = path.Join(fdd, dirnameMap[rpds[i]]) } } else { fdd = path.Join(fdd, strings.Join(rpds[:len(rpds)-1], pathSeparator)) } log.Infof("file output dir %v", fdd) o.cryptFile(cipher, p, fdd) return nil }) } func (o *options) cryptFile(cipher *rcCrypt.Cipher, src string, dst string) { fileInfo, err := os.Stat(src) if err != nil { log.Fatalf("get file %v info failed,err:%v", src, err) } fd, err := os.OpenFile(src, os.O_RDWR, 0666) if err != nil { log.Fatalf("open file %v failed,err:%v", src, err) } defer fd.Close() var cryptSrcReader io.Reader var outFile string if o.op == "encrypt" || o.op == "en" { filename := fileInfo.Name() if o.filenameEncryption != "off" { filename = cipher.EncryptFileName(fileInfo.Name()) log.Infof("encrypt file name %v to %v", fileInfo.Name(), filename) } else { filename = fileInfo.Name() + o.suffix } cryptSrcReader, err = cipher.EncryptData(fd) if err != nil { log.Fatalf("encrypt file %v failed,err:%v", src, err) } outFile = path.Join(dst, filename) } else { filename := fileInfo.Name() if o.filenameEncryption != "off" { filename, err = cipher.DecryptFileName(filename) if err != nil { log.Fatalf("decrypt file name %v failed,err:%v", src, err) } log.Infof("decrypt file name %v to %v, ", fileInfo.Name(), filename) } else { filename = strings.TrimSuffix(filename, o.suffix) } cryptSrcReader, err = cipher.DecryptData(fd) if err != nil { log.Fatalf("decrypt file %v failed,err:%v", src, err) } outFile = path.Join(dst, filename) } //write new file wr, err := os.OpenFile(outFile, os.O_CREATE|os.O_WRONLY, 0755) if err != nil { log.Fatalf("create file %v failed,err:%v", outFile, err) } defer wr.Close() _, err = io.Copy(wr, cryptSrcReader) if err != nil { log.Fatalf("write file %v failed,err:%v", outFile, err) } } // check dir exist ,if not ,create func checkCreateDir(dir string) { _, err := os.Stat(dir) if os.IsNotExist(err) { err := os.MkdirAll(dir, 0755) if err != nil { log.Fatalf("create dir %v failed,err:%v", dir, err) } return } else if err != nil { log.Fatalf("read dir %v err: %v", dir, err) } } func updateObfusParm(str string) string { obfuscatedPrefix := "___Obfuscated___" if !strings.HasPrefix(str, obfuscatedPrefix) { str, err := obscure.Obscure(str) if err != nil { log.Fatalf("update obfuscated parameter failed,err:%v", str) } } else { str, _ = strings.CutPrefix(str, obfuscatedPrefix) } return str } ================================================ FILE: cmd/flags/config.go ================================================ package flags var ( DataDir string ConfigPath string Debug bool NoPrefix bool Dev bool ForceBinDir bool LogStd bool ) ================================================ FILE: cmd/kill.go ================================================ package cmd import ( "os" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) // KillCmd represents the kill command var KillCmd = &cobra.Command{ Use: "kill", Short: "Force kill openlist server process by daemon/pid file", Run: func(cmd *cobra.Command, args []string) { kill() }, } func kill() { initDaemon() if pid == -1 { log.Info("Seems not have been started. Try use `openlist start` to start server.") return } process, err := os.FindProcess(pid) if err != nil { log.Errorf("failed to find process by pid: %d, reason: %v", pid, process) return } err = process.Kill() if err != nil { log.Errorf("failed to kill process %d: %v", pid, err) } else { log.Info("killed process: ", pid) } err = os.Remove(pidFile) if err != nil { log.Errorf("failed to remove pid file") } pid = -1 } func init() { RootCmd.AddCommand(KillCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // stopCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: // stopCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } ================================================ FILE: cmd/lang.go ================================================ /* Package cmd Copyright © 2022 Noah Hsu */ package cmd import ( "fmt" "io" "os" "strings" _ "github.com/OpenListTeam/OpenList/v4/drivers" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/data" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) type KV[V any] map[string]V type Drivers KV[KV[interface{}]] var frontendPath string func firstUpper(s string) string { if s == "" { return "" } return strings.ToUpper(s[:1]) + s[1:] } func convert(s string) string { ss := strings.Split(s, "_") ans := strings.Join(ss, " ") return firstUpper(ans) } func writeFile(name string, data interface{}) { f, err := os.Open(fmt.Sprintf("%s/src/lang/en/%s.json", frontendPath, name)) if err != nil { log.Errorf("failed to open %s.json: %+v", name, err) return } defer f.Close() content, err := io.ReadAll(f) if err != nil { log.Errorf("failed to read %s.json: %+v", name, err) return } oldData := make(map[string]interface{}) newData := make(map[string]interface{}) err = utils.Json.Unmarshal(content, &oldData) if err != nil { log.Errorf("failed to unmarshal %s.json: %+v", name, err) return } content, err = utils.Json.Marshal(data) if err != nil { log.Errorf("failed to marshal json: %+v", err) return } err = utils.Json.Unmarshal(content, &newData) if err != nil { log.Errorf("failed to unmarshal json: %+v", err) return } if mergeJson(newData, oldData) { log.Infof("%s.json no changed, skip", name) } else { log.Infof("%s.json changed, update file", name) //log.Infof("old: %+v\nnew:%+v", oldData, data) utils.WriteJsonToFile(fmt.Sprintf("lang/%s.json", name), oldData, true) } } func mergeJson(source, target map[string]interface{}) bool { equal := true for k, v := range source { tgtV, tgtOk := target[k] if !tgtOk { equal = false target[k] = v } else { srcMap, srcIsMap := v.(map[string]interface{}) tgtMap, tgtIsMap := tgtV.(map[string]interface{}) if srcIsMap && tgtIsMap { equal = mergeJson(srcMap, tgtMap) && equal } } } return equal } func generateDriversJson() { drivers := make(Drivers) drivers["drivers"] = make(KV[interface{}]) drivers["config"] = make(KV[interface{}]) driverInfoMap := op.GetDriverInfoMap() for k, v := range driverInfoMap { drivers["drivers"][k] = convert(k) items := make(KV[interface{}]) config := map[string]string{} if v.Config.Alert != "" { alert := strings.SplitN(v.Config.Alert, "|", 2) if len(alert) > 1 { config["alert"] = alert[1] } } drivers["config"][k] = config for i := range v.Additional { item := v.Additional[i] items[item.Name] = convert(item.Name) if item.Help != "" { items[fmt.Sprintf("%s-tips", item.Name)] = item.Help } if item.Type == conf.TypeSelect && len(item.Options) > 0 { options := make(KV[string]) _options := strings.Split(item.Options, ",") for _, o := range _options { options[o] = convert(o) } items[fmt.Sprintf("%ss", item.Name)] = options } } drivers[k] = items } writeFile("drivers", drivers) } func generateSettingsJson() { settings := data.InitialSettings() settingsLang := make(KV[any]) for _, setting := range settings { settingsLang[setting.Key] = convert(setting.Key) if setting.Help != "" { settingsLang[fmt.Sprintf("%s-tips", setting.Key)] = setting.Help } if setting.Type == conf.TypeSelect && len(setting.Options) > 0 { options := make(KV[string]) _options := strings.Split(setting.Options, ",") for _, o := range _options { options[o] = convert(o) } settingsLang[fmt.Sprintf("%ss", setting.Key)] = options } } writeFile("settings", settingsLang) //utils.WriteJsonToFile("lang/settings.json", settingsLang) } // LangCmd represents the lang command var LangCmd = &cobra.Command{ Use: "lang", Short: "Generate language json file", Run: func(cmd *cobra.Command, args []string) { frontendPath, _ = cmd.Flags().GetString("frontend-path") bootstrap.InitConfig() err := os.MkdirAll("lang", 0777) if err != nil { utils.Log.Fatalf("failed create folder: %s", err.Error()) } generateDriversJson() generateSettingsJson() }, } func init() { RootCmd.AddCommand(LangCmd) // Add frontend-path flag LangCmd.Flags().String("frontend-path", "../OpenList-Frontend", "Path to the frontend project directory") // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // langCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: // langCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } ================================================ FILE: cmd/restart.go ================================================ /* Copyright © 2022 NAME HERE */ package cmd import ( "github.com/spf13/cobra" ) // RestartCmd represents the restart command var RestartCmd = &cobra.Command{ Use: "restart", Short: "Restart openlist server by daemon/pid file", Run: func(cmd *cobra.Command, args []string) { stop() start() }, } func init() { RootCmd.AddCommand(RestartCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // restartCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: // restartCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } ================================================ FILE: cmd/root.go ================================================ package cmd import ( "fmt" "os" "github.com/OpenListTeam/OpenList/v4/cmd/flags" _ "github.com/OpenListTeam/OpenList/v4/drivers" _ "github.com/OpenListTeam/OpenList/v4/internal/archive" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download" "github.com/spf13/cobra" ) var RootCmd = &cobra.Command{ Use: "openlist", Short: "A file list program that supports multiple storage.", Long: `A file list program that supports multiple storage, built with love by OpenListTeam. Complete documentation is available at https://doc.oplist.org/`, } func Execute() { if err := RootCmd.Execute(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } } func init() { RootCmd.PersistentFlags().StringVar(&flags.DataDir, "data", "data", "data directory (relative paths are resolved against the current working directory)") RootCmd.PersistentFlags().StringVar(&flags.ConfigPath, "config", "", "path to config.json (relative to current working directory; defaults to [data directory]/config.json, where [data directory] is set by --data)") RootCmd.PersistentFlags().BoolVar(&flags.Debug, "debug", false, "start with debug mode") RootCmd.PersistentFlags().BoolVar(&flags.NoPrefix, "no-prefix", false, "disable env prefix") RootCmd.PersistentFlags().BoolVar(&flags.Dev, "dev", false, "start with dev mode") RootCmd.PersistentFlags().BoolVar(&flags.ForceBinDir, "force-bin-dir", false, "Force to use the directory where the binary file is located as data directory") RootCmd.PersistentFlags().BoolVar(&flags.LogStd, "log-std", false, "Force to log to std") } ================================================ FILE: cmd/server.go ================================================ package cmd import ( "os" "os/signal" "syscall" "time" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" "github.com/spf13/cobra" ) // ServerCmd represents the server command var ServerCmd = &cobra.Command{ Use: "server", Short: "Start the server at the specified address", Long: `Start the server at the specified address the address is defined in config file`, Run: func(cmd *cobra.Command, args []string) { bootstrap.Init() defer bootstrap.Release() bootstrap.Start() // Wait for interrupt signal to gracefully shutdown the server with // a timeout of 1 second. quit := make(chan os.Signal, 1) // kill (no param) default send syscanll.SIGTERM // kill -2 is syscall.SIGINT // kill -9 is syscall. SIGKILL but can"t be catch, so don't need add it signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) <-quit bootstrap.Shutdown(1 * time.Second) }, } func init() { RootCmd.AddCommand(ServerCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // serverCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: // serverCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } // OutOpenListInit 暴露用于外部启动server的函数 func OutOpenListInit() { var ( cmd *cobra.Command args []string ) ServerCmd.Run(cmd, args) } ================================================ FILE: cmd/start.go ================================================ /* Copyright © 2022 NAME HERE */ package cmd import ( "os" "os/exec" "path/filepath" "strconv" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) // StartCmd represents the start command var StartCmd = &cobra.Command{ Use: "start", Short: "Silent start openlist server with `--force-bin-dir`", Run: func(cmd *cobra.Command, args []string) { start() }, } func start() { initDaemon() if pid != -1 { _, err := os.FindProcess(pid) if err == nil { log.Info("openlist already started, pid ", pid) return } } args := os.Args args[1] = "server" args = append(args, "--force-bin-dir") cmd := &exec.Cmd{ Path: args[0], Args: args, Env: os.Environ(), } stdout, err := os.OpenFile(filepath.Join(filepath.Dir(pidFile), "start.log"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) if err != nil { log.Fatal(os.Getpid(), ": failed to open start log file:", err) } cmd.Stderr = stdout cmd.Stdout = stdout err = cmd.Start() if err != nil { log.Fatal("failed to start children process: ", err) } log.Infof("success start pid: %d", cmd.Process.Pid) err = os.WriteFile(pidFile, []byte(strconv.Itoa(cmd.Process.Pid)), 0666) if err != nil { log.Warn("failed to record pid, you may not be able to stop the program with `./openlist stop`") } } func init() { RootCmd.AddCommand(StartCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // startCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: // startCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } ================================================ FILE: cmd/stop_default.go ================================================ //go:build !windows package cmd import ( "os" "syscall" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) // StopCmd represents the stop command var StopCmd = &cobra.Command{ Use: "stop", Short: "Stop openlist server by daemon/pid file", Run: func(cmd *cobra.Command, args []string) { stop() }, } func stop() { initDaemon() if pid == -1 { log.Info("Seems not have been started. Try use `openlist start` to start server.") return } process, err := os.FindProcess(pid) if err != nil { log.Errorf("failed to find process by pid: %d, reason: %v", pid, process) return } err = process.Signal(syscall.SIGTERM) if err != nil { log.Errorf("failed to terminate process %d: %v", pid, err) } else { log.Info("terminated process: ", pid) } err = os.Remove(pidFile) if err != nil { log.Errorf("failed to remove pid file") } pid = -1 } func init() { RootCmd.AddCommand(StopCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // stopCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: // stopCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } ================================================ FILE: cmd/stop_windows.go ================================================ //go:build windows package cmd import ( "github.com/spf13/cobra" ) // StopCmd represents the stop command var StopCmd = &cobra.Command{ Use: "stop", Short: "Same as the kill command", Run: func(cmd *cobra.Command, args []string) { stop() }, } func stop() { kill() } func init() { RootCmd.AddCommand(StopCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // stopCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: // stopCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } ================================================ FILE: cmd/storage.go ================================================ /* Copyright © 2023 NAME HERE */ package cmd import ( "fmt" "os" "strconv" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/charmbracelet/bubbles/table" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" "github.com/spf13/cobra" ) // storageCmd represents the storage command var storageCmd = &cobra.Command{ Use: "storage", Short: "Manage storage", } var disableStorageCmd = &cobra.Command{ Use: "disable [mount path]", Short: "Disable a storage by mount path", RunE: func(cmd *cobra.Command, args []string) error { if len(args) < 1 { return fmt.Errorf("mount path is required") } mountPath := args[0] bootstrap.Init() defer bootstrap.Release() storage, err := db.GetStorageByMountPath(mountPath) if err != nil { return fmt.Errorf("failed to query storage: %+v", err) } storage.Disabled = true err = db.UpdateStorage(storage) if err != nil { return fmt.Errorf("failed to update storage: %+v", err) } utils.Log.Infof("Storage with mount path [%s] has been disabled from CLI", mountPath) fmt.Printf("Storage with mount path [%s] has been disabled\n", mountPath) return nil }, } var deleteStorageCmd = &cobra.Command{ Use: "delete [id]", Short: "Delete a storage by id", RunE: func(cmd *cobra.Command, args []string) error { if len(args) < 1 { return fmt.Errorf("id is required") } id, err := strconv.Atoi(args[0]) if err != nil { return fmt.Errorf("id must be a number") } if force, _ := cmd.Flags().GetBool("force"); force { fmt.Printf("Are you sure you want to delete storage with id [%d]? [y/N]: ", id) var confirm string fmt.Scanln(&confirm) if confirm != "y" && confirm != "Y" { fmt.Println("Delete operation cancelled.") return nil } } bootstrap.Init() defer bootstrap.Release() err = db.DeleteStorageById(uint(id)) if err != nil { return fmt.Errorf("failed to delete storage by id: %+v", err) } utils.Log.Infof("Storage with id [%d] have been deleted from CLI", id) fmt.Printf("Storage with id [%d] have been deleted\n", id) return nil }, } var baseStyle = lipgloss.NewStyle(). BorderStyle(lipgloss.NormalBorder()). BorderForeground(lipgloss.Color("240")) type model struct { table table.Model } func (m model) Init() tea.Cmd { return nil } func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { var cmd tea.Cmd switch msg := msg.(type) { case tea.KeyMsg: switch msg.String() { case "esc": if m.table.Focused() { m.table.Blur() } else { m.table.Focus() } case "q", "ctrl+c": return m, tea.Quit //case "enter": // return m, tea.Batch( // tea.Printf("Let's go to %s!", m.table.SelectedRow()[1]), // ) } } m.table, cmd = m.table.Update(msg) return m, cmd } func (m model) View() string { return baseStyle.Render(m.table.View()) + "\n" } var storageTableHeight int var listStorageCmd = &cobra.Command{ Use: "list", Short: "List all storages", RunE: func(cmd *cobra.Command, args []string) error { bootstrap.Init() defer bootstrap.Release() storages, _, err := db.GetStorages(1, -1) if err != nil { return fmt.Errorf("failed to query storages: %+v", err) } else { fmt.Printf("Found %d storages\n", len(storages)) columns := []table.Column{ {Title: "ID", Width: 4}, {Title: "Driver", Width: 16}, {Title: "Mount Path", Width: 30}, {Title: "Enabled", Width: 7}, } var rows []table.Row for i := range storages { storage := storages[i] enabled := "true" if storage.Disabled { enabled = "false" } rows = append(rows, table.Row{ strconv.Itoa(int(storage.ID)), storage.Driver, storage.MountPath, enabled, }) } t := table.New( table.WithColumns(columns), table.WithRows(rows), table.WithFocused(true), table.WithHeight(storageTableHeight), ) s := table.DefaultStyles() s.Header = s.Header. BorderStyle(lipgloss.NormalBorder()). BorderForeground(lipgloss.Color("240")). BorderBottom(true). Bold(false) s.Selected = s.Selected. Foreground(lipgloss.Color("229")). Background(lipgloss.Color("57")). Bold(false) t.SetStyles(s) m := model{t} if _, err := tea.NewProgram(m).Run(); err != nil { fmt.Printf("failed to run program: %+v\n", err) os.Exit(1) } } return nil }, } func init() { RootCmd.AddCommand(storageCmd) storageCmd.AddCommand(disableStorageCmd) storageCmd.AddCommand(listStorageCmd) storageCmd.PersistentFlags().IntVarP(&storageTableHeight, "height", "H", 10, "Table height") storageCmd.AddCommand(deleteStorageCmd) deleteStorageCmd.Flags().BoolP("force", "f", false, "Force delete without confirmation") // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // storageCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: // storageCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } ================================================ FILE: cmd/user.go ================================================ package cmd import ( "crypto/tls" "fmt" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) func DelAdminCacheOnline() { admin, err := op.GetAdmin() if err != nil { utils.Log.Errorf("[del_admin_cache] get admin error: %+v", err) return } DelUserCacheOnline(admin.Username) } func DelUserCacheOnline(username string) { client := resty.New().SetTimeout(1 * time.Second).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) token := setting.GetStr(conf.Token) port := conf.Conf.Scheme.HttpPort u := fmt.Sprintf("http://localhost:%d/api/admin/user/del_cache", port) if port == -1 { if conf.Conf.Scheme.HttpsPort == -1 { utils.Log.Warnf("[del_user_cache] no open port") return } u = fmt.Sprintf("https://localhost:%d/api/admin/user/del_cache", conf.Conf.Scheme.HttpsPort) } res, err := client.R().SetHeader("Authorization", token).SetQueryParam("username", username).Post(u) if err != nil { utils.Log.Warnf("[del_user_cache_online] failed: %+v", err) return } if res.StatusCode() != 200 { utils.Log.Warnf("[del_user_cache_online] failed: %+v", res.String()) return } code := utils.Json.Get(res.Body(), "code").ToInt() msg := utils.Json.Get(res.Body(), "message").ToString() if code != 200 { utils.Log.Errorf("[del_user_cache_online] error: %s", msg) return } utils.Log.Debugf("[del_user_cache_online] del user [%s] cache success", username) } ================================================ FILE: cmd/version.go ================================================ /* Copyright © 2022 NAME HERE */ package cmd import ( "fmt" "os" "runtime" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/spf13/cobra" ) // VersionCmd represents the version command var VersionCmd = &cobra.Command{ Use: "version", Short: "Show current version of OpenList", Run: func(cmd *cobra.Command, args []string) { goVersion := fmt.Sprintf("%s %s/%s", runtime.Version(), runtime.GOOS, runtime.GOARCH) fmt.Printf(`Built At: %s Go Version: %s Author: %s Commit ID: %s Version: %s WebVersion: %s `, conf.BuiltAt, goVersion, conf.GitAuthor, conf.GitCommit, conf.Version, conf.WebVersion) os.Exit(0) }, } func init() { RootCmd.AddCommand(VersionCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // versionCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: // versionCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } ================================================ FILE: docker-compose.yml ================================================ services: openlist: restart: always volumes: - '/etc/openlist:/opt/openlist/data' ports: - '5244:5244' - '5245:5245' user: '0:0' environment: - UMASK=022 - TZ=Asia/Shanghai container_name: openlist image: 'openlistteam/openlist:latest' ================================================ FILE: drivers/115/appver.go ================================================ package _115 import ( "errors" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/pkg/utils" driver115 "github.com/SheltonZhu/115driver/pkg/driver" log "github.com/sirupsen/logrus" ) var ( md5Salt = "Qclm8MGWUv59TnrR0XPg" appVer = "35.6.0.3" ) func (d *Pan115) getAppVersion() (string, error) { result := VersionResp{} res, err := base.RestyClient.R().Get(driver115.ApiGetVersion) if err != nil { return "", err } err = utils.Json.Unmarshal(res.Body(), &result) if err != nil { return "", err } if len(result.Error) > 0 { return "", errors.New(result.Error) } return result.Data.Win.Version, nil } func (d *Pan115) getAppVer() string { ver, err := d.getAppVersion() if err != nil { log.Warnf("[115] get app version failed: %v", err) return appVer } if len(ver) > 0 { return ver } return appVer } func (d *Pan115) initAppVer() { appVer = d.getAppVer() log.Debugf("use app version: %v", appVer) } type VersionResp struct { Error string `json:"error,omitempty"` Data Versions `json:"data"` } type Versions struct { Win Version `json:"win"` } type Version struct { Version string `json:"version_code"` } ================================================ FILE: drivers/115/driver.go ================================================ package _115 import ( "context" "strings" "sync" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/pkg/errors" "golang.org/x/time/rate" ) type Pan115 struct { model.Storage Addition client *driver115.Pan115Client limiter *rate.Limiter appVerOnce sync.Once } func (d *Pan115) Config() driver.Config { return config } func (d *Pan115) GetAddition() driver.Additional { return &d.Addition } func (d *Pan115) Init(ctx context.Context) error { d.appVerOnce.Do(d.initAppVer) if d.LimitRate > 0 { d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) } return d.login() } func (d *Pan115) WaitLimit(ctx context.Context) error { if d.limiter != nil { return d.limiter.Wait(ctx) } return nil } func (d *Pan115) Drop(ctx context.Context) error { return nil } func (d *Pan115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } files, err := d.getFiles(dir.GetID()) if err != nil && !errors.Is(err, driver115.ErrNotExist) { return nil, err } return utils.SliceConvert(files, func(src FileObj) (model.Obj, error) { return &src, nil }) } func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } userAgent := args.Header.Get("User-Agent") downloadInfo, err := d.client.DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { return nil, err } link := &model.Link{ URL: downloadInfo.Url.Url, Header: downloadInfo.Header, } return link, nil } func (d *Pan115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } result := driver115.MkdirResp{} form := map[string]string{ "pid": parentDir.GetID(), "cname": dirName, } req := d.client.NewRequest(). SetFormData(form). SetResult(&result). ForceContentType("application/json;charset=UTF-8") resp, err := req.Post(driver115.ApiDirAdd) err = driver115.CheckErr(err, &result, resp) if err != nil { return nil, err } f, err := d.getNewFile(result.FileID) if err != nil { return nil, nil } return f, nil } func (d *Pan115) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } if err := d.client.Move(dstDir.GetID(), srcObj.GetID()); err != nil { return nil, err } f, err := d.getNewFile(srcObj.GetID()) if err != nil { return nil, nil } return f, nil } func (d *Pan115) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } if err := d.client.Rename(srcObj.GetID(), newName); err != nil { return nil, err } f, err := d.getNewFile((srcObj.GetID())) if err != nil { return nil, nil } return f, nil } func (d *Pan115) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { if err := d.WaitLimit(ctx); err != nil { return err } return d.client.Copy(dstDir.GetID(), srcObj.GetID()) } func (d *Pan115) Remove(ctx context.Context, obj model.Obj) error { if err := d.WaitLimit(ctx); err != nil { return err } return d.client.Delete(obj.GetID()) } func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } var ( fastInfo *driver115.UploadInitResp dirID = dstDir.GetID() ) if ok, err := d.client.UploadAvailable(); err != nil || !ok { return nil, err } if stream.GetSize() > d.client.UploadMetaInfo.SizeLimit { return nil, driver115.ErrUploadTooLarge } //if digest, err = d.client.GetDigestResult(stream); err != nil { // return err //} const PreHashSize int64 = 128 * utils.KB hashSize := PreHashSize if stream.GetSize() < PreHashSize { hashSize = stream.GetSize() } reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: hashSize}) if err != nil { return nil, err } preHash, err := utils.HashReader(utils.SHA1, reader) if err != nil { return nil, err } preHash = strings.ToUpper(preHash) fullHash := stream.GetHash().GetHash(utils.SHA1) if len(fullHash) != utils.SHA1.Width { _, fullHash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1) if err != nil { return nil, err } } fullHash = strings.ToUpper(fullHash) // rapid-upload // note that 115 add timeout for rapid-upload, // and "sig invalid" err is thrown even when the hash is correct after timeout. if fastInfo, err = d.rapidUpload(stream.GetSize(), stream.GetName(), dirID, preHash, fullHash, stream); err != nil { return nil, err } if matched, err := fastInfo.Ok(); err != nil { return nil, err } else if matched { f, err := d.getNewFileByPickCode(fastInfo.PickCode) if err != nil { return nil, nil } return f, nil } var uploadResult *UploadResult // 闪传失败，上传 if stream.GetSize() <= 10*utils.MB { // 文件大小小于10MB，改用普通模式上传 if uploadResult, err = d.UploadByOSS(ctx, &fastInfo.UploadOSSParams, stream, dirID, up); err != nil { return nil, err } } else { // 分片上传 if uploadResult, err = d.UploadByMultipart(ctx, &fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID, up); err != nil { return nil, err } } file, err := d.getNewFile(uploadResult.Data.FileID) if err != nil { return nil, nil } return file, nil } func (d *Pan115) OfflineList(ctx context.Context) ([]*driver115.OfflineTask, error) { resp, err := d.client.ListOfflineTask(0) if err != nil { return nil, err } return resp.Tasks, nil } func (d *Pan115) OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { return d.client.AddOfflineTaskURIs(uris, dstDir.GetID(), driver115.WithAppVer(appVer)) } func (d *Pan115) DeleteOfflineTasks(ctx context.Context, hashes []string, deleteFiles bool) error { return d.client.DeleteOfflineTasks(hashes, deleteFiles) } func (d *Pan115) GetDetails(ctx context.Context) (*model.StorageDetails, error) { info, err := d.client.GetInfo() if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: info.SpaceInfo.AllTotal.Size, UsedSpace: info.SpaceInfo.AllUse.Size, }, }, nil } var _ driver.Driver = (*Pan115)(nil) ================================================ FILE: drivers/115/meta.go ================================================ package _115 import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"` PageSize int64 `json:"page_size" type:"number" default:"1000" help:"list api per page size of 115 driver"` LimitRate float64 `json:"limit_rate" type:"float" default:"2" help:"limit all api request rate ([limit]r/1s)"` driver.RootID } var config = driver.Config{ Name: "115 Cloud", DefaultRoot: "0", LinkCacheMode: driver.LinkCacheUA, } func init() { op.RegisterDriver(func() driver.Driver { return &Pan115{} }) } ================================================ FILE: drivers/115/types.go ================================================ package _115 import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/SheltonZhu/115driver/pkg/driver" ) var _ model.Obj = (*FileObj)(nil) type FileObj struct { driver.File } func (f *FileObj) CreateTime() time.Time { return f.File.CreateTime } func (f *FileObj) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.SHA1, f.Sha1) } func (f *FileObj) Thumb() string { return f.ThumbURL } type UploadResult struct { driver.BasicResp Data struct { PickCode string `json:"pick_code"` FileSize int `json:"file_size"` FileID string `json:"file_id"` ThumbURL string `json:"thumb_url"` Sha1 string `json:"sha1"` Aid int `json:"aid"` FileName string `json:"file_name"` Cid string `json:"cid"` IsVideo int `json:"is_video"` } `json:"data"` } ================================================ FILE: drivers/115/util.go ================================================ package _115 import ( "bytes" "context" "crypto/md5" "crypto/tls" "encoding/hex" "encoding/json" "fmt" "io" "net/url" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" netutil "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" cipher "github.com/SheltonZhu/115driver/pkg/crypto/ec115" driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/pkg/errors" ) // var UserAgent = driver115.UA115Browser func (d *Pan115) login() error { var err error opts := []driver115.Option{ driver115.UA(d.getUA()), func(c *driver115.Pan115Client) { c.Client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) }, } d.client = driver115.New(opts...) cr := &driver115.Credential{} if d.QRCodeToken != "" { s := &driver115.QRCodeSession{ UID: d.QRCodeToken, } if cr, err = d.client.QRCodeLoginWithApp(s, driver115.LoginApp(d.QRCodeSource)); err != nil { return errors.Wrap(err, "failed to login by qrcode") } d.Cookie = fmt.Sprintf("UID=%s;CID=%s;SEID=%s;KID=%s", cr.UID, cr.CID, cr.SEID, cr.KID) d.QRCodeToken = "" } else if d.Cookie != "" { if err = cr.FromCookie(d.Cookie); err != nil { return errors.Wrap(err, "failed to login by cookies") } d.client.ImportCredential(cr) } else { return errors.New("missing cookie or qrcode account") } return d.client.LoginCheck() } func (d *Pan115) getFiles(fileId string) ([]FileObj, error) { res := make([]FileObj, 0) if d.PageSize <= 0 { d.PageSize = driver115.FileListLimit } files, err := d.client.ListWithLimit(fileId, d.PageSize, driver115.WithMultiUrls()) if err != nil { return nil, err } for _, file := range *files { res = append(res, FileObj{file}) } return res, nil } func (d *Pan115) getNewFile(fileId string) (*FileObj, error) { file, err := d.client.GetFile(fileId) if err != nil { return nil, err } return &FileObj{*file}, nil } func (d *Pan115) getNewFileByPickCode(pickCode string) (*FileObj, error) { result := driver115.GetFileInfoResponse{} req := d.client.NewRequest(). SetQueryParam("pick_code", pickCode). ForceContentType("application/json;charset=UTF-8"). SetResult(&result) resp, err := req.Get(driver115.ApiFileInfo) if err := driver115.CheckErr(err, &result, resp); err != nil { return nil, err } if len(result.Files) == 0 { return nil, errors.New("not get file info") } fileInfo := result.Files[0] f := &FileObj{} f.From(fileInfo) return f, nil } func (d *Pan115) getUA() string { return fmt.Sprintf("Mozilla/5.0 115Browser/%s", appVer) } func (c *Pan115) GenerateToken(fileID, preID, timeStamp, fileSize, signKey, signVal string) string { userID := strconv.FormatInt(c.client.UserID, 10) userIDMd5 := md5.Sum([]byte(userID)) tokenMd5 := md5.Sum([]byte(md5Salt + fileID + fileSize + signKey + signVal + userID + timeStamp + hex.EncodeToString(userIDMd5[:]) + appVer)) return hex.EncodeToString(tokenMd5[:]) } func (d *Pan115) rapidUpload(fileSize int64, fileName, dirID, preID, fileID string, stream model.FileStreamer) (*driver115.UploadInitResp, error) { var ( ecdhCipher *cipher.EcdhCipher encrypted []byte decrypted []byte encodedToken string err error target = "U_1_" + dirID bodyBytes []byte result = driver115.UploadInitResp{} fileSizeStr = strconv.FormatInt(fileSize, 10) ) if ecdhCipher, err = cipher.NewEcdhCipher(); err != nil { return nil, err } userID := strconv.FormatInt(d.client.UserID, 10) form := url.Values{} form.Set("appid", "0") form.Set("appversion", appVer) form.Set("userid", userID) form.Set("filename", fileName) form.Set("filesize", fileSizeStr) form.Set("fileid", fileID) form.Set("target", target) form.Set("sig", d.client.GenerateSignature(fileID, target)) signKey, signVal := "", "" for retry := true; retry; { t := driver115.NowMilli() if encodedToken, err = ecdhCipher.EncodeToken(t.ToInt64()); err != nil { return nil, err } params := map[string]string{ "k_ec": encodedToken, } form.Set("t", t.String()) form.Set("token", d.GenerateToken(fileID, preID, t.String(), fileSizeStr, signKey, signVal)) if signKey != "" && signVal != "" { form.Set("sign_key", signKey) form.Set("sign_val", signVal) } if encrypted, err = ecdhCipher.Encrypt([]byte(form.Encode())); err != nil { return nil, err } req := d.client.NewRequest(). SetQueryParams(params). SetBody(encrypted). SetHeaderVerbatim("Content-Type", "application/x-www-form-urlencoded"). SetDoNotParseResponse(true) resp, err := req.Post(driver115.ApiUploadInit) if err != nil { return nil, err } data := resp.RawBody() defer data.Close() if bodyBytes, err = io.ReadAll(data); err != nil { return nil, err } if decrypted, err = ecdhCipher.Decrypt(bodyBytes); err != nil { return nil, err } if err = driver115.CheckErr(json.Unmarshal(decrypted, &result), &result, resp); err != nil { return nil, err } if result.Status == 7 { // Update signKey & signVal signKey = result.SignKey signVal, err = UploadDigestRange(stream, result.SignCheck) if err != nil { return nil, err } } else { retry = false } result.SHA1 = fileID } return &result, nil } func UploadDigestRange(stream model.FileStreamer, rangeSpec string) (result string, err error) { var start, end int64 if _, err = fmt.Sscanf(rangeSpec, "%d-%d", &start, &end); err != nil { return } length := end - start + 1 reader, err := stream.RangeRead(http_range.Range{Start: start, Length: length}) if err != nil { return "", err } hashStr, err := utils.HashReader(utils.SHA1, reader) if err != nil { return "", err } result = strings.ToUpper(hashStr) return } // UploadByOSS use aliyun sdk to upload func (c *Pan115) UploadByOSS(ctx context.Context, params *driver115.UploadOSSParams, s model.FileStreamer, dirID string, up driver.UpdateProgress) (*UploadResult, error) { ossToken, err := c.client.GetOSSToken() if err != nil { return nil, err } ossClient, err := netutil.NewOSSClient(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret) if err != nil { return nil, err } bucket, err := ossClient.Bucket(params.Bucket) if err != nil { return nil, err } var bodyBytes []byte r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, }) if err = bucket.PutObject(params.Object, r, append( driver115.OssOption(params, ossToken), oss.CallbackResult(&bodyBytes), )...); err != nil { return nil, err } var uploadResult UploadResult if err = json.Unmarshal(bodyBytes, &uploadResult); err != nil { return nil, err } return &uploadResult, uploadResult.Err(string(bodyBytes)) } // UploadByMultipart upload by mutipart blocks func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.UploadOSSParams, fileSize int64, s model.FileStreamer, dirID string, up driver.UpdateProgress, opts ...driver115.UploadMultipartOption, ) (*UploadResult, error) { var ( chunks []oss.FileChunk parts []oss.UploadPart imur oss.InitiateMultipartUploadResult ossClient *oss.Client bucket *oss.Bucket ossToken *driver115.UploadOSSTokenResp bodyBytes []byte err error ) tmpF, err := s.CacheFullAndWriter(&up, nil) if err != nil { return nil, err } options := driver115.DefalutUploadMultipartOptions() if len(opts) > 0 { for _, f := range opts { f(options) } } // oss 启用Sequential必须按顺序上传 options.ThreadsNum = 1 if ossToken, err = d.client.GetOSSToken(); err != nil { return nil, err } if ossClient, err = netutil.NewOSSClient(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret, oss.EnableMD5(true), oss.EnableCRC(true)); err != nil { return nil, err } if bucket, err = ossClient.Bucket(params.Bucket); err != nil { return nil, err } // ossToken一小时后就会失效，所以每50分钟重新获取一次 ticker := time.NewTicker(options.TokenRefreshTime) defer ticker.Stop() // 设置超时 timeout := time.NewTimer(options.Timeout) if chunks, err = SplitFile(fileSize); err != nil { return nil, err } if imur, err = bucket.InitiateMultipartUpload(params.Object, oss.SetHeader(driver115.OssSecurityTokenHeaderName, ossToken.SecurityToken), oss.UserAgentHeader(driver115.OSSUserAgent), oss.EnableSha1(), oss.Sequential(), ); err != nil { return nil, err } wg := sync.WaitGroup{} wg.Add(len(chunks)) chunksCh := make(chan oss.FileChunk) errCh := make(chan error) UploadedPartsCh := make(chan oss.UploadPart) quit := make(chan struct{}) // producer go chunksProducer(chunksCh, chunks) go func() { wg.Wait() quit <- struct{}{} }() completedNum := atomic.Int32{} // consumers for i := 0; i < options.ThreadsNum; i++ { go func(threadId int) { defer func() { if r := recover(); r != nil { errCh <- fmt.Errorf("recovered in %v", r) } }() for chunk := range chunksCh { var part oss.UploadPart // 出现错误就继续尝试，共尝试3次 for retry := 0; retry < 3; retry++ { select { case <-ctx.Done(): break case <-ticker.C: if ossToken, err = d.client.GetOSSToken(); err != nil { // 到时重新获取ossToken errCh <- errors.Wrap(err, "刷新token时出现错误") } default: } buf := make([]byte, chunk.Size) if _, err = tmpF.ReadAt(buf, chunk.Offset); err != nil && !errors.Is(err, io.EOF) { continue } if part, err = bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, bytes.NewReader(buf)), chunk.Size, chunk.Number, driver115.OssOption(params, ossToken)...); err == nil { break } } if err != nil { errCh <- errors.Wrap(err, fmt.Sprintf("上传 %s 的第%d个分片时出现错误：%v", s.GetName(), chunk.Number, err)) } else { num := completedNum.Add(1) up(float64(num) * 100.0 / float64(len(chunks))) } UploadedPartsCh <- part } }(i) } go func() { for part := range UploadedPartsCh { parts = append(parts, part) wg.Done() } }() LOOP: for { select { case <-ticker.C: // 到时重新获取ossToken if ossToken, err = d.client.GetOSSToken(); err != nil { return nil, err } case <-quit: break LOOP case <-errCh: return nil, err case <-timeout.C: return nil, fmt.Errorf("time out") } } // 不知道啥原因，oss那边分片上传不计算sha1，导致115服务器校验错误 // params.Callback.Callback = strings.ReplaceAll(params.Callback.Callback, "${sha1}", params.SHA1) if _, err := bucket.CompleteMultipartUpload(imur, parts, append( driver115.OssOption(params, ossToken), oss.CallbackResult(&bodyBytes), )...); err != nil { return nil, err } var uploadResult UploadResult if err = json.Unmarshal(bodyBytes, &uploadResult); err != nil { return nil, err } return &uploadResult, uploadResult.Err(string(bodyBytes)) } func chunksProducer(ch chan oss.FileChunk, chunks []oss.FileChunk) { for _, chunk := range chunks { ch <- chunk } } func SplitFile(fileSize int64) (chunks []oss.FileChunk, err error) { for i := int64(1); i < 10; i++ { if fileSize < i*utils.GB { // 文件大小小于iGB时分为i*1000片 if chunks, err = SplitFileByPartNum(fileSize, int(i*1000)); err != nil { return } break } } if fileSize > 9*utils.GB { // 文件大小大于9GB时分为10000片 if chunks, err = SplitFileByPartNum(fileSize, 10000); err != nil { return } } // 单个分片大小不能小于100KB if chunks[0].Size < 100*utils.KB { if chunks, err = SplitFileByPartSize(fileSize, 100*utils.KB); err != nil { return } } return } // SplitFileByPartNum splits big file into parts by the num of parts. // Split the file with specified parts count, returns the split result when error is nil. func SplitFileByPartNum(fileSize int64, chunkNum int) ([]oss.FileChunk, error) { if chunkNum <= 0 || chunkNum > 10000 { return nil, errors.New("chunkNum invalid") } if int64(chunkNum) > fileSize { return nil, errors.New("oss: chunkNum invalid") } var chunks []oss.FileChunk chunk := oss.FileChunk{} chunkN := (int64)(chunkNum) for i := int64(0); i < chunkN; i++ { chunk.Number = int(i + 1) chunk.Offset = i * (fileSize / chunkN) if i == chunkN-1 { chunk.Size = fileSize/chunkN + fileSize%chunkN } else { chunk.Size = fileSize / chunkN } chunks = append(chunks, chunk) } return chunks, nil } // SplitFileByPartSize splits big file into parts by the size of parts. // Splits the file by the part size. Returns the FileChunk when error is nil. func SplitFileByPartSize(fileSize int64, chunkSize int64) ([]oss.FileChunk, error) { if chunkSize <= 0 { return nil, errors.New("chunkSize invalid") } chunkN := fileSize / chunkSize if chunkN >= 10000 { return nil, errors.New("Too many parts, please increase part size") } var chunks []oss.FileChunk chunk := oss.FileChunk{} for i := int64(0); i < chunkN; i++ { chunk.Number = int(i + 1) chunk.Offset = i * chunkSize chunk.Size = chunkSize chunks = append(chunks, chunk) } if fileSize%chunkSize > 0 { chunk.Number = len(chunks) + 1 chunk.Offset = int64(len(chunks)) * chunkSize chunk.Size = fileSize % chunkSize chunks = append(chunks, chunk) } return chunks, nil } ================================================ FILE: drivers/115_open/driver.go ================================================ package _115_open import ( "context" "fmt" "net/http" "strconv" "strings" "time" sdk "github.com/OpenListTeam/115-sdk-go" "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "golang.org/x/time/rate" ) type Open115 struct { model.Storage Addition client *sdk.Client limiter *rate.Limiter } func (d *Open115) Config() driver.Config { return config } func (d *Open115) GetAddition() driver.Additional { return &d.Addition } func (d *Open115) Init(ctx context.Context) error { d.client = sdk.New(sdk.WithRefreshToken(d.Addition.RefreshToken), sdk.WithAccessToken(d.Addition.AccessToken), sdk.WithOnRefreshToken(func(s1, s2 string) { d.Addition.AccessToken = s1 d.Addition.RefreshToken = s2 op.MustSaveDriverStorage(d) })) if flags.Debug || flags.Dev { d.client.SetDebug(true) } _, err := d.client.UserInfo(ctx) if err != nil { return err } if d.Addition.LimitRate > 0 { d.limiter = rate.NewLimiter(rate.Limit(d.Addition.LimitRate), 1) } if d.PageSize <= 0 { d.PageSize = 200 } else if d.PageSize > 1150 { d.PageSize = 1150 } return nil } func (d *Open115) WaitLimit(ctx context.Context) error { if d.limiter != nil { return d.limiter.Wait(ctx) } return nil } func (d *Open115) Drop(ctx context.Context) error { return nil } func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var res []model.Obj pageSize := int64(d.PageSize) offset := int64(0) for { if err := d.WaitLimit(ctx); err != nil { return nil, err } resp, err := d.client.GetFiles(ctx, &sdk.GetFilesReq{ CID: dir.GetID(), Limit: pageSize, Offset: offset, ASC: d.Addition.OrderDirection == "asc", O: d.Addition.OrderBy, // Cur: 1, ShowDir: true, }) if err != nil { return nil, err } res = append(res, utils.MustSliceConvert(resp.Data, func(src sdk.GetFilesResp_File) model.Obj { obj := Obj(src) return &obj })...) if len(res) >= int(resp.Count) { break } offset += pageSize } return res, nil } func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } var ua string if args.Header != nil { ua = args.Header.Get("User-Agent") } if ua == "" { ua = base.UserAgent } obj, ok := file.(*Obj) if !ok { return nil, fmt.Errorf("can't convert obj") } pc := obj.Pc resp, err := d.client.DownURL(ctx, pc, ua) if err != nil { return nil, err } u, ok := resp[obj.GetID()] if !ok { return nil, fmt.Errorf("can't get link") } return &model.Link{ URL: u.URL.URL, Header: http.Header{ "User-Agent": []string{ua}, }, }, nil } func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } resp, err := d.client.Mkdir(ctx, parentDir.GetID(), dirName) if err != nil { return nil, err } return &Obj{ Fid: resp.FileID, Pid: parentDir.GetID(), Fn: dirName, Fc: "0", Upt: time.Now().Unix(), Uet: time.Now().Unix(), UpPt: time.Now().Unix(), }, nil } func (d *Open115) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } _, err := d.client.Move(ctx, &sdk.MoveReq{ FileIDs: srcObj.GetID(), ToCid: dstDir.GetID(), }) if err != nil { return nil, err } return srcObj, nil } func (d *Open115) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } _, err := d.client.UpdateFile(ctx, &sdk.UpdateFileReq{ FileID: srcObj.GetID(), FileName: newName, }) if err != nil { return nil, err } obj, ok := srcObj.(*Obj) if ok { obj.Fn = newName } return srcObj, nil } func (d *Open115) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } _, err := d.client.Copy(ctx, &sdk.CopyReq{ PID: dstDir.GetID(), FileID: srcObj.GetID(), NoDupli: "1", }) if err != nil { return nil, err } return srcObj, nil } func (d *Open115) Remove(ctx context.Context, obj model.Obj) error { if err := d.WaitLimit(ctx); err != nil { return err } _obj, ok := obj.(*Obj) if !ok { return fmt.Errorf("can't convert obj") } _, err := d.client.DelFile(ctx, &sdk.DelFileReq{ FileIDs: _obj.GetID(), ParentID: _obj.Pid, }) if err != nil { return err } return nil } func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { err := d.WaitLimit(ctx) if err != nil { return err } sha1 := file.GetHash().GetHash(utils.SHA1) if len(sha1) != utils.SHA1.Width { _, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1) if err != nil { return err } } const PreHashSize int64 = 128 * utils.KB hashSize := PreHashSize if file.GetSize() < PreHashSize { hashSize = file.GetSize() } reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) if err != nil { return err } sha1128k, err := utils.HashReader(utils.SHA1, reader) if err != nil { return err } // 1. Init resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ FileName: file.GetName(), FileSize: file.GetSize(), Target: dstDir.GetID(), FileID: strings.ToUpper(sha1), PreID: strings.ToUpper(sha1128k), }) if err != nil { return err } if resp.Status == 2 { up(100) return nil } // 2. two way verify if utils.SliceContains([]int{6, 7, 8}, resp.Status) { signCheck := strings.Split(resp.SignCheck, "-") //"sign_check": "2392148-2392298" 取2392148-2392298之间的内容(包含2392148、2392298)的sha1 start, err := strconv.ParseInt(signCheck[0], 10, 64) if err != nil { return err } end, err := strconv.ParseInt(signCheck[1], 10, 64) if err != nil { return err } reader, err = file.RangeRead(http_range.Range{Start: start, Length: end - start + 1}) if err != nil { return err } signVal, err := utils.HashReader(utils.SHA1, reader) if err != nil { return err } resp, err = d.client.UploadInit(ctx, &sdk.UploadInitReq{ FileName: file.GetName(), FileSize: file.GetSize(), Target: dstDir.GetID(), FileID: strings.ToUpper(sha1), PreID: strings.ToUpper(sha1128k), SignKey: resp.SignKey, SignVal: strings.ToUpper(signVal), }) if err != nil { return err } if resp.Status == 2 { up(100) return nil } } // 3. get upload token tokenResp, err := d.client.UploadGetToken(ctx) if err != nil { return err } // 4. upload err = d.multpartUpload(ctx, file, up, tokenResp, resp) if err != nil { return err } return nil } func (d *Open115) OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { return d.client.AddOfflineTaskURIs(ctx, uris, dstDir.GetID()) } func (d *Open115) DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error { return d.client.DeleteOfflineTask(ctx, infoHash, deleteFiles) } func (d *Open115) OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) { resp, err := d.client.OfflineTaskList(ctx, 1) if err != nil { return nil, err } return resp, nil } func (d *Open115) GetDetails(ctx context.Context) (*model.StorageDetails, error) { userInfo, err := d.client.UserInfo(ctx) if err != nil { return nil, err } total, err := ParseInt64(userInfo.RtSpaceInfo.AllTotal.Size) if err != nil { return nil, err } used, err := ParseInt64(userInfo.RtSpaceInfo.AllUse.Size) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: used, }, }, nil } // func (d *Open115) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { // // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional // return nil, errs.NotImplement // } // func (d *Open115) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { // // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional // return nil, errs.NotImplement // } // func (d *Open115) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { // // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional // return nil, errs.NotImplement // } // func (d *Open115) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { // // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional // // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir // // return errs.NotImplement to use an internal archive tool // return nil, errs.NotImplement // } //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*Open115)(nil) ================================================ FILE: drivers/115_open/meta.go ================================================ package _115_open import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { // Usually one of two driver.RootID // define other OrderBy string `json:"order_by" type:"select" options:"file_name,file_size,user_utime,file_type"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` LimitRate float64 `json:"limit_rate" type:"float" default:"1" help:"limit all api request rate ([limit]r/1s)"` PageSize int64 `json:"page_size" type:"number" default:"200" help:"list api per page size of 115open driver"` AccessToken string `json:"access_token" required:"true"` RefreshToken string `json:"refresh_token" required:"true"` } var config = driver.Config{ Name: "115 Open", DefaultRoot: "0", LinkCacheMode: driver.LinkCacheUA, } func init() { op.RegisterDriver(func() driver.Driver { return &Open115{} }) } ================================================ FILE: drivers/115_open/types.go ================================================ package _115_open import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" sdk "github.com/OpenListTeam/115-sdk-go" ) type Obj sdk.GetFilesResp_File // Thumb implements model.Thumb. func (o *Obj) Thumb() string { return o.Thumbnail } // CreateTime implements model.Obj. func (o *Obj) CreateTime() time.Time { return time.Unix(o.UpPt, 0) } // GetHash implements model.Obj. func (o *Obj) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.SHA1, o.Sha1) } // GetID implements model.Obj. func (o *Obj) GetID() string { return o.Fid } // GetName implements model.Obj. func (o *Obj) GetName() string { return o.Fn } // GetPath implements model.Obj. func (o *Obj) GetPath() string { return "" } // GetSize implements model.Obj. func (o *Obj) GetSize() int64 { return o.FS } // IsDir implements model.Obj. func (o *Obj) IsDir() bool { return o.Fc == "0" } // ModTime implements model.Obj. func (o *Obj) ModTime() time.Time { return time.Unix(o.Upt, 0) } var _ model.Obj = (*Obj)(nil) var _ model.Thumb = (*Obj)(nil) ================================================ FILE: drivers/115_open/upload.go ================================================ package _115_open import ( "context" "encoding/base64" "io" "time" sdk "github.com/OpenListTeam/115-sdk-go" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" netutil "github.com/OpenListTeam/OpenList/v4/internal/net" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/avast/retry-go" ) func calPartSize(fileSize int64) int64 { var partSize int64 = 20 * utils.MB if fileSize > partSize { if fileSize > 1*utils.TB { // file Size over 1TB partSize = 5 * utils.GB // file part size 5GB } else if fileSize > 768*utils.GB { // over 768GB partSize = 109951163 // ≈ 104.8576MB, split 1TB into 10,000 part } else if fileSize > 512*utils.GB { // over 512GB partSize = 82463373 // ≈ 78.6432MB } else if fileSize > 384*utils.GB { // over 384GB partSize = 54975582 // ≈ 52.4288MB } else if fileSize > 256*utils.GB { // over 256GB partSize = 41231687 // ≈ 39.3216MB } else if fileSize > 128*utils.GB { // over 128GB partSize = 27487791 // ≈ 26.2144MB } } return partSize } func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { ossClient, err := netutil.NewOSSClient(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) if err != nil { return err } bucket, err := ossClient.Bucket(initResp.Bucket) if err != nil { return err } err = bucket.PutObject(initResp.Object, tempF, oss.Callback(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.Callback))), oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.CallbackVar))), ) return err } // type CallbackResult struct { // State bool `json:"state"` // Code int `json:"code"` // Message string `json:"message"` // Data struct { // PickCode string `json:"pick_code"` // FileName string `json:"file_name"` // FileSize int64 `json:"file_size"` // FileID string `json:"file_id"` // ThumbURL string `json:"thumb_url"` // Sha1 string `json:"sha1"` // Aid int `json:"aid"` // Cid string `json:"cid"` // } `json:"data"` // } func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { ossClient, err := netutil.NewOSSClient(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) if err != nil { return err } bucket, err := ossClient.Bucket(initResp.Bucket) if err != nil { return err } imur, err := bucket.InitiateMultipartUpload(initResp.Object, oss.Sequential()) if err != nil { return err } fileSize := stream.GetSize() chunkSize := calPartSize(fileSize) ss, err := streamPkg.NewStreamSectionReader(stream, int(chunkSize), &up) if err != nil { return err } partNum := (stream.GetSize() + chunkSize - 1) / chunkSize parts := make([]oss.UploadPart, partNum) offset := int64(0) for i := int64(1); i <= partNum; i++ { if utils.IsCanceled(ctx) { return ctx.Err() } partSize := chunkSize if i == partNum { partSize = fileSize - (i-1)*chunkSize } rd, err := ss.GetSectionReader(offset, partSize) if err != nil { return err } err = retry.Do(func() error { rd.Seek(0, io.SeekStart) part, err := bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, rd), partSize, int(i)) if err != nil { return err } parts[i-1] = part return nil }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) ss.FreeSectionReader(rd) if err != nil { return err } if i == partNum { offset = fileSize } else { offset += partSize } up(float64(offset) * 100 / float64(fileSize)) } // callbackRespBytes := make([]byte, 1024) _, err = bucket.CompleteMultipartUpload( imur, parts, oss.Callback(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.Callback))), oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.CallbackVar))), // oss.CallbackResult(&callbackRespBytes), ) if err != nil { return err } return nil } ================================================ FILE: drivers/115_open/util.go ================================================ package _115_open import "encoding/json" func ParseInt64(v json.Number) (int64, error) { i, err := v.Int64() if err == nil { return i, nil } f, e1 := v.Float64() if e1 == nil { return int64(f), nil } return int64(0), err } ================================================ FILE: drivers/115_share/driver.go ================================================ package _115_share import ( "context" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" driver115 "github.com/SheltonZhu/115driver/pkg/driver" "golang.org/x/time/rate" ) type Pan115Share struct { model.Storage Addition client *driver115.Pan115Client limiter *rate.Limiter } func (d *Pan115Share) Config() driver.Config { return config } func (d *Pan115Share) GetAddition() driver.Additional { return &d.Addition } func (d *Pan115Share) Init(ctx context.Context) error { if d.LimitRate > 0 { d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) } return d.login() } func (d *Pan115Share) WaitLimit(ctx context.Context) error { if d.limiter != nil { return d.limiter.Wait(ctx) } return nil } func (d *Pan115Share) Drop(ctx context.Context) error { return nil } func (d *Pan115Share) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } var ua string // TODO: will use user agent from header // if args.Header != nil { // ua = args.Header.Get("User-Agent") // } if ua == "" { ua = base.UserAgentNT } files := make([]driver115.ShareFile, 0) fileResp, err := d.client.GetShareSnapWithUA(ua, d.ShareCode, d.ReceiveCode, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) if err != nil { return nil, err } files = append(files, fileResp.Data.List...) total := fileResp.Data.Count count := len(fileResp.Data.List) for total > count { fileResp, err := d.client.GetShareSnap( d.ShareCode, d.ReceiveCode, dir.GetID(), driver115.QueryLimit(int(d.PageSize)), driver115.QueryOffset(count), ) if err != nil { return nil, err } files = append(files, fileResp.Data.List...) count += len(fileResp.Data.List) } return utils.SliceConvert(files, transFunc) } func (d *Pan115Share) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } var ua string if args.Header != nil { ua = args.Header.Get("User-Agent") } if ua == "" { ua = base.UserAgent } downloadInfo, err := d.client.DownloadByShareCodeWithUA(ua, d.ShareCode, d.ReceiveCode, file.GetID()) if err != nil { return nil, err } return &model.Link{URL: downloadInfo.URL.URL}, nil } func (d *Pan115Share) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { return errs.NotSupport } func (d *Pan115Share) Move(ctx context.Context, srcObj, dstDir model.Obj) error { return errs.NotSupport } func (d *Pan115Share) Rename(ctx context.Context, srcObj model.Obj, newName string) error { return errs.NotSupport } func (d *Pan115Share) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return errs.NotSupport } func (d *Pan115Share) Remove(ctx context.Context, obj model.Obj) error { return errs.NotSupport } func (d *Pan115Share) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { return errs.NotSupport } var _ driver.Driver = (*Pan115Share)(nil) ================================================ FILE: drivers/115_share/meta.go ================================================ package _115_share import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"` PageSize int64 `json:"page_size" type:"number" default:"1000" help:"list api per page size of 115 driver"` LimitRate float64 `json:"limit_rate" type:"float" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` ShareCode string `json:"share_code" type:"text" required:"true" help:"share code of 115 share link"` ReceiveCode string `json:"receive_code" type:"text" required:"true" help:"receive code of 115 share link"` driver.RootID } var config = driver.Config{ Name: "115 Share", DefaultRoot: "0", NoUpload: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Pan115Share{} }) } ================================================ FILE: drivers/115_share/utils.go ================================================ package _115_share import ( "fmt" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/pkg/errors" ) var _ model.Obj = (*FileObj)(nil) type FileObj struct { Size int64 Sha1 string Utm time.Time FileName string isDir bool FileID string ThumbURL string } func (f *FileObj) CreateTime() time.Time { return f.Utm } func (f *FileObj) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.SHA1, f.Sha1) } func (f *FileObj) GetSize() int64 { return f.Size } func (f *FileObj) GetName() string { return f.FileName } func (f *FileObj) ModTime() time.Time { return f.Utm } func (f *FileObj) IsDir() bool { return f.isDir } func (f *FileObj) GetID() string { return f.FileID } func (f *FileObj) GetPath() string { return "" } func (f *FileObj) Thumb() string { return f.ThumbURL } func transFunc(sf driver115.ShareFile) (model.Obj, error) { timeInt, err := strconv.ParseInt(sf.UpdateTime, 10, 64) if err != nil { return nil, err } var ( utm = time.Unix(timeInt, 0) isDir = (sf.IsFile == 0) fileID = string(sf.FileID) ) if isDir { fileID = string(sf.CategoryID) } return &FileObj{ Size: int64(sf.Size), Sha1: sf.Sha1, Utm: utm, FileName: string(sf.FileName), isDir: isDir, FileID: fileID, ThumbURL: sf.ThumbURL, }, nil } func (d *Pan115Share) login() error { var err error opts := []driver115.Option{ driver115.UA(base.UserAgentNT), } d.client = driver115.New(opts...) if _, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, ""); err != nil { return errors.Wrap(err, "failed to get share snap") } cr := &driver115.Credential{} if d.QRCodeToken != "" { s := &driver115.QRCodeSession{ UID: d.QRCodeToken, } if cr, err = d.client.QRCodeLoginWithApp(s, driver115.LoginApp(d.QRCodeSource)); err != nil { return errors.Wrap(err, "failed to login by qrcode") } d.Cookie = fmt.Sprintf("UID=%s;CID=%s;SEID=%s;KID=%s", cr.UID, cr.CID, cr.SEID, cr.KID) d.QRCodeToken = "" } else if d.Cookie != "" { if err = cr.FromCookie(d.Cookie); err != nil { return errors.Wrap(err, "failed to login by cookies") } d.client.ImportCredential(cr) } else { return errors.New("missing cookie or qrcode account") } return d.client.LoginCheck() } ================================================ FILE: drivers/123/driver.go ================================================ package _123 import ( "context" "encoding/base64" "fmt" "net/http" "net/url" "strings" "sync" "time" "golang.org/x/time/rate" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) type Pan123 struct { model.Storage Addition apiRateLimit sync.Map } func (d *Pan123) Config() driver.Config { return config } func (d *Pan123) GetAddition() driver.Additional { return &d.Addition } func (d *Pan123) Init(ctx context.Context) error { _, err := d.Request(UserInfo, http.MethodGet, func(req *resty.Request) { req.SetHeader("platform", "web") }, nil) return err } func (d *Pan123) Drop(ctx context.Context) error { _, _ = d.Request(Logout, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{}) }, nil) return nil } func (d *Pan123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(ctx, dir.GetID(), dir.GetName()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return src, nil }) } func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if f, ok := file.(File); ok { data := base.Json{ "driveId": 0, "etag": f.Etag, "fileId": f.FileId, "fileName": f.FileName, "s3keyFlag": f.S3KeyFlag, "size": f.Size, "type": f.Type, } resp, err := d.Request(DownloadInfo, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) if err != nil { return nil, err } downloadUrl := utils.Json.Get(resp, "data", "DownloadUrl").ToString() ou, err := url.Parse(downloadUrl) if err != nil { return nil, err } u_ := ou.String() nu := ou.Query().Get("params") if nu != "" { du, _ := base64.StdEncoding.DecodeString(nu) u, err := url.Parse(string(du)) if err != nil { return nil, err } u_ = u.String() } log.Debug("download url: ", u_) res, err := base.NoRedirectClient.R().SetHeader("Referer", "https://www.123pan.com/").Get(u_) if err != nil { return nil, err } log.Debug(res.String()) link := model.Link{ URL: u_, } log.Debugln("res code: ", res.StatusCode()) if res.StatusCode() == 302 { link.URL = res.Header().Get("location") } else if res.StatusCode() < 300 { link.URL = utils.Json.Get(res.Body(), "data", "redirect_url").ToString() } link.Header = http.Header{ "Referer": []string{fmt.Sprintf("%s://%s/", ou.Scheme, ou.Host)}, } return &link, nil } else { return nil, fmt.Errorf("can't convert obj") } } func (d *Pan123) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { data := base.Json{ "driveId": 0, "etag": "", "fileName": dirName, "parentFileId": parentDir.GetID(), "size": 0, "type": 1, } _, err := d.Request(Mkdir, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *Pan123) Move(ctx context.Context, srcObj, dstDir model.Obj) error { data := base.Json{ "fileIdList": []base.Json{{"FileId": srcObj.GetID()}}, "parentFileId": dstDir.GetID(), } _, err := d.Request(Move, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *Pan123) Rename(ctx context.Context, srcObj model.Obj, newName string) error { data := base.Json{ "driveId": 0, "fileId": srcObj.GetID(), "fileName": newName, } _, err := d.Request(Rename, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *Pan123) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return errs.NotSupport } func (d *Pan123) Remove(ctx context.Context, obj model.Obj) error { if f, ok := obj.(File); ok { data := base.Json{ "driveId": 0, "operation": true, "fileTrashInfoList": []File{f}, } _, err := d.Request(Trash, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } else { return fmt.Errorf("can't convert obj") } } func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { etag := file.GetHash().GetHash(utils.MD5) var err error if len(etag) < utils.MD5.Width { _, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5) if err != nil { return err } } data := base.Json{ "driveId": 0, "duplicate": 2, // 2->覆盖 1->重命名 0->默认 "etag": strings.ToLower(etag), "fileName": file.GetName(), "parentFileId": dstDir.GetID(), "size": file.GetSize(), "type": 0, } var resp UploadResp res, err := d.Request(UploadRequest, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetContext(ctx) }, &resp) if err != nil { return err } log.Debugln("upload request res: ", string(res)) if resp.Data.Reuse || resp.Data.Key == "" { return nil } if resp.Data.AccessKeyId == "" || resp.Data.SecretAccessKey == "" || resp.Data.SessionToken == "" { err = d.newUpload(ctx, &resp, file, up) return err } else { cfg := &aws.Config{ Credentials: credentials.NewStaticCredentials(resp.Data.AccessKeyId, resp.Data.SecretAccessKey, resp.Data.SessionToken), Region: aws.String("123pan"), Endpoint: aws.String(resp.Data.EndPoint), S3ForcePathStyle: aws.Bool(true), } s, err := session.NewSession(cfg) if err != nil { return err } uploader := s3manager.NewUploader(s) if file.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { uploader.PartSize = file.GetSize() / (s3manager.MaxUploadParts - 1) } input := &s3manager.UploadInput{ Bucket: &resp.Data.Bucket, Key: &resp.Data.Key, Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: file, UpdateProgress: up, }), } _, err = uploader.UploadWithContext(ctx, input) if err != nil { return err } } _, err = d.Request(UploadComplete, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "fileId": resp.Data.FileId, }).SetContext(ctx) }, nil) return err } func (d *Pan123) APIRateLimit(ctx context.Context, api string) error { value, _ := d.apiRateLimit.LoadOrStore(api, rate.NewLimiter(rate.Every(700*time.Millisecond), 1)) limiter := value.(*rate.Limiter) return limiter.Wait(ctx) } func (d *Pan123) GetDetails(ctx context.Context) (*model.StorageDetails, error) { userInfo, err := d.getUserInfo(ctx) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: userInfo.Data.SpacePermanent + userInfo.Data.SpaceTemp, UsedSpace: userInfo.Data.SpaceUsed, }, }, nil } var _ driver.Driver = (*Pan123)(nil) ================================================ FILE: drivers/123/meta.go ================================================ package _123 import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` driver.RootID //OrderBy string `json:"order_by" type:"select" options:"file_id,file_name,size,update_at" default:"file_name"` //OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` AccessToken string UploadThread int `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"` Platform string `json:"platform" type:"string" default:"web" help:"the platform header value, sent with API requests"` } var config = driver.Config{ Name: "123Pan", DefaultRoot: "0", LocalSort: true, PreferProxy: true, } func init() { op.RegisterDriver(func() driver.Driver { // 新增默认选项 要在RegisterDriver初始化设置 才会对正在使用的用户生效 return &Pan123{ Addition: Addition{ UploadThread: 3, Platform: "web", }, } }) } ================================================ FILE: drivers/123/types.go ================================================ package _123 import ( "net/url" "path" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type File struct { FileName string `json:"FileName"` Size int64 `json:"Size"` UpdateAt time.Time `json:"UpdateAt"` FileId int64 `json:"FileId"` Type int `json:"Type"` Etag string `json:"Etag"` S3KeyFlag string `json:"S3KeyFlag"` DownloadUrl string `json:"DownloadUrl"` } func (f File) CreateTime() time.Time { return f.UpdateAt } func (f File) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.MD5, f.Etag) } func (f File) GetPath() string { return "" } func (f File) GetSize() int64 { return f.Size } func (f File) GetName() string { return f.FileName } func (f File) ModTime() time.Time { return f.UpdateAt } func (f File) IsDir() bool { return f.Type == 1 } func (f File) GetID() string { return strconv.FormatInt(f.FileId, 10) } func (f File) Thumb() string { if f.DownloadUrl == "" { return "" } du, err := url.Parse(f.DownloadUrl) if err != nil { return "" } du.Path = strings.TrimSuffix(du.Path, "_24_24") + "_70_70" query := du.Query() query.Set("w", "70") query.Set("h", "70") if !query.Has("type") { query.Set("type", strings.TrimPrefix(path.Base(f.FileName), ".")) } if !query.Has("trade_key") { query.Set("trade_key", "123pan-thumbnail") } du.RawQuery = query.Encode() return du.String() } var _ model.Obj = (*File)(nil) var _ model.Thumb = (*File)(nil) //func (f File) Thumb() string { // //} //var _ model.Thumb = (*File)(nil) type Files struct { //BaseResp Data struct { Next string `json:"Next"` Total int `json:"Total"` InfoList []File `json:"InfoList"` } `json:"data"` } //type DownResp struct { // //BaseResp // Data struct { // DownloadUrl string `json:"DownloadUrl"` // } `json:"data"` //} type UploadResp struct { //BaseResp Data struct { AccessKeyId string `json:"AccessKeyId"` Bucket string `json:"Bucket"` Key string `json:"Key"` SecretAccessKey string `json:"SecretAccessKey"` SessionToken string `json:"SessionToken"` FileId int64 `json:"FileId"` Reuse bool `json:"Reuse"` EndPoint string `json:"EndPoint"` StorageNode string `json:"StorageNode"` UploadId string `json:"UploadId"` } `json:"data"` } type S3PreSignedURLs struct { Data struct { PreSignedUrls map[string]string `json:"presignedUrls"` } `json:"data"` } type UserInfoResp struct { Data struct { Uid int64 `json:"UID"` Nickname string `json:"Nickname"` SpaceUsed int64 `json:"SpaceUsed"` SpacePermanent int64 `json:"SpacePermanent"` SpaceTemp int64 `json:"SpaceTemp"` FileCount int `json:"FileCount"` } `json:"data"` } type offlineResolveResp struct { Data struct { List []struct { Result int `json:"result"` ID int64 `json:"id"` ErrCode int `json:"err_code"` ErrMsg string `json:"err_msg"` Files []struct { ID int64 `json:"id"` } `json:"files"` } `json:"list"` } `json:"data"` } type offlineSubmitResp struct { Data struct { TaskList []struct { TaskID int64 `json:"task_id"` Result int `json:"result"` } `json:"task_list"` } `json:"data"` } type offlineTaskListResp struct { Data struct { HasRun bool `json:"has_run"` List []offlineTask `json:"list"` Total int `json:"total"` } `json:"data"` } type offlineTask struct { TaskID int64 `json:"task_id"` Name string `json:"name"` Status int `json:"status"` Size int64 `json:"size"` ThirdTask string `json:"third_task_id"` Downloaded int64 `json:"downloaded"` Progress float64 `json:"progress"` UploadIDR int64 `json:"upload_idr"` UploadName string `json:"upload_name"` Type string `json:"type"` Speed int64 `json:"speed"` } ================================================ FILE: drivers/123/upload.go ================================================ package _123 import ( "context" "fmt" "io" "net/http" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" ) func (d *Pan123) getS3PreSignedUrls(ctx context.Context, upReq *UploadResp, start, end int) (*S3PreSignedURLs, error) { data := base.Json{ "bucket": upReq.Data.Bucket, "key": upReq.Data.Key, "partNumberEnd": end, "partNumberStart": start, "uploadId": upReq.Data.UploadId, "StorageNode": upReq.Data.StorageNode, } var s3PreSignedUrls S3PreSignedURLs _, err := d.Request(S3PreSignedUrls, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetContext(ctx) }, &s3PreSignedUrls) if err != nil { return nil, err } return &s3PreSignedUrls, nil } func (d *Pan123) getS3Auth(ctx context.Context, upReq *UploadResp, start, end int) (*S3PreSignedURLs, error) { data := base.Json{ "StorageNode": upReq.Data.StorageNode, "bucket": upReq.Data.Bucket, "key": upReq.Data.Key, "partNumberEnd": end, "partNumberStart": start, "uploadId": upReq.Data.UploadId, } var s3PreSignedUrls S3PreSignedURLs _, err := d.Request(S3Auth, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetContext(ctx) }, &s3PreSignedUrls) if err != nil { return nil, err } return &s3PreSignedUrls, nil } func (d *Pan123) completeS3(ctx context.Context, upReq *UploadResp, file model.FileStreamer, isMultipart bool) error { data := base.Json{ "StorageNode": upReq.Data.StorageNode, "bucket": upReq.Data.Bucket, "fileId": upReq.Data.FileId, "fileSize": file.GetSize(), "isMultipart": isMultipart, "key": upReq.Data.Key, "uploadId": upReq.Data.UploadId, } _, err := d.Request(UploadCompleteV2, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetContext(ctx) }, nil) return err } func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.FileStreamer, up driver.UpdateProgress) error { // fetch s3 pre signed urls size := file.GetSize() chunkSize := int64(16 * utils.MB) chunkCount := 1 if size > chunkSize { chunkCount = int((size + chunkSize - 1) / chunkSize) } ss, err := stream.NewStreamSectionReader(file, int(chunkSize), &up) if err != nil { return err } lastChunkSize := size % chunkSize if lastChunkSize == 0 { lastChunkSize = chunkSize } // only 1 batch is allowed batchSize := 1 getS3UploadUrl := d.getS3Auth if chunkCount > 1 { batchSize = 10 getS3UploadUrl = d.getS3PreSignedUrls } thread := min(int(chunkCount), d.UploadThread) threadG, uploadCtx := errgroup.NewOrderedGroupWithContext(ctx, thread, retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) for i := 1; i <= chunkCount; i += batchSize { if utils.IsCanceled(uploadCtx) { break } start := i end := min(i+batchSize, chunkCount+1) s3PreSignedUrls, err := getS3UploadUrl(uploadCtx, upReq, start, end) if err != nil { return err } // upload each chunk for cur := start; cur < end; cur++ { if utils.IsCanceled(uploadCtx) { break } offset := int64(cur-1) * chunkSize curSize := chunkSize if cur == chunkCount { curSize = lastChunkSize } var reader io.ReadSeeker threadG.GoWithLifecycle(errgroup.Lifecycle{ Before: func(ctx context.Context) (err error) { reader, err = ss.GetSectionReader(offset, curSize) return }, Do: func(ctx context.Context) (err error) { reader.Seek(0, io.SeekStart) uploadUrl := s3PreSignedUrls.Data.PreSignedUrls[strconv.Itoa(cur)] if uploadUrl == "" { return fmt.Errorf("upload url is empty, s3PreSignedUrls: %+v", s3PreSignedUrls) } req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, driver.NewLimitedUploadStream(ctx, reader)) if err != nil { return err } req.ContentLength = curSize //req.Header.Set("Content-Length", strconv.FormatInt(curSize, 10)) res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() if res.StatusCode == http.StatusForbidden { _, err, _ = singleflight.AnyGroup.Do(fmt.Sprintf("Pan123.newUpload_%p", threadG), func() (any, error) { newS3PreSignedUrls, err := getS3UploadUrl(ctx, upReq, cur, end) if err != nil { return nil, err } s3PreSignedUrls.Data.PreSignedUrls = newS3PreSignedUrls.Data.PreSignedUrls return nil, nil }) if err != nil { return err } return fmt.Errorf("upload s3 chunk %d failed, status code: %d", cur, res.StatusCode) } if res.StatusCode != http.StatusOK { body, err := io.ReadAll(res.Body) if err != nil { return err } return fmt.Errorf("upload s3 chunk %d failed, status code: %d, body: %s", cur, res.StatusCode, body) } progress := 100 * float64(threadG.Success()+1) / float64(chunkCount+1) up(progress) return nil }, After: func(err error) { ss.FreeSectionReader(reader) }, }) } } if err := threadG.Wait(); err != nil { return err } defer up(100) // complete s3 upload return d.completeS3(ctx, upReq, file, chunkCount > 1) } ================================================ FILE: drivers/123/util.go ================================================ package _123 import ( "context" "errors" "fmt" "hash/crc32" "math" "math/rand" "net/http" "net/url" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" ) // do others that not defined in Driver interface const ( Api = "https://www.123pan.com/api" AApi = "https://www.123pan.com/a/api" BApi = "https://www.123pan.com/b/api" LoginApi = "https://login.123pan.com/api" MainApi = BApi SignIn = LoginApi + "/user/sign_in" Logout = MainApi + "/user/logout" UserInfo = MainApi + "/user/info" FileList = MainApi + "/file/list/new" DownloadInfo = MainApi + "/file/download_info" Mkdir = MainApi + "/file/upload_request" Move = MainApi + "/file/mod_pid" Rename = MainApi + "/file/rename" Trash = MainApi + "/file/trash" UploadRequest = MainApi + "/file/upload_request" UploadComplete = MainApi + "/file/upload_complete" S3PreSignedUrls = MainApi + "/file/s3_repare_upload_parts_batch" S3Auth = MainApi + "/file/s3_upload_object/auth" UploadCompleteV2 = MainApi + "/file/upload_complete/v2" S3Complete = MainApi + "/file/s3_complete_multipart_upload" OfflineResolve = MainApi + "/v2/offline_download/task/resolve" OfflineSubmit = MainApi + "/v2/offline_download/task/submit" OfflineTaskList = MainApi + "/offline_download/task/list" OfflineTaskDelete = MainApi + "/offline_download/task/delete" // AuthKeySalt = "8-8D$sL8gPjom7bk#cY" ) var ErrOfflineTaskNotFound = errors.New("offline task not found") func signPath(path string, os string, version string) (k string, v string) { table := []byte{'a', 'd', 'e', 'f', 'g', 'h', 'l', 'm', 'y', 'i', 'j', 'n', 'o', 'p', 'k', 'q', 'r', 's', 't', 'u', 'b', 'c', 'v', 'w', 's', 'z'} random := fmt.Sprintf("%.f", math.Round(1e7*rand.Float64())) now := time.Now().In(time.FixedZone("CST", 8*3600)) timestamp := fmt.Sprint(now.Unix()) nowStr := []byte(now.Format("200601021504")) for i := 0; i < len(nowStr); i++ { nowStr[i] = table[nowStr[i]-48] } timeSign := fmt.Sprint(crc32.ChecksumIEEE(nowStr)) data := strings.Join([]string{timestamp, random, path, os, version, timeSign}, "|") dataSign := fmt.Sprint(crc32.ChecksumIEEE([]byte(data))) return timeSign, strings.Join([]string{timestamp, random, dataSign}, "-") } func GetApi(rawUrl string) string { u, _ := url.Parse(rawUrl) query := u.Query() query.Add(signPath(u.Path, "web", "3")) u.RawQuery = query.Encode() return u.String() } //func GetApi(url string) string { // vm := js.New() // vm.Set("url", url[22:]) // r, err := vm.RunString(` // (function(e){ // function A(t, e) { // e = 1 < arguments.length && void 0 !== e ? e : 10; // for (var n = function() { // for (var t = [], e = 0; e < 256; e++) { // for (var n = e, r = 0; r < 8; r++) // n = 1 & n ? 3988292384 ^ n >>> 1 : n >>> 1; // t[e] = n // } // return t // }(), r = function(t) { // t = t.replace(/\\r\\n/g, "\\n"); // for (var e = "", n = 0; n < t.length; n++) { // var r = t.charCodeAt(n); // r < 128 ? e += String.fromCharCode(r) : e = 127 < r && r < 2048 ? (e += String.fromCharCode(r >> 6 | 192)) + String.fromCharCode(63 & r | 128) : (e = (e += String.fromCharCode(r >> 12 | 224)) + String.fromCharCode(r >> 6 & 63 | 128)) + String.fromCharCode(63 & r | 128) // } // return e // }(t), a = -1, i = 0; i < r.length; i++) // a = a >>> 8 ^ n[255 & (a ^ r.charCodeAt(i))]; // return (a = (-1 ^ a) >>> 0).toString(e) // } // // function v(t) { // return (v = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function(t) { // return typeof t // } // : function(t) { // return t && "function" == typeof Symbol && t.constructor === Symbol && t !== Symbol.prototype ? "symbol" : typeof t // } // )(t) // } // // for (p in a = Math.round(1e7 * Math.random()), // o = Math.round(((new Date).getTime() + 60 * (new Date).getTimezoneOffset() * 1e3 + 288e5) / 1e3).toString(), // m = ["a", "d", "e", "f", "g", "h", "l", "m", "y", "i", "j", "n", "o", "p", "k", "q", "r", "s", "t", "u", "b", "c", "v", "w", "s", "z"], // u = function(t, e, n) { // var r; // n = 2 < arguments.length && void 0 !== n ? n : 8; // return 0 === arguments.length ? null : (r = "object" === v(t) ? t : (10 === "".concat(t).length && (t = 1e3 * Number.parseInt(t)), // new Date(t)), // t += 6e4 * new Date(t).getTimezoneOffset(), // { // y: (r = new Date(t + 36e5 * n)).getFullYear(), // m: r.getMonth() + 1 < 10 ? "0".concat(r.getMonth() + 1) : r.getMonth() + 1, // d: r.getDate() < 10 ? "0".concat(r.getDate()) : r.getDate(), // h: r.getHours() < 10 ? "0".concat(r.getHours()) : r.getHours(), // f: r.getMinutes() < 10 ? "0".concat(r.getMinutes()) : r.getMinutes() // }) // }(o), // h = u.y, // g = u.m, // l = u.d, // c = u.h, // u = u.f, // d = [h, g, l, c, u].join(""), // f = [], // d) // f.push(m[Number(d[p])]); // return h = A(f.join("")), // g = A("".concat(o, "|").concat(a, "|").concat(e, "|").concat("web", "|").concat("3", "|").concat(h)), // "".concat(h, "=").concat(o, "-").concat(a, "-").concat(g); // })(url) // `) // if err != nil { // fmt.Println(err) // return url // } // v, _ := r.Export().(string) // return url + "?" + v //} func (d *Pan123) login() error { var body base.Json if utils.IsEmailFormat(d.Username) { body = base.Json{ "mail": d.Username, "password": d.Password, "type": 2, } } else { body = base.Json{ "passport": d.Username, "password": d.Password, "remember": true, } } res, err := base.RestyClient.R(). SetHeaders(map[string]string{ "origin": "https://www.123pan.com", "referer": "https://www.123pan.com/", "user-agent": "Dart/2.19(dart:io)-openlist", "platform": "web", "app-version": "3", //"user-agent": base.UserAgent, }). SetBody(body).Post(SignIn) if err != nil { return err } if utils.Json.Get(res.Body(), "code").ToInt() != 200 { err = fmt.Errorf(utils.Json.Get(res.Body(), "message").ToString()) } else { d.AccessToken = utils.Json.Get(res.Body(), "data", "token").ToString() } return err } //func authKey(reqUrl string) (*string, error) { // reqURL, err := url.Parse(reqUrl) // if err != nil { // return nil, err // } // // nowUnix := time.Now().Unix() // random := rand.Intn(0x989680) // // p4 := fmt.Sprintf("%d|%d|%s|%s|%s|%s", nowUnix, random, reqURL.Path, "web", "3", AuthKeySalt) // authKey := fmt.Sprintf("%d-%d-%x", nowUnix, random, md5.Sum([]byte(p4))) // return &authKey, nil //} func (d *Pan123) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { isRetry := false do: req := base.RestyClient.R() req.SetHeaders(map[string]string{ "origin": "https://www.123pan.com", "referer": "https://www.123pan.com/", "authorization": "Bearer " + d.AccessToken, "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) openlist-client", "platform": d.Platform, "app-version": "3", //"user-agent": base.UserAgent, }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } //authKey, err := authKey(url) //if err != nil { // return nil, err //} //req.SetQueryParam("auth-key", *authKey) res, err := req.Execute(method, GetApi(url)) if err != nil { return nil, err } body := res.Body() code := utils.Json.Get(body, "code").ToInt() if code != 0 { if !isRetry && code == 401 { err := d.login() if err != nil { return nil, err } isRetry = true goto do } return nil, errors.New(jsoniter.Get(body, "message").ToString()) } return body, nil } func (d *Pan123) OfflineDownload(ctx context.Context, uri string, dstDir model.Obj) (int64, error) { var resolveResp offlineResolveResp _, err := d.Request(OfflineResolve, http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "urls": uri, }) }, &resolveResp) if err != nil { return 0, err } if len(resolveResp.Data.List) == 0 { return 0, fmt.Errorf("offline resolve failed: empty response") } if resolveResp.Data.List[0].Result != 0 { msg := resolveResp.Data.List[0].ErrMsg if msg == "" { msg = "offline resolve failed" } return 0, fmt.Errorf("%s", msg) } resourceID := resolveResp.Data.List[0].ID if resourceID == 0 { return 0, fmt.Errorf("offline resolve failed: empty resource id") } selectFileIDs := make([]int64, 0, len(resolveResp.Data.List[0].Files)) for _, f := range resolveResp.Data.List[0].Files { if f.ID > 0 { selectFileIDs = append(selectFileIDs, f.ID) } } if len(selectFileIDs) == 0 { return 0, fmt.Errorf("offline resolve failed: empty file list") } uploadDir, err := strconv.ParseInt(dstDir.GetID(), 10, 64) if err != nil { return 0, fmt.Errorf("invalid destination dir id: %s", dstDir.GetID()) } var submitResp offlineSubmitResp _, err = d.Request(OfflineSubmit, http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "resource_list": []base.Json{ { "resource_id": resourceID, "select_file_id": selectFileIDs, }, }, "upload_dir": uploadDir, }) }, &submitResp) if err != nil { return 0, err } if len(submitResp.Data.TaskList) == 0 { return 0, fmt.Errorf("offline submit failed: empty task list") } if submitResp.Data.TaskList[0].Result != 0 { return 0, fmt.Errorf("offline submit failed") } if submitResp.Data.TaskList[0].TaskID == 0 { return 0, fmt.Errorf("offline submit failed: empty task id") } return submitResp.Data.TaskList[0].TaskID, nil } func (d *Pan123) GetOfflineTask(ctx context.Context, taskID int64) (*offlineTask, error) { if taskID == 0 { return nil, fmt.Errorf("invalid task id") } page := 1 pageSize := 100 statusArr := []int{0, 1, 2, 3} for { var listResp offlineTaskListResp _, err := d.Request(OfflineTaskList, http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "current_page": page, "page_size": pageSize, "status_arr": statusArr, }) }, &listResp) if err != nil { return nil, err } for i := range listResp.Data.List { if listResp.Data.List[i].TaskID == taskID { return &listResp.Data.List[i], nil } } if len(listResp.Data.List) == 0 || page*pageSize >= listResp.Data.Total { break } page++ } return nil, ErrOfflineTaskNotFound } func (d *Pan123) DeleteOfflineTasks(ctx context.Context, taskIDs []int64) error { if len(taskIDs) == 0 { return nil } _, err := d.Request(OfflineTaskDelete, http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "task_ids": taskIDs, }) }, nil) return err } func (d *Pan123) getFiles(ctx context.Context, parentId string, name string) ([]File, error) { page := 1 total := 0 res := make([]File, 0) // 2024-02-06 fix concurrency by 123pan for { if err := d.APIRateLimit(ctx, FileList); err != nil { return nil, err } var resp Files query := map[string]string{ "driveId": "0", "limit": "100", "next": "0", "orderBy": "file_id", "orderDirection": "desc", "parentFileId": parentId, "trashed": "false", "SearchData": "", "Page": strconv.Itoa(page), "OnlyLookAbnormalFile": "0", "event": "homeListFile", "operateType": "4", "inDirectSpace": "false", } _res, err := d.Request(FileList, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return nil, err } log.Debug(string(_res)) page++ res = append(res, resp.Data.InfoList...) total = resp.Data.Total if len(resp.Data.InfoList) == 0 || resp.Data.Next == "-1" { break } } if len(res) != total { log.Warnf("incorrect file count from remote at %s: expected %d, got %d", name, total, len(res)) } return res, nil } func (d *Pan123) getUserInfo(ctx context.Context) (*UserInfoResp, error) { var resp UserInfoResp _, err := d.Request(UserInfo, http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }, &resp) if err != nil { return nil, err } return &resp, nil } ================================================ FILE: drivers/123_link/driver.go ================================================ package _123Link import ( "context" stdpath "path" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type Pan123Link struct { model.Storage Addition root *Node } func (d *Pan123Link) Config() driver.Config { return config } func (d *Pan123Link) GetAddition() driver.Additional { return &d.Addition } func (d *Pan123Link) Init(ctx context.Context) error { node, err := BuildTree(d.OriginURLs) if err != nil { return err } node.calSize() d.root = node return nil } func (d *Pan123Link) Drop(ctx context.Context) error { return nil } func (Addition) GetRootPath() string { return "/" } func (d *Pan123Link) Get(ctx context.Context, path string) (model.Obj, error) { node := GetNodeFromRootByPath(d.root, path) return nodeToObj(node, path) } func (d *Pan123Link) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { node := GetNodeFromRootByPath(d.root, dir.GetPath()) if node == nil { return nil, errs.ObjectNotFound } if node.isFile() { return nil, errs.NotFolder } return utils.SliceConvert(node.Children, func(node *Node) (model.Obj, error) { return nodeToObj(node, stdpath.Join(dir.GetPath(), node.Name)) }) } func (d *Pan123Link) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { node := GetNodeFromRootByPath(d.root, file.GetPath()) if node == nil { return nil, errs.ObjectNotFound } if node.isFile() { signUrl, err := SignURL(node.Url, d.PrivateKey, d.UID, time.Duration(d.ValidDuration)*time.Minute) if err != nil { return nil, err } return &model.Link{ URL: signUrl, }, nil } return nil, errs.NotFile } var _ driver.Driver = (*Pan123Link)(nil) ================================================ FILE: drivers/123_link/meta.go ================================================ package _123Link import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { OriginURLs string `json:"origin_urls" type:"text" required:"true" default:"https://vip.123pan.com/29/folder/file.mp3" help:"structure:FolderName:\n [FileSize:][Modified:]Url"` PrivateKey string `json:"private_key"` UID uint64 `json:"uid" type:"number"` ValidDuration int64 `json:"valid_duration" type:"number" default:"30" help:"minutes"` } var config = driver.Config{ Name: "123PanLink", } func init() { op.RegisterDriver(func() driver.Driver { return &Pan123Link{} }) } ================================================ FILE: drivers/123_link/parse.go ================================================ package _123Link import ( "fmt" url2 "net/url" stdpath "path" "strconv" "strings" "time" ) // build tree from text, text structure definition: /** * FolderName: * [FileSize:][Modified:]Url */ /** * For example: * folder1: * name1:url1 * url2 * folder2: * url3 * url4 * url5 * folder3: * url6 * url7 * url8 */ // if there are no name, use the last segment of url as name func BuildTree(text string) (*Node, error) { lines := strings.Split(text, "\n") var root = &Node{Level: -1, Name: "root"} stack := []*Node{root} for _, line := range lines { // calculate indent indent := 0 for i := 0; i < len(line); i++ { if line[i] != ' ' { break } indent++ } // if indent is not a multiple of 2, it is an error if indent%2 != 0 { return nil, fmt.Errorf("the line '%s' is not a multiple of 2", line) } // calculate level level := indent / 2 line = strings.TrimSpace(line[indent:]) // if the line is empty, skip if line == "" { continue } // if level isn't greater than the level of the top of the stack // it is not the child of the top of the stack for level <= stack[len(stack)-1].Level { // pop the top of the stack stack = stack[:len(stack)-1] } // if the line is a folder if isFolder(line) { // create a new node node := &Node{ Level: level, Name: strings.TrimSuffix(line, ":"), } // add the node to the top of the stack stack[len(stack)-1].Children = append(stack[len(stack)-1].Children, node) // push the node to the stack stack = append(stack, node) } else { // if the line is a file // create a new node node, err := parseFileLine(line) if err != nil { return nil, err } node.Level = level // add the node to the top of the stack stack[len(stack)-1].Children = append(stack[len(stack)-1].Children, node) } } return root, nil } func isFolder(line string) bool { return strings.HasSuffix(line, ":") } // line definition: // [FileSize:][Modified:]Url func parseFileLine(line string) (*Node, error) { // if there is no url, it is an error if !strings.Contains(line, "http://") && !strings.Contains(line, "https://") { return nil, fmt.Errorf("invalid line: %s, because url is required for file", line) } index := strings.Index(line, "http://") if index == -1 { index = strings.Index(line, "https://") } url := line[index:] info := line[:index] node := &Node{ Url: url, } name := stdpath.Base(url) unescape, err := url2.PathUnescape(name) if err == nil { name = unescape } node.Name = name if index > 0 { if !strings.HasSuffix(info, ":") { return nil, fmt.Errorf("invalid line: %s, because file info must end with ':'", line) } info = info[:len(info)-1] if info == "" { return nil, fmt.Errorf("invalid line: %s, because file name can't be empty", line) } infoParts := strings.Split(info, ":") size, err := strconv.ParseInt(infoParts[0], 10, 64) if err != nil { return nil, fmt.Errorf("invalid line: %s, because file size must be an integer", line) } node.Size = size if len(infoParts) > 1 { modified, err := strconv.ParseInt(infoParts[1], 10, 64) if err != nil { return nil, fmt.Errorf("invalid line: %s, because file modified must be an unix timestamp", line) } node.Modified = modified } else { node.Modified = time.Now().Unix() } } return node, nil } func splitPath(path string) []string { if path == "/" { return []string{"root"} } parts := strings.Split(path, "/") parts[0] = "root" return parts } func GetNodeFromRootByPath(root *Node, path string) *Node { return root.getByPath(splitPath(path)) } ================================================ FILE: drivers/123_link/types.go ================================================ package _123Link import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" ) // Node is a node in the folder tree type Node struct { Url string Name string Level int Modified int64 Size int64 Children []*Node } func (node *Node) getByPath(paths []string) *Node { if len(paths) == 0 || node == nil { return nil } if node.Name != paths[0] { return nil } if len(paths) == 1 { return node } for _, child := range node.Children { tmp := child.getByPath(paths[1:]) if tmp != nil { return tmp } } return nil } func (node *Node) isFile() bool { return node.Url != "" } func (node *Node) calSize() int64 { if node.isFile() { return node.Size } var size int64 = 0 for _, child := range node.Children { size += child.calSize() } node.Size = size return size } func nodeToObj(node *Node, path string) (model.Obj, error) { if node == nil { return nil, errs.ObjectNotFound } return &model.Object{ Name: node.Name, Size: node.Size, Modified: time.Unix(node.Modified, 0), IsFolder: !node.isFile(), Path: path, }, nil } ================================================ FILE: drivers/123_link/util.go ================================================ package _123Link import ( "crypto/md5" "fmt" "math/rand" "net/url" "time" ) func SignURL(originURL, privateKey string, uid uint64, validDuration time.Duration) (newURL string, err error) { if privateKey == "" { return originURL, nil } var ( ts = time.Now().Add(validDuration).Unix() // 有效时间戳 rInt = rand.Int() // 随机正整数 objURL *url.URL ) objURL, err = url.Parse(originURL) if err != nil { return "", err } authKey := fmt.Sprintf("%d-%d-%d-%x", ts, rInt, uid, md5.Sum([]byte(fmt.Sprintf("%s-%d-%d-%d-%s", objURL.Path, ts, rInt, uid, privateKey)))) v := objURL.Query() v.Add("auth_key", authKey) objURL.RawQuery = v.Encode() return objURL.String(), nil } ================================================ FILE: drivers/123_open/driver.go ================================================ package _123_open import ( "context" "fmt" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type Open123 struct { model.Storage Addition UID uint64 tm *tokenManager } func (d *Open123) Config() driver.Config { return config } func (d *Open123) GetAddition() driver.Additional { return &d.Addition } func (d *Open123) Init(ctx context.Context) error { if d.UploadThread < 1 || d.UploadThread > 32 { d.UploadThread = 3 } if d.RefreshToken != "" { // refresh token 直接主动刷新 d.AccessToken = "" d.tm = &tokenManager{} } else { // 避免个人 token 刷新产生的多个登录，被动刷新 // 默认过期时间90天，jwt exp 不可靠 d.tm = &tokenManager{ // accessToken: d.AccessToken, expiredAt: time.Now().Add(90 * 24 * time.Hour), } } _, err := d.getAccessToken(false) if err != nil { return fmt.Errorf("init get access token error: %w", err) } return nil } func (d *Open123) Drop(ctx context.Context) error { op.MustSaveDriverStorage(d) return nil } func (d *Open123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { fileLastId := int64(0) parentFileId, err := strconv.ParseInt(dir.GetID(), 10, 64) if err != nil { return nil, err } res := make([]File, 0) for fileLastId != -1 { files, err := d.getFiles(parentFileId, 100, fileLastId) if err != nil { return nil, err } // 目前123panAPI请求，trashed失效，只能通过遍历过滤 for i := range files.Data.FileList { if files.Data.FileList[i].Trashed == 0 { res = append(res, files.Data.FileList[i]) } } fileLastId = files.Data.LastFileId } return utils.SliceConvert(res, func(src File) (model.Obj, error) { return src, nil }) } func (d *Open123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { fileId, _ := strconv.ParseInt(file.GetID(), 10, 64) if d.DirectLink { res, err := d.getDirectLink(fileId) if err != nil { return nil, err } if d.DirectLinkPrivateKey == "" { duration := 365 * 24 * time.Hour // 缓存1年 return &model.Link{ URL: res.Data.URL, Expiration: &duration, }, nil } uid, err := d.getUID(ctx) if err != nil { return nil, err } duration := time.Duration(d.DirectLinkValidDuration) * time.Minute newURL, err := d.SignURL(res.Data.URL, d.DirectLinkPrivateKey, uid, duration) if err != nil { return nil, err } return &model.Link{ URL: newURL, Expiration: &duration, }, nil } res, err := d.getDownloadInfo(fileId) if err != nil { return nil, err } return &model.Link{URL: res.Data.DownloadUrl}, nil } func (d *Open123) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { parentFileId, _ := strconv.ParseInt(parentDir.GetID(), 10, 64) return d.mkdir(parentFileId, dirName) } func (d *Open123) Move(ctx context.Context, srcObj, dstDir model.Obj) error { toParentFileID, _ := strconv.ParseInt(dstDir.GetID(), 10, 64) return d.move(srcObj.(File).FileId, toParentFileID) } func (d *Open123) Rename(ctx context.Context, srcObj model.Obj, newName string) error { fileId, _ := strconv.ParseInt(srcObj.GetID(), 10, 64) return d.rename(fileId, newName) } func (d *Open123) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { // 尝试使用上传+MD5秒传功能实现复制 // 1. 创建文件 // parentFileID 父目录id，上传到根目录时填写 0 parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) if err != nil { return fmt.Errorf("parse parentFileID error: %v", err) } etag := srcObj.(File).Etag createResp, err := d.create(parentFileId, srcObj.GetName(), etag, srcObj.GetSize(), 2, false) if err != nil { return err } // 是否秒传 if createResp.Data.Reuse { return nil } return errs.NotSupport } func (d *Open123) Remove(ctx context.Context, obj model.Obj) error { fileId, _ := strconv.ParseInt(obj.GetID(), 10, 64) return d.trash(fileId) } func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { // 1. 创建文件 // parentFileID 父目录id，上传到根目录时填写 0 parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) if err != nil { return nil, fmt.Errorf("parse parentFileID error: %v", err) } // 尝试 SHA1 秒传 sha1Hash := file.GetHash().GetHash(utils.SHA1) if len(sha1Hash) == utils.SHA1.Width { resp, err := d.sha1Reuse(parentFileId, file.GetName(), sha1Hash, file.GetSize(), 2) if err == nil && resp.Data.Reuse { return File{ FileName: file.GetName(), Size: file.GetSize(), FileId: resp.Data.FileID, Type: 2, SHA1: sha1Hash, }, nil } } // etag 文件md5 etag := file.GetHash().GetHash(utils.MD5) if len(etag) < utils.MD5.Width { _, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5) if err != nil { return nil, err } } createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { return nil, err } // 是否秒传 if createResp.Data.Reuse { // 秒传成功才会返回正确的 FileID，否则为 0 if createResp.Data.FileID != 0 { return File{ FileName: file.GetName(), Size: file.GetSize(), FileId: createResp.Data.FileID, Type: 2, Etag: etag, }, nil } } // 2. 上传分片 err = d.Upload(ctx, file, createResp, up) if err != nil { return nil, err } // 3. 上传完毕 for range 60 { uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID) // 返回错误代码未知，如：20103，文档也没有具体说 if err == nil && uploadCompleteResp.Data.Completed && uploadCompleteResp.Data.FileID != 0 { up(100) return File{ FileName: file.GetName(), Size: file.GetSize(), FileId: uploadCompleteResp.Data.FileID, Type: 2, Etag: etag, }, nil } // 若接口返回的completed为 false 时，则需间隔1秒继续轮询此接口，获取上传最终结果。 time.Sleep(time.Second) } return nil, fmt.Errorf("upload complete timeout") } func (d *Open123) GetDetails(ctx context.Context) (*model.StorageDetails, error) { userInfo, err := d.getUserInfo(ctx) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: userInfo.Data.SpacePermanent + userInfo.Data.SpaceTemp, UsedSpace: userInfo.Data.SpaceUsed, }, }, nil } func (d *Open123) OfflineDownload(ctx context.Context, url string, dir model.Obj, callback string) (int, error) { return d.createOfflineDownloadTask(ctx, url, dir.GetID(), callback) } func (d *Open123) OfflineDownloadProcess(ctx context.Context, taskID int) (float64, int, error) { return d.queryOfflineDownloadStatus(ctx, taskID) } var ( _ driver.Driver = (*Open123)(nil) _ driver.PutResult = (*Open123)(nil) ) ================================================ FILE: drivers/123_open/meta.go ================================================ package _123_open import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { // refresh_token方式的AccessToken 【对个人开发者暂未开放】 RefreshToken string `json:"RefreshToken" required:"false"` // 通过 https://www.123pan.com/developer 申请 ClientID string `json:"ClientID" required:"false"` ClientSecret string `json:"ClientSecret" required:"false"` // 直接写入AccessToken, AccessToken有过期时间，不建议直接填写 AccessToken string `json:"AccessToken" required:"false"` // 用户名+密码方式登录的AccessToken可以兼容 //Username string `json:"username" required:"false"` //Password string `json:"password" required:"false"` // 上传线程数 UploadThread int `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"` // 使用直链 DirectLink bool `json:"DirectLink" type:"bool" default:"false" required:"false" help:"use direct link when download file"` DirectLinkPrivateKey string `json:"DirectLinkPrivateKey" required:"false" help:"private key for direct link, if URL authentication is enabled"` DirectLinkValidDuration int64 `json:"DirectLinkValidDuration" type:"number" default:"30" required:"false" help:"minutes, if URL authentication is enabled"` driver.RootID } var config = driver.Config{ Name: "123 Open", DefaultRoot: "0", LocalSort: true, PreferProxy: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Open123{} }) } ================================================ FILE: drivers/123_open/token.go ================================================ package _123_open import ( "encoding/json" "errors" "fmt" "net/http" "sync" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/op" ) var ( AccessToken = "https://open-api.123pan.com/api/v1/access_token" RefreshToken = "https://open-api.123pan.com/api/v1/oauth2/access_token" ) type tokenManager struct { // accessToken string expiredAt time.Time mu sync.Mutex blockRefresh bool } func (d *Open123) getAccessToken(forceRefresh bool) (string, error) { tm := d.tm tm.mu.Lock() defer tm.mu.Unlock() if tm.blockRefresh { return "", errors.New("Authentication expired") } if !forceRefresh && d.AccessToken != "" && time.Now().Before(tm.expiredAt.Add(-5*time.Minute)) { return d.AccessToken, nil } if err := d.flushAccessToken(); err != nil { // token expired and failed to refresh, block further refresh attempts tm.blockRefresh = true return "", err } return d.AccessToken, nil } func (d *Open123) flushAccessToken() error { // directly send request to avoid deadlock req := base.RestyClient.R() req.SetHeaders(map[string]string{ "authorization": "Bearer " + d.AccessToken, "platform": "open_platform", "Content-Type": "application/json", }) if d.ClientID != "" { if d.RefreshToken != "" { var resp RefreshTokenResp req.SetQueryParam("client_id", d.ClientID) if d.ClientSecret != "" { req.SetQueryParam("client_secret", d.ClientSecret) } req.SetQueryParam("grant_type", "refresh_token") req.SetQueryParam("refresh_token", d.RefreshToken) req.SetResult(&resp) res, err := req.Execute(http.MethodPost, RefreshToken) if err != nil { return err } body := res.Body() var baseResp BaseResp if err = json.Unmarshal(body, &baseResp); err != nil { return err } if baseResp.Code != 0 { return fmt.Errorf("get access token failed: %s", baseResp.Message) } d.AccessToken = resp.AccessToken // add token expire time d.tm.expiredAt = time.Now().Add(time.Duration(resp.ExpiresIn) * time.Second) d.RefreshToken = resp.RefreshToken op.MustSaveDriverStorage(d) d.tm.blockRefresh = false return nil } else if d.ClientSecret != "" { var resp AccessTokenResp req.SetBody(base.Json{ "clientID": d.ClientID, "clientSecret": d.ClientSecret, }) req.SetResult(&resp) res, err := req.Execute(http.MethodPost, AccessToken) if err != nil { return err } body := res.Body() var baseResp BaseResp if err = json.Unmarshal(body, &baseResp); err != nil { return err } if baseResp.Code != 0 { return fmt.Errorf("get access token failed: %s", baseResp.Message) } d.AccessToken = resp.Data.AccessToken // parse token expire time d.tm.expiredAt, err = time.Parse(time.RFC3339, resp.Data.ExpiredAt) if err != nil { return fmt.Errorf("parse expire time failed: %w", err) } op.MustSaveDriverStorage(d) d.tm.blockRefresh = false return nil } } return errors.New("no valid authentication method available") } ================================================ FILE: drivers/123_open/types.go ================================================ package _123_open import ( "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type ApiInfo struct { url string qps int token chan struct{} } func (a *ApiInfo) Require() { if a.qps > 0 { a.token <- struct{}{} } } func (a *ApiInfo) Release() { if a.qps > 0 { time.AfterFunc(time.Second, func() { <-a.token }) } } func (a *ApiInfo) SetQPS(qps int) { a.qps = qps a.token = make(chan struct{}, qps) } func (a *ApiInfo) NowLen() int { return len(a.token) } func InitApiInfo(url string, qps int) *ApiInfo { return &ApiInfo{ url: url, qps: qps, token: make(chan struct{}, qps), } } type File struct { FileName string `json:"filename"` Size int64 `json:"size"` CreateAt string `json:"createAt"` UpdateAt string `json:"updateAt"` FileId int64 `json:"fileId"` Type int `json:"type"` Etag string `json:"etag"` S3KeyFlag string `json:"s3KeyFlag"` ParentFileId int `json:"parentFileId"` Category int `json:"category"` Status int `json:"status"` Trashed int `json:"trashed"` SHA1 string } func (f File) GetHash() utils.HashInfo { if len(f.SHA1) == utils.SHA1.Width && len(f.Etag) != utils.MD5.Width { return utils.NewHashInfo(utils.SHA1, f.SHA1) } return utils.NewHashInfo(utils.MD5, f.Etag) } func (f File) GetPath() string { return "" } func (f File) GetSize() int64 { return f.Size } func (f File) GetName() string { return f.FileName } func (f File) CreateTime() time.Time { // 返回的时间没有时区信息，默认 UTC+8 loc := time.FixedZone("UTC+8", 8*60*60) parsedTime, err := time.ParseInLocation("2006-01-02 15:04:05", f.CreateAt, loc) if err != nil { return time.Now() } return parsedTime } func (f File) ModTime() time.Time { // 返回的时间没有时区信息，默认 UTC+8 loc := time.FixedZone("UTC+8", 8*60*60) parsedTime, err := time.ParseInLocation("2006-01-02 15:04:05", f.UpdateAt, loc) if err != nil { return time.Now() } return parsedTime } func (f File) IsDir() bool { return f.Type == 1 } func (f File) GetID() string { return strconv.FormatInt(f.FileId, 10) } var _ model.Obj = (*File)(nil) type BaseResp struct { Code int `json:"code"` Message string `json:"message"` XTraceID string `json:"x-traceID"` } type AccessTokenResp struct { BaseResp Data struct { AccessToken string `json:"accessToken"` ExpiredAt string `json:"expiredAt"` } `json:"data"` } type RefreshTokenResp struct { AccessToken string `json:"access_token"` ExpiresIn int `json:"expires_in"` RefreshToken string `json:"refresh_token"` Scope string `json:"scope"` TokenType string `json:"token_type"` } type UserInfoResp struct { BaseResp Data struct { UID uint64 `json:"uid"` // Username string `json:"username"` // DisplayName string `json:"displayName"` // HeadImage string `json:"headImage"` // Passport string `json:"passport"` // Mail string `json:"mail"` SpaceUsed int64 `json:"spaceUsed"` SpacePermanent int64 `json:"spacePermanent"` SpaceTemp int64 `json:"spaceTemp"` // SpaceTempExpr int64 `json:"spaceTempExpr"` // Vip bool `json:"vip"` // DirectTraffic int64 `json:"directTraffic"` // IsHideUID bool `json:"isHideUID"` } `json:"data"` } type FileListResp struct { BaseResp Data struct { LastFileId int64 `json:"lastFileId"` FileList []File `json:"fileList"` } `json:"data"` } type DownloadInfoResp struct { BaseResp Data struct { DownloadUrl string `json:"downloadUrl"` } `json:"data"` } type DirectLinkResp struct { BaseResp Data struct { URL string `json:"url"` } `json:"data"` } // 创建文件V2返回 type UploadCreateResp struct { BaseResp Data struct { FileID int64 `json:"fileID"` PreuploadID string `json:"preuploadID"` Reuse bool `json:"reuse"` SliceSize int64 `json:"sliceSize"` Servers []string `json:"servers"` } `json:"data"` } // 上传完毕V2返回 type UploadCompleteResp struct { BaseResp Data struct { Completed bool `json:"completed"` FileID int64 `json:"fileID"` } `json:"data"` } type SHA1ReuseResp struct { BaseResp Data struct { FileID int64 `json:"fileID"` Reuse bool `json:"reuse"` } `json:"data"` } type OfflineDownloadResp struct { BaseResp Data struct { TaskID int `json:"taskID"` } `json:"data"` } type OfflineDownloadProcessResp struct { BaseResp Data struct { Process float64 `json:"process"` Status int `json:"status"` } `json:"data"` } ================================================ FILE: drivers/123_open/upload.go ================================================ package _123_open import ( "bytes" "context" "encoding/json" "fmt" "io" "mime/multipart" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" ) // 创建文件 V2 func (d *Open123) create(parentFileID int64, filename string, etag string, size int64, duplicate int, containDir bool) (*UploadCreateResp, error) { var resp UploadCreateResp _, err := d.Request(UploadCreate, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "parentFileId": parentFileID, "filename": filename, "etag": strings.ToLower(etag), "size": size, "duplicate": duplicate, "containDir": containDir, }) }, &resp) if err != nil { return nil, err } return &resp, nil } // 上传分片 V2 func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createResp *UploadCreateResp, up driver.UpdateProgress) error { uploadDomain := createResp.Data.Servers[0] size := file.GetSize() chunkSize := createResp.Data.SliceSize ss, err := stream.NewStreamSectionReader(file, int(chunkSize), &up) if err != nil { return err } uploadNums := (size + chunkSize - 1) / chunkSize thread := min(int(uploadNums), d.UploadThread) threadG, uploadCtx := errgroup.NewOrderedGroupWithContext(ctx, thread, retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) for partIndex := range uploadNums { if utils.IsCanceled(uploadCtx) { break } partIndex := partIndex partNumber := partIndex + 1 // 分片号从1开始 offset := partIndex * chunkSize size := min(chunkSize, size-offset) var reader io.ReadSeeker var rateLimitedRd io.Reader sliceMD5 := "" // 表单 b := bytes.NewBuffer(make([]byte, 0, 2048)) threadG.GoWithLifecycle(errgroup.Lifecycle{ Before: func(ctx context.Context) (err error) { reader, err = ss.GetSectionReader(offset, size) return }, Do: func(ctx context.Context) (err error) { reader.Seek(0, io.SeekStart) if sliceMD5 == "" { // 把耗时的计算放在这里，避免阻塞其他协程 sliceMD5, err = utils.HashReader(utils.MD5, reader) if err != nil { return err } reader.Seek(0, io.SeekStart) } b.Reset() w := multipart.NewWriter(b) // 添加表单字段 err = w.WriteField("preuploadID", createResp.Data.PreuploadID) if err != nil { return err } err = w.WriteField("sliceNo", strconv.FormatInt(partNumber, 10)) if err != nil { return err } err = w.WriteField("sliceMD5", sliceMD5) if err != nil { return err } // 写入文件内容 _, err = w.CreateFormFile("slice", fmt.Sprintf("%s.part%d", file.GetName(), partNumber)) if err != nil { return err } headSize := b.Len() err = w.Close() if err != nil { return err } head := bytes.NewReader(b.Bytes()[:headSize]) tail := bytes.NewReader(b.Bytes()[headSize:]) rateLimitedRd = driver.NewLimitedUploadStream(ctx, io.MultiReader(head, reader, tail)) token, err := d.getAccessToken(false) if err != nil { return err } // 创建请求并设置header req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadDomain+"/upload/v2/file/slice", rateLimitedRd) if err != nil { return err } // 设置请求头 req.Header.Add("Authorization", "Bearer "+token) req.Header.Add("Content-Type", w.FormDataContentType()) req.Header.Add("Platform", "open_platform") res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() if res.StatusCode != 200 { return fmt.Errorf("slice %d upload failed, status code: %d", partNumber, res.StatusCode) } b.Reset() _, err = b.ReadFrom(res.Body) if err != nil { return err } var resp BaseResp err = json.Unmarshal(b.Bytes(), &resp) if err != nil { return err } if resp.Code != 0 { return fmt.Errorf("slice %d upload failed: %s", partNumber, resp.Message) } progress := 100 * float64(threadG.Success()+1) / float64(uploadNums+1) up(progress) return nil }, After: func(err error) { ss.FreeSectionReader(reader) }, }) } if err := threadG.Wait(); err != nil { return err } return nil } // 上传完毕 func (d *Open123) complete(preuploadID string) (*UploadCompleteResp, error) { var resp UploadCompleteResp _, err := d.Request(UploadComplete, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "preuploadID": preuploadID, }) }, &resp) if err != nil { return nil, err } return &resp, nil } // SHA1 秒传 func (d *Open123) sha1Reuse(parentFileID int64, filename string, sha1Hash string, size int64, duplicate int) (*SHA1ReuseResp, error) { var resp SHA1ReuseResp _, err := d.Request(UploadSHA1Reuse, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "parentFileID": parentFileID, "filename": filename, "sha1": strings.ToLower(sha1Hash), "size": size, "duplicate": duplicate, }) }, &resp) if err != nil { return nil, err } return &resp, nil } ================================================ FILE: drivers/123_open/util.go ================================================ package _123_open import ( "context" "crypto/md5" "encoding/json" "errors" "fmt" "net/http" "net/url" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/go-resty/resty/v2" "github.com/google/uuid" log "github.com/sirupsen/logrus" ) var ( // 不同情况下获取的AccessTokenQPS限制不同 如下模块化易于拓展 Api = "https://open-api.123pan.com" UserInfo = InitApiInfo(Api+"/api/v1/user/info", 1) FileList = InitApiInfo(Api+"/api/v2/file/list", 3) DownloadInfo = InitApiInfo(Api+"/api/v1/file/download_info", 5) DirectLink = InitApiInfo(Api+"/api/v1/direct-link/url", 5) Mkdir = InitApiInfo(Api+"/upload/v1/file/mkdir", 2) Move = InitApiInfo(Api+"/api/v1/file/move", 1) Rename = InitApiInfo(Api+"/api/v1/file/name", 1) Trash = InitApiInfo(Api+"/api/v1/file/trash", 2) UploadCreate = InitApiInfo(Api+"/upload/v2/file/create", 2) UploadComplete = InitApiInfo(Api+"/upload/v2/file/upload_complete", 0) UploadSHA1Reuse = InitApiInfo(Api+"/upload/v2/file/sha1_reuse", 2) OfflineDownload = InitApiInfo(Api+"/api/v1/offline/download", 1) OfflineDownloadProcess = InitApiInfo(Api+"/api/v1/offline/download/process", 5) ) func (d *Open123) Request(apiInfo *ApiInfo, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { for { token, err := d.getAccessToken(false) if err != nil { return nil, err } req := base.RestyClient.R() req.SetHeaders(map[string]string{ "authorization": "Bearer " + token, "platform": "open_platform", "Content-Type": "application/json", }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } log.Debugf("API: %s, QPS: %d, NowLen: %d", apiInfo.url, apiInfo.qps, apiInfo.NowLen()) apiInfo.Require() defer apiInfo.Release() res, err := req.Execute(method, apiInfo.url) if err != nil { return nil, err } body := res.Body() // 解析为通用响应 var baseResp BaseResp if err = json.Unmarshal(body, &baseResp); err != nil { return nil, err } if baseResp.Code == 0 { return body, nil } else if baseResp.Code == 401 { // 强制刷新Token, 有小概率会 race condition 导致多次刷新Token，但不影响正确运行 if _, err := d.getAccessToken(true); err != nil { return nil, err } } else if baseResp.Code == 429 { time.Sleep(500 * time.Millisecond) log.Warningf("API: %s, QPS: %d, 请求太频繁，对应API提示过多请减小QPS", apiInfo.url, apiInfo.qps) } else { return nil, errors.New(baseResp.Message) } } } func (d *Open123) SignURL(originURL, privateKey string, uid uint64, validDuration time.Duration) (newURL string, err error) { // 生成Unix时间戳 ts := time.Now().Add(validDuration).Unix() // 生成随机数（建议使用UUID，不能包含中划线（-）） rand := strings.ReplaceAll(uuid.New().String(), "-", "") // 解析URL objURL, err := url.Parse(originURL) if err != nil { return "", err } // 待签名字符串，格式：path-timestamp-rand-uid-privateKey unsignedStr := fmt.Sprintf("%s-%d-%s-%d-%s", objURL.Path, ts, rand, uid, privateKey) md5Hash := md5.Sum([]byte(unsignedStr)) // 生成鉴权参数，格式：timestamp-rand-uid-md5hash authKey := fmt.Sprintf("%d-%s-%d-%x", ts, rand, uid, md5Hash) // 添加鉴权参数到URL查询参数 v := objURL.Query() v.Add("auth_key", authKey) objURL.RawQuery = v.Encode() return objURL.String(), nil } func (d *Open123) getUserInfo(ctx context.Context) (*UserInfoResp, error) { var resp UserInfoResp if _, err := d.Request(UserInfo, http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }, &resp); err != nil { return nil, err } return &resp, nil } func (d *Open123) getUID(ctx context.Context) (uint64, error) { if d.UID != 0 { return d.UID, nil } resp, err := d.getUserInfo(ctx) if err != nil { return 0, err } d.UID = resp.Data.UID return resp.Data.UID, nil } func (d *Open123) getFiles(parentFileId int64, limit int, lastFileId int64) (*FileListResp, error) { var resp FileListResp _, err := d.Request(FileList, http.MethodGet, func(req *resty.Request) { req.SetQueryParams( map[string]string{ "parentFileId": strconv.FormatInt(parentFileId, 10), "limit": strconv.Itoa(limit), "lastFileId": strconv.FormatInt(lastFileId, 10), "trashed": "false", "searchMode": "", "searchData": "", }) }, &resp) if err != nil { return nil, err } return &resp, nil } func (d *Open123) getDownloadInfo(fileId int64) (*DownloadInfoResp, error) { var resp DownloadInfoResp _, err := d.Request(DownloadInfo, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "fileId": strconv.FormatInt(fileId, 10), }) }, &resp) if err != nil { return nil, err } return &resp, nil } func (d *Open123) getDirectLink(fileId int64) (*DirectLinkResp, error) { var resp DirectLinkResp _, err := d.Request(DirectLink, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "fileID": strconv.FormatInt(fileId, 10), }) }, &resp) if err != nil { return nil, err } return &resp, nil } func (d *Open123) mkdir(parentID int64, name string) error { _, err := d.Request(Mkdir, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "parentID": strconv.FormatInt(parentID, 10), "name": name, }) }, nil) if err != nil { return err } return nil } func (d *Open123) move(fileID, toParentFileID int64) error { _, err := d.Request(Move, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "fileIDs": []int64{fileID}, "toParentFileID": toParentFileID, }) }, nil) if err != nil { return err } return nil } func (d *Open123) rename(fileId int64, fileName string) error { _, err := d.Request(Rename, http.MethodPut, func(req *resty.Request) { req.SetBody(base.Json{ "fileId": fileId, "fileName": fileName, }) }, nil) if err != nil { return err } return nil } func (d *Open123) trash(fileId int64) error { _, err := d.Request(Trash, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "fileIDs": []int64{fileId}, }) }, nil) if err != nil { return err } return nil } func (d *Open123) createOfflineDownloadTask(ctx context.Context, url string, dirID, callback string) (taskID int, err error) { body := base.Json{ "url": url, "dirID": dirID, } if len(callback) > 0 { body["callBackUrl"] = callback } var resp OfflineDownloadResp _, err = d.Request(OfflineDownload, http.MethodPost, func(req *resty.Request) { req.SetBody(body) }, &resp) if err != nil { return 0, err } return resp.Data.TaskID, nil } func (d *Open123) queryOfflineDownloadStatus(ctx context.Context, taskID int) (process float64, status int, err error) { var resp OfflineDownloadProcessResp _, err = d.Request(OfflineDownloadProcess, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "taskID": strconv.Itoa(taskID), }) }, &resp) if err != nil { return .0, 0, err } return resp.Data.Process, resp.Data.Status, nil } ================================================ FILE: drivers/123_share/driver.go ================================================ package _123Share import ( "context" "encoding/base64" "fmt" "net/http" "net/url" "sync" "time" "golang.org/x/time/rate" _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) type Pan123Share struct { model.Storage Addition apiRateLimit sync.Map ref *_123.Pan123 } func (d *Pan123Share) Config() driver.Config { return config } func (d *Pan123Share) GetAddition() driver.Additional { return &d.Addition } func (d *Pan123Share) Init(ctx context.Context) error { // TODO login / refresh token //op.MustSaveDriverStorage(d) return nil } func (d *Pan123Share) InitReference(storage driver.Driver) error { refStorage, ok := storage.(*_123.Pan123) if ok { d.ref = refStorage return nil } return fmt.Errorf("ref: storage is not 123Pan") } func (d *Pan123Share) Drop(ctx context.Context) error { d.ref = nil return nil } func (d *Pan123Share) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { // TODO return the files list, required files, err := d.getFiles(ctx, dir.GetID()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return src, nil }) } func (d *Pan123Share) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { // TODO return link of file, required if f, ok := file.(File); ok { data := base.Json{ "shareKey": d.ShareKey, "SharePwd": d.SharePwd, "etag": f.Etag, "fileId": f.FileId, "s3keyFlag": f.S3KeyFlag, "size": f.Size, } resp, err := d.request(DownloadInfo, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) if err != nil { return nil, err } downloadUrl := utils.Json.Get(resp, "data", "DownloadURL").ToString() ou, err := url.Parse(downloadUrl) if err != nil { return nil, err } u_ := ou.String() nu := ou.Query().Get("params") if nu != "" { du, _ := base64.StdEncoding.DecodeString(nu) u, err := url.Parse(string(du)) if err != nil { return nil, err } u_ = u.String() } log.Debug("download url: ", u_) res, err := base.NoRedirectClient.R().SetHeader("Referer", "https://www.123pan.com/").Get(u_) if err != nil { return nil, err } log.Debug(res.String()) link := model.Link{ URL: u_, } log.Debugln("res code: ", res.StatusCode()) if res.StatusCode() == 302 { link.URL = res.Header().Get("location") } else if res.StatusCode() < 300 { link.URL = utils.Json.Get(res.Body(), "data", "redirect_url").ToString() } link.Header = http.Header{ "Referer": []string{fmt.Sprintf("%s://%s/", ou.Scheme, ou.Host)}, } return &link, nil } return nil, fmt.Errorf("can't convert obj") } func (d *Pan123Share) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { // TODO create folder, optional return errs.NotSupport } func (d *Pan123Share) Move(ctx context.Context, srcObj, dstDir model.Obj) error { // TODO move obj, optional return errs.NotSupport } func (d *Pan123Share) Rename(ctx context.Context, srcObj model.Obj, newName string) error { // TODO rename obj, optional return errs.NotSupport } func (d *Pan123Share) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { // TODO copy obj, optional return errs.NotSupport } func (d *Pan123Share) Remove(ctx context.Context, obj model.Obj) error { // TODO remove obj, optional return errs.NotSupport } func (d *Pan123Share) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { // TODO upload file, optional return errs.NotSupport } //func (d *Pan123Share) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} func (d *Pan123Share) APIRateLimit(ctx context.Context, api string) error { value, _ := d.apiRateLimit.LoadOrStore(api, rate.NewLimiter(rate.Every(700*time.Millisecond), 1)) limiter := value.(*rate.Limiter) return limiter.Wait(ctx) } var _ driver.Driver = (*Pan123Share)(nil) ================================================ FILE: drivers/123_share/meta.go ================================================ package _123Share import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { ShareKey string `json:"sharekey" required:"true"` SharePwd string `json:"sharepassword"` driver.RootID //OrderBy string `json:"order_by" type:"select" options:"file_name,size,update_at" default:"file_name"` //OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` AccessToken string `json:"accesstoken" type:"text"` } var config = driver.Config{ Name: "123PanShare", LocalSort: true, NoUpload: true, DefaultRoot: "0", PreferProxy: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Pan123Share{} }) } ================================================ FILE: drivers/123_share/types.go ================================================ package _123Share import ( "net/url" "path" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type File struct { FileName string `json:"FileName"` Size int64 `json:"Size"` UpdateAt time.Time `json:"UpdateAt"` FileId int64 `json:"FileId"` Type int `json:"Type"` Etag string `json:"Etag"` S3KeyFlag string `json:"S3KeyFlag"` DownloadUrl string `json:"DownloadUrl"` } func (f File) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.MD5, f.Etag) } func (f File) GetPath() string { return "" } func (f File) GetSize() int64 { return f.Size } func (f File) GetName() string { return f.FileName } func (f File) ModTime() time.Time { return f.UpdateAt } func (f File) CreateTime() time.Time { return f.UpdateAt } func (f File) IsDir() bool { return f.Type == 1 } func (f File) GetID() string { return strconv.FormatInt(f.FileId, 10) } func (f File) Thumb() string { if f.DownloadUrl == "" { return "" } du, err := url.Parse(f.DownloadUrl) if err != nil { return "" } du.Path = strings.TrimSuffix(du.Path, "_24_24") + "_70_70" query := du.Query() query.Set("w", "70") query.Set("h", "70") if !query.Has("type") { query.Set("type", strings.TrimPrefix(path.Base(f.FileName), ".")) } if !query.Has("trade_key") { query.Set("trade_key", "123pan-thumbnail") } du.RawQuery = query.Encode() return du.String() } var _ model.Obj = (*File)(nil) var _ model.Thumb = (*File)(nil) //func (f File) Thumb() string { // //} //var _ model.Thumb = (*File)(nil) type Files struct { //BaseResp Data struct { InfoList []File `json:"InfoList"` Next string `json:"Next"` } `json:"data"` } //type DownResp struct { // //BaseResp // Data struct { // DownloadUrl string `json:"DownloadUrl"` // } `json:"data"` //} ================================================ FILE: drivers/123_share/util.go ================================================ package _123Share import ( "context" "errors" "fmt" "hash/crc32" "math" "math/rand" "net/http" "net/url" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" ) const ( Api = "https://www.123pan.com/api" AApi = "https://www.123pan.com/a/api" BApi = "https://www.123pan.com/b/api" MainApi = BApi FileList = MainApi + "/share/get" DownloadInfo = MainApi + "/share/download/info" //AuthKeySalt = "8-8D$sL8gPjom7bk#cY" ) func signPath(path string, os string, version string) (k string, v string) { table := []byte{'a', 'd', 'e', 'f', 'g', 'h', 'l', 'm', 'y', 'i', 'j', 'n', 'o', 'p', 'k', 'q', 'r', 's', 't', 'u', 'b', 'c', 'v', 'w', 's', 'z'} random := fmt.Sprintf("%.f", math.Round(1e7*rand.Float64())) now := time.Now().In(time.FixedZone("CST", 8*3600)) timestamp := fmt.Sprint(now.Unix()) nowStr := []byte(now.Format("200601021504")) for i := 0; i < len(nowStr); i++ { nowStr[i] = table[nowStr[i]-48] } timeSign := fmt.Sprint(crc32.ChecksumIEEE(nowStr)) data := strings.Join([]string{timestamp, random, path, os, version, timeSign}, "|") dataSign := fmt.Sprint(crc32.ChecksumIEEE([]byte(data))) return timeSign, strings.Join([]string{timestamp, random, dataSign}, "-") } func GetApi(rawUrl string) string { u, _ := url.Parse(rawUrl) query := u.Query() query.Add(signPath(u.Path, "web", "3")) u.RawQuery = query.Encode() return u.String() } func (d *Pan123Share) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { if d.ref != nil { return d.ref.Request(url, method, callback, resp) } req := base.RestyClient.R() req.SetHeaders(map[string]string{ "origin": "https://www.123pan.com", "referer": "https://www.123pan.com/", "authorization": "Bearer " + d.AccessToken, "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) openlist-client", "platform": "web", "app-version": "3", //"user-agent": base.UserAgent, }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } res, err := req.Execute(method, GetApi(url)) if err != nil { return nil, err } body := res.Body() code := utils.Json.Get(body, "code").ToInt() if code != 0 { return nil, errors.New(jsoniter.Get(body, "message").ToString()) } return body, nil } func (d *Pan123Share) getFiles(ctx context.Context, parentId string) ([]File, error) { page := 1 res := make([]File, 0) for { if err := d.APIRateLimit(ctx, FileList); err != nil { return nil, err } var resp Files query := map[string]string{ "limit": "100", "next": "0", "orderBy": "file_id", "orderDirection": "desc", "parentFileId": parentId, "Page": strconv.Itoa(page), "shareKey": d.ShareKey, "SharePwd": d.SharePwd, } _, err := d.request(FileList, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return nil, err } page++ res = append(res, resp.Data.InfoList...) if len(resp.Data.InfoList) == 0 || resp.Data.Next == "-1" { break } } return res, nil } // do others that not defined in Driver interface ================================================ FILE: drivers/139/driver.go ================================================ package _139 import ( "context" "encoding/xml" "fmt" "io" "net/http" "path" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" log "github.com/sirupsen/logrus" ) type Yun139 struct { model.Storage Addition cron *cron.Cron Account string ref *Yun139 PersonalCloudHost string RootPath string } func (d *Yun139) Config() driver.Config { return config } func (d *Yun139) GetAddition() driver.Additional { return &d.Addition } func (d *Yun139) Init(ctx context.Context) error { if d.ref == nil { if len(d.Authorization) == 0 { if d.Username != "" && d.Password != "" { log.Infof("139yun: authorization is empty, trying to login with password.") newAuth, err := d.loginWithPassword() log.Debugf("newAuth: Ok: %s", newAuth) if err != nil { return fmt.Errorf("login with password failed: %w", err) } } else { return fmt.Errorf("authorization is empty and username/password is not provided") } } err := d.refreshToken() if err != nil { return err } // Query Route Policy var resp QueryRoutePolicyResp _, err = d.requestRoute(base.Json{ "userInfo": base.Json{ "userType": 1, "accountType": 1, "accountName": d.Account, }, "modAddrType": 1, }, &resp) if err != nil { return err } for _, policyItem := range resp.Data.RoutePolicyList { if policyItem.ModName == "personal" { d.PersonalCloudHost = policyItem.HttpsUrl break } } if len(d.PersonalCloudHost) == 0 { return fmt.Errorf("PersonalCloudHost is empty") } d.cron = cron.NewCron(time.Hour * 12) d.cron.Do(func() { err := d.refreshToken() if err != nil { log.Errorf("%+v", err) } }) } switch d.Addition.Type { case MetaPersonalNew: if len(d.Addition.RootFolderID) == 0 { d.RootFolderID = "/" } case MetaPersonal: if len(d.Addition.RootFolderID) == 0 { d.RootFolderID = "root" } case MetaGroup: if len(d.Addition.RootFolderID) == 0 { d.RootFolderID = d.CloudID } _, err := d.groupGetFiles(d.RootFolderID) if err != nil { return err } case MetaFamily: if len(d.Addition.RootFolderID) == 0 { // Attempt to obtain data.path as the root via a query and persist it. if root, err := d.getFamilyRootPath(d.CloudID); err == nil && root != "" { d.RootFolderID = root op.MustSaveDriverStorage(d) } } _, err := d.familyGetFiles(d.RootFolderID) if err != nil { return err } default: return errs.NotImplement } return nil } func (d *Yun139) InitReference(storage driver.Driver) error { refStorage, ok := storage.(*Yun139) if ok { d.ref = refStorage return nil } return errs.NotSupport } func (d *Yun139) Drop(ctx context.Context) error { if d.cron != nil { d.cron.Stop() } d.ref = nil return nil } func (d *Yun139) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { switch d.Addition.Type { case MetaPersonalNew: return d.personalGetFiles(dir.GetID()) case MetaPersonal: return d.getFiles(dir.GetID()) case MetaFamily: return d.familyGetFiles(dir.GetID()) case MetaGroup: return d.groupGetFiles(dir.GetID()) default: return nil, errs.NotImplement } } func (d *Yun139) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var url string var err error switch d.Addition.Type { case MetaPersonalNew: url, err = d.personalGetLink(file.GetID()) case MetaPersonal: url, err = d.getLink(file.GetID()) case MetaFamily: url, err = d.familyGetLink(file.GetID(), file.GetPath()) case MetaGroup: url, err = d.groupGetLink(file.GetID(), file.GetPath()) default: return nil, errs.NotImplement } if err != nil { return nil, err } return &model.Link{URL: url}, nil } func (d *Yun139) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { var err error switch d.Addition.Type { case MetaPersonalNew: data := base.Json{ "parentFileId": parentDir.GetID(), "name": dirName, "description": "", "type": "folder", "fileRenameMode": "force_rename", } pathname := "/file/create" _, err = d.personalPost(pathname, data, nil) case MetaPersonal: data := base.Json{ "createCatalogExtReq": base.Json{ "parentCatalogID": parentDir.GetID(), "newCatalogName": dirName, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, }, } pathname := "/orchestration/personalCloud/catalog/v1.0/createCatalogExt" _, err = d.post(pathname, data, nil) case MetaFamily: data := base.Json{ "cloudID": d.CloudID, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, "docLibName": dirName, "path": path.Join(parentDir.GetPath(), parentDir.GetID()), } pathname := "/orchestration/familyCloud-rebuild/cloudCatalog/v1.0/createCloudDoc" _, err = d.post(pathname, data, nil) case MetaGroup: data := base.Json{ "catalogName": dirName, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, "groupID": d.CloudID, "parentFileId": parentDir.GetID(), "path": path.Join(parentDir.GetPath(), parentDir.GetID()), } pathname := "/orchestration/group-rebuild/catalog/v1.0/createGroupCatalog" _, err = d.post(pathname, data, nil) default: err = errs.NotImplement } return err } func (d *Yun139) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { switch d.Addition.Type { case MetaPersonalNew: data := base.Json{ "fileIds": []string{srcObj.GetID()}, "toParentFileId": dstDir.GetID(), } pathname := "/file/batchMove" _, err := d.personalPost(pathname, data, nil) if err != nil { return nil, err } return srcObj, nil case MetaGroup: var contentList []string var catalogList []string if srcObj.IsDir() { catalogList = append(catalogList, srcObj.GetID()) } else { contentList = append(contentList, srcObj.GetID()) } data := base.Json{ "taskType": 3, "srcType": 2, "srcGroupID": d.CloudID, "destType": 2, "destGroupID": d.CloudID, "destPath": dstDir.GetPath(), "contentList": contentList, "catalogList": catalogList, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, } pathname := "/orchestration/group-rebuild/task/v1.0/createBatchOprTask" _, err := d.post(pathname, data, nil) if err != nil { return nil, err } return srcObj, nil case MetaPersonal: var contentInfoList []string var catalogInfoList []string if srcObj.IsDir() { catalogInfoList = append(catalogInfoList, srcObj.GetID()) } else { contentInfoList = append(contentInfoList, srcObj.GetID()) } data := base.Json{ "createBatchOprTaskReq": base.Json{ "taskType": 3, "actionType": "304", "taskInfo": base.Json{ "contentInfoList": contentInfoList, "catalogInfoList": catalogInfoList, "newCatalogID": dstDir.GetID(), }, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, }, } pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" _, err := d.post(pathname, data, nil) if err != nil { return nil, err } return srcObj, nil case MetaFamily: pathname := "/isbo/openApi/createBatchOprTask" var contentList []string var catalogList []string if srcObj.IsDir() { catalogList = append(catalogList, path.Join(srcObj.GetPath(), srcObj.GetID())) } else { contentList = append(contentList, path.Join(srcObj.GetPath(), srcObj.GetID())) } body := base.Json{ "catalogList": catalogList, "accountInfo": base.Json{ "accountName": d.getAccount(), "accountType": "1", }, "contentList": contentList, "destCatalogID": dstDir.GetID(), "destGroupID": d.CloudID, "destPath": path.Join(dstDir.GetPath(), dstDir.GetID()), "destType": 0, "srcGroupID": d.CloudID, "srcType": 0, "taskType": 3, } var resp CreateBatchOprTaskResp _, err := d.isboPost(pathname, body, &resp) if err != nil { return nil, err } log.Debugf("[139] Move MetaFamily CreateBatchOprTaskResp.Result.ResultCode: %s", resp.Result.ResultCode) if resp.Result.ResultCode != "0" { return nil, fmt.Errorf("failed to move in family cloud: %s", resp.Result.ResultDesc) } return srcObj, nil default: return nil, errs.NotImplement } } func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) error { var err error switch d.Addition.Type { case MetaPersonalNew: data := base.Json{ "fileId": srcObj.GetID(), "name": newName, "description": "", } pathname := "/file/update" _, err = d.personalPost(pathname, data, nil) case MetaPersonal: var data base.Json var pathname string if srcObj.IsDir() { data = base.Json{ "catalogID": srcObj.GetID(), "catalogName": newName, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, } pathname = "/orchestration/personalCloud/catalog/v1.0/updateCatalogInfo" } else { data = base.Json{ "contentID": srcObj.GetID(), "contentName": newName, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, } pathname = "/orchestration/personalCloud/content/v1.0/updateContentInfo" } _, err = d.post(pathname, data, nil) case MetaGroup: var data base.Json var pathname string if srcObj.IsDir() { data = base.Json{ "groupID": d.CloudID, "modifyCatalogID": srcObj.GetID(), "modifyCatalogName": newName, "path": srcObj.GetPath(), "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, } pathname = "/orchestration/group-rebuild/catalog/v1.0/modifyGroupCatalog" } else { data = base.Json{ "groupID": d.CloudID, "contentID": srcObj.GetID(), "contentName": newName, "path": srcObj.GetPath(), "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, } pathname = "/orchestration/group-rebuild/content/v1.0/modifyGroupContent" } _, err = d.post(pathname, data, nil) case MetaFamily: var data base.Json var pathname string if srcObj.IsDir() { pathname = "/modifyCloudDocV2" data = base.Json{ "catalogType": 3, "cloudID": d.CloudID, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": "1", }, "docLibName": newName, "docLibraryID": srcObj.GetID(), "path": path.Join(srcObj.GetPath(), srcObj.GetID()), } var resp ModifyCloudDocV2Resp _, err = d.andAlbumRequest(pathname, data, &resp) if err != nil { return err } if resp.Result.ResultCode != "0" { return fmt.Errorf("failed to rename family folder: %s", resp.Result.ResultDesc) } return nil } else { data = base.Json{ "contentID": srcObj.GetID(), "contentName": newName, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, "path": srcObj.GetPath(), } pathname = "/orchestration/familyCloud-rebuild/photoContent/v1.0/modifyContentInfo" } _, err = d.post(pathname, data, nil) default: err = errs.NotImplement } return err } func (d *Yun139) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { var err error switch d.Addition.Type { case MetaPersonalNew: data := base.Json{ "fileIds": []string{srcObj.GetID()}, "toParentFileId": dstDir.GetID(), } pathname := "/file/batchCopy" _, err := d.personalPost(pathname, data, nil) return err case MetaPersonal: var contentInfoList []string var catalogInfoList []string if srcObj.IsDir() { catalogInfoList = append(catalogInfoList, srcObj.GetID()) } else { contentInfoList = append(contentInfoList, srcObj.GetID()) } data := base.Json{ "createBatchOprTaskReq": base.Json{ "taskType": 3, "actionType": 309, "taskInfo": base.Json{ "contentInfoList": contentInfoList, "catalogInfoList": catalogInfoList, "newCatalogID": dstDir.GetID(), }, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, }, } pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" _, err = d.post(pathname, data, nil) case MetaGroup: err = d.handleMetaGroupCopy(ctx, srcObj, dstDir) case MetaFamily: pathname := "/copyContentCatalog" var sourceContentIDs []string var sourceCatalogIDs []string if srcObj.IsDir() { sourceCatalogIDs = append(sourceCatalogIDs, srcObj.GetID()) } else { sourceContentIDs = append(sourceContentIDs, srcObj.GetID()) } body := base.Json{ "commonAccountInfo": base.Json{ "accountType": "1", "accountUserId": d.ref.UserDomainID, }, "destCatalogID": dstDir.GetID(), "destCloudID": d.CloudID, "sourceCatalogIDs": sourceCatalogIDs, "sourceCloudID": d.CloudID, "sourceContentIDs": sourceContentIDs, } var resp base.Json // Assuming a generic JSON response for success/failure _, err = d.andAlbumRequest(pathname, body, &resp) // For now, we assume no error means success. default: err = errs.NotImplement } return err } func (d *Yun139) Remove(ctx context.Context, obj model.Obj) error { switch d.Addition.Type { case MetaPersonalNew: data := base.Json{ "fileIds": []string{obj.GetID()}, } pathname := "/recyclebin/batchTrash" _, err := d.personalPost(pathname, data, nil) return err case MetaGroup: var contentList []string var catalogList []string // 必须使用完整路径删除 if obj.IsDir() { catalogList = append(catalogList, obj.GetPath()) } else { contentList = append(contentList, path.Join(obj.GetPath(), obj.GetID())) } data := base.Json{ "taskType": 2, "srcGroupID": d.CloudID, "contentList": contentList, "catalogList": catalogList, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, } pathname := "/orchestration/group-rebuild/task/v1.0/createBatchOprTask" _, err := d.post(pathname, data, nil) return err case MetaPersonal: fallthrough case MetaFamily: var contentInfoList []string var catalogInfoList []string if obj.IsDir() { catalogInfoList = append(catalogInfoList, obj.GetID()) } else { contentInfoList = append(contentInfoList, obj.GetID()) } data := base.Json{ "createBatchOprTaskReq": base.Json{ "taskType": 2, "actionType": 201, "taskInfo": base.Json{ "newCatalogID": "", "contentInfoList": contentInfoList, "catalogInfoList": catalogInfoList, }, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, }, } pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" if d.isFamily() { data = base.Json{ "catalogList": catalogInfoList, "contentList": contentInfoList, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, "sourceCloudID": d.CloudID, "sourceCatalogType": 1002, "taskType": 2, "path": obj.GetPath(), } pathname = "/orchestration/familyCloud-rebuild/batchOprTask/v1.0/createBatchOprTask" } _, err := d.post(pathname, data, nil) return err default: return errs.NotImplement } } func (d *Yun139) getPartSize(size int64) int64 { if d.CustomUploadPartSize != 0 { return d.CustomUploadPartSize } // 网盘对于分片数量存在上限 if size/utils.GB > 30 { return 512 * utils.MB } return 100 * utils.MB } func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { switch d.Addition.Type { case MetaPersonalNew: var err error fullHash := stream.GetHash().GetHash(utils.SHA256) if len(fullHash) != utils.SHA256.Width { _, fullHash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA256) if err != nil { return err } } size := stream.GetSize() partSize := d.getPartSize(size) part := int64(1) if size > partSize { part = (size + partSize - 1) / partSize } // 生成所有 partInfos partInfos := make([]PartInfo, 0, part) for i := int64(0); i < part; i++ { if utils.IsCanceled(ctx) { return ctx.Err() } start := i * partSize byteSize := min(size-start, partSize) partNumber := i + 1 partInfo := PartInfo{ PartNumber: partNumber, PartSize: byteSize, ParallelHashCtx: ParallelHashCtx{ PartOffset: start, }, } partInfos = append(partInfos, partInfo) } // 筛选出前 100 个 partInfos firstPartInfos := partInfos if len(firstPartInfos) > 100 { firstPartInfos = firstPartInfos[:100] } // 创建任务，获取上传信息和前100个分片的上传地址 data := base.Json{ "contentHash": fullHash, "contentHashAlgorithm": "SHA256", "contentType": "application/octet-stream", "parallelUpload": false, "partInfos": firstPartInfos, "size": size, "parentFileId": dstDir.GetID(), "name": stream.GetName(), "type": "file", "fileRenameMode": "auto_rename", } pathname := "/file/create" var resp PersonalUploadResp _, err = d.personalPost(pathname, data, &resp) if err != nil { return err } // 判断文件是否已存在 // resp.Data.Exist: true 已存在同名文件且校验相同，云端不会重复增加文件，无需手动处理冲突 if resp.Data.Exist { return nil } // 判断文件是否支持快传 // resp.Data.RapidUpload: true 支持快传，但此处直接检测是否返回分片的上传地址 // 快传的情况下同样需要手动处理冲突 if resp.Data.PartInfos != nil { // Progress p := driver.NewProgress(size, up) rateLimited := driver.NewLimitedUploadStream(ctx, stream) // 先上传前100个分片 err = d.uploadPersonalParts(ctx, partInfos, resp.Data.PartInfos, rateLimited, p) if err != nil { return err } // 如果还有剩余分片，分批获取上传地址并上传 for i := 100; i < len(partInfos); i += 100 { end := min(i+100, len(partInfos)) batchPartInfos := partInfos[i:end] moredata := base.Json{ "fileId": resp.Data.FileId, "uploadId": resp.Data.UploadId, "partInfos": batchPartInfos, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, } pathname := "/file/getUploadUrl" var moreresp PersonalUploadUrlResp _, err = d.personalPost(pathname, moredata, &moreresp) if err != nil { return err } err = d.uploadPersonalParts(ctx, partInfos, moreresp.Data.PartInfos, rateLimited, p) if err != nil { return err } } // 全部分片上传完毕后，complete data = base.Json{ "contentHash": fullHash, "contentHashAlgorithm": "SHA256", "fileId": resp.Data.FileId, "uploadId": resp.Data.UploadId, } _, err = d.personalPost("/file/complete", data, nil) if err != nil { return err } } // 处理冲突 if resp.Data.FileName != stream.GetName() { log.Debugf("[139] conflict detected: %s != %s", resp.Data.FileName, stream.GetName()) // 给服务器一定时间处理数据，避免无法刷新文件列表 time.Sleep(time.Millisecond * 500) // 刷新并获取文件列表 files, err := d.List(ctx, dstDir, model.ListArgs{Refresh: true}) if err != nil { return err } // 删除旧文件 for _, file := range files { if file.GetName() == stream.GetName() { log.Debugf("[139] conflict: removing old: %s", file.GetName()) // 删除前重命名旧文件，避免仍旧冲突 err = d.Rename(ctx, file, stream.GetName()+random.String(4)) if err != nil { return err } err = d.Remove(ctx, file) if err != nil { return err } break } } // 重命名新文件 for _, file := range files { if file.GetName() == resp.Data.FileName { log.Debugf("[139] conflict: renaming new: %s => %s", file.GetName(), stream.GetName()) err = d.Rename(ctx, file, stream.GetName()) if err != nil { return err } break } } } return nil case MetaPersonal: fallthrough case MetaGroup: fallthrough case MetaFamily: // 处理冲突 // 获取文件列表 files, err := d.List(ctx, dstDir, model.ListArgs{}) if err != nil { return err } // 删除旧文件 for _, file := range files { if file.GetName() == stream.GetName() { log.Debugf("[139] conflict: removing old: %s", file.GetName()) // 删除前重命名旧文件，避免仍旧冲突 err = d.Rename(ctx, file, stream.GetName()+random.String(4)) if err != nil { return err } err = d.Remove(ctx, file) if err != nil { return err } break } } var reportSize int64 if d.ReportRealSize { reportSize = stream.GetSize() } else { reportSize = 0 } data := base.Json{ "manualRename": 2, "operation": 0, "fileCount": 1, "totalSize": reportSize, "uploadContentList": []base.Json{{ "contentName": stream.GetName(), "contentSize": reportSize, // "digest": "5a3231986ce7a6b46e408612d385bafa" }}, "parentCatalogID": dstDir.GetID(), "newCatalogName": "", "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, } pathname := "/orchestration/personalCloud/uploadAndDownload/v1.0/pcUploadFileRequest" if d.isFamily() || d.Addition.Type == MetaGroup { uploadPath := path.Join(dstDir.GetPath(), dstDir.GetID()) // if dstDir is root folder if dstDir.GetID() == d.RootFolderID { uploadPath = d.RootPath } data = d.newJson(base.Json{ "fileCount": 1, "manualRename": 2, "operation": 0, "path": uploadPath, "seqNo": random.String(32), // 序列号不能为空 "totalSize": reportSize, "uploadContentList": []base.Json{{ "contentName": stream.GetName(), "contentSize": reportSize, // "digest": "5a3231986ce7a6b46e408612d385bafa" }}, }) pathname = "/orchestration/familyCloud-rebuild/content/v1.0/getFileUploadURL" } var resp UploadResp log.Debugf("[139] upload request body: %+v", data) _, err = d.post(pathname, data, &resp) if err != nil { return err } if resp.Data.Result.ResultCode != "0" { return fmt.Errorf("get file upload url failed with result code: %s, message: %s", resp.Data.Result.ResultCode, resp.Data.Result.ResultDesc) } size := stream.GetSize() // Progress p := driver.NewProgress(size, up) partSize := d.getPartSize(size) part := int64(1) if size > partSize { part = (size + partSize - 1) / partSize } rateLimited := driver.NewLimitedUploadStream(ctx, stream) for i := int64(0); i < part; i++ { if utils.IsCanceled(ctx) { return ctx.Err() } start := i * partSize byteSize := min(size-start, partSize) limitReader := io.LimitReader(rateLimited, byteSize) // Update Progress r := io.TeeReader(limitReader, p) req, err := http.NewRequestWithContext(ctx, http.MethodPost, resp.Data.UploadResult.RedirectionURL, r) if err != nil { return err } req.Header.Set("Content-Type", "text/plain;name="+unicode(stream.GetName())) req.Header.Set("contentSize", strconv.FormatInt(size, 10)) req.Header.Set("range", fmt.Sprintf("bytes=%d-%d", start, start+byteSize-1)) req.Header.Set("uploadtaskID", resp.Data.UploadResult.UploadTaskID) req.Header.Set("rangeType", "0") req.ContentLength = byteSize res, err := base.HttpClient.Do(req) if err != nil { return err } if res.StatusCode != http.StatusOK { res.Body.Close() return fmt.Errorf("unexpected status code: %d", res.StatusCode) } bodyBytes, err := io.ReadAll(res.Body) if err != nil { return fmt.Errorf("error reading response body: %v", err) } var result InterLayerUploadResult err = xml.Unmarshal(bodyBytes, &result) if err != nil { return fmt.Errorf("error parsing XML: %v", err) } if result.ResultCode != 0 { return fmt.Errorf("upload failed with result code: %d, message: %s", result.ResultCode, result.Msg) } } return nil default: return errs.NotImplement } } func (d *Yun139) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { switch d.Addition.Type { case MetaPersonalNew: var resp base.Json var uri string data := base.Json{ "category": "video", "fileId": args.Obj.GetID(), } switch args.Method { case "video_preview": uri = "/videoPreview/getPreviewInfo" default: return nil, errs.NotSupport } _, err := d.personalPost(uri, data, &resp) if err != nil { return nil, err } return resp["data"], nil default: return nil, errs.NotImplement } } func (d *Yun139) GetDetails(ctx context.Context) (*model.StorageDetails, error) { if d.UserDomainID == "" { return nil, errs.NotImplement } var total, used int64 if d.isFamily() { diskInfo, err := d.getFamilyDiskInfo(ctx) if err != nil { return nil, err } totalMb, err := strconv.ParseInt(diskInfo.Data.DiskSize, 10, 64) if err != nil { return nil, fmt.Errorf("failed convert disk size into integer: %+v", err) } usedMb, err := strconv.ParseInt(diskInfo.Data.UsedSize, 10, 64) if err != nil { return nil, fmt.Errorf("failed convert used size into integer: %+v", err) } total = totalMb * 1024 * 1024 used = usedMb * 1024 * 1024 } else { diskInfo, err := d.getPersonalDiskInfo(ctx) if err != nil { return nil, err } totalMb, err := strconv.ParseInt(diskInfo.Data.DiskSize, 10, 64) if err != nil { return nil, fmt.Errorf("failed convert disk size into integer: %+v", err) } freeMb, err := strconv.ParseInt(diskInfo.Data.FreeDiskSize, 10, 64) if err != nil { return nil, fmt.Errorf("failed convert free size into integer: %+v", err) } total = totalMb * 1024 * 1024 used = total - (freeMb * 1024 * 1024) } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: used, }, }, nil } var _ driver.Driver = (*Yun139)(nil) ================================================ FILE: drivers/139/meta.go ================================================ package _139 import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { //Account string `json:"account" required:"true"` Authorization string `json:"authorization" type:"text" required:"true"` Username string `json:"username" required:"true"` Password string `json:"password" required:"true" secret:"true"` MailCookies string `json:"mail_cookies" required:"true" type:"text" help:"Cookies from mail.139.com used for login authentication."` driver.RootID Type string `json:"type" type:"select" options:"personal_new,family,group,personal" default:"personal_new"` CloudID string `json:"cloud_id"` UserDomainID string `json:"user_domain_id" help:"ud_id in Cookie, fill in to show disk usage"` CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` ReportRealSize bool `json:"report_real_size" type:"bool" default:"true" help:"Enable to report the real file size during upload"` UseLargeThumbnail bool `json:"use_large_thumbnail" type:"bool" default:"false" help:"Enable to use large thumbnail for images"` } var config = driver.Config{ Name: "139Yun", LocalSort: true, ProxyRangeOption: true, } func init() { op.RegisterDriver(func() driver.Driver { d := &Yun139{} d.ProxyRange = true return d }) } ================================================ FILE: drivers/139/types.go ================================================ package _139 import ( "encoding/xml" ) const ( MetaPersonal string = "personal" MetaFamily string = "family" MetaGroup string = "group" MetaPersonalNew string = "personal_new" ) type BaseResp struct { Success bool `json:"success"` Code string `json:"code"` Message string `json:"message"` } type Catalog struct { CatalogID string `json:"catalogID"` CatalogName string `json:"catalogName"` //CatalogType int `json:"catalogType"` CreateTime string `json:"createTime"` UpdateTime string `json:"updateTime"` //IsShared bool `json:"isShared"` //CatalogLevel int `json:"catalogLevel"` //ShareDoneeCount int `json:"shareDoneeCount"` //OpenType int `json:"openType"` //ParentCatalogID string `json:"parentCatalogId"` //DirEtag int `json:"dirEtag"` //Tombstoned int `json:"tombstoned"` //ProxyID interface{} `json:"proxyID"` //Moved int `json:"moved"` //IsFixedDir int `json:"isFixedDir"` //IsSynced interface{} `json:"isSynced"` //Owner string `json:"owner"` //Modifier interface{} `json:"modifier"` //Path string `json:"path"` //ShareType int `json:"shareType"` //SoftLink interface{} `json:"softLink"` //ExtProp1 interface{} `json:"extProp1"` //ExtProp2 interface{} `json:"extProp2"` //ExtProp3 interface{} `json:"extProp3"` //ExtProp4 interface{} `json:"extProp4"` //ExtProp5 interface{} `json:"extProp5"` //ETagOprType int `json:"ETagOprType"` } type Content struct { ContentID string `json:"contentID"` ContentName string `json:"contentName"` //ContentSuffix string `json:"contentSuffix"` ContentSize int64 `json:"contentSize"` //ContentDesc string `json:"contentDesc"` //ContentType int `json:"contentType"` //ContentOrigin int `json:"contentOrigin"` CreateTime string `json:"createTime"` UpdateTime string `json:"updateTime"` //CommentCount int `json:"commentCount"` ThumbnailURL string `json:"thumbnailURL"` //BigthumbnailURL string `json:"bigthumbnailURL"` //PresentURL string `json:"presentURL"` //PresentLURL string `json:"presentLURL"` //PresentHURL string `json:"presentHURL"` //ContentTAGList interface{} `json:"contentTAGList"` //ShareDoneeCount int `json:"shareDoneeCount"` //Safestate int `json:"safestate"` //Transferstate int `json:"transferstate"` //IsFocusContent int `json:"isFocusContent"` //UpdateShareTime interface{} `json:"updateShareTime"` //UploadTime string `json:"uploadTime"` //OpenType int `json:"openType"` //AuditResult int `json:"auditResult"` //ParentCatalogID string `json:"parentCatalogId"` //Channel string `json:"channel"` //GeoLocFlag string `json:"geoLocFlag"` Digest string `json:"digest"` //Version string `json:"version"` //FileEtag string `json:"fileEtag"` //FileVersion string `json:"fileVersion"` //Tombstoned int `json:"tombstoned"` //ProxyID string `json:"proxyID"` //Moved int `json:"moved"` //MidthumbnailURL string `json:"midthumbnailURL"` //Owner string `json:"owner"` //Modifier string `json:"modifier"` //ShareType int `json:"shareType"` //ExtInfo struct { // Uploader string `json:"uploader"` // Address string `json:"address"` //} `json:"extInfo"` //Exif struct { // CreateTime string `json:"createTime"` // Longitude interface{} `json:"longitude"` // Latitude interface{} `json:"latitude"` // LocalSaveTime interface{} `json:"localSaveTime"` //} `json:"exif"` //CollectionFlag interface{} `json:"collectionFlag"` //TreeInfo interface{} `json:"treeInfo"` //IsShared bool `json:"isShared"` //ETagOprType int `json:"ETagOprType"` } type GetDiskResp struct { BaseResp Data struct { Result struct { ResultCode string `json:"resultCode"` ResultDesc interface{} `json:"resultDesc"` } `json:"result"` GetDiskResult struct { ParentCatalogID string `json:"parentCatalogID"` NodeCount int `json:"nodeCount"` CatalogList []Catalog `json:"catalogList"` ContentList []Content `json:"contentList"` IsCompleted int `json:"isCompleted"` } `json:"getDiskResult"` } `json:"data"` } type UploadResp struct { BaseResp Data struct { Result struct { ResultCode string `json:"resultCode"` ResultDesc interface{} `json:"resultDesc"` } `json:"result"` UploadResult struct { UploadTaskID string `json:"uploadTaskID"` RedirectionURL string `json:"redirectionUrl"` NewContentIDList []struct { ContentID string `json:"contentID"` ContentName string `json:"contentName"` IsNeedUpload string `json:"isNeedUpload"` FileEtag int64 `json:"fileEtag"` FileVersion int64 `json:"fileVersion"` OverridenFlag int `json:"overridenFlag"` } `json:"newContentIDList"` CatalogIDList interface{} `json:"catalogIDList"` IsSlice interface{} `json:"isSlice"` } `json:"uploadResult"` } `json:"data"` } type InterLayerUploadResult struct { XMLName xml.Name `xml:"result"` Text string `xml:",chardata"` ResultCode int `xml:"resultCode"` Msg string `xml:"msg"` } type CloudContent struct { ContentID string `json:"contentID"` //Modifier string `json:"modifier"` //Nickname string `json:"nickname"` //CloudNickName string `json:"cloudNickName"` ContentName string `json:"contentName"` //ContentType int `json:"contentType"` //ContentSuffix string `json:"contentSuffix"` ContentSize int64 `json:"contentSize"` //ContentDesc string `json:"contentDesc"` CreateTime string `json:"createTime"` //Shottime interface{} `json:"shottime"` LastUpdateTime string `json:"lastUpdateTime"` ThumbnailURL string `json:"thumbnailURL"` //MidthumbnailURL string `json:"midthumbnailURL"` //BigthumbnailURL string `json:"bigthumbnailURL"` //PresentURL string `json:"presentURL"` //PresentLURL string `json:"presentLURL"` //PresentHURL string `json:"presentHURL"` //ParentCatalogID string `json:"parentCatalogID"` //Uploader string `json:"uploader"` //UploaderNickName string `json:"uploaderNickName"` //TreeInfo interface{} `json:"treeInfo"` //UpdateTime interface{} `json:"updateTime"` //ExtInfo struct { // Uploader string `json:"uploader"` //} `json:"extInfo"` //EtagOprType interface{} `json:"etagOprType"` } type CloudCatalog struct { CatalogID string `json:"catalogID"` CatalogName string `json:"catalogName"` //CloudID string `json:"cloudID"` CreateTime string `json:"createTime"` LastUpdateTime string `json:"lastUpdateTime"` //Creator string `json:"creator"` //CreatorNickname string `json:"creatorNickname"` } type QueryContentListResp struct { BaseResp Data struct { Result struct { ResultCode string `json:"resultCode"` ResultDesc string `json:"resultDesc"` } `json:"result"` Path string `json:"path"` CloudContentList []CloudContent `json:"cloudContentList"` CloudCatalogList []CloudCatalog `json:"cloudCatalogList"` TotalCount int `json:"totalCount"` RecallContent interface{} `json:"recallContent"` } `json:"data"` } type QueryGroupContentListResp struct { BaseResp Data struct { Result struct { ResultCode string `json:"resultCode"` ResultDesc string `json:"resultDesc"` } `json:"result"` GetGroupContentResult struct { ParentCatalogID string `json:"parentCatalogID"` // 根目录是"0" CatalogList []struct { Catalog Path string `json:"path"` } `json:"catalogList"` ContentList []Content `json:"contentList"` NodeCount int `json:"nodeCount"` // 文件+文件夹数量 CtlgCnt int `json:"ctlgCnt"` // 文件夹数量 ContCnt int `json:"contCnt"` // 文件数量 } `json:"getGroupContentResult"` } `json:"data"` } type ParallelHashCtx struct { PartOffset int64 `json:"partOffset"` } type PartInfo struct { PartNumber int64 `json:"partNumber"` PartSize int64 `json:"partSize"` ParallelHashCtx ParallelHashCtx `json:"parallelHashCtx"` } type PersonalThumbnail struct { Style string `json:"style"` Url string `json:"url"` } type PersonalFileItem struct { FileId string `json:"fileId"` Name string `json:"name"` Size int64 `json:"size"` Type string `json:"type"` CreatedAt string `json:"createdAt"` UpdatedAt string `json:"updatedAt"` Thumbnails []PersonalThumbnail `json:"thumbnailUrls"` } type PersonalListResp struct { BaseResp Data struct { Items []PersonalFileItem `json:"items"` NextPageCursor string `json:"nextPageCursor"` } } type PersonalPartInfo struct { PartNumber int `json:"partNumber"` UploadUrl string `json:"uploadUrl"` } type PersonalUploadResp struct { BaseResp Data struct { FileId string `json:"fileId"` FileName string `json:"fileName"` PartInfos []PersonalPartInfo `json:"partInfos"` Exist bool `json:"exist"` RapidUpload bool `json:"rapidUpload"` UploadId string `json:"uploadId"` } } type PersonalUploadUrlResp struct { BaseResp Data struct { FileId string `json:"fileId"` UploadId string `json:"uploadId"` PartInfos []PersonalPartInfo `json:"partInfos"` } } type QueryRoutePolicyResp struct { Success bool `json:"success"` Code string `json:"code"` Message string `json:"message"` Data struct { RoutePolicyList []struct { SiteID string `json:"siteID"` SiteCode string `json:"siteCode"` ModName string `json:"modName"` HttpUrl string `json:"httpUrl"` HttpsUrl string `json:"httpsUrl"` EnvID string `json:"envID"` ExtInfo string `json:"extInfo"` HashName string `json:"hashName"` ModAddrType int `json:"modAddrType"` } `json:"routePolicyList"` } `json:"data"` } type RefreshTokenResp struct { XMLName xml.Name `xml:"root"` Return string `xml:"return"` Token string `xml:"token"` Expiretime int32 `xml:"expiretime"` AccessToken string `xml:"accessToken"` Desc string `xml:"desc"` } type PersonalDiskInfoResp struct { BaseResp Data struct { FreeDiskSize string `json:"freeDiskSize"` DiskSize string `json:"diskSize"` IsInfinitePicStorage *bool `json:"isInfinitePicStorage"` } `json:"data"` } type FamilyDiskInfoResp struct { BaseResp Data struct { UsedSize string `json:"usedSize"` DiskSize string `json:"diskSize"` } `json:"data"` } type AndAlbumUploadResp struct { Result struct { ResultCode string `json:"resultCode"` ResultDesc string `json:"resultDesc"` } `json:"result"` UploadResult struct { UploadTaskID string `json:"uploadTaskID"` RedirectionURL string `json:"redirectionUrl"` NewContentIDList []struct { ContentID string `json:"contentID"` ContentName string `json:"contentName"` } `json:"newContentIDList"` } `json:"uploadResult"` } type ModifyCloudDocV2Req struct { CatalogType int `json:"catalogType"` CloudID string `json:"cloudID"` CommonAccountInfo struct { Account string `json:"account"` AccountType string `json:"accountType"` } `json:"commonAccountInfo"` DocLibName string `json:"docLibName"` DocLibraryID string `json:"docLibraryID"` Path string `json:"path"` } type ModifyCloudDocV2Resp struct { Result struct { ResultCode string `json:"resultCode"` ResultDesc string `json:"resultDesc"` } `json:"result"` } type CreateBatchOprTaskReq struct { CatalogList []string `json:"catalogList"` CommonAccountInfo struct { Account string `json:"account"` AccountType string `json:"accountType"` } `json:"commonAccountInfo"` ContentList []string `json:"contentList"` DestCatalogID string `json:"destCatalogID"` DestGroupID string `json:"destGroupID"` DestPath string `json:"destPath"` DestType int `json:"destType"` SourceCatalogType int `json:"sourceCatalogType"` SourceCloudID string `json:"sourceCloudID"` SourceType int `json:"sourceType"` TaskType int `json:"taskType"` } type CreateBatchOprTaskResp struct { Result struct { ResultCode string `json:"resultCode"` ResultDesc string `json:"resultDesc"` } `json:"result"` TaskID string `json:"taskID"` } ================================================ FILE: drivers/139/util.go ================================================ package _139 import ( "bytes" "context" "crypto/aes" "crypto/cipher" "crypto/md5" crypto_rand "crypto/rand" "crypto/sha1" "encoding/base64" "encoding/hex" "errors" "fmt" "io" "net/http" "net/url" "path" "regexp" "sort" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" ) const ( KEY_HEX_1 = "73634235495062495331515373756c734e7253306c673d3d" // 第一层 AES 解密密钥 KEY_HEX_2 = "7150714477323633586746674c337538" // 第二层 AES 解密密钥 ) // do others that not defined in Driver interface func (d *Yun139) isFamily() bool { return d.Type == "family" } func encodeURIComponent(str string) string { r := url.QueryEscape(str) r = strings.Replace(r, "+", "%20", -1) r = strings.Replace(r, "%21", "!", -1) r = strings.Replace(r, "%27", "'", -1) r = strings.Replace(r, "%28", "(", -1) r = strings.Replace(r, "%29", ")", -1) r = strings.Replace(r, "%2A", "*", -1) return r } func calSign(body, ts, randStr string) string { body = encodeURIComponent(body) strs := strings.Split(body, "") sort.Strings(strs) body = strings.Join(strs, "") body = base64.StdEncoding.EncodeToString([]byte(body)) res := utils.GetMD5EncodeStr(body) + utils.GetMD5EncodeStr(ts+":"+randStr) res = strings.ToUpper(utils.GetMD5EncodeStr(res)) return res } func getTime(t string) time.Time { stamp, _ := time.ParseInLocation("20060102150405", t, utils.CNLoc) return stamp } func (d *Yun139) refreshToken() error { if d.ref != nil { return d.ref.refreshToken() } decode, err := base64.StdEncoding.DecodeString(d.Authorization) if err != nil { return fmt.Errorf("authorization decode failed: %s", err) } decodeStr := string(decode) splits := strings.Split(decodeStr, ":") if len(splits) < 3 { return fmt.Errorf("authorization is invalid, splits < 3") } d.Account = splits[1] strs := strings.Split(splits[2], "|") if len(strs) < 4 { return fmt.Errorf("authorization is invalid, strs < 4") } expiration, err := strconv.ParseInt(strs[3], 10, 64) if err != nil { return fmt.Errorf("authorization is invalid") } expiration -= time.Now().UnixMilli() if expiration > 1000*60*60*24*15 { // Authorization有效期大于15天无需刷新 return nil } if expiration < 0 { return fmt.Errorf("authorization has expired") } url := "https://aas.caiyun.feixin.10086.cn:443/tellin/authTokenRefresh.do" var resp RefreshTokenResp reqBody := "" + splits[2] + "" + splits[1] + "656" _, err = base.RestyClient.R(). ForceContentType("application/xml"). SetBody(reqBody). SetResult(&resp). Post(url) if err != nil || resp.Return != "0" { log.Warnf("139yun: failed to refresh token with old token: %v, desc: %s. trying to login with password.", err, resp.Desc) newAuth, loginErr := d.loginWithPassword() log.Debugf("newAuth: Ok: %s", newAuth) if loginErr != nil { return fmt.Errorf("failed to login with password after refresh failed: %w", loginErr) } return nil } d.Authorization = base64.StdEncoding.EncodeToString([]byte(splits[0] + ":" + splits[1] + ":" + resp.Token)) op.MustSaveDriverStorage(d) return nil } func (d *Yun139) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() randStr := random.String(16) ts := time.Now().Format("2006-01-02 15:04:05") if callback != nil { callback(req) } body, err := utils.Json.Marshal(req.Body) if err != nil { return nil, err } sign := calSign(string(body), ts, randStr) svcType := "1" if d.isFamily() { svcType = "2" } req.SetHeaders(map[string]string{ "Accept": "application/json, text/plain, */*", "CMS-DEVICE": "default", "Authorization": "Basic " + d.getAuthorization(), "mcloud-channel": "1000101", "mcloud-client": "10701", //"mcloud-route": "001", "mcloud-sign": fmt.Sprintf("%s,%s,%s", ts, randStr, sign), //"mcloud-skey":"", "mcloud-version": "7.14.0", "Origin": "https://yun.139.com", "Referer": "https://yun.139.com/w/", "x-DeviceInfo": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||", "x-huawei-channelSrc": "10000034", "x-inner-ntwk": "2", "x-m4c-caller": "PC", "x-m4c-src": "10002", "x-SvcType": svcType, "Inner-Hcy-Router-Https": "1", }) var e BaseResp req.SetResult(&e) log.Debugf("[139] request: %s %s, body: %s", method, url, string(body)) res, err := req.Execute(method, url) if err != nil { log.Debugf("[139] request error: %v", err) return nil, err } log.Debugf("[139] response body: %s", res.String()) if !e.Success { // Always try to unmarshal to the specific response type first if 'resp' is provided. if resp != nil { err = utils.Json.Unmarshal(res.Body(), resp) if err != nil { log.Debugf("[139] failed to unmarshal response to specific type: %v", err) return nil, err // Return unmarshal error } if createBatchOprTaskResp, ok := resp.(*CreateBatchOprTaskResp); ok { log.Debugf("[139] CreateBatchOprTaskResp.Result.ResultCode: %s", createBatchOprTaskResp.Result.ResultCode) if createBatchOprTaskResp.Result.ResultCode == "0" { goto SUCCESS_PROCESS } } } return nil, errors.New(e.Message) // Fallback to original error if not handled } if resp != nil { err = utils.Json.Unmarshal(res.Body(), resp) if err != nil { return nil, err } } SUCCESS_PROCESS: return res.Body(), nil } func (d *Yun139) requestRoute(data interface{}, resp interface{}) ([]byte, error) { url := "https://user-njs.yun.139.com/user/route/qryRoutePolicy" req := base.RestyClient.R() randStr := random.String(16) ts := time.Now().Format("2006-01-02 15:04:05") callback := func(req *resty.Request) { req.SetBody(data) } if callback != nil { callback(req) } body, err := utils.Json.Marshal(req.Body) if err != nil { return nil, err } sign := calSign(string(body), ts, randStr) svcType := "1" if d.isFamily() { svcType = "2" } req.SetHeaders(map[string]string{ "Accept": "application/json, text/plain, */*", "CMS-DEVICE": "default", "Authorization": "Basic " + d.getAuthorization(), "mcloud-channel": "1000101", "mcloud-client": "10701", //"mcloud-route": "001", "mcloud-sign": fmt.Sprintf("%s,%s,%s", ts, randStr, sign), //"mcloud-skey":"", "mcloud-version": "7.14.0", "Origin": "https://yun.139.com", "Referer": "https://yun.139.com/w/", "x-DeviceInfo": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||", "x-huawei-channelSrc": "10000034", "x-inner-ntwk": "2", "x-m4c-caller": "PC", "x-m4c-src": "10002", "x-SvcType": svcType, "Inner-Hcy-Router-Https": "1", }) var e BaseResp req.SetResult(&e) res, err := req.Execute(http.MethodPost, url) log.Debugln(res.String()) if !e.Success { return nil, errors.New(e.Message) } if resp != nil { err = utils.Json.Unmarshal(res.Body(), resp) if err != nil { return nil, err } } return res.Body(), nil } func (d *Yun139) post(pathname string, data interface{}, resp interface{}) ([]byte, error) { return d.request("https://yun.139.com"+pathname, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, resp) } func (d *Yun139) getFiles(catalogID string) ([]model.Obj, error) { start := 0 limit := 100 files := make([]model.Obj, 0) for { data := base.Json{ "catalogID": catalogID, "sortDirection": 1, "startNumber": start + 1, "endNumber": start + limit, "filterType": 0, "catalogSortType": 0, "contentSortType": 0, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, } var resp GetDiskResp _, err := d.post("/orchestration/personalCloud/catalog/v1.0/getDisk", data, &resp) if err != nil { return nil, err } for _, catalog := range resp.Data.GetDiskResult.CatalogList { f := model.Object{ ID: catalog.CatalogID, Name: catalog.CatalogName, Size: 0, Modified: getTime(catalog.UpdateTime), Ctime: getTime(catalog.CreateTime), IsFolder: true, } files = append(files, &f) } for _, content := range resp.Data.GetDiskResult.ContentList { f := model.ObjThumb{ Object: model.Object{ ID: content.ContentID, Name: content.ContentName, Size: content.ContentSize, Modified: getTime(content.UpdateTime), HashInfo: utils.NewHashInfo(utils.MD5, content.Digest), }, Thumbnail: model.Thumbnail{Thumbnail: content.ThumbnailURL}, // Thumbnail: content.BigthumbnailURL, } files = append(files, &f) } if start+limit >= resp.Data.GetDiskResult.NodeCount { break } start += limit } return files, nil } func (d *Yun139) newJson(data map[string]interface{}) base.Json { common := map[string]interface{}{ "catalogType": 3, "cloudID": d.CloudID, "cloudType": 1, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, } return utils.MergeMap(data, common) } func (d *Yun139) familyGetFiles(catalogID string) ([]model.Obj, error) { pageNum := 1 files := make([]model.Obj, 0) for { data := d.newJson(base.Json{ "catalogID": catalogID, "contentSortType": 0, "pageInfo": base.Json{ "pageNum": pageNum, "pageSize": 100, }, "sortDirection": 1, }) var resp QueryContentListResp _, err := d.post("/orchestration/familyCloud-rebuild/content/v1.2/queryContentList", data, &resp) if err != nil { return nil, err } path := resp.Data.Path if catalogID == d.RootFolderID { d.RootPath = path } for _, catalog := range resp.Data.CloudCatalogList { f := model.Object{ ID: catalog.CatalogID, Name: catalog.CatalogName, Size: 0, IsFolder: true, Modified: getTime(catalog.LastUpdateTime), Ctime: getTime(catalog.CreateTime), Path: path, // 文件夹上一级的Path } files = append(files, &f) } for _, content := range resp.Data.CloudContentList { f := model.ObjThumb{ Object: model.Object{ ID: content.ContentID, Name: content.ContentName, Size: content.ContentSize, Modified: getTime(content.LastUpdateTime), Ctime: getTime(content.CreateTime), Path: path, // 文件所在目录的Path }, Thumbnail: model.Thumbnail{Thumbnail: content.ThumbnailURL}, // Thumbnail: content.BigthumbnailURL, } files = append(files, &f) } if resp.Data.TotalCount == 0 { break } pageNum++ } return files, nil } func (d *Yun139) groupGetFiles(catalogID string) ([]model.Obj, error) { pageNum := 1 files := make([]model.Obj, 0) for { data := d.newJson(base.Json{ "groupID": d.CloudID, "catalogID": path.Base(catalogID), "contentSortType": 0, "sortDirection": 1, "startNumber": pageNum, "endNumber": pageNum + 99, "path": path.Join(d.RootFolderID, catalogID), }) var resp QueryGroupContentListResp _, err := d.post("/orchestration/group-rebuild/content/v1.0/queryGroupContentList", data, &resp) if err != nil { return nil, err } path := resp.Data.GetGroupContentResult.ParentCatalogID if catalogID == d.RootFolderID { d.RootPath = path } for _, catalog := range resp.Data.GetGroupContentResult.CatalogList { f := model.Object{ ID: catalog.CatalogID, Name: catalog.CatalogName, Size: 0, IsFolder: true, Modified: getTime(catalog.UpdateTime), Ctime: getTime(catalog.CreateTime), Path: catalog.Path, // 文件夹的真实Path， root:/开头 } files = append(files, &f) } for _, content := range resp.Data.GetGroupContentResult.ContentList { f := model.ObjThumb{ Object: model.Object{ ID: content.ContentID, Name: content.ContentName, Size: content.ContentSize, Modified: getTime(content.UpdateTime), Ctime: getTime(content.CreateTime), Path: path, // 文件所在目录的Path }, Thumbnail: model.Thumbnail{Thumbnail: content.ThumbnailURL}, // Thumbnail: content.BigthumbnailURL, } files = append(files, &f) } if (pageNum + 99) > resp.Data.GetGroupContentResult.NodeCount { break } pageNum = pageNum + 100 } return files, nil } func (d *Yun139) getLink(contentId string) (string, error) { data := base.Json{ "appName": "", "contentID": contentId, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, } res, err := d.post("/orchestration/personalCloud/uploadAndDownload/v1.0/downloadRequest", data, nil) if err != nil { return "", err } return jsoniter.Get(res, "data", "downloadURL").ToString(), nil } func (d *Yun139) familyGetLink(contentId string, path string) (string, error) { data := d.newJson(base.Json{ "contentID": contentId, "path": path, }) res, err := d.post("/orchestration/familyCloud-rebuild/content/v1.0/getFileDownLoadURL", data, nil) if err != nil { return "", err } return jsoniter.Get(res, "data", "downloadURL").ToString(), nil } func (d *Yun139) groupGetLink(contentId string, path string) (string, error) { data := d.newJson(base.Json{ "contentID": contentId, "groupID": d.CloudID, "path": path, }) res, err := d.post("/orchestration/group-rebuild/groupManage/v1.0/getGroupFileDownLoadURL", data, nil) if err != nil { return "", err } return jsoniter.Get(res, "data", "downloadURL").ToString(), nil } func unicode(str string) string { textQuoted := strconv.QuoteToASCII(str) textUnquoted := textQuoted[1 : len(textQuoted)-1] return textUnquoted } func (d *Yun139) personalRequest(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { url := d.getPersonalCloudHost() + pathname req := base.RestyClient.R() randStr := random.String(16) ts := time.Now().Format("2006-01-02 15:04:05") if callback != nil { callback(req) } body, err := utils.Json.Marshal(req.Body) if err != nil { return nil, err } sign := calSign(string(body), ts, randStr) svcType := "1" if d.isFamily() { svcType = "2" } req.SetHeaders(map[string]string{ "Accept": "application/json, text/plain, */*", "Authorization": "Basic " + d.getAuthorization(), "Caller": "web", "Cms-Device": "default", "Mcloud-Channel": "1000101", "Mcloud-Client": "10701", "Mcloud-Route": "001", "Mcloud-Sign": fmt.Sprintf("%s,%s,%s", ts, randStr, sign), "Mcloud-Version": "7.14.0", "x-DeviceInfo": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||", "x-huawei-channelSrc": "10000034", "x-inner-ntwk": "2", "x-m4c-caller": "PC", "x-m4c-src": "10002", "x-SvcType": svcType, "X-Yun-Api-Version": "v1", "X-Yun-App-Channel": "10000034", "X-Yun-Channel-Source": "10000034", "X-Yun-Client-Info": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||", "X-Yun-Module-Type": "100", "X-Yun-Svc-Type": "1", }) var e BaseResp req.SetResult(&e) log.Debugf("[139] personal request: %s %s, body: %s", method, url, string(body)) res, err := req.Execute(method, url) if err != nil { log.Debugf("[139] personal request error: %v", err) return nil, err } log.Debugf("[139] personal response body: %s", res.String()) if !e.Success { return nil, errors.New(e.Message) } if resp != nil { err = utils.Json.Unmarshal(res.Body(), resp) if err != nil { return nil, err } } return res.Body(), nil } func (d *Yun139) personalPost(pathname string, data interface{}, resp interface{}) ([]byte, error) { return d.personalRequest(pathname, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, resp) } func (d *Yun139) isboPost(pathname string, data interface{}, resp interface{}) ([]byte, error) { url := "https://group.yun.139.com/hcy/mutual/adapter" + pathname return d.request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, resp) } func getPersonalTime(t string) time.Time { stamp, err := time.ParseInLocation("2006-01-02T15:04:05.999-07:00", t, utils.CNLoc) if err != nil { panic(err) } return stamp } func (d *Yun139) personalGetFiles(fileId string) ([]model.Obj, error) { files := make([]model.Obj, 0) nextPageCursor := "" for { data := base.Json{ "imageThumbnailStyleList": []string{"Small", "Large"}, "orderBy": "updated_at", "orderDirection": "DESC", "pageInfo": base.Json{ "pageCursor": nextPageCursor, "pageSize": 100, }, "parentFileId": fileId, } var resp PersonalListResp _, err := d.personalPost("/file/list", data, &resp) if err != nil { return nil, err } nextPageCursor = resp.Data.NextPageCursor for _, item := range resp.Data.Items { isFolder := (item.Type == "folder") var f model.Obj if isFolder { f = &model.Object{ ID: item.FileId, Name: item.Name, Size: 0, Modified: getPersonalTime(item.UpdatedAt), Ctime: getPersonalTime(item.CreatedAt), IsFolder: isFolder, } } else { Thumbnails := item.Thumbnails var ThumbnailUrl string if d.UseLargeThumbnail { for _, thumb := range Thumbnails { if strings.Contains(thumb.Style, "Large") { ThumbnailUrl = thumb.Url break } } } if ThumbnailUrl == "" && len(Thumbnails) > 0 { ThumbnailUrl = Thumbnails[len(Thumbnails)-1].Url } f = &model.ObjThumb{ Object: model.Object{ ID: item.FileId, Name: item.Name, Size: item.Size, Modified: getPersonalTime(item.UpdatedAt), Ctime: getPersonalTime(item.CreatedAt), IsFolder: isFolder, }, Thumbnail: model.Thumbnail{Thumbnail: ThumbnailUrl}, } } files = append(files, f) } if len(nextPageCursor) == 0 { break } } return files, nil } func (d *Yun139) personalGetLink(fileId string) (string, error) { data := base.Json{ "fileId": fileId, } res, err := d.personalPost("/file/getDownloadUrl", data, nil) if err != nil { return "", err } cdnUrl := jsoniter.Get(res, "data", "cdnUrl").ToString() if cdnUrl != "" { return cdnUrl, nil } else { return jsoniter.Get(res, "data", "url").ToString(), nil } } func (d *Yun139) getAuthorization() string { if d.ref != nil { return d.ref.getAuthorization() } return d.Authorization } func (d *Yun139) getAccount() string { if d.ref != nil { return d.ref.getAccount() } return d.Account } func (d *Yun139) getPersonalCloudHost() string { if d.ref != nil { return d.ref.getPersonalCloudHost() } return d.PersonalCloudHost } func (d *Yun139) uploadPersonalParts(ctx context.Context, partInfos []PartInfo, uploadPartInfos []PersonalPartInfo, rateLimited *driver.RateLimitReader, p *driver.Progress) error { // 确保数组以 PartNumber 从小到大排序 sort.Slice(uploadPartInfos, func(i, j int) bool { return uploadPartInfos[i].PartNumber < uploadPartInfos[j].PartNumber }) for _, uploadPartInfo := range uploadPartInfos { index := uploadPartInfo.PartNumber - 1 if index < 0 || index >= len(partInfos) { return fmt.Errorf("invalid PartNumber %d: index out of bounds (partInfos length: %d)", uploadPartInfo.PartNumber, len(partInfos)) } partSize := partInfos[index].PartSize log.Debugf("[139] uploading part %+v/%+v", index, len(partInfos)) limitReader := io.LimitReader(rateLimited, partSize) r := io.TeeReader(limitReader, p) req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadPartInfo.UploadUrl, r) if err != nil { return err } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Content-Length", fmt.Sprint(partSize)) req.Header.Set("Origin", "https://yun.139.com") req.Header.Set("Referer", "https://yun.139.com/") req.ContentLength = partSize err = func() error { res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() log.Debugf("[139] uploaded: %+v", res) if res.StatusCode != http.StatusOK { body, _ := io.ReadAll(res.Body) return fmt.Errorf("unexpected status code: %d, body: %s", res.StatusCode, string(body)) } return nil }() if err != nil { return err } } return nil } func (d *Yun139) getPersonalDiskInfo(ctx context.Context) (*PersonalDiskInfoResp, error) { data := map[string]interface{}{ "userDomainId": d.UserDomainID, } var resp PersonalDiskInfoResp _, err := d.request("https://user-njs.yun.139.com/user/disk/getPersonalDiskInfo", http.MethodPost, func(req *resty.Request) { req.SetBody(data) req.SetContext(ctx) }, &resp) if err != nil { return nil, err } return &resp, nil } func (d *Yun139) getFamilyDiskInfo(ctx context.Context) (*FamilyDiskInfoResp, error) { data := map[string]interface{}{ "userDomainId": d.UserDomainID, } var resp FamilyDiskInfoResp _, err := d.request("https://user-njs.yun.139.com/user/disk/getFamilyDiskInfo", http.MethodPost, func(req *resty.Request) { req.SetBody(data) req.SetContext(ctx) }, &resp) if err != nil { return nil, err } return &resp, nil } func getMd5(dataStr string) string { hash := md5.Sum([]byte(dataStr)) return fmt.Sprintf("%x", hash) } func (d *Yun139) step1_password_login() (string, error) { log.Debugf("--- 执行步骤 1: 登录 API ---") loginURL := "https://mail.10086.cn/Login/Login.ashx" // 密码 SHA1 哈希 hashedPassword := sha1Hash(fmt.Sprintf("fetion.com.cn:%s", d.Password)) log.Debugf("DEBUG: 原始密码: %s", d.Password) log.Debugf("DEBUG: SHA1 输入: fetion.com.cn:%s", d.Password) log.Debugf("DEBUG: 生成的 Password 哈希: %s", hashedPassword) cguid := strconv.FormatInt(time.Now().UnixMilli(), 10) // 随机生成 cguid loginHeaders := map[string]string{ "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", "accept-language": "zh-CN,zh;q=0.9,zh-TW;q=0.8,en-US;q=0.7,en;q=0.6,en-GB;q=0.5", "cache-control": "max-age=0", "content-type": "application/x-www-form-urlencoded", "dnt": "1", "origin": "https://mail.10086.cn", "priority": "u=0, i", "referer": fmt.Sprintf("https://mail.10086.cn/default.html?&s=1&v=0&u=%s&m=1&ec=S001&resource=indexLogin&clientid=1003&auto=on&cguid=%s&mtime=45", base64.StdEncoding.EncodeToString([]byte(d.Username)), cguid), "sec-ch-ua": "\"Microsoft Edge\";v=\"141\", \"Not?A_Brand\";v=\"8\", \"Chromium\";v=\"141\"", "sec-ch-ua-mobile": "?0", "sec-ch-ua-platform": "\"Windows\"", "sec-fetch-dest": "document", "sec-fetch-mode": "navigate", "sec-fetch-site": "same-origin", "sec-fetch-user": "?1", "upgrade-insecure-requests": "1", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 Edg/141.0.0.0", "Cookie": d.MailCookies, } loginData := url.Values{} loginData.Set("UserName", d.Username) loginData.Set("passOld", "") loginData.Set("auto", "on") loginData.Set("Password", hashedPassword) loginData.Set("webIndexPagePwdLogin", "1") loginData.Set("pwdType", "1") loginData.Set("clientId", "1003") loginData.Set("authType", "2") log.Debugf("DEBUG: 登录请求 URL: %s", loginURL) log.Debugf("DEBUG: 登录请求 Headers: %+v", loginHeaders) log.Debugf("DEBUG: 登录请求 Body: %s", loginData.Encode()) // 设置客户端不跟随重定向 client := base.RestyClient.SetRedirectPolicy(resty.NoRedirectPolicy()) res, err := client.R(). SetHeaders(loginHeaders). SetFormDataFromValues(loginData). Post(loginURL) if err != nil { // 如果是重定向错误，则不作为失败处理，因为我们禁止了自动重定向 if res != nil && res.StatusCode() >= 300 && res.StatusCode() < 400 { log.Debugf("DEBUG: 登录响应 Status Code: %d (Redirect)", res.StatusCode()) } else { return "", fmt.Errorf("step1 login request failed: %w", err) } } else { log.Debugf("DEBUG: 登录响应 Status Code: %d", res.StatusCode()) } // 恢复客户端的默认重定向策略，以免影响后续请求 base.RestyClient.SetRedirectPolicy(resty.FlexibleRedirectPolicy(10)) log.Debugf("DEBUG: 登录响应 Headers: %+v", res.Header()) var sid, extractedCguid string // 从 Location 头部提取 sid 和 cguid locationHeader := res.Header().Get("Location") if locationHeader != "" { sidMatch := regexp.MustCompile(`sid=([^&]+)`).FindStringSubmatch(locationHeader) cguidMatch := regexp.MustCompile(`cguid=([^&]+)`).FindStringSubmatch(locationHeader) if len(sidMatch) > 1 { sid = sidMatch[1] log.Debugf("DEBUG: 从 Location 提取到 sid: %s", sid) } if len(cguidMatch) > 1 { extractedCguid = cguidMatch[1] log.Debugf("DEBUG: 从 Location 提取到 cguid: %s", extractedCguid) } } // 如果 Location 中没有，尝试从 Set-Cookie 中提取 if sid == "" || extractedCguid == "" { setCookieHeaders := res.Header().Values("Set-Cookie") for _, cookieStr := range setCookieHeaders { ssoSidMatch := regexp.MustCompile(`Os_SSo_Sid=([^;]+)`).FindStringSubmatch(cookieStr) cookieCguidMatch := regexp.MustCompile(`cguid=([^;]+)`).FindStringSubmatch(cookieStr) if len(ssoSidMatch) > 1 && sid == "" { sid = ssoSidMatch[1] log.Debugf("DEBUG: 从 Set-Cookie 提取到 sid: %s", sid) } if len(cookieCguidMatch) > 1 && extractedCguid == "" { extractedCguid = cookieCguidMatch[1] log.Debugf("DEBUG: 从 Set-Cookie 提取到 cguid: %s", extractedCguid) } } } if sid == "" || extractedCguid == "" { return "", errors.New("failed to extract sid or cguid from login response") } // 提取并记录 cookies loginUrlObj, _ := url.Parse(loginURL) cookies := base.RestyClient.GetClient().Jar.Cookies(loginUrlObj) var cookieStrings []string for _, cookie := range cookies { cookieStrings = append(cookieStrings, cookie.Name+"="+cookie.Value) } cookieStr := strings.Join(cookieStrings, "; ") log.Debugf("DEBUG: 提取到的 Cookies: %s", cookieStr) d.MailCookies = cookieStr return sid, nil } func (d *Yun139) step2_get_single_token(sid string) (string, error) { log.Debugf("\n--- 执行步骤 2: 换artifact API ---") cguid := strconv.FormatInt(time.Now().UnixMilli(), 10) exchangeArtifactURL := fmt.Sprintf("https://smsrebuild1.mail.10086.cn/setting/s?func=%s&sid=%s&cguid=%s", url.QueryEscape("umc:getArtifact"), sid, cguid) // 从 MailCookies 中提取 RMKEY var rmkey string cookies := strings.Split(d.MailCookies, ";") for _, cookie := range cookies { cookie = strings.TrimSpace(cookie) if strings.HasPrefix(cookie, "RMKEY=") { rmkey = cookie break } } if rmkey == "" { return "", errors.New("RMKEY not found in MailCookies") } exchangePassidHeaders := map[string]string{ "Host": "smsrebuild1.mail.10086.cn", "Cookie": rmkey, "Content-Type": "text/xml; charset=utf-8", "Accept-Encoding": "gzip", "User-Agent": "okhttp/4.12.0", } log.Debugf("DEBUG: 换passid 请求 URL: %s", exchangeArtifactURL) log.Debugf("DEBUG: 换passid 请求 Headers: %+v", exchangePassidHeaders) res, err := base.RestyClient.R(). SetHeaders(exchangePassidHeaders). Post(exchangeArtifactURL) if err != nil { return "", fmt.Errorf("step2 exchange artifact request failed: %w", err) } log.Debugf("DEBUG: 换passid 响应 Status Code: %d", res.StatusCode()) log.Debugf("DEBUG: 换passid 响应 Headers: %+v", res.Header()) log.Debugf("DEBUG: 换passid 响应 Body: %s...", res.String()[:min(len(res.String()), 500)]) dycpwd := jsoniter.Get(res.Body(), "var", "artifact").ToString() if dycpwd == "" { return "", errors.New("failed to extract dycpwd from artifact exchange response") } log.Debugf("DEBUG: 提取到 dycpwd: %s", dycpwd) return dycpwd, nil } // --- 辅助函数：加密/解密 --- // sha1Hash 计算 SHA1 哈希值，返回十六进制字符串。 func sha1Hash(data string) string { h := sha1.New() h.Write([]byte(data)) return hex.EncodeToString(h.Sum(nil)) } // pkcs7_pad PKCS7 填充 func pkcs7_pad(data []byte, blockSize int) []byte { padding := blockSize - len(data)%blockSize padtext := bytes.Repeat([]byte{byte(padding)}, padding) return append(data, padtext...) } // pkcs7_unpad PKCS7 去填充 func pkcs7_unpad(data []byte) ([]byte, error) { length := len(data) if length == 0 { return nil, errors.New("pkcs7: data is empty") } unpadding := int(data[length-1]) if unpadding > length { return nil, errors.New("pkcs7: invalid padding") } return data[:(length - unpadding)], nil } // aes_ecb_decrypt AES/ECB/Pkcs7 解密，输入为十六进制字符串。 func aes_ecb_decrypt(ciphertext []byte, key []byte) ([]byte, error) { block, err := aes.NewCipher(key) if err != nil { return nil, err } if len(ciphertext)%block.BlockSize() != 0 { return nil, errors.New("AES ECB decrypt: ciphertext is not a multiple of the block size") } decrypted := make([]byte, len(ciphertext)) blockSize := block.BlockSize() for bs, be := 0, blockSize; bs < len(ciphertext); bs, be = bs+blockSize, be+blockSize { block.Decrypt(decrypted[bs:be], ciphertext[bs:be]) } return pkcs7_unpad(decrypted) } // 以下提供 camelCase 的 AES CBC 加解密，供文件中其它位置调用（并支持传入 IV）。 func aesCbcEncrypt(plaintext []byte, key []byte, iv []byte) ([]byte, error) { block, err := aes.NewCipher(key) if err != nil { return nil, err } if len(iv) != block.BlockSize() { return nil, fmt.Errorf("aesCbcEncrypt: iv length %d does not match block size %d", len(iv), block.BlockSize()) } padded := pkcs7_pad(plaintext, block.BlockSize()) ciphertext := make([]byte, len(padded)) mode := cipher.NewCBCEncrypter(block, iv) mode.CryptBlocks(ciphertext, padded) return ciphertext, nil } func aesCbcDecrypt(ciphertext []byte, key []byte, iv []byte) ([]byte, error) { block, err := aes.NewCipher(key) if err != nil { return nil, err } if len(iv) != block.BlockSize() { return nil, fmt.Errorf("aesCbcDecrypt: iv length %d does not match block size %d", len(iv), block.BlockSize()) } if len(ciphertext)%block.BlockSize() != 0 { return nil, errors.New("aesCbcDecrypt: ciphertext is not a multiple of the block size") } decrypted := make([]byte, len(ciphertext)) mode := cipher.NewCBCDecrypter(block, iv) mode.CryptBlocks(decrypted, ciphertext) return pkcs7_unpad(decrypted) } // sortedJsonStringify 对 JSON 对象进行排序并字符串化。 func sortedJsonStringify(obj interface{}) (string, error) { if obj == nil { return "null", nil } switch v := obj.(type) { case string: // 尝试解析为 JSON，如果成功则递归处理 var parsed interface{} if err := jsoniter.Unmarshal([]byte(v), &parsed); err == nil { return sortedJsonStringify(parsed) } // 如果不是 JSON 字符串，则直接返回 JSON 字符串化的结果 return jsoniter.MarshalToString(v) case int, float64, bool: return fmt.Sprintf("%v", v), nil case []interface{}: var items []string for _, item := range v { s, err := sortedJsonStringify(item) if err != nil { return "", err } items = append(items, s) } return fmt.Sprintf("[%s]", strings.Join(items, ",")), nil case map[string]interface{}: sortedKeys := make([]string, 0, len(v)) for key := range v { sortedKeys = append(sortedKeys, key) } sort.Strings(sortedKeys) var pairs []string for _, key := range sortedKeys { value := v[key] s, err := sortedJsonStringify(value) if err != nil { return "", err } // Use jsoniter.MarshalToString for the key to ensure it's quoted correctly keyStr, err := jsoniter.MarshalToString(key) if err != nil { return "", err } pairs = append(pairs, fmt.Sprintf("%s:%s", keyStr, s)) } return fmt.Sprintf("{%s}", strings.Join(pairs, ",")), nil default: // Fallback for other types, e.g., numbers, booleans, or unhandled complex types // Use jsoniter's default marshalling for these return jsoniter.MarshalToString(v) } } // yun139EncryptedRequest handles the common encrypted request/response flow. func (d *Yun139) yun139EncryptedRequest(url string, body interface{}, headers map[string]string, aesKeyHex string, resp interface{}) ([]byte, error) { // 1. Decode AES key aesKey, err := hex.DecodeString(aesKeyHex) if err != nil { return nil, fmt.Errorf("yun139EncryptedRequest: failed to decode AES key: %w", err) } // 2. Marshal and sort the request body sortedJson, err := sortedJsonStringify(body) if err != nil { return nil, fmt.Errorf("yun139EncryptedRequest: failed to marshal and sort body: %w", err) } log.Debugf("yun139EncryptedRequest: Request Body (plaintext): %s", sortedJson) // 3. Encrypt the body using AES/CBC iv := make([]byte, 16) // 16 bytes for AES-128 if _, err := crypto_rand.Read(iv); err != nil { return nil, fmt.Errorf("yun139EncryptedRequest: failed to generate IV: %w", err) } encryptedBody, err := aesCbcEncrypt([]byte(sortedJson), aesKey, iv) if err != nil { return nil, fmt.Errorf("yun139EncryptedRequest: failed to encrypt body: %w", err) } payload := base64.StdEncoding.EncodeToString(append(iv, encryptedBody...)) // 4. Make the request res, err := base.RestyClient.R(). SetHeaders(headers). SetBody(payload). Post(url) if err != nil { return nil, fmt.Errorf("yun139EncryptedRequest: http request failed: %w", err) } if res.StatusCode() != 200 { return nil, fmt.Errorf("yun139EncryptedRequest: unexpected status code %d: %s", res.StatusCode(), res.String()) } // 5. Decrypt the response respBody := res.Body() var decryptedBytes []byte if len(respBody) > 0 && respBody[0] == '{' { log.Warnf("yun139EncryptedRequest: received a plain JSON response, not an encrypted string. Body: %s", string(respBody)) decryptedBytes = respBody } else { decodedResp, err := base64.StdEncoding.DecodeString(string(respBody)) if err != nil { return nil, fmt.Errorf("yun139EncryptedRequest: response base64 decode failed: %w. Body: '%s'", err, string(respBody)) } if len(decodedResp) < 16 { return nil, fmt.Errorf("yun139EncryptedRequest: decoded response is too short to be encrypted. Length: %d", len(decodedResp)) } respIv := decodedResp[:16] respCiphertext := decodedResp[16:] decryptedBytes, err = aesCbcDecrypt(respCiphertext, aesKey, respIv) if err != nil { return nil, fmt.Errorf("yun139EncryptedRequest: response aes decrypt failed: %w", err) } } log.Debugf("yun139EncryptedRequest: Response Body (decrypted): %s", string(decryptedBytes)) // 6. Unmarshal to the final response struct if resp != nil { err = utils.Json.Unmarshal(decryptedBytes, resp) if err != nil { return nil, fmt.Errorf("yun139EncryptedRequest: failed to unmarshal decrypted response: %w", err) } } return decryptedBytes, nil } func (d *Yun139) step3_third_party_login(dycpwd string) (string, error) { log.Debugf("\n--- 执行步骤 3: 单点登录 API ---") ssoLoginURL := "https://user-njs.yun.139.com/user/thirdlogin" // 构建原始请求体 ssoRequestBodyRaw := base.Json{ "clientkey_decrypt": "l3TryM&Q+X7@dzwk)qP", "clienttype": "886", "cpid": "507", "dycpwd": dycpwd, "extInfo": base.Json{"ifOpenAccount": "0"}, "loginMode": "0", "msisdn": d.Username, "pintype": "13", "secinfo": strings.ToUpper(sha1Hash(fmt.Sprintf("fetion.com.cn:%s", dycpwd))), "version": "20250901", } ssoLoginHeaders := map[string]string{ "hcy-cool-flag": "1", "x-huawei-channelSrc": "10246600", "x-sdk-channelSrc": "", "x-MM-Source": "0", "x-UserAgent": "android|23116PN5BC|android15|1.2.6|||1440x3200|10246600", "x-DeviceInfo": "4|127.0.0.1|5|1.2.6|Xiaomi|23116PN5BC||02-00-00-00-00-00|android 15|1440x3200|android|||", "Content-Type": "text/plain;charset=UTF-8", "Host": "user-njs.yun.139.com", "Connection": "Keep-Alive", "Accept-Encoding": "gzip", "User-Agent": "okhttp/3.12.2", } // 使用通用加密请求函数 decryptedLayer1StrBytes, err := d.yun139EncryptedRequest(ssoLoginURL, ssoRequestBodyRaw, ssoLoginHeaders, KEY_HEX_1, nil) if err != nil { return "", fmt.Errorf("step3 encrypted request failed: %w", err) } hexInner := jsoniter.Get(decryptedLayer1StrBytes, "data").ToString() if hexInner == "" { return "", errors.New("missing data field in first layer decryption result") } log.Debugf("DEBUG: 第一层解密提取到 hex_inner: %s...", hexInner[:min(len(hexInner), 50)]) // 第二层解密 key2, err := hex.DecodeString(KEY_HEX_2) if err != nil { return "", fmt.Errorf("failed to decode KEY_HEX_2: %w", err) } hexInnerBytes, err := hex.DecodeString(hexInner) if err != nil { return "", fmt.Errorf("failed to decode hex_inner: %w", err) } finalJsonStrBytes, err := aes_ecb_decrypt(hexInnerBytes, key2) if err != nil { return "", fmt.Errorf("step3 response layer2 aes ecb decrypt failed: %w", err) } log.Debugf("DEBUG: 最终解密结果: %s", string(finalJsonStrBytes)) // 提取 authToken authToken := jsoniter.Get(finalJsonStrBytes, "authToken").ToString() if authToken == "" { return "", errors.New("failed to extract authToken from final decryption result") } log.Debugf("DEBUG: 提取到 authToken: %s", authToken) // 提取 account 和 userDomainId account := jsoniter.Get(finalJsonStrBytes, "account").ToString() userDomainId := jsoniter.Get(finalJsonStrBytes, "userDomainId").ToString() if account == "" || userDomainId == "" { return "", errors.New("failed to extract account or userDomainId from final decryption result") } d.UserDomainID = userDomainId newAuthorization := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("pc:%s:%s", account, authToken))) return newAuthorization, nil } func (d *Yun139) loginWithPassword() (string, error) { if d.Username == "" || d.Password == "" || d.MailCookies == "" { return "", errors.New("username, password or mail_cookies is empty") } passId, err := d.step1_password_login() if err != nil { return "", err } log.Infof("Step 1 success, passId: %s", passId) token, err := d.step2_get_single_token(passId) if err != nil { return "", err } log.Infof("Step 2 success, token: %s", token) newAuth, err := d.step3_third_party_login(token) if err != nil { return "", err } log.Infof("Step 3 success, new authorization generated.") d.Authorization = newAuth // Ensure Authorization is also updated before saving op.MustSaveDriverStorage(d) return newAuth, nil } func (d *Yun139) andAlbumRequest(pathname string, body interface{}, resp interface{}) ([]byte, error) { url := "https://group.yun.139.com/hcy/family/adapter/andAlbum/openApi" + pathname headers := map[string]string{ "Host": "group.yun.139.com", "authorization": "Basic " + d.getAuthorization(), "x-svctype": "2", "hcy-cool-flag": "1", "api-version": "v2", "x-huawei-channelsrc": "10246600", "x-sdk-channelsrc": "", "x-mm-source": "0", "x-deviceinfo": "1|127.0.0.1|1|12.3.2|Xiaomi|23116PN5BC||02-00-00-00-00-00|android 15|1440x3200|android|zh||||032|0|", //重要参数 "content-type": "application/json; charset=utf-8", "user-agent": "okhttp/4.11.0", "accept-encoding": "gzip", } return d.yun139EncryptedRequest(url, body, headers, KEY_HEX_1, resp) } func (d *Yun139) handleMetaGroupCopy(ctx context.Context, srcObj, dstDir model.Obj) error { pathname := "/copyContentCatalog" var sourceContentIDs []string var sourceCatalogIDs []string if srcObj.IsDir() { sourceCatalogIDs = append(sourceCatalogIDs, path.Join("root:/", srcObj.GetPath(), srcObj.GetID())) } else { sourceContentIDs = append(sourceContentIDs, path.Join("root:/", srcObj.GetPath(), srcObj.GetID())) } destCatalogID := path.Join("root:/", dstDir.GetPath(), dstDir.GetID()) log.Debugf("[139Yun Group Copy] srcObj ID: %s, srcObj Path: %s, dstDir ID: %s, dstDir Path: %s, destCatalogID: %s", srcObj.GetID(), srcObj.GetPath(), dstDir.GetID(), dstDir.GetPath(), destCatalogID) body := base.Json{ "commonAccountInfo": base.Json{ "accountType": "1", "accountUserId": d.UserDomainID, }, "destCatalogID": destCatalogID, "destCloudID": d.CloudID, "sourceCatalogIDs": sourceCatalogIDs, "sourceCloudID": d.CloudID, "sourceContentIDs": sourceContentIDs, } var resp base.Json _, err := d.andAlbumRequest(pathname, body, &resp) return err } // getGroupRootByCloudID 查询 group 上层信息，优先返回 parentCatalogID，回退到 catalogList[0].path func (d *Yun139) getGroupRootByCloudID(cloudID string) (string, error) { pathname := "/orchestration/group-rebuild/catalog/v1.0/queryGroupContentList" body := base.Json{ "groupID": cloudID, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, "pageInfo": base.Json{ "pageNum": 1, "pageSize": 1, }, } var resp base.Json _, err := d.post(pathname, body, &resp) if err != nil { return "", err } dataObj, _ := resp["data"].(map[string]interface{}) if dataObj == nil { return "", fmt.Errorf("invalid group response data") } if gcr, ok := dataObj["getGroupContentResult"].(map[string]interface{}); ok { if pid, ok := gcr["parentCatalogID"].(string); ok && pid != "" { return pid, nil } if cl, ok := gcr["catalogList"].([]interface{}); ok && len(cl) > 0 { if first, ok := cl[0].(map[string]interface{}); ok { if p, ok := first["path"].(string); ok && p != "" { return p, nil } } } } return "", fmt.Errorf("no root found in group response") } // getFamilyRootPath 查询 family 的上层 path（data.path） // 返回值已去除前缀 "root:/"（或 "root:"），直接返回纯 ID 或 path 部分，便于持久化为 RootFolderID。 func (d *Yun139) getFamilyRootPath(cloudID string) (string, error) { // 使用 v1.2 接口（代码日志中已有该请求），pageSize 取 1 足够获取 path 字段 pathname := "/orchestration/familyCloud-rebuild/content/v1.2/queryContentList" body := base.Json{ "catalogID": "", "catalogType": 3, "cloudID": cloudID, "cloudType": 1, "commonAccountInfo": base.Json{ "account": d.getAccount(), "accountType": 1, }, "contentSortType": 0, "pageInfo": base.Json{ "pageNum": 1, "pageSize": 1, }, "sortDirection": 1, } var resp base.Json _, err := d.post(pathname, body, &resp) if err != nil { return "", err } dataObj, _ := resp["data"].(map[string]interface{}) if dataObj == nil { return "", fmt.Errorf("invalid family response data") } // helper to strip "root:/" or "root:" prefix stripRoot := func(s string) string { s = strings.TrimSpace(s) s = strings.TrimPrefix(s, "root:/") s = strings.TrimPrefix(s, "root:") return s } if p, ok := dataObj["path"].(string); ok && p != "" { return stripRoot(p), nil } // 回退：有时 path 在 cloudCatalogList.catalogList 中 if cl, ok := dataObj["cloudCatalogList"].([]interface{}); ok && len(cl) > 0 { if first, ok := cl[0].(map[string]interface{}); ok { if p, ok := first["path"].(string); ok && p != "" { return stripRoot(p), nil } } } return "", fmt.Errorf("no path found in family response") } ================================================ FILE: drivers/189/driver.go ================================================ package _189 import ( "context" "net/http" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) type Cloud189 struct { model.Storage Addition client *resty.Client rsa Rsa sessionKey string } func (d *Cloud189) Config() driver.Config { return config } func (d *Cloud189) GetAddition() driver.Additional { return &d.Addition } func (d *Cloud189) Init(ctx context.Context) error { d.client = base.NewRestyClient(). SetHeader("Referer", "https://cloud.189.cn/") return d.newLogin() } func (d *Cloud189) Drop(ctx context.Context) error { return nil } func (d *Cloud189) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { return d.getFiles(dir.GetID()) } func (d *Cloud189) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp DownResp u := "https://cloud.189.cn/api/portal/getFileInfo.action" _, err := d.request(u, http.MethodGet, func(req *resty.Request) { req.SetQueryParam("fileId", file.GetID()) }, &resp) if err != nil { return nil, err } client := resty.NewWithClient(d.client.GetClient()).SetRedirectPolicy( resty.RedirectPolicyFunc(func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse })) res, err := client.R().SetHeader("User-Agent", base.UserAgent).Get("https:" + resp.FileDownloadUrl) if err != nil { return nil, err } log.Debugln(res.Status()) log.Debugln(res.String()) link := model.Link{} log.Debugln("first url:", resp.FileDownloadUrl) if res.StatusCode() == 302 { link.URL = res.Header().Get("location") log.Debugln("second url:", link.URL) _, _ = client.R().Get(link.URL) if res.StatusCode() == 302 { link.URL = res.Header().Get("location") } log.Debugln("third url:", link.URL) } else { link.URL = resp.FileDownloadUrl } link.URL = strings.Replace(link.URL, "http://", "https://", 1) return &link, nil } func (d *Cloud189) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { form := map[string]string{ "parentFolderId": parentDir.GetID(), "folderName": dirName, } _, err := d.request("https://cloud.189.cn/api/open/file/createFolder.action", http.MethodPost, func(req *resty.Request) { req.SetFormData(form) }, nil) return err } func (d *Cloud189) Move(ctx context.Context, srcObj, dstDir model.Obj) error { isFolder := 0 if srcObj.IsDir() { isFolder = 1 } taskInfos := []base.Json{ { "fileId": srcObj.GetID(), "fileName": srcObj.GetName(), "isFolder": isFolder, }, } taskInfosBytes, err := utils.Json.Marshal(taskInfos) if err != nil { return err } form := map[string]string{ "type": "MOVE", "targetFolderId": dstDir.GetID(), "taskInfos": string(taskInfosBytes), } _, err = d.request("https://cloud.189.cn/api/open/batch/createBatchTask.action", http.MethodPost, func(req *resty.Request) { req.SetFormData(form) }, nil) return err } func (d *Cloud189) Rename(ctx context.Context, srcObj model.Obj, newName string) error { url := "https://cloud.189.cn/api/open/file/renameFile.action" idKey := "fileId" nameKey := "destFileName" if srcObj.IsDir() { url = "https://cloud.189.cn/api/open/file/renameFolder.action" idKey = "folderId" nameKey = "destFolderName" } form := map[string]string{ idKey: srcObj.GetID(), nameKey: newName, } _, err := d.request(url, http.MethodPost, func(req *resty.Request) { req.SetFormData(form) }, nil) return err } func (d *Cloud189) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { isFolder := 0 if srcObj.IsDir() { isFolder = 1 } taskInfos := []base.Json{ { "fileId": srcObj.GetID(), "fileName": srcObj.GetName(), "isFolder": isFolder, }, } taskInfosBytes, err := utils.Json.Marshal(taskInfos) if err != nil { return err } form := map[string]string{ "type": "COPY", "targetFolderId": dstDir.GetID(), "taskInfos": string(taskInfosBytes), } _, err = d.request("https://cloud.189.cn/api/open/batch/createBatchTask.action", http.MethodPost, func(req *resty.Request) { req.SetFormData(form) }, nil) return err } func (d *Cloud189) Remove(ctx context.Context, obj model.Obj) error { isFolder := 0 if obj.IsDir() { isFolder = 1 } taskInfos := []base.Json{ { "fileId": obj.GetID(), "fileName": obj.GetName(), "isFolder": isFolder, }, } taskInfosBytes, err := utils.Json.Marshal(taskInfos) if err != nil { return err } form := map[string]string{ "type": "DELETE", "targetFolderId": "", "taskInfos": string(taskInfosBytes), } _, err = d.request("https://cloud.189.cn/api/open/batch/createBatchTask.action", http.MethodPost, func(req *resty.Request) { req.SetFormData(form) }, nil) return err } func (d *Cloud189) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { return d.newUpload(ctx, dstDir, stream, up) } func (d *Cloud189) GetDetails(ctx context.Context) (*model.StorageDetails, error) { capacityInfo, err := d.getCapacityInfo(ctx) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: capacityInfo.CloudCapacityInfo.TotalSize, UsedSpace: capacityInfo.CloudCapacityInfo.UsedSize, }, }, nil } var _ driver.Driver = (*Cloud189)(nil) ================================================ FILE: drivers/189/help.go ================================================ package _189 import ( "bytes" "crypto/aes" "crypto/hmac" "crypto/md5" "crypto/rand" "crypto/rsa" "crypto/sha1" "crypto/x509" "encoding/base64" "encoding/hex" "encoding/pem" "fmt" "net/url" "regexp" "strconv" "strings" myrand "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" log "github.com/sirupsen/logrus" ) func random() string { return fmt.Sprintf("0.%17v", myrand.Rand.Int63n(100000000000000000)) } func RsaEncode(origData []byte, j_rsakey string, hex bool) string { publicKey := []byte("-----BEGIN PUBLIC KEY-----\n" + j_rsakey + "\n-----END PUBLIC KEY-----") block, _ := pem.Decode(publicKey) pubInterface, _ := x509.ParsePKIXPublicKey(block.Bytes) pub := pubInterface.(*rsa.PublicKey) b, err := rsa.EncryptPKCS1v15(rand.Reader, pub, origData) if err != nil { log.Errorf("err: %s", err.Error()) } res := base64.StdEncoding.EncodeToString(b) if hex { return b64tohex(res) } return res } var b64map = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" var BI_RM = "0123456789abcdefghijklmnopqrstuvwxyz" func int2char(a int) string { return strings.Split(BI_RM, "")[a] } func b64tohex(a string) string { d := "" e := 0 c := 0 for i := 0; i < len(a); i++ { m := strings.Split(a, "")[i] if m != "=" { v := strings.Index(b64map, m) if 0 == e { e = 1 d += int2char(v >> 2) c = 3 & v } else if 1 == e { e = 2 d += int2char(c<<2 | v>>4) c = 15 & v } else if 2 == e { e = 3 d += int2char(c) d += int2char(v >> 2) c = 3 & v } else { e = 0 d += int2char(c<<2 | v>>4) d += int2char(15 & v) } } } if e == 1 { d += int2char(c << 2) } return d } func qs(form map[string]string) string { f := make(url.Values) for k, v := range form { f.Set(k, v) } return EncodeParam(f) //strList := make([]string, 0) //for k, v := range form { // strList = append(strList, fmt.Sprintf("%s=%s", k, url.QueryEscape(v))) //} //return strings.Join(strList, "&") } func EncodeParam(v url.Values) string { if v == nil { return "" } var buf strings.Builder keys := make([]string, 0, len(v)) for k := range v { keys = append(keys, k) } for _, k := range keys { vs := v[k] for _, v := range vs { if buf.Len() > 0 { buf.WriteByte('&') } buf.WriteString(k) buf.WriteByte('=') //if k == "fileName" { // buf.WriteString(encode(v)) //} else { buf.WriteString(v) //} } } return buf.String() } func encode(str string) string { //str = strings.ReplaceAll(str, "%", "%25") //str = strings.ReplaceAll(str, "&", "%26") //str = strings.ReplaceAll(str, "+", "%2B") //return str return url.QueryEscape(str) } func AesEncrypt(data, key []byte) []byte { block, _ := aes.NewCipher(key) if block == nil { return []byte{} } data = PKCS7Padding(data, block.BlockSize()) decrypted := make([]byte, len(data)) size := block.BlockSize() for bs, be := 0, size; bs < len(data); bs, be = bs+size, be+size { block.Encrypt(decrypted[bs:be], data[bs:be]) } return decrypted } func PKCS7Padding(ciphertext []byte, blockSize int) []byte { padding := blockSize - len(ciphertext)%blockSize padtext := bytes.Repeat([]byte{byte(padding)}, padding) return append(ciphertext, padtext...) } func hmacSha1(data string, secret string) string { h := hmac.New(sha1.New, []byte(secret)) h.Write([]byte(data)) return hex.EncodeToString(h.Sum(nil)) } func getMd5(data []byte) []byte { h := md5.New() h.Write(data) return h.Sum(nil) } func decodeURIComponent(str string) string { r, _ := url.PathUnescape(str) //r = strings.ReplaceAll(r, " ", "+") return r } func Random(v string) string { reg := regexp.MustCompilePOSIX("[xy]") data := reg.ReplaceAllFunc([]byte(v), func(msg []byte) []byte { var i int64 t := int64(16 * myrand.Rand.Float32()) if msg[0] == 120 { i = t } else { i = 3&t | 8 } return []byte(strconv.FormatInt(i, 16)) }) return string(data) } ================================================ FILE: drivers/189/login.go ================================================ package _189 import ( "errors" "strconv" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" ) type AppConf struct { Data struct { AccountType string `json:"accountType"` AgreementCheck string `json:"agreementCheck"` AppKey string `json:"appKey"` ClientType int `json:"clientType"` IsOauth2 bool `json:"isOauth2"` LoginSort string `json:"loginSort"` MailSuffix string `json:"mailSuffix"` PageKey string `json:"pageKey"` ParamId string `json:"paramId"` RegReturnUrl string `json:"regReturnUrl"` ReqId string `json:"reqId"` ReturnUrl string `json:"returnUrl"` ShowFeedback string `json:"showFeedback"` ShowPwSaveName string `json:"showPwSaveName"` ShowQrSaveName string `json:"showQrSaveName"` ShowSmsSaveName string `json:"showSmsSaveName"` Sso string `json:"sso"` } `json:"data"` Msg string `json:"msg"` Result string `json:"result"` } type EncryptConf struct { Result int `json:"result"` Data struct { UpSmsOn string `json:"upSmsOn"` Pre string `json:"pre"` PreDomain string `json:"preDomain"` PubKey string `json:"pubKey"` } `json:"data"` } func (d *Cloud189) newLogin() error { url := "https://cloud.189.cn/api/portal/loginUrl.action?redirectURL=https%3A%2F%2Fcloud.189.cn%2Fmain.action" res, err := d.client.R().Get(url) if err != nil { return err } // Is logged in redirectURL := res.RawResponse.Request.URL if redirectURL.String() == "https://cloud.189.cn/web/main" { return nil } lt := redirectURL.Query().Get("lt") reqId := redirectURL.Query().Get("reqId") appId := redirectURL.Query().Get("appId") headers := map[string]string{ "lt": lt, "reqid": reqId, "referer": redirectURL.String(), "origin": "https://open.e.189.cn", } // get app Conf var appConf AppConf res, err = d.client.R().SetHeaders(headers).SetFormData(map[string]string{ "version": "2.0", "appKey": appId, }).SetResult(&appConf).Post("https://open.e.189.cn/api/logbox/oauth2/appConf.do") if err != nil { return err } log.Debugf("189 AppConf resp body: %s", res.String()) if appConf.Result != "0" { return errors.New(appConf.Msg) } // get encrypt conf var encryptConf EncryptConf res, err = d.client.R().SetHeaders(headers).SetFormData(map[string]string{ "appId": appId, }).Post("https://open.e.189.cn/api/logbox/config/encryptConf.do") if err != nil { return err } err = utils.Json.Unmarshal(res.Body(), &encryptConf) if err != nil { return err } log.Debugf("189 EncryptConf resp body: %s\n%+v", res.String(), encryptConf) if encryptConf.Result != 0 { return errors.New("get EncryptConf error:" + res.String()) } // TODO: getUUID? needcaptcha // login loginData := map[string]string{ "version": "v2.0", "apToken": "", "appKey": appId, "accountType": appConf.Data.AccountType, "userName": encryptConf.Data.Pre + RsaEncode([]byte(d.Username), encryptConf.Data.PubKey, true), "epd": encryptConf.Data.Pre + RsaEncode([]byte(d.Password), encryptConf.Data.PubKey, true), "captchaType": "", "validateCode": "", "smsValidateCode": "", "captchaToken": "", "returnUrl": appConf.Data.ReturnUrl, "mailSuffix": appConf.Data.MailSuffix, "dynamicCheck": "FALSE", "clientType": strconv.Itoa(appConf.Data.ClientType), "cb_SaveName": "3", "isOauth2": strconv.FormatBool(appConf.Data.IsOauth2), "state": "", "paramId": appConf.Data.ParamId, } res, err = d.client.R().SetHeaders(headers).SetFormData(loginData).Post("https://open.e.189.cn/api/logbox/oauth2/loginSubmit.do") if err != nil { return err } log.Debugf("189 login resp body: %s", res.String()) loginResult := utils.Json.Get(res.Body(), "result").ToInt() if loginResult != 0 { return errors.New(utils.Json.Get(res.Body(), "msg").ToString()) } return nil } ================================================ FILE: drivers/189/meta.go ================================================ package _189 import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` Cookie string `json:"cookie" help:"Fill in the cookie if need captcha"` driver.RootID } var config = driver.Config{ Name: "189Cloud", LocalSort: true, DefaultRoot: "-11", Alert: `info|You can try to use 189PC driver if this driver does not work.`, } func init() { op.RegisterDriver(func() driver.Driver { return &Cloud189{} }) } ================================================ FILE: drivers/189/types.go ================================================ package _189 type LoginResp struct { Msg string `json:"msg"` Result int `json:"result"` ToUrl string `json:"toUrl"` } type Error struct { ErrorCode string `json:"errorCode"` ErrorMsg string `json:"errorMsg"` } type File struct { Id int64 `json:"id"` LastOpTime string `json:"lastOpTime"` Name string `json:"name"` Size int64 `json:"size"` Icon struct { SmallUrl string `json:"smallUrl"` //LargeUrl string `json:"largeUrl"` } `json:"icon"` Url string `json:"url"` } type Folder struct { Id int64 `json:"id"` LastOpTime string `json:"lastOpTime"` Name string `json:"name"` } type Files struct { ResCode int `json:"res_code"` ResMessage string `json:"res_message"` FileListAO struct { Count int `json:"count"` FileList []File `json:"fileList"` FolderList []Folder `json:"folderList"` } `json:"fileListAO"` } type UploadUrlsResp struct { Code string `json:"code"` UploadUrls map[string]Part `json:"uploadUrls"` } type Part struct { RequestURL string `json:"requestURL"` RequestHeader string `json:"requestHeader"` } type Rsa struct { Expire int64 `json:"expire"` PkId string `json:"pkId"` PubKey string `json:"pubKey"` } type Down struct { ResCode int `json:"res_code"` ResMessage string `json:"res_message"` FileDownloadUrl string `json:"fileDownloadUrl"` } type DownResp struct { ResCode int `json:"res_code"` ResMessage string `json:"res_message"` FileDownloadUrl string `json:"downloadUrl"` } type CapacityResp struct { ResCode int `json:"res_code"` ResMessage string `json:"res_message"` Account string `json:"account"` CloudCapacityInfo struct { FreeSize int64 `json:"freeSize"` MailUsedSize int64 `json:"mail189UsedSize"` TotalSize int64 `json:"totalSize"` UsedSize int64 `json:"usedSize"` } `json:"cloudCapacityInfo"` FamilyCapacityInfo struct { FreeSize int64 `json:"freeSize"` TotalSize int64 `json:"totalSize"` UsedSize int64 `json:"usedSize"` } `json:"familyCapacityInfo"` TotalSize uint64 `json:"totalSize"` } ================================================ FILE: drivers/189/util.go ================================================ package _189 import ( "bytes" "context" "crypto/md5" "encoding/base64" "encoding/hex" "errors" "fmt" "io" "math" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" myrand "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" ) // do others that not defined in Driver interface //func (d *Cloud189) login() error { // url := "https://cloud.189.cn/api/portal/loginUrl.action?redirectURL=https%3A%2F%2Fcloud.189.cn%2Fmain.action" // b := "" // lt := "" // ltText := regexp.MustCompile(`lt = "(.+?)"`) // var res *resty.Response // var err error // for i := 0; i < 3; i++ { // res, err = d.client.R().Get(url) // if err != nil { // return err // } // // 已经登陆 // if res.RawResponse.Request.URL.String() == "https://cloud.189.cn/web/main" { // return nil // } // b = res.String() // ltTextArr := ltText.FindStringSubmatch(b) // if len(ltTextArr) > 0 { // lt = ltTextArr[1] // break // } else { // <-time.After(time.Second) // } // } // if lt == "" { // return fmt.Errorf("get page: %s \nstatus: %d \nrequest url: %s\nredirect url: %s", // b, res.StatusCode(), res.RawResponse.Request.URL.String(), res.Header().Get("location")) // } // captchaToken := regexp.MustCompile(`captchaToken' value='(.+?)'`).FindStringSubmatch(b)[1] // returnUrl := regexp.MustCompile(`returnUrl = '(.+?)'`).FindStringSubmatch(b)[1] // paramId := regexp.MustCompile(`paramId = "(.+?)"`).FindStringSubmatch(b)[1] // //reqId := regexp.MustCompile(`reqId = "(.+?)"`).FindStringSubmatch(b)[1] // jRsakey := regexp.MustCompile(`j_rsaKey" value="(\S+)"`).FindStringSubmatch(b)[1] // vCodeID := regexp.MustCompile(`picCaptcha\.do\?token\=([A-Za-z0-9\&\=]+)`).FindStringSubmatch(b)[1] // vCodeRS := "" // if vCodeID != "" { // // need ValidateCode // log.Debugf("try to identify verification codes") // timeStamp := strconv.FormatInt(time.Now().UnixNano()/1e6, 10) // u := "https://open.e.189.cn/api/logbox/oauth2/picCaptcha.do?token=" + vCodeID + timeStamp // imgRes, err := d.client.R().SetHeaders(map[string]string{ // "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:74.0) Gecko/20100101 Firefox/76.0", // "Referer": "https://open.e.189.cn/api/logbox/oauth2/unifyAccountLogin.do", // "Sec-Fetch-Dest": "image", // "Sec-Fetch-Mode": "no-cors", // "Sec-Fetch-Site": "same-origin", // }).Get(u) // if err != nil { // return err // } // // Enter the verification code manually // //err = message.GetMessenger().WaitSend(message.Message{ // // Type: "image", // // Content: "data:image/png;base64," + base64.StdEncoding.EncodeToString(imgRes.Body()), // //}, 10) // //if err != nil { // // return err // //} // //vCodeRS, err = message.GetMessenger().WaitReceive(30) // // use ocr api // vRes, err := base.RestyClient.R().SetMultipartField( // "image", "validateCode.png", "image/png", bytes.NewReader(imgRes.Body())). // Post(setting.GetStr(conf.OcrApi)) // if err != nil { // return err // } // if jsoniter.Get(vRes.Body(), "status").ToInt() != 200 { // return errors.New("ocr error:" + jsoniter.Get(vRes.Body(), "msg").ToString()) // } // vCodeRS = jsoniter.Get(vRes.Body(), "result").ToString() // log.Debugln("code: ", vCodeRS) // } // userRsa := RsaEncode([]byte(d.Username), jRsakey, true) // passwordRsa := RsaEncode([]byte(d.Password), jRsakey, true) // url = "https://open.e.189.cn/api/logbox/oauth2/loginSubmit.do" // var loginResp LoginResp // res, err = d.client.R(). // SetHeaders(map[string]string{ // "lt": lt, // "User-Agent": base.UserAgentNT, // "Referer": "https://open.e.189.cn/", // "accept": "application/json;charset=UTF-8", // }).SetFormData(map[string]string{ // "appKey": "cloud", // "accountType": "01", // "userName": "{RSA}" + userRsa, // "password": "{RSA}" + passwordRsa, // "validateCode": vCodeRS, // "captchaToken": captchaToken, // "returnUrl": returnUrl, // "mailSuffix": "@pan.cn", // "paramId": paramId, // "clientType": "10010", // "dynamicCheck": "FALSE", // "cb_SaveName": "1", // "isOauth2": "false", // }).Post(url) // if err != nil { // return err // } // err = utils.Json.Unmarshal(res.Body(), &loginResp) // if err != nil { // log.Error(err.Error()) // return err // } // if loginResp.Result != 0 { // return fmt.Errorf(loginResp.Msg) // } // _, err = d.client.R().Get(loginResp.ToUrl) // return err //} func (d *Cloud189) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { var e Error req := d.client.R().SetError(&e). SetHeader("Accept", "application/json;charset=UTF-8"). SetQueryParams(map[string]string{ "noCache": random(), }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } res, err := req.Execute(method, url) if err != nil { return nil, err } // log.Debug(res.String()) if e.ErrorCode != "" { if e.ErrorCode == "InvalidSessionKey" { err = d.newLogin() if err != nil { return nil, err } return d.request(url, method, callback, resp) } } if jsoniter.Get(res.Body(), "res_code").ToInt() != 0 { err = errors.New(jsoniter.Get(res.Body(), "res_message").ToString()) } return res.Body(), err } func (d *Cloud189) getFiles(fileId string) ([]model.Obj, error) { res := make([]model.Obj, 0) pageNum := 1 for { var resp Files _, err := d.request("https://cloud.189.cn/api/open/file/listFiles.action", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(map[string]string{ //"noCache": random(), "pageSize": "60", "pageNum": strconv.Itoa(pageNum), "mediaType": "0", "folderId": fileId, "iconOption": "5", "orderBy": "lastOpTime", // account.OrderBy "descending": "true", // account.OrderDirection }) }, &resp) if err != nil { return nil, err } if resp.FileListAO.Count == 0 { break } for _, folder := range resp.FileListAO.FolderList { lastOpTime := utils.MustParseCNTime(folder.LastOpTime) res = append(res, &model.Object{ ID: strconv.FormatInt(folder.Id, 10), Name: folder.Name, Modified: lastOpTime, IsFolder: true, }) } for _, file := range resp.FileListAO.FileList { lastOpTime := utils.MustParseCNTime(file.LastOpTime) res = append(res, &model.ObjThumb{ Object: model.Object{ ID: strconv.FormatInt(file.Id, 10), Name: file.Name, Modified: lastOpTime, Size: file.Size, }, Thumbnail: model.Thumbnail{Thumbnail: file.Icon.SmallUrl}, }) } pageNum++ } return res, nil } func (d *Cloud189) oldUpload(dstDir model.Obj, file model.FileStreamer) error { res, err := d.client.R().SetMultipartFormData(map[string]string{ "parentId": dstDir.GetID(), "sessionKey": "??", "opertype": "1", "fname": file.GetName(), }).SetMultipartField("Filedata", file.GetName(), file.GetMimetype(), file).Post("https://hb02.upload.cloud.189.cn/v1/DCIWebUploadAction") if err != nil { return err } if utils.Json.Get(res.Body(), "MD5").ToString() != "" { return nil } log.Debugf(res.String()) return errors.New(res.String()) } func (d *Cloud189) getSessionKey() (string, error) { resp, err := d.request("https://cloud.189.cn/v2/getUserBriefInfo.action", http.MethodGet, nil, nil) if err != nil { return "", err } sessionKey := utils.Json.Get(resp, "sessionKey").ToString() return sessionKey, nil } func (d *Cloud189) getResKey() (string, string, error) { now := time.Now().UnixMilli() if d.rsa.Expire > now { return d.rsa.PubKey, d.rsa.PkId, nil } resp, err := d.request("https://cloud.189.cn/api/security/generateRsaKey.action", http.MethodGet, nil, nil) if err != nil { return "", "", err } pubKey, pkId := utils.Json.Get(resp, "pubKey").ToString(), utils.Json.Get(resp, "pkId").ToString() d.rsa.PubKey, d.rsa.PkId = pubKey, pkId d.rsa.Expire = utils.Json.Get(resp, "expire").ToInt64() return pubKey, pkId, nil } func (d *Cloud189) uploadRequest(uri string, form map[string]string, resp interface{}) ([]byte, error) { c := strconv.FormatInt(time.Now().UnixMilli(), 10) r := Random("xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx") l := Random("xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx") l = l[0 : 16+int(16*myrand.Rand.Float32())] e := qs(form) data := AesEncrypt([]byte(e), []byte(l[0:16])) h := hex.EncodeToString(data) sessionKey := d.sessionKey signature := hmacSha1(fmt.Sprintf("SessionKey=%s&Operate=GET&RequestURI=%s&Date=%s¶ms=%s", sessionKey, uri, c, h), l) pubKey, pkId, err := d.getResKey() if err != nil { return nil, err } b := RsaEncode([]byte(l), pubKey, false) req := d.client.R().SetHeaders(map[string]string{ "accept": "application/json;charset=UTF-8", "SessionKey": sessionKey, "Signature": signature, "X-Request-Date": c, "X-Request-ID": r, "EncryptionText": b, "PkId": pkId, }) if resp != nil { req.SetResult(resp) } res, err := req.Get("https://upload.cloud.189.cn" + uri + "?params=" + h) if err != nil { return nil, err } data = res.Body() if utils.Json.Get(data, "code").ToString() != "SUCCESS" { return nil, errors.New(uri + "---" + jsoniter.Get(data, "msg").ToString()) } return data, nil } func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { sessionKey, err := d.getSessionKey() if err != nil { return err } d.sessionKey = sessionKey const DEFAULT int64 = 10485760 count := int64(math.Ceil(float64(file.GetSize()) / float64(DEFAULT))) res, err := d.uploadRequest("/person/initMultiUpload", map[string]string{ "parentFolderId": dstDir.GetID(), "fileName": encode(file.GetName()), "fileSize": strconv.FormatInt(file.GetSize(), 10), "sliceSize": strconv.FormatInt(DEFAULT, 10), "lazyCheck": "1", }, nil) if err != nil { return err } uploadFileId := jsoniter.Get(res, "data", "uploadFileId").ToString() //_, err = d.uploadRequest("/person/getUploadedPartsInfo", map[string]string{ // "uploadFileId": uploadFileId, //}, nil) var finish int64 = 0 var i int64 var byteSize int64 md5s := make([]string, 0) md5Sum := md5.New() for i = 1; i <= count; i++ { if utils.IsCanceled(ctx) { return ctx.Err() } byteSize = file.GetSize() - finish if DEFAULT < byteSize { byteSize = DEFAULT } // log.Debugf("%d,%d", byteSize, finish) byteData := make([]byte, byteSize) n, err := io.ReadFull(file, byteData) // log.Debug(err, n) if err != nil { return err } finish += int64(n) md5Bytes := getMd5(byteData) md5Hex := hex.EncodeToString(md5Bytes) md5Base64 := base64.StdEncoding.EncodeToString(md5Bytes) md5s = append(md5s, strings.ToUpper(md5Hex)) md5Sum.Write(byteData) var resp UploadUrlsResp res, err = d.uploadRequest("/person/getMultiUploadUrls", map[string]string{ "partInfo": fmt.Sprintf("%s-%s", strconv.FormatInt(i, 10), md5Base64), "uploadFileId": uploadFileId, }, &resp) if err != nil { return err } uploadData := resp.UploadUrls["partNumber_"+strconv.FormatInt(i, 10)] log.Debugf("uploadData: %+v", uploadData) requestURL := uploadData.RequestURL uploadHeaders := strings.Split(decodeURIComponent(uploadData.RequestHeader), "&") req, err := http.NewRequestWithContext(ctx, http.MethodPut, requestURL, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) if err != nil { return err } for _, v := range uploadHeaders { i := strings.Index(v, "=") req.Header.Set(v[0:i], v[i+1:]) } r, err := base.HttpClient.Do(req) if err != nil { return err } log.Debugf("%+v %+v", r, r.Request.Header) _ = r.Body.Close() up(float64(i) * 100 / float64(count)) } fileMd5 := hex.EncodeToString(md5Sum.Sum(nil)) sliceMd5 := fileMd5 if file.GetSize() > DEFAULT { sliceMd5 = utils.GetMD5EncodeStr(strings.Join(md5s, "\n")) } res, err = d.uploadRequest("/person/commitMultiUploadFile", map[string]string{ "uploadFileId": uploadFileId, "fileMd5": fileMd5, "sliceMd5": sliceMd5, "lazyCheck": "1", "opertype": "3", }, nil) return err } func (d *Cloud189) getCapacityInfo(ctx context.Context) (*CapacityResp, error) { var resp CapacityResp _, err := d.request("https://cloud.189.cn/api/portal/getUserSizeInfo.action", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }, &resp) if err != nil { return nil, err } return &resp, nil } ================================================ FILE: drivers/189_tv/driver.go ================================================ package _189_tv import ( "context" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/go-resty/resty/v2" ) type Cloud189TV struct { model.Storage Addition client *resty.Client tokenInfo *AppSessionResp uploadThread int storageConfig driver.Config TempUuid string cron *cron.Cron // 新增 cron 字段 } func (y *Cloud189TV) Config() driver.Config { if y.storageConfig.Name == "" { y.storageConfig = config } return y.storageConfig } func (y *Cloud189TV) GetAddition() driver.Additional { return &y.Addition } func (y *Cloud189TV) Init(ctx context.Context) (err error) { // 兼容旧上传接口 y.storageConfig.NoOverwriteUpload = y.isFamily() && y.Addition.RapidUpload // 处理个人云和家庭云参数 if y.isFamily() && y.RootFolderID == "-11" { y.RootFolderID = "" } if !y.isFamily() && y.RootFolderID == "" { y.RootFolderID = "-11" } // 限制上传线程数 y.uploadThread, _ = strconv.Atoi(y.UploadThread) if y.uploadThread < 1 || y.uploadThread > 32 { y.uploadThread, y.UploadThread = 3, "3" } // 初始化请求客户端 if y.client == nil { y.client = base.NewRestyClient().SetHeaders( map[string]string{ "Accept": "application/json;charset=UTF-8", "User-Agent": "EcloudTV/6.5.5 (PJX110; unknown; home02) Android/35", }, ) } // 避免重复登陆 if !y.isLogin() || y.Addition.AccessToken == "" { if err = y.login(); err != nil { return err } } // 处理家庭云ID if y.FamilyID == "" { if y.FamilyID, err = y.getFamilyID(); err != nil { return err } } y.cron = cron.NewCron(time.Minute * 5) y.cron.Do(y.keepAlive) return err } func (y *Cloud189TV) Drop(ctx context.Context) error { if y.cron != nil { y.cron.Stop() y.cron = nil } return nil } func (y *Cloud189TV) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { return y.getFiles(ctx, dir.GetID(), y.isFamily()) } func (y *Cloud189TV) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var downloadUrl struct { URL string `json:"fileDownloadUrl"` } isFamily := y.isFamily() fullUrl := ApiUrl if isFamily { fullUrl += "/family/file" } fullUrl += "/getFileDownloadUrl.action" _, err := y.get(fullUrl, func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParam("fileId", file.GetID()) if isFamily { r.SetQueryParams(map[string]string{ "familyId": y.FamilyID, }) } else { r.SetQueryParams(map[string]string{ "dt": "3", "flag": "1", }) } }, &downloadUrl, isFamily) if err != nil { return nil, err } // 重定向获取真实链接 downloadUrl.URL = strings.Replace(strings.ReplaceAll(downloadUrl.URL, "&", "&"), "http://", "https://", 1) res, err := base.NoRedirectClient.R().SetContext(ctx).SetDoNotParseResponse(true).Get(downloadUrl.URL) if err != nil { return nil, err } defer res.RawBody().Close() if res.StatusCode() == 302 { downloadUrl.URL = res.Header().Get("location") } like := &model.Link{ URL: downloadUrl.URL, Header: http.Header{ "User-Agent": []string{base.UserAgent}, }, } return like, nil } func (y *Cloud189TV) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { isFamily := y.isFamily() fullUrl := ApiUrl if isFamily { fullUrl += "/family/file" } fullUrl += "/createFolder.action" var newFolder Cloud189Folder _, err := y.post(fullUrl, func(req *resty.Request) { req.SetContext(ctx) req.SetQueryParams(map[string]string{ "folderName": dirName, "relativePath": "", }) if isFamily { req.SetQueryParams(map[string]string{ "familyId": y.FamilyID, "parentId": parentDir.GetID(), }) } else { req.SetQueryParams(map[string]string{ "parentFolderId": parentDir.GetID(), }) } }, &newFolder, isFamily) if err != nil { return nil, err } return &newFolder, nil } func (y *Cloud189TV) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { isFamily := y.isFamily() other := map[string]string{"targetFileName": dstDir.GetName()} resp, err := y.CreateBatchTask("MOVE", IF(isFamily, y.FamilyID, ""), dstDir.GetID(), other, BatchTaskInfo{ FileId: srcObj.GetID(), FileName: srcObj.GetName(), IsFolder: BoolToNumber(srcObj.IsDir()), }) if err != nil { return nil, err } if err = y.WaitBatchTask("MOVE", resp.TaskID, time.Millisecond*400); err != nil { return nil, err } return srcObj, nil } func (y *Cloud189TV) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { isFamily := y.isFamily() queryParam := make(map[string]string) fullUrl := ApiUrl method := http.MethodPost if isFamily { fullUrl += "/family/file" method = http.MethodGet queryParam["familyId"] = y.FamilyID } var newObj model.Obj switch f := srcObj.(type) { case *Cloud189File: fullUrl += "/renameFile.action" queryParam["fileId"] = srcObj.GetID() queryParam["destFileName"] = newName newObj = &Cloud189File{Icon: f.Icon} // 复用预览 case *Cloud189Folder: fullUrl += "/renameFolder.action" queryParam["folderId"] = srcObj.GetID() queryParam["destFolderName"] = newName newObj = &Cloud189Folder{} default: return nil, errs.NotSupport } _, err := y.request(fullUrl, method, func(req *resty.Request) { req.SetContext(ctx).SetQueryParams(queryParam) }, nil, newObj, isFamily) if err != nil { return nil, err } return newObj, nil } func (y *Cloud189TV) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { isFamily := y.isFamily() other := map[string]string{"targetFileName": dstDir.GetName()} resp, err := y.CreateBatchTask("COPY", IF(isFamily, y.FamilyID, ""), dstDir.GetID(), other, BatchTaskInfo{ FileId: srcObj.GetID(), FileName: srcObj.GetName(), IsFolder: BoolToNumber(srcObj.IsDir()), }) if err != nil { return err } return y.WaitBatchTask("COPY", resp.TaskID, time.Second) } func (y *Cloud189TV) Remove(ctx context.Context, obj model.Obj) error { isFamily := y.isFamily() resp, err := y.CreateBatchTask("DELETE", IF(isFamily, y.FamilyID, ""), "", nil, BatchTaskInfo{ FileId: obj.GetID(), FileName: obj.GetName(), IsFolder: BoolToNumber(obj.IsDir()), }) if err != nil { return err } // 批量任务数量限制，过快会导致无法删除 return y.WaitBatchTask("DELETE", resp.TaskID, time.Millisecond*200) } func (y *Cloud189TV) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (newObj model.Obj, err error) { overwrite := true isFamily := y.isFamily() // 响应时间长,按需启用 if y.Addition.RapidUpload && !stream.IsForceStreamUpload() { if newObj, err := y.RapidUpload(ctx, dstDir, stream, isFamily, overwrite); err == nil { return newObj, nil } } return y.OldUpload(ctx, dstDir, stream, up, isFamily, overwrite) } func (y *Cloud189TV) GetDetails(ctx context.Context) (*model.StorageDetails, error) { capacityInfo, err := y.getCapacityInfo(ctx) if err != nil { return nil, err } var total, used int64 if y.isFamily() { total = capacityInfo.FamilyCapacityInfo.TotalSize used = capacityInfo.FamilyCapacityInfo.UsedSize } else { total = capacityInfo.CloudCapacityInfo.TotalSize used = capacityInfo.CloudCapacityInfo.UsedSize } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: used, }, }, nil } ================================================ FILE: drivers/189_tv/help.go ================================================ package _189_tv import ( "bytes" "crypto/hmac" "crypto/sha1" "encoding/hex" "encoding/xml" "fmt" "net/http" "regexp" "strings" "time" ) func clientSuffix() map[string]string { return map[string]string{ "clientType": AndroidTV, "version": TvVersion, "channelId": TvChannelId, "clientSn": "unknown", "model": "PJX110", "osFamily": "Android", "osVersion": "35", "networkAccessMode": "WIFI", "telecomsOperator": "46011", } } // SessionKeySignatureOfHmac HMAC签名 func SessionKeySignatureOfHmac(sessionSecret, sessionKey, operate, fullUrl, dateOfGmt string) string { urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(fullUrl)[1] mac := hmac.New(sha1.New, []byte(sessionSecret)) data := fmt.Sprintf("SessionKey=%s&Operate=%s&RequestURI=%s&Date=%s", sessionKey, operate, urlpath, dateOfGmt) mac.Write([]byte(data)) return strings.ToUpper(hex.EncodeToString(mac.Sum(nil))) } // AppKeySignatureOfHmac HMAC签名 func AppKeySignatureOfHmac(sessionSecret, appKey, operate, fullUrl string, timestamp int64) string { urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(fullUrl)[1] mac := hmac.New(sha1.New, []byte(sessionSecret)) data := fmt.Sprintf("AppKey=%s&Operate=%s&RequestURI=%s&Timestamp=%d", appKey, operate, urlpath, timestamp) mac.Write([]byte(data)) return strings.ToUpper(hex.EncodeToString(mac.Sum(nil))) } // 获取http规范的时间 func getHttpDateStr() string { return time.Now().UTC().Format(http.TimeFormat) } // 时间戳 func timestamp() int64 { return time.Now().UTC().UnixNano() / 1e6 } type Time time.Time func (t *Time) UnmarshalJSON(b []byte) error { return t.Unmarshal(b) } func (t *Time) UnmarshalXML(e *xml.Decoder, ee xml.StartElement) error { b, err := e.Token() if err != nil { return err } if b, ok := b.(xml.CharData); ok { if err = t.Unmarshal(b); err != nil { return err } } return e.Skip() } func (t *Time) Unmarshal(b []byte) error { bs := strings.Trim(string(b), "\"") var v time.Time var err error for _, f := range []string{"2006-01-02 15:04:05 -07", "Jan 2, 2006 15:04:05 PM -07"} { v, err = time.ParseInLocation(f, bs+" +08", time.Local) if err == nil { break } } *t = Time(v) return err } type String string func (t *String) UnmarshalJSON(b []byte) error { return t.Unmarshal(b) } func (t *String) UnmarshalXML(e *xml.Decoder, ee xml.StartElement) error { b, err := e.Token() if err != nil { return err } if b, ok := b.(xml.CharData); ok { if err = t.Unmarshal(b); err != nil { return err } } return e.Skip() } func (s *String) Unmarshal(b []byte) error { *s = String(bytes.Trim(b, "\"")) return nil } func toFamilyOrderBy(o string) string { switch o { case "filename": return "1" case "filesize": return "2" case "lastOpTime": return "3" default: return "1" } } func toDesc(o string) string { switch o { case "desc": return "true" case "asc": fallthrough default: return "false" } } func ParseHttpHeader(str string) map[string]string { header := make(map[string]string) for _, value := range strings.Split(str, "&") { if k, v, found := strings.Cut(value, "="); found { header[k] = v } } return header } func MustString(str string, err error) string { return str } func BoolToNumber(b bool) int { if b { return 1 } return 0 } func isBool(bs ...bool) bool { for _, b := range bs { if b { return true } } return false } func IF[V any](o bool, t V, f V) V { if o { return t } return f } ================================================ FILE: drivers/189_tv/meta.go ================================================ package _189_tv import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootID AccessToken string `json:"access_token"` OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` Type string `json:"type" type:"select" options:"personal,family" default:"personal"` FamilyID string `json:"family_id"` UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` RapidUpload bool `json:"rapid_upload"` } var config = driver.Config{ Name: "189CloudTV", DefaultRoot: "-11", CheckStatus: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Cloud189TV{} }) } ================================================ FILE: drivers/189_tv/types.go ================================================ package _189_tv import ( "encoding/xml" "fmt" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) // 居然有四种返回方式 type RespErr struct { ResCode any `json:"res_code"` // int or string ResMessage string `json:"res_message"` Error_ string `json:"error"` XMLName xml.Name `xml:"error"` Code string `json:"code" xml:"code"` Message string `json:"message" xml:"message"` Msg string `json:"msg"` ErrorCode string `json:"errorCode"` ErrorMsg string `json:"errorMsg"` } func (e *RespErr) HasError() bool { switch v := e.ResCode.(type) { case int, int64, int32: return v != 0 case string: return e.ResCode != "" } return (e.Code != "" && e.Code != "SUCCESS") || e.ErrorCode != "" || e.Error_ != "" } func (e *RespErr) Error() string { switch v := e.ResCode.(type) { case int, int64, int32: if v != 0 { return fmt.Sprintf("res_code: %d ,res_msg: %s", v, e.ResMessage) } case string: if e.ResCode != "" { return fmt.Sprintf("res_code: %s ,res_msg: %s", e.ResCode, e.ResMessage) } } if e.Code != "" && e.Code != "SUCCESS" { if e.Msg != "" { return fmt.Sprintf("code: %s ,msg: %s", e.Code, e.Msg) } if e.Message != "" { return fmt.Sprintf("code: %s ,msg: %s", e.Code, e.Message) } return "code: " + e.Code } if e.ErrorCode != "" { return fmt.Sprintf("err_code: %s ,err_msg: %s", e.ErrorCode, e.ErrorMsg) } if e.Error_ != "" { return fmt.Sprintf("error: %s ,message: %s", e.ErrorCode, e.Message) } return "" } // 刷新session返回 type UserSessionResp struct { ResCode int `json:"res_code"` ResMessage string `json:"res_message"` LoginName string `json:"loginName"` KeepAlive int `json:"keepAlive"` GetFileDiffSpan int `json:"getFileDiffSpan"` GetUserInfoSpan int `json:"getUserInfoSpan"` // 个人云 SessionKey string `json:"sessionKey"` SessionSecret string `json:"sessionSecret"` // 家庭云 FamilySessionKey string `json:"familySessionKey"` FamilySessionSecret string `json:"familySessionSecret"` } type UuidInfoResp struct { Uuid string `json:"uuid"` } type E189AccessTokenResp struct { E189AccessToken string `json:"accessToken"` ExpiresIn int64 `json:"expiresIn"` } // 登录返回 type AppSessionResp struct { UserSessionResp IsSaveName string `json:"isSaveName"` // 会话刷新Token AccessToken string `json:"accessToken"` //Token刷新 RefreshToken string `json:"refreshToken"` } // 家庭云账户 type FamilyInfoListResp struct { FamilyInfoResp []FamilyInfoResp `json:"familyInfoResp"` } type FamilyInfoResp struct { Count int `json:"count"` CreateTime string `json:"createTime"` FamilyID int64 `json:"familyId"` RemarkName string `json:"remarkName"` Type int `json:"type"` UseFlag int `json:"useFlag"` UserRole int `json:"userRole"` } /*文件部分*/ // 文件 type Cloud189File struct { ID String `json:"id"` Name string `json:"name"` Size int64 `json:"size"` Md5 string `json:"md5"` LastOpTime Time `json:"lastOpTime"` CreateDate Time `json:"createDate"` Icon struct { //iconOption 5 SmallUrl string `json:"smallUrl"` LargeUrl string `json:"largeUrl"` // iconOption 10 Max600 string `json:"max600"` MediumURL string `json:"mediumUrl"` } `json:"icon"` // Orientation int64 `json:"orientation"` // FileCata int64 `json:"fileCata"` // MediaType int `json:"mediaType"` // Rev string `json:"rev"` // StarLabel int64 `json:"starLabel"` } func (c *Cloud189File) CreateTime() time.Time { return time.Time(c.CreateDate) } func (c *Cloud189File) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.MD5, c.Md5) } func (c *Cloud189File) GetSize() int64 { return c.Size } func (c *Cloud189File) GetName() string { return c.Name } func (c *Cloud189File) ModTime() time.Time { return time.Time(c.LastOpTime) } func (c *Cloud189File) IsDir() bool { return false } func (c *Cloud189File) GetID() string { return string(c.ID) } func (c *Cloud189File) GetPath() string { return "" } func (c *Cloud189File) Thumb() string { return c.Icon.SmallUrl } // 文件夹 type Cloud189Folder struct { ID String `json:"id"` ParentID int64 `json:"parentId"` Name string `json:"name"` LastOpTime Time `json:"lastOpTime"` CreateDate Time `json:"createDate"` // FileListSize int64 `json:"fileListSize"` // FileCount int64 `json:"fileCount"` // FileCata int64 `json:"fileCata"` // Rev string `json:"rev"` // StarLabel int64 `json:"starLabel"` } func (c *Cloud189Folder) CreateTime() time.Time { return time.Time(c.CreateDate) } func (c *Cloud189Folder) GetHash() utils.HashInfo { return utils.HashInfo{} } func (c *Cloud189Folder) GetSize() int64 { return 0 } func (c *Cloud189Folder) GetName() string { return c.Name } func (c *Cloud189Folder) ModTime() time.Time { return time.Time(c.LastOpTime) } func (c *Cloud189Folder) IsDir() bool { return true } func (c *Cloud189Folder) GetID() string { return string(c.ID) } func (c *Cloud189Folder) GetPath() string { return "" } type Cloud189FilesResp struct { //ResCode int `json:"res_code"` //ResMessage string `json:"res_message"` FileListAO struct { Count int `json:"count"` FileList []Cloud189File `json:"fileList"` FolderList []Cloud189Folder `json:"folderList"` } `json:"fileListAO"` } // TaskInfo 任务信息 type BatchTaskInfo struct { // FileId 文件ID FileId string `json:"fileId"` // FileName 文件名 FileName string `json:"fileName"` // IsFolder 是否是文件夹，0-否，1-是 IsFolder int `json:"isFolder"` // SrcParentId 文件所在父目录ID SrcParentId string `json:"srcParentId,omitempty"` /* 冲突管理 */ // 1 -> 跳过 2 -> 保留 3 -> 覆盖 DealWay int `json:"dealWay,omitempty"` IsConflict int `json:"isConflict,omitempty"` } /* 上传部分 */ type InitMultiUploadResp struct { //Code string `json:"code"` Data struct { UploadType int `json:"uploadType"` UploadHost string `json:"uploadHost"` UploadFileID string `json:"uploadFileId"` FileDataExists int `json:"fileDataExists"` } `json:"data"` } type UploadUrlsResp struct { Code string `json:"code"` Data map[string]UploadUrlsData `json:"uploadUrls"` } type UploadUrlsData struct { RequestURL string `json:"requestURL"` RequestHeader string `json:"requestHeader"` } /* 第二种上传方式 */ type CreateUploadFileResp struct { // 上传文件请求ID UploadFileId int64 `json:"uploadFileId"` // 上传文件数据的URL路径 FileUploadUrl string `json:"fileUploadUrl"` // 上传文件完成后确认路径 FileCommitUrl string `json:"fileCommitUrl"` // 文件是否已存在云盘中，0-未存在，1-已存在 FileDataExists int `json:"fileDataExists"` } type GetUploadFileStatusResp struct { CreateUploadFileResp // 已上传的大小 DataSize int64 `json:"dataSize"` Size int64 `json:"size"` } func (r *GetUploadFileStatusResp) GetSize() int64 { return r.DataSize + r.Size } type CommitMultiUploadFileResp struct { File struct { UserFileID String `json:"userFileId"` FileName string `json:"fileName"` FileSize int64 `json:"fileSize"` FileMd5 string `json:"fileMd5"` CreateDate Time `json:"createDate"` } `json:"file"` } type OldCommitUploadFileResp struct { XMLName xml.Name `xml:"file"` ID String `xml:"id"` Name string `xml:"name"` Size int64 `xml:"size"` Md5 string `xml:"md5"` CreateDate Time `xml:"createDate"` } func (f *OldCommitUploadFileResp) toFile() *Cloud189File { return &Cloud189File{ ID: f.ID, Name: f.Name, Size: f.Size, Md5: f.Md5, CreateDate: f.CreateDate, LastOpTime: f.CreateDate, } } type CreateBatchTaskResp struct { TaskID string `json:"taskId"` } type BatchTaskStateResp struct { FailedCount int `json:"failedCount"` Process int `json:"process"` SkipCount int `json:"skipCount"` SubTaskCount int `json:"subTaskCount"` SuccessedCount int `json:"successedCount"` SuccessedFileIDList []int64 `json:"successedFileIdList"` TaskID string `json:"taskId"` TaskStatus int `json:"taskStatus"` //1 初始化 2 存在冲突 3 执行中，4 完成 } type BatchTaskConflictTaskInfoResp struct { SessionKey string `json:"sessionKey"` TargetFolderID int `json:"targetFolderId"` TaskID string `json:"taskId"` TaskInfos []BatchTaskInfo TaskType int `json:"taskType"` } type CapacityResp struct { ResCode int `json:"res_code"` ResMessage string `json:"res_message"` Account string `json:"account"` CloudCapacityInfo struct { FreeSize int64 `json:"freeSize"` MailUsedSize int64 `json:"mail189UsedSize"` TotalSize int64 `json:"totalSize"` UsedSize int64 `json:"usedSize"` } `json:"cloudCapacityInfo"` FamilyCapacityInfo struct { FreeSize int64 `json:"freeSize"` TotalSize int64 `json:"totalSize"` UsedSize int64 `json:"usedSize"` } `json:"familyCapacityInfo"` TotalSize uint64 `json:"totalSize"` } ================================================ FILE: drivers/189_tv/utils.go ================================================ package _189_tv import ( "context" "encoding/base64" "encoding/xml" "fmt" "io" "net/http" "strconv" "strings" "time" "github.com/skip2/go-qrcode" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" "github.com/google/uuid" jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" ) const ( TVAppKey = "600100885" TVAppSignatureSecre = "fe5734c74c2f96a38157f420b32dc995" TvVersion = "6.5.5" AndroidTV = "FAMILY_TV" TvChannelId = "home02" ApiUrl = "https://api.cloud.189.cn" ) func (y *Cloud189TV) SignatureHeader(url, method string, isFamily bool) map[string]string { dateOfGmt := getHttpDateStr() sessionKey := y.tokenInfo.SessionKey sessionSecret := y.tokenInfo.SessionSecret if isFamily { sessionKey = y.tokenInfo.FamilySessionKey sessionSecret = y.tokenInfo.FamilySessionSecret } header := map[string]string{ "Date": dateOfGmt, "SessionKey": sessionKey, "X-Request-ID": uuid.NewString(), "Signature": SessionKeySignatureOfHmac(sessionSecret, sessionKey, method, url, dateOfGmt), } return header } func (y *Cloud189TV) AppKeySignatureHeader(url, method string) map[string]string { tempTime := timestamp() header := map[string]string{ "Timestamp": strconv.FormatInt(tempTime, 10), "X-Request-ID": uuid.NewString(), "AppKey": TVAppKey, "AppSignature": AppKeySignatureOfHmac(TVAppSignatureSecre, TVAppKey, method, url, tempTime), } return header } func (y *Cloud189TV) request(url, method string, callback base.ReqCallback, params map[string]string, resp interface{}, isFamily ...bool) ([]byte, error) { return y.requestWithRetry(url, method, callback, params, resp, 0, isFamily...) } func (y *Cloud189TV) requestWithRetry(url, method string, callback base.ReqCallback, params map[string]string, resp interface{}, retryCount int, isFamily ...bool) ([]byte, error) { if y.tokenInfo == nil { return nil, fmt.Errorf("login failed") } req := y.client.R().SetQueryParams(clientSuffix()) if params != nil { req.SetQueryParams(params) } // Signature req.SetHeaders(y.SignatureHeader(url, method, isBool(isFamily...))) var erron RespErr req.SetError(&erron) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } res, err := req.Execute(method, url) if err != nil { return nil, err } if strings.Contains(res.String(), "userSessionBO is null") || strings.Contains(res.String(), "InvalidSessionKey") { // 限制重试次数，避免无限递归 if retryCount >= 3 { y.Addition.AccessToken = "" op.MustSaveDriverStorage(y) return nil, errors.New("session expired after retry") } // 尝试刷新会话 if err := y.refreshSession(); err != nil { // 如果刷新失败，说明AccessToken也已过期，需要重新登录 y.Addition.AccessToken = "" op.MustSaveDriverStorage(y) return nil, errors.New("session expired") } // 如果刷新成功，则重试原始请求（增加重试计数） return y.requestWithRetry(url, method, callback, params, resp, retryCount+1, isFamily...) } // 处理错误 if erron.HasError() { return nil, &erron } return res.Body(), nil } func (y *Cloud189TV) get(url string, callback base.ReqCallback, resp interface{}, isFamily ...bool) ([]byte, error) { return y.request(url, http.MethodGet, callback, nil, resp, isFamily...) } func (y *Cloud189TV) post(url string, callback base.ReqCallback, resp interface{}, isFamily ...bool) ([]byte, error) { return y.request(url, http.MethodPost, callback, nil, resp, isFamily...) } func (y *Cloud189TV) put(ctx context.Context, url string, headers map[string]string, sign bool, file io.Reader, isFamily bool) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, file) if err != nil { return nil, err } query := req.URL.Query() for key, value := range clientSuffix() { query.Add(key, value) } req.URL.RawQuery = query.Encode() for key, value := range headers { req.Header.Add(key, value) } if sign { for key, value := range y.SignatureHeader(url, http.MethodPut, isFamily) { req.Header.Add(key, value) } } // 请求完成后http.Client会Close Request.Body resp, err := base.HttpClient.Do(req) if err != nil { return nil, err } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } var erron RespErr jsoniter.Unmarshal(body, &erron) xml.Unmarshal(body, &erron) if erron.HasError() { return nil, &erron } if resp.StatusCode != http.StatusOK { return nil, errors.Errorf("put fail,err:%s", string(body)) } return body, nil } func (y *Cloud189TV) getFiles(ctx context.Context, fileId string, isFamily bool) ([]model.Obj, error) { fullUrl := ApiUrl if isFamily { fullUrl += "/family/file" } fullUrl += "/listFiles.action" res := make([]model.Obj, 0, 130) for pageNum := 1; ; pageNum++ { var resp Cloud189FilesResp _, err := y.get(fullUrl, func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(map[string]string{ "folderId": fileId, "fileType": "0", "mediaAttr": "0", "iconOption": "5", "pageNum": fmt.Sprint(pageNum), "pageSize": "130", }) if isFamily { r.SetQueryParams(map[string]string{ "familyId": y.FamilyID, "orderBy": toFamilyOrderBy(y.OrderBy), "descending": toDesc(y.OrderDirection), }) } else { r.SetQueryParams(map[string]string{ "recursive": "0", "orderBy": y.OrderBy, "descending": toDesc(y.OrderDirection), }) } }, &resp, isFamily) if err != nil { return nil, err } // 获取完毕跳出 if resp.FileListAO.Count == 0 { break } for i := 0; i < len(resp.FileListAO.FolderList); i++ { res = append(res, &resp.FileListAO.FolderList[i]) } for i := 0; i < len(resp.FileListAO.FileList); i++ { res = append(res, &resp.FileListAO.FileList[i]) } } return res, nil } func (y *Cloud189TV) login() (err error) { req := y.client.R().SetQueryParams(clientSuffix()) var erron RespErr var tokenInfo AppSessionResp if y.Addition.AccessToken == "" { if y.TempUuid == "" { // 获取登录参数 var uuidInfo UuidInfoResp req.SetResult(&uuidInfo).SetError(&erron) // Signature req.SetHeaders(y.AppKeySignatureHeader(ApiUrl+"/family/manage/getQrCodeUUID.action", http.MethodGet)) _, err = req.Execute(http.MethodGet, ApiUrl+"/family/manage/getQrCodeUUID.action") if err != nil { return err } if erron.HasError() { return &erron } if uuidInfo.Uuid == "" { return errors.New("uuidInfo is empty") } y.TempUuid = uuidInfo.Uuid op.MustSaveDriverStorage(y) // 展示二维码 qrTemplate := `
Or Click here: %s ` // Generate QR code qrCode, err := qrcode.Encode(uuidInfo.Uuid, qrcode.Medium, 256) if err != nil { return fmt.Errorf("failed to generate QR code: %v", err) } // Encode QR code to base64 qrCodeBase64 := base64.StdEncoding.EncodeToString(qrCode) // Create the HTML page qrPage := fmt.Sprintf(qrTemplate, qrCodeBase64, uuidInfo.Uuid, uuidInfo.Uuid) return fmt.Errorf("need verify: \n%s", qrPage) } else { var accessTokenResp E189AccessTokenResp req.SetResult(&accessTokenResp).SetError(&erron) // Signature req.SetHeaders(y.AppKeySignatureHeader(ApiUrl+"/family/manage/qrcodeLoginResult.action", http.MethodGet)) req.SetQueryParam("uuid", y.TempUuid) _, err = req.Execute(http.MethodGet, ApiUrl+"/family/manage/qrcodeLoginResult.action") if err != nil { return err } if erron.HasError() { return &erron } if accessTokenResp.E189AccessToken == "" { return errors.New("E189AccessToken is empty") } y.Addition.AccessToken = accessTokenResp.E189AccessToken } } // 获取SessionKey 和 SessionSecret reqb := y.client.R().SetQueryParams(clientSuffix()) reqb.SetResult(&tokenInfo).SetError(&erron) // Signature reqb.SetHeaders(y.AppKeySignatureHeader(ApiUrl+"/family/manage/loginFamilyMerge.action", http.MethodGet)) reqb.SetQueryParam("e189AccessToken", y.Addition.AccessToken) _, err = reqb.Execute(http.MethodGet, ApiUrl+"/family/manage/loginFamilyMerge.action") if err != nil { return err } if erron.HasError() { return &erron } y.tokenInfo = &tokenInfo op.MustSaveDriverStorage(y) return err } // refreshSession 尝试使用现有的 AccessToken 刷新会话 func (y *Cloud189TV) refreshSession() (err error) { var erron RespErr var tokenInfo AppSessionResp reqb := y.client.R().SetQueryParams(clientSuffix()) reqb.SetResult(&tokenInfo).SetError(&erron) // Signature reqb.SetHeaders(y.AppKeySignatureHeader(ApiUrl+"/family/manage/loginFamilyMerge.action", http.MethodGet)) reqb.SetQueryParam("e189AccessToken", y.Addition.AccessToken) _, err = reqb.Execute(http.MethodGet, ApiUrl+"/family/manage/loginFamilyMerge.action") if err != nil { return err } if erron.HasError() { return &erron } y.tokenInfo = &tokenInfo return nil } func (y *Cloud189TV) keepAlive() { _, err := y.get(ApiUrl+"/keepUserSession.action", func(r *resty.Request) { r.SetQueryParams(clientSuffix()) }, nil) if err != nil { utils.Log.Warnf("189tv: Failed to keep user session alive: %v", err) // 如果keepAlive失败，尝试刷新session if refreshErr := y.refreshSession(); refreshErr != nil { utils.Log.Errorf("189tv: Failed to refresh session after keepAlive error: %v", refreshErr) } } else { utils.Log.Debugf("189tv: User session kept alive successfully.") } } func (y *Cloud189TV) RapidUpload(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, isFamily bool, overwrite bool) (model.Obj, error) { fileMd5 := stream.GetHash().GetHash(utils.MD5) if len(fileMd5) < utils.MD5.Width { return nil, errors.New("invalid hash") } uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, stream.GetName(), fmt.Sprint(stream.GetSize()), isFamily) if err != nil { return nil, err } if uploadInfo.FileDataExists != 1 { return nil, errors.New("rapid upload fail") } return y.OldUploadCommit(ctx, uploadInfo.FileCommitUrl, uploadInfo.UploadFileId, isFamily, overwrite) } // 旧版本上传，家庭云不支持覆盖 func (y *Cloud189TV) OldUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { fileMd5 := file.GetHash().GetHash(utils.MD5) tempFile := file.GetFile() var err error if len(fileMd5) != utils.MD5.Width { tempFile, fileMd5, err = stream.CacheFullAndHash(file, &up, utils.MD5) } else if tempFile == nil { tempFile, err = file.CacheFullAndWriter(&up, nil) } if err != nil { return nil, err } // 创建上传会话 uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, file.GetName(), fmt.Sprint(file.GetSize()), isFamily) if err != nil { return nil, err } // 网盘中不存在该文件，开始上传 status := GetUploadFileStatusResp{CreateUploadFileResp: *uploadInfo} // driver.RateLimitReader会尝试Close底层的reader // 但这里的tempFile是一个*os.File，Close后就没法继续读了 // 所以这里用io.NopCloser包一层 rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.NopCloser(tempFile)) for status.GetSize() < file.GetSize() && status.FileDataExists != 1 { if utils.IsCanceled(ctx) { return nil, ctx.Err() } header := map[string]string{ "ResumePolicy": "1", "Expect": "100-continue", } if isFamily { header["FamilyId"] = fmt.Sprint(y.FamilyID) header["UploadFileId"] = fmt.Sprint(status.UploadFileId) } else { header["Edrive-UploadFileId"] = fmt.Sprint(status.UploadFileId) } _, err := y.put(ctx, status.FileUploadUrl, header, true, rateLimitedRd, isFamily) if err, ok := err.(*RespErr); ok && err.Code != "InputStreamReadError" { return nil, err } // 获取断点状态 fullUrl := ApiUrl + "/getUploadFileStatus.action" if y.isFamily() { fullUrl = ApiUrl + "/family/file/getFamilyFileStatus.action" } _, err = y.get(fullUrl, func(req *resty.Request) { req.SetContext(ctx).SetQueryParams(map[string]string{ "uploadFileId": fmt.Sprint(status.UploadFileId), "resumePolicy": "1", }) if isFamily { req.SetQueryParam("familyId", fmt.Sprint(y.FamilyID)) } }, &status, isFamily) if err != nil { return nil, err } if _, err := tempFile.Seek(status.GetSize(), io.SeekStart); err != nil { return nil, err } up(float64(status.GetSize()) / float64(file.GetSize()) * 100) } return y.OldUploadCommit(ctx, status.FileCommitUrl, status.UploadFileId, isFamily, overwrite) } // 创建上传会话 func (y *Cloud189TV) OldUploadCreate(ctx context.Context, parentID string, fileMd5, fileName, fileSize string, isFamily bool) (*CreateUploadFileResp, error) { var uploadInfo CreateUploadFileResp fullUrl := ApiUrl + "/createUploadFile.action" if isFamily { fullUrl = ApiUrl + "/family/file/createFamilyFile.action" } _, err := y.post(fullUrl, func(req *resty.Request) { req.SetContext(ctx) if isFamily { req.SetQueryParams(map[string]string{ "familyId": y.FamilyID, "parentId": parentID, "fileMd5": fileMd5, "fileName": fileName, "fileSize": fileSize, "resumePolicy": "1", }) } else { req.SetFormData(map[string]string{ "parentFolderId": parentID, "fileName": fileName, "size": fileSize, "md5": fileMd5, "opertype": "3", "flag": "1", "resumePolicy": "1", "isLog": "0", }) } }, &uploadInfo, isFamily) if err != nil { return nil, err } return &uploadInfo, nil } // 提交上传文件 func (y *Cloud189TV) OldUploadCommit(ctx context.Context, fileCommitUrl string, uploadFileID int64, isFamily bool, overwrite bool) (model.Obj, error) { var resp OldCommitUploadFileResp _, err := y.post(fileCommitUrl, func(req *resty.Request) { req.SetContext(ctx) if isFamily { req.SetHeaders(map[string]string{ "ResumePolicy": "1", "UploadFileId": fmt.Sprint(uploadFileID), "FamilyId": fmt.Sprint(y.FamilyID), }) } else { req.SetFormData(map[string]string{ "opertype": IF(overwrite, "3", "1"), "resumePolicy": "1", "uploadFileId": fmt.Sprint(uploadFileID), "isLog": "0", }) } }, &resp, isFamily) if err != nil { return nil, err } return resp.toFile(), nil } func (y *Cloud189TV) isFamily() bool { return y.Type == "family" } func (y *Cloud189TV) isLogin() bool { if y.tokenInfo == nil { return false } _, err := y.get(ApiUrl+"/getUserInfo.action", nil, nil) return err == nil } // 获取家庭云所有用户信息 func (y *Cloud189TV) getFamilyInfoList() ([]FamilyInfoResp, error) { var resp FamilyInfoListResp _, err := y.get(ApiUrl+"/family/manage/getFamilyList.action", nil, &resp, true) if err != nil { return nil, err } return resp.FamilyInfoResp, nil } // 抽取家庭云ID func (y *Cloud189TV) getFamilyID() (string, error) { infos, err := y.getFamilyInfoList() if err != nil { return "", err } if len(infos) == 0 { return "", fmt.Errorf("cannot get automatically,please input family_id") } for _, info := range infos { if strings.Contains(y.tokenInfo.LoginName, info.RemarkName) { return fmt.Sprint(info.FamilyID), nil } } return fmt.Sprint(infos[0].FamilyID), nil } func (y *Cloud189TV) CreateBatchTask(aType string, familyID string, targetFolderId string, other map[string]string, taskInfos ...BatchTaskInfo) (*CreateBatchTaskResp, error) { var resp CreateBatchTaskResp _, err := y.post(ApiUrl+"/batch/createBatchTask.action", func(req *resty.Request) { req.SetFormData(map[string]string{ "type": aType, "taskInfos": MustString(utils.Json.MarshalToString(taskInfos)), }) if targetFolderId != "" { req.SetFormData(map[string]string{"targetFolderId": targetFolderId}) } if familyID != "" { req.SetFormData(map[string]string{"familyId": familyID}) } req.SetFormData(other) }, &resp, familyID != "") if err != nil { return nil, err } return &resp, nil } // 检测任务状态 func (y *Cloud189TV) CheckBatchTask(aType string, taskID string) (*BatchTaskStateResp, error) { var resp BatchTaskStateResp _, err := y.post(ApiUrl+"/batch/checkBatchTask.action", func(req *resty.Request) { req.SetFormData(map[string]string{ "type": aType, "taskId": taskID, }) }, &resp) if err != nil { return nil, err } return &resp, nil } // 获取冲突的任务信息 func (y *Cloud189TV) GetConflictTaskInfo(aType string, taskID string) (*BatchTaskConflictTaskInfoResp, error) { var resp BatchTaskConflictTaskInfoResp _, err := y.post(ApiUrl+"/batch/getConflictTaskInfo.action", func(req *resty.Request) { req.SetFormData(map[string]string{ "type": aType, "taskId": taskID, }) }, &resp) if err != nil { return nil, err } return &resp, nil } // 处理冲突 func (y *Cloud189TV) ManageBatchTask(aType string, taskID string, targetFolderId string, taskInfos ...BatchTaskInfo) error { _, err := y.post(ApiUrl+"/batch/manageBatchTask.action", func(req *resty.Request) { req.SetFormData(map[string]string{ "targetFolderId": targetFolderId, "type": aType, "taskId": taskID, "taskInfos": MustString(utils.Json.MarshalToString(taskInfos)), }) }, nil) return err } var ErrIsConflict = errors.New("there is a conflict with the target object") // 等待任务完成 func (y *Cloud189TV) WaitBatchTask(aType string, taskID string, t time.Duration) error { for { state, err := y.CheckBatchTask(aType, taskID) if err != nil { return err } switch state.TaskStatus { case 2: return ErrIsConflict case 4: return nil } time.Sleep(t) } } func (y *Cloud189TV) getCapacityInfo(ctx context.Context) (*CapacityResp, error) { fullUrl := ApiUrl + "/portal/getUserSizeInfo.action" var resp CapacityResp _, err := y.get(fullUrl, func(req *resty.Request) { req.SetContext(ctx) }, &resp) if err != nil { return nil, err } return &resp, nil } ================================================ FILE: drivers/189pc/driver.go ================================================ package _189pc import ( "context" "fmt" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" "github.com/google/uuid" ) type Cloud189PC struct { model.Storage Addition client *resty.Client loginParam *LoginParam qrcodeParam *QRLoginParam tokenInfo *AppSessionResp uploadThread int familyTransferFolder *Cloud189Folder cleanFamilyTransferFile func() storageConfig driver.Config ref *Cloud189PC cron *cron.Cron } func (y *Cloud189PC) Config() driver.Config { if y.storageConfig.Name == "" { y.storageConfig = config } return y.storageConfig } func (y *Cloud189PC) GetAddition() driver.Additional { return &y.Addition } func (y *Cloud189PC) Init(ctx context.Context) (err error) { y.storageConfig = config if y.isFamily() { // 兼容旧上传接口 if y.Addition.RapidUpload || y.Addition.UploadMethod == "old" { y.storageConfig.NoOverwriteUpload = true } } else { // 家庭云转存，不支持覆盖上传 if y.Addition.FamilyTransfer { y.storageConfig.NoOverwriteUpload = true } } // 处理个人云和家庭云参数 if y.isFamily() && y.RootFolderID == "-11" { y.RootFolderID = "" } if !y.isFamily() && y.RootFolderID == "" { y.RootFolderID = "-11" } // 限制上传线程数 y.uploadThread, _ = strconv.Atoi(y.UploadThread) if y.uploadThread < 1 || y.uploadThread > 32 { y.uploadThread, y.UploadThread = 3, "3" } if y.ref == nil { // 初始化请求客户端 if y.client == nil { y.client = base.NewRestyClient().SetHeaders(map[string]string{ "Accept": "application/json;charset=UTF-8", "Referer": WEB_URL, }) } // 先尝试用Token刷新，之后尝试登陆 if y.Addition.RefreshToken != "" { y.tokenInfo = &AppSessionResp{RefreshToken: y.Addition.RefreshToken} if err = y.refreshToken(); err != nil { return err } } else { if err = y.login(); err != nil { return err } } // 初始化并启动 cron 任务 y.cron = cron.NewCron(time.Duration(time.Minute * 5)) // 每5分钟执行一次 keepAlive y.cron.Do(y.keepAlive) } // 处理家庭云ID if y.FamilyID == "" { if y.FamilyID, err = y.getFamilyID(); err != nil { return err } } // 创建中转文件夹 if y.FamilyTransfer { if err := y.createFamilyTransferFolder(); err != nil { return err } } // 清理转存文件节流 y.cleanFamilyTransferFile = utils.NewThrottle2(time.Minute, func() { if err := y.cleanFamilyTransfer(context.TODO()); err != nil { utils.Log.Errorf("cleanFamilyTransferFolderError:%s", err) } }) return err } func (d *Cloud189PC) InitReference(storage driver.Driver) error { refStorage, ok := storage.(*Cloud189PC) if ok { d.ref = refStorage return nil } return errs.NotSupport } func (y *Cloud189PC) Drop(ctx context.Context) error { y.ref = nil if y.cron != nil { y.cron.Stop() y.cron = nil } return nil } func (y *Cloud189PC) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { return y.getFiles(ctx, dir.GetID(), y.isFamily()) } func (y *Cloud189PC) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var downloadUrl struct { URL string `json:"fileDownloadUrl"` } isFamily := y.isFamily() fullUrl := API_URL if isFamily { fullUrl += "/family/file" } fullUrl += "/getFileDownloadUrl.action" _, err := y.get(fullUrl, func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParam("fileId", file.GetID()) if isFamily { r.SetQueryParams(map[string]string{ "familyId": y.FamilyID, }) } else { r.SetQueryParams(map[string]string{ "dt": "3", "flag": "1", }) } }, &downloadUrl, isFamily) if err != nil { return nil, err } // 重定向获取真实链接 downloadUrl.URL = strings.Replace(strings.ReplaceAll(downloadUrl.URL, "&", "&"), "http://", "https://", 1) res, err := base.NoRedirectClient.R().SetContext(ctx).SetDoNotParseResponse(true).Get(downloadUrl.URL) if err != nil { return nil, err } defer res.RawBody().Close() if res.StatusCode() == 302 { downloadUrl.URL = res.Header().Get("location") } like := &model.Link{ URL: downloadUrl.URL, Header: http.Header{ "User-Agent": []string{base.UserAgent}, }, } /* // 获取链接有效时常 strs := regexp.MustCompile(`(?i)expire[^=]*=([0-9]*)`).FindStringSubmatch(downloadUrl.URL) if len(strs) == 2 { timestamp, err := strconv.ParseInt(strs[1], 10, 64) if err == nil { expired := time.Duration(timestamp-time.Now().Unix()) * time.Second like.Expiration = &expired } } */ return like, nil } func (y *Cloud189PC) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { isFamily := y.isFamily() fullUrl := API_URL if isFamily { fullUrl += "/family/file" } fullUrl += "/createFolder.action" var newFolder Cloud189Folder _, err := y.post(fullUrl, func(req *resty.Request) { req.SetContext(ctx) req.SetQueryParams(map[string]string{ "folderName": dirName, "relativePath": "", }) if isFamily { req.SetQueryParams(map[string]string{ "familyId": y.FamilyID, "parentId": parentDir.GetID(), }) } else { req.SetQueryParams(map[string]string{ "parentFolderId": parentDir.GetID(), }) } }, &newFolder, isFamily) if err != nil { return nil, err } return &newFolder, nil } func (y *Cloud189PC) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { isFamily := y.isFamily() other := map[string]string{"targetFileName": dstDir.GetName()} resp, err := y.CreateBatchTask("MOVE", IF(isFamily, y.FamilyID, ""), dstDir.GetID(), other, BatchTaskInfo{ FileId: srcObj.GetID(), FileName: srcObj.GetName(), IsFolder: BoolToNumber(srcObj.IsDir()), }) if err != nil { return nil, err } if err = y.WaitBatchTask("MOVE", resp.TaskID, time.Millisecond*400); err != nil { return nil, err } return srcObj, nil } func (y *Cloud189PC) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { isFamily := y.isFamily() queryParam := make(map[string]string) fullUrl := API_URL method := http.MethodPost if isFamily { fullUrl += "/family/file" method = http.MethodGet queryParam["familyId"] = y.FamilyID } var newObj model.Obj switch f := srcObj.(type) { case *Cloud189File: fullUrl += "/renameFile.action" queryParam["fileId"] = srcObj.GetID() queryParam["destFileName"] = newName newObj = &Cloud189File{Icon: f.Icon} // 复用预览 case *Cloud189Folder: fullUrl += "/renameFolder.action" queryParam["folderId"] = srcObj.GetID() queryParam["destFolderName"] = newName newObj = &Cloud189Folder{} default: return nil, errs.NotSupport } _, err := y.request(fullUrl, method, func(req *resty.Request) { req.SetContext(ctx).SetQueryParams(queryParam) }, nil, newObj, isFamily) if err != nil { return nil, err } return newObj, nil } func (y *Cloud189PC) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { isFamily := y.isFamily() other := map[string]string{"targetFileName": dstDir.GetName()} resp, err := y.CreateBatchTask("COPY", IF(isFamily, y.FamilyID, ""), dstDir.GetID(), other, BatchTaskInfo{ FileId: srcObj.GetID(), FileName: srcObj.GetName(), IsFolder: BoolToNumber(srcObj.IsDir()), }) if err != nil { return err } return y.WaitBatchTask("COPY", resp.TaskID, time.Second) } func (y *Cloud189PC) Remove(ctx context.Context, obj model.Obj) error { isFamily := y.isFamily() resp, err := y.CreateBatchTask("DELETE", IF(isFamily, y.FamilyID, ""), "", nil, BatchTaskInfo{ FileId: obj.GetID(), FileName: obj.GetName(), IsFolder: BoolToNumber(obj.IsDir()), }) if err != nil { return err } // 批量任务数量限制，过快会导致无法删除 return y.WaitBatchTask("DELETE", resp.TaskID, time.Millisecond*200) } func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (newObj model.Obj, err error) { overwrite := true isFamily := y.isFamily() // 响应时间长,按需启用 if y.Addition.RapidUpload && !stream.IsForceStreamUpload() { if newObj, err := y.RapidUpload(ctx, dstDir, stream, isFamily, overwrite); err == nil { return newObj, nil } } uploadMethod := y.UploadMethod if stream.IsForceStreamUpload() { uploadMethod = "stream" } // 旧版上传家庭云也有限制 if uploadMethod == "old" { return y.OldUpload(ctx, dstDir, stream, up, isFamily, overwrite) } // 开启家庭云转存 if !isFamily && y.FamilyTransfer { // 修改上传目标为家庭云文件夹 transferDstDir := dstDir dstDir = y.familyTransferFolder // 使用临时文件名 srcName := stream.GetName() stream = &WrapFileStreamer{ FileStreamer: stream, Name: fmt.Sprintf("0%s.transfer", uuid.NewString()), } // 使用家庭云上传 isFamily = true overwrite = false defer func() { if newObj != nil { // 转存家庭云文件到个人云 err = y.SaveFamilyFileToPersonCloud(context.TODO(), y.FamilyID, newObj, transferDstDir, true) // 删除家庭云源文件 go y.Delete(context.TODO(), y.FamilyID, newObj) // 批量任务有概率删不掉 go y.cleanFamilyTransferFile() // 转存失败返回错误 if err != nil { return } // 查找转存文件 var file *Cloud189File file, err = y.findFileByName(context.TODO(), newObj.GetName(), transferDstDir.GetID(), false) if err != nil { if err == errs.ObjectNotFound { err = fmt.Errorf("unknown error: No transfer file obtained %s", newObj.GetName()) } return } // 重命名转存文件 newObj, err = y.Rename(context.TODO(), file, srcName) if err != nil { // 重命名失败删除源文件 _ = y.Delete(context.TODO(), "", file) } return } }() } switch uploadMethod { case "rapid": return y.FastUpload(ctx, dstDir, stream, up, isFamily, overwrite) case "stream": if stream.GetSize() == 0 { return y.FastUpload(ctx, dstDir, stream, up, isFamily, overwrite) } fallthrough default: return y.StreamUpload(ctx, dstDir, stream, up, isFamily, overwrite) } } func (y *Cloud189PC) GetDetails(ctx context.Context) (*model.StorageDetails, error) { capacityInfo, err := y.getCapacityInfo(ctx) if err != nil { return nil, err } var total, used int64 if y.isFamily() { total = capacityInfo.FamilyCapacityInfo.TotalSize used = capacityInfo.FamilyCapacityInfo.UsedSize } else { total = capacityInfo.CloudCapacityInfo.TotalSize used = capacityInfo.CloudCapacityInfo.UsedSize } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: used, }, }, nil } ================================================ FILE: drivers/189pc/help.go ================================================ package _189pc import ( "bytes" "crypto/aes" "crypto/hmac" "crypto/rand" "crypto/rsa" "crypto/sha1" "crypto/x509" "encoding/hex" "encoding/pem" "encoding/xml" "fmt" "math" "net/http" "regexp" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" ) func clientSuffix() map[string]string { rand := random.Rand return map[string]string{ "clientType": PC, "version": VERSION, "channelId": CHANNEL_ID, "rand": fmt.Sprintf("%d_%d", rand.Int63n(1e5), rand.Int63n(1e10)), } } // 带params的SignatureOfHmac HMAC签名 func signatureOfHmac(sessionSecret, sessionKey, operate, fullUrl, dateOfGmt, param string) string { urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(fullUrl)[1] mac := hmac.New(sha1.New, []byte(sessionSecret)) data := fmt.Sprintf("SessionKey=%s&Operate=%s&RequestURI=%s&Date=%s", sessionKey, operate, urlpath, dateOfGmt) if param != "" { data += fmt.Sprintf("¶ms=%s", param) } mac.Write([]byte(data)) return strings.ToUpper(hex.EncodeToString(mac.Sum(nil))) } // RAS 加密用户名密码 func RsaEncrypt(publicKey, origData string) string { block, _ := pem.Decode([]byte(publicKey)) pubInterface, _ := x509.ParsePKIXPublicKey(block.Bytes) data, _ := rsa.EncryptPKCS1v15(rand.Reader, pubInterface.(*rsa.PublicKey), []byte(origData)) return strings.ToUpper(hex.EncodeToString(data)) } // aes 加密params func AesECBEncrypt(data, key string) string { block, _ := aes.NewCipher([]byte(key)) paddingData := PKCS7Padding([]byte(data), block.BlockSize()) decrypted := make([]byte, len(paddingData)) size := block.BlockSize() for src, dst := paddingData, decrypted; len(src) > 0; src, dst = src[size:], dst[size:] { block.Encrypt(dst[:size], src[:size]) } return strings.ToUpper(hex.EncodeToString(decrypted)) } func PKCS7Padding(ciphertext []byte, blockSize int) []byte { padding := blockSize - len(ciphertext)%blockSize padtext := bytes.Repeat([]byte{byte(padding)}, padding) return append(ciphertext, padtext...) } // 获取http规范的时间 func getHttpDateStr() string { return time.Now().UTC().Format(http.TimeFormat) } // 时间戳 func timestamp() int64 { return time.Now().UTC().UnixNano() / 1e6 } // formatDate formats a time.Time object into the "YYYY-MM-DDHH:mm:ssSSS" format. func formatDate(t time.Time) string { // The layout string "2006-01-0215:04:05.000" corresponds to: // 2006 -> Year (YYYY) // 01 -> Month (MM) // 02 -> Day (DD) // 15 -> Hour (HH) // 04 -> Minute (mm) // 05 -> Second (ss) // 000 -> Millisecond (SSS) with leading zeros // Note the lack of a separator between the date and hour, matching the desired output. return t.Format("2006-01-0215:04:05.000") } func MustParseTime(str string) *time.Time { lastOpTime, _ := time.ParseInLocation("2006-01-02 15:04:05 -07", str+" +08", time.Local) return &lastOpTime } type Time time.Time func (t *Time) UnmarshalJSON(b []byte) error { return t.Unmarshal(b) } func (t *Time) UnmarshalXML(e *xml.Decoder, ee xml.StartElement) error { b, err := e.Token() if err != nil { return err } if b, ok := b.(xml.CharData); ok { if err = t.Unmarshal(b); err != nil { return err } } return e.Skip() } func (t *Time) Unmarshal(b []byte) error { bs := strings.Trim(string(b), "\"") var v time.Time var err error for _, f := range []string{"2006-01-02 15:04:05 -07", "Jan 2, 2006 15:04:05 PM -07"} { v, err = time.ParseInLocation(f, bs+" +08", time.Local) if err == nil { break } } *t = Time(v) return err } type String string func (t *String) UnmarshalJSON(b []byte) error { return t.Unmarshal(b) } func (t *String) UnmarshalXML(e *xml.Decoder, ee xml.StartElement) error { b, err := e.Token() if err != nil { return err } if b, ok := b.(xml.CharData); ok { if err = t.Unmarshal(b); err != nil { return err } } return e.Skip() } func (s *String) Unmarshal(b []byte) error { *s = String(bytes.Trim(b, "\"")) return nil } func toFamilyOrderBy(o string) string { switch o { case "filename": return "1" case "filesize": return "2" case "lastOpTime": return "3" default: return "1" } } func toDesc(o string) string { switch o { case "desc": return "true" case "asc": fallthrough default: return "false" } } func ParseHttpHeader(str string) map[string]string { header := make(map[string]string) for _, value := range strings.Split(str, "&") { if k, v, found := strings.Cut(value, "="); found { header[k] = v } } return header } func MustString(str string, err error) string { return str } func BoolToNumber(b bool) int { if b { return 1 } return 0 } // 计算分片大小 // 对分片数量有限制 // 10MIB 20 MIB 999片 // 50MIB 60MIB 70MIB 80MIB ∞MIB 1999片 func partSize(size int64) int64 { const DEFAULT = 1024 * 1024 * 10 // 10MIB if size > DEFAULT*2*999 { return int64(math.Max(math.Ceil((float64(size)/1999) /*=单个切片大小*/ /float64(DEFAULT)) /*=倍率*/, 5) * DEFAULT) } if size > DEFAULT*999 { return DEFAULT * 2 // 20MIB } return DEFAULT } func isBool(bs ...bool) bool { for _, b := range bs { if b { return true } } return false } func IF[V any](o bool, t V, f V) V { if o { return t } return f } type WrapFileStreamer struct { model.FileStreamer Name string } func (w *WrapFileStreamer) GetName() string { return w.Name } ================================================ FILE: drivers/189pc/meta.go ================================================ package _189pc import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { LoginType string `json:"login_type" type:"select" options:"password,qrcode" default:"password" required:"true"` Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` VCode string `json:"validate_code"` RefreshToken string `json:"refresh_token" help:"To switch accounts, please clear this field"` driver.RootID OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` Type string `json:"type" type:"select" options:"personal,family" default:"personal"` FamilyID string `json:"family_id"` UploadMethod string `json:"upload_method" type:"select" options:"stream,rapid,old" default:"stream"` UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` FamilyTransfer bool `json:"family_transfer"` RapidUpload bool `json:"rapid_upload"` NoUseOcr bool `json:"no_use_ocr"` } var config = driver.Config{ Name: "189CloudPC", DefaultRoot: "-11", CheckStatus: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Cloud189PC{} }) } ================================================ FILE: drivers/189pc/types.go ================================================ package _189pc import ( "encoding/xml" "fmt" "sort" "strings" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) // 居然有四种返回方式 type RespErr struct { ResCode any `json:"res_code"` // int or string ResMessage string `json:"res_message"` Error_ string `json:"error"` XMLName xml.Name `xml:"error"` Code string `json:"code" xml:"code"` Message string `json:"message" xml:"message"` Msg string `json:"msg"` ErrorCode string `json:"errorCode"` ErrorMsg string `json:"errorMsg"` } func (e *RespErr) HasError() bool { switch v := e.ResCode.(type) { case int, int64, int32: return v != 0 case string: return e.ResCode != "" } return (e.Code != "" && e.Code != "SUCCESS") || e.ErrorCode != "" || e.Error_ != "" } func (e *RespErr) Error() string { switch v := e.ResCode.(type) { case int, int64, int32: if v != 0 { return fmt.Sprintf("res_code: %d ,res_msg: %s", v, e.ResMessage) } case string: if e.ResCode != "" { return fmt.Sprintf("res_code: %s ,res_msg: %s", e.ResCode, e.ResMessage) } } if e.Code != "" && e.Code != "SUCCESS" { if e.Msg != "" { return fmt.Sprintf("code: %s ,msg: %s", e.Code, e.Msg) } if e.Message != "" { return fmt.Sprintf("code: %s ,msg: %s", e.Code, e.Message) } return "code: " + e.Code } if e.ErrorCode != "" { return fmt.Sprintf("err_code: %s ,err_msg: %s", e.ErrorCode, e.ErrorMsg) } if e.Error_ != "" { return fmt.Sprintf("error: %s ,message: %s", e.ErrorCode, e.Message) } return "" } type BaseLoginParam struct { // 请求头参数 Lt string ReqId string // 表单参数 ParamId string // 验证码 CaptchaToken string } // QRLoginParam 用于暂存二维码登录过程中的参数 type QRLoginParam struct { BaseLoginParam UUID string `json:"uuid"` EncodeUUID string `json:"encodeuuid"` EncryUUID string `json:"encryuuid"` } // 登陆需要的参数 type LoginParam struct { // 加密后的用户名和密码 RsaUsername string RsaPassword string // rsa密钥 jRsaKey string BaseLoginParam } // 登陆加密相关 type EncryptConfResp struct { Result int `json:"result"` Data struct { UpSmsOn string `json:"upSmsOn"` Pre string `json:"pre"` PreDomain string `json:"preDomain"` PubKey string `json:"pubKey"` } `json:"data"` } type LoginResp struct { Msg string `json:"msg"` Result int `json:"result"` ToUrl string `json:"toUrl"` } // 刷新session返回 type UserSessionResp struct { ResCode int `json:"res_code"` ResMessage string `json:"res_message"` LoginName string `json:"loginName"` KeepAlive int `json:"keepAlive"` GetFileDiffSpan int `json:"getFileDiffSpan"` GetUserInfoSpan int `json:"getUserInfoSpan"` // 个人云 SessionKey string `json:"sessionKey"` SessionSecret string `json:"sessionSecret"` // 家庭云 FamilySessionKey string `json:"familySessionKey"` FamilySessionSecret string `json:"familySessionSecret"` } // 登录返回 type AppSessionResp struct { UserSessionResp IsSaveName string `json:"isSaveName"` // 会话刷新Token AccessToken string `json:"accessToken"` //Token刷新 RefreshToken string `json:"refreshToken"` } // 家庭云账户 type FamilyInfoListResp struct { FamilyInfoResp []FamilyInfoResp `json:"familyInfoResp"` } type FamilyInfoResp struct { Count int `json:"count"` CreateTime string `json:"createTime"` FamilyID int64 `json:"familyId"` RemarkName string `json:"remarkName"` Type int `json:"type"` UseFlag int `json:"useFlag"` UserRole int `json:"userRole"` } /*文件部分*/ // 文件 type Cloud189File struct { ID String `json:"id"` Name string `json:"name"` Size int64 `json:"size"` Md5 string `json:"md5"` LastOpTime Time `json:"lastOpTime"` CreateDate Time `json:"createDate"` Icon struct { //iconOption 5 SmallUrl string `json:"smallUrl"` LargeUrl string `json:"largeUrl"` // iconOption 10 Max600 string `json:"max600"` MediumURL string `json:"mediumUrl"` } `json:"icon"` // Orientation int64 `json:"orientation"` // FileCata int64 `json:"fileCata"` // MediaType int `json:"mediaType"` // Rev string `json:"rev"` // StarLabel int64 `json:"starLabel"` } func (c *Cloud189File) CreateTime() time.Time { return time.Time(c.CreateDate) } func (c *Cloud189File) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.MD5, c.Md5) } func (c *Cloud189File) GetSize() int64 { return c.Size } func (c *Cloud189File) GetName() string { return c.Name } func (c *Cloud189File) ModTime() time.Time { return time.Time(c.LastOpTime) } func (c *Cloud189File) IsDir() bool { return false } func (c *Cloud189File) GetID() string { return string(c.ID) } func (c *Cloud189File) GetPath() string { return "" } func (c *Cloud189File) Thumb() string { return c.Icon.SmallUrl } // 文件夹 type Cloud189Folder struct { ID String `json:"id"` ParentID int64 `json:"parentId"` Name string `json:"name"` LastOpTime Time `json:"lastOpTime"` CreateDate Time `json:"createDate"` // FileListSize int64 `json:"fileListSize"` // FileCount int64 `json:"fileCount"` // FileCata int64 `json:"fileCata"` // Rev string `json:"rev"` // StarLabel int64 `json:"starLabel"` } func (c *Cloud189Folder) CreateTime() time.Time { return time.Time(c.CreateDate) } func (c *Cloud189Folder) GetHash() utils.HashInfo { return utils.HashInfo{} } func (c *Cloud189Folder) GetSize() int64 { return 0 } func (c *Cloud189Folder) GetName() string { return c.Name } func (c *Cloud189Folder) ModTime() time.Time { return time.Time(c.LastOpTime) } func (c *Cloud189Folder) IsDir() bool { return true } func (c *Cloud189Folder) GetID() string { return string(c.ID) } func (c *Cloud189Folder) GetPath() string { return "" } type Cloud189FilesResp struct { //ResCode int `json:"res_code"` //ResMessage string `json:"res_message"` FileListAO struct { Count int `json:"count"` FileList []Cloud189File `json:"fileList"` FolderList []Cloud189Folder `json:"folderList"` } `json:"fileListAO"` } // TaskInfo 任务信息 type BatchTaskInfo struct { // FileId 文件ID FileId string `json:"fileId"` // FileName 文件名 FileName string `json:"fileName"` // IsFolder 是否是文件夹，0-否，1-是 IsFolder int `json:"isFolder"` // SrcParentId 文件所在父目录ID SrcParentId string `json:"srcParentId,omitempty"` /* 冲突管理 */ // 1 -> 跳过 2 -> 保留 3 -> 覆盖 DealWay int `json:"dealWay,omitempty"` IsConflict int `json:"isConflict,omitempty"` } /* 上传部分 */ type InitMultiUploadResp struct { //Code string `json:"code"` Data struct { UploadType int `json:"uploadType"` UploadHost string `json:"uploadHost"` UploadFileID string `json:"uploadFileId"` FileDataExists int `json:"fileDataExists"` } `json:"data"` } type UploadUrlsResp struct { Code string `json:"code"` Data map[string]UploadUrlsData `json:"uploadUrls"` } type UploadUrlsData struct { RequestURL string `json:"requestURL"` RequestHeader string `json:"requestHeader"` } type UploadUrlInfo struct { PartNumber int Headers map[string]string UploadUrlsData } type UploadProgress struct { UploadInfo InitMultiUploadResp UploadParts []string } /* 第二种上传方式 */ type CreateUploadFileResp struct { // 上传文件请求ID UploadFileId int64 `json:"uploadFileId"` // 上传文件数据的URL路径 FileUploadUrl string `json:"fileUploadUrl"` // 上传文件完成后确认路径 FileCommitUrl string `json:"fileCommitUrl"` // 文件是否已存在云盘中，0-未存在，1-已存在 FileDataExists int `json:"fileDataExists"` } type GetUploadFileStatusResp struct { CreateUploadFileResp // 已上传的大小 DataSize int64 `json:"dataSize"` Size int64 `json:"size"` } func (r *GetUploadFileStatusResp) GetSize() int64 { return r.DataSize + r.Size } type CommitMultiUploadFileResp struct { File struct { UserFileID String `json:"userFileId"` FileName string `json:"fileName"` FileSize int64 `json:"fileSize"` FileMd5 string `json:"fileMd5"` CreateDate Time `json:"createDate"` } `json:"file"` } func (f *CommitMultiUploadFileResp) toFile() *Cloud189File { return &Cloud189File{ ID: f.File.UserFileID, Name: f.File.FileName, Size: f.File.FileSize, Md5: f.File.FileMd5, LastOpTime: f.File.CreateDate, CreateDate: f.File.CreateDate, } } type OldCommitUploadFileResp struct { XMLName xml.Name `xml:"file"` ID String `xml:"id"` Name string `xml:"name"` Size int64 `xml:"size"` Md5 string `xml:"md5"` CreateDate Time `xml:"createDate"` } func (f *OldCommitUploadFileResp) toFile() *Cloud189File { return &Cloud189File{ ID: f.ID, Name: f.Name, Size: f.Size, Md5: f.Md5, CreateDate: f.CreateDate, LastOpTime: f.CreateDate, } } type CreateBatchTaskResp struct { TaskID string `json:"taskId"` } type BatchTaskStateResp struct { FailedCount int `json:"failedCount"` Process int `json:"process"` SkipCount int `json:"skipCount"` SubTaskCount int `json:"subTaskCount"` SuccessedCount int `json:"successedCount"` SuccessedFileIDList []int64 `json:"successedFileIdList"` TaskID string `json:"taskId"` TaskStatus int `json:"taskStatus"` //1 初始化 2 存在冲突 3 执行中，4 完成 } type BatchTaskConflictTaskInfoResp struct { SessionKey string `json:"sessionKey"` TargetFolderID int `json:"targetFolderId"` TaskID string `json:"taskId"` TaskInfos []BatchTaskInfo TaskType int `json:"taskType"` } /* query 加密参数*/ type Params map[string]string func (p Params) Set(k, v string) { p[k] = v } func (p Params) Encode() string { if p == nil { return "" } var buf strings.Builder keys := make([]string, 0, len(p)) for k := range p { keys = append(keys, k) } sort.Strings(keys) for i := range keys { if buf.Len() > 0 { buf.WriteByte('&') } buf.WriteString(keys[i]) buf.WriteByte('=') buf.WriteString(p[keys[i]]) } return buf.String() } type CapacityResp struct { ResCode int `json:"res_code"` ResMessage string `json:"res_message"` Account string `json:"account"` CloudCapacityInfo struct { FreeSize int64 `json:"freeSize"` MailUsedSize int64 `json:"mail189UsedSize"` TotalSize int64 `json:"totalSize"` UsedSize int64 `json:"usedSize"` } `json:"cloudCapacityInfo"` FamilyCapacityInfo struct { FreeSize int64 `json:"freeSize"` TotalSize int64 `json:"totalSize"` UsedSize int64 `json:"usedSize"` } `json:"familyCapacityInfo"` TotalSize uint64 `json:"totalSize"` } ================================================ FILE: drivers/189pc/utils.go ================================================ package _189pc import ( "bytes" "context" "encoding/base64" "encoding/hex" "encoding/xml" "fmt" "hash" "io" "net/http" "net/http/cookiejar" "net/url" "os" "regexp" "sort" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/skip2/go-qrcode" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" "github.com/google/uuid" jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" ) const ( ACCOUNT_TYPE = "02" APP_ID = "8025431004" CLIENT_TYPE = "10020" VERSION = "6.2" WEB_URL = "https://cloud.189.cn" AUTH_URL = "https://open.e.189.cn" API_URL = "https://api.cloud.189.cn" UPLOAD_URL = "https://upload.cloud.189.cn" RETURN_URL = "https://m.cloud.189.cn/zhuanti/2020/loginErrorPc/index.html" PC = "TELEPC" MAC = "TELEMAC" CHANNEL_ID = "web_cloud.189.cn" // Error codes UserInvalidOpenTokenError = "UserInvalidOpenToken" ) func (y *Cloud189PC) SignatureHeader(url, method, params string, isFamily bool) map[string]string { dateOfGmt := getHttpDateStr() sessionKey := y.getTokenInfo().SessionKey sessionSecret := y.getTokenInfo().SessionSecret if isFamily { sessionKey = y.getTokenInfo().FamilySessionKey sessionSecret = y.getTokenInfo().FamilySessionSecret } header := map[string]string{ "Date": dateOfGmt, "SessionKey": sessionKey, "X-Request-ID": uuid.NewString(), "Signature": signatureOfHmac(sessionSecret, sessionKey, method, url, dateOfGmt, params), } return header } func (y *Cloud189PC) EncryptParams(params Params, isFamily bool) string { sessionSecret := y.getTokenInfo().SessionSecret if isFamily { sessionSecret = y.getTokenInfo().FamilySessionSecret } if params != nil { return AesECBEncrypt(params.Encode(), sessionSecret[:16]) } return "" } func (y *Cloud189PC) request(url, method string, callback base.ReqCallback, params Params, resp interface{}, isFamily ...bool) ([]byte, error) { if y.getTokenInfo() == nil { return nil, fmt.Errorf("login failed") } req := y.getClient().R().SetQueryParams(clientSuffix()) // 设置params paramsData := y.EncryptParams(params, isBool(isFamily...)) if paramsData != "" { req.SetQueryParam("params", paramsData) } // Signature req.SetHeaders(y.SignatureHeader(url, method, paramsData, isBool(isFamily...))) var erron RespErr req.SetError(&erron) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } res, err := req.Execute(method, url) if err != nil { return nil, err } if strings.Contains(res.String(), "userSessionBO is null") { if err = y.refreshSession(); err != nil { return nil, err } return y.request(url, method, callback, params, resp, isFamily...) } // if erron.ErrorCode == "InvalidSessionKey" || erron.Code == "InvalidSessionKey" { if strings.Contains(res.String(), "InvalidSessionKey") { if err = y.refreshSession(); err != nil { return nil, err } return y.request(url, method, callback, params, resp, isFamily...) } // 处理错误 if erron.HasError() { return nil, &erron } return res.Body(), nil } func (y *Cloud189PC) get(url string, callback base.ReqCallback, resp interface{}, isFamily ...bool) ([]byte, error) { return y.request(url, http.MethodGet, callback, nil, resp, isFamily...) } func (y *Cloud189PC) post(url string, callback base.ReqCallback, resp interface{}, isFamily ...bool) ([]byte, error) { return y.request(url, http.MethodPost, callback, nil, resp, isFamily...) } func (y *Cloud189PC) put(ctx context.Context, url string, headers map[string]string, sign bool, file io.Reader, isFamily bool) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, file) if err != nil { return nil, err } query := req.URL.Query() for key, value := range clientSuffix() { query.Add(key, value) } req.URL.RawQuery = query.Encode() for key, value := range headers { req.Header.Add(key, value) } if sign { for key, value := range y.SignatureHeader(url, http.MethodPut, "", isFamily) { req.Header.Add(key, value) } } resp, err := base.HttpClient.Do(req) if err != nil { return nil, err } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } var erron RespErr _ = jsoniter.Unmarshal(body, &erron) _ = xml.Unmarshal(body, &erron) if erron.HasError() { return nil, &erron } if resp.StatusCode != http.StatusOK { return nil, errors.Errorf("put fail,err:%s", string(body)) } return body, nil } func (y *Cloud189PC) getFiles(ctx context.Context, fileId string, isFamily bool) ([]model.Obj, error) { res := make([]model.Obj, 0, 100) for pageNum := 1; ; pageNum++ { resp, err := y.getFilesWithPage(ctx, fileId, isFamily, pageNum, 1000, y.OrderBy, y.OrderDirection) if err != nil { return nil, err } // 获取完毕跳出 if resp.FileListAO.Count == 0 { break } for i := 0; i < len(resp.FileListAO.FolderList); i++ { res = append(res, &resp.FileListAO.FolderList[i]) } for i := 0; i < len(resp.FileListAO.FileList); i++ { res = append(res, &resp.FileListAO.FileList[i]) } } return res, nil } func (y *Cloud189PC) getFilesWithPage(ctx context.Context, fileId string, isFamily bool, pageNum int, pageSize int, orderBy string, orderDirection string) (*Cloud189FilesResp, error) { fullUrl := API_URL if isFamily { fullUrl += "/family/file" } fullUrl += "/listFiles.action" var resp Cloud189FilesResp _, err := y.get(fullUrl, func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(map[string]string{ "folderId": fileId, "fileType": "0", "mediaAttr": "0", "iconOption": "5", "pageNum": fmt.Sprint(pageNum), "pageSize": fmt.Sprint(pageSize), }) if isFamily { r.SetQueryParams(map[string]string{ "familyId": y.FamilyID, "orderBy": toFamilyOrderBy(orderBy), "descending": toDesc(orderDirection), }) } else { r.SetQueryParams(map[string]string{ "recursive": "0", "orderBy": orderBy, "descending": toDesc(orderDirection), }) } }, &resp, isFamily) if err != nil { return nil, err } return &resp, nil } func (y *Cloud189PC) findFileByName(ctx context.Context, searchName string, folderId string, isFamily bool) (*Cloud189File, error) { for pageNum := 1; ; pageNum++ { resp, err := y.getFilesWithPage(ctx, folderId, isFamily, pageNum, 10, "filename", "asc") if err != nil { return nil, err } // 获取完毕跳出 if resp.FileListAO.Count == 0 { return nil, errs.ObjectNotFound } for i := 0; i < len(resp.FileListAO.FileList); i++ { file := resp.FileListAO.FileList[i] if file.Name == searchName { return &file, nil } } } } func (y *Cloud189PC) login() error { if y.LoginType == "qrcode" { return y.loginByQRCode() } return y.loginByPassword() } func (y *Cloud189PC) loginByPassword() (err error) { // 初始化登陆所需参数 if y.loginParam == nil { if err = y.initLoginParam(); err != nil { // 验证码也通过错误返回 return err } } defer func() { // 销毁验证码 y.VCode = "" // 销毁登陆参数 y.loginParam = nil // 遇到错误，重新加载登陆参数(刷新验证码) if err != nil { if y.NoUseOcr { if err1 := y.initLoginParam(); err1 != nil { err = fmt.Errorf("err1: %s \nerr2: %s", err, err1) } } y.Status = err.Error() op.MustSaveDriverStorage(y) } }() param := y.loginParam var loginresp LoginResp _, err = y.client.R(). ForceContentType("application/json;charset=UTF-8").SetResult(&loginresp). SetHeaders(map[string]string{ "REQID": param.ReqId, "lt": param.Lt, }). SetFormData(map[string]string{ "appKey": APP_ID, "accountType": ACCOUNT_TYPE, "userName": param.RsaUsername, "password": param.RsaPassword, "validateCode": y.VCode, "captchaToken": param.CaptchaToken, "returnUrl": RETURN_URL, // "mailSuffix": "@189.cn", "dynamicCheck": "FALSE", "clientType": CLIENT_TYPE, "cb_SaveName": "1", "isOauth2": "false", "state": "", "paramId": param.ParamId, }). Post(AUTH_URL + "/api/logbox/oauth2/loginSubmit.do") if err != nil { return err } if loginresp.ToUrl == "" { return fmt.Errorf("login failed,No toUrl obtained, msg: %s", loginresp.Msg) } // 获取Session var erron RespErr var tokenInfo AppSessionResp _, err = y.client.R(). SetResult(&tokenInfo).SetError(&erron). SetQueryParams(clientSuffix()). SetQueryParam("redirectURL", loginresp.ToUrl). Post(API_URL + "/getSessionForPC.action") if err != nil { return err } if erron.HasError() { return &erron } if tokenInfo.ResCode != 0 { err = fmt.Errorf(tokenInfo.ResMessage) return err } y.Addition.RefreshToken = tokenInfo.RefreshToken y.tokenInfo = &tokenInfo op.MustSaveDriverStorage(y) return err } func (y *Cloud189PC) loginByQRCode() error { if y.qrcodeParam == nil { if err := y.initQRCodeParam(); err != nil { // 二维码也通过错误返回 return err } } var state struct { Status int `json:"status"` RedirectUrl string `json:"redirectUrl"` Msg string `json:"msg"` } now := time.Now() _, err := y.client.R(). SetHeaders(map[string]string{ "Referer": AUTH_URL, "Reqid": y.qrcodeParam.ReqId, "lt": y.qrcodeParam.Lt, }). SetFormData(map[string]string{ "appId": APP_ID, "clientType": CLIENT_TYPE, "returnUrl": RETURN_URL, "paramId": y.qrcodeParam.ParamId, "uuid": y.qrcodeParam.UUID, "encryuuid": y.qrcodeParam.EncryUUID, "date": formatDate(now), "timeStamp": fmt.Sprint(now.UTC().UnixNano() / 1e6), }). ForceContentType("application/json;charset=UTF-8"). SetResult(&state). Post(AUTH_URL + "/api/logbox/oauth2/qrcodeLoginState.do") if err != nil { return fmt.Errorf("failed to check QR code state: %w", err) } switch state.Status { case 0: // 登录成功 var tokenInfo AppSessionResp _, err = y.client.R(). SetResult(&tokenInfo). SetQueryParams(clientSuffix()). SetQueryParam("redirectURL", state.RedirectUrl). Post(API_URL + "/getSessionForPC.action") if err != nil { return err } if tokenInfo.ResCode != 0 { return fmt.Errorf(tokenInfo.ResMessage) } y.Addition.RefreshToken = tokenInfo.RefreshToken y.tokenInfo = &tokenInfo op.MustSaveDriverStorage(y) return nil case -11001: // 二维码过期 y.qrcodeParam = nil return errors.New("QR code expired, please try again") case -106: // 等待扫描 return y.genQRCode("QR code has not been scanned yet, please scan and save again") case -11002: // 等待确认 return y.genQRCode("QR code has been scanned, please confirm the login on your phone and save again") default: // 其他错误 y.qrcodeParam = nil return fmt.Errorf("QR code login failed with status %d: %s", state.Status, state.Msg) } } func (y *Cloud189PC) genQRCode(text string) error { // 展示二维码 qrTemplate := ` state: %s

Or Click here: Login ` // Generate QR code qrCode, err := qrcode.Encode(y.qrcodeParam.UUID, qrcode.Medium, 256) if err != nil { return fmt.Errorf("failed to generate QR code: %v", err) } // Encode QR code to base64 qrCodeBase64 := base64.StdEncoding.EncodeToString(qrCode) // Create the HTML page qrPage := fmt.Sprintf(qrTemplate, text, qrCodeBase64, y.qrcodeParam.UUID) return fmt.Errorf("need verify: \n%s", qrPage) } func (y *Cloud189PC) initBaseParams() (*BaseLoginParam, error) { // 清除cookie jar, _ := cookiejar.New(nil) y.client.SetCookieJar(jar) res, err := y.client.R(). SetQueryParams(map[string]string{ "appId": APP_ID, "clientType": CLIENT_TYPE, "returnURL": RETURN_URL, "timeStamp": fmt.Sprint(timestamp()), }). Get(WEB_URL + "/api/portal/unifyLoginForPC.action") if err != nil { return nil, err } return &BaseLoginParam{ CaptchaToken: regexp.MustCompile(`'captchaToken' value='(.+?)'`).FindStringSubmatch(res.String())[1], Lt: regexp.MustCompile(`lt = "(.+?)"`).FindStringSubmatch(res.String())[1], ParamId: regexp.MustCompile(`paramId = "(.+?)"`).FindStringSubmatch(res.String())[1], ReqId: regexp.MustCompile(`reqId = "(.+?)"`).FindStringSubmatch(res.String())[1], }, nil } /* 初始化登陆需要的参数 * 如果遇到验证码返回错误 */ func (y *Cloud189PC) initLoginParam() error { y.loginParam = nil baseParam, err := y.initBaseParams() if err != nil { return err } y.loginParam = &LoginParam{BaseLoginParam: *baseParam} // 获取rsa公钥 var encryptConf EncryptConfResp _, err = y.client.R(). ForceContentType("application/json;charset=UTF-8").SetResult(&encryptConf). SetFormData(map[string]string{"appId": APP_ID}). Post(AUTH_URL + "/api/logbox/config/encryptConf.do") if err != nil { return err } y.loginParam.jRsaKey = fmt.Sprintf("-----BEGIN PUBLIC KEY-----\n%s\n-----END PUBLIC KEY-----", encryptConf.Data.PubKey) y.loginParam.RsaUsername = encryptConf.Data.Pre + RsaEncrypt(y.loginParam.jRsaKey, y.Username) y.loginParam.RsaPassword = encryptConf.Data.Pre + RsaEncrypt(y.loginParam.jRsaKey, y.Password) // 判断是否需要验证码 resp, err := y.client.R(). SetHeader("REQID", y.loginParam.ReqId). SetFormData(map[string]string{ "appKey": APP_ID, "accountType": ACCOUNT_TYPE, "userName": y.loginParam.RsaUsername, }).Post(AUTH_URL + "/api/logbox/oauth2/needcaptcha.do") if err != nil { return err } if resp.String() == "0" { return nil } // 拉取验证码 imgRes, err := y.client.R(). SetQueryParams(map[string]string{ "token": y.loginParam.CaptchaToken, "REQID": y.loginParam.ReqId, "rnd": fmt.Sprint(timestamp()), }). Get(AUTH_URL + "/api/logbox/oauth2/picCaptcha.do") if err != nil { return fmt.Errorf("failed to obtain verification code") } if imgRes.Size() > 20 { if setting.GetStr(conf.OcrApi) != "" && !y.NoUseOcr { vRes, err := base.RestyClient.R(). SetMultipartField("image", "validateCode.png", "image/png", bytes.NewReader(imgRes.Body())). Post(setting.GetStr(conf.OcrApi)) if err != nil { return err } if jsoniter.Get(vRes.Body(), "status").ToInt() == 200 { y.VCode = jsoniter.Get(vRes.Body(), "result").ToString() return nil } } // 返回验证码图片给前端 return fmt.Errorf(`need img validate code: `, base64.StdEncoding.EncodeToString(imgRes.Body())) } return nil } // getQRCode 获取并返回二维码 func (y *Cloud189PC) initQRCodeParam() (err error) { y.qrcodeParam = nil baseParam, err := y.initBaseParams() if err != nil { return err } var qrcodeParam QRLoginParam _, err = y.client.R(). SetFormData(map[string]string{"appId": APP_ID}). ForceContentType("application/json;charset=UTF-8"). SetResult(&qrcodeParam). Post(AUTH_URL + "/api/logbox/oauth2/getUUID.do") if err != nil { return err } qrcodeParam.BaseLoginParam = *baseParam y.qrcodeParam = &qrcodeParam return y.genQRCode("please scan the QR code with the 189 Cloud app, then save the settings again.") } // 刷新会话 func (y *Cloud189PC) refreshSession() (err error) { return y.refreshSessionWithRetry(0) } func (y *Cloud189PC) refreshSessionWithRetry(retryCount int) (err error) { if y.ref != nil { return y.ref.refreshSessionWithRetry(retryCount) } var erron RespErr var userSessionResp UserSessionResp _, err = y.client.R(). SetResult(&userSessionResp).SetError(&erron). SetQueryParams(clientSuffix()). SetQueryParams(map[string]string{ "appId": APP_ID, "accessToken": y.tokenInfo.AccessToken, }). SetHeader("X-Request-ID", uuid.NewString()). Get(API_URL + "/getSessionForPC.action") if err != nil { return err } // token生效刷新token if erron.HasError() { if erron.ResCode == UserInvalidOpenTokenError { return y.refreshTokenWithRetry(retryCount) } return &erron } y.tokenInfo.UserSessionResp = userSessionResp return nil } // refreshToken 刷新token，失败时返回错误，不再直接调用login func (y *Cloud189PC) refreshToken() (err error) { return y.refreshTokenWithRetry(0) } func (y *Cloud189PC) refreshTokenWithRetry(retryCount int) (err error) { if y.ref != nil { return y.ref.refreshTokenWithRetry(retryCount) } // 限制重试次数，避免无限递归 if retryCount >= 3 { if y.Addition.RefreshToken != "" { y.Addition.RefreshToken = "" op.MustSaveDriverStorage(y) } return errors.New("refresh token failed after maximum retries") } var erron RespErr var tokenInfo AppSessionResp _, err = y.client.R(). SetResult(&tokenInfo). ForceContentType("application/json;charset=UTF-8"). SetError(&erron). SetFormData(map[string]string{ "clientId": APP_ID, "refreshToken": y.tokenInfo.RefreshToken, "grantType": "refresh_token", "format": "json", }). Post(AUTH_URL + "/api/oauth2/refreshToken.do") if err != nil { return err } // 如果刷新失败，返回错误给上层处理 if erron.HasError() { if y.Addition.RefreshToken != "" { y.Addition.RefreshToken = "" op.MustSaveDriverStorage(y) } // 根据登录类型决定下一步行为 if y.LoginType == "qrcode" { return errors.New("QR code session has expired, please re-scan the code to log in") } // 密码登录模式下，尝试回退到完整登录 return y.login() } y.Addition.RefreshToken = tokenInfo.RefreshToken y.tokenInfo = &tokenInfo op.MustSaveDriverStorage(y) return y.refreshSessionWithRetry(retryCount + 1) } func (y *Cloud189PC) keepAlive() { _, err := y.get(API_URL+"/keepUserSession.action", func(r *resty.Request) { r.SetQueryParams(clientSuffix()) }, nil) if err != nil { utils.Log.Warnf("189pc: Failed to keep user session alive: %v", err) // 如果keepAlive失败，尝试刷新session if refreshErr := y.refreshSession(); refreshErr != nil { utils.Log.Errorf("189pc: Failed to refresh session after keepAlive error: %v", refreshErr) } } else { utils.Log.Debugf("189pc: User session kept alive successfully.") } } // 普通上传 // 无法上传大小为0的文件 func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { // 文件大小 fileSize := file.GetSize() // 分片大小，不得为文件大小 sliceSize := partSize(fileSize) params := Params{ "parentFolderId": dstDir.GetID(), "fileName": url.QueryEscape(file.GetName()), "fileSize": fmt.Sprint(fileSize), "sliceSize": fmt.Sprint(sliceSize), // 必须为特定分片大小 "lazyCheck": "1", } fullUrl := UPLOAD_URL if isFamily { params.Set("familyId", y.FamilyID) fullUrl += "/family" } else { // params.Set("extend", `{"opScene":"1","relativepath":"","rootfolderid":""}`) fullUrl += "/person" } // 初始化上传 var initMultiUpload InitMultiUploadResp _, err := y.request(fullUrl+"/initMultiUpload", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }, params, &initMultiUpload, isFamily) if err != nil { return nil, err } ss, err := stream.NewStreamSectionReader(file, int(sliceSize), &up) if err != nil { return nil, err } threadG, upCtx := errgroup.NewOrderedGroupWithContext(ctx, y.uploadThread, retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) count := 1 if fileSize > sliceSize { count = int((fileSize + sliceSize - 1) / sliceSize) } lastPartSize := fileSize % sliceSize if lastPartSize == 0 { lastPartSize = sliceSize } silceMd5Hexs := make([]string, 0, count) silceMd5 := utils.MD5.NewFunc() var writers io.Writer = silceMd5 fileMd5Hex := file.GetHash().GetHash(utils.MD5) var fileMd5 hash.Hash if len(fileMd5Hex) != utils.MD5.Width { fileMd5 = utils.MD5.NewFunc() writers = io.MultiWriter(silceMd5, fileMd5) } for i := 1; i <= count; i++ { if utils.IsCanceled(upCtx) { break } offset := int64((i)-1) * sliceSize partSize := sliceSize if i == count { partSize = lastPartSize } partInfo := "" var reader io.ReadSeeker threadG.GoWithLifecycle(errgroup.Lifecycle{ Before: func(ctx context.Context) (err error) { reader, err = ss.GetSectionReader(offset, partSize) if err != nil { return err } silceMd5.Reset() w, err := utils.CopyWithBuffer(writers, reader) if w != partSize { return fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", partSize, w, err) } // 计算块md5并进行hex和base64编码 md5Bytes := silceMd5.Sum(nil) silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Bytes))) partInfo = fmt.Sprintf("%d-%s", i, base64.StdEncoding.EncodeToString(md5Bytes)) return nil }, Do: func(ctx context.Context) (err error) { reader.Seek(0, io.SeekStart) uploadUrls, err := y.GetMultiUploadUrls(ctx, isFamily, initMultiUpload.Data.UploadFileID, partInfo) if err != nil { return err } // step.4 上传切片 uploadUrl := uploadUrls[0] _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, driver.NewLimitedUploadStream(ctx, reader), isFamily) if err != nil { return err } up(float64(threadG.Success()+1) * 100 / float64(count+1)) return nil }, After: func(err error) { ss.FreeSectionReader(reader) }, }, ) } if err = threadG.Wait(); err != nil { return nil, err } defer up(100) if fileMd5 != nil { fileMd5Hex = strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil))) } sliceMd5Hex := fileMd5Hex if fileSize > sliceSize { sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(silceMd5Hexs, "\n"))) } // 提交上传 var resp CommitMultiUploadFileResp _, err = y.request(fullUrl+"/commitMultiUploadFile", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }, Params{ "uploadFileId": initMultiUpload.Data.UploadFileID, "fileMd5": fileMd5Hex, "sliceMd5": sliceMd5Hex, "lazyCheck": "1", "isLog": "0", "opertype": IF(overwrite, "3", "1"), }, &resp, isFamily) if err != nil { return nil, err } return resp.toFile(), nil } func (y *Cloud189PC) RapidUpload(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, isFamily bool, overwrite bool) (model.Obj, error) { fileMd5 := stream.GetHash().GetHash(utils.MD5) if len(fileMd5) < utils.MD5.Width { return nil, errors.New("invalid hash") } uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, stream.GetName(), fmt.Sprint(stream.GetSize()), isFamily) if err != nil { return nil, err } if uploadInfo.FileDataExists != 1 { return nil, errors.New("rapid upload fail") } return y.OldUploadCommit(ctx, uploadInfo.FileCommitUrl, uploadInfo.UploadFileId, isFamily, overwrite) } // 快传 func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { var ( cache = file.GetFile() tmpF *os.File err error ) size := file.GetSize() if _, ok := cache.(io.ReaderAt); !ok && size > 0 { tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") if err != nil { return nil, err } defer func() { _ = tmpF.Close() _ = os.Remove(tmpF.Name()) }() cache = tmpF } sliceSize := partSize(size) count := 1 if size > sliceSize { count = int((size + sliceSize - 1) / sliceSize) } lastSliceSize := size % sliceSize if lastSliceSize == 0 { lastSliceSize = sliceSize } // step.1 优先计算所需信息 byteSize := sliceSize fileMd5 := utils.MD5.NewFunc() sliceMd5 := utils.MD5.NewFunc() sliceMd5Hexs := make([]string, 0, count) partInfos := make([]string, 0, count) writers := []io.Writer{fileMd5, sliceMd5} if tmpF != nil { writers = append(writers, tmpF) } written := int64(0) for i := 1; i <= count; i++ { if utils.IsCanceled(ctx) { return nil, ctx.Err() } if i == count { byteSize = lastSliceSize } n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), file, byteSize) written += n if err != nil && err != io.EOF { return nil, err } md5Byte := sliceMd5.Sum(nil) sliceMd5Hexs = append(sliceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Byte))) partInfos = append(partInfos, fmt.Sprint(i, "-", base64.StdEncoding.EncodeToString(md5Byte))) sliceMd5.Reset() } if tmpF != nil { if size > 0 && written != size { return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, size) } _, err = tmpF.Seek(0, io.SeekStart) if err != nil { return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") } } fileMd5Hex := strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil))) sliceMd5Hex := fileMd5Hex if size > sliceSize { sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(sliceMd5Hexs, "\n"))) } fullUrl := UPLOAD_URL if isFamily { fullUrl += "/family" } else { // params.Set("extend", `{"opScene":"1","relativepath":"","rootfolderid":""}`) fullUrl += "/person" } // 尝试恢复进度 uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, y.getTokenInfo().SessionKey, fileMd5Hex) if !ok { // step.2 预上传 params := Params{ "parentFolderId": dstDir.GetID(), "fileName": url.QueryEscape(file.GetName()), "fileSize": fmt.Sprint(file.GetSize()), "fileMd5": fileMd5Hex, "sliceSize": fmt.Sprint(sliceSize), "sliceMd5": sliceMd5Hex, } if isFamily { params.Set("familyId", y.FamilyID) } var uploadInfo InitMultiUploadResp _, err = y.request(fullUrl+"/initMultiUpload", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }, params, &uploadInfo, isFamily) if err != nil { return nil, err } uploadProgress = &UploadProgress{ UploadInfo: uploadInfo, UploadParts: partInfos, } } uploadInfo := uploadProgress.UploadInfo.Data // 网盘中不存在该文件，开始上传 if uploadInfo.FileDataExists != 1 { threadG, upCtx := errgroup.NewGroupWithContext(ctx, y.uploadThread, retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) for i, uploadPart := range uploadProgress.UploadParts { if utils.IsCanceled(upCtx) { break } i, uploadPart := i, uploadPart threadG.Go(func(ctx context.Context) error { // step.3 获取上传链接 uploadUrls, err := y.GetMultiUploadUrls(ctx, isFamily, uploadInfo.UploadFileID, uploadPart) if err != nil { return err } uploadUrl := uploadUrls[0] byteSize, offset := sliceSize, int64(uploadUrl.PartNumber-1)*sliceSize if uploadUrl.PartNumber == count { byteSize = lastSliceSize } // step.4 上传切片 rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.NewSectionReader(cache, offset, byteSize)) _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, rateLimitedRd, isFamily) if err != nil { return err } up(float64(threadG.Success()+1) * 100 / float64(len(uploadUrls)+1)) uploadProgress.UploadParts[i] = "" return nil }) } if err = threadG.Wait(); err != nil { if errors.Is(err, context.Canceled) { uploadProgress.UploadParts = utils.SliceFilter(uploadProgress.UploadParts, func(s string) bool { return s != "" }) base.SaveUploadProgress(y, uploadProgress, y.getTokenInfo().SessionKey, fileMd5Hex) } return nil, err } defer up(100) } // step.5 提交 var resp CommitMultiUploadFileResp _, err = y.request(fullUrl+"/commitMultiUploadFile", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }, Params{ "uploadFileId": uploadInfo.UploadFileID, "isLog": "0", "opertype": IF(overwrite, "3", "1"), }, &resp, isFamily) if err != nil { return nil, err } return resp.toFile(), nil } // 获取上传切片信息 // 对http body有大小限制，分片信息太多会出错 func (y *Cloud189PC) GetMultiUploadUrls(ctx context.Context, isFamily bool, uploadFileId string, partInfo ...string) ([]UploadUrlInfo, error) { fullUrl := UPLOAD_URL if isFamily { fullUrl += "/family" } else { fullUrl += "/person" } var uploadUrlsResp UploadUrlsResp _, err := y.request(fullUrl+"/getMultiUploadUrls", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }, Params{ "uploadFileId": uploadFileId, "partInfo": strings.Join(partInfo, ","), }, &uploadUrlsResp, isFamily) if err != nil { return nil, err } uploadUrls := uploadUrlsResp.Data if len(uploadUrls) != len(partInfo) { return nil, fmt.Errorf("uploadUrls get error, due to get length %d, real length %d", len(partInfo), len(uploadUrls)) } uploadUrlInfos := make([]UploadUrlInfo, 0, len(uploadUrls)) for k, uploadUrl := range uploadUrls { partNumber, err := strconv.Atoi(strings.TrimPrefix(k, "partNumber_")) if err != nil { return nil, err } uploadUrlInfos = append(uploadUrlInfos, UploadUrlInfo{ PartNumber: partNumber, Headers: ParseHttpHeader(uploadUrl.RequestHeader), UploadUrlsData: uploadUrl, }) } sort.Slice(uploadUrlInfos, func(i, j int) bool { return uploadUrlInfos[i].PartNumber < uploadUrlInfos[j].PartNumber }) return uploadUrlInfos, nil } // 旧版本上传，家庭云不支持覆盖 func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { tempFile, fileMd5, err := stream.CacheFullAndHash(file, &up, utils.MD5) if err != nil { return nil, err } rateLimited := driver.NewLimitedUploadStream(ctx, io.NopCloser(tempFile)) // 创建上传会话 uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, file.GetName(), fmt.Sprint(file.GetSize()), isFamily) if err != nil { return nil, err } // 网盘中不存在该文件，开始上传 status := GetUploadFileStatusResp{CreateUploadFileResp: *uploadInfo} for status.GetSize() < file.GetSize() && status.FileDataExists != 1 { if utils.IsCanceled(ctx) { return nil, ctx.Err() } header := map[string]string{ "ResumePolicy": "1", "Expect": "100-continue", } if isFamily { header["FamilyId"] = fmt.Sprint(y.FamilyID) header["UploadFileId"] = fmt.Sprint(status.UploadFileId) } else { header["Edrive-UploadFileId"] = fmt.Sprint(status.UploadFileId) } _, err := y.put(ctx, status.FileUploadUrl, header, true, rateLimited, isFamily) if err, ok := err.(*RespErr); ok && err.Code != "InputStreamReadError" { return nil, err } // 获取断点状态 fullUrl := API_URL + "/getUploadFileStatus.action" if y.isFamily() { fullUrl = API_URL + "/family/file/getFamilyFileStatus.action" } _, err = y.get(fullUrl, func(req *resty.Request) { req.SetContext(ctx).SetQueryParams(map[string]string{ "uploadFileId": fmt.Sprint(status.UploadFileId), "resumePolicy": "1", }) if isFamily { req.SetQueryParam("familyId", fmt.Sprint(y.FamilyID)) } }, &status, isFamily) if err != nil { return nil, err } if _, err := tempFile.Seek(status.GetSize(), io.SeekStart); err != nil { return nil, err } up(float64(status.GetSize()) / float64(file.GetSize()) * 100) } return y.OldUploadCommit(ctx, status.FileCommitUrl, status.UploadFileId, isFamily, overwrite) } // 创建上传会话 func (y *Cloud189PC) OldUploadCreate(ctx context.Context, parentID string, fileMd5, fileName, fileSize string, isFamily bool) (*CreateUploadFileResp, error) { var uploadInfo CreateUploadFileResp fullUrl := API_URL + "/createUploadFile.action" if isFamily { fullUrl = API_URL + "/family/file/createFamilyFile.action" } _, err := y.post(fullUrl, func(req *resty.Request) { req.SetContext(ctx) if isFamily { req.SetQueryParams(map[string]string{ "familyId": y.FamilyID, "parentId": parentID, "fileMd5": fileMd5, "fileName": fileName, "fileSize": fileSize, "resumePolicy": "1", }) } else { req.SetFormData(map[string]string{ "parentFolderId": parentID, "fileName": fileName, "size": fileSize, "md5": fileMd5, "opertype": "3", "flag": "1", "resumePolicy": "1", "isLog": "0", }) } }, &uploadInfo, isFamily) if err != nil { return nil, err } return &uploadInfo, nil } // 提交上传文件 func (y *Cloud189PC) OldUploadCommit(ctx context.Context, fileCommitUrl string, uploadFileID int64, isFamily bool, overwrite bool) (model.Obj, error) { var resp OldCommitUploadFileResp _, err := y.post(fileCommitUrl, func(req *resty.Request) { req.SetContext(ctx) if isFamily { req.SetHeaders(map[string]string{ "ResumePolicy": "1", "UploadFileId": fmt.Sprint(uploadFileID), "FamilyId": fmt.Sprint(y.FamilyID), }) } else { req.SetFormData(map[string]string{ "opertype": IF(overwrite, "3", "1"), "resumePolicy": "1", "uploadFileId": fmt.Sprint(uploadFileID), "isLog": "0", }) } }, &resp, isFamily) if err != nil { return nil, err } return resp.toFile(), nil } func (y *Cloud189PC) isFamily() bool { return y.Type == "family" } func (y *Cloud189PC) isLogin() bool { if y.tokenInfo == nil { return false } _, err := y.get(API_URL+"/getUserInfo.action", nil, nil) return err == nil } // 创建家庭云中转文件夹 func (y *Cloud189PC) createFamilyTransferFolder() error { var rootFolder Cloud189Folder _, err := y.post(API_URL+"/family/file/createFolder.action", func(req *resty.Request) { req.SetQueryParams(map[string]string{ "folderName": "FamilyTransferFolder", "familyId": y.FamilyID, }) }, &rootFolder, true) if err != nil { return err } y.familyTransferFolder = &rootFolder return nil } // 清理中转文件夹 func (y *Cloud189PC) cleanFamilyTransfer(ctx context.Context) error { transferFolderId := y.familyTransferFolder.GetID() for pageNum := 1; ; pageNum++ { resp, err := y.getFilesWithPage(ctx, transferFolderId, true, pageNum, 100, "lastOpTime", "asc") if err != nil { return err } // 获取完毕跳出 if resp.FileListAO.Count == 0 { break } var tasks []BatchTaskInfo for i := 0; i < len(resp.FileListAO.FolderList); i++ { folder := resp.FileListAO.FolderList[i] tasks = append(tasks, BatchTaskInfo{ FileId: folder.GetID(), FileName: folder.GetName(), IsFolder: BoolToNumber(folder.IsDir()), }) } for i := 0; i < len(resp.FileListAO.FileList); i++ { file := resp.FileListAO.FileList[i] tasks = append(tasks, BatchTaskInfo{ FileId: file.GetID(), FileName: file.GetName(), IsFolder: BoolToNumber(file.IsDir()), }) } if len(tasks) > 0 { // 删除 resp, err := y.CreateBatchTask("DELETE", y.FamilyID, "", nil, tasks...) if err != nil { return err } err = y.WaitBatchTask("DELETE", resp.TaskID, time.Second) if err != nil { return err } // 永久删除 resp, err = y.CreateBatchTask("CLEAR_RECYCLE", y.FamilyID, "", nil, tasks...) if err != nil { return err } err = y.WaitBatchTask("CLEAR_RECYCLE", resp.TaskID, time.Second) return err } } return nil } // 获取家庭云所有用户信息 func (y *Cloud189PC) getFamilyInfoList() ([]FamilyInfoResp, error) { var resp FamilyInfoListResp _, err := y.get(API_URL+"/family/manage/getFamilyList.action", nil, &resp, true) if err != nil { return nil, err } return resp.FamilyInfoResp, nil } // 抽取家庭云ID func (y *Cloud189PC) getFamilyID() (string, error) { infos, err := y.getFamilyInfoList() if err != nil { return "", err } if len(infos) == 0 { return "", fmt.Errorf("cannot get automatically,please input family_id") } for _, info := range infos { if strings.Contains(y.getTokenInfo().LoginName, info.RemarkName) { return fmt.Sprint(info.FamilyID), nil } } return fmt.Sprint(infos[0].FamilyID), nil } // 保存家庭云中的文件到个人云 func (y *Cloud189PC) SaveFamilyFileToPersonCloud(ctx context.Context, familyId string, srcObj, dstDir model.Obj, overwrite bool) error { // _, err := y.post(API_URL+"/family/file/saveFileToMember.action", func(req *resty.Request) { // req.SetQueryParams(map[string]string{ // "channelId": "home", // "familyId": familyId, // "destParentId": destParentId, // "fileIdList": familyFileId, // }) // }, nil) // return err task := BatchTaskInfo{ FileId: srcObj.GetID(), FileName: srcObj.GetName(), IsFolder: BoolToNumber(srcObj.IsDir()), } resp, err := y.CreateBatchTask("COPY", familyId, dstDir.GetID(), map[string]string{ "groupId": "null", "copyType": "2", "shareId": "null", }, task) if err != nil { return err } for { state, err := y.CheckBatchTask("COPY", resp.TaskID) if err != nil { return err } switch state.TaskStatus { case 2: task.DealWay = IF(overwrite, 3, 2) // 冲突时覆盖文件 if err := y.ManageBatchTask("COPY", resp.TaskID, dstDir.GetID(), task); err != nil { return err } case 4: return nil } time.Sleep(time.Millisecond * 400) } } // 永久删除文件 func (y *Cloud189PC) Delete(ctx context.Context, familyId string, srcObj model.Obj) error { task := BatchTaskInfo{ FileId: srcObj.GetID(), FileName: srcObj.GetName(), IsFolder: BoolToNumber(srcObj.IsDir()), } // 删除源文件 resp, err := y.CreateBatchTask("DELETE", familyId, "", nil, task) if err != nil { return err } err = y.WaitBatchTask("DELETE", resp.TaskID, time.Second) if err != nil { return err } // 清除回收站 resp, err = y.CreateBatchTask("CLEAR_RECYCLE", familyId, "", nil, task) if err != nil { return err } err = y.WaitBatchTask("CLEAR_RECYCLE", resp.TaskID, time.Second) if err != nil { return err } return nil } func (y *Cloud189PC) CreateBatchTask(aType string, familyID string, targetFolderId string, other map[string]string, taskInfos ...BatchTaskInfo) (*CreateBatchTaskResp, error) { var resp CreateBatchTaskResp _, err := y.post(API_URL+"/batch/createBatchTask.action", func(req *resty.Request) { req.SetFormData(map[string]string{ "type": aType, "taskInfos": MustString(utils.Json.MarshalToString(taskInfos)), }) if targetFolderId != "" { req.SetFormData(map[string]string{"targetFolderId": targetFolderId}) } if familyID != "" { req.SetFormData(map[string]string{"familyId": familyID}) } req.SetFormData(other) }, &resp, familyID != "") if err != nil { return nil, err } return &resp, nil } // 检测任务状态 func (y *Cloud189PC) CheckBatchTask(aType string, taskID string) (*BatchTaskStateResp, error) { var resp BatchTaskStateResp _, err := y.post(API_URL+"/batch/checkBatchTask.action", func(req *resty.Request) { req.SetFormData(map[string]string{ "type": aType, "taskId": taskID, }) }, &resp) if err != nil { return nil, err } return &resp, nil } // 获取冲突的任务信息 func (y *Cloud189PC) GetConflictTaskInfo(aType string, taskID string) (*BatchTaskConflictTaskInfoResp, error) { var resp BatchTaskConflictTaskInfoResp _, err := y.post(API_URL+"/batch/getConflictTaskInfo.action", func(req *resty.Request) { req.SetFormData(map[string]string{ "type": aType, "taskId": taskID, }) }, &resp) if err != nil { return nil, err } return &resp, nil } // 处理冲突 func (y *Cloud189PC) ManageBatchTask(aType string, taskID string, targetFolderId string, taskInfos ...BatchTaskInfo) error { _, err := y.post(API_URL+"/batch/manageBatchTask.action", func(req *resty.Request) { req.SetFormData(map[string]string{ "targetFolderId": targetFolderId, "type": aType, "taskId": taskID, "taskInfos": MustString(utils.Json.MarshalToString(taskInfos)), }) }, nil) return err } var ErrIsConflict = errors.New("there is a conflict with the target object") // 等待任务完成 func (y *Cloud189PC) WaitBatchTask(aType string, taskID string, t time.Duration) error { for { state, err := y.CheckBatchTask(aType, taskID) if err != nil { return err } switch state.TaskStatus { case 2: return ErrIsConflict case 4: return nil } time.Sleep(t) } } func (y *Cloud189PC) getTokenInfo() *AppSessionResp { if y.ref != nil { return y.ref.getTokenInfo() } return y.tokenInfo } func (y *Cloud189PC) getClient() *resty.Client { if y.ref != nil { return y.ref.getClient() } return y.client } func (y *Cloud189PC) getCapacityInfo(ctx context.Context) (*CapacityResp, error) { fullUrl := API_URL + "/portal/getUserSizeInfo.action" var resp CapacityResp _, err := y.get(fullUrl, func(req *resty.Request) { req.SetContext(ctx) }, &resp) if err != nil { return nil, err } return &resp, nil } ================================================ FILE: drivers/alias/driver.go ================================================ package alias import ( "context" "errors" "fmt" "io" "math/rand" "net/url" stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" ) type Alias struct { model.Storage Addition rootOrder []string pathMap map[string][]string root model.Obj } func (d *Alias) Config() driver.Config { return config } func (d *Alias) GetAddition() driver.Additional { return &d.Addition } func (d *Alias) Init(ctx context.Context) error { paths := strings.Split(d.Paths, "\n") d.rootOrder = make([]string, 0, len(paths)) d.pathMap = make(map[string][]string) for _, path := range paths { path = strings.TrimSpace(path) if path == "" { continue } k, v := getPair(path) temp, ok := d.pathMap[k] if !ok { d.rootOrder = append(d.rootOrder, k) } d.pathMap[k] = append(temp, v) } switch len(d.rootOrder) { case 0: return errors.New("paths is required") case 1: paths := d.pathMap[d.rootOrder[0]] roots := make(BalancedObjs, 0, len(paths)) roots = append(roots, &model.Object{ Name: "root", Path: paths[0], IsFolder: true, Modified: d.Modified, Mask: model.Locked, }) for _, path := range paths[1:] { roots = append(roots, &model.Object{ Path: path, }) } d.root = roots default: d.root = &model.Object{ Name: "root", Path: "/", IsFolder: true, Modified: d.Modified, Mask: model.ReadOnly, } } if !utils.SliceContains(ValidReadConflictPolicy, d.ReadConflictPolicy) { d.ReadConflictPolicy = FirstRWP } if !utils.SliceContains(ValidWriteConflictPolicy, d.WriteConflictPolicy) { d.WriteConflictPolicy = DisabledWP } if !utils.SliceContains(ValidPutConflictPolicy, d.PutConflictPolicy) { d.PutConflictPolicy = DisabledWP } return nil } func (d *Alias) Drop(ctx context.Context) error { d.rootOrder = nil d.pathMap = nil d.root = nil return nil } func (d *Alias) GetRoot(ctx context.Context) (model.Obj, error) { if d.root == nil { return nil, errs.StorageNotInit } return d.root, nil } // 通过op.Get调用的话，path一定是子路径(/开头) func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { roots, sub := d.getRootsAndPath(path) if len(roots) == 0 { return nil, errs.ObjectNotFound } for idx, root := range roots { rawPath := stdpath.Join(root, sub) obj, err := fs.Get(ctx, rawPath, &fs.GetArgs{NoLog: true}) if err != nil { continue } mask := model.GetObjMask(obj) &^ model.Temp if sub == "" { // 根目录 mask |= model.Locked | model.Virtual } ret := model.Object{ Path: rawPath, Name: obj.GetName(), Size: obj.GetSize(), Modified: obj.ModTime(), IsFolder: obj.IsDir(), HashInfo: obj.GetHash(), Mask: mask, } obj = &ret if d.ProviderPassThrough && !obj.IsDir() { if storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{}); err == nil { obj = &model.ObjectProvider{ Object: ret, Provider: model.Provider{ Provider: storage.Config().Name, }, } } } roots = roots[idx+1:] var objs BalancedObjs if idx > 0 { objs = make(BalancedObjs, 0, len(roots)+2) } else { objs = make(BalancedObjs, 0, len(roots)+1) } objs = append(objs, obj) if idx > 0 { objs = append(objs, nil) } for _, d := range roots { objs = append(objs, &tempObj{model.Object{ Path: stdpath.Join(d, sub), }}) } return objs, nil } return nil, errs.ObjectNotFound } func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { dirs, ok := dir.(BalancedObjs) if !ok { return d.listRoot(ctx, args.WithStorageDetails && d.DetailsPassThrough, args.Refresh), nil } // 因为alias是NoCache且Get方法不会返回NotSupport或NotImplement错误 // 所以这里对象不会传回到alias，也就不需要返回BalancedObjs了 objMap := make(map[string]model.Obj) for _, dir := range dirs { if dir == nil { continue } dirPath := dir.GetPath() tmp, err := fs.List(ctx, dirPath, &fs.ListArgs{ NoLog: true, Refresh: args.Refresh, WithStorageDetails: args.WithStorageDetails && d.DetailsPassThrough, }) if err != nil { continue } for _, obj := range tmp { name := obj.GetName() if _, exists := objMap[name]; exists { continue } mask := model.GetObjMask(obj) &^ model.Temp objRes := model.Object{ Name: name, Path: stdpath.Join(dirPath, name), Size: obj.GetSize(), Modified: obj.ModTime(), IsFolder: obj.IsDir(), Mask: mask, } var objRet model.Obj if thumb, ok := model.GetThumb(obj); ok { objRet = &model.ObjThumb{ Object: objRes, Thumbnail: model.Thumbnail{ Thumbnail: thumb, }, } } else { objRet = &objRes } if details, ok := model.GetStorageDetails(obj); ok { objRet = &model.ObjStorageDetails{ Obj: objRet, StorageDetails: details, } } objMap[name] = objRet } } objs := make([]model.Obj, 0, len(objMap)) for _, obj := range objMap { objs = append(objs, obj) } if d.OrderBy == "" { sort := getAllSort(dirs) if sort.OrderBy != "" { model.SortFiles(objs, sort.OrderBy, sort.OrderDirection) } if d.ExtractFolder == "" && sort.ExtractFolder != "" { model.ExtractFolder(objs, sort.ExtractFolder) } } return objs, nil } func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if d.ReadConflictPolicy == AllRWP && !args.Redirect { files, err := d.getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllRWP)) if err != nil { return nil, err } linkClosers := make([]io.Closer, 0, len(files)) rrf := make([]model.RangeReaderIF, 0, len(files)) for _, f := range files { link, fi, err := d.link(ctx, f.GetPath(), args) if err != nil { continue } if fi.GetSize() != files.GetSize() { _ = link.Close() continue } l := *link // 复制一份，避免修改到原始link if l.ContentLength == 0 { l.ContentLength = fi.GetSize() } if d.DownloadConcurrency > 0 { l.Concurrency = d.DownloadConcurrency } if d.DownloadPartSize > 0 { l.PartSize = d.DownloadPartSize * utils.KB } rr, err := stream.GetRangeReaderFromLink(l.ContentLength, &l) if err != nil { _ = link.Close() continue } linkClosers = append(linkClosers, link) rrf = append(rrf, rr) } rr := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { return rrf[rand.Intn(len(rrf))].RangeRead(ctx, httpRange) } return &model.Link{ RangeReader: stream.RangeReaderFunc(rr), SyncClosers: utils.NewSyncClosers(linkClosers...), }, nil } var link *model.Link var fi model.Obj var err error files := file.(BalancedObjs) if d.ReadConflictPolicy == RandomBalancedRP || d.ReadConflictPolicy == AllRWP { rand.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] }) } for _, f := range files { if f == nil { continue } link, fi, err = d.link(ctx, f.GetPath(), args) if err == nil { if link == nil { // 重定向且需要通过代理 return &model.Link{ URL: fmt.Sprintf("%s/p%s?sign=%s", common.GetApiUrl(ctx), utils.EncodePath(f.GetPath(), true), sign.Sign(f.GetPath())), }, nil } break } } if err != nil { return nil, err } resultLink := *link // 复制一份，避免修改到原始link resultLink.Expiration = nil resultLink.SyncClosers = utils.NewSyncClosers(link) if args.Redirect { return &resultLink, nil } if resultLink.ContentLength == 0 { resultLink.ContentLength = fi.GetSize() } if d.DownloadConcurrency > 0 { resultLink.Concurrency = d.DownloadConcurrency } if d.DownloadPartSize > 0 { resultLink.PartSize = d.DownloadPartSize * utils.KB } return &resultLink, nil } func (d *Alias) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // Other 不应负载均衡，这是因为前端是否调用 /fs/other 的判断条件是返回的 provider 的值 // 而 ProviderPassThrough 开启时，返回的 provider 固定为第一个 obj 的后端驱动 storage, actualPath, err := op.GetStorageAndActualPath(args.Obj.GetPath()) if err != nil { return nil, err } return op.Other(ctx, storage, model.FsOtherArgs{ Path: actualPath, Method: args.Method, Data: args.Data, }) } func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { objs, err := d.getWriteObjs(ctx, parentDir) if err == nil { for _, obj := range objs { err = errors.Join(err, fs.MakeDir(ctx, stdpath.Join(obj.GetPath(), dirName))) } } return err } func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { srcs, dsts, err := d.getMoveObjs(ctx, srcObj, dstDir) if err == nil { for i, dst := range dsts { src := srcs[i] _, e := fs.Move(ctx, src.GetPath(), dst.GetPath()) err = errors.Join(err, e) } srcs = srcs[len(dsts):] for _, src := range srcs { e := fs.Remove(ctx, src.GetPath()) err = errors.Join(err, e) } } return err } func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) error { objs, err := d.getWriteObjs(ctx, srcObj) if err == nil { for _, obj := range objs { err = errors.Join(err, fs.Rename(ctx, obj.GetPath(), newName)) } } return err } func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { srcs, dsts, err := d.getCopyObjs(ctx, srcObj, dstDir) if err == nil { for i, src := range srcs { dst := dsts[i] _, e := fs.Copy(ctx, src.GetPath(), dst.GetPath()) err = errors.Join(err, e) } } return err } func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { objs, err := d.getWriteObjs(ctx, obj) if err == nil { for _, obj := range objs { err = errors.Join(err, fs.Remove(ctx, obj.GetPath())) } } return err } func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { objs, err := d.getPutObjs(ctx, dstDir) if err == nil { if len(objs) == 1 { storage, reqActualPath, err := op.GetStorageAndActualPath(objs.GetPath()) if err != nil { return err } return op.Put(ctx, storage, reqActualPath, &stream.FileStream{ Obj: s, Mimetype: s.GetMimetype(), Reader: s, }, up) } else { file, err := s.CacheFullAndWriter(nil, nil) if err != nil { return err } count := float64(len(objs) + 1) up(100 / count) for i, obj := range objs { err = errors.Join(err, fs.PutDirectly(ctx, obj.GetPath(), &stream.FileStream{ Obj: s, Mimetype: s.GetMimetype(), Reader: file, })) up(float64(i+2) / float64(count) * 100) _, e := file.Seek(0, io.SeekStart) if e != nil { return errors.Join(err, e) } } return err } } return err } func (d *Alias) PutURL(ctx context.Context, dstDir model.Obj, name, url string) error { objs, err := d.getPutObjs(ctx, dstDir) if err == nil { for _, obj := range objs { err = errors.Join(err, fs.PutURL(ctx, obj.GetPath(), name, url)) } return err } return err } func (d *Alias) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { reqPath := d.getBalancedPath(ctx, obj) if reqPath == "" { return nil, errs.NotFile } meta, err := d.getArchiveMeta(ctx, reqPath, args) if err == nil { return meta, nil } return nil, errs.NotImplement } func (d *Alias) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { reqPath := d.getBalancedPath(ctx, obj) if reqPath == "" { return nil, errs.NotFile } l, err := d.listArchive(ctx, reqPath, args) if err == nil { return l, nil } return nil, errs.NotImplement } func (d *Alias) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { // alias的两个驱动，一个支持驱动提取，一个不支持，如何兼容？ // 如果访问的是不支持驱动提取的驱动内的压缩文件，GetArchiveMeta就会返回errs.NotImplement，提取URL前缀就会是/ae，Extract就不会被调用 // 如果访问的是支持驱动提取的驱动内的压缩文件，GetArchiveMeta就会返回有效值，提取URL前缀就会是/ad，Extract就会被调用 reqPath := d.getBalancedPath(ctx, obj) if reqPath == "" { return nil, errs.NotFile } link, err := d.extract(ctx, reqPath, args) if err != nil { return nil, errs.NotImplement } if link == nil { return &model.Link{ URL: fmt.Sprintf("%s/ap%s?inner=%s&pass=%s&sign=%s", common.GetApiUrl(ctx), utils.EncodePath(reqPath, true), utils.EncodePath(args.InnerPath, true), url.QueryEscape(args.Password), sign.SignArchive(reqPath)), }, nil } resultLink := *link resultLink.SyncClosers = utils.NewSyncClosers(link) return &resultLink, nil } func (d *Alias) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { srcs, dsts, err := d.getCopyObjs(ctx, srcObj, dstDir) if err == nil { for i, src := range srcs { dst := dsts[i] _, e := fs.ArchiveDecompress(ctx, src.GetPath(), dst.GetPath(), args) err = errors.Join(err, e) } } return err } func (d *Alias) GetDetails(ctx context.Context) (*model.StorageDetails, error) { if !d.DetailsPassThrough { return nil, errs.NotImplement } if len(d.rootOrder) != 1 { return nil, errs.NotImplement } backends := d.pathMap[d.rootOrder[0]] var storage driver.Driver for _, backend := range backends { s, err := fs.GetStorage(backend, &fs.GetStoragesArgs{}) if err != nil { return nil, errs.NotImplement } if storage == nil { storage = s } else if storage.GetStorage().MountPath != s.GetStorage().MountPath { return nil, errs.NotImplement } } if storage == nil { // should never access return nil, errs.NotImplement } return op.GetStorageDetails(ctx, storage) } func (d *Alias) ResolveLinkCacheMode(path string) driver.LinkCacheMode { roots, sub := d.getRootsAndPath(path) if len(roots) == 0 { return 0 } for _, root := range roots { storage, actualPath, err := op.GetStorageAndActualPath(stdpath.Join(root, sub)) if err != nil { continue } if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { continue } mode := storage.Config().LinkCacheMode if mode == -1 { return storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(actualPath) } else { return mode } } return 0 } var _ driver.Driver = (*Alias)(nil) ================================================ FILE: drivers/alias/meta.go ================================================ package alias import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { Paths string `json:"paths" required:"true" type:"text"` ReadConflictPolicy string `json:"read_conflict_policy" type:"select" options:"first,random,all" default:"first"` WriteConflictPolicy string `json:"write_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict" default:"disabled" help:"How the driver handles identical backend paths when renaming, removing, or making directories."` PutConflictPolicy string `json:"put_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict,random,quota,quota_strict" default:"disabled" help:"How the driver handles identical backend paths when uploading, copying, moving, or decompressing."` FileConsistencyCheck bool `json:"file_consistency_check" type:"bool" default:"false"` DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"` DownloadPartSize int `json:"download_part_size" default:"0" type:"number" required:"false" help:"Need to enable proxy. Unit: KB"` ProviderPassThrough bool `json:"provider_pass_through" type:"bool" default:"false"` DetailsPassThrough bool `json:"details_pass_through" type:"bool" default:"false"` } var config = driver.Config{ Name: "Alias", LocalSort: true, NoCache: true, NoUpload: false, DefaultRoot: "/", ProxyRangeOption: true, LinkCacheMode: driver.LinkCacheAuto, } func init() { op.RegisterDriver(func() driver.Driver { return &Alias{} }) } ================================================ FILE: drivers/alias/types.go ================================================ package alias import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" ) const ( DisabledWP = "disabled" FirstRWP = "first" DeterministicWP = "deterministic" DeterministicOrAllWP = "deterministic_or_all" AllRWP = "all" AllStrictWP = "all_strict" RandomBalancedRP = "random" BalancedByQuotaP = "quota" BalancedByQuotaStrictP = "quota_strict" ) var ( ValidReadConflictPolicy = []string{FirstRWP, RandomBalancedRP, AllRWP} ValidWriteConflictPolicy = []string{DisabledWP, FirstRWP, DeterministicWP, DeterministicOrAllWP, AllRWP, AllStrictWP} ValidPutConflictPolicy = []string{DisabledWP, FirstRWP, DeterministicWP, DeterministicOrAllWP, AllRWP, AllStrictWP, RandomBalancedRP, BalancedByQuotaP, BalancedByQuotaStrictP} ) var ( ErrPathConflict = errors.New("path conflict") ErrSamePathLeak = errors.New("leak some of same-name dirs") ErrNoEnoughSpace = errors.New("none of same-name dirs has enough space") ErrNotEnoughSrcObjs = errors.New("cannot move fewer objs to more paths, please try copying") ) type BalancedObjs []model.Obj func (b BalancedObjs) GetSize() int64 { return b[0].GetSize() } func (b BalancedObjs) ModTime() time.Time { return b[0].ModTime() } func (b BalancedObjs) CreateTime() time.Time { return b[0].CreateTime() } func (b BalancedObjs) IsDir() bool { return b[0].IsDir() } func (b BalancedObjs) GetHash() utils.HashInfo { return b[0].GetHash() } func (b BalancedObjs) GetName() string { return b[0].GetName() } func (b BalancedObjs) GetPath() string { return b[0].GetPath() } func (b BalancedObjs) GetID() string { return b[0].GetID() } func (b BalancedObjs) Unwrap() model.Obj { return b[0] } var _ model.Obj = (BalancedObjs)(nil) type tempObj struct{ model.Object } ================================================ FILE: drivers/alias/util.go ================================================ package alias import ( "context" "math/rand" stdpath "path" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) type detailWithIndex struct { idx int val *model.StorageDetails } func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model.Obj { var objs []model.Obj detailsChan := make(chan detailWithIndex, len(d.pathMap)) workerCount := 0 for _, k := range d.rootOrder { obj := &model.Object{ Name: k, Path: "/" + k, IsFolder: true, Modified: d.Modified, Mask: model.Locked | model.Virtual, } idx := len(objs) objs = append(objs, obj) v := d.pathMap[k] if !withDetails || len(v) != 1 { continue } remoteDriver, err := op.GetStorageByMountPath(v[0]) if err != nil { continue } obj.Modified = remoteDriver.GetStorage().Modified _, ok := remoteDriver.(driver.WithDetails) if !ok { continue } objs[idx] = &model.ObjStorageDetails{ Obj: objs[idx], StorageDetails: nil, } workerCount++ go func(dri driver.Driver, i int) { details, e := op.GetStorageDetails(ctx, dri, refresh) if e != nil { if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) { log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, e) } } detailsChan <- detailWithIndex{idx: i, val: details} }(remoteDriver, idx) } for workerCount > 0 { select { case r := <-detailsChan: objs[r.idx].(*model.ObjStorageDetails).StorageDetails = r.val workerCount-- case <-time.After(time.Second): workerCount = 0 } } return objs } // do others that not defined in Driver interface func getPair(path string) (string, string) { if name, path, ok := strings.Cut(path, ":"); ok && !strings.Contains(name, "/") { return name, path } return stdpath.Base(path), path } func (d *Alias) getRootsAndPath(path string) (roots []string, sub string) { if len(d.rootOrder) == 1 { return d.pathMap[d.rootOrder[0]], path } path = strings.TrimPrefix(path, "/") before, after, ok := strings.Cut(path, "/") if !ok { return d.pathMap[path], "" } return d.pathMap[before], after } func (d *Alias) link(ctx context.Context, reqPath string, args model.LinkArgs) (*model.Link, model.Obj, error) { storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { return nil, nil, err } if args.Redirect && common.ShouldProxy(storage, stdpath.Base(reqPath)) { return nil, nil, nil } return op.Link(ctx, storage, reqActualPath, args) } func isConsistent(a, b model.Obj) bool { if a.GetSize() != b.GetSize() { return false } for ht, v := range a.GetHash().All() { ah := b.GetHash().GetHash(ht) if ah != "" && ah != v { return false } } return true } func (d *Alias) getAllObjs(ctx context.Context, bObj model.Obj, ifContinue func(err error) (bool, error)) (BalancedObjs, error) { objs := bObj.(BalancedObjs) length := 0 for _, o := range objs { var err error var obj model.Obj temp, isTemp := o.(*tempObj) if isTemp { obj, err = fs.Get(ctx, o.GetPath(), &fs.GetArgs{NoLog: true}) if err == nil { if !bObj.IsDir() { if obj.IsDir() { err = errs.NotFile } else if d.FileConsistencyCheck && !isConsistent(bObj, obj) { err = errs.ObjectNotFound } } else if !obj.IsDir() { err = errs.NotFolder } } } else if o == nil { err = errs.ObjectNotFound } cont, err := ifContinue(err) if err != nil { if cont { continue } return nil, err } if isTemp { objRes := temp.Object // objRes.Name = obj.GetName() // objRes.Size = obj.GetSize() // objRes.Modified = obj.ModTime() // objRes.HashInfo = obj.GetHash() objs[length] = &objRes } else { objs[length] = o } length++ if !cont { break } } if length == 0 { return nil, errs.ObjectNotFound } return objs[:length], nil } func (d *Alias) getBalancedPath(ctx context.Context, file model.Obj) string { if d.ReadConflictPolicy == FirstRWP { return file.GetPath() } files := file.(BalancedObjs) if rand.Intn(len(files)) == 0 { return file.GetPath() } files, _ = d.getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllRWP)) return files[rand.Intn(len(files))].GetPath() } func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { if policy == AllRWP { return func(err error) (bool, error) { return true, err } } all := true l := 0 return func(err error) (bool, error) { if err != nil { switch policy { case AllStrictWP: return false, ErrSamePathLeak case DeterministicOrAllWP: if l >= 2 { return false, ErrSamePathLeak } } all = false } else { switch policy { case FirstRWP: return false, nil case DeterministicWP: if l > 0 { return false, ErrPathConflict } case DeterministicOrAllWP: if l > 0 && !all { return false, ErrSamePathLeak } } l += 1 } return true, err } } func (d *Alias) getWriteObjs(ctx context.Context, obj model.Obj) (BalancedObjs, error) { if d.WriteConflictPolicy == DisabledWP { return nil, errs.PermissionDenied } return d.getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.WriteConflictPolicy)) } func (d *Alias) getPutObjs(ctx context.Context, obj model.Obj) (BalancedObjs, error) { if d.PutConflictPolicy == DisabledWP { return nil, errs.PermissionDenied } objs, err := d.getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.PutConflictPolicy)) if err != nil { return nil, err } strict := false switch d.PutConflictPolicy { case RandomBalancedRP: ri := rand.Intn(len(objs)) return objs[ri : ri+1], nil case BalancedByQuotaStrictP: strict = true fallthrough case BalancedByQuotaP: objs, ok := getRandomObjByQuotaBalanced(ctx, objs, strict, obj.GetSize()) if !ok { return nil, ErrNoEnoughSpace } return objs, nil default: return objs, nil } } func getRandomObjByQuotaBalanced(ctx context.Context, reqPath BalancedObjs, strict bool, objSize int64) (BalancedObjs, bool) { // Get all space details := make([]*model.StorageDetails, len(reqPath)) detailsChan := make(chan detailWithIndex, len(reqPath)) workerCount := 0 for i, p := range reqPath { s, err := fs.GetStorage(p.GetPath(), &fs.GetStoragesArgs{}) if err != nil { continue } if _, ok := s.(driver.WithDetails); !ok { continue } workerCount++ go func(dri driver.Driver, i int) { d, e := op.GetStorageDetails(ctx, dri) if e != nil { if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) { log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, e) } } detailsChan <- detailWithIndex{idx: i, val: d} }(s, i) } for workerCount > 0 { select { case r := <-detailsChan: details[r.idx] = r.val workerCount-- case <-time.After(time.Second): workerCount = 0 } } // Try select one that has space info selected, ok := selectRandom(details, func(d *model.StorageDetails) uint64 { if d == nil || d.FreeSpace() < objSize { return 0 } return uint64(d.FreeSpace()) }) if !ok { if strict { return nil, false } else { // No strict mode, return any of non-details ones noDetails := make([]int, 0, len(details)) for i, d := range details { if d == nil { noDetails = append(noDetails, i) } } if len(noDetails) == 0 { return nil, false } selected = noDetails[rand.Intn(len(noDetails))] } } return reqPath[selected : selected+1], true } func selectRandom[Item any](arr []Item, getWeight func(Item) uint64) (int, bool) { var totalWeight uint64 = 0 for _, i := range arr { totalWeight += getWeight(i) } if totalWeight == 0 { return 0, false } r := rand.Uint64() % totalWeight for i, item := range arr { w := getWeight(item) if r < w { return i, true } r -= w } return 0, false } func (d *Alias) getCopyObjs(ctx context.Context, srcObj, dstDir model.Obj) (BalancedObjs, BalancedObjs, error) { if d.PutConflictPolicy == DisabledWP { return nil, nil, errs.PermissionDenied } dstObjs, err := d.getAllObjs(ctx, dstDir, getWriteAndPutFilterFunc(d.PutConflictPolicy)) if err != nil { return nil, nil, err } dstStorageMap := make(map[string][]model.Obj) allocatingDst := make(map[model.Obj]struct{}) for _, o := range dstObjs { storage, e := fs.GetStorage(o.GetPath(), &fs.GetStoragesArgs{}) if e != nil { return nil, nil, errors.WithMessagef(e, "cannot copy to virtual path [%s]", o.GetPath()) } mp := storage.GetStorage().MountPath dstStorageMap[mp] = append(dstStorageMap[mp], o) allocatingDst[o] = struct{}{} } tmpSrcObjs, err := d.getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllRWP)) if err != nil { return nil, nil, err } srcObjs := make(BalancedObjs, 0, len(dstObjs)) for _, src := range tmpSrcObjs { storage, e := fs.GetStorage(src.GetPath(), &fs.GetStoragesArgs{}) if e != nil { continue } mp := storage.GetStorage().MountPath if tmp, ok := dstStorageMap[mp]; ok { for _, dst := range tmp { dstObjs[len(srcObjs)] = dst srcObjs = append(srcObjs, src) delete(allocatingDst, dst) } delete(dstStorageMap, mp) } } dstObjs = dstObjs[:len(srcObjs)] for dst := range allocatingDst { src := tmpSrcObjs[0] if d.ReadConflictPolicy == RandomBalancedRP || d.ReadConflictPolicy == AllRWP { src = tmpSrcObjs[rand.Intn(len(tmpSrcObjs))] } srcObjs = append(srcObjs, src) dstObjs = append(dstObjs, dst) } return srcObjs, dstObjs, nil } func (d *Alias) getMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) (BalancedObjs, BalancedObjs, error) { if d.PutConflictPolicy == DisabledWP { return nil, nil, errs.PermissionDenied } dstObjs, err := d.getAllObjs(ctx, dstDir, getWriteAndPutFilterFunc(d.PutConflictPolicy)) if err != nil { return nil, nil, err } tmpSrcObjs, err := d.getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllRWP)) if err != nil { return nil, nil, err } if len(tmpSrcObjs) < len(dstObjs) { return nil, nil, ErrNotEnoughSrcObjs } dstStorageMap := make(map[string][]model.Obj) allocatingDst := make(map[model.Obj]struct{}) for _, o := range dstObjs { storage, e := fs.GetStorage(o.GetPath(), &fs.GetStoragesArgs{}) if e != nil { return nil, nil, errors.WithMessagef(e, "cannot move to virtual path [%s]", o.GetPath()) } mp := storage.GetStorage().MountPath dstStorageMap[mp] = append(dstStorageMap[mp], o) allocatingDst[o] = struct{}{} } srcObjs := make(BalancedObjs, 0, len(tmpSrcObjs)) restSrcObjs := make(BalancedObjs, 0, len(tmpSrcObjs)-len(dstObjs)) for _, src := range tmpSrcObjs { storage, e := fs.GetStorage(src.GetPath(), &fs.GetStoragesArgs{}) if e != nil { continue } mp := storage.GetStorage().MountPath if tmp, ok := dstStorageMap[mp]; ok { dst := tmp[0] if len(tmp) == 1 { delete(dstStorageMap, mp) } else { dstStorageMap[mp] = tmp[1:] } dstObjs[len(srcObjs)] = dst srcObjs = append(srcObjs, src) delete(allocatingDst, dst) } else { restSrcObjs = append(restSrcObjs, src) } } dstObjs = dstObjs[:len(srcObjs)] // len(restSrcObjs) >= len(allocatingDst) srcObjs = append(srcObjs, restSrcObjs...) for dst := range allocatingDst { dstObjs = append(dstObjs, dst) } return srcObjs, dstObjs, nil } func (d *Alias) getArchiveMeta(ctx context.Context, reqPath string, args model.ArchiveArgs) (model.ArchiveMeta, error) { storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { return nil, err } if _, ok := storage.(driver.ArchiveReader); ok { return op.GetArchiveMeta(ctx, storage, reqActualPath, model.ArchiveMetaArgs{ ArchiveArgs: args, Refresh: true, }) } return nil, errs.NotImplement } func (d *Alias) listArchive(ctx context.Context, reqPath string, args model.ArchiveInnerArgs) ([]model.Obj, error) { storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { return nil, err } if _, ok := storage.(driver.ArchiveReader); ok { return op.ListArchive(ctx, storage, reqActualPath, model.ArchiveListArgs{ ArchiveInnerArgs: args, Refresh: true, }) } return nil, errs.NotImplement } func (d *Alias) extract(ctx context.Context, reqPath string, args model.ArchiveInnerArgs) (*model.Link, error) { storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { return nil, err } if _, ok := storage.(driver.ArchiveReader); !ok { return nil, errs.NotImplement } if args.Redirect && common.ShouldProxy(storage, stdpath.Base(reqPath)) { _, err := fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) if err == nil { return nil, err } return nil, nil } link, _, err := op.DriverExtract(ctx, storage, reqActualPath, args) return link, err } func getAllSort(dirs []model.Obj) model.Sort { ret := model.Sort{} noSort := false noExtractFolder := false for _, dir := range dirs { if dir == nil { continue } storage, err := fs.GetStorage(dir.GetPath(), &fs.GetStoragesArgs{}) if err != nil { continue } if !noSort && storage.GetStorage().OrderBy != "" { if ret.OrderBy == "" { ret.OrderBy = storage.GetStorage().OrderBy ret.OrderDirection = storage.GetStorage().OrderDirection if ret.OrderDirection == "" { ret.OrderDirection = "asc" } } else if ret.OrderBy != storage.GetStorage().OrderBy || ret.OrderDirection != storage.GetStorage().OrderDirection { ret.OrderBy = "" ret.OrderDirection = "" noSort = true } } if !noExtractFolder && storage.GetStorage().ExtractFolder != "" { if ret.ExtractFolder == "" { ret.ExtractFolder = storage.GetStorage().ExtractFolder } else if ret.ExtractFolder != storage.GetStorage().ExtractFolder { ret.ExtractFolder = "" noExtractFolder = true } } if noSort && noExtractFolder { break } } return ret } ================================================ FILE: drivers/alist_v3/driver.go ================================================ package alist_v3 import ( "context" "fmt" "io" "net/http" "net/url" "path" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) type AListV3 struct { model.Storage Addition } func (d *AListV3) Config() driver.Config { return config } func (d *AListV3) GetAddition() driver.Additional { return &d.Addition } func (d *AListV3) Init(ctx context.Context) error { d.Addition.Address = strings.TrimSuffix(d.Addition.Address, "/") var resp common.Resp[MeResp] _, _, err := d.request("/me", http.MethodGet, func(req *resty.Request) { req.SetResult(&resp) }) if err != nil { return err } // if the username is not empty and the username is not the same as the current username, then login again if d.Username != resp.Data.Username { err = d.login() if err != nil { return err } } // re-get the user info _, _, err = d.request("/me", http.MethodGet, func(req *resty.Request) { req.SetResult(&resp) }) if err != nil { return err } if utils.SliceContains(resp.Data.Role, model.GUEST) { u := d.Address + "/api/public/settings" res, err := base.RestyClient.R().Get(u) if err != nil { return err } allowMounted := utils.Json.Get(res.Body(), "data", conf.AllowMounted).ToString() == "true" if !allowMounted { return fmt.Errorf("the site does not allow mounted") } } return err } func (d *AListV3) Drop(ctx context.Context) error { return nil } func (d *AListV3) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var resp common.Resp[FsListResp] _, _, err := d.request("/fs/list", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(ListReq{ PageReq: model.PageReq{ Page: 1, PerPage: 0, }, Path: dir.GetPath(), Password: d.MetaPassword, Refresh: false, }) }) if err != nil { return nil, err } var files []model.Obj for _, f := range resp.Data.Content { file := model.ObjThumb{ Object: model.Object{ Name: f.Name, Modified: f.Modified, Ctime: f.Created, Size: f.Size, IsFolder: f.IsDir, HashInfo: utils.FromString(f.HashInfo), }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumb}, } files = append(files, &file) } return files, nil } func (d *AListV3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp common.Resp[FsGetResp] headers := map[string]string{ "User-Agent": base.UserAgent, } // if PassUAToUpsteam is true, then pass the user-agent to the upstream if d.PassUAToUpsteam { userAgent := args.Header.Get("user-agent") if userAgent != "" { headers["User-Agent"] = userAgent } } // if PassIPToUpsteam is true, then pass the ip address to the upstream if d.PassIPToUpsteam { ip := args.IP if ip != "" { headers["X-Forwarded-For"] = ip headers["X-Real-Ip"] = ip } } _, _, err := d.request("/fs/get", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(FsGetReq{ Path: file.GetPath(), Password: d.MetaPassword, }).SetHeaders(headers) }) if err != nil { return nil, err } return &model.Link{ URL: resp.Data.RawURL, }, nil } func (d *AListV3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { _, _, err := d.request("/fs/mkdir", http.MethodPost, func(req *resty.Request) { req.SetBody(MkdirOrLinkReq{ Path: path.Join(parentDir.GetPath(), dirName), }) }) return err } func (d *AListV3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { _, _, err := d.request("/fs/move", http.MethodPost, func(req *resty.Request) { req.SetBody(MoveCopyReq{ SrcDir: path.Dir(srcObj.GetPath()), DstDir: dstDir.GetPath(), Names: []string{srcObj.GetName()}, }) }) return err } func (d *AListV3) Rename(ctx context.Context, srcObj model.Obj, newName string) error { _, _, err := d.request("/fs/rename", http.MethodPost, func(req *resty.Request) { req.SetBody(RenameReq{ Path: srcObj.GetPath(), Name: newName, }) }) return err } func (d *AListV3) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { _, _, err := d.request("/fs/copy", http.MethodPost, func(req *resty.Request) { req.SetBody(MoveCopyReq{ SrcDir: path.Dir(srcObj.GetPath()), DstDir: dstDir.GetPath(), Names: []string{srcObj.GetName()}, }) }) return err } func (d *AListV3) Remove(ctx context.Context, obj model.Obj) error { _, _, err := d.request("/fs/remove", http.MethodPost, func(req *resty.Request) { req.SetBody(RemoveReq{ Dir: path.Dir(obj.GetPath()), Names: []string{obj.GetName()}, }) }) return err } func (d *AListV3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, }) req, err := http.NewRequestWithContext(ctx, http.MethodPut, d.Address+"/api/fs/put", reader) if err != nil { return err } req.Header.Set("Authorization", d.Token) req.Header.Set("File-Path", path.Join(dstDir.GetPath(), s.GetName())) req.Header.Set("Password", d.MetaPassword) if md5 := s.GetHash().GetHash(utils.MD5); len(md5) > 0 { req.Header.Set("X-File-Md5", md5) } if sha1 := s.GetHash().GetHash(utils.SHA1); len(sha1) > 0 { req.Header.Set("X-File-Sha1", sha1) } if sha256 := s.GetHash().GetHash(utils.SHA256); len(sha256) > 0 { req.Header.Set("X-File-Sha256", sha256) } req.ContentLength = s.GetSize() // client := base.NewHttpClient() // client.Timeout = time.Hour * 6 res, err := base.HttpClient.Do(req) if err != nil { return err } bytes, err := io.ReadAll(res.Body) if err != nil { return err } log.Debugf("[openlist] response body: %s", string(bytes)) if res.StatusCode >= 400 { return fmt.Errorf("request failed, status: %s", res.Status) } code := utils.Json.Get(bytes, "code").ToInt() if code != 200 { if code == 401 || code == 403 { err = d.login() if err != nil { return err } } return fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(bytes, "message").ToString()) } return nil } func (d *AListV3) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { if !d.ForwardArchiveReq { return nil, errs.NotImplement } var resp common.Resp[ArchiveMetaResp] _, code, err := d.request("/fs/archive/meta", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(ArchiveMetaReq{ ArchivePass: args.Password, Password: d.MetaPassword, Path: obj.GetPath(), Refresh: false, }) }) if code == 202 { return nil, errs.WrongArchivePassword } if err != nil { return nil, err } var tree []model.ObjTree if resp.Data.Content != nil { tree = make([]model.ObjTree, 0, len(resp.Data.Content)) for _, content := range resp.Data.Content { tree = append(tree, &content) } } return &model.ArchiveMetaInfo{ Comment: resp.Data.Comment, Encrypted: resp.Data.Encrypted, Tree: tree, }, nil } func (d *AListV3) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { if !d.ForwardArchiveReq { return nil, errs.NotImplement } var resp common.Resp[ArchiveListResp] _, code, err := d.request("/fs/archive/list", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(ArchiveListReq{ ArchiveMetaReq: ArchiveMetaReq{ ArchivePass: args.Password, Password: d.MetaPassword, Path: obj.GetPath(), Refresh: false, }, PageReq: model.PageReq{ Page: 1, PerPage: 0, }, InnerPath: args.InnerPath, }) }) if code == 202 { return nil, errs.WrongArchivePassword } if err != nil { return nil, err } var files []model.Obj for _, f := range resp.Data.Content { file := model.ObjThumb{ Object: model.Object{ Name: f.Name, Modified: f.Modified, Ctime: f.Created, Size: f.Size, IsFolder: f.IsDir, HashInfo: utils.FromString(f.HashInfo), }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumb}, } files = append(files, &file) } return files, nil } func (d *AListV3) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { if !d.ForwardArchiveReq { return nil, errs.NotSupport } var resp common.Resp[ArchiveMetaResp] _, _, err := d.request("/fs/archive/meta", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(ArchiveMetaReq{ ArchivePass: args.Password, Password: d.MetaPassword, Path: obj.GetPath(), Refresh: false, }) }) if err != nil { return nil, err } return &model.Link{ URL: fmt.Sprintf("%s?inner=%s&pass=%s&sign=%s", resp.Data.RawURL, utils.EncodePath(args.InnerPath, true), url.QueryEscape(args.Password), resp.Data.Sign), }, nil } func (d *AListV3) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { if !d.ForwardArchiveReq { return errs.NotImplement } dir, name := path.Split(srcObj.GetPath()) _, _, err := d.request("/fs/archive/decompress", http.MethodPost, func(req *resty.Request) { req.SetBody(DecompressReq{ ArchivePass: args.Password, CacheFull: args.CacheFull, DstDir: dstDir.GetPath(), InnerPath: args.InnerPath, Name: []string{name}, PutIntoNewDir: args.PutIntoNewDir, SrcDir: dir, }) }) return err } func (d *AListV3) ResolveLinkCacheMode(_ string) driver.LinkCacheMode { var mode driver.LinkCacheMode if d.PassIPToUpsteam { mode |= driver.LinkCacheIP } if d.PassUAToUpsteam { mode |= driver.LinkCacheUA } return mode } var _ driver.Driver = (*AListV3)(nil) ================================================ FILE: drivers/alist_v3/meta.go ================================================ package alist_v3 import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Address string `json:"url" required:"true"` MetaPassword string `json:"meta_password"` Username string `json:"username"` Password string `json:"password"` Token string `json:"token"` PassIPToUpsteam bool `json:"pass_ip_to_upsteam" default:"true"` PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"` ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"` } var config = driver.Config{ Name: "AList V3", LocalSort: true, DefaultRoot: "/", ProxyRangeOption: true, LinkCacheMode: driver.LinkCacheAuto, } func init() { op.RegisterDriver(func() driver.Driver { return &AListV3{} }) } ================================================ FILE: drivers/alist_v3/types.go ================================================ package alist_v3 import ( "encoding/json" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type ListReq struct { model.PageReq Path string `json:"path" form:"path"` Password string `json:"password" form:"password"` Refresh bool `json:"refresh"` } type ObjResp struct { Name string `json:"name"` Size int64 `json:"size"` IsDir bool `json:"is_dir"` Modified time.Time `json:"modified"` Created time.Time `json:"created"` Sign string `json:"sign"` Thumb string `json:"thumb"` Type int `json:"type"` HashInfo string `json:"hashinfo"` } type FsListResp struct { Content []ObjResp `json:"content"` Total int64 `json:"total"` Readme string `json:"readme"` Write bool `json:"write"` Provider string `json:"provider"` } type FsGetReq struct { Path string `json:"path" form:"path"` Password string `json:"password" form:"password"` } type FsGetResp struct { ObjResp RawURL string `json:"raw_url"` Readme string `json:"readme"` Provider string `json:"provider"` Related []ObjResp `json:"related"` } type MkdirOrLinkReq struct { Path string `json:"path" form:"path"` } type MoveCopyReq struct { SrcDir string `json:"src_dir"` DstDir string `json:"dst_dir"` Names []string `json:"names"` } type RenameReq struct { Path string `json:"path"` Name string `json:"name"` } type RemoveReq struct { Dir string `json:"dir"` Names []string `json:"names"` } type LoginResp struct { Token string `json:"token"` } type MeResp struct { Id int `json:"id"` Username string `json:"username"` Password string `json:"password"` BasePath string `json:"base_path"` Role IntSlice `json:"role"` Disabled bool `json:"disabled"` Permission int `json:"permission"` SsoId string `json:"sso_id"` Otp bool `json:"otp"` } type IntSlice []int func (s *IntSlice) UnmarshalJSON(b []byte) error { var i int if json.Unmarshal(b, &i) == nil { *s = []int{i} return nil } return json.Unmarshal(b, (*[]int)(s)) } type ArchiveMetaReq struct { ArchivePass string `json:"archive_pass"` Password string `json:"password"` Path string `json:"path"` Refresh bool `json:"refresh"` } type TreeResp struct { ObjResp Children []TreeResp `json:"children"` hashCache *utils.HashInfo } func (t *TreeResp) GetSize() int64 { return t.Size } func (t *TreeResp) GetName() string { return t.Name } func (t *TreeResp) ModTime() time.Time { return t.Modified } func (t *TreeResp) CreateTime() time.Time { return t.Created } func (t *TreeResp) IsDir() bool { return t.ObjResp.IsDir } func (t *TreeResp) GetHash() utils.HashInfo { return utils.FromString(t.HashInfo) } func (t *TreeResp) GetID() string { return "" } func (t *TreeResp) GetPath() string { return "" } func (t *TreeResp) GetChildren() []model.ObjTree { ret := make([]model.ObjTree, 0, len(t.Children)) for _, child := range t.Children { ret = append(ret, &child) } return ret } func (t *TreeResp) Thumb() string { return t.ObjResp.Thumb } type ArchiveMetaResp struct { Comment string `json:"comment"` Encrypted bool `json:"encrypted"` Content []TreeResp `json:"content"` RawURL string `json:"raw_url"` Sign string `json:"sign"` } type ArchiveListReq struct { model.PageReq ArchiveMetaReq InnerPath string `json:"inner_path"` } type ArchiveListResp struct { Content []ObjResp `json:"content"` Total int64 `json:"total"` } type DecompressReq struct { ArchivePass string `json:"archive_pass"` CacheFull bool `json:"cache_full"` DstDir string `json:"dst_dir"` InnerPath string `json:"inner_path"` Name []string `json:"name"` PutIntoNewDir bool `json:"put_into_new_dir"` SrcDir string `json:"src_dir"` } ================================================ FILE: drivers/alist_v3/util.go ================================================ package alist_v3 import ( "fmt" "net/http" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) func (d *AListV3) login() error { if d.Username == "" { return nil } var resp common.Resp[LoginResp] _, _, err := d.request("/auth/login", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(base.Json{ "username": d.Username, "password": d.Password, }) }) if err != nil { return err } d.Token = resp.Data.Token op.MustSaveDriverStorage(d) return nil } func (d *AListV3) request(api, method string, callback base.ReqCallback, retry ...bool) ([]byte, int, error) { url := d.Address + "/api" + api req := base.RestyClient.R() req.SetHeader("Authorization", d.Token) if callback != nil { callback(req) } res, err := req.Execute(method, url) if err != nil { code := 0 if res != nil { code = res.StatusCode() } return nil, code, err } log.Debugf("[openlist] response body: %s", res.String()) if res.StatusCode() >= 400 { return nil, res.StatusCode(), fmt.Errorf("request failed, status: %s", res.Status()) } code := utils.Json.Get(res.Body(), "code").ToInt() if code != 200 { if (code == 401 || code == 403) && !utils.IsBool(retry...) { err = d.login() if err != nil { return nil, code, err } return d.request(api, method, callback, true) } return nil, code, fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(res.Body(), "message").ToString()) } return res.Body(), 200, nil } ================================================ FILE: drivers/aliyundrive/driver.go ================================================ package aliyundrive import ( "bytes" "context" "crypto/sha1" "encoding/base64" "encoding/hex" "fmt" "io" "math" "math/big" "net/http" "os" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) type AliDrive struct { model.Storage Addition AccessToken string cron *cron.Cron DriveId string UserID string } func (d *AliDrive) Config() driver.Config { return config } func (d *AliDrive) GetAddition() driver.Additional { return &d.Addition } func (d *AliDrive) Init(ctx context.Context) error { // TODO login / refresh token // op.MustSaveDriverStorage(d) err := d.refreshToken() if err != nil { return err } // get driver id res, err, _ := d.request("https://api.alipan.com/v2/user/get", http.MethodPost, nil, nil) if err != nil { return err } d.DriveId = utils.Json.Get(res, "default_drive_id").ToString() d.UserID = utils.Json.Get(res, "user_id").ToString() d.cron = cron.NewCron(time.Hour * 2) d.cron.Do(func() { err := d.refreshToken() if err != nil { log.Errorf("%+v", err) } }) if global.Has(d.UserID) { return nil } // init deviceID deviceID := utils.HashData(utils.SHA256, []byte(d.UserID)) // init privateKey privateKey, _ := NewPrivateKeyFromHex(deviceID) state := State{ privateKey: privateKey, deviceID: deviceID, } // store state global.Store(d.UserID, &state) // init signature d.sign() return nil } func (d *AliDrive) Drop(ctx context.Context) error { if d.cron != nil { d.cron.Stop() } return nil } func (d *AliDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(dir.GetID()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return fileToObj(src), nil }) } func (d *AliDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { data := base.Json{ "drive_id": d.DriveId, "file_id": file.GetID(), "expire_sec": 14400, } res, err, _ := d.request("https://api.alipan.com/v2/file/get_download_url", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) if err != nil { return nil, err } return &model.Link{ Header: http.Header{ "Referer": []string{"https://www.alipan.com/"}, }, URL: utils.Json.Get(res, "url").ToString(), }, nil } func (d *AliDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { _, err, _ := d.request("https://api.alipan.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "check_name_mode": "refuse", "drive_id": d.DriveId, "name": dirName, "parent_file_id": parentDir.GetID(), "type": "folder", }) }, nil) return err } func (d *AliDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { err := d.batch(srcObj.GetID(), dstDir.GetID(), "/file/move") return err } func (d *AliDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) error { _, err, _ := d.request("https://api.alipan.com/v3/file/update", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "check_name_mode": "refuse", "drive_id": d.DriveId, "file_id": srcObj.GetID(), "name": newName, }) }, nil) return err } func (d *AliDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { err := d.batch(srcObj.GetID(), dstDir.GetID(), "/file/copy") return err } func (d *AliDrive) Remove(ctx context.Context, obj model.Obj) error { _, err, _ := d.request("https://api.alipan.com/v2/recyclebin/trash", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": obj.GetID(), }) }, nil) return err } func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.FileStreamer, up driver.UpdateProgress) error { file := &stream.FileStream{ Obj: streamer, Reader: streamer, Mimetype: streamer.GetMimetype(), } const DEFAULT int64 = 10485760 count := int(math.Ceil(float64(streamer.GetSize()) / float64(DEFAULT))) partInfoList := make([]base.Json, 0, count) for i := 1; i <= count; i++ { partInfoList = append(partInfoList, base.Json{"part_number": i}) } reqBody := base.Json{ "check_name_mode": "overwrite", "drive_id": d.DriveId, "name": file.GetName(), "parent_file_id": dstDir.GetID(), "part_info_list": partInfoList, "size": file.GetSize(), "type": "file", } var localFile *os.File if fileStream, ok := file.Reader.(*stream.FileStream); ok { localFile, _ = fileStream.Reader.(*os.File) } if d.RapidUpload { buf := bytes.NewBuffer(make([]byte, 0, 1024)) _, err := utils.CopyWithBufferN(buf, file, 1024) if err != nil { return err } reqBody["pre_hash"] = utils.HashData(utils.SHA1, buf.Bytes()) if localFile != nil { if _, err := localFile.Seek(0, io.SeekStart); err != nil { return err } } else { // 把头部拼接回去 file.Reader = struct { io.Reader io.Closer }{ Reader: io.MultiReader(buf, file), Closer: file, } } } else { reqBody["content_hash_name"] = "none" reqBody["proof_version"] = "v1" } var resp UploadResp _, err, e := d.request("https://api.alipan.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { req.SetBody(reqBody) }, &resp) if err != nil && e.Code != "PreHashMatched" { return err } if d.RapidUpload && e.Code == "PreHashMatched" { delete(reqBody, "pre_hash") h := sha1.New() if localFile != nil { if err = utils.CopyWithCtx(ctx, h, localFile, 0, nil); err != nil { return err } if _, err = localFile.Seek(0, io.SeekStart); err != nil { return err } } else { tempFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*") if err != nil { return err } defer func() { _ = tempFile.Close() _ = os.Remove(tempFile.Name()) }() if err = utils.CopyWithCtx(ctx, io.MultiWriter(tempFile, h), file, 0, nil); err != nil { return err } localFile = tempFile } reqBody["content_hash"] = hex.EncodeToString(h.Sum(nil)) reqBody["content_hash_name"] = "sha1" reqBody["proof_version"] = "v1" /* js 隐性转换太坑不知道有没有bug var n = e.access_token， r = new BigNumber('0x'.concat(md5(n).slice(0, 16)))， i = new BigNumber(t.file.size)， o = i ? r.mod(i) : new gt.BigNumber(0); (t.file.slice(o.toNumber(), Math.min(o.plus(8).toNumber(), t.file.size))) */ buf := make([]byte, 8) r, _ := new(big.Int).SetString(utils.GetMD5EncodeStr(d.AccessToken)[:16], 16) i := new(big.Int).SetInt64(file.GetSize()) o := new(big.Int).SetInt64(0) if file.GetSize() > 0 { o = r.Mod(r, i) } n, _ := io.NewSectionReader(localFile, o.Int64(), 8).Read(buf[:8]) reqBody["proof_code"] = base64.StdEncoding.EncodeToString(buf[:n]) _, err, e := d.request("https://api.alipan.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { req.SetBody(reqBody) }, &resp) if err != nil && e.Code != "PreHashMatched" { return err } if resp.RapidUpload { return nil } // 秒传失败 if _, err = localFile.Seek(0, io.SeekStart); err != nil { return err } file.Reader = localFile } rateLimited := driver.NewLimitedUploadStream(ctx, file) for i, partInfo := range resp.PartInfoList { if utils.IsCanceled(ctx) { return ctx.Err() } url := partInfo.UploadUrl if d.InternalUpload { url = partInfo.InternalUploadUrl } req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, io.LimitReader(rateLimited, DEFAULT)) if err != nil { return err } res, err := base.HttpClient.Do(req) if err != nil { return err } _ = res.Body.Close() if count > 0 { up(float64(i) * 100 / float64(count)) } } var resp2 base.Json _, err, e = d.request("https://api.alipan.com/v2/file/complete", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": resp.FileId, "upload_id": resp.UploadId, }) }, &resp2) if err != nil && e.Code != "PreHashMatched" { return err } if resp2["file_id"] == resp.FileId { return nil } return fmt.Errorf("%+v", resp2) } func (d *AliDrive) GetDetails(ctx context.Context) (*model.StorageDetails, error) { res, err, _ := d.request("https://api.aliyundrive.com/adrive/v1/user/driveCapacityDetails", http.MethodPost, func(req *resty.Request) { req.SetContext(ctx) }, nil) if err != nil { return nil, err } used := utils.Json.Get(res, "drive_used_size").ToInt64() total := utils.Json.Get(res, "drive_total_size").ToInt64() return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: used, }, }, nil } func (d *AliDrive) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { var resp base.Json var url string data := base.Json{ "drive_id": d.DriveId, "file_id": args.Obj.GetID(), } switch args.Method { case "doc_preview": url = "https://api.alipan.com/v2/file/get_office_preview_url" data["access_token"] = d.AccessToken case "video_preview": url = "https://api.alipan.com/v2/file/get_video_preview_play_info" data["category"] = "live_transcoding" data["url_expire_sec"] = 14400 default: return nil, errs.NotSupport } _, err, _ := d.request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) if err != nil { return nil, err } return resp, nil } var _ driver.Driver = (*AliDrive)(nil) ================================================ FILE: drivers/aliyundrive/global.go ================================================ package aliyundrive import ( "crypto/ecdsa" "github.com/OpenListTeam/OpenList/v4/pkg/generic_sync" ) type State struct { deviceID string signature string retry int privateKey *ecdsa.PrivateKey } var global = generic_sync.MapOf[string, *State]{} ================================================ FILE: drivers/aliyundrive/help.go ================================================ package aliyundrive import ( "crypto/ecdsa" "crypto/rand" "encoding/hex" "math/big" "github.com/dustinxie/ecc" ) func NewPrivateKey() (*ecdsa.PrivateKey, error) { p256k1 := ecc.P256k1() return ecdsa.GenerateKey(p256k1, rand.Reader) } func NewPrivateKeyFromHex(hex_ string) (*ecdsa.PrivateKey, error) { data, err := hex.DecodeString(hex_) if err != nil { return nil, err } return NewPrivateKeyFromBytes(data), nil } func NewPrivateKeyFromBytes(priv []byte) *ecdsa.PrivateKey { p256k1 := ecc.P256k1() x, y := p256k1.ScalarBaseMult(priv) return &ecdsa.PrivateKey{ PublicKey: ecdsa.PublicKey{ Curve: p256k1, X: x, Y: y, }, D: new(big.Int).SetBytes(priv), } } func PrivateKeyToHex(private *ecdsa.PrivateKey) string { return hex.EncodeToString(PrivateKeyToBytes(private)) } func PrivateKeyToBytes(private *ecdsa.PrivateKey) []byte { return private.D.Bytes() } func PublicKeyToHex(public *ecdsa.PublicKey) string { return hex.EncodeToString(PublicKeyToBytes(public)) } func PublicKeyToBytes(public *ecdsa.PublicKey) []byte { x := public.X.Bytes() if len(x) < 32 { for i := 0; i < 32-len(x); i++ { x = append([]byte{0}, x...) } } y := public.Y.Bytes() if len(y) < 32 { for i := 0; i < 32-len(y); i++ { y = append([]byte{0}, y...) } } return append(x, y...) } ================================================ FILE: drivers/aliyundrive/meta.go ================================================ package aliyundrive import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootID RefreshToken string `json:"refresh_token" required:"true"` //DeviceID string `json:"device_id" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` RapidUpload bool `json:"rapid_upload"` InternalUpload bool `json:"internal_upload"` } var config = driver.Config{ Name: "Aliyundrive", DefaultRoot: "root", Alert: `warning|There may be an infinite loop bug in this driver. Deprecated, no longer maintained and will be removed in a future version. We recommend using the official driver AliyundriveOpen.`, } func init() { op.RegisterDriver(func() driver.Driver { return &AliDrive{} }) } ================================================ FILE: drivers/aliyundrive/types.go ================================================ package aliyundrive import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type RespErr struct { Code string `json:"code"` Message string `json:"message"` } type Files struct { Items []File `json:"items"` NextMarker string `json:"next_marker"` } type File struct { DriveId string `json:"drive_id"` CreatedAt *time.Time `json:"created_at"` FileExtension string `json:"file_extension"` FileId string `json:"file_id"` Type string `json:"type"` Name string `json:"name"` Category string `json:"category"` ParentFileId string `json:"parent_file_id"` UpdatedAt time.Time `json:"updated_at"` Size int64 `json:"size"` Thumbnail string `json:"thumbnail"` Url string `json:"url"` } func fileToObj(f File) *model.ObjThumb { return &model.ObjThumb{ Object: model.Object{ ID: f.FileId, Name: f.Name, Size: f.Size, Modified: f.UpdatedAt, IsFolder: f.Type == "folder", }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbnail}, } } type UploadResp struct { FileId string `json:"file_id"` UploadId string `json:"upload_id"` PartInfoList []struct { UploadUrl string `json:"upload_url"` InternalUploadUrl string `json:"internal_upload_url"` } `json:"part_info_list"` RapidUpload bool `json:"rapid_upload"` } ================================================ FILE: drivers/aliyundrive/util.go ================================================ package aliyundrive import ( "crypto/sha256" "encoding/hex" "errors" "fmt" "net/http" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/dustinxie/ecc" "github.com/go-resty/resty/v2" "github.com/google/uuid" ) func (d *AliDrive) createSession() error { state, ok := global.Load(d.UserID) if !ok { return fmt.Errorf("can't load user state, user_id: %s", d.UserID) } d.sign() state.retry++ if state.retry > 3 { state.retry = 0 return fmt.Errorf("createSession failed after three retries") } _, err, _ := d.request("https://api.alipan.com/users/v1/users/device/create_session", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "deviceName": "samsung", "modelName": "SM-G9810", "nonce": 0, "pubKey": PublicKeyToHex(&state.privateKey.PublicKey), "refreshToken": d.RefreshToken, }) }, nil) if err == nil { state.retry = 0 } return err } // func (d *AliDrive) renewSession() error { // _, err, _ := d.request("https://api.alipan.com/users/v1/users/device/renew_session", http.MethodPost, nil, nil) // return err // } func (d *AliDrive) sign() { state, _ := global.Load(d.UserID) secpAppID := "5dde4e1bdf9e4966b387ba58f4b3fdc3" singdata := fmt.Sprintf("%s:%s:%s:%d", secpAppID, state.deviceID, d.UserID, 0) hash := sha256.Sum256([]byte(singdata)) data, _ := ecc.SignBytes(state.privateKey, hash[:], ecc.RecID|ecc.LowerS) state.signature = hex.EncodeToString(data) //strconv.Itoa(state.nonce) } // do others that not defined in Driver interface func (d *AliDrive) refreshToken() error { url := "https://auth.alipan.com/v2/account/token" var resp base.TokenResp var e RespErr _, err := base.RestyClient.R(). //ForceContentType("application/json"). SetBody(base.Json{"refresh_token": d.RefreshToken, "grant_type": "refresh_token"}). SetResult(&resp). SetError(&e). Post(url) if err != nil { return err } if e.Code != "" { return fmt.Errorf("failed to refresh token: %s", e.Message) } if resp.RefreshToken == "" { return errors.New("failed to refresh token: refresh token is empty") } d.RefreshToken, d.AccessToken = resp.RefreshToken, resp.AccessToken op.MustSaveDriverStorage(d) return nil } func (d *AliDrive) request(url, method string, callback base.ReqCallback, resp interface{}) ([]byte, error, RespErr) { req := base.RestyClient.R() state, ok := global.Load(d.UserID) if !ok { if url == "https://api.alipan.com/v2/user/get" { state = &State{} } else { return nil, fmt.Errorf("can't load user state, user_id: %s", d.UserID), RespErr{} } } req.SetHeaders(map[string]string{ "Authorization": "Bearer\t" + d.AccessToken, "content-type": "application/json", "origin": "https://www.alipan.com", "Referer": "https://alipan.com/", "X-Signature": state.signature, "x-request-id": uuid.NewString(), "X-Canary": "client=Android,app=adrive,version=v4.1.0", "X-Device-Id": state.deviceID, }) if callback != nil { callback(req) } else { req.SetBody("{}") } if resp != nil { req.SetResult(resp) } var e RespErr req.SetError(&e) res, err := req.Execute(method, url) if err != nil { return nil, err, e } if e.Code != "" { switch e.Code { case "AccessTokenInvalid": err = d.refreshToken() if err != nil { return nil, err, e } case "DeviceSessionSignatureInvalid": err = d.createSession() if err != nil { return nil, err, e } default: return nil, errors.New(e.Message), e } return d.request(url, method, callback, resp) } else if res.IsError() { return nil, errors.New("bad status code " + res.Status()), e } return res.Body(), nil, e } func (d *AliDrive) getFiles(fileId string) ([]File, error) { marker := "first" res := make([]File, 0) for marker != "" { if marker == "first" { marker = "" } var resp Files data := base.Json{ "drive_id": d.DriveId, "fields": "*", "image_thumbnail_process": "image/resize,w_400/format,jpeg", "image_url_process": "image/resize,w_1920/format,jpeg", "limit": 200, "marker": marker, "order_by": d.OrderBy, "order_direction": d.OrderDirection, "parent_file_id": fileId, "video_thumbnail_process": "video/snapshot,t_0,f_jpg,ar_auto,w_300", "url_expire_sec": 14400, } _, err, _ := d.request("https://api.alipan.com/v2/file/list", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) if err != nil { return nil, err } marker = resp.NextMarker res = append(res, resp.Items...) } return res, nil } func (d *AliDrive) batch(srcId, dstId string, url string) error { res, err, _ := d.request("https://api.alipan.com/v3/batch", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "requests": []base.Json{ { "headers": base.Json{ "Content-Type": "application/json", }, "method": "POST", "id": srcId, "body": base.Json{ "drive_id": d.DriveId, "file_id": srcId, "to_drive_id": d.DriveId, "to_parent_file_id": dstId, }, "url": url, }, }, "resource": "file", }) }, nil) if err != nil { return err } status := utils.Json.Get(res, "responses", 0, "status").ToInt() if status < 400 && status >= 100 { return nil } return errors.New(string(res)) } ================================================ FILE: drivers/aliyundrive_open/driver.go ================================================ package aliyundrive_open import ( "context" "errors" "net/http" "path/filepath" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) type AliyundriveOpen struct { model.Storage Addition DriveId string limiter *limiter ref *AliyundriveOpen } func (d *AliyundriveOpen) Config() driver.Config { return config } func (d *AliyundriveOpen) GetAddition() driver.Additional { return &d.Addition } func (d *AliyundriveOpen) Init(ctx context.Context) error { d.limiter = getLimiterForUser(globalLimiterUserID) // First create a globally shared limiter to limit the initial requests. if d.LIVPDownloadFormat == "" { d.LIVPDownloadFormat = "jpeg" } if d.DriveType == "" { d.DriveType = "default" } res, err := d.request(ctx, limiterOther, "/adrive/v1.0/user/getDriveInfo", http.MethodPost, nil) if err != nil { d.limiter.free() d.limiter = nil return err } d.DriveId = utils.Json.Get(res, d.DriveType+"_drive_id").ToString() userid := utils.Json.Get(res, "user_id").ToString() d.limiter.free() d.limiter = getLimiterForUser(userid) // Allocate a corresponding limiter for each user. return nil } func (d *AliyundriveOpen) InitReference(storage driver.Driver) error { refStorage, ok := storage.(*AliyundriveOpen) if ok { d.ref = refStorage return nil } return errs.NotSupport } func (d *AliyundriveOpen) Drop(ctx context.Context) error { d.limiter.free() d.limiter = nil d.ref = nil return nil } // GetRoot implements the driver.GetRooter interface to properly set up the root object func (d *AliyundriveOpen) GetRoot(ctx context.Context) (model.Obj, error) { return &model.Object{ ID: d.RootFolderID, Path: "/", Name: "root", Modified: d.Modified, IsFolder: true, }, nil } func (d *AliyundriveOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(ctx, dir.GetID()) if err != nil { return nil, err } objs, err := utils.SliceConvert(files, func(src File) (model.Obj, error) { obj := fileToObj(src) // Set the correct path for the object if dir.GetPath() != "" { obj.Path = filepath.Join(dir.GetPath(), obj.GetName()) } return obj, nil }) return objs, err } func (d *AliyundriveOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { res, err := d.request(ctx, limiterLink, "/adrive/v1.0/openFile/getDownloadUrl", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": file.GetID(), "expire_sec": 14400, }) }) if err != nil { return nil, err } url := utils.Json.Get(res, "url").ToString() if url == "" { if utils.Ext(file.GetName()) != "livp" { return nil, errors.New("get download url failed: " + string(res)) } url = utils.Json.Get(res, "streamsUrl", d.LIVPDownloadFormat).ToString() } exp := time.Minute return &model.Link{ URL: url, Expiration: &exp, }, nil } func (d *AliyundriveOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { nowTime, _ := getNowTime() newDir := File{CreatedAt: nowTime, UpdatedAt: nowTime} _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "parent_file_id": parentDir.GetID(), "name": dirName, "type": "folder", "check_name_mode": "refuse", }).SetResult(&newDir) }) if err != nil { return nil, err } obj := fileToObj(newDir) // Set the correct Path for the returned directory object if parentDir.GetPath() != "" { obj.Path = filepath.Join(parentDir.GetPath(), dirName) } else { obj.Path = "/" + dirName } return obj, nil } func (d *AliyundriveOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { var resp MoveOrCopyResp _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/move", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), "to_parent_file_id": dstDir.GetID(), "check_name_mode": "ignore", // optional:ignore,auto_rename,refuse //"new_name": "newName", // The new name to use when a file of the same name exists }).SetResult(&resp) }) if err != nil { return nil, err } if srcObj, ok := srcObj.(*model.ObjThumb); ok { srcObj.ID = resp.FileID srcObj.Modified = time.Now() srcObj.Path = filepath.Join(dstDir.GetPath(), srcObj.GetName()) // Check for duplicate files in the destination directory if err := d.removeDuplicateFiles(ctx, dstDir.GetPath(), srcObj.GetName(), srcObj.GetID()); err != nil { // Only log a warning instead of returning an error since the move operation has already completed successfully log.Warnf("Failed to remove duplicate files after move: %v", err) } return srcObj, nil } return nil, nil } func (d *AliyundriveOpen) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { var newFile File _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/update", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), "name": newName, }).SetResult(&newFile) }) if err != nil { return nil, err } // Check for duplicate files in the parent directory parentPath := filepath.Dir(srcObj.GetPath()) if err := d.removeDuplicateFiles(ctx, parentPath, newName, newFile.FileId); err != nil { // Only log a warning instead of returning an error since the rename operation has already completed successfully log.Warnf("Failed to remove duplicate files after rename: %v", err) } obj := fileToObj(newFile) // Set the correct Path for the renamed object if parentPath != "" && parentPath != "." { obj.Path = filepath.Join(parentPath, newName) } else { obj.Path = "/" + newName } return obj, nil } func (d *AliyundriveOpen) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { var resp MoveOrCopyResp _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/copy", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), "to_parent_file_id": dstDir.GetID(), "auto_rename": false, }).SetResult(&resp) }) if err != nil { return err } // Check for duplicate files in the destination directory if err := d.removeDuplicateFiles(ctx, dstDir.GetPath(), srcObj.GetName(), resp.FileID); err != nil { // Only log a warning instead of returning an error since the copy operation has already completed successfully log.Warnf("Failed to remove duplicate files after copy: %v", err) } return nil } func (d *AliyundriveOpen) Remove(ctx context.Context, obj model.Obj) error { uri := "/adrive/v1.0/openFile/recyclebin/trash" if d.RemoveWay == "delete" { uri = "/adrive/v1.0/openFile/delete" } _, err := d.request(ctx, limiterOther, uri, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": obj.GetID(), }) }) return err } func (d *AliyundriveOpen) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { obj, err := d.upload(ctx, dstDir, stream, up) // Set the correct Path for the returned file object if obj != nil && obj.GetPath() == "" { if dstDir.GetPath() != "" { if objWithPath, ok := obj.(model.SetPath); ok { objWithPath.SetPath(filepath.Join(dstDir.GetPath(), obj.GetName())) } } } return obj, err } func (d *AliyundriveOpen) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { var resp base.Json var uri string data := base.Json{ "drive_id": d.DriveId, "file_id": args.Obj.GetID(), } switch args.Method { case "video_preview": uri = "/adrive/v1.0/openFile/getVideoPreviewPlayInfo" data["category"] = "live_transcoding" data["url_expire_sec"] = 14400 default: return nil, errs.NotSupport } _, err := d.request(ctx, limiterOther, uri, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetResult(&resp) }) if err != nil { return nil, err } return resp, nil } func (d *AliyundriveOpen) GetDetails(ctx context.Context) (*model.StorageDetails, error) { res, err := d.request(ctx, limiterOther, "/adrive/v1.0/user/getSpaceInfo", http.MethodPost, nil) if err != nil { return nil, err } total := utils.Json.Get(res, "personal_space_info", "total_size").ToInt64() used := utils.Json.Get(res, "personal_space_info", "used_size").ToInt64() return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: used, }, }, nil } var _ driver.Driver = (*AliyundriveOpen)(nil) var _ driver.MkdirResult = (*AliyundriveOpen)(nil) var _ driver.MoveResult = (*AliyundriveOpen)(nil) var _ driver.RenameResult = (*AliyundriveOpen)(nil) var _ driver.PutResult = (*AliyundriveOpen)(nil) var _ driver.GetRooter = (*AliyundriveOpen)(nil) ================================================ FILE: drivers/aliyundrive_open/limiter.go ================================================ package aliyundrive_open import ( "context" "fmt" "sync" "golang.org/x/time/rate" ) // See document https://www.yuque.com/aliyundrive/zpfszx/mqocg38hlxzc5vcd // See issue https://github.com/OpenListTeam/OpenList/issues/724 // We got limit per user per app, so the limiter should be global. type limiterType int const ( limiterList limiterType = iota limiterLink limiterOther ) const ( listRateLimit = 3.9 // 4 per second in document, but we use 3.9 per second to be safe linkRateLimit = 0.9 // 1 per second in document, but we use 0.9 per second to be safe otherRateLimit = 14.9 // 15 per second in document, but we use 14.9 per second to be safe globalLimiterUserID = "" // Global limiter user ID, used to limit the initial requests. ) type limiter struct { usedBy int list *rate.Limiter link *rate.Limiter other *rate.Limiter } var limiters = make(map[string]*limiter) var limitersLock = &sync.Mutex{} func getLimiterForUser(userid string) *limiter { limitersLock.Lock() defer limitersLock.Unlock() defer func() { // Clean up limiters that are no longer used. for id, lim := range limiters { if lim.usedBy <= 0 && id != globalLimiterUserID { // Do not delete the global limiter. delete(limiters, id) } } }() if lim, ok := limiters[userid]; ok { lim.usedBy++ return lim } lim := &limiter{ usedBy: 1, list: rate.NewLimiter(rate.Limit(listRateLimit), 1), link: rate.NewLimiter(rate.Limit(linkRateLimit), 1), other: rate.NewLimiter(rate.Limit(otherRateLimit), 1), } limiters[userid] = lim return lim } func (l *limiter) wait(ctx context.Context, typ limiterType) error { if l == nil { return fmt.Errorf("driver not init") } switch typ { case limiterList: return l.list.Wait(ctx) case limiterLink: return l.link.Wait(ctx) case limiterOther: return l.other.Wait(ctx) default: return fmt.Errorf("unknown limiter type") } } func (l *limiter) free() { if l == nil { return } limitersLock.Lock() defer limitersLock.Unlock() l.usedBy-- } func (d *AliyundriveOpen) wait(ctx context.Context, typ limiterType) error { if d == nil { return fmt.Errorf("driver not init") } if d.ref != nil { return d.ref.wait(ctx, typ) // If this is a reference driver, wait on the reference driver. } return d.limiter.wait(ctx, typ) } ================================================ FILE: drivers/aliyundrive_open/meta.go ================================================ package aliyundrive_open import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { DriveType string `json:"drive_type" type:"select" options:"default,resource,backup" default:"resource"` driver.RootID RefreshToken string `json:"refresh_token" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` UseOnlineAPI bool `json:"use_online_api" default:"true"` AlipanType string `json:"alipan_type" required:"true" type:"select" default:"default" options:"default,alipanTV"` APIAddress string `json:"api_url_address" default:"https://api.oplist.org/alicloud/renewapi"` ClientID string `json:"client_id" help:"Keep it empty if you don't have one"` ClientSecret string `json:"client_secret" help:"Keep it empty if you don't have one"` RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` RapidUpload bool `json:"rapid_upload" help:"If you enable this option, the file will be uploaded to the server first, so the progress will be incorrect"` InternalUpload bool `json:"internal_upload" help:"If you are using Aliyun ECS is located in Beijing, you can turn it on to boost the upload speed"` LIVPDownloadFormat string `json:"livp_download_format" type:"select" options:"jpeg,mov" default:"jpeg"` AccessToken string } var config = driver.Config{ Name: "AliyundriveOpen", DefaultRoot: "root", NoOverwriteUpload: true, } var API_URL = "https://openapi.alipan.com" func init() { op.RegisterDriver(func() driver.Driver { return &AliyundriveOpen{} }) } ================================================ FILE: drivers/aliyundrive_open/types.go ================================================ package aliyundrive_open import ( "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type ErrResp struct { Code string `json:"code"` Message string `json:"message"` } type Files struct { Items []File `json:"items"` NextMarker string `json:"next_marker"` } type File struct { DriveId string `json:"drive_id"` FileId string `json:"file_id"` ParentFileId string `json:"parent_file_id"` Name string `json:"name"` Size int64 `json:"size"` FileExtension string `json:"file_extension"` ContentHash string `json:"content_hash"` Category string `json:"category"` Type string `json:"type"` Thumbnail string `json:"thumbnail"` Url string `json:"url"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` // create only FileName string `json:"file_name"` } func fileToObj(f File) *model.ObjThumb { if f.Name == "" { f.Name = f.FileName } return &model.ObjThumb{ Object: model.Object{ ID: f.FileId, Name: f.Name, Size: f.Size, Modified: f.UpdatedAt, IsFolder: f.Type == "folder", Ctime: f.CreatedAt, HashInfo: utils.NewHashInfo(utils.SHA1, f.ContentHash), }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbnail}, } } type PartInfo struct { Etag interface{} `json:"etag"` PartNumber int `json:"part_number"` PartSize interface{} `json:"part_size"` UploadUrl string `json:"upload_url"` ContentType string `json:"content_type"` } type CreateResp struct { //Type string `json:"type"` //ParentFileId string `json:"parent_file_id"` //DriveId string `json:"drive_id"` FileId string `json:"file_id"` //RevisionId string `json:"revision_id"` //EncryptMode string `json:"encrypt_mode"` //DomainId string `json:"domain_id"` //FileName string `json:"file_name"` UploadId string `json:"upload_id"` //Location string `json:"location"` RapidUpload bool `json:"rapid_upload"` PartInfoList []PartInfo `json:"part_info_list"` } type MoveOrCopyResp struct { Exist bool `json:"exist"` DriveID string `json:"drive_id"` FileID string `json:"file_id"` } ================================================ FILE: drivers/aliyundrive_open/upload.go ================================================ package aliyundrive_open import ( "context" "encoding/base64" "fmt" "io" "math" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) func makePartInfos(size int) []base.Json { partInfoList := make([]base.Json, size) for i := 0; i < size; i++ { partInfoList[i] = base.Json{"part_number": 1 + i} } return partInfoList } func calPartSize(fileSize int64) int64 { var partSize int64 = 20 * utils.MB if fileSize > partSize { if fileSize > 1*utils.TB { // file Size over 1TB partSize = 5 * utils.GB // file part size 5GB } else if fileSize > 768*utils.GB { // over 768GB partSize = 109951163 // ≈ 104.8576MB, split 1TB into 10,000 part } else if fileSize > 512*utils.GB { // over 512GB partSize = 82463373 // ≈ 78.6432MB } else if fileSize > 384*utils.GB { // over 384GB partSize = 54975582 // ≈ 52.4288MB } else if fileSize > 256*utils.GB { // over 256GB partSize = 41231687 // ≈ 39.3216MB } else if fileSize > 128*utils.GB { // over 128GB partSize = 27487791 // ≈ 26.2144MB } } return partSize } func (d *AliyundriveOpen) getUploadUrl(ctx context.Context, count int, fileId, uploadId string) ([]PartInfo, error) { partInfoList := makePartInfos(count) var resp CreateResp _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/getUploadUrl", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": fileId, "part_info_list": partInfoList, "upload_id": uploadId, }).SetResult(&resp) }) return resp.PartInfoList, err } func (d *AliyundriveOpen) uploadPart(ctx context.Context, r io.Reader, partInfo PartInfo) error { uploadUrl := partInfo.UploadUrl if d.InternalUpload { uploadUrl = strings.ReplaceAll(uploadUrl, "https://cn-beijing-data.aliyundrive.net/", "http://ccp-bj29-bj-1592982087.oss-cn-beijing-internal.aliyuncs.com/") } req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, r) if err != nil { return err } res, err := base.HttpClient.Do(req) if err != nil { return err } _ = res.Body.Close() if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusConflict { return fmt.Errorf("upload status: %d", res.StatusCode) } return nil } func (d *AliyundriveOpen) completeUpload(ctx context.Context, fileId, uploadId string) (model.Obj, error) { // 3. complete var newFile File _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/complete", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": fileId, "upload_id": uploadId, }).SetResult(&newFile) }) if err != nil { return nil, err } return fileToObj(newFile), nil } type ProofRange struct { Start int64 End int64 } func getProofRange(input string, size int64) (*ProofRange, error) { if size == 0 { return &ProofRange{}, nil } tmpStr := utils.GetMD5EncodeStr(input)[0:16] tmpInt, err := strconv.ParseUint(tmpStr, 16, 64) if err != nil { return nil, err } index := tmpInt % uint64(size) pr := &ProofRange{ Start: int64(index), End: int64(index) + 8, } if pr.End >= size { pr.End = size } return pr, nil } func (d *AliyundriveOpen) calProofCode(stream model.FileStreamer) (string, error) { proofRange, err := getProofRange(d.getAccessToken(), stream.GetSize()) if err != nil { return "", err } length := proofRange.End - proofRange.Start reader, err := stream.RangeRead(http_range.Range{Start: proofRange.Start, Length: length}) if err != nil { return "", err } buf := make([]byte, length) n, err := io.ReadFull(reader, buf) if n != int(length) { return "", fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) } return base64.StdEncoding.EncodeToString(buf), nil } func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { // 1. create // Part Size Unit: Bytes, Default: 20MB, // Maximum number of slices 10,000, ≈195.3125GB var partSize = calPartSize(stream.GetSize()) const dateFormat = "2006-01-02T15:04:05.000Z" mtimeStr := stream.ModTime().UTC().Format(dateFormat) ctimeStr := stream.CreateTime().UTC().Format(dateFormat) createData := base.Json{ "drive_id": d.DriveId, "parent_file_id": dstDir.GetID(), "name": stream.GetName(), "type": "file", "check_name_mode": "ignore", "local_modified_at": mtimeStr, "local_created_at": ctimeStr, } count := int(math.Ceil(float64(stream.GetSize()) / float64(partSize))) createData["part_info_list"] = makePartInfos(count) // rapid upload rapidUpload := !stream.IsForceStreamUpload() && stream.GetSize() > 100*utils.KB && d.RapidUpload if rapidUpload { log.Debugf("[aliyundrive_open] start cal pre_hash") // read 1024 bytes to calculate pre hash reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: 1024}) if err != nil { return nil, err } hash, err := utils.HashReader(utils.SHA1, reader) if err != nil { return nil, err } createData["size"] = stream.GetSize() createData["pre_hash"] = hash } var createResp CreateResp _, err, e := d.requestReturnErrResp(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(createData).SetResult(&createResp) }) if err != nil { if e.Code != "PreHashMatched" || !rapidUpload { return nil, err } log.Debugf("[aliyundrive_open] pre_hash matched, start rapid upload") hash := stream.GetHash().GetHash(utils.SHA1) if len(hash) != utils.SHA1.Width { _, hash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1) if err != nil { return nil, err } } delete(createData, "pre_hash") createData["proof_version"] = "v1" createData["content_hash_name"] = "sha1" createData["content_hash"] = hash createData["proof_code"], err = d.calProofCode(stream) if err != nil { return nil, fmt.Errorf("cal proof code error: %s", err.Error()) } _, err = d.request(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(createData).SetResult(&createResp) }) if err != nil { return nil, err } } if !createResp.RapidUpload { // 2. normal upload log.Debugf("[aliyundive_open] normal upload") ss, err := streamPkg.NewStreamSectionReader(stream, int(partSize), &up) if err != nil { return nil, err } preTime := time.Now() var offset, length int64 = 0, partSize for i := 0; i < len(createResp.PartInfoList); i++ { if utils.IsCanceled(ctx) { return nil, ctx.Err() } // refresh upload url if 50 minutes passed if time.Since(preTime) > 50*time.Minute { createResp.PartInfoList, err = d.getUploadUrl(ctx, count, createResp.FileId, createResp.UploadId) if err != nil { return nil, err } preTime = time.Now() } if remain := stream.GetSize() - offset; length > remain { length = remain } rd, err := ss.GetSectionReader(offset, length) if err != nil { return nil, err } err = retry.Do(func() error { rd.Seek(0, io.SeekStart) return d.uploadPart(ctx, driver.NewLimitedUploadStream(ctx, rd), createResp.PartInfoList[i]) }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) ss.FreeSectionReader(rd) if err != nil { return nil, err } offset += partSize up(float64(i*100) / float64(count)) } } else { log.Debugf("[aliyundrive_open] rapid upload success, file id: %s", createResp.FileId) } log.Debugf("[aliyundrive_open] create file success, resp: %+v", createResp) // 3. complete return d.completeUpload(ctx, createResp.FileId, createResp.UploadId) } ================================================ FILE: drivers/aliyundrive_open/util.go ================================================ package aliyundrive_open import ( "context" "encoding/base64" "errors" "fmt" "net/http" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) // do others that not defined in Driver interface func (d *AliyundriveOpen) _refreshToken(ctx context.Context) (string, string, error) { if d.UseOnlineAPI && d.APIAddress != "" { u := d.APIAddress var resp struct { RefreshToken string `json:"refresh_token"` AccessToken string `json:"access_token"` ErrorMessage string `json:"text"` } // 根据AlipanType选项设置driver_txt driverTxt := "alicloud_qr" if d.AlipanType == "alipanTV" { driverTxt = "alicloud_tv" } err := d.wait(ctx, limiterOther) if err != nil { return "", "", err } _, err = base.RestyClient.R(). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, "server_use": "true", "driver_txt": driverTxt, }). Get(u) if err != nil { return "", "", err } if resp.RefreshToken == "" || resp.AccessToken == "" { if resp.ErrorMessage != "" { return "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) } return "", "", fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") } return resp.RefreshToken, resp.AccessToken, nil } // 本地刷新逻辑，必须要求 client_id 和 client_secret if d.ClientID == "" || d.ClientSecret == "" { return "", "", fmt.Errorf("empty ClientID or ClientSecret") } err := d.wait(ctx, limiterOther) if err != nil { return "", "", err } url := API_URL + "/oauth/access_token" //var resp base.TokenResp var e ErrResp res, err := base.RestyClient.R(). //ForceContentType("application/json"). SetBody(base.Json{ "client_id": d.ClientID, "client_secret": d.ClientSecret, "grant_type": "refresh_token", "refresh_token": d.RefreshToken, }). //SetResult(&resp). SetError(&e). Post(url) if err != nil { return "", "", err } log.Debugf("[ali_open] refresh token response: %s", res.String()) if e.Code != "" { return "", "", fmt.Errorf("failed to refresh token: %s", e.Message) } refresh, access := utils.Json.Get(res.Body(), "refresh_token").ToString(), utils.Json.Get(res.Body(), "access_token").ToString() if refresh == "" { return "", "", fmt.Errorf("failed to refresh token: refresh token is empty, resp: %s", res.String()) } curSub, err := getSub(d.RefreshToken) if err != nil { return "", "", err } newSub, err := getSub(refresh) if err != nil { return "", "", err } if curSub != newSub { return "", "", errors.New("failed to refresh token: sub not match") } return refresh, access, nil } func getSub(token string) (string, error) { segments := strings.Split(token, ".") if len(segments) != 3 { return "", errors.New("not a jwt token because of invalid segments") } bs, err := base64.RawStdEncoding.DecodeString(segments[1]) if err != nil { return "", errors.New("failed to decode jwt token") } return utils.Json.Get(bs, "sub").ToString(), nil } func (d *AliyundriveOpen) refreshToken(ctx context.Context) error { if d.ref != nil { return d.ref.refreshToken(ctx) } refresh, access, err := d._refreshToken(ctx) for i := 0; i < 3; i++ { if err == nil { break } else { log.Errorf("[ali_open] failed to refresh token: %s", err) } refresh, access, err = d._refreshToken(ctx) } if err != nil { return err } log.Infof("[ali_open] token exchange: %s -> %s", d.RefreshToken, refresh) d.RefreshToken, d.AccessToken = refresh, access op.MustSaveDriverStorage(d) return nil } func (d *AliyundriveOpen) request(ctx context.Context, limitTy limiterType, uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { b, err, _ := d.requestReturnErrResp(ctx, limitTy, uri, method, callback, retry...) return b, err } func (d *AliyundriveOpen) requestReturnErrResp(ctx context.Context, limitTy limiterType, uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error, *ErrResp) { req := base.RestyClient.R() // TODO check whether access_token is expired req.SetHeader("Authorization", "Bearer "+d.getAccessToken()) if method == http.MethodPost { req.SetHeader("Content-Type", "application/json") } if callback != nil { callback(req) } var e ErrResp req.SetError(&e) err := d.wait(ctx, limitTy) if err != nil { return nil, err, nil } res, err := req.Execute(method, API_URL+uri) if err != nil { if res != nil { log.Errorf("[aliyundrive_open] request error: %s", res.String()) } return nil, err, nil } isRetry := len(retry) > 0 && retry[0] if e.Code != "" { if !isRetry && (utils.SliceContains([]string{"AccessTokenInvalid", "AccessTokenExpired", "I400JD"}, e.Code) || d.getAccessToken() == "") { err = d.refreshToken(ctx) if err != nil { return nil, err, nil } return d.requestReturnErrResp(ctx, limitTy, uri, method, callback, true) } return nil, fmt.Errorf("%s:%s", e.Code, e.Message), &e } return res.Body(), nil, nil } func (d *AliyundriveOpen) list(ctx context.Context, data base.Json) (*Files, error) { var resp Files _, err := d.request(ctx, limiterList, "/adrive/v1.0/openFile/list", http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetResult(&resp) }) if err != nil { return nil, err } return &resp, nil } func (d *AliyundriveOpen) getFiles(ctx context.Context, fileId string) ([]File, error) { marker := "first" res := make([]File, 0) for marker != "" { if marker == "first" { marker = "" } data := base.Json{ "drive_id": d.DriveId, "limit": 200, "marker": marker, "order_by": d.OrderBy, "order_direction": d.OrderDirection, "parent_file_id": fileId, //"category": "", //"type": "", //"video_thumbnail_time": 120000, //"video_thumbnail_width": 480, //"image_thumbnail_width": 480, } resp, err := d.list(ctx, data) if err != nil { return nil, err } marker = resp.NextMarker res = append(res, resp.Items...) } return res, nil } func getNowTime() (time.Time, string) { nowTime := time.Now() nowTimeStr := nowTime.Format("2006-01-02T15:04:05.000Z") return nowTime, nowTimeStr } func (d *AliyundriveOpen) getAccessToken() string { if d.ref != nil { return d.ref.getAccessToken() } return d.AccessToken } // Remove duplicate files with the same name in the given directory path, // preserving the file with the given skipID if provided func (d *AliyundriveOpen) removeDuplicateFiles(ctx context.Context, parentPath string, fileName string, skipID string) error { // Handle empty path (root directory) case if parentPath == "" { parentPath = "/" } // List all files in the parent directory files, err := op.List(ctx, d, parentPath, model.ListArgs{}) if err != nil { return err } // Find all files with the same name var duplicates []model.Obj for _, file := range files { if file.GetName() == fileName && file.GetID() != skipID { duplicates = append(duplicates, file) } } // Remove all duplicates files, except the file with the given ID for _, file := range duplicates { err := d.Remove(ctx, file) if err != nil { return err } } return nil } ================================================ FILE: drivers/aliyundrive_share/driver.go ================================================ package aliyundrive_share import ( "context" "net/http" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) type AliyundriveShare struct { model.Storage Addition AccessToken string ShareToken string DriveId string cron *cron.Cron limiter *limiter } func (d *AliyundriveShare) Config() driver.Config { return config } func (d *AliyundriveShare) GetAddition() driver.Additional { return &d.Addition } func (d *AliyundriveShare) Init(ctx context.Context) error { d.limiter = getLimiter() err := d.refreshToken(ctx) if err != nil { d.limiter.free() d.limiter = nil return err } err = d.getShareToken(ctx) if err != nil { d.limiter.free() d.limiter = nil return err } d.cron = cron.NewCron(time.Hour * 2) d.cron.Do(func() { err := d.refreshToken(ctx) if err != nil { log.Errorf("%+v", err) } }) return nil } func (d *AliyundriveShare) Drop(ctx context.Context) error { if d.cron != nil { d.cron.Stop() } d.limiter.free() d.limiter = nil d.DriveId = "" return nil } func (d *AliyundriveShare) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(ctx, dir.GetID()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return fileToObj(src), nil }) } func (d *AliyundriveShare) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { data := base.Json{ "drive_id": d.DriveId, "file_id": file.GetID(), // // Only ten minutes lifetime "expire_sec": 600, "share_id": d.ShareId, } var resp ShareLinkResp _, err := d.request(ctx, limiterLink, "https://api.alipan.com/v2/file/get_share_link_download_url", http.MethodPost, func(req *resty.Request) { req.SetHeader(CanaryHeaderKey, CanaryHeaderValue).SetBody(data).SetResult(&resp) }) if err != nil { return nil, err } return &model.Link{ Header: http.Header{ "Referer": []string{"https://www.alipan.com/"}, }, URL: resp.DownloadUrl, }, nil } func (d *AliyundriveShare) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { var resp base.Json var url string data := base.Json{ "share_id": d.ShareId, "file_id": args.Obj.GetID(), } switch args.Method { case "doc_preview": url = "https://api.alipan.com/v2/file/get_office_preview_url" case "video_preview": url = "https://api.alipan.com/v2/file/get_video_preview_play_info" data["category"] = "live_transcoding" default: return nil, errs.NotSupport } _, err := d.request(ctx, limiterOther, url, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetResult(&resp) }) if err != nil { return nil, err } return resp, nil } var _ driver.Driver = (*AliyundriveShare)(nil) ================================================ FILE: drivers/aliyundrive_share/limiter.go ================================================ package aliyundrive_share import ( "context" "fmt" "golang.org/x/time/rate" ) // See issue https://github.com/OpenListTeam/OpenList/issues/724 // Seems there is no limit per user. type limiterType int const ( limiterList limiterType = iota limiterLink limiterOther ) const ( listRateLimit = 3.9 // 4 per second in document, but we use 3.9 per second to be safe linkRateLimit = 0.9 // 1 per second in document, but we use 0.9 per second to be safe otherRateLimit = 14.9 // 15 per second in document, but we use 14.9 per second to be safe ) type limiter struct { list *rate.Limiter link *rate.Limiter other *rate.Limiter } func getLimiter() *limiter { return &limiter{ list: rate.NewLimiter(rate.Limit(listRateLimit), 1), link: rate.NewLimiter(rate.Limit(linkRateLimit), 1), other: rate.NewLimiter(rate.Limit(otherRateLimit), 1), } } func (l *limiter) wait(ctx context.Context, typ limiterType) error { if l == nil { return fmt.Errorf("driver not init") } switch typ { case limiterList: return l.list.Wait(ctx) case limiterLink: return l.link.Wait(ctx) case limiterOther: return l.other.Wait(ctx) default: return fmt.Errorf("unknown limiter type") } } func (l *limiter) free() { } func (d *AliyundriveShare) wait(ctx context.Context, typ limiterType) error { if d == nil { return fmt.Errorf("driver not init") } //if d.ref != nil { // return d.ref.wait(ctx, typ) // If this is a reference driver, wait on the reference driver. //} return d.limiter.wait(ctx, typ) } ================================================ FILE: drivers/aliyundrive_share/meta.go ================================================ package aliyundrive_share import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { RefreshToken string `json:"refresh_token" required:"true"` ShareId string `json:"share_id" required:"true"` SharePwd string `json:"share_pwd"` driver.RootID OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` } var config = driver.Config{ Name: "AliyundriveShare", LocalSort: false, OnlyProxy: false, NoUpload: true, DefaultRoot: "root", } func init() { op.RegisterDriver(func() driver.Driver { return &AliyundriveShare{} }) } ================================================ FILE: drivers/aliyundrive_share/types.go ================================================ package aliyundrive_share import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type ErrorResp struct { Code string `json:"code"` Message string `json:"message"` } type ShareTokenResp struct { ShareToken string `json:"share_token"` ExpireTime time.Time `json:"expire_time"` ExpiresIn int `json:"expires_in"` } type ListResp struct { Items []File `json:"items"` NextMarker string `json:"next_marker"` PunishedFileCount int `json:"punished_file_count"` } type File struct { DriveId string `json:"drive_id"` DomainId string `json:"domain_id"` FileId string `json:"file_id"` ShareId string `json:"share_id"` Name string `json:"name"` Type string `json:"type"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` ParentFileId string `json:"parent_file_id"` Size int64 `json:"size"` Thumbnail string `json:"thumbnail"` } func fileToObj(f File) *model.ObjThumb { return &model.ObjThumb{ Object: model.Object{ ID: f.FileId, Name: f.Name, Size: f.Size, Modified: f.UpdatedAt, Ctime: f.CreatedAt, IsFolder: f.Type == "folder", }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbnail}, } } type ShareLinkResp struct { DownloadUrl string `json:"download_url"` Url string `json:"url"` Thumbnail string `json:"thumbnail"` } ================================================ FILE: drivers/aliyundrive_share/util.go ================================================ package aliyundrive_share import ( "context" "errors" "fmt" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/op" log "github.com/sirupsen/logrus" ) const ( // CanaryHeaderKey CanaryHeaderValue for lifting rate limit restrictions CanaryHeaderKey = "X-Canary" CanaryHeaderValue = "client=web,app=share,version=v2.3.1" ) func (d *AliyundriveShare) refreshToken(ctx context.Context) error { err := d.wait(ctx, limiterOther) if err != nil { return err } url := "https://auth.alipan.com/v2/account/token" var resp base.TokenResp var e ErrorResp _, err = base.RestyClient.R(). SetBody(base.Json{"refresh_token": d.RefreshToken, "grant_type": "refresh_token"}). SetResult(&resp). SetError(&e). Post(url) if err != nil { return err } if e.Code != "" { return fmt.Errorf("failed to refresh token: %s", e.Message) } d.RefreshToken, d.AccessToken = resp.RefreshToken, resp.AccessToken op.MustSaveDriverStorage(d) return nil } // do others that not defined in Driver interface func (d *AliyundriveShare) getShareToken(ctx context.Context) error { err := d.wait(ctx, limiterOther) if err != nil { return err } data := base.Json{ "share_id": d.ShareId, } if d.SharePwd != "" { data["share_pwd"] = d.SharePwd } var e ErrorResp var resp ShareTokenResp _, err = base.RestyClient.R(). SetResult(&resp).SetError(&e).SetBody(data). Post("https://api.alipan.com/v2/share_link/get_share_token") if err != nil { return err } if e.Code != "" { return errors.New(e.Message) } d.ShareToken = resp.ShareToken return nil } func (d *AliyundriveShare) request(ctx context.Context, limitTy limiterType, url, method string, callback base.ReqCallback) ([]byte, error) { var e ErrorResp req := base.RestyClient.R(). SetError(&e). SetHeader("content-type", "application/json"). SetHeader("Authorization", "Bearer\t"+d.AccessToken). SetHeader(CanaryHeaderKey, CanaryHeaderValue). SetHeader("x-share-token", d.ShareToken) if callback != nil { callback(req) } else { req.SetBody("{}") } err := d.wait(ctx, limitTy) if err != nil { return nil, err } resp, err := req.Execute(method, url) if err != nil { return nil, err } if e.Code != "" { if e.Code == "AccessTokenInvalid" || e.Code == "ShareLinkTokenInvalid" { if e.Code == "AccessTokenInvalid" { err = d.refreshToken(ctx) } else { err = d.getShareToken(ctx) } if err != nil { return nil, err } return d.request(ctx, limitTy, url, method, callback) } else { return nil, errors.New(e.Code + ": " + e.Message) } } return resp.Body(), nil } func (d *AliyundriveShare) getFiles(ctx context.Context, fileId string) ([]File, error) { files := make([]File, 0) data := base.Json{ "image_thumbnail_process": "image/resize,w_160/format,jpeg", "image_url_process": "image/resize,w_1920/format,jpeg", "limit": 200, "order_by": d.OrderBy, "order_direction": d.OrderDirection, "parent_file_id": fileId, "share_id": d.ShareId, "video_thumbnail_process": "video/snapshot,t_1000,f_jpg,ar_auto,w_300", "marker": "first", } for data["marker"] != "" { if data["marker"] == "first" { data["marker"] = "" } err := d.wait(ctx, limiterList) if err != nil { return nil, err } var e ErrorResp var resp ListResp res, err := base.RestyClient.R(). SetHeader("x-share-token", d.ShareToken). SetHeader(CanaryHeaderKey, CanaryHeaderValue). SetResult(&resp).SetError(&e).SetBody(data). Post("https://api.alipan.com/adrive/v3/file/list") if err != nil { return nil, err } log.Debugf("aliyundrive share get files: %s", res.String()) if e.Code != "" { if e.Code == "AccessTokenInvalid" || e.Code == "ShareLinkTokenInvalid" { err = d.getShareToken(ctx) if err != nil { return nil, err } return d.getFiles(ctx, fileId) } return nil, errors.New(e.Message) } data["marker"] = resp.NextMarker files = append(files, resp.Items...) } if len(files) > 0 && d.DriveId == "" { d.DriveId = files[0].DriveId } return files, nil } ================================================ FILE: drivers/all.go ================================================ package drivers import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/115" _ "github.com/OpenListTeam/OpenList/v4/drivers/115_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/115_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/123" _ "github.com/OpenListTeam/OpenList/v4/drivers/123_link" _ "github.com/OpenListTeam/OpenList/v4/drivers/123_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/123_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/139" _ "github.com/OpenListTeam/OpenList/v4/drivers/189" _ "github.com/OpenListTeam/OpenList/v4/drivers/189_tv" _ "github.com/OpenListTeam/OpenList/v4/drivers/189pc" _ "github.com/OpenListTeam/OpenList/v4/drivers/alias" _ "github.com/OpenListTeam/OpenList/v4/drivers/alist_v3" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/autoindex" _ "github.com/OpenListTeam/OpenList/v4/drivers/azure_blob" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_netdisk" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_photo" _ "github.com/OpenListTeam/OpenList/v4/drivers/chaoxing" _ "github.com/OpenListTeam/OpenList/v4/drivers/chunk" _ "github.com/OpenListTeam/OpenList/v4/drivers/cloudreve" _ "github.com/OpenListTeam/OpenList/v4/drivers/cloudreve_v4" _ "github.com/OpenListTeam/OpenList/v4/drivers/cnb_releases" _ "github.com/OpenListTeam/OpenList/v4/drivers/crypt" _ "github.com/OpenListTeam/OpenList/v4/drivers/degoo" _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao" _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/dropbox" _ "github.com/OpenListTeam/OpenList/v4/drivers/febbox" _ "github.com/OpenListTeam/OpenList/v4/drivers/ftp" _ "github.com/OpenListTeam/OpenList/v4/drivers/github" _ "github.com/OpenListTeam/OpenList/v4/drivers/github_releases" _ "github.com/OpenListTeam/OpenList/v4/drivers/google_drive" _ "github.com/OpenListTeam/OpenList/v4/drivers/google_photo" _ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud" _ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/ilanzou" _ "github.com/OpenListTeam/OpenList/v4/drivers/ipfs_api" _ "github.com/OpenListTeam/OpenList/v4/drivers/kodbox" _ "github.com/OpenListTeam/OpenList/v4/drivers/lanzou" _ "github.com/OpenListTeam/OpenList/v4/drivers/lenovonas_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/local" _ "github.com/OpenListTeam/OpenList/v4/drivers/mediafire" _ "github.com/OpenListTeam/OpenList/v4/drivers/mediatrack" _ "github.com/OpenListTeam/OpenList/v4/drivers/mega" _ "github.com/OpenListTeam/OpenList/v4/drivers/misskey" _ "github.com/OpenListTeam/OpenList/v4/drivers/mopan" _ "github.com/OpenListTeam/OpenList/v4/drivers/netease_music" _ "github.com/OpenListTeam/OpenList/v4/drivers/onedrive" _ "github.com/OpenListTeam/OpenList/v4/drivers/onedrive_app" _ "github.com/OpenListTeam/OpenList/v4/drivers/onedrive_sharelink" _ "github.com/OpenListTeam/OpenList/v4/drivers/openlist" _ "github.com/OpenListTeam/OpenList/v4/drivers/openlist_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" _ "github.com/OpenListTeam/OpenList/v4/drivers/pikpak_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/proton_drive" _ "github.com/OpenListTeam/OpenList/v4/drivers/quark_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/quark_uc" _ "github.com/OpenListTeam/OpenList/v4/drivers/quark_uc_tv" _ "github.com/OpenListTeam/OpenList/v4/drivers/s3" _ "github.com/OpenListTeam/OpenList/v4/drivers/seafile" _ "github.com/OpenListTeam/OpenList/v4/drivers/sftp" _ "github.com/OpenListTeam/OpenList/v4/drivers/smb" _ "github.com/OpenListTeam/OpenList/v4/drivers/strm" _ "github.com/OpenListTeam/OpenList/v4/drivers/teambition" _ "github.com/OpenListTeam/OpenList/v4/drivers/teldrive" _ "github.com/OpenListTeam/OpenList/v4/drivers/terabox" _ "github.com/OpenListTeam/OpenList/v4/drivers/thunder" _ "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" _ "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" _ "github.com/OpenListTeam/OpenList/v4/drivers/url_tree" _ "github.com/OpenListTeam/OpenList/v4/drivers/uss" _ "github.com/OpenListTeam/OpenList/v4/drivers/virtual" _ "github.com/OpenListTeam/OpenList/v4/drivers/webdav" _ "github.com/OpenListTeam/OpenList/v4/drivers/weiyun" _ "github.com/OpenListTeam/OpenList/v4/drivers/wopan" _ "github.com/OpenListTeam/OpenList/v4/drivers/wps" _ "github.com/OpenListTeam/OpenList/v4/drivers/yandex_disk" ) // All do nothing,just for import // same as _ import func All() { } ================================================ FILE: drivers/autoindex/driver.go ================================================ package autoindex import ( "context" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/antchfx/htmlquery" "github.com/antchfx/xpath" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) type AutoIndex struct { model.Storage Addition itemXPath *xpath.Expr nameXPath *xpath.Expr modifiedXPath *xpath.Expr sizeXPath *xpath.Expr ignores map[string]any } func (d *AutoIndex) Config() driver.Config { return config } func (d *AutoIndex) GetAddition() driver.Additional { return &d.Addition } func (d *AutoIndex) Init(ctx context.Context) error { var err error d.itemXPath, err = xpath.Compile(d.ItemXPath) if err != nil { return errors.WithMessage(err, "failed to compile Item XPath") } d.nameXPath, err = xpath.Compile(d.NameXPath) if err != nil { return errors.WithMessage(err, "failed to compile Name XPath") } if len(d.ModifiedXPath) > 0 { d.modifiedXPath, err = xpath.Compile(d.ModifiedXPath) if err != nil { return errors.WithMessage(err, "failed to compile Modified XPath") } } if len(d.SizeXPath) > 0 { d.sizeXPath, err = xpath.Compile(d.SizeXPath) if err != nil { return errors.WithMessage(err, "failed to compile Size XPath") } } ignores := strings.Split(d.IgnoreFileNames, "\n") d.ignores = make(map[string]any, len(ignores)) for _, i := range ignores { i = strings.TrimSpace(i) if len(i) == 0 { continue } d.ignores[i] = struct{}{} } hasScheme := strings.Contains(d.URL, "://") hasSuffix := strings.HasSuffix(d.URL, "/") if !hasScheme || !hasSuffix { if !hasSuffix { d.URL = d.URL + "/" } if !hasScheme { d.URL = "https://" + d.URL } op.MustSaveDriverStorage(d) } return nil } func (d *AutoIndex) Drop(ctx context.Context) error { return nil } func (d *AutoIndex) GetRoot(ctx context.Context) (model.Obj, error) { return &model.Object{ Name: op.RootName, Path: d.URL, Modified: d.Modified, Mask: model.Locked, IsFolder: true, }, nil } func (d *AutoIndex) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { res, err := base.RestyClient.R(). SetContext(ctx). SetDoNotParseResponse(true). Get(dir.GetPath()) if err != nil { return nil, errors.WithMessagef(err, "failed to get url [%s]", dir.GetPath()) } defer res.RawResponse.Body.Close() doc, err := htmlquery.Parse(res.RawBody()) if err != nil { return nil, errors.WithMessagef(err, "failed to parse [%s]", dir.GetPath()) } itemsIter := d.itemXPath.Select(htmlquery.CreateXPathNavigator(doc)) var objs []model.Obj for itemsIter.MoveNext() { nameFull, err := parseString(d.nameXPath.Evaluate(itemsIter.Current().Copy())) if err != nil { log.Warnf("skip invalid name evaluating result: %v", err) continue } nameFull = strings.TrimSpace(nameFull) name, isDir := strings.CutSuffix(nameFull, "/") if _, ok := d.ignores[name]; ok { continue } var size int64 = 0 exact := false modified := time.Now() if d.sizeXPath != nil { size, exact, err = parseSize(d.sizeXPath.Evaluate(itemsIter.Current().Copy())) if err != nil { log.Errorf("failed to parse size of %s: %v", name, err) } } if d.modifiedXPath != nil { modified, err = parseTime(d.modifiedXPath.Evaluate(itemsIter.Current().Copy()), d.ModifiedTimeFormat) if err != nil { log.Errorf("failed to parse modified time of %s: %v", name, err) } } var o model.Obj = &model.Object{ Name: name, IsFolder: isDir, Path: dir.GetPath() + nameFull, Modified: modified, Size: size, } if exact { o = &exactSizeObj{Obj: o} } objs = append(objs, o) } return objs, nil } func (d *AutoIndex) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if _, ok := file.(*exactSizeObj); ok || args.Redirect { return &model.Link{URL: file.GetPath()}, nil } res, err := base.RestyClient.R(). SetContext(ctx). SetDoNotParseResponse(true). Head(file.GetPath()) if err != nil { return nil, errors.WithMessagef(err, "failed to head [%s]", file.GetPath()) } _ = res.RawResponse.Body.Close() return &model.Link{ URL: file.GetPath(), ContentLength: res.RawResponse.ContentLength, }, nil } var _ driver.Driver = (*AutoIndex)(nil) ================================================ FILE: drivers/autoindex/meta.go ================================================ package autoindex import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { URL string `json:"url" required:"true"` ItemXPath string `json:"item_xpath" required:"true"` NameXPath string `json:"name_xpath" required:"true"` ModifiedXPath string `json:"modified_xpath"` SizeXPath string `json:"size_xpath"` IgnoreFileNames string `json:"ignore_file_names" type:"text" default:".\n..\nParent Directory\nUp"` ModifiedTimeFormat string `json:"modified_time_format" default:"02-Jan-2006 15:04" help:"Must be based on the time point Mon Jan 2 15:04:05 -0700 MST 2006"` } var config = driver.Config{ Name: "AutoIndex", LocalSort: true, CheckStatus: true, NoUpload: true, } func init() { op.RegisterDriver(func() driver.Driver { return &AutoIndex{} }) } ================================================ FILE: drivers/autoindex/types.go ================================================ package autoindex import ( "fmt" "github.com/OpenListTeam/OpenList/v4/internal/model" ) var ( errEmptyEvaluateResult = fmt.Errorf("empty result") ) type exactSizeObj struct{ model.Obj } ================================================ FILE: drivers/autoindex/util.go ================================================ package autoindex import ( "fmt" "strconv" "strings" "time" "github.com/antchfx/xpath" "github.com/pkg/errors" ) var units = map[string]int64{ "": 1, "b": 1, "byte": 1, "bytes": 1, "k": 1 << 10, "kb": 1 << 10, "kib": 1 << 10, "m": 1 << 20, "mb": 1 << 20, "mib": 1 << 20, "g": 1 << 30, "gb": 1 << 30, "gib": 1 << 30, "t": 1 << 40, "tb": 1 << 40, "tib": 1 << 40, "p": 1 << 50, "pb": 1 << 50, "pib": 1 << 50, } func splitUnit(s string) (string, string) { for i := len(s) - 1; i >= 0; i-- { if s[i] >= '0' && s[i] <= '9' { return strings.TrimSpace(s[:i+1]), strings.TrimSpace(s[i+1:]) } } return "", s } func parseSize(a any) (int64, bool, error) { // 第二个返回值exact表示大小是否精确 if f, ok := a.(float64); ok { return int64(f), false, nil } s, err := parseString(a) if errors.Is(err, errEmptyEvaluateResult) { // 可能是错误，也可能确实大小为0 // 如果确实大小为0，大概率不会下载，exact返回false也不会有什么性能损失 // 如果是错误，exact返回true会导致本地代理出错，综合来看返回false更好 return 0, false, nil } if err != nil { return 0, false, err } s = strings.TrimSpace(s) if s == "-" { return 0, false, nil } nbs, unit := splitUnit(s) mul, ok := units[strings.ToLower(unit)] exact := mul == 1 if !ok { mul = 1 // 推测无单位，exact应为false } nb, err := strconv.ParseInt(nbs, 10, 64) if err != nil { fnb, err := strconv.ParseFloat(nbs, 64) if err != nil { return 0, false, fmt.Errorf("failed to convert %s to number", nbs) } nb = int64(fnb * float64(mul)) exact = false } else { nb = nb * mul } return nb, exact, nil } func parseString(res any) (string, error) { if r, ok := res.(string); ok { if len(r) == 0 { return "", errEmptyEvaluateResult } return r, nil } n, ok := res.(*xpath.NodeIterator) if !ok { return "", fmt.Errorf("unsupported evaluating result") } if !n.MoveNext() { return "", fmt.Errorf("no matched nodes") } ns := n.Current().Value() if len(ns) == 0 { return "", errEmptyEvaluateResult } return ns, nil } func parseTime(res any, format string) (time.Time, error) { s, err := parseString(res) if err != nil { return time.Now(), err } s = strings.TrimSpace(s) t, err := time.Parse(format, s) if err != nil { return time.Now(), errors.WithMessagef(err, "failed to convert %s to time", s) } return t, nil } ================================================ FILE: drivers/autoindex/util_test.go ================================================ package autoindex import ( "testing" ) type wantType struct { v int64 exact bool error bool } func TestParseSize(t *testing.T) { tests := []struct { input string want wantType }{ {"100", wantType{100, true, false}}, {"1k", wantType{1024, false, false}}, {"1kb", wantType{1024, false, false}}, {"1K", wantType{1024, false, false}}, // case insensitive {"1.5m", wantType{1572864, false, false}}, // 1.5 * 1024^2 {"500 bytes", wantType{500, true, false}}, {"-", wantType{0, false, false}}, {"", wantType{0, false, false}}, {"abc", wantType{0, false, true}}, {"1.5GB", wantType{1610612736, false, false}}, // 1.5 * 1024^3 {"2t", wantType{2199023255552, false, false}}, // 2 * 1024^4 {"1p", wantType{1125899906842624, false, false}}, // 1 * 1024^5 {"0", wantType{0, true, false}}, {" 100 ", wantType{100, true, false}}, // trimmed {"100b", wantType{100, true, false}}, {"1gib", wantType{1073741824, false, false}}, // 1024^3 {"1z", wantType{1, false, false}}, // invalid unit, mul=1 {"1.5", wantType{1, false, false}}, // float without unit, truncated {"2.7k", wantType{2764, false, false}}, // 2.7 * 1024 truncated {"1.0g", wantType{1073741824, false, false}}, // 1.0 * 1024^3 {"invalid", wantType{0, false, true}}, {"123xyz", wantType{123, false, false}}, // unit not found, mul=1 } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { got, exact, err := parseSize(tt.input) if got != tt.want.v || exact != tt.want.exact || (err != nil) != tt.want.error { t.Errorf("ParseSize(%q) = (%d, %t, %t), want (%d, %t, %t)", tt.input, got, exact, err != nil, tt.want.v, tt.want.exact, tt.want.error) } }) } } ================================================ FILE: drivers/azure_blob/driver.go ================================================ package azure_blob import ( "context" "fmt" "io" "path" "regexp" "strings" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" ) // Azure Blob Storage based on the blob APIs // Link: https://learn.microsoft.com/rest/api/storageservices/blob-service-rest-api type AzureBlob struct { model.Storage Addition client *azblob.Client containerClient *container.Client } // Config returns the driver configuration. func (d *AzureBlob) Config() driver.Config { return config } // GetAddition returns additional settings specific to Azure Blob Storage. func (d *AzureBlob) GetAddition() driver.Additional { return &d.Addition } // Init initializes the Azure Blob Storage client using shared key authentication. func (d *AzureBlob) Init(ctx context.Context) error { // Validate the endpoint URL accountName := extractAccountName(d.Addition.Endpoint) if !regexp.MustCompile(`^[a-z0-9]+$`).MatchString(accountName) { return fmt.Errorf("invalid storage account name: must be chars of lowercase letters or numbers only") } credential, err := azblob.NewSharedKeyCredential(accountName, d.Addition.AccessKey) if err != nil { return fmt.Errorf("failed to create credential: %w", err) } // Check if Endpoint is just account name endpoint := d.Addition.Endpoint if accountName == endpoint { endpoint = fmt.Sprintf("https://%s.blob.core.windows.net/", accountName) } // Initialize Azure Blob client with retry policy client, err := azblob.NewClientWithSharedKeyCredential(endpoint, credential, &azblob.ClientOptions{ClientOptions: azcore.ClientOptions{ Retry: policy.RetryOptions{ MaxRetries: MaxRetries, RetryDelay: RetryDelay, }, }}) if err != nil { return fmt.Errorf("failed to create client: %w", err) } d.client = client // Ensure container exists or create it containerName := strings.Trim(d.Addition.ContainerName, "/ \\") if containerName == "" { return fmt.Errorf("container name cannot be empty") } return d.createContainerIfNotExists(ctx, containerName) } // Drop releases resources associated with the Azure Blob client. func (d *AzureBlob) Drop(ctx context.Context) error { d.client = nil return nil } // List retrieves blobs and directories under the specified path. func (d *AzureBlob) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { prefix := ensureTrailingSlash(dir.GetPath()) if prefix == "/" { prefix = "" } pager := d.containerClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{ Prefix: &prefix, }) var objs []model.Obj for pager.More() { page, err := pager.NextPage(ctx) if err != nil { return nil, fmt.Errorf("failed to list blobs: %w", err) } // Process directories for _, blobPrefix := range page.Segment.BlobPrefixes { objs = append(objs, &model.Object{ Name: path.Base(strings.TrimSuffix(*blobPrefix.Name, "/")), Path: *blobPrefix.Name, // Azure does not support properties now. //Modified: *blobPrefix.Properties.LastModified, //Ctime: *blobPrefix.Properties.CreationTime, IsFolder: true, }) } // Process files for _, blob := range page.Segment.BlobItems { if strings.HasSuffix(*blob.Name, "/") { continue } objs = append(objs, &model.Object{ Name: path.Base(*blob.Name), Path: *blob.Name, Size: *blob.Properties.ContentLength, Modified: *blob.Properties.LastModified, Ctime: *blob.Properties.CreationTime, IsFolder: false, }) } } return objs, nil } // Link generates a temporary SAS URL for accessing a blob. func (d *AzureBlob) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { blobClient := d.containerClient.NewBlobClient(file.GetPath()) expireDuration := time.Hour * time.Duration(d.SignURLExpire) sasURL, err := blobClient.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(expireDuration), nil) if err != nil { return nil, fmt.Errorf("failed to generate SAS URL: %w", err) } return &model.Link{URL: sasURL}, nil } // MakeDir creates a virtual directory by uploading an empty blob as a marker. func (d *AzureBlob) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { dirPath := path.Join(parentDir.GetPath(), dirName) if err := d.mkDir(ctx, dirPath); err != nil { return nil, fmt.Errorf("failed to create directory marker: %w", err) } return &model.Object{ Path: dirPath, Name: dirName, IsFolder: true, }, nil } // Move relocates an object (file or directory) to a new directory. func (d *AzureBlob) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { srcPath := srcObj.GetPath() dstPath := path.Join(dstDir.GetPath(), srcObj.GetName()) if err := d.moveOrRename(ctx, srcPath, dstPath, srcObj.IsDir(), srcObj.GetSize()); err != nil { return nil, fmt.Errorf("move operation failed: %w", err) } return &model.Object{ Path: dstPath, Name: srcObj.GetName(), Modified: time.Now(), IsFolder: srcObj.IsDir(), Size: srcObj.GetSize(), }, nil } // Rename changes the name of an existing object. func (d *AzureBlob) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { srcPath := srcObj.GetPath() dstPath := path.Join(path.Dir(srcPath), newName) if err := d.moveOrRename(ctx, srcPath, dstPath, srcObj.IsDir(), srcObj.GetSize()); err != nil { return nil, fmt.Errorf("rename operation failed: %w", err) } return &model.Object{ Path: dstPath, Name: newName, Modified: time.Now(), IsFolder: srcObj.IsDir(), Size: srcObj.GetSize(), }, nil } // Copy duplicates an object (file or directory) to a specified destination directory. func (d *AzureBlob) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { dstPath := path.Join(dstDir.GetPath(), srcObj.GetName()) // Handle directory copying using flat listing if srcObj.IsDir() { srcPrefix := srcObj.GetPath() srcPrefix = ensureTrailingSlash(srcPrefix) // Get all blobs under the source directory blobs, err := d.flattenListBlobs(ctx, srcPrefix) if err != nil { return nil, fmt.Errorf("failed to list source directory contents: %w", err) } // Process each blob - copy to destination for _, blob := range blobs { // Skip the directory marker itself if *blob.Name == srcPrefix { continue } // Calculate relative path from source relPath := strings.TrimPrefix(*blob.Name, srcPrefix) itemDstPath := path.Join(dstPath, relPath) if strings.HasSuffix(itemDstPath, "/") || (blob.Metadata["hdi_isfolder"] != nil && *blob.Metadata["hdi_isfolder"] == "true") { // Create directory marker at destination err := d.mkDir(ctx, itemDstPath) if err != nil { return nil, fmt.Errorf("failed to create directory marker [%s]: %w", itemDstPath, err) } } else { // Copy the blob if err := d.copyFile(ctx, *blob.Name, itemDstPath); err != nil { return nil, fmt.Errorf("failed to copy %s: %w", *blob.Name, err) } } } // Create directory marker at destination if needed if len(blobs) == 0 { err := d.mkDir(ctx, dstPath) if err != nil { return nil, fmt.Errorf("failed to create directory [%s]: %w", dstPath, err) } } return &model.Object{ Path: dstPath, Name: srcObj.GetName(), Modified: time.Now(), IsFolder: true, }, nil } // Copy a single file if err := d.copyFile(ctx, srcObj.GetPath(), dstPath); err != nil { return nil, fmt.Errorf("failed to copy blob: %w", err) } return &model.Object{ Path: dstPath, Name: srcObj.GetName(), Size: srcObj.GetSize(), Modified: time.Now(), IsFolder: false, }, nil } // Remove deletes a specified blob or recursively deletes a directory and its contents. func (d *AzureBlob) Remove(ctx context.Context, obj model.Obj) error { path := obj.GetPath() // Handle recursive directory deletion if obj.IsDir() { return d.deleteFolder(ctx, path) } // Delete single file return d.deleteFile(ctx, path, false) } // Put uploads a file stream to Azure Blob Storage with progress tracking. func (d *AzureBlob) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { blobPath := path.Join(dstDir.GetPath(), stream.GetName()) blobClient := d.containerClient.NewBlockBlobClient(blobPath) // Determine optimal upload options based on file size options := optimizedUploadOptions(stream.GetSize()) // Track upload progress progressTracker := &progressTracker{ total: stream.GetSize(), updateProgress: up, } // Wrap stream to handle context cancellation and progress tracking limitedStream := driver.NewLimitedUploadStream(ctx, io.TeeReader(stream, progressTracker)) // Upload the stream to Azure Blob Storage _, err := blobClient.UploadStream(ctx, limitedStream, options) if err != nil { return nil, fmt.Errorf("failed to upload file: %w", err) } return &model.Object{ Path: blobPath, Name: stream.GetName(), Size: stream.GetSize(), Modified: time.Now(), IsFolder: false, }, nil } // The following methods related to archive handling are not implemented yet. // func (d *AzureBlob) GetArchiveMeta(...) {...} // func (d *AzureBlob) ListArchive(...) {...} // func (d *AzureBlob) Extract(...) {...} // func (d *AzureBlob) ArchiveDecompress(...) {...} // Ensure AzureBlob implements the driver.Driver interface. var _ driver.Driver = (*AzureBlob)(nil) ================================================ FILE: drivers/azure_blob/meta.go ================================================ package azure_blob import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Endpoint string `json:"endpoint" required:"true" default:"https://.blob.core.windows.net/" help:"e.g. https://accountname.blob.core.windows.net/. The full endpoint URL for Azure Storage, including the unique storage account name (3 ~ 24 numbers and lowercase letters only)."` AccessKey string `json:"access_key" required:"true" help:"The access key for Azure Storage, used for authentication. https://learn.microsoft.com/azure/storage/common/storage-account-keys-manage"` ContainerName string `json:"container_name" required:"true" help:"The name of the container in Azure Storage (created in the Azure portal). https://learn.microsoft.com/azure/storage/blobs/blob-containers-portal"` SignURLExpire int `json:"sign_url_expire" type:"number" default:"4" help:"The expiration time for SAS URLs, in hours."` } var config = driver.Config{ Name: "Azure Blob Storage", LocalSort: true, CheckStatus: true, } func init() { op.RegisterDriver(func() driver.Driver { return &AzureBlob{} }) } ================================================ FILE: drivers/azure_blob/types.go ================================================ package azure_blob import "github.com/OpenListTeam/OpenList/v4/internal/driver" // progressTracker is used to track upload progress type progressTracker struct { total int64 current int64 updateProgress driver.UpdateProgress } // Write implements io.Writer to track progress func (pt *progressTracker) Write(p []byte) (n int, err error) { n = len(p) pt.current += int64(n) if pt.updateProgress != nil && pt.total > 0 { pt.updateProgress(float64(pt.current) * 100 / float64(pt.total)) } return n, nil } ================================================ FILE: drivers/azure_blob/util.go ================================================ package azure_blob import ( "bytes" "context" "errors" "fmt" "io" "path" "sort" "strings" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" log "github.com/sirupsen/logrus" ) const ( // MaxRetries defines the maximum number of retry attempts for Azure operations MaxRetries = 3 // RetryDelay defines the base delay between retries RetryDelay = 3 * time.Second // MaxBatchSize defines the maximum number of operations in a single batch request MaxBatchSize = 128 ) // extractAccountName 从 Azure 存储 Endpoint 中提取账户名 func extractAccountName(endpoint string) string { // 移除协议前缀 endpoint = strings.TrimPrefix(endpoint, "https://") endpoint = strings.TrimPrefix(endpoint, "http://") // 获取第一个点之前的部分（即账户名） parts := strings.Split(endpoint, ".") if len(parts) > 0 { // to lower case return strings.ToLower(parts[0]) } return "" } // isNotFoundError checks if the error is a "not found" type error func isNotFoundError(err error) bool { var storageErr *azcore.ResponseError if errors.As(err, &storageErr) { return storageErr.StatusCode == 404 } // Fallback to string matching for backwards compatibility return err != nil && strings.Contains(err.Error(), "BlobNotFound") } // flattenListBlobs - Optimize blob listing to handle pagination better func (d *AzureBlob) flattenListBlobs(ctx context.Context, prefix string) ([]container.BlobItem, error) { // Standardize prefix format prefix = ensureTrailingSlash(prefix) var blobItems []container.BlobItem pager := d.containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ Prefix: &prefix, Include: container.ListBlobsInclude{ Metadata: true, }, }) for pager.More() { page, err := pager.NextPage(ctx) if err != nil { return nil, fmt.Errorf("failed to list blobs: %w", err) } for _, blob := range page.Segment.BlobItems { blobItems = append(blobItems, *blob) } } return blobItems, nil } // batchDeleteBlobs - Simplify batch deletion logic func (d *AzureBlob) batchDeleteBlobs(ctx context.Context, blobPaths []string) error { if len(blobPaths) == 0 { return nil } // Process in batches of MaxBatchSize for i := 0; i < len(blobPaths); i += MaxBatchSize { end := min(i+MaxBatchSize, len(blobPaths)) currentBatch := blobPaths[i:end] // Create batch builder batchBuilder, err := d.containerClient.NewBatchBuilder() if err != nil { return fmt.Errorf("failed to create batch builder: %w", err) } // Add delete operations for _, blobPath := range currentBatch { if err := batchBuilder.Delete(blobPath, nil); err != nil { return fmt.Errorf("failed to add delete operation for %s: %w", blobPath, err) } } // Submit batch responses, err := d.containerClient.SubmitBatch(ctx, batchBuilder, nil) if err != nil { return fmt.Errorf("batch delete request failed: %w", err) } // Check responses for _, resp := range responses.Responses { if resp.Error != nil && !isNotFoundError(resp.Error) { // 获取 blob 名称以提供更好的错误信息 blobName := "unknown" if resp.BlobName != nil { blobName = *resp.BlobName } return fmt.Errorf("failed to delete blob %s: %v", blobName, resp.Error) } } } return nil } // deleteFolder recursively deletes a directory and all its contents func (d *AzureBlob) deleteFolder(ctx context.Context, prefix string) error { // Ensure directory path ends with slash prefix = ensureTrailingSlash(prefix) // Get all blobs under the directory using flattenListBlobs globs, err := d.flattenListBlobs(ctx, prefix) if err != nil { return fmt.Errorf("failed to list blobs for deletion: %w", err) } // If there are blobs in the directory, delete them if len(globs) > 0 { // 分离文件和目录标记 var filePaths []string var dirPaths []string for _, blob := range globs { blobName := *blob.Name if isDirectory(blob) { // remove trailing slash for directory names dirPaths = append(dirPaths, strings.TrimSuffix(blobName, "/")) } else { filePaths = append(filePaths, blobName) } } // 先删除文件，再删除目录 if len(filePaths) > 0 { if err := d.batchDeleteBlobs(ctx, filePaths); err != nil { return err } } if len(dirPaths) > 0 { // 按路径深度分组 depthMap := make(map[int][]string) for _, dir := range dirPaths { depth := strings.Count(dir, "/") // 计算目录深度 depthMap[depth] = append(depthMap[depth], dir) } // 按深度从大到小排序 var depths []int for depth := range depthMap { depths = append(depths, depth) } sort.Sort(sort.Reverse(sort.IntSlice(depths))) // 按深度逐层批量删除 for _, depth := range depths { batch := depthMap[depth] if err := d.batchDeleteBlobs(ctx, batch); err != nil { return err } } } } // 最后删除目录标记本身 return d.deleteEmptyDirectory(ctx, prefix) } // deleteFile deletes a single file or blob with better error handling func (d *AzureBlob) deleteFile(ctx context.Context, path string, isDir bool) error { blobClient := d.containerClient.NewBlobClient(path) _, err := blobClient.Delete(ctx, nil) if err != nil && !(isDir && isNotFoundError(err)) { return err } return nil } // copyFile copies a single blob from source path to destination path func (d *AzureBlob) copyFile(ctx context.Context, srcPath, dstPath string) error { srcBlob := d.containerClient.NewBlobClient(srcPath) dstBlob := d.containerClient.NewBlobClient(dstPath) // Use configured expiration time for SAS URL expireDuration := time.Hour * time.Duration(d.SignURLExpire) srcURL, err := srcBlob.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(expireDuration), nil) if err != nil { return fmt.Errorf("failed to generate source SAS URL: %w", err) } _, err = dstBlob.StartCopyFromURL(ctx, srcURL, nil) return err } // createContainerIfNotExists - Create container if not exists // Clean up commented code func (d *AzureBlob) createContainerIfNotExists(ctx context.Context, containerName string) error { serviceClient := d.client.ServiceClient() containerClient := serviceClient.NewContainerClient(containerName) var options = service.CreateContainerOptions{} _, err := containerClient.Create(ctx, &options) if err != nil { var responseErr *azcore.ResponseError if errors.As(err, &responseErr) && responseErr.ErrorCode != "ContainerAlreadyExists" { return fmt.Errorf("failed to create or access container [%s]: %w", containerName, err) } } d.containerClient = containerClient return nil } // mkDir creates a virtual directory marker by uploading an empty blob with metadata. func (d *AzureBlob) mkDir(ctx context.Context, fullDirName string) error { dirPath := ensureTrailingSlash(fullDirName) blobClient := d.containerClient.NewBlockBlobClient(dirPath) // Upload an empty blob with metadata indicating it's a directory _, err := blobClient.Upload(ctx, struct { *bytes.Reader io.Closer }{ Reader: bytes.NewReader([]byte{}), Closer: io.NopCloser(nil), }, &blockblob.UploadOptions{ Metadata: map[string]*string{ "hdi_isfolder": to.Ptr("true"), }, }) return err } // ensureTrailingSlash ensures the provided path ends with a trailing slash. func ensureTrailingSlash(path string) string { if !strings.HasSuffix(path, "/") { return path + "/" } return path } // moveOrRename moves or renames blobs or directories from source to destination. func (d *AzureBlob) moveOrRename(ctx context.Context, srcPath, dstPath string, isDir bool, srcSize int64) error { if isDir { // Normalize paths for directory operations srcPath = ensureTrailingSlash(srcPath) dstPath = ensureTrailingSlash(dstPath) // List all blobs under the source directory blobs, err := d.flattenListBlobs(ctx, srcPath) if err != nil { return fmt.Errorf("failed to list blobs: %w", err) } // Iterate and copy each blob to the destination for _, item := range blobs { srcBlobName := *item.Name relPath := strings.TrimPrefix(srcBlobName, srcPath) itemDstPath := path.Join(dstPath, relPath) if isDirectory(item) { // Create directory marker at destination if err := d.mkDir(ctx, itemDstPath); err != nil { return fmt.Errorf("failed to create directory marker [%s]: %w", itemDstPath, err) } } else { // Copy file blob to destination if err := d.copyFile(ctx, srcBlobName, itemDstPath); err != nil { return fmt.Errorf("failed to copy blob [%s]: %w", srcBlobName, err) } } } // Handle empty directories by creating a marker at destination if len(blobs) == 0 { if err := d.mkDir(ctx, dstPath); err != nil { return fmt.Errorf("failed to create directory [%s]: %w", dstPath, err) } } // Delete source directory and its contents if err := d.deleteFolder(ctx, srcPath); err != nil { log.Warnf("failed to delete source directory [%s]: %v\n, and try again", srcPath, err) // Retry deletion once more and ignore the result if err := d.deleteFolder(ctx, srcPath); err != nil { log.Errorf("Retry deletion of source directory [%s] failed: %v", srcPath, err) } } return nil } // Single file move or rename operation if err := d.copyFile(ctx, srcPath, dstPath); err != nil { return fmt.Errorf("failed to copy file: %w", err) } // Delete source file after successful copy if err := d.deleteFile(ctx, srcPath, false); err != nil { log.Errorf("Error deleting source file [%s]: %v", srcPath, err) } return nil } // optimizedUploadOptions returns the optimal upload options based on file size func optimizedUploadOptions(fileSize int64) *azblob.UploadStreamOptions { options := &azblob.UploadStreamOptions{ BlockSize: 4 * 1024 * 1024, // 4MB block size Concurrency: 4, // Default concurrency } // For large files, increase block size and concurrency if fileSize > 256*1024*1024 { // For files larger than 256MB options.BlockSize = 8 * 1024 * 1024 // 8MB blocks options.Concurrency = 8 // More concurrent uploads } // For very large files (>1GB) if fileSize > 1024*1024*1024 { options.BlockSize = 16 * 1024 * 1024 // 16MB blocks options.Concurrency = 16 // Higher concurrency } return options } // isDirectory determines if a blob represents a directory // Checks multiple indicators: path suffix, metadata, and content type func isDirectory(blob container.BlobItem) bool { // Check path suffix if strings.HasSuffix(*blob.Name, "/") { return true } // Check metadata for directory marker if blob.Metadata != nil { if val, ok := blob.Metadata["hdi_isfolder"]; ok && val != nil && *val == "true" { return true } // Azure Storage Explorer and other tools may use different metadata keys if val, ok := blob.Metadata["is_directory"]; ok && val != nil && strings.ToLower(*val) == "true" { return true } } // Check content type (some tools mark directories with specific content types) if blob.Properties != nil && blob.Properties.ContentType != nil { contentType := strings.ToLower(*blob.Properties.ContentType) if blob.Properties.ContentLength != nil && *blob.Properties.ContentLength == 0 && (contentType == "application/directory" || contentType == "directory") { return true } } return false } // deleteEmptyDirectory deletes a directory only if it's empty func (d *AzureBlob) deleteEmptyDirectory(ctx context.Context, dirPath string) error { // Directory is empty, delete the directory marker blobClient := d.containerClient.NewBlobClient(strings.TrimSuffix(dirPath, "/")) _, err := blobClient.Delete(ctx, nil) // Also try deleting with trailing slash (for different directory marker formats) if err != nil && isNotFoundError(err) { blobClient = d.containerClient.NewBlobClient(dirPath) _, err = blobClient.Delete(ctx, nil) } // Ignore not found errors if err != nil && isNotFoundError(err) { log.Infof("Directory [%s] not found during deletion: %v", dirPath, err) return nil } return err } ================================================ FILE: drivers/baidu_netdisk/driver.go ================================================ package baidu_netdisk import ( "bytes" "context" "crypto/md5" "encoding/hex" "errors" "io" "mime/multipart" "net/http" "net/url" "os" stdpath "path" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" log "github.com/sirupsen/logrus" ) type BaiduNetdisk struct { model.Storage Addition uploadThread int vipType int // 会员类型，0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M) } var ErrUploadIDExpired = errors.New("uploadid expired") func (d *BaiduNetdisk) Config() driver.Config { return config } func (d *BaiduNetdisk) GetAddition() driver.Additional { return &d.Addition } func (d *BaiduNetdisk) Init(ctx context.Context) error { d.uploadThread, _ = strconv.Atoi(d.UploadThread) if d.uploadThread < 1 { d.uploadThread, d.UploadThread = 1, "1" } else if d.uploadThread > 32 { d.uploadThread, d.UploadThread = 32, "32" } if _, err := url.Parse(d.UploadAPI); d.UploadAPI == "" || err != nil { d.UploadAPI = UPLOAD_FALLBACK_API } res, err := d.get("/xpan/nas", map[string]string{ "method": "uinfo", }, nil) log.Debugf("[baidu_netdisk] get uinfo: %s", string(res)) if err != nil { return err } d.vipType = utils.Json.Get(res, "vip_type").ToInt() return nil } func (d *BaiduNetdisk) Drop(ctx context.Context) error { return nil } func (d *BaiduNetdisk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(dir.GetPath()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return fileToObj(src), nil }) } func (d *BaiduNetdisk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { switch d.DownloadAPI { case "crack": return d.linkCrack(file, args) case "crack_video": return d.linkCrackVideo(file, args) } return d.linkOfficial(file, args) } func (d *BaiduNetdisk) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { var newDir File _, err := d.create(stdpath.Join(parentDir.GetPath(), dirName), 0, 1, "", "", &newDir, 0, 0) if err != nil { return nil, err } return fileToObj(newDir), nil } func (d *BaiduNetdisk) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { data := []base.Json{ { "path": srcObj.GetPath(), "dest": dstDir.GetPath(), "newname": srcObj.GetName(), }, } _, err := d.manage("move", data) if err != nil { return nil, err } if srcObj, ok := srcObj.(*model.ObjThumb); ok { srcObj.SetPath(stdpath.Join(dstDir.GetPath(), srcObj.GetName())) srcObj.Modified = time.Now() return srcObj, nil } return nil, nil } func (d *BaiduNetdisk) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { data := []base.Json{ { "path": srcObj.GetPath(), "newname": newName, }, } _, err := d.manage("rename", data) if err != nil { return nil, err } if srcObj, ok := srcObj.(*model.ObjThumb); ok { srcObj.SetPath(stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName)) srcObj.Name = newName srcObj.Modified = time.Now() return srcObj, nil } return nil, nil } func (d *BaiduNetdisk) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { data := []base.Json{ { "path": srcObj.GetPath(), "dest": dstDir.GetPath(), "newname": srcObj.GetName(), }, } _, err := d.manage("copy", data) return err } func (d *BaiduNetdisk) Remove(ctx context.Context, obj model.Obj) error { data := []string{obj.GetPath()} _, err := d.manage("delete", data) return err } func (d *BaiduNetdisk) PutRapid(ctx context.Context, dstDir model.Obj, stream model.FileStreamer) (model.Obj, error) { contentMd5 := stream.GetHash().GetHash(utils.MD5) if len(contentMd5) < utils.MD5.Width { return nil, errors.New("invalid hash") } streamSize := stream.GetSize() path := stdpath.Join(dstDir.GetPath(), stream.GetName()) mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() blockList, _ := utils.Json.MarshalToString([]string{contentMd5}) var newFile File _, err := d.create(path, streamSize, 0, "", blockList, &newFile, mtime, ctime) if err != nil { return nil, err } // 修复时间，具体原因见 Put 方法注释的 **注意** newFile.Ctime = stream.CreateTime().Unix() newFile.Mtime = stream.ModTime().Unix() return fileToObj(newFile), nil } // Put // // **注意**: 截至 2024/04/20 百度云盘 api 接口返回的时间永远是当前时间，而不是文件时间。 // 而实际上云盘存储的时间是文件时间，所以此处需要覆盖时间，保证缓存与云盘的数据一致 func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { // 百度网盘不允许上传空文件 if stream.GetSize() < 1 { return nil, ErrBaiduEmptyFilesNotAllowed } // rapid upload if newObj, err := d.PutRapid(ctx, dstDir, stream); err == nil { return newObj, nil } var ( cache = stream.GetFile() tmpF *os.File err error ) if cache == nil { tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") if err != nil { return nil, err } defer func() { _ = tmpF.Close() _ = os.Remove(tmpF.Name()) }() cache = tmpF } streamSize := stream.GetSize() sliceSize := d.getSliceSize(streamSize) count := 1 if streamSize > sliceSize { count = int((streamSize + sliceSize - 1) / sliceSize) } lastBlockSize := streamSize % sliceSize if lastBlockSize == 0 { lastBlockSize = sliceSize } // cal md5 for first 256k data const SliceSize int64 = 256 * utils.KB blockList := make([]string, 0, count) byteSize := sliceSize fileMd5H := md5.New() sliceMd5H := md5.New() sliceMd5H2 := md5.New() slicemd5H2Write := utils.LimitWriter(sliceMd5H2, SliceSize) writers := []io.Writer{fileMd5H, sliceMd5H, slicemd5H2Write} if tmpF != nil { writers = append(writers, tmpF) } written := int64(0) for i := 1; i <= count; i++ { if utils.IsCanceled(ctx) { return nil, ctx.Err() } if i == count { byteSize = lastBlockSize } n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) written += n if err != nil && err != io.EOF { return nil, err } blockList = append(blockList, hex.EncodeToString(sliceMd5H.Sum(nil))) sliceMd5H.Reset() } if tmpF != nil { if written != streamSize { return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize) } _, err = tmpF.Seek(0, io.SeekStart) if err != nil { return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") } } contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) blockListStr, _ := utils.Json.MarshalToString(blockList) path := stdpath.Join(dstDir.GetPath(), stream.GetName()) mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() // step.1 尝试读取已保存进度 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { // 没有进度，走预上传 precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) if err != nil { return nil, err } if precreateResp.ReturnType == 2 { // rapid upload, since got md5 match from baidu server // 修复时间，具体原因见 Put 方法注释的 **注意** precreateResp.File.Ctime = ctime precreateResp.File.Mtime = mtime return fileToObj(precreateResp.File), nil } } ensureUploadURL := func() { if precreateResp.UploadURL != "" { return } precreateResp.UploadURL = d.getUploadUrl(path, precreateResp.Uploadid) } // step.2 上传分片 uploadLoop: for range 2 { // 获取上传域名 ensureUploadURL() // 并发上传 threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, retry.Attempts(UPLOAD_RETRY_COUNT), retry.Delay(UPLOAD_RETRY_WAIT_TIME), retry.MaxDelay(UPLOAD_RETRY_MAX_WAIT_TIME), retry.DelayType(retry.BackOffDelay), retry.RetryIf(func(err error) bool { return !errors.Is(err, ErrUploadIDExpired) }), retry.LastErrorOnly(true)) totalParts := len(precreateResp.BlockList) for i, partseq := range precreateResp.BlockList { if utils.IsCanceled(upCtx) { break } if partseq < 0 { continue } i, partseq := i, partseq offset, size := int64(partseq)*sliceSize, sliceSize if partseq+1 == count { size = lastBlockSize } threadG.Go(func(ctx context.Context) error { params := map[string]string{ "method": "upload", "access_token": d.AccessToken, "type": "tmpfile", "path": path, "uploadid": precreateResp.Uploadid, "partseq": strconv.Itoa(partseq), } section := io.NewSectionReader(cache, offset, size) err := d.uploadSlice(ctx, precreateResp.UploadURL, params, stream.GetName(), section) if err != nil { return err } precreateResp.BlockList[i] = -1 progress := float64(threadG.Success()+1) * 100 / float64(totalParts+1) up(progress) return nil }) } err = threadG.Wait() if err == nil { break uploadLoop } // 保存进度（所有错误都会保存） precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) if errors.Is(err, context.Canceled) { return nil, err } if errors.Is(err, ErrUploadIDExpired) { log.Warn("[baidu_netdisk] uploadid expired, will restart from scratch") // 重新 precreate（所有分片都要重传） newPre, err2 := d.precreate(ctx, path, streamSize, blockListStr, "", "", ctime, mtime) if err2 != nil { return nil, err2 } if newPre.ReturnType == 2 { return fileToObj(newPre.File), nil } precreateResp = newPre precreateResp.UploadURL = "" // 覆盖掉旧的进度 base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) continue uploadLoop } return nil, err } defer up(100) // step.3 创建文件 var newFile File _, err = d.create(path, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile, mtime, ctime) if err != nil { return nil, err } // 修复时间，具体原因见 Put 方法注释的 **注意** newFile.Ctime = ctime newFile.Mtime = mtime // 上传成功清理进度 base.SaveUploadProgress(d, nil, d.AccessToken, contentMd5) return fileToObj(newFile), nil } // precreate 执行预上传操作，支持首次上传和 uploadid 过期重试 func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize int64, blockListStr, contentMd5, sliceMd5 string, ctime, mtime int64) (*PrecreateResp, error) { params := map[string]string{"method": "precreate"} form := map[string]string{ "path": path, "size": strconv.FormatInt(streamSize, 10), "isdir": "0", "autoinit": "1", "rtype": "3", "block_list": blockListStr, } // 只有在首次上传时才包含 content-md5 和 slice-md5 if contentMd5 != "" && sliceMd5 != "" { form["content-md5"] = contentMd5 form["slice-md5"] = sliceMd5 } joinTime(form, ctime, mtime) var precreateResp PrecreateResp _, err := d.postForm("/xpan/file", params, form, &precreateResp) if err != nil { return nil, err } // 修复时间，具体原因见 Put 方法注释的 **注意** if precreateResp.ReturnType == 2 { precreateResp.File.Ctime = ctime precreateResp.File.Mtime = mtime } return &precreateResp, nil } func (d *BaiduNetdisk) uploadSlice(ctx context.Context, uploadUrl string, params map[string]string, fileName string, file *io.SectionReader) error { b := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) mw := multipart.NewWriter(b) _, err := mw.CreateFormFile("file", fileName) if err != nil { return err } headSize := b.Len() err = mw.Close() if err != nil { return err } head := bytes.NewReader(b.Bytes()[:headSize]) tail := bytes.NewReader(b.Bytes()[headSize:]) rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, file, tail)) req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"/rest/2.0/pcs/superfile2", rateLimitedRd) if err != nil { return err } query := req.URL.Query() for k, v := range params { query.Set(k, v) } req.URL.RawQuery = query.Encode() req.Header.Set("Content-Type", mw.FormDataContentType()) req.ContentLength = int64(b.Len()) + file.Size() client := net.NewHttpClient() if d.UploadSliceTimeout > 0 { client.Timeout = time.Second * time.Duration(d.UploadSliceTimeout) } else { client.Timeout = DEFAULT_UPLOAD_SLICE_TIMEOUT } resp, err := client.Do(req) if err != nil { return err } defer resp.Body.Close() b.Reset() _, err = b.ReadFrom(resp.Body) if err != nil { return err } body := b.Bytes() respStr := string(body) log.Debugln(respStr) lower := strings.ToLower(respStr) // 合并 uploadid 过期检测逻辑 if strings.Contains(lower, "uploadid") && (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { return ErrUploadIDExpired } errCode := utils.Json.Get(body, "error_code").ToInt() errNo := utils.Json.Get(body, "errno").ToInt() if errCode != 0 || errNo != 0 { return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", respStr) } return nil } func (d *BaiduNetdisk) GetDetails(ctx context.Context) (*model.StorageDetails, error) { du, err := d.quota(ctx) if err != nil { return nil, err } return &model.StorageDetails{DiskUsage: du}, nil } var _ driver.Driver = (*BaiduNetdisk)(nil) ================================================ FILE: drivers/baidu_netdisk/meta.go ================================================ package baidu_netdisk import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` DownloadAPI string `json:"download_api" type:"select" options:"official,crack,crack_video" default:"official"` UseOnlineAPI bool `json:"use_online_api" default:"true"` APIAddress string `json:"api_url_address" default:"https://api.oplist.org/baiduyun/renewapi"` ClientID string `json:"client_id"` ClientSecret string `json:"client_secret"` CustomCrackUA string `json:"custom_crack_ua" required:"true" default:"netdisk"` AccessToken string RefreshToken string `json:"refresh_token" required:"true"` UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` UploadSliceTimeout int `json:"upload_timeout" type:"number" default:"60" help:"per-slice upload timeout in seconds"` UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` UseDynamicUploadAPI bool `json:"use_dynamic_upload_api" default:"true" help:"dynamically get upload api domain, when enabled, the 'Upload API' setting will be used as a fallback if failed to get"` CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` LowBandwithUploadMode bool `json:"low_bandwith_upload_mode" default:"false"` OnlyListVideoFile bool `json:"only_list_video_file" default:"false"` } const ( UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" // 备用上传地址 UPLOAD_URL_EXPIRE_TIME = time.Minute * 60 // 上传地址有效期(分钟) DEFAULT_UPLOAD_SLICE_TIMEOUT = time.Second * 60 // 上传分片请求默认超时时间 UPLOAD_RETRY_COUNT = 3 UPLOAD_RETRY_WAIT_TIME = time.Second * 1 UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 ) var config = driver.Config{ Name: "BaiduNetdisk", DefaultRoot: "/", PreferProxy: true, } func init() { op.RegisterDriver(func() driver.Driver { return &BaiduNetdisk{} }) } ================================================ FILE: drivers/baidu_netdisk/types.go ================================================ package baidu_netdisk import ( "errors" "path" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) var ( ErrBaiduEmptyFilesNotAllowed = errors.New("empty files are not allowed by baidu netdisk") ) type TokenErrResp struct { ErrorDescription string `json:"error_description"` Error string `json:"error"` } type File struct { //TkbindId int `json:"tkbind_id"` //OwnerType int `json:"owner_type"` Category int `json:"category"` //RealCategory string `json:"real_category"` FsId int64 `json:"fs_id"` //OperId int `json:"oper_id"` Thumbs struct { //Icon string `json:"icon"` Url3 string `json:"url3"` //Url2 string `json:"url2"` //Url1 string `json:"url1"` } `json:"thumbs"` //Wpfile int `json:"wpfile"` Size int64 `json:"size"` //ExtentTinyint7 int `json:"extent_tinyint7"` Path string `json:"path"` //Share int `json:"share"` //Pl int `json:"pl"` ServerFilename string `json:"server_filename"` Md5 string `json:"md5"` //OwnerId int `json:"owner_id"` //Unlist int `json:"unlist"` Isdir int `json:"isdir"` // list resp ServerCtime int64 `json:"server_ctime"` ServerMtime int64 `json:"server_mtime"` LocalMtime int64 `json:"local_mtime"` LocalCtime int64 `json:"local_ctime"` //ServerAtime int64 `json:"server_atime"` ` // only create and precreate resp Ctime int64 `json:"ctime"` Mtime int64 `json:"mtime"` } func fileToObj(f File) *model.ObjThumb { if f.ServerFilename == "" { f.ServerFilename = path.Base(f.Path) } if f.ServerCtime == 0 { f.ServerCtime = f.Ctime } if f.ServerMtime == 0 { f.ServerMtime = f.Mtime } return &model.ObjThumb{ Object: model.Object{ ID: strconv.FormatInt(f.FsId, 10), Path: f.Path, Name: f.ServerFilename, Size: f.Size, Modified: time.Unix(f.ServerMtime, 0), Ctime: time.Unix(f.ServerCtime, 0), IsFolder: f.Isdir == 1, // 百度API返回的MD5不可信，不使用HashInfo }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, } } type ListResp struct { Errno int `json:"errno"` GuidInfo string `json:"guid_info"` List []File `json:"list"` RequestId int64 `json:"request_id"` Guid int `json:"guid"` } type DownloadResp struct { Errmsg string `json:"errmsg"` Errno int `json:"errno"` List []struct { //Category int `json:"category"` //DateTaken int `json:"date_taken,omitempty"` Dlink string `json:"dlink"` //Filename string `json:"filename"` //FsId int64 `json:"fs_id"` //Height int `json:"height,omitempty"` //Isdir int `json:"isdir"` //Md5 string `json:"md5"` //OperId int `json:"oper_id"` //Path string `json:"path"` //ServerCtime int `json:"server_ctime"` //ServerMtime int `json:"server_mtime"` //Size int `json:"size"` //Thumbs struct { // Icon string `json:"icon,omitempty"` // Url1 string `json:"url1,omitempty"` // Url2 string `json:"url2,omitempty"` // Url3 string `json:"url3,omitempty"` //} `json:"thumbs"` //Width int `json:"width,omitempty"` } `json:"list"` //Names struct { //} `json:"names"` RequestId string `json:"request_id"` } type DownloadResp2 struct { Errno int `json:"errno"` Info []struct { //ExtentTinyint4 int `json:"extent_tinyint4"` //ExtentTinyint1 int `json:"extent_tinyint1"` //Bitmap string `json:"bitmap"` //Category int `json:"category"` //Isdir int `json:"isdir"` //Videotag int `json:"videotag"` Dlink string `json:"dlink"` //OperID int64 `json:"oper_id"` //PathMd5 int `json:"path_md5"` //Wpfile int `json:"wpfile"` //LocalMtime int `json:"local_mtime"` /*Thumbs struct { Icon string `json:"icon"` URL3 string `json:"url3"` URL2 string `json:"url2"` URL1 string `json:"url1"` } `json:"thumbs"`*/ //PlaySource int `json:"play_source"` //Share int `json:"share"` //FileKey string `json:"file_key"` //Errno int `json:"errno"` //LocalCtime int `json:"local_ctime"` //Rotate int `json:"rotate"` //Metadata time.Time `json:"metadata"` //Height int `json:"height"` //SampleRate int `json:"sample_rate"` //Width int `json:"width"` //OwnerType int `json:"owner_type"` //Privacy int `json:"privacy"` //ExtentInt3 int64 `json:"extent_int3"` //RealCategory string `json:"real_category"` //SrcLocation string `json:"src_location"` //MetaInfo string `json:"meta_info"` //ID string `json:"id"` //Duration int `json:"duration"` //FileSize string `json:"file_size"` //Channels int `json:"channels"` //UseSegment int `json:"use_segment"` //ServerCtime int `json:"server_ctime"` //Resolution string `json:"resolution"` //OwnerID int `json:"owner_id"` //ExtraInfo string `json:"extra_info"` //Size int `json:"size"` //FsID int64 `json:"fs_id"` //ExtentTinyint3 int `json:"extent_tinyint3"` //Md5 string `json:"md5"` //Path string `json:"path"` //FrameRate int `json:"frame_rate"` //ExtentTinyint2 int `json:"extent_tinyint2"` //ServerFilename string `json:"server_filename"` //ServerMtime int `json:"server_mtime"` //TkbindID int `json:"tkbind_id"` } `json:"info"` RequestID int64 `json:"request_id"` } type PrecreateResp struct { Errno int `json:"errno"` RequestId int64 `json:"request_id"` ReturnType int `json:"return_type"` // return_type=1 Path string `json:"path"` Uploadid string `json:"uploadid"` BlockList []int `json:"block_list"` // return_type=2 File File `json:"info"` UploadURL string `json:"-"` // 保存断点续传对应的上传域名 } type UploadServerResp struct { BakServer []any `json:"bak_server"` BakServers []struct { Server string `json:"server"` } `json:"bak_servers"` ClientIP string `json:"client_ip"` ErrorCode int `json:"error_code"` ErrorMsg string `json:"error_msg"` Expire int `json:"expire"` Host string `json:"host"` Newno string `json:"newno"` QuicServer []any `json:"quic_server"` QuicServers []struct { Server string `json:"server"` } `json:"quic_servers"` RequestID int64 `json:"request_id"` Server []any `json:"server"` ServerTime int `json:"server_time"` Servers []struct { Server string `json:"server"` } `json:"servers"` Sl int `json:"sl"` } type QuotaResp struct { Errno int `json:"errno"` RequestId int64 `json:"request_id"` Total int64 `json:"total"` Used int64 `json:"used"` //FreeSpace uint64 `json:"free"` //Expire bool `json:"expire"` } ================================================ FILE: drivers/baidu_netdisk/util.go ================================================ package baidu_netdisk import ( "context" "encoding/hex" "errors" "fmt" "net/http" "strconv" "strings" "time" "unicode" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) // do others that not defined in Driver interface func (d *BaiduNetdisk) refreshToken() error { err := d._refreshToken() if err != nil && errors.Is(err, errs.EmptyToken) { err = d._refreshToken() } return err } func (d *BaiduNetdisk) _refreshToken() error { // 使用在线API刷新Token，无需ClientID和ClientSecret if d.UseOnlineAPI && len(d.APIAddress) > 0 { u := d.APIAddress var resp struct { RefreshToken string `json:"refresh_token"` AccessToken string `json:"access_token"` ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, "server_use": "true", "driver_txt": "baiduyun_go", }). Get(u) if err != nil { return err } if resp.RefreshToken == "" || resp.AccessToken == "" { if resp.ErrorMessage != "" { return fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) } return fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") } d.AccessToken = resp.AccessToken d.RefreshToken = resp.RefreshToken op.MustSaveDriverStorage(d) return nil } // 使用本地客户端的情况下检查是否为空 if d.ClientID == "" || d.ClientSecret == "" { return fmt.Errorf("empty ClientID or ClientSecret") } // 走原有的刷新逻辑 u := "https://openapi.baidu.com/oauth/2.0/token" var resp base.TokenResp var e TokenErrResp _, err := base.RestyClient.R(). SetResult(&resp). SetError(&e). SetQueryParams(map[string]string{ "grant_type": "refresh_token", "refresh_token": d.RefreshToken, "client_id": d.ClientID, "client_secret": d.ClientSecret, }). Get(u) if err != nil { return err } if e.Error != "" { return fmt.Errorf("%s : %s", e.Error, e.ErrorDescription) } if resp.RefreshToken == "" { return errs.EmptyToken } d.AccessToken, d.RefreshToken = resp.AccessToken, resp.RefreshToken op.MustSaveDriverStorage(d) return nil } func (d *BaiduNetdisk) request(furl string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { var result []byte err := retry.Do(func() error { req := base.RestyClient.R() req.SetQueryParam("access_token", d.AccessToken) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } res, err := req.Execute(method, furl) if err != nil { return err } log.Debugf("[baidu_netdisk] req: %s, resp: %s", furl, res.String()) errno := utils.Json.Get(res.Body(), "errno").ToInt() if errno != 0 { if utils.SliceContains([]int{111, -6}, errno) { log.Info("[baidu_netdisk] refreshing baidu_netdisk token.") err2 := d.refreshToken() if err2 != nil { return retry.Unrecoverable(err2) } } if errno == 31023 && d.DownloadAPI == "crack_video" { result = res.Body() return nil } return fmt.Errorf("req: [%s] ,errno: %d, refer to https://pan.baidu.com/union/doc/", furl, errno) } result = res.Body() return nil }, retry.LastErrorOnly(true), retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) return result, err } func (d *BaiduNetdisk) get(pathname string, params map[string]string, resp interface{}) ([]byte, error) { return d.request("https://pan.baidu.com/rest/2.0"+pathname, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(params) }, resp) } func (d *BaiduNetdisk) postForm(pathname string, params map[string]string, form map[string]string, resp interface{}) ([]byte, error) { return d.request("https://pan.baidu.com/rest/2.0"+pathname, http.MethodPost, func(req *resty.Request) { req.SetQueryParams(params) req.SetFormData(form) }, resp) } func (d *BaiduNetdisk) getFiles(dir string) ([]File, error) { start := 0 limit := 1000 params := map[string]string{ "method": "list", "dir": dir, "web": "web", } if d.OrderBy != "" { params["order"] = d.OrderBy if d.OrderDirection == "desc" { params["desc"] = "1" } } res := make([]File, 0) for { params["start"] = strconv.Itoa(start) params["limit"] = strconv.Itoa(limit) var resp ListResp _, err := d.get("/xpan/file", params, &resp) if err != nil { return nil, err } if len(resp.List) == 0 { break } if d.OnlyListVideoFile { for _, file := range resp.List { if file.Isdir == 1 || file.Category == 1 { res = append(res, file) } } } else { res = append(res, resp.List...) } if len(resp.List) < limit { break } start += limit } return res, nil } func (d *BaiduNetdisk) linkOfficial(file model.Obj, _ model.LinkArgs) (*model.Link, error) { var resp DownloadResp params := map[string]string{ "method": "filemetas", "fsids": fmt.Sprintf("[%s]", file.GetID()), "dlink": "1", } _, err := d.get("/xpan/multimedia", params, &resp) if err != nil { return nil, err } u := fmt.Sprintf("%s&access_token=%s", resp.List[0].Dlink, d.AccessToken) res, err := base.NoRedirectClient.R().SetHeader("User-Agent", "pan.baidu.com").Head(u) if err != nil { return nil, err } // if res.StatusCode() == 302 { u = res.Header().Get("location") //} return &model.Link{ URL: u, Header: http.Header{ "User-Agent": []string{"pan.baidu.com"}, }, }, nil } func (d *BaiduNetdisk) linkCrack(file model.Obj, _ model.LinkArgs) (*model.Link, error) { var resp DownloadResp2 param := map[string]string{ "target": fmt.Sprintf("[\"%s\"]", file.GetPath()), "dlink": "1", "web": "5", "origin": "dlna", } _, err := d.request("https://pan.baidu.com/api/filemetas", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(param) }, &resp) if err != nil { return nil, err } return &model.Link{ URL: resp.Info[0].Dlink, Header: http.Header{ "User-Agent": []string{d.CustomCrackUA}, }, }, nil } func (d *BaiduNetdisk) linkCrackVideo(file model.Obj, _ model.LinkArgs) (*model.Link, error) { param := map[string]string{ "type": "VideoURL", "path": file.GetPath(), "fs_id": file.GetID(), "devuid": "0%1", "clienttype": "1", "channel": "android_15_25010PN30C_bd-netdisk_1523a", "nom3u8": "1", "dlink": "1", "media": "1", "origin": "dlna", } resp, err := d.request("https://pan.baidu.com/api/mediainfo", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(param) }, nil) if err != nil { return nil, err } return &model.Link{ URL: utils.Json.Get(resp, "info", "dlink").ToString(), Header: http.Header{ "User-Agent": []string{d.CustomCrackUA}, }, }, nil } func (d *BaiduNetdisk) manage(opera string, filelist any) ([]byte, error) { params := map[string]string{ "method": "filemanager", "opera": opera, } marshal, _ := utils.Json.MarshalToString(filelist) return d.postForm("/xpan/file", params, map[string]string{ "async": "0", "filelist": marshal, "ondup": "fail", }, nil) } func (d *BaiduNetdisk) create(path string, size int64, isdir int, uploadid, block_list string, resp any, mtime, ctime int64) ([]byte, error) { params := map[string]string{ "method": "create", } form := map[string]string{ "path": path, "size": strconv.FormatInt(size, 10), "isdir": strconv.Itoa(isdir), "rtype": "3", } if mtime != 0 && ctime != 0 { joinTime(form, ctime, mtime) } if uploadid != "" { form["uploadid"] = uploadid } if block_list != "" { form["block_list"] = block_list } return d.postForm("/xpan/file", params, form, resp) } func joinTime(form map[string]string, ctime, mtime int64) { form["local_mtime"] = strconv.FormatInt(mtime, 10) form["local_ctime"] = strconv.FormatInt(ctime, 10) } const ( DefaultSliceSize int64 = 4 * utils.MB VipSliceSize int64 = 16 * utils.MB SVipSliceSize int64 = 32 * utils.MB MaxSliceNum = 2048 // 文档写的是 1024/没写 ，但实际测试是 2048 SliceStep int64 = 1 * utils.MB ) func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { // 非会员固定为 4MB if d.vipType == 0 { if d.CustomUploadPartSize != 0 { log.Warnf("[baidu_netdisk] CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize") } if filesize > MaxSliceNum*DefaultSliceSize { log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize) } return DefaultSliceSize } if d.CustomUploadPartSize != 0 { if d.CustomUploadPartSize < DefaultSliceSize { log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize) return DefaultSliceSize } if d.vipType == 1 && d.CustomUploadPartSize > VipSliceSize { log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize) return VipSliceSize } if d.vipType == 2 && d.CustomUploadPartSize > SVipSliceSize { log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize) return SVipSliceSize } return d.CustomUploadPartSize } maxSliceSize := DefaultSliceSize switch d.vipType { case 1: maxSliceSize = VipSliceSize case 2: maxSliceSize = SVipSliceSize } // upload on low bandwidth if d.LowBandwithUploadMode { size := DefaultSliceSize for size <= maxSliceSize { if filesize <= MaxSliceNum*size { return size } size += SliceStep } } if filesize > MaxSliceNum*maxSliceSize { log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize) } return maxSliceSize } func (d *BaiduNetdisk) quota(ctx context.Context) (model.DiskUsage, error) { var resp QuotaResp _, err := d.request("https://pan.baidu.com/api/quota", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }, &resp) if err != nil { return model.DiskUsage{}, err } return model.DiskUsage{TotalSpace: resp.Total, UsedSpace: resp.Used}, nil } // getUploadUrl 从开放平台获取上传域名/地址，并发请求会被合并，结果会在 uploadid 生命周期内复用。 // 如果获取失败，则返回 Upload API设置项。 func (d *BaiduNetdisk) getUploadUrl(path, uploadId string) string { if !d.UseDynamicUploadAPI || uploadId == "" { return d.UploadAPI } uploadUrl, err := d.requestForUploadUrl(path, uploadId) if err != nil { return d.UploadAPI } return uploadUrl } // requestForUploadUrl 请求获取上传地址。 // 实测此接口不需要认证，传method和upload_version就行，不过还是按文档规范调用。 // https://pan.baidu.com/union/doc/Mlvw5hfnr func (d *BaiduNetdisk) requestForUploadUrl(path, uploadId string) (string, error) { params := map[string]string{ "method": "locateupload", "appid": "250528", "path": path, "uploadid": uploadId, "upload_version": "2.0", } apiUrl := "https://d.pcs.baidu.com/rest/2.0/pcs/file" var resp UploadServerResp _, err := d.request(apiUrl, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(params) }, &resp) if err != nil { return "", err } // 应该是https开头的一个地址 var uploadUrl string if len(resp.Servers) > 0 { uploadUrl = resp.Servers[0].Server } else if len(resp.BakServers) > 0 { uploadUrl = resp.BakServers[0].Server } if uploadUrl == "" { return "", errors.New("upload URL is empty") } return uploadUrl, nil } // func encodeURIComponent(str string) string { // r := url.QueryEscape(str) // r = strings.ReplaceAll(r, "+", "%20") // return r // } func DecryptMd5(encryptMd5 string) string { if _, err := hex.DecodeString(encryptMd5); err == nil { return encryptMd5 } var out strings.Builder out.Grow(len(encryptMd5)) for i, n := 0, int64(0); i < len(encryptMd5); i++ { if i == 9 { n = int64(unicode.ToLower(rune(encryptMd5[i])) - 'g') } else { n, _ = strconv.ParseInt(encryptMd5[i:i+1], 16, 64) } out.WriteString(strconv.FormatInt(n^int64(15&i), 16)) } encryptMd5 = out.String() return encryptMd5[8:16] + encryptMd5[:8] + encryptMd5[24:32] + encryptMd5[16:24] } func EncryptMd5(originalMd5 string) string { reversed := originalMd5[8:16] + originalMd5[:8] + originalMd5[24:32] + originalMd5[16:24] var out strings.Builder out.Grow(len(reversed)) for i, n := 0, int64(0); i < len(reversed); i++ { n, _ = strconv.ParseInt(reversed[i:i+1], 16, 64) n ^= int64(15 & i) if i == 9 { out.WriteRune(rune(n) + 'g') } else { out.WriteString(strconv.FormatInt(n, 16)) } } return out.String() } ================================================ FILE: drivers/baidu_photo/driver.go ================================================ package baiduphoto import ( "context" "crypto/md5" "encoding/hex" "errors" "fmt" "io" "os" "regexp" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" ) type BaiduPhoto struct { model.Storage Addition // AccessToken string Uk int64 bdstoken string root model.Obj uploadThread int } func (d *BaiduPhoto) Config() driver.Config { return config } func (d *BaiduPhoto) GetAddition() driver.Additional { return &d.Addition } func (d *BaiduPhoto) Init(ctx context.Context) error { d.uploadThread, _ = strconv.Atoi(d.UploadThread) if d.uploadThread < 1 || d.uploadThread > 32 { d.uploadThread, d.UploadThread = 3, "3" } // if err := d.refreshToken(); err != nil { // return err // } // root if d.AlbumID != "" { albumID := strings.Split(d.AlbumID, "|")[0] album, err := d.GetAlbumDetail(ctx, albumID) if err != nil { return err } d.root = album } else { d.root = &Root{ Name: "root", Modified: d.Modified, IsFolder: true, } } // uk info, err := d.uInfo() if err != nil { return err } d.bdstoken, err = d.getBDStoken() if err != nil { return err } d.Uk, err = strconv.ParseInt(info.YouaID, 10, 64) return err } func (d *BaiduPhoto) GetRoot(ctx context.Context) (model.Obj, error) { return d.root, nil } func (d *BaiduPhoto) Drop(ctx context.Context) error { // d.AccessToken = "" d.Uk = 0 d.root = nil return nil } func (d *BaiduPhoto) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var err error /* album */ if album, ok := dir.(*Album); ok { var files []AlbumFile files, err = d.GetAllAlbumFile(ctx, album, "") if err != nil { return nil, err } return utils.MustSliceConvert(files, func(file AlbumFile) model.Obj { return &file }), nil } /* root */ var albums []Album if d.ShowType != "root_only_file" { albums, err = d.GetAllAlbum(ctx) if err != nil { return nil, err } } var files []File if d.ShowType != "root_only_album" { files, err = d.GetAllFile(ctx) if err != nil { return nil, err } } return append( utils.MustSliceConvert(albums, func(album Album) model.Obj { return &album }), utils.MustSliceConvert(files, func(album File) model.Obj { return &album })..., ), nil } func (d *BaiduPhoto) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { switch file := file.(type) { case *File: return d.linkFile(ctx, file, args) case *AlbumFile: // 处理共享相册 if d.Uk != file.Uk { // 有概率无法获取到链接 // return d.linkAlbum(ctx, file, args) f, err := d.CopyAlbumFile(ctx, file) if err != nil { return nil, err } return d.linkFile(ctx, f, args) } return d.linkFile(ctx, &file.File, args) } return nil, errs.NotFile } var joinReg = regexp.MustCompile(`(?i)join:([\S]*)`) func (d *BaiduPhoto) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if _, ok := parentDir.(*Root); ok { code := joinReg.FindStringSubmatch(dirName) if len(code) > 1 { return d.JoinAlbum(ctx, code[1]) } return d.CreateAlbum(ctx, dirName) } return nil, errs.NotSupport } func (d *BaiduPhoto) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { switch file := srcObj.(type) { case *File: if album, ok := dstDir.(*Album); ok { //rootfile -> album return d.AddAlbumFile(ctx, album, file) } case *AlbumFile: switch album := dstDir.(type) { case *Root: //albumfile -> root return d.CopyAlbumFile(ctx, file) case *Album: // albumfile -> root -> album rootfile, err := d.CopyAlbumFile(ctx, file) if err != nil { return nil, err } return d.AddAlbumFile(ctx, album, rootfile) } } return nil, errs.NotSupport } func (d *BaiduPhoto) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if file, ok := srcObj.(*AlbumFile); ok { switch dstDir.(type) { case *Album, *Root: // albumfile -> root -> album or albumfile -> root newObj, err := d.Copy(ctx, srcObj, dstDir) if err != nil { return nil, err } // 删除原相册文件 _ = d.DeleteAlbumFile(ctx, file) return newObj, nil } } return nil, errs.NotSupport } func (d *BaiduPhoto) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { // 仅支持相册改名 if album, ok := srcObj.(*Album); ok { return d.SetAlbumName(ctx, album, newName) } return nil, errs.NotSupport } func (d *BaiduPhoto) Remove(ctx context.Context, obj model.Obj) error { switch obj := obj.(type) { case *File: return d.DeleteFile(ctx, obj) case *AlbumFile: return d.DeleteAlbumFile(ctx, obj) case *Album: return d.DeleteAlbum(ctx, obj) } return errs.NotSupport } func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { // 不支持大小为0的文件 if stream.GetSize() == 0 { return nil, fmt.Errorf("file size cannot be zero") } // TODO: // 暂时没有找到妙传方式 var ( cache = stream.GetFile() tmpF *os.File err error ) if _, ok := cache.(io.ReaderAt); !ok { tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") if err != nil { return nil, err } defer func() { _ = tmpF.Close() _ = os.Remove(tmpF.Name()) }() cache = tmpF } const DEFAULT int64 = 1 << 22 const SliceSize int64 = 1 << 18 // 计算需要的数据 streamSize := stream.GetSize() count := 1 if streamSize > DEFAULT { count = int((streamSize + DEFAULT - 1) / DEFAULT) } lastBlockSize := streamSize % DEFAULT if lastBlockSize == 0 { lastBlockSize = DEFAULT } // step.1 计算MD5 sliceMD5List := make([]string, 0, count) byteSize := int64(DEFAULT) fileMd5H := md5.New() sliceMd5H := md5.New() sliceMd5H2 := md5.New() slicemd5H2Write := utils.LimitWriter(sliceMd5H2, SliceSize) writers := []io.Writer{fileMd5H, sliceMd5H, slicemd5H2Write} if tmpF != nil { writers = append(writers, tmpF) } written := int64(0) for i := 1; i <= count; i++ { if utils.IsCanceled(ctx) { return nil, ctx.Err() } if i == count { byteSize = lastBlockSize } n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) written += n if err != nil && err != io.EOF { return nil, err } sliceMD5List = append(sliceMD5List, hex.EncodeToString(sliceMd5H.Sum(nil))) sliceMd5H.Reset() } if tmpF != nil { if written != streamSize { return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, streamSize) } _, err = tmpF.Seek(0, io.SeekStart) if err != nil { return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") } } contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) blockListStr, _ := utils.Json.MarshalToString(sliceMD5List) // step.2 预上传 params := map[string]string{ "autoinit": "1", "isdir": "0", "rtype": "1", "ctype": "11", "path": fmt.Sprintf("/%s", stream.GetName()), "size": fmt.Sprint(streamSize), "slice-md5": sliceMd5, "content-md5": contentMd5, "block_list": blockListStr, } // 尝试获取之前的进度 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, strconv.FormatInt(d.Uk, 10), contentMd5) if !ok { _, err = d.Post(FILE_API_URL_V1+"/precreate", func(r *resty.Request) { r.SetContext(ctx) r.SetFormData(params) r.SetQueryParam("bdstoken", d.bdstoken) }, &precreateResp) if err != nil { return nil, err } } switch precreateResp.ReturnType { case 1: //step.3 上传文件切片 threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) for i, partseq := range precreateResp.BlockList { if utils.IsCanceled(upCtx) { break } i, partseq, offset, byteSize := i, partseq, int64(partseq)*DEFAULT, DEFAULT if partseq+1 == count { byteSize = lastBlockSize } threadG.Go(func(ctx context.Context) error { uploadParams := map[string]string{ "method": "upload", "path": params["path"], "partseq": fmt.Sprint(partseq), "uploadid": precreateResp.UploadID, "app_id": "16051585", } _, err = d.Post("https://c3.pcs.baidu.com/rest/2.0/pcs/superfile2", func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(uploadParams) r.SetFileReader("file", stream.GetName(), driver.NewLimitedUploadStream(ctx, io.NewSectionReader(cache, offset, byteSize))) }, nil) if err != nil { return err } up(float64(threadG.Success()+1) * 100 / float64(len(precreateResp.BlockList)+1)) precreateResp.BlockList[i] = -1 return nil }) } if err = threadG.Wait(); err != nil { if errors.Is(err, context.Canceled) { precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) base.SaveUploadProgress(d, strconv.FormatInt(d.Uk, 10), contentMd5) } return nil, err } defer up(100) fallthrough case 2: //step.4 创建文件 params["uploadid"] = precreateResp.UploadID _, err = d.Post(FILE_API_URL_V1+"/create", func(r *resty.Request) { r.SetContext(ctx) r.SetFormData(params) r.SetQueryParam("bdstoken", d.bdstoken) }, &precreateResp) if err != nil { return nil, err } fallthrough case 3: //step.5 增加到相册 rootfile := precreateResp.Data.toFile() if album, ok := dstDir.(*Album); ok { return d.AddAlbumFile(ctx, album, rootfile) } return rootfile, nil } return nil, errs.NotSupport } var _ driver.Driver = (*BaiduPhoto)(nil) var _ driver.GetRooter = (*BaiduPhoto)(nil) var _ driver.MkdirResult = (*BaiduPhoto)(nil) var _ driver.CopyResult = (*BaiduPhoto)(nil) var _ driver.MoveResult = (*BaiduPhoto)(nil) var _ driver.Remove = (*BaiduPhoto)(nil) var _ driver.PutResult = (*BaiduPhoto)(nil) var _ driver.RenameResult = (*BaiduPhoto)(nil) ================================================ FILE: drivers/baidu_photo/help.go ================================================ package baiduphoto import ( "fmt" "math" "math/rand" "strings" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) // Tid生成 func getTid() string { return fmt.Sprintf("3%d%.0f", time.Now().Unix(), math.Floor(9000000*rand.Float64()+1000000)) } func toTime(t int64) *time.Time { tm := time.Unix(t, 0) return &tm } func fsidsFormatNotUk(ids ...int64) string { buf := utils.MustSliceConvert(ids, func(id int64) string { return fmt.Sprintf(`{"fsid":%d}`, id) }) return fmt.Sprintf("[%s]", strings.Join(buf, ",")) } func getFileName(path string) string { return path[strings.LastIndex(path, "/")+1:] } func MustString(str string, err error) string { return str } /* * 处理文件变化 * 最大程度利用重复数据 **/ func copyFile(file *AlbumFile, cf *CopyFile) *File { return &File{ Fsid: cf.Fsid, Path: cf.Path, Ctime: cf.Ctime, Mtime: cf.Ctime, Size: file.Size, Thumburl: file.Thumburl, } } func moveFileToAlbumFile(file *File, album *Album, uk int64) *AlbumFile { return &AlbumFile{ File: *file, AlbumID: album.AlbumID, Tid: album.Tid, Uk: uk, } } func renameAlbum(album *Album, newName string) *Album { return &Album{ AlbumID: album.AlbumID, Tid: album.Tid, JoinTime: album.JoinTime, CreationTime: album.CreationTime, Title: newName, Mtime: time.Now().Unix(), } } func BoolToIntStr(b bool) string { if b { return "1" } return "0" } ================================================ FILE: drivers/baidu_photo/meta.go ================================================ package baiduphoto import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { // RefreshToken string `json:"refresh_token" required:"true"` Cookie string `json:"cookie" required:"true"` ShowType string `json:"show_type" type:"select" options:"root,root_only_album,root_only_file" default:"root"` AlbumID string `json:"album_id"` //AlbumPassword string `json:"album_password"` DeleteOrigin bool `json:"delete_origin"` // ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"` // ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"` UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` } var config = driver.Config{ Name: "BaiduPhoto", LocalSort: true, LinkCacheMode: driver.LinkCacheUA, } func init() { op.RegisterDriver(func() driver.Driver { return &BaiduPhoto{} }) } ================================================ FILE: drivers/baidu_photo/types.go ================================================ package baiduphoto import ( "fmt" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type TokenErrResp struct { ErrorDescription string `json:"error_description"` ErrorMsg string `json:"error"` } func (e *TokenErrResp) Error() string { return fmt.Sprint(e.ErrorMsg, " : ", e.ErrorDescription) } type Erron struct { Errno int `json:"errno"` RequestID int `json:"request_id"` } // 用户信息 type UInfo struct { // uk YouaID string `json:"youa_id"` } type Page struct { HasMore int `json:"has_more"` Cursor string `json:"cursor"` } func (p Page) HasNextPage() bool { return p.HasMore == 1 } type Root = model.Object type ( FileListResp struct { Page List []File `json:"list"` } File struct { Fsid int64 `json:"fsid"` // 文件ID Path string `json:"path"` // 文件路径 Size int64 `json:"size"` Ctime int64 `json:"ctime"` // 创建时间 s Mtime int64 `json:"mtime"` // 修改时间 s Thumburl []string `json:"thumburl"` Md5 string `json:"md5"` } ) func (c *File) GetSize() int64 { return c.Size } func (c *File) GetName() string { return getFileName(c.Path) } func (c *File) CreateTime() time.Time { return time.Unix(c.Ctime, 0) } func (c *File) ModTime() time.Time { return time.Unix(c.Mtime, 0) } func (c *File) IsDir() bool { return false } func (c *File) GetID() string { return "" } func (c *File) GetPath() string { return "" } func (c *File) Thumb() string { if len(c.Thumburl) > 0 { return c.Thumburl[0] } return "" } func (c *File) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.MD5, DecryptMd5(c.Md5)) } /*相册部分*/ type ( AlbumListResp struct { Page List []Album `json:"list"` Reset int64 `json:"reset"` TotalCount int64 `json:"total_count"` } Album struct { AlbumID string `json:"album_id"` Tid int64 `json:"tid"` Title string `json:"title"` JoinTime int64 `json:"join_time"` CreationTime int64 `json:"create_time"` Mtime int64 `json:"mtime"` parseTime *time.Time } AlbumFileListResp struct { Page List []AlbumFile `json:"list"` Reset int64 `json:"reset"` TotalCount int64 `json:"total_count"` } AlbumFile struct { File AlbumID string `json:"album_id"` Tid int64 `json:"tid"` Uk int64 `json:"uk"` } ) func (a *Album) GetHash() utils.HashInfo { return utils.HashInfo{} } func (a *Album) GetSize() int64 { return 0 } func (a *Album) GetName() string { return a.Title } func (a *Album) CreateTime() time.Time { return time.Unix(a.CreationTime, 0) } func (a *Album) ModTime() time.Time { return time.Unix(a.Mtime, 0) } func (a *Album) IsDir() bool { return true } func (a *Album) GetID() string { return "" } func (a *Album) GetPath() string { return "" } type ( CopyFileResp struct { List []CopyFile `json:"list"` } CopyFile struct { FromFsid int64 `json:"from_fsid"` // 源ID Ctime int64 `json:"ctime"` Fsid int64 `json:"fsid"` // 目标ID Path string `json:"path"` ShootTime int `json:"shoot_time"` } ) /*上传部分*/ type ( UploadFile struct { FsID int64 `json:"fs_id"` Size int64 `json:"size"` Md5 string `json:"md5"` ServerFilename string `json:"server_filename"` Path string `json:"path"` Ctime int64 `json:"ctime"` Mtime int64 `json:"mtime"` Isdir int `json:"isdir"` Category int `json:"category"` ServerMd5 string `json:"server_md5"` ShootTime int `json:"shoot_time"` } CreateFileResp struct { Data UploadFile `json:"data"` } PrecreateResp struct { ReturnType int `json:"return_type"` //存在返回2 不存在返回1 已经保存3 //存在返回 CreateFileResp //不存在返回 Path string `json:"path"` UploadID string `json:"uploadid"` BlockList []int `json:"block_list"` } ) func (f *UploadFile) toFile() *File { return &File{ Fsid: f.FsID, Path: f.Path, Size: f.Size, Ctime: f.Ctime, Mtime: f.Mtime, Thumburl: nil, } } /* 共享相册部分 */ type InviteResp struct { Pdata struct { // 邀请码 InviteCode string `json:"invite_code"` // 有效时间 ExpireTime int `json:"expire_time"` ShareID string `json:"share_id"` } `json:"pdata"` } /* 加入相册部分 */ type JoinOrCreateAlbumResp struct { AlbumID string `json:"album_id"` AlreadyExists int `json:"already_exists"` } ================================================ FILE: drivers/baidu_photo/utils.go ================================================ package baiduphoto import ( "context" "encoding/hex" "fmt" "net/http" "strconv" "strings" "unicode" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) const ( API_URL = "https://photo.baidu.com/youai" USER_API_URL = API_URL + "/user/v1" ALBUM_API_URL = API_URL + "/album/v1" FILE_API_URL_V1 = API_URL + "/file/v1" FILE_API_URL_V2 = API_URL + "/file/v2" ) func (d *BaiduPhoto) Request(client *resty.Client, furl string, method string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { req := client.R(). // SetQueryParam("access_token", d.AccessToken) SetHeader("Cookie", d.Cookie) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } res, err := req.Execute(method, furl) if err != nil { return nil, err } erron := utils.Json.Get(res.Body(), "errno").ToInt() switch erron { case 0: break case 50805: return nil, fmt.Errorf("you have joined album") case 50820: return nil, fmt.Errorf("no shared albums found") case 50100: return nil, fmt.Errorf("illegal title, only supports 50 characters") // case -6: // if err = d.refreshToken(); err != nil { // return nil, err // } default: return nil, fmt.Errorf("errno: %d, refer to https://photo.baidu.com/union/doc", erron) } return res, nil } //func (d *BaiduPhoto) Request(furl string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { // res, err := d.request(furl, method, callback, resp) // if err != nil { // return nil, err // } // return res.Body(), nil //} // func (d *BaiduPhoto) refreshToken() error { // u := "https://openapi.baidu.com/oauth/2.0/token" // var resp base.TokenResp // var e TokenErrResp // _, err := base.RestyClient.R().SetResult(&resp).SetError(&e).SetQueryParams(map[string]string{ // "grant_type": "refresh_token", // "refresh_token": d.RefreshToken, // "client_id": d.ClientID, // "client_secret": d.ClientSecret, // }).Get(u) // if err != nil { // return err // } // if e.ErrorMsg != "" { // return &e // } // if resp.RefreshToken == "" { // return errs.EmptyToken // } // d.AccessToken, d.RefreshToken = resp.AccessToken, resp.RefreshToken // op.MustSaveDriverStorage(d) // return nil // } func (d *BaiduPhoto) Get(furl string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { return d.Request(base.RestyClient, furl, http.MethodGet, callback, resp) } func (d *BaiduPhoto) Post(furl string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { return d.Request(base.RestyClient, furl, http.MethodPost, callback, resp) } // 获取所有文件 func (d *BaiduPhoto) GetAllFile(ctx context.Context) (files []File, err error) { var cursor string for { var resp FileListResp _, err = d.Get(FILE_API_URL_V1+"/list", func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(map[string]string{ "need_thumbnail": "1", "need_filter_hidden": "0", "cursor": cursor, }) }, &resp) if err != nil { return } files = append(files, resp.List...) if !resp.HasNextPage() { return } cursor = resp.Cursor } } // 删除根文件 func (d *BaiduPhoto) DeleteFile(ctx context.Context, file *File) error { _, err := d.Get(FILE_API_URL_V1+"/delete", func(req *resty.Request) { req.SetContext(ctx) req.SetQueryParams(map[string]string{ "fsid_list": fmt.Sprintf("[%d]", file.Fsid), }) }, nil) return err } // 获取所有相册 func (d *BaiduPhoto) GetAllAlbum(ctx context.Context) (albums []Album, err error) { var cursor string for { var resp AlbumListResp _, err = d.Get(ALBUM_API_URL+"/list", func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(map[string]string{ "need_amount": "1", "limit": "100", "cursor": cursor, }) }, &resp) if err != nil { return } if albums == nil { albums = make([]Album, 0, resp.TotalCount) } cursor = resp.Cursor albums = append(albums, resp.List...) if !resp.HasNextPage() { return } } } // 获取相册中所有文件 func (d *BaiduPhoto) GetAllAlbumFile(ctx context.Context, album *Album, passwd string) (files []AlbumFile, err error) { var cursor string for { var resp AlbumFileListResp _, err = d.Get(ALBUM_API_URL+"/listfile", func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(map[string]string{ "album_id": album.AlbumID, "need_amount": "1", "limit": "1000", "passwd": passwd, "cursor": cursor, }) }, &resp) if err != nil { return } if files == nil { files = make([]AlbumFile, 0, resp.TotalCount) } cursor = resp.Cursor files = append(files, resp.List...) if !resp.HasNextPage() { return } } } // 创建相册 func (d *BaiduPhoto) CreateAlbum(ctx context.Context, name string) (*Album, error) { var resp JoinOrCreateAlbumResp _, err := d.Post(ALBUM_API_URL+"/create", func(r *resty.Request) { r.SetContext(ctx).SetResult(&resp) r.SetQueryParams(map[string]string{ "title": name, "tid": getTid(), "source": "0", }) }, nil) if err != nil { return nil, err } return d.GetAlbumDetail(ctx, resp.AlbumID) } // 相册改名 func (d *BaiduPhoto) SetAlbumName(ctx context.Context, album *Album, name string) (*Album, error) { _, err := d.Post(ALBUM_API_URL+"/settitle", func(r *resty.Request) { r.SetContext(ctx) r.SetFormData(map[string]string{ "title": name, "album_id": album.AlbumID, "tid": fmt.Sprint(album.Tid), }) }, nil) if err != nil { return nil, err } return renameAlbum(album, name), nil } // 删除相册 func (d *BaiduPhoto) DeleteAlbum(ctx context.Context, album *Album) error { _, err := d.Post(ALBUM_API_URL+"/delete", func(r *resty.Request) { r.SetContext(ctx) r.SetFormData(map[string]string{ "album_id": album.AlbumID, "tid": fmt.Sprint(album.Tid), "delete_origin_image": BoolToIntStr(d.DeleteOrigin), // 是否删除原图 0 不删除 1 删除 }) }, nil) return err } // 删除相册文件 func (d *BaiduPhoto) DeleteAlbumFile(ctx context.Context, file *AlbumFile) error { _, err := d.Post(ALBUM_API_URL+"/delfile", func(r *resty.Request) { r.SetContext(ctx) r.SetFormData(map[string]string{ "album_id": fmt.Sprint(file.AlbumID), "tid": fmt.Sprint(file.Tid), "list": fmt.Sprintf(`[{"fsid":%d,"uk":%d}]`, file.Fsid, file.Uk), "del_origin": BoolToIntStr(d.DeleteOrigin), // 是否删除原图 0 不删除 1 删除 }) }, nil) return err } // 增加相册文件 func (d *BaiduPhoto) AddAlbumFile(ctx context.Context, album *Album, file *File) (*AlbumFile, error) { _, err := d.Get(ALBUM_API_URL+"/addfile", func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(map[string]string{ "album_id": fmt.Sprint(album.AlbumID), "tid": fmt.Sprint(album.Tid), "list": fsidsFormatNotUk(file.Fsid), }) }, nil) if err != nil { return nil, err } return moveFileToAlbumFile(file, album, d.Uk), nil } // 保存相册文件为根文件 func (d *BaiduPhoto) CopyAlbumFile(ctx context.Context, file *AlbumFile) (*File, error) { var resp CopyFileResp _, err := d.Post(ALBUM_API_URL+"/copyfile", func(r *resty.Request) { r.SetContext(ctx) r.SetFormData(map[string]string{ "album_id": file.AlbumID, "tid": fmt.Sprint(file.Tid), "uk": fmt.Sprint(file.Uk), "list": fsidsFormatNotUk(file.Fsid), }) r.SetResult(&resp) }, nil) if err != nil { return nil, err } return copyFile(file, &resp.List[0]), nil } // 加入相册 func (d *BaiduPhoto) JoinAlbum(ctx context.Context, code string) (*Album, error) { var resp InviteResp _, err := d.Get(ALBUM_API_URL+"/querypcode", func(req *resty.Request) { req.SetContext(ctx) req.SetQueryParams(map[string]string{ "pcode": code, "web": "1", }) }, &resp) if err != nil { return nil, err } var resp2 JoinOrCreateAlbumResp _, err = d.Get(ALBUM_API_URL+"/join", func(req *resty.Request) { req.SetContext(ctx) req.SetQueryParams(map[string]string{ "invite_code": resp.Pdata.InviteCode, }) }, &resp2) if err != nil { return nil, err } return d.GetAlbumDetail(ctx, resp2.AlbumID) } // 获取相册详细信息 func (d *BaiduPhoto) GetAlbumDetail(ctx context.Context, albumID string) (*Album, error) { var album Album _, err := d.Get(ALBUM_API_URL+"/detail", func(req *resty.Request) { req.SetContext(ctx).SetResult(&album) req.SetQueryParams(map[string]string{ "album_id": albumID, }) }, &album) if err != nil { return nil, err } return &album, nil } func (d *BaiduPhoto) linkAlbum(ctx context.Context, file *AlbumFile, args model.LinkArgs) (*model.Link, error) { headers := map[string]string{ "User-Agent": base.UserAgent, } if args.Header.Get("User-Agent") != "" { headers["User-Agent"] = args.Header.Get("User-Agent") } if !utils.IsLocalIPAddr(args.IP) { headers["X-Forwarded-For"] = args.IP } resp, err := d.Request(base.NoRedirectClient, ALBUM_API_URL+"/download", http.MethodHead, func(r *resty.Request) { r.SetContext(ctx) r.SetHeaders(headers) r.SetQueryParams(map[string]string{ "fsid": fmt.Sprint(file.Fsid), "album_id": file.AlbumID, "tid": fmt.Sprint(file.Tid), "uk": fmt.Sprint(file.Uk), }) }, nil) if err != nil { return nil, err } if resp.StatusCode() != 302 { return nil, fmt.Errorf("not found 302 redirect") } location := resp.Header().Get("Location") link := &model.Link{ URL: location, Header: http.Header{ "User-Agent": []string{headers["User-Agent"]}, "Referer": []string{"https://photo.baidu.com/"}, }, } return link, nil } func (d *BaiduPhoto) linkFile(ctx context.Context, file *File, args model.LinkArgs) (*model.Link, error) { headers := map[string]string{ "User-Agent": base.UserAgent, } if args.Header.Get("User-Agent") != "" { headers["User-Agent"] = args.Header.Get("User-Agent") } if !utils.IsLocalIPAddr(args.IP) { headers["X-Forwarded-For"] = args.IP } var downloadUrl struct { Dlink string `json:"dlink"` } _, err := d.Get(FILE_API_URL_V2+"/download", func(r *resty.Request) { r.SetContext(ctx) r.SetHeaders(headers) r.SetQueryParams(map[string]string{ "fsid": fmt.Sprint(file.Fsid), }) }, &downloadUrl) // resp, err := d.Request(base.NoRedirectClient, FILE_API_URL_V1+"/download", http.MethodHead, func(r *resty.Request) { // r.SetContext(ctx) // r.SetHeaders(headers) // r.SetQueryParams(map[string]string{ // "fsid": fmt.Sprint(file.Fsid), // }) // }, nil) if err != nil { return nil, err } // if resp.StatusCode() != 302 { // return nil, fmt.Errorf("not found 302 redirect") // } // location := resp.Header().Get("Location") link := &model.Link{ URL: downloadUrl.Dlink, Header: http.Header{ "User-Agent": []string{headers["User-Agent"]}, "Referer": []string{"https://photo.baidu.com/"}, }, } return link, nil } /*func (d *BaiduPhoto) linkStreamAlbum(ctx context.Context, file *AlbumFile) (*model.Link, error) { return &model.Link{ Header: http.Header{}, Writer: func(w io.Writer) error { res, err := d.Get(ALBUM_API_URL+"/streaming", func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(map[string]string{ "fsid": fmt.Sprint(file.Fsid), "album_id": file.AlbumID, "tid": fmt.Sprint(file.Tid), "uk": fmt.Sprint(file.Uk), }).SetDoNotParseResponse(true) }, nil) if err != nil { return err } defer res.RawBody().Close() _, err = io.Copy(w, res.RawBody()) return err }, }, nil }*/ /*func (d *BaiduPhoto) linkStream(ctx context.Context, file *File) (*model.Link, error) { return &model.Link{ Header: http.Header{}, Writer: func(w io.Writer) error { res, err := d.Get(FILE_API_URL_V1+"/streaming", func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(map[string]string{ "fsid": fmt.Sprint(file.Fsid), }).SetDoNotParseResponse(true) }, nil) if err != nil { return err } defer res.RawBody().Close() _, err = io.Copy(w, res.RawBody()) return err }, }, nil }*/ // 获取uk func (d *BaiduPhoto) uInfo() (*UInfo, error) { var info UInfo _, err := d.Get(USER_API_URL+"/getuinfo", func(req *resty.Request) { }, &info) if err != nil { return nil, err } return &info, nil } func (d *BaiduPhoto) getBDStoken() (string, error) { var info struct { Result struct { Bdstoken string `json:"bdstoken"` Token string `json:"token"` Uk int64 `json:"uk"` } `json:"result"` } _, err := d.Get("https://pan.baidu.com/api/gettemplatevariable?fields=[%22bdstoken%22,%22token%22,%22uk%22]", nil, &info) if err != nil { return "", err } return info.Result.Bdstoken, nil } func DecryptMd5(encryptMd5 string) string { if _, err := hex.DecodeString(encryptMd5); err == nil { return encryptMd5 } var out strings.Builder out.Grow(len(encryptMd5)) for i, n := 0, int64(0); i < len(encryptMd5); i++ { if i == 9 { n = int64(unicode.ToLower(rune(encryptMd5[i])) - 'g') } else { n, _ = strconv.ParseInt(encryptMd5[i:i+1], 16, 64) } out.WriteString(strconv.FormatInt(n^int64(15&i), 16)) } encryptMd5 = out.String() return encryptMd5[8:16] + encryptMd5[:8] + encryptMd5[24:32] + encryptMd5[16:24] } func EncryptMd5(originalMd5 string) string { reversed := originalMd5[8:16] + originalMd5[:8] + originalMd5[24:32] + originalMd5[16:24] var out strings.Builder out.Grow(len(reversed)) for i, n := 0, int64(0); i < len(reversed); i++ { n, _ = strconv.ParseInt(reversed[i:i+1], 16, 64) n ^= int64(15 & i) if i == 9 { out.WriteRune(rune(n) + 'g') } else { out.WriteString(strconv.FormatInt(n, 16)) } } return out.String() } ================================================ FILE: drivers/base/client.go ================================================ package base import ( "crypto/tls" "net/http" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/go-resty/resty/v2" ) var ( NoRedirectClient *resty.Client RestyClient *resty.Client HttpClient *http.Client ) var DefaultTimeout = time.Second * 30 const UserAgent = "Mozilla/5.0 (Macintosh; Apple macOS 26_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/142.0.0.0 OpenList/425.6.30" const UserAgentNT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/142.0.0.0 OpenList/425.6.30" func InitClient() { NoRedirectClient = resty.New().SetRedirectPolicy( resty.RedirectPolicyFunc(func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }), ).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) NoRedirectClient.SetHeader("user-agent", UserAgent) net.SetRestyProxyIfConfigured(NoRedirectClient) RestyClient = NewRestyClient() HttpClient = net.NewHttpClient() } func NewRestyClient() *resty.Client { client := resty.New(). SetHeader("user-agent", UserAgent). SetRetryCount(3). SetRetryResetReaders(true). SetTimeout(DefaultTimeout). SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) net.SetRestyProxyIfConfigured(client) return client } ================================================ FILE: drivers/base/types.go ================================================ package base import "github.com/go-resty/resty/v2" type Json map[string]interface{} type TokenResp struct { AccessToken string `json:"access_token"` RefreshToken string `json:"refresh_token"` } type ReqCallback func(req *resty.Request) ================================================ FILE: drivers/base/upload.go ================================================ package base import ( "fmt" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/go-cache" ) // storage upload progress, for upload recovery var UploadStateCache = cache.NewMemCache(cache.WithShards[any](32)) // Save upload progress for 20 minutes func SaveUploadProgress(driver driver.Driver, state any, keys ...string) bool { return UploadStateCache.Set( fmt.Sprint(driver.Config().Name, "-upload-", strings.Join(keys, "-")), state, cache.WithEx[any](time.Minute*20)) } // An upload progress can only be made by one process alone, // so here you need to get it and then delete it. func GetUploadProgress[T any](driver driver.Driver, keys ...string) (state T, ok bool) { v, ok := UploadStateCache.GetDel(fmt.Sprint(driver.Config().Name, "-upload-", strings.Join(keys, "-"))) if ok { state, ok = v.(T) } return } ================================================ FILE: drivers/base/util.go ================================================ package base ================================================ FILE: drivers/chaoxing/driver.go ================================================ package chaoxing import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "mime/multipart" "net/http" "net/url" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" "google.golang.org/appengine/log" ) type ChaoXing struct { model.Storage Addition cron *cron.Cron config driver.Config conf Conf } func (d *ChaoXing) Config() driver.Config { return d.config } func (d *ChaoXing) GetAddition() driver.Additional { return &d.Addition } func (d *ChaoXing) refreshCookie() error { cookie, err := d.Login() if err != nil { d.Status = err.Error() op.MustSaveDriverStorage(d) return nil } d.Addition.Cookie = cookie op.MustSaveDriverStorage(d) return nil } func (d *ChaoXing) Init(ctx context.Context) error { err := d.refreshCookie() if err != nil { log.Errorf(ctx, err.Error()) } d.cron = cron.NewCron(time.Hour * 12) d.cron.Do(func() { err = d.refreshCookie() if err != nil { log.Errorf(ctx, err.Error()) } }) return nil } func (d *ChaoXing) Drop(ctx context.Context) error { if d.cron != nil { d.cron.Stop() } return nil } func (d *ChaoXing) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.GetFiles(dir.GetID()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return fileToObj(src), nil }) } func (d *ChaoXing) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp DownResp ua := d.conf.ua fileId := strings.Split(file.GetID(), "$")[1] _, err := d.requestDownload("/screen/note_note/files/status/"+fileId, http.MethodPost, func(req *resty.Request) { req.SetHeader("User-Agent", ua) }, &resp) if err != nil { return nil, err } u := resp.Download return &model.Link{ URL: u, Header: http.Header{ "Cookie": []string{d.Cookie}, "Referer": []string{d.conf.referer}, "User-Agent": []string{ua}, }, Concurrency: 2, PartSize: 10 * utils.MB, }, nil } func (d *ChaoXing) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { query := map[string]string{ "bbsid": d.Addition.Bbsid, "name": dirName, "pid": parentDir.GetID(), } var resp ListFileResp _, err := d.request("/pc/resource/addResourceFolder", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return err } if resp.Result != 1 { msg := fmt.Sprintf("error:%s", resp.Msg) return errors.New(msg) } return nil } func (d *ChaoXing) Move(ctx context.Context, srcObj, dstDir model.Obj) error { query := map[string]string{ "bbsid": d.Addition.Bbsid, "folderIds": srcObj.GetID(), "targetId": dstDir.GetID(), } if !srcObj.IsDir() { query = map[string]string{ "bbsid": d.Addition.Bbsid, "recIds": strings.Split(srcObj.GetID(), "$")[0], "targetId": dstDir.GetID(), } } var resp ListFileResp _, err := d.request("/pc/resource/moveResource", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return err } if !resp.Status { msg := fmt.Sprintf("error:%s", resp.Msg) return errors.New(msg) } return nil } func (d *ChaoXing) Rename(ctx context.Context, srcObj model.Obj, newName string) error { query := map[string]string{ "bbsid": d.Addition.Bbsid, "folderId": srcObj.GetID(), "name": newName, } path := "/pc/resource/updateResourceFolderName" if !srcObj.IsDir() { // path = "/pc/resource/updateResourceFileName" // query = map[string]string{ // "bbsid": d.Addition.Bbsid, // "recIds": strings.Split(srcObj.GetID(), "$")[0], // "name": newName, // } return errors.New("此网盘不支持修改文件名") } var resp ListFileResp _, err := d.request(path, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return err } if resp.Result != 1 { msg := fmt.Sprintf("error:%s", resp.Msg) return errors.New(msg) } return nil } func (d *ChaoXing) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { // TODO copy obj, optional return errs.NotImplement } func (d *ChaoXing) Remove(ctx context.Context, obj model.Obj) error { query := map[string]string{ "bbsid": d.Addition.Bbsid, "folderIds": obj.GetID(), } path := "/pc/resource/deleteResourceFolder" var resp ListFileResp if !obj.IsDir() { path = "/pc/resource/deleteResourceFile" query = map[string]string{ "bbsid": d.Addition.Bbsid, "recIds": strings.Split(obj.GetID(), "$")[0], } } _, err := d.request(path, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return err } if resp.Result != 1 { msg := fmt.Sprintf("error:%s", resp.Msg) return errors.New(msg) } return nil } func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { var resp UploadDataRsp _, err := d.request("https://noteyd.chaoxing.com/pc/files/getUploadConfig", http.MethodGet, func(req *resty.Request) { }, &resp) if err != nil { return err } if resp.Result != 1 { return errors.New("get upload data error") } body := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) writer := multipart.NewWriter(body) _, err = writer.CreateFormFile("file", file.GetName()) if err != nil { return err } headSize := body.Len() err = writer.WriteField("_token", resp.Msg.Token) if err != nil { return err } err = writer.WriteField("puid", strconv.Itoa(resp.Msg.Puid)) if err != nil { fmt.Println("Error writing param2 to request body:", err) return err } err = writer.Close() if err != nil { return err } head := bytes.NewReader(body.Bytes()[:headSize]) tail := bytes.NewReader(body.Bytes()[headSize:]) r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: &driver.SimpleReaderWithSize{ Reader: io.MultiReader(head, file, tail), Size: int64(body.Len()) + file.GetSize(), }, UpdateProgress: up, }) req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://pan-yz.chaoxing.com/upload", r) if err != nil { return err } req.Header.Set("Content-Type", writer.FormDataContentType()) req.ContentLength = int64(body.Len()) + file.GetSize() resps, err := http.DefaultClient.Do(req) if err != nil { return err } defer resps.Body.Close() body.Reset() _, err = body.ReadFrom(resps.Body) if err != nil { return err } var fileRsp UploadFileDataRsp err = json.Unmarshal(body.Bytes(), &fileRsp) if err != nil { return err } if fileRsp.Msg != "success" { return errors.New(fileRsp.Msg) } uploadDoneParam := UploadDoneParam{Key: fileRsp.ObjectID, Cataid: "100000019", Param: fileRsp.Data} params, err := json.Marshal(uploadDoneParam) if err != nil { return err } query := map[string]string{ "bbsid": d.Addition.Bbsid, "pid": dstDir.GetID(), "type": "yunpan", "params": url.QueryEscape("[" + string(params) + "]"), } var respd ListFileResp _, err = d.request("/pc/resource/addResource", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &respd) if err != nil { return err } if respd.Result != 1 { msg := fmt.Sprintf("error:%v", resp.Msg) return errors.New(msg) } return nil } var _ driver.Driver = (*ChaoXing)(nil) ================================================ FILE: drivers/chaoxing/meta.go ================================================ package chaoxing import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) // 此程序挂载的是超星小组网盘，需要代理才能使用； // 登录超星后进入个人空间，进入小组，新建小组，点击进去。 // url中就有bbsid的参数，系统限制单文件大小2G，没有总容量限制 type Addition struct { // 超星用户名及密码 UserName string `json:"user_name" required:"true"` Password string `json:"password" required:"true"` // 从自己新建的小组url里获取 Bbsid string `json:"bbsid" required:"true"` driver.RootID // 可不填，程序会自动登录获取 Cookie string `json:"cookie"` } type Conf struct { ua string referer string api string DowloadApi string } func init() { op.RegisterDriver(func() driver.Driver { return &ChaoXing{ config: driver.Config{ Name: "ChaoXingGroupDrive", OnlyProxy: true, DefaultRoot: "-1", NoOverwriteUpload: true, }, conf: Conf{ ua: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch", referer: "https://chaoxing.com/", api: "https://groupweb.chaoxing.com", DowloadApi: "https://noteyd.chaoxing.com", }, } }) } ================================================ FILE: drivers/chaoxing/types.go ================================================ package chaoxing import ( "bytes" "fmt" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type Resp struct { Result int `json:"result"` } type UserAuth struct { GroupAuth struct { AddData int `json:"addData"` AddDataFolder int `json:"addDataFolder"` AddLebel int `json:"addLebel"` AddManager int `json:"addManager"` AddMem int `json:"addMem"` AddTopicFolder int `json:"addTopicFolder"` AnonymousAddReply int `json:"anonymousAddReply"` AnonymousAddTopic int `json:"anonymousAddTopic"` BatchOperation int `json:"batchOperation"` DelData int `json:"delData"` DelDataFolder int `json:"delDataFolder"` DelMem int `json:"delMem"` DelTopicFolder int `json:"delTopicFolder"` Dismiss int `json:"dismiss"` ExamEnc string `json:"examEnc"` GroupChat int `json:"groupChat"` IsShowCircleChatButton int `json:"isShowCircleChatButton"` IsShowCircleCloudButton int `json:"isShowCircleCloudButton"` IsShowCompanyButton int `json:"isShowCompanyButton"` Join int `json:"join"` MemberShowRankSet int `json:"memberShowRankSet"` ModifyDataFolder int `json:"modifyDataFolder"` ModifyExpose int `json:"modifyExpose"` ModifyName int `json:"modifyName"` ModifyShowPic int `json:"modifyShowPic"` ModifyTopicFolder int `json:"modifyTopicFolder"` ModifyVisibleState int `json:"modifyVisibleState"` OnlyMgrScoreSet int `json:"onlyMgrScoreSet"` Quit int `json:"quit"` SendNotice int `json:"sendNotice"` ShowActivityManage int `json:"showActivityManage"` ShowActivitySet int `json:"showActivitySet"` ShowAttentionSet int `json:"showAttentionSet"` ShowAutoClearStatus int `json:"showAutoClearStatus"` ShowBarcode int `json:"showBarcode"` ShowChatRoomSet int `json:"showChatRoomSet"` ShowCircleActivitySet int `json:"showCircleActivitySet"` ShowCircleSet int `json:"showCircleSet"` ShowCmem int `json:"showCmem"` ShowDataFolder int `json:"showDataFolder"` ShowDelReason int `json:"showDelReason"` ShowForward int `json:"showForward"` ShowGroupChat int `json:"showGroupChat"` ShowGroupChatSet int `json:"showGroupChatSet"` ShowGroupSquareSet int `json:"showGroupSquareSet"` ShowLockAddSet int `json:"showLockAddSet"` ShowManager int `json:"showManager"` ShowManagerIdentitySet int `json:"showManagerIdentitySet"` ShowNeedDelReasonSet int `json:"showNeedDelReasonSet"` ShowNotice int `json:"showNotice"` ShowOnlyManagerReplySet int `json:"showOnlyManagerReplySet"` ShowRank int `json:"showRank"` ShowRank2 int `json:"showRank2"` ShowRecycleBin int `json:"showRecycleBin"` ShowReplyByClass int `json:"showReplyByClass"` ShowReplyNeedCheck int `json:"showReplyNeedCheck"` ShowSignbanSet int `json:"showSignbanSet"` ShowSpeechSet int `json:"showSpeechSet"` ShowTopicCheck int `json:"showTopicCheck"` ShowTopicNeedCheck int `json:"showTopicNeedCheck"` ShowTransferSet int `json:"showTransferSet"` } `json:"groupAuth"` OperationAuth struct { Add int `json:"add"` AddTopicToFolder int `json:"addTopicToFolder"` ChoiceSet int `json:"choiceSet"` DelTopicFromFolder int `json:"delTopicFromFolder"` Delete int `json:"delete"` Reply int `json:"reply"` ScoreSet int `json:"scoreSet"` TopSet int `json:"topSet"` Update int `json:"update"` } `json:"operationAuth"` } // 手机端学习通上传的文件的json内容(content字段)与网页端上传的有所不同 // 网页端json `"puid": 54321, "size": 12345` // 手机端json `"puid": "54321". "size": "12345"` type int_str int // json 字符串数字和纯数字解析 func (ios *int_str) UnmarshalJSON(data []byte) error { intValue, err := strconv.Atoi(string(bytes.Trim(data, "\""))) if err != nil { return err } *ios = int_str(intValue) return nil } type File struct { Cataid int `json:"cataid"` Cfid int `json:"cfid"` Content struct { Cfid int `json:"cfid"` Pid int `json:"pid"` FolderName string `json:"folderName"` ShareType int `json:"shareType"` Preview string `json:"preview"` Filetype string `json:"filetype"` PreviewURL string `json:"previewUrl"` IsImg bool `json:"isImg"` ParentPath string `json:"parentPath"` Icon string `json:"icon"` Suffix string `json:"suffix"` Duration int `json:"duration"` Pantype string `json:"pantype"` Puid int_str `json:"puid"` Filepath string `json:"filepath"` Crc string `json:"crc"` Isfile bool `json:"isfile"` Residstr string `json:"residstr"` ObjectID string `json:"objectId"` Extinfo string `json:"extinfo"` Thumbnail string `json:"thumbnail"` Creator int `json:"creator"` ResTypeValue int `json:"resTypeValue"` UploadDateFormat string `json:"uploadDateFormat"` DisableOpt bool `json:"disableOpt"` DownPath string `json:"downPath"` Sort int `json:"sort"` Topsort int `json:"topsort"` Restype string `json:"restype"` Size int_str `json:"size"` UploadDate int64 `json:"uploadDate"` FileSize string `json:"fileSize"` Name string `json:"name"` FileID string `json:"fileId"` } `json:"content"` CreatorID int `json:"creatorId"` DesID string `json:"des_id"` ID int `json:"id"` Inserttime int64 `json:"inserttime"` Key string `json:"key"` Norder int `json:"norder"` OwnerID int `json:"ownerId"` OwnerType int `json:"ownerType"` Path string `json:"path"` Rid int `json:"rid"` Status int `json:"status"` Topsign int `json:"topsign"` } type ListFileResp struct { Msg string `json:"msg"` Result int `json:"result"` Status bool `json:"status"` UserAuth UserAuth `json:"userAuth"` List []File `json:"list"` } type DownResp struct { Msg string `json:"msg"` Duration int `json:"duration"` Download string `json:"download"` FileStatus string `json:"fileStatus"` URL string `json:"url"` Status bool `json:"status"` } type UploadDataRsp struct { Result int `json:"result"` Msg struct { Puid int `json:"puid"` Token string `json:"token"` } `json:"msg"` } type UploadFileDataRsp struct { Result bool `json:"result"` Msg string `json:"msg"` Crc string `json:"crc"` ObjectID string `json:"objectId"` Resid int64 `json:"resid"` Puid int `json:"puid"` Data struct { DisableOpt bool `json:"disableOpt"` Resid int64 `json:"resid"` Crc string `json:"crc"` Puid int `json:"puid"` Isfile bool `json:"isfile"` Pantype string `json:"pantype"` Size int `json:"size"` Name string `json:"name"` ObjectID string `json:"objectId"` Restype string `json:"restype"` UploadDate int64 `json:"uploadDate"` ModifyDate int64 `json:"modifyDate"` UploadDateFormat string `json:"uploadDateFormat"` Residstr string `json:"residstr"` Suffix string `json:"suffix"` Preview string `json:"preview"` Thumbnail string `json:"thumbnail"` Creator int `json:"creator"` Duration int `json:"duration"` IsImg bool `json:"isImg"` PreviewURL string `json:"previewUrl"` Filetype string `json:"filetype"` Filepath string `json:"filepath"` Sort int `json:"sort"` Topsort int `json:"topsort"` ResTypeValue int `json:"resTypeValue"` Extinfo string `json:"extinfo"` } `json:"data"` } type UploadDoneParam struct { Cataid string `json:"cataid"` Key string `json:"key"` Param struct { DisableOpt bool `json:"disableOpt"` Resid int64 `json:"resid"` Crc string `json:"crc"` Puid int `json:"puid"` Isfile bool `json:"isfile"` Pantype string `json:"pantype"` Size int `json:"size"` Name string `json:"name"` ObjectID string `json:"objectId"` Restype string `json:"restype"` UploadDate int64 `json:"uploadDate"` ModifyDate int64 `json:"modifyDate"` UploadDateFormat string `json:"uploadDateFormat"` Residstr string `json:"residstr"` Suffix string `json:"suffix"` Preview string `json:"preview"` Thumbnail string `json:"thumbnail"` Creator int `json:"creator"` Duration int `json:"duration"` IsImg bool `json:"isImg"` PreviewURL string `json:"previewUrl"` Filetype string `json:"filetype"` Filepath string `json:"filepath"` Sort int `json:"sort"` Topsort int `json:"topsort"` ResTypeValue int `json:"resTypeValue"` Extinfo string `json:"extinfo"` } `json:"param"` } func fileToObj(f File) *model.Object { if len(f.Content.FolderName) > 0 { return &model.Object{ ID: strconv.Itoa(f.ID), Name: f.Content.FolderName, Size: 0, Modified: time.UnixMilli(f.Inserttime), IsFolder: true, } } paserTime := time.UnixMilli(f.Content.UploadDate) return &model.Object{ ID: fmt.Sprintf("%d$%s", f.ID, f.Content.FileID), Name: f.Content.Name, Size: int64(f.Content.Size), Modified: paserTime, IsFolder: false, } } ================================================ FILE: drivers/chaoxing/util.go ================================================ package chaoxing import ( "bytes" "crypto/aes" "crypto/cipher" "encoding/base64" "errors" "fmt" "mime/multipart" "net/http" "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/go-resty/resty/v2" ) func (d *ChaoXing) requestDownload(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { u := d.conf.DowloadApi + pathname req := base.RestyClient.R() req.SetHeaders(map[string]string{ "Cookie": d.Cookie, "Accept": "application/json, text/plain, */*", "Referer": d.conf.referer, }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e Resp req.SetError(&e) res, err := req.Execute(method, u) if err != nil { return nil, err } return res.Body(), nil } func (d *ChaoXing) request(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { u := d.conf.api + pathname if strings.Contains(pathname, "getUploadConfig") { u = pathname } req := base.RestyClient.R() req.SetHeaders(map[string]string{ "Cookie": d.Cookie, "Accept": "application/json, text/plain, */*", "Referer": d.conf.referer, }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e Resp req.SetError(&e) res, err := req.Execute(method, u) if err != nil { return nil, err } return res.Body(), nil } func (d *ChaoXing) GetFiles(parent string) ([]File, error) { files := make([]File, 0) query := map[string]string{ "bbsid": d.Addition.Bbsid, "folderId": parent, "recType": "1", } var resp ListFileResp _, err := d.request("/pc/resource/getResourceList", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return nil, err } if resp.Result != 1 { msg := fmt.Sprintf("error code is:%d", resp.Result) return nil, errors.New(msg) } if len(resp.List) > 0 { files = append(files, resp.List...) } querys := map[string]string{ "bbsid": d.Addition.Bbsid, "folderId": parent, "recType": "2", } var resps ListFileResp _, err = d.request("/pc/resource/getResourceList", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(querys) }, &resps) if err != nil { return nil, err } for _, file := range resps.List { // 手机端超星上传的文件没有fileID字段，但ObjectID与fileID相同，可代替 if file.Content.FileID == "" { file.Content.FileID = file.Content.ObjectID } files = append(files, file) } return files, nil } func EncryptByAES(message, key string) (string, error) { aesKey := []byte(key) plainText := []byte(message) block, err := aes.NewCipher(aesKey) if err != nil { return "", err } iv := aesKey[:aes.BlockSize] mode := cipher.NewCBCEncrypter(block, iv) padding := aes.BlockSize - len(plainText)%aes.BlockSize paddedText := append(plainText, byte(padding)) for i := 0; i < padding-1; i++ { paddedText = append(paddedText, byte(padding)) } ciphertext := make([]byte, len(paddedText)) mode.CryptBlocks(ciphertext, paddedText) encrypted := base64.StdEncoding.EncodeToString(ciphertext) return encrypted, nil } func CookiesToString(cookies []*http.Cookie) string { var cookieStr string for _, cookie := range cookies { cookieStr += cookie.Name + "=" + cookie.Value + "; " } if len(cookieStr) > 2 { cookieStr = cookieStr[:len(cookieStr)-2] } return cookieStr } func (d *ChaoXing) Login() (string, error) { transferKey := "u2oh6Vu^HWe4_AES" body := &bytes.Buffer{} writer := multipart.NewWriter(body) uname, err := EncryptByAES(d.Addition.UserName, transferKey) if err != nil { return "", err } password, err := EncryptByAES(d.Addition.Password, transferKey) if err != nil { return "", err } err = writer.WriteField("uname", uname) if err != nil { return "", err } err = writer.WriteField("password", password) if err != nil { return "", err } err = writer.WriteField("t", "true") if err != nil { return "", err } err = writer.Close() if err != nil { return "", err } // Create the request req, err := http.NewRequest(http.MethodPost, "https://passport2.chaoxing.com/fanyalogin", body) if err != nil { return "", err } req.Header.Set("Content-Type", writer.FormDataContentType()) req.Header.Set("Content-Length", strconv.Itoa(body.Len())) resp, err := http.DefaultClient.Do(req) if err != nil { return "", err } defer resp.Body.Close() return CookiesToString(resp.Cookies()), nil } ================================================ FILE: drivers/chunk/driver.go ================================================ package chunk import ( "bytes" "context" "errors" "fmt" "io" stdpath "path" "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/avast/retry-go" ) type Chunk struct { model.Storage Addition } func (d *Chunk) Config() driver.Config { return config } func (d *Chunk) GetAddition() driver.Additional { return &d.Addition } func (d *Chunk) Init(ctx context.Context) error { if d.PartSize <= 0 { return errors.New("part size must be positive") } if len(d.ChunkPrefix) <= 0 { return errors.New("chunk folder prefix must not be empty") } d.RemotePath = utils.FixAndCleanPath(d.RemotePath) return nil } func (d *Chunk) Drop(ctx context.Context) error { return nil } func (Addition) GetRootPath() string { return "/" } func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(d.RemotePath) if err != nil { return nil, err } remoteActualPath = stdpath.Join(remoteActualPath, path) if remoteObj, err := op.Get(ctx, remoteStorage, remoteActualPath); err == nil { return &model.Object{ Path: path, Name: remoteObj.GetName(), Size: remoteObj.GetSize(), Modified: remoteObj.ModTime(), IsFolder: remoteObj.IsDir(), HashInfo: remoteObj.GetHash(), }, nil } remoteActualDir, name := stdpath.Split(remoteActualPath) chunkName := d.ChunkPrefix + name chunkObjs, err := op.List(ctx, remoteStorage, stdpath.Join(remoteActualDir, chunkName), model.ListArgs{}) if err != nil { return nil, err } var totalSize int64 = 0 // 0号块默认为-1 以支持空文件 chunkSizes := []int64{-1} h := make(map[*utils.HashType]string) var first model.Obj for _, o := range chunkObjs { if o.IsDir() { continue } if after, ok := strings.CutPrefix(o.GetName(), "hash_"); ok { hn, value, ok := strings.Cut(strings.TrimSuffix(after, d.CustomExt), "_") if ok { ht, ok := utils.GetHashByName(hn) if ok { h[ht] = value } } continue } idx, err := strconv.Atoi(strings.TrimSuffix(o.GetName(), d.CustomExt)) if err != nil { continue } totalSize += o.GetSize() if len(chunkSizes) > idx { if idx == 0 { first = o } chunkSizes[idx] = o.GetSize() } else if len(chunkSizes) == idx { chunkSizes = append(chunkSizes, o.GetSize()) } else { newChunkSizes := make([]int64, idx+1) copy(newChunkSizes, chunkSizes) chunkSizes = newChunkSizes chunkSizes[idx] = o.GetSize() } } reqDir, _ := stdpath.Split(path) objRes := chunkObject{ Object: model.Object{ Path: stdpath.Join(reqDir, chunkName), Name: name, Size: totalSize, Modified: first.ModTime(), Ctime: first.CreateTime(), }, chunkSizes: chunkSizes, } if len(h) > 0 { objRes.HashInfo = utils.NewHashInfoByMap(h) } return &objRes, nil } func (d *Chunk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(d.RemotePath) if err != nil { return nil, err } remoteActualDir := stdpath.Join(remoteActualPath, dir.GetPath()) remoteObjs, err := op.List(ctx, remoteStorage, remoteActualDir, model.ListArgs{ ReqPath: args.ReqPath, Refresh: args.Refresh, }) if err != nil { return nil, err } result := make([]model.Obj, 0, len(remoteObjs)) listG, listCtx := errgroup.NewGroupWithContext(ctx, d.NumListWorkers, retry.Attempts(3)) for _, obj := range remoteObjs { if utils.IsCanceled(listCtx) { break } rawName := obj.GetName() if obj.IsDir() { if name, ok := strings.CutPrefix(rawName, d.ChunkPrefix); ok { resultIdx := len(result) result = append(result, nil) listG.Go(func(ctx context.Context) error { chunkObjs, err := op.List(ctx, remoteStorage, stdpath.Join(remoteActualDir, rawName), model.ListArgs{ ReqPath: stdpath.Join(args.ReqPath, rawName), Refresh: args.Refresh, }) if err != nil { return err } totalSize := int64(0) h := make(map[*utils.HashType]string) first := obj for _, o := range chunkObjs { if o.IsDir() { continue } if after, ok := strings.CutPrefix(strings.TrimSuffix(o.GetName(), d.CustomExt), "hash_"); ok { hn, value, ok := strings.Cut(after, "_") if ok { ht, ok := utils.GetHashByName(hn) if ok { h[ht] = value } continue } } idx, err := strconv.Atoi(strings.TrimSuffix(o.GetName(), d.CustomExt)) if err != nil { continue } if idx == 0 { first = o } totalSize += o.GetSize() } objRes := model.Object{ Name: name, Size: totalSize, Modified: first.ModTime(), Ctime: first.CreateTime(), } if len(h) > 0 { objRes.HashInfo = utils.NewHashInfoByMap(h) } if !d.Thumbnail { result[resultIdx] = &objRes } else { thumbPath := stdpath.Join(args.ReqPath, ".thumbnails", name+".webp") thumb := fmt.Sprintf("%s/d%s?sign=%s", common.GetApiUrl(ctx), utils.EncodePath(thumbPath, true), sign.Sign(thumbPath)) result[resultIdx] = &model.ObjThumb{ Object: objRes, Thumbnail: model.Thumbnail{ Thumbnail: thumb, }, } } return nil }) continue } } if !d.ShowHidden && strings.HasPrefix(rawName, ".") { continue } thumb, ok := model.GetThumb(obj) objRes := model.Object{ Name: rawName, Size: obj.GetSize(), Modified: obj.ModTime(), IsFolder: obj.IsDir(), HashInfo: obj.GetHash(), } if !ok { result = append(result, &objRes) } else { result = append(result, &model.ObjThumb{ Object: objRes, Thumbnail: model.Thumbnail{ Thumbnail: thumb, }, }) } } if err = listG.Wait(); err != nil { return nil, err } return result, nil } func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(d.RemotePath) if err != nil { return nil, err } chunkFile, ok := file.(*chunkObject) remoteActualPath = stdpath.Join(remoteActualPath, file.GetPath()) if !ok { l, _, err := op.Link(ctx, remoteStorage, remoteActualPath, args) if err != nil { return nil, err } resultLink := *l resultLink.SyncClosers = utils.NewSyncClosers(l) return &resultLink, nil } // 检查0号块不等于-1 以支持空文件 // 如果块数量大于1 最后一块不可能为0 // 只检查中间块是否有0 if chunkFile.chunkSizes[0] == -1 { return nil, fmt.Errorf("chunk part[%d] are missing", 0) } for i, l := 1, len(chunkFile.chunkSizes)-1; i < l; i++ { if chunkFile.chunkSizes[i] == 0 { return nil, fmt.Errorf("chunk part[%d] are missing", i) } } fileSize := chunkFile.GetSize() mergedRrf := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { start := httpRange.Start length := httpRange.Length if length < 0 || start+length > fileSize { length = fileSize - start } if length == 0 { return io.NopCloser(strings.NewReader("")), nil } rs := make([]io.Reader, 0) cs := make(utils.Closers, 0) var ( rc io.ReadCloser readFrom bool ) for idx, chunkSize := range chunkFile.chunkSizes { if readFrom { l, o, err := op.Link(ctx, remoteStorage, stdpath.Join(remoteActualPath, d.getPartName(idx)), args) if err != nil { _ = cs.Close() return nil, err } cs = append(cs, l) chunkSize2 := l.ContentLength if chunkSize2 <= 0 { chunkSize2 = o.GetSize() } if chunkSize2 != chunkSize { _ = cs.Close() return nil, fmt.Errorf("chunk part[%d] size not match", idx) } rrf, err := stream.GetRangeReaderFromLink(chunkSize2, l) if err != nil { _ = cs.Close() return nil, err } newLength := length - chunkSize2 if newLength >= 0 { length = newLength rc, err = rrf.RangeRead(ctx, http_range.Range{Length: -1}) } else { rc, err = rrf.RangeRead(ctx, http_range.Range{Length: length}) } if err != nil { _ = cs.Close() return nil, err } rs = append(rs, rc) cs = append(cs, rc) if newLength <= 0 { return utils.ReadCloser{ Reader: io.MultiReader(rs...), Closer: &cs, }, nil } } else if newStart := start - chunkSize; newStart >= 0 { start = newStart } else { l, o, err := op.Link(ctx, remoteStorage, stdpath.Join(remoteActualPath, d.getPartName(idx)), args) if err != nil { _ = cs.Close() return nil, err } cs = append(cs, l) chunkSize2 := l.ContentLength if chunkSize2 <= 0 { chunkSize2 = o.GetSize() } if chunkSize2 != chunkSize { _ = cs.Close() return nil, fmt.Errorf("chunk part[%d] size not match", idx) } rrf, err := stream.GetRangeReaderFromLink(chunkSize2, l) if err != nil { _ = cs.Close() return nil, err } rc, err = rrf.RangeRead(ctx, http_range.Range{Start: start, Length: -1}) if err != nil { _ = cs.Close() return nil, err } length -= chunkSize2 - start cs = append(cs, rc) if length <= 0 { return utils.ReadCloser{ Reader: rc, Closer: &cs, }, nil } rs = append(rs, rc) readFrom = true } } return nil, fmt.Errorf("invalid range: start=%d,length=%d,fileSize=%d", httpRange.Start, httpRange.Length, fileSize) } return &model.Link{ RangeReader: stream.RangeReaderFunc(mergedRrf), }, nil } func (d *Chunk) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { path := stdpath.Join(d.RemotePath, parentDir.GetPath(), dirName) return fs.MakeDir(ctx, path) } func (d *Chunk) Move(ctx context.Context, srcObj, dstDir model.Obj) error { src := stdpath.Join(d.RemotePath, srcObj.GetPath()) dst := stdpath.Join(d.RemotePath, dstDir.GetPath()) _, err := fs.Move(ctx, src, dst) return err } func (d *Chunk) Rename(ctx context.Context, srcObj model.Obj, newName string) error { if _, ok := srcObj.(*chunkObject); ok { newName = d.ChunkPrefix + newName } return fs.Rename(ctx, stdpath.Join(d.RemotePath, srcObj.GetPath()), newName) } func (d *Chunk) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { dst := stdpath.Join(d.RemotePath, dstDir.GetPath()) src := stdpath.Join(d.RemotePath, srcObj.GetPath()) _, err := fs.Copy(ctx, src, dst) return err } func (d *Chunk) Remove(ctx context.Context, obj model.Obj) error { return fs.Remove(ctx, stdpath.Join(d.RemotePath, obj.GetPath())) } func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(d.RemotePath) if err != nil { return err } if (d.Thumbnail && dstDir.GetName() == ".thumbnails") || (d.ChunkLargeFileOnly && file.GetSize() <= d.PartSize) { return op.Put(ctx, remoteStorage, stdpath.Join(remoteActualPath, dstDir.GetPath()), file, up) } upReader := &driver.ReaderUpdatingProgress{ Reader: file, UpdateProgress: up, } dst := stdpath.Join(remoteActualPath, dstDir.GetPath(), d.ChunkPrefix+file.GetName()) skipHookCtx := context.WithValue(ctx, conf.SkipHookKey, struct{}{}) if d.StoreHash { for ht, value := range file.GetHash().All() { _ = op.Put(skipHookCtx, remoteStorage, dst, &stream.FileStream{ Obj: &model.Object{ Name: fmt.Sprintf("hash_%s_%s%s", ht.Name, value, d.CustomExt), Size: 1, Modified: file.ModTime(), }, Mimetype: "application/octet-stream", Reader: bytes.NewReader([]byte{0}), // 兼容不支持空文件的驱动 }, nil) } } fullPartCount := int(file.GetSize() / d.PartSize) tailSize := file.GetSize() % d.PartSize if tailSize == 0 && fullPartCount > 0 { fullPartCount-- tailSize = d.PartSize } partIndex := 0 for partIndex < fullPartCount { err = op.Put(skipHookCtx, remoteStorage, dst, &stream.FileStream{ Obj: &model.Object{ Name: d.getPartName(partIndex), Size: d.PartSize, Modified: file.ModTime(), }, Mimetype: file.GetMimetype(), Reader: io.LimitReader(upReader, d.PartSize), }, nil) if err != nil { _ = op.Remove(ctx, remoteStorage, dst) return err } partIndex++ } err = op.Put(ctx, remoteStorage, dst, &stream.FileStream{ Obj: &model.Object{ Name: d.getPartName(fullPartCount), Size: tailSize, Modified: file.ModTime(), }, Mimetype: file.GetMimetype(), Reader: upReader, }, nil) if err != nil { _ = op.Remove(ctx, remoteStorage, dst) } return err } func (d *Chunk) getPartName(part int) string { return fmt.Sprintf("%d%s", part, d.CustomExt) } func (d *Chunk) GetDetails(ctx context.Context) (*model.StorageDetails, error) { remoteStorage, err := fs.GetStorage(d.RemotePath, &fs.GetStoragesArgs{}) if err != nil { return nil, errs.NotImplement } remoteDetails, err := op.GetStorageDetails(ctx, remoteStorage) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: remoteDetails.DiskUsage, }, nil } var _ driver.Driver = (*Chunk)(nil) ================================================ FILE: drivers/chunk/meta.go ================================================ package chunk import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { RemotePath string `json:"remote_path" required:"true"` PartSize int64 `json:"part_size" required:"true" type:"number" help:"bytes"` ChunkLargeFileOnly bool `json:"chunk_large_file_only" default:"false" help:"chunk only if file size > part_size"` ChunkPrefix string `json:"chunk_prefix" type:"string" default:"[openlist_chunk]" help:"the prefix of chunk folder"` CustomExt string `json:"custom_ext" type:"string"` StoreHash bool `json:"store_hash" type:"bool" default:"true"` NumListWorkers int `json:"num_list_workers" required:"true" type:"number" default:"5"` Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` } var config = driver.Config{ Name: "Chunk", LocalSort: true, OnlyProxy: true, NoCache: true, DefaultRoot: "/", NoLinkURL: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Chunk{ Addition: Addition{ ChunkPrefix: "[openlist_chunk]", NumListWorkers: 5, }, } }) } ================================================ FILE: drivers/chunk/obj.go ================================================ package chunk import "github.com/OpenListTeam/OpenList/v4/internal/model" type chunkObject struct { model.Object chunkSizes []int64 } ================================================ FILE: drivers/cloudreve/driver.go ================================================ package cloudreve import ( "context" "io" "net/http" "path" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) type Cloudreve struct { model.Storage Addition ref *Cloudreve } func (d *Cloudreve) Config() driver.Config { return config } func (d *Cloudreve) GetAddition() driver.Additional { return &d.Addition } func (d *Cloudreve) Init(ctx context.Context) error { if d.Cookie != "" { return nil } // removing trailing slash d.Address = strings.TrimSuffix(d.Address, "/") return d.login() } func (d *Cloudreve) InitReference(storage driver.Driver) error { refStorage, ok := storage.(*Cloudreve) if ok { d.ref = refStorage return nil } return errs.NotSupport } func (d *Cloudreve) Drop(ctx context.Context) error { d.Cookie = "" d.ref = nil return nil } func (d *Cloudreve) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var r DirectoryResp err := d.request(http.MethodGet, "/directory"+dir.GetPath(), nil, &r) if err != nil { return nil, err } return utils.SliceConvert(r.Objects, func(src Object) (model.Obj, error) { thumb, err := d.GetThumb(src) if err != nil { return nil, err } if src.Type == "dir" && d.EnableThumbAndFolderSize { var dprop DirectoryProp err = d.request(http.MethodGet, "/object/property/"+src.Id+"?is_folder=true", nil, &dprop) if err != nil { return nil, err } src.Size = dprop.Size } src.Path = path.Join(dir.GetPath(), src.Name) return objectToObj(src, thumb), nil }) } func (d *Cloudreve) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var dUrl string err := d.request(http.MethodPut, "/file/download/"+file.GetID(), nil, &dUrl) if err != nil { return nil, err } if strings.HasPrefix(dUrl, "/api") { dUrl = d.Address + dUrl } return &model.Link{ URL: dUrl, }, nil } func (d *Cloudreve) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { return d.request(http.MethodPut, "/directory", func(req *resty.Request) { req.SetBody(base.Json{ "path": parentDir.GetPath() + "/" + dirName, }) }, nil) } func (d *Cloudreve) Move(ctx context.Context, srcObj, dstDir model.Obj) error { body := base.Json{ "action": "move", "src_dir": path.Dir(srcObj.GetPath()), "dst": dstDir.GetPath(), "src": convertSrc(srcObj), } return d.request(http.MethodPatch, "/object", func(req *resty.Request) { req.SetBody(body) }, nil) } func (d *Cloudreve) Rename(ctx context.Context, srcObj model.Obj, newName string) error { body := base.Json{ "action": "rename", "new_name": newName, "src": convertSrc(srcObj), } return d.request(http.MethodPatch, "/object/rename", func(req *resty.Request) { req.SetBody(body) }, nil) } func (d *Cloudreve) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { body := base.Json{ "src_dir": path.Dir(srcObj.GetPath()), "dst": dstDir.GetPath(), "src": convertSrc(srcObj), } return d.request(http.MethodPost, "/object/copy", func(req *resty.Request) { req.SetBody(body) }, nil) } func (d *Cloudreve) Remove(ctx context.Context, obj model.Obj) error { body := convertSrc(obj) err := d.request(http.MethodDelete, "/object", func(req *resty.Request) { req.SetBody(body) }, nil) return err } func (d *Cloudreve) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { if io.ReadCloser(stream) == http.NoBody { return d.create(ctx, dstDir, stream) } // 获取存储策略 var r DirectoryResp err := d.request(http.MethodGet, "/directory"+dstDir.GetPath(), nil, &r) if err != nil { return err } uploadBody := base.Json{ "path": dstDir.GetPath(), "size": stream.GetSize(), "name": stream.GetName(), "policy_id": r.Policy.Id, "last_modified": stream.ModTime().UnixMilli(), } // 获取上传会话信息 var u UploadInfo err = d.request(http.MethodPut, "/file/upload", func(req *resty.Request) { req.SetBody(uploadBody) }, &u) if err != nil { return err } // 根据存储方式选择分片上传的方法 switch r.Policy.Type { case "onedrive": err = d.upOneDrive(ctx, stream, u, up) case "s3": err = d.upS3(ctx, stream, u, up) case "remote": // 从机存储 err = d.upRemote(ctx, stream, u, up) case "local": // 本机存储 err = d.upLocal(ctx, stream, u, up) default: err = errs.NotImplement } if err != nil { // 删除失败的会话 _ = d.request(http.MethodDelete, "/file/upload/"+u.SessionID, nil, nil) return err } return nil } func (d *Cloudreve) create(ctx context.Context, dir model.Obj, file model.Obj) error { body := base.Json{"path": dir.GetPath() + "/" + file.GetName()} if file.IsDir() { err := d.request(http.MethodPut, "directory", func(req *resty.Request) { req.SetBody(body) }, nil) return err } return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { req.SetBody(body) }, nil) } func (d *Cloudreve) GetDetails(ctx context.Context) (*model.StorageDetails, error) { var r StorageDetails d.request(http.MethodGet, "/user/storage", nil, &r) return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: r.Total, UsedSpace: r.Used, }, }, nil } //func (d *Cloudreve) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*Cloudreve)(nil) ================================================ FILE: drivers/cloudreve/meta.go ================================================ package cloudreve import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { // Usually one of two driver.RootPath // define other Address string `json:"address" required:"true"` Username string `json:"username"` Password string `json:"password"` Cookie string `json:"cookie"` CustomUA string `json:"custom_ua"` EnableThumbAndFolderSize bool `json:"enable_thumb_and_folder_size"` } var config = driver.Config{ Name: "Cloudreve", DefaultRoot: "/", LocalSort: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Cloudreve{} }) } ================================================ FILE: drivers/cloudreve/types.go ================================================ package cloudreve import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type Resp struct { Code int `json:"code"` Msg string `json:"msg"` Data interface{} `json:"data"` } type Policy struct { Id string `json:"id"` Name string `json:"name"` Type string `json:"type"` MaxSize int `json:"max_size"` FileType []string `json:"file_type"` } type UploadInfo struct { SessionID string `json:"sessionID"` ChunkSize int `json:"chunkSize"` Expires int `json:"expires"` UploadURLs []string `json:"uploadURLs"` Credential string `json:"credential,omitempty"` // local CompleteURL string `json:"completeURL,omitempty"` // s3 } type DirectoryResp struct { Parent string `json:"parent"` Objects []Object `json:"objects"` Policy Policy `json:"policy"` } type Object struct { Id string `json:"id"` Name string `json:"name"` Path string `json:"path"` Pic string `json:"pic"` Size int `json:"size"` Type string `json:"type"` Date time.Time `json:"date"` CreateDate time.Time `json:"create_date"` SourceEnabled bool `json:"source_enabled"` } type DirectoryProp struct { Size int `json:"size"` } func objectToObj(f Object, t model.Thumbnail) *model.ObjThumb { return &model.ObjThumb{ Object: model.Object{ ID: f.Id, Name: f.Name, Size: int64(f.Size), Modified: f.Date, IsFolder: f.Type == "dir", Path: f.Path, }, Thumbnail: t, } } type Config struct { LoginCaptcha bool `json:"loginCaptcha"` CaptchaType string `json:"captcha_type"` } type StorageDetails struct { Used int64 `json:"used"` Free int64 `json:"free"` Total int64 `json:"total"` } ================================================ FILE: drivers/cloudreve/util.go ================================================ package cloudreve import ( "bytes" "context" "encoding/base64" "encoding/json" "errors" "fmt" "io" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/setting" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/cookie" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" ) // do others that not defined in Driver interface const loginPath = "/user/session" func (d *Cloudreve) getUA() string { if d.CustomUA != "" { return d.CustomUA } return base.UserAgent } func (d *Cloudreve) request(method string, path string, callback base.ReqCallback, out interface{}) error { if d.ref != nil { return d.ref.request(method, path, callback, out) } u := d.Address + "/api/v3" + path req := base.RestyClient.R() req.SetHeaders(map[string]string{ "Cookie": "cloudreve-session=" + d.Cookie, "Accept": "application/json, text/plain, */*", "User-Agent": d.getUA(), }) var r Resp req.SetResult(&r) if callback != nil { callback(req) } resp, err := req.Execute(method, u) if err != nil { return err } if !resp.IsSuccess() { return errors.New(resp.String()) } if r.Code != 0 { // 刷新 cookie if r.Code == http.StatusUnauthorized && path != loginPath { if d.Username != "" && d.Password != "" { err = d.login() if err != nil { return err } return d.request(method, path, callback, out) } } return errors.New(r.Msg) } sess := cookie.GetCookie(resp.Cookies(), "cloudreve-session") if sess != nil { d.Cookie = sess.Value } if out != nil && r.Data != nil { var marshal []byte marshal, err = jsoniter.Marshal(r.Data) if err != nil { return err } err = jsoniter.Unmarshal(marshal, out) if err != nil { return err } } return nil } func (d *Cloudreve) login() error { var siteConfig Config err := d.request(http.MethodGet, "/site/config", nil, &siteConfig) if err != nil { return err } for i := 0; i < 5; i++ { err = d.doLogin(siteConfig.LoginCaptcha) if err == nil { break } if err.Error() != "CAPTCHA not match." { break } } return err } func (d *Cloudreve) doLogin(needCaptcha bool) error { var captchaCode string var err error if needCaptcha { var captcha string err = d.request(http.MethodGet, "/site/captcha", nil, &captcha) if err != nil { return err } if len(captcha) == 0 { return errors.New("can not get captcha") } i := strings.Index(captcha, ",") dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(captcha[i+1:])) vRes, err := base.RestyClient.R().SetMultipartField( "image", "validateCode.png", "image/png", dec). Post(setting.GetStr(conf.OcrApi)) if err != nil { return err } if jsoniter.Get(vRes.Body(), "status").ToInt() != 200 { return errors.New("ocr error:" + jsoniter.Get(vRes.Body(), "msg").ToString()) } captchaCode = jsoniter.Get(vRes.Body(), "result").ToString() } var resp Resp err = d.request(http.MethodPost, loginPath, func(req *resty.Request) { req.SetBody(base.Json{ "username": d.Addition.Username, "Password": d.Addition.Password, "captchaCode": captchaCode, }) }, &resp) return err } func convertSrc(obj model.Obj) map[string]interface{} { m := make(map[string]interface{}) var dirs []string var items []string if obj.IsDir() { dirs = append(dirs, obj.GetID()) } else { items = append(items, obj.GetID()) } m["dirs"] = dirs m["items"] = items return m } func (d *Cloudreve) GetThumb(file Object) (model.Thumbnail, error) { if !d.Addition.EnableThumbAndFolderSize { return model.Thumbnail{}, nil } req := base.NoRedirectClient.R() req.SetHeaders(map[string]string{ "Cookie": "cloudreve-session=" + d.Cookie, "Accept": "image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", "User-Agent": d.getUA(), }) resp, err := req.Execute(http.MethodGet, d.Address+"/api/v3/file/thumb/"+file.Id) if err != nil { return model.Thumbnail{}, err } return model.Thumbnail{ Thumbnail: resp.Header().Get("Location"), }, nil } func (d *Cloudreve) upLocal(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error { var finish int64 = 0 var chunk int = 0 DEFAULT := int64(u.ChunkSize) for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } left := stream.GetSize() - finish byteSize := min(left, DEFAULT) utils.Log.Debugf("[Cloudreve-Local] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) byteData := make([]byte, byteSize) n, err := io.ReadFull(stream, byteData) utils.Log.Debug(err, n) if err != nil { return err } err = d.request(http.MethodPost, "/file/upload/"+u.SessionID+"/"+strconv.Itoa(chunk), func(req *resty.Request) { req.SetHeader("Content-Type", "application/octet-stream") req.SetContentLength(true) req.SetHeader("Content-Length", strconv.FormatInt(byteSize, 10)) req.SetHeader("User-Agent", d.getUA()) req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) req.AddRetryCondition(func(r *resty.Response, err error) bool { if err != nil { return true } if r.IsError() { return true } var retryResp Resp jErr := base.RestyClient.JSONUnmarshal(r.Body(), &retryResp) if jErr != nil { return true } if retryResp.Code != 0 { return true } return false }) }, nil) if err != nil { return err } finish += byteSize up(float64(finish) * 100 / float64(stream.GetSize())) chunk++ } return nil } func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error { DEFAULT := int64(u.ChunkSize) ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT), &up) if err != nil { return err } uploadUrl := u.UploadURLs[0] credential := u.Credential var finish int64 = 0 var chunk int = 0 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } left := stream.GetSize() - finish byteSize := min(left, DEFAULT) utils.Log.Debugf("[Cloudreve-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) rd, err := ss.GetSectionReader(finish, byteSize) if err != nil { return err } err = retry.Do( func() error { rd.Seek(0, io.SeekStart) req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"?chunk="+strconv.Itoa(chunk), driver.NewLimitedUploadStream(ctx, rd)) if err != nil { return err } req.ContentLength = byteSize req.Header.Set("Authorization", fmt.Sprint(credential)) req.Header.Set("User-Agent", d.getUA()) res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() if res.StatusCode != 200 { return fmt.Errorf("server error: %d", res.StatusCode) } body, err := io.ReadAll(res.Body) if err != nil { return err } var up Resp err = json.Unmarshal(body, &up) if err != nil { return err } if up.Code != 0 { return errors.New(up.Msg) } return nil }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) ss.FreeSectionReader(rd) if err != nil { return err } finish += byteSize up(float64(finish) * 100 / float64(stream.GetSize())) chunk++ } return nil } func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error { DEFAULT := int64(u.ChunkSize) ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT), &up) if err != nil { return err } uploadUrl := u.UploadURLs[0] var finish int64 = 0 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } left := stream.GetSize() - finish byteSize := min(left, DEFAULT) utils.Log.Debugf("[Cloudreve-OneDrive] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) rd, err := ss.GetSectionReader(finish, byteSize) if err != nil { return err } err = retry.Do( func() error { rd.Seek(0, io.SeekStart) req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, driver.NewLimitedUploadStream(ctx, rd)) if err != nil { return err } req.ContentLength = byteSize req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) req.Header.Set("User-Agent", d.getUA()) res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession switch { case res.StatusCode >= 500 && res.StatusCode <= 504: return fmt.Errorf("server error: %d", res.StatusCode) case res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200: data, _ := io.ReadAll(res.Body) return errors.New(string(data)) default: return nil } }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) ss.FreeSectionReader(rd) if err != nil { return err } finish += byteSize up(float64(finish) * 100 / float64(stream.GetSize())) } // 上传成功发送回调请求 return d.request(http.MethodPost, "/callback/onedrive/finish/"+u.SessionID, func(req *resty.Request) { req.SetBody("{}") }, nil) } func (d *Cloudreve) upS3(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error { DEFAULT := int64(u.ChunkSize) ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT), &up) if err != nil { return err } var finish int64 = 0 var chunk int = 0 var etags []string for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } left := stream.GetSize() - finish byteSize := min(left, DEFAULT) utils.Log.Debugf("[Cloudreve-S3] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) rd, err := ss.GetSectionReader(finish, byteSize) if err != nil { return err } err = retry.Do( func() error { rd.Seek(0, io.SeekStart) req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.UploadURLs[chunk], driver.NewLimitedUploadStream(ctx, rd)) if err != nil { return err } req.ContentLength = byteSize req.Header.Set("User-Agent", d.getUA()) res, err := base.HttpClient.Do(req) if err != nil { return err } etag := res.Header.Get("ETag") res.Body.Close() switch { case res.StatusCode != 200: return fmt.Errorf("server error: %d", res.StatusCode) case etag == "": return errors.New("failed to get ETag from header") default: etags = append(etags, etag) return nil } }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) ss.FreeSectionReader(rd) if err != nil { return err } finish += byteSize up(float64(finish) * 100 / float64(stream.GetSize())) chunk++ } // s3LikeFinishUpload // https://github.com/cloudreve/frontend/blob/b485bf297974cbe4834d2e8e744ae7b7e5b2ad39/src/component/Uploader/core/api/index.ts#L204-L252 bodyBuilder := &strings.Builder{} bodyBuilder.WriteString("") for i, etag := range etags { bodyBuilder.WriteString(fmt.Sprintf( `%d%s`, i+1, // PartNumber 从 1 开始 etag, )) } bodyBuilder.WriteString("") req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.CompleteURL, strings.NewReader(bodyBuilder.String()), ) if err != nil { return err } req.Header.Set("Content-Type", "application/xml") req.Header.Set("User-Agent", d.getUA()) res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() if res.StatusCode != http.StatusOK { body, _ := io.ReadAll(res.Body) return fmt.Errorf("up status: %d, error: %s", res.StatusCode, string(body)) } // 上传成功发送回调请求 err = d.request(http.MethodGet, "/callback/s3/"+u.SessionID, nil, nil) if err != nil { return err } return nil } ================================================ FILE: drivers/cloudreve_v4/driver.go ================================================ package cloudreve_v4 import ( "context" "errors" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) type CloudreveV4 struct { model.Storage Addition ref *CloudreveV4 AccessExpires string RefreshExpires string } func (d *CloudreveV4) Config() driver.Config { if d.ref != nil { return d.ref.Config() } if d.EnableVersionUpload { config.NoOverwriteUpload = false } return config } func (d *CloudreveV4) GetAddition() driver.Additional { return &d.Addition } func (d *CloudreveV4) Init(ctx context.Context) error { // removing trailing slash d.Address = strings.TrimSuffix(d.Address, "/") op.MustSaveDriverStorage(d) if d.ref != nil { return nil } if d.canLogin() { return d.login() } if d.RefreshToken != "" { return d.refreshToken() } if d.AccessToken == "" { return errors.New("no way to authenticate. At least AccessToken is required") } // ensure AccessToken is valid return d.parseJWT(d.AccessToken, &AccessJWT{}) } func (d *CloudreveV4) InitReference(storage driver.Driver) error { refStorage, ok := storage.(*CloudreveV4) if ok { d.ref = refStorage return nil } return errs.NotSupport } func (d *CloudreveV4) Drop(ctx context.Context) error { d.ref = nil return nil } func (d *CloudreveV4) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { const pageSize int = 100 var f []File var r FileResp params := map[string]string{ "page_size": strconv.Itoa(pageSize), "uri": dir.GetPath(), "order_by": d.OrderBy, "order_direction": d.OrderDirection, "page": "0", } for { err := d.request(http.MethodGet, "/file", func(req *resty.Request) { req.SetQueryParams(params) }, &r) if err != nil { return nil, err } f = append(f, r.Files...) if r.Pagination.NextToken == "" || len(r.Files) < pageSize { break } params["next_page_token"] = r.Pagination.NextToken } if d.HideUploading { f = utils.SliceFilter(f, func(src File) bool { return src.Metadata == nil || src.Metadata[MetadataUploadSessionID] == nil }) } return utils.SliceConvert(f, func(src File) (model.Obj, error) { if d.EnableFolderSize && src.Type == 1 { var ds FolderSummaryResp err := d.request(http.MethodGet, "/file/info", func(req *resty.Request) { req.SetQueryParam("uri", src.Path) req.SetQueryParam("folder_summary", "true") }, &ds) if err == nil && ds.FolderSummary.Size > 0 { src.Size = ds.FolderSummary.Size } } var thumb model.Thumbnail if d.EnableThumb && src.Type == 0 && (src.Metadata == nil || src.Metadata[MetadataThumbDisabled] == "") { var t FileThumbResp err := d.request(http.MethodGet, "/file/thumb", func(req *resty.Request) { req.SetQueryParam("uri", src.Path) }, &t) if err == nil && t.URL != "" { thumb = model.Thumbnail{ Thumbnail: t.URL, } } } return &model.ObjThumb{ Object: *fileToObject(&src), Thumbnail: thumb, }, nil }) } func (d *CloudreveV4) Get(ctx context.Context, path string) (model.Obj, error) { var info File err := d.request(http.MethodGet, "/file/info", func(req *resty.Request) { req.SetQueryParam("uri", d.RootFolderPath+path) }, &info) if err != nil { return nil, err } return fileToObject(&info), nil } func (d *CloudreveV4) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var url FileUrlResp err := d.request(http.MethodPost, "/file/url", func(req *resty.Request) { req.SetBody(base.Json{ "uris": []string{file.GetPath()}, "download": true, }) }, &url) if err != nil { return nil, err } if len(url.Urls) == 0 { return nil, errors.New("server returns no url") } exp := time.Until(url.Expires) return &model.Link{ URL: url.Urls[0].URL, Expiration: &exp, }, nil } func (d *CloudreveV4) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { req.SetBody(base.Json{ "type": "folder", "uri": parentDir.GetPath() + "/" + dirName, "error_on_conflict": true, }) }, nil) } func (d *CloudreveV4) Move(ctx context.Context, srcObj, dstDir model.Obj) error { return d.request(http.MethodPost, "/file/move", func(req *resty.Request) { req.SetBody(base.Json{ "uris": []string{srcObj.GetPath()}, "dst": dstDir.GetPath(), "copy": false, }) }, nil) } func (d *CloudreveV4) Rename(ctx context.Context, srcObj model.Obj, newName string) error { return d.request(http.MethodPost, "/file/rename", func(req *resty.Request) { req.SetBody(base.Json{ "new_name": newName, "uri": srcObj.GetPath(), }) }, nil) } func (d *CloudreveV4) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return d.request(http.MethodPost, "/file/move", func(req *resty.Request) { req.SetBody(base.Json{ "uris": []string{srcObj.GetPath()}, "dst": dstDir.GetPath(), "copy": true, }) }, nil) } func (d *CloudreveV4) Remove(ctx context.Context, obj model.Obj) error { var r FileDeleteResp err := d.request(http.MethodDelete, "/file", func(req *resty.Request) { req.SetBody(base.Json{ "uris": []string{obj.GetPath()}, "unlink": false, "skip_soft_delete": true, }) req.SetResult(&r) }, nil) if err != nil { return err } if r.Code == 0 { return nil } if r.Code == 40073 && r.Msg == "Lock conflict" && len(r.Data) > 0 { tokens := make([]string, 0, len(r.Data)) for _, item := range r.Data { tokens = append(tokens, item.Token) } err = d.request(http.MethodDelete, "/file/lock", func(req *resty.Request) { req.SetBody(base.Json{ "tokens": tokens, }) }, nil) if err != nil { return err } return d.request(http.MethodDelete, "/file", func(req *resty.Request) { req.SetBody(base.Json{ "uris": []string{obj.GetPath()}, "unlink": false, "skip_soft_delete": true, }) }, nil) } return errors.New(r.Msg) } func (d *CloudreveV4) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { if file.GetSize() == 0 { // 空文件使用新建文件方法，避免上传卡锁 return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { req.SetBody(base.Json{ "type": "file", "uri": dstDir.GetPath() + "/" + file.GetName(), "error_on_conflict": true, }) }, nil) } var p StoragePolicy var r FileResp var u FileUploadResp var err error params := map[string]string{ "page_size": "10", "uri": dstDir.GetPath(), "order_by": "created_at", "order_direction": "asc", "page": "0", } err = d.request(http.MethodGet, "/file", func(req *resty.Request) { req.SetQueryParams(params) }, &r) if err != nil { return err } p = r.StoragePolicy body := base.Json{ "uri": dstDir.GetPath() + "/" + file.GetName(), "size": file.GetSize(), "policy_id": p.ID, "last_modified": file.ModTime().UnixMilli(), "mime_type": "", } if d.EnableVersionUpload { body["entity_type"] = "version" } err = d.request(http.MethodPut, "/file/upload", func(req *resty.Request) { req.SetBody(body) }, &u) if err != nil { return err } if u.StoragePolicy.Relay { err = d.upLocal(ctx, file, u, up) } else { switch u.StoragePolicy.Type { case "local": err = d.upLocal(ctx, file, u, up) case "remote": err = d.upRemote(ctx, file, u, up) case "onedrive": err = d.upOneDrive(ctx, file, u, up) case "s3": err = d.upS3(ctx, file, u, up, "s3") case "ks3": err = d.upS3(ctx, file, u, up, "ks3") default: return errs.NotImplement } } if err != nil { // 删除失败的会话 _ = d.request(http.MethodDelete, "/file/upload", func(req *resty.Request) { req.SetBody(base.Json{ "id": u.SessionID, "uri": u.URI, }) }, nil) return err } return nil } func (d *CloudreveV4) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *CloudreveV4) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *CloudreveV4) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *CloudreveV4) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir // return errs.NotImplement to use an internal archive tool return nil, errs.NotImplement } func (d *CloudreveV4) GetDetails(ctx context.Context) (*model.StorageDetails, error) { // TODO return storage details (total space, free space, etc.) var r CapacityResp err := d.request(http.MethodGet, "/user/capacity", func(req *resty.Request) { req.SetContext(ctx) }, &r) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: r.Total, UsedSpace: r.Used, }, }, nil } //func (d *CloudreveV4) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*CloudreveV4)(nil) ================================================ FILE: drivers/cloudreve_v4/meta.go ================================================ package cloudreve_v4 import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { // Usually one of two driver.RootPath // driver.RootID // define other Address string `json:"address" required:"true"` Username string `json:"username"` Password string `json:"password"` AccessToken string `json:"access_token"` RefreshToken string `json:"refresh_token"` CustomUA string `json:"custom_ua"` EnableFolderSize bool `json:"enable_folder_size"` EnableThumb bool `json:"enable_thumb"` EnableVersionUpload bool `json:"enable_version_upload"` HideUploading bool `json:"hide_uploading"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at" default:"name" required:"true"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc" required:"true"` } var config = driver.Config{ Name: "Cloudreve V4", DefaultRoot: "cloudreve://my", CheckStatus: true, NoOverwriteUpload: true, } func init() { op.RegisterDriver(func() driver.Driver { return &CloudreveV4{} }) } ================================================ FILE: drivers/cloudreve_v4/types.go ================================================ package cloudreve_v4 import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) const ( MetadataUploadSessionID = "sys:upload_session_id" MetadataThumbDisabled = "thumb:disabled" ) type Object struct { model.Object StoragePolicy StoragePolicy } type Resp struct { Code int `json:"code"` Msg string `json:"msg"` Data any `json:"data"` } type BasicConfigResp struct { InstanceID string `json:"instance_id"` // Title string `json:"title"` // Themes string `json:"themes"` // DefaultTheme string `json:"default_theme"` User struct { ID string `json:"id"` // Nickname string `json:"nickname"` // CreatedAt time.Time `json:"created_at"` // Anonymous bool `json:"anonymous"` Group struct { ID string `json:"id"` Name string `json:"name"` Permission string `json:"permission"` } `json:"group"` } `json:"user"` // Logo string `json:"logo"` // LogoLight string `json:"logo_light"` // CaptchaReCaptchaKey string `json:"captcha_ReCaptchaKey"` CaptchaType string `json:"captcha_type"` // support 'normal' only // AppPromotion bool `json:"app_promotion"` } type SiteLoginConfigResp struct { LoginCaptcha bool `json:"login_captcha"` // RegCaptcha bool `json:"reg_captcha"` // ForgetCaptcha bool `json:"forget_captcha"` // RegisterEnabled bool `json:"register_enabled"` // TosURL string `json:"tos_url"` // PrivacyPolicyURL string `json:"privacy_policy_url"` // SsoDisplayName string `json:"sso_display_name"` // OidcDisplayName string `json:"oidc_display_name"` } type PrepareLoginResp struct { WebauthnEnabled bool `json:"webauthn_enabled"` PasswordEnabled bool `json:"password_enabled"` } type CaptchaResp struct { Image string `json:"image"` Ticket string `json:"ticket"` } type AccessJWT struct { TokenType string `json:"token_type"` Sub string `json:"sub"` Exp int64 `json:"exp"` Nbf int64 `json:"nbf"` } type RefreshJWT struct { TokenType string `json:"token_type"` Sub string `json:"sub"` Exp int `json:"exp"` Nbf int `json:"nbf"` StateHash string `json:"state_hash"` RootTokenID string `json:"root_token_id"` } type Token struct { AccessToken string `json:"access_token"` RefreshToken string `json:"refresh_token"` AccessExpires string `json:"access_expires"` RefreshExpires string `json:"refresh_expires"` } type TokenResponse struct { User struct { ID string `json:"id"` // Email string `json:"email"` // Nickname string `json:"nickname"` Status string `json:"status"` // CreatedAt time.Time `json:"created_at"` Group struct { ID string `json:"id"` Name string `json:"name"` Permission string `json:"permission"` // DirectLinkBatchSize int `json:"direct_link_batch_size"` // TrashRetention int `json:"trash_retention"` } `json:"group"` // Language string `json:"language"` } `json:"user"` Token Token `json:"token"` } type File struct { Type int `json:"type"` // 0: file, 1: folder ID string `json:"id"` Name string `json:"name"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` Size int64 `json:"size"` Metadata map[string]any `json:"metadata,omitempty"` Path string `json:"path"` Capability string `json:"capability"` Owned bool `json:"owned"` PrimaryEntity string `json:"primary_entity"` } func fileToObject(f *File) *model.Object { return &model.Object{ ID: f.ID, Path: f.Path, Name: f.Name, Size: f.Size, Modified: f.UpdatedAt, Ctime: f.CreatedAt, IsFolder: f.Type == 1, } } type StoragePolicy struct { ID string `json:"id"` Name string `json:"name"` Type string `json:"type"` MaxSize int64 `json:"max_size"` Relay bool `json:"relay,omitempty"` } type Pagination struct { Page int `json:"page"` PageSize int `json:"page_size"` IsCursor bool `json:"is_cursor"` NextToken string `json:"next_token,omitempty"` } type Props struct { Capability string `json:"capability"` MaxPageSize int `json:"max_page_size"` OrderByOptions []string `json:"order_by_options"` OrderDirectionOptions []string `json:"order_direction_options"` } type FileResp struct { Files []File `json:"files"` Parent File `json:"parent"` Pagination Pagination `json:"pagination"` Props Props `json:"props"` ContextHint string `json:"context_hint"` MixedType bool `json:"mixed_type"` StoragePolicy StoragePolicy `json:"storage_policy"` } type FileUrlResp struct { Urls []struct { URL string `json:"url"` } `json:"urls"` Expires time.Time `json:"expires"` } type FileUploadResp struct { // UploadID string `json:"upload_id"` SessionID string `json:"session_id"` ChunkSize int64 `json:"chunk_size"` Expires int64 `json:"expires"` StoragePolicy StoragePolicy `json:"storage_policy"` URI string `json:"uri"` CompleteURL string `json:"completeURL,omitempty"` // for S3-like CallbackSecret string `json:"callback_secret,omitempty"` // for S3-like, OneDrive UploadUrls []string `json:"upload_urls,omitempty"` // for not-local Credential string `json:"credential,omitempty"` // for local } type FileDeleteResp struct { Resp Data []struct { Path string `json:"path"` Token string `json:"token"` // Owner struct { // Owner string `json:"owner"` // Application struct { // Type string `json:"type"` // } `json:"application"` // } `json:"owner"` Type int `json:"type"` } `json:"data,omitempty"` } type FileThumbResp struct { URL string `json:"url"` Expires time.Time `json:"expires"` } type FolderSummaryResp struct { File FolderSummary struct { Size int64 `json:"size"` Files int64 `json:"files"` Folders int64 `json:"folders"` Completed bool `json:"completed"` CalculatedAt time.Time `json:"calculated_at"` } `json:"folder_summary"` } type CapacityResp struct { Total int64 `json:"total"` Used int64 `json:"used"` // StoragePackTotal uint64 `json:"storage_pack_total"` } ================================================ FILE: drivers/cloudreve_v4/util.go ================================================ package cloudreve_v4 import ( "bytes" "context" "encoding/base64" "encoding/json" "errors" "fmt" "io" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" ) // do others that not defined in Driver interface const ( CodeLoginRequired = http.StatusUnauthorized CodePathNotExist = 40016 // Path not exist CodeCredentialInvalid = 40020 // Failed to issue token ) var ( ErrorIssueToken = errors.New("failed to issue token") ) func (d *CloudreveV4) getUA() string { if d.CustomUA != "" { return d.CustomUA } return base.UserAgent } func (d *CloudreveV4) request(method string, path string, callback base.ReqCallback, out any) error { if d.ref != nil { return d.ref.request(method, path, callback, out) } // ensure token if d.isTokenExpired() { err := d.refreshToken() if err != nil { return err } } return d._request(method, path, callback, out) } func (d *CloudreveV4) _request(method string, path string, callback base.ReqCallback, out any) error { if d.ref != nil { return d.ref._request(method, path, callback, out) } u := d.Address + "/api/v4" + path req := base.RestyClient.R() req.SetHeaders(map[string]string{ "Accept": "application/json, text/plain, */*", "User-Agent": d.getUA(), }) if d.AccessToken != "" { req.SetHeader("Authorization", "Bearer "+d.AccessToken) } var r Resp req.SetResult(&r) if callback != nil { callback(req) } resp, err := req.Execute(method, u) if err != nil { return err } if !resp.IsSuccess() { return errors.New(resp.String()) } if r.Code != 0 { if r.Code == CodeLoginRequired && d.canLogin() && path != "/session/token/refresh" { err = d.login() if err != nil { return err } return d.request(method, path, callback, out) } if r.Code == CodeCredentialInvalid { return ErrorIssueToken } if r.Code == CodePathNotExist { return errs.ObjectNotFound } return fmt.Errorf("%d: %s", r.Code, r.Msg) } if out != nil && r.Data != nil { var marshal []byte marshal, err = json.Marshal(r.Data) if err != nil { return err } err = json.Unmarshal(marshal, out) if err != nil { return err } } return nil } func (d *CloudreveV4) canLogin() bool { return d.Username != "" && d.Password != "" } func (d *CloudreveV4) login() error { var siteConfig SiteLoginConfigResp err := d._request(http.MethodGet, "/site/config/login", nil, &siteConfig) if err != nil { return err } var prepareLogin PrepareLoginResp err = d._request(http.MethodGet, "/session/prepare?email="+d.Addition.Username, nil, &prepareLogin) if err != nil { return err } if !prepareLogin.PasswordEnabled { return errors.New("password not enabled") } if prepareLogin.WebauthnEnabled { return errors.New("webauthn not support") } for range 5 { err = d.doLogin(siteConfig.LoginCaptcha) if err == nil { break } if err.Error() != "CAPTCHA not match." { break } } return err } func (d *CloudreveV4) doLogin(needCaptcha bool) error { var err error loginBody := base.Json{ "email": d.Username, "password": d.Password, } if needCaptcha { var config BasicConfigResp err = d._request(http.MethodGet, "/site/config/basic", nil, &config) if err != nil { return err } if config.CaptchaType != "normal" { return fmt.Errorf("captcha type %s not support", config.CaptchaType) } var captcha CaptchaResp err = d._request(http.MethodGet, "/site/captcha", nil, &captcha) if err != nil { return err } if !strings.HasPrefix(captcha.Image, "data:image/png;base64,") { return errors.New("can not get captcha") } loginBody["ticket"] = captcha.Ticket i := strings.Index(captcha.Image, ",") dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(captcha.Image[i+1:])) vRes, err := base.RestyClient.R().SetMultipartField( "image", "validateCode.png", "image/png", dec). Post(setting.GetStr(conf.OcrApi)) if err != nil { return err } if jsoniter.Get(vRes.Body(), "status").ToInt() != 200 { return errors.New("ocr error:" + jsoniter.Get(vRes.Body(), "msg").ToString()) } captchaCode := jsoniter.Get(vRes.Body(), "result").ToString() if captchaCode == "" { return errors.New("ocr error: empty result") } loginBody["captcha"] = captchaCode } var token TokenResponse err = d._request(http.MethodPost, "/session/token", func(req *resty.Request) { req.SetBody(loginBody) }, &token) if err != nil { return err } d.AccessToken, d.RefreshToken = token.Token.AccessToken, token.Token.RefreshToken d.AccessExpires, d.RefreshExpires = token.Token.AccessExpires, token.Token.RefreshExpires op.MustSaveDriverStorage(d) return nil } func (d *CloudreveV4) refreshToken() error { // if no refresh token, try to login if possible if d.RefreshToken == "" { if d.canLogin() { err := d.login() if err != nil { return fmt.Errorf("cannot login to get refresh token, error: %s", err) } } return nil } // parse jwt to check if refresh token is valid var jwt RefreshJWT err := d.parseJWT(d.RefreshToken, &jwt) if err != nil { // if refresh token is invalid, try to login if possible if d.canLogin() { return d.login() } d.GetStorage().SetStatus(fmt.Sprintf("Invalid RefreshToken: %s", err.Error())) op.MustSaveDriverStorage(d) return fmt.Errorf("invalid refresh token: %w", err) } // do refresh token var token Token err = d._request(http.MethodPost, "/session/token/refresh", func(req *resty.Request) { req.SetBody(base.Json{ "refresh_token": d.RefreshToken, }) }, &token) if err != nil { if errors.Is(err, ErrorIssueToken) { if d.canLogin() { // try to login again return d.login() } d.GetStorage().SetStatus("This session is no longer valid") op.MustSaveDriverStorage(d) return ErrorIssueToken } return err } d.AccessToken, d.RefreshToken = token.AccessToken, token.RefreshToken d.AccessExpires, d.RefreshExpires = token.AccessExpires, token.RefreshExpires op.MustSaveDriverStorage(d) return nil } func (d *CloudreveV4) parseJWT(token string, jwt any) error { split := strings.Split(token, ".") if len(split) != 3 { return fmt.Errorf("invalid token length: %d, ensure the token is a valid JWT", len(split)) } data, err := base64.RawURLEncoding.DecodeString(split[1]) if err != nil { return fmt.Errorf("invalid token encoding: %w, ensure the token is a valid JWT", err) } err = json.Unmarshal(data, &jwt) if err != nil { return fmt.Errorf("invalid token content: %w, ensure the token is a valid JWT", err) } return nil } // check if token is expired // https://github.com/cloudreve/frontend/blob/ddfacc1c31c49be03beb71de4cc114c8811038d6/src/session/index.ts#L177-L200 func (d *CloudreveV4) isTokenExpired() bool { if d.RefreshToken == "" { // login again if username and password is set if d.canLogin() { return true } // no refresh token, cannot refresh return false } if d.AccessToken == "" { return true } var ( err error expires time.Time ) // check if token is expired if d.AccessExpires != "" { // use expires field if possible to prevent timezone issue // only available after login or refresh token // 2025-08-28T02:43:07.645109985+08:00 expires, err = time.Parse(time.RFC3339Nano, d.AccessExpires) if err != nil { return false } } else { // fallback to parse jwt // if failed, disable the storage var jwt AccessJWT err = d.parseJWT(d.AccessToken, &jwt) if err != nil { d.GetStorage().SetStatus(fmt.Sprintf("Invalid AccessToken: %s", err.Error())) op.MustSaveDriverStorage(d) return false } // may be have timezone issue expires = time.Unix(jwt.Exp, 0) } // add a 10 minutes safe margin ddl := time.Now().Add(10 * time.Minute) if expires.Before(ddl) { // current access token expired, check if refresh token is expired // warning: cannot parse refresh token from jwt, because the exp field is not standard if d.RefreshExpires != "" { refreshExpires, err := time.Parse(time.RFC3339Nano, d.RefreshExpires) if err != nil { return false } if refreshExpires.Before(time.Now()) { // This session is no longer valid if d.canLogin() { // try to login again return true } d.GetStorage().SetStatus("This session is no longer valid") op.MustSaveDriverStorage(d) return false } } return true } return false } func (d *CloudreveV4) upLocal(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error { var finish int64 = 0 var chunk int = 0 DEFAULT := int64(u.ChunkSize) if DEFAULT == 0 { // support relay DEFAULT = file.GetSize() } for finish < file.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } left := file.GetSize() - finish byteSize := min(left, DEFAULT) utils.Log.Debugf("[CloudreveV4-Local] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) byteData := make([]byte, byteSize) n, err := io.ReadFull(file, byteData) utils.Log.Debug(err, n) if err != nil { return err } err = d.request(http.MethodPost, "/file/upload/"+u.SessionID+"/"+strconv.Itoa(chunk), func(req *resty.Request) { req.SetHeader("Content-Type", "application/octet-stream") req.SetContentLength(true) req.SetHeader("Content-Length", strconv.FormatInt(byteSize, 10)) req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) req.AddRetryCondition(func(r *resty.Response, err error) bool { if err != nil { return true } if r.IsError() { return true } var retryResp Resp jErr := base.RestyClient.JSONUnmarshal(r.Body(), &retryResp) if jErr != nil { return true } if retryResp.Code != 0 { return true } return false }) }, nil) if err != nil { return err } finish += byteSize up(float64(finish) * 100 / float64(file.GetSize())) chunk++ } return nil } func (d *CloudreveV4) upRemote(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error { DEFAULT := int64(u.ChunkSize) ss, err := stream.NewStreamSectionReader(file, int(DEFAULT), &up) if err != nil { return err } uploadUrl := u.UploadUrls[0] credential := u.Credential var finish int64 = 0 var chunk int = 0 for finish < file.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } left := file.GetSize() - finish byteSize := min(left, DEFAULT) utils.Log.Debugf("[CloudreveV4-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) rd, err := ss.GetSectionReader(finish, byteSize) if err != nil { return err } err = retry.Do( func() error { rd.Seek(0, io.SeekStart) req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"?chunk="+strconv.Itoa(chunk), driver.NewLimitedUploadStream(ctx, rd)) if err != nil { return err } req.ContentLength = byteSize req.Header.Set("Authorization", fmt.Sprint(credential)) req.Header.Set("User-Agent", d.getUA()) res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() if res.StatusCode != 200 { return fmt.Errorf("server error: %d", res.StatusCode) } body, err := io.ReadAll(res.Body) if err != nil { return err } var up Resp err = json.Unmarshal(body, &up) if err != nil { return err } if up.Code != 0 { return errors.New(up.Msg) } return nil }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) ss.FreeSectionReader(rd) if err != nil { return err } finish += byteSize up(float64(finish) * 100 / float64(file.GetSize())) chunk++ } return nil } func (d *CloudreveV4) upOneDrive(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error { DEFAULT := int64(u.ChunkSize) ss, err := stream.NewStreamSectionReader(file, int(DEFAULT), &up) if err != nil { return err } uploadUrl := u.UploadUrls[0] var finish int64 = 0 for finish < file.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } left := file.GetSize() - finish byteSize := min(left, DEFAULT) utils.Log.Debugf("[CloudreveV4-OneDrive] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) rd, err := ss.GetSectionReader(finish, byteSize) if err != nil { return err } err = retry.Do( func() error { rd.Seek(0, io.SeekStart) req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, driver.NewLimitedUploadStream(ctx, rd)) if err != nil { return err } req.ContentLength = byteSize req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, file.GetSize())) req.Header.Set("User-Agent", d.getUA()) res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession switch { case res.StatusCode >= 500 && res.StatusCode <= 504: return fmt.Errorf("server error: %d", res.StatusCode) case res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200: data, _ := io.ReadAll(res.Body) return errors.New(string(data)) default: return nil } }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) ss.FreeSectionReader(rd) if err != nil { return err } finish += byteSize up(float64(finish) * 100 / float64(file.GetSize())) } // 上传成功发送回调请求 return d.request(http.MethodPost, "/callback/onedrive/"+u.SessionID+"/"+u.CallbackSecret, func(req *resty.Request) { req.SetBody("{}") }, nil) } func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress, s3Type string) error { DEFAULT := int64(u.ChunkSize) ss, err := stream.NewStreamSectionReader(file, int(DEFAULT), &up) if err != nil { return err } var finish int64 = 0 var chunk int = 0 var etags []string for finish < file.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } left := file.GetSize() - finish byteSize := min(left, DEFAULT) utils.Log.Debugf("[CloudreveV4-S3] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) rd, err := ss.GetSectionReader(finish, byteSize) if err != nil { return err } err = retry.Do( func() error { rd.Seek(0, io.SeekStart) req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.UploadUrls[chunk], driver.NewLimitedUploadStream(ctx, rd)) if err != nil { return err } req.ContentLength = byteSize req.Header.Set("User-Agent", d.getUA()) if s3Type == "ks3" { req.Header.Set("Content-Type", "application/octet-stream") } res, err := base.HttpClient.Do(req) if err != nil { return err } etag := res.Header.Get("ETag") res.Body.Close() switch { case res.StatusCode != 200: return fmt.Errorf("server error: %d", res.StatusCode) case etag == "": return errors.New("failed to get ETag from header") default: etags = append(etags, etag) return nil } }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) ss.FreeSectionReader(rd) if err != nil { return err } finish += byteSize up(float64(finish) * 100 / float64(file.GetSize())) chunk++ } // s3LikeFinishUpload bodyBuilder := &strings.Builder{} bodyBuilder.WriteString("") for i, etag := range etags { bodyBuilder.WriteString(fmt.Sprintf( `%d%s`, i+1, // PartNumber 从 1 开始 etag, )) } bodyBuilder.WriteString("") req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.CompleteURL, strings.NewReader(bodyBuilder.String()), ) if err != nil { return err } if s3Type == "ks3" { req.Header.Set("Content-Type", "application/octet-stream") } else { req.Header.Set("Content-Type", "application/xml") } req.Header.Set("User-Agent", d.getUA()) res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() if res.StatusCode != http.StatusOK { body, _ := io.ReadAll(res.Body) return fmt.Errorf("up status: %d, error: %s", res.StatusCode, string(body)) } // 上传成功发送回调请求 return d.request(http.MethodGet, "/callback/"+s3Type+"/"+u.SessionID+"/"+u.CallbackSecret, nil, nil) } ================================================ FILE: drivers/cnb_releases/driver.go ================================================ package cnb_releases import ( "bytes" "context" "fmt" "io" "mime/multipart" "net/http" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) type CnbReleases struct { model.Storage Addition ref *CnbReleases } func (d *CnbReleases) Config() driver.Config { return config } func (d *CnbReleases) GetAddition() driver.Additional { return &d.Addition } func (d *CnbReleases) Init(ctx context.Context) error { return nil } func (d *CnbReleases) InitReference(storage driver.Driver) error { refStorage, ok := storage.(*CnbReleases) if ok { d.ref = refStorage return nil } return fmt.Errorf("ref: storage is not CnbReleases") } func (d *CnbReleases) Drop(ctx context.Context) error { d.ref = nil return nil } func (d *CnbReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { dirID := dir.GetID() if dirID == "" { // get all releases for root dir var resp ReleaseList err := d.Request(http.MethodGet, "/{repo}/-/releases", func(req *resty.Request) { req.SetPathParam("repo", d.Repo) }, &resp) if err != nil { return nil, err } return utils.SliceConvert(resp, func(src Release) (model.Obj, error) { name := src.Name if d.UseTagName { name = src.TagName } return &model.Object{ ID: src.ID, Name: name, Size: d.sumAssetsSize(src.Assets), Ctime: src.CreatedAt, Modified: src.UpdatedAt, IsFolder: true, }, nil }) } var resp Release err := d.Request(http.MethodGet, "/{repo}/-/releases/{release_id}", func(req *resty.Request) { req.SetPathParam("repo", d.Repo) req.SetPathParam("release_id", dirID) }, &resp) if err != nil { return nil, err } return utils.SliceConvert(resp.Assets, func(src ReleaseAsset) (model.Obj, error) { return &Object{ Object: model.Object{ ID: src.ID, Path: src.Path, Name: src.Name, Size: src.Size, Ctime: src.CreatedAt, Modified: src.UpdatedAt, IsFolder: false, }, ParentID: dirID, }, nil }) } func (d *CnbReleases) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { return &model.Link{ URL: "https://cnb.cool" + file.GetPath(), }, nil } func (d *CnbReleases) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { if parentDir.GetPath() == "/" { // create a new release branch := d.DefaultBranch if branch == "" { branch = "main" // fallback to "main" if not set } return d.Request(http.MethodPost, "/{repo}/-/releases", func(req *resty.Request) { req.SetPathParam("repo", d.Repo) req.SetBody(base.Json{ "name": dirName, "tag_name": dirName, "target_commitish": branch, }) }, nil) } return errs.NotImplement } func (d *CnbReleases) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return nil, errs.NotImplement } func (d *CnbReleases) Rename(ctx context.Context, srcObj model.Obj, newName string) error { if srcObj.IsDir() && !d.UseTagName { return d.Request(http.MethodPatch, "/{repo}/-/releases/{release_id}", func(req *resty.Request) { req.SetPathParam("repo", d.Repo) req.SetPathParam("release_id", srcObj.GetID()) req.SetFormData(map[string]string{ "name": newName, }) }, nil) } return errs.NotImplement } func (d *CnbReleases) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return nil, errs.NotImplement } func (d *CnbReleases) Remove(ctx context.Context, obj model.Obj) error { if obj.IsDir() { return d.Request(http.MethodDelete, "/{repo}/-/releases/{release_id}", func(req *resty.Request) { req.SetPathParam("repo", d.Repo) req.SetPathParam("release_id", obj.GetID()) }, nil) } if o, ok := obj.(*Object); ok { return d.Request(http.MethodDelete, "/{repo}/-/releases/{release_id}/assets/{asset_id}", func(req *resty.Request) { req.SetPathParam("repo", d.Repo) req.SetPathParam("release_id", o.ParentID) req.SetPathParam("asset_id", obj.GetID()) }, nil) } else { return fmt.Errorf("unable to get release ID") } } func (d *CnbReleases) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { // 1. get upload info var resp ReleaseAssetUploadURL err := d.Request(http.MethodPost, "/{repo}/-/releases/{release_id}/asset-upload-url", func(req *resty.Request) { req.SetPathParam("repo", d.Repo) req.SetPathParam("release_id", dstDir.GetID()) req.SetBody(base.Json{ "asset_name": file.GetName(), "overwrite": true, "size": file.GetSize(), }) }, &resp) if err != nil { return err } // 2. upload file // use multipart to create form file var b bytes.Buffer w := multipart.NewWriter(&b) _, err = w.CreateFormFile("file", file.GetName()) if err != nil { return err } headSize := b.Len() err = w.Close() if err != nil { return err } head := bytes.NewReader(b.Bytes()[:headSize]) tail := bytes.NewReader(b.Bytes()[headSize:]) r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: &driver.SimpleReaderWithSize{ Reader: io.MultiReader(head, file, tail), Size: int64(b.Len()) + file.GetSize(), }, UpdateProgress: up, }) // use net/http to upload file ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(resp.ExpiresInSec+1)*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctxWithTimeout, http.MethodPost, resp.UploadURL, r) if err != nil { return err } req.Header.Set("Content-Type", w.FormDataContentType()) req.Header.Set("User-Agent", base.UserAgent) httpResp, err := base.HttpClient.Do(req) if err != nil { return err } defer httpResp.Body.Close() if httpResp.StatusCode != http.StatusNoContent { return fmt.Errorf("upload file failed: %s", httpResp.Status) } // 3. verify upload return d.Request(http.MethodPost, resp.VerifyURL, nil, nil) } var _ driver.Driver = (*CnbReleases)(nil) ================================================ FILE: drivers/cnb_releases/meta.go ================================================ package cnb_releases import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootID Repo string `json:"repo" type:"string" required:"true"` Token string `json:"token" type:"string" required:"true"` UseTagName bool `json:"use_tag_name" type:"bool" default:"false" help:"Use tag name instead of release name"` DefaultBranch string `json:"default_branch" type:"string" default:"main" help:"Default branch for new releases"` } var config = driver.Config{ Name: "CNB Releases", LocalSort: true, } func init() { op.RegisterDriver(func() driver.Driver { return &CnbReleases{} }) } ================================================ FILE: drivers/cnb_releases/types.go ================================================ package cnb_releases import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type Object struct { model.Object ParentID string } type TagList []Tag type Tag struct { Commit struct { Author UserInfo `json:"author"` Commit CommitObject `json:"commit"` Committer UserInfo `json:"committer"` Parents []CommitParent `json:"parents"` Sha string `json:"sha"` } `json:"commit"` Name string `json:"name"` Target string `json:"target"` TargetType string `json:"target_type"` Verification TagObjectVerification `json:"verification"` } type UserInfo struct { Freeze bool `json:"freeze"` Nickname string `json:"nickname"` Username string `json:"username"` } type CommitObject struct { Author Signature `json:"author"` CommentCount int `json:"comment_count"` Committer Signature `json:"committer"` Message string `json:"message"` Tree CommitObjectTree `json:"tree"` Verification CommitObjectVerification `json:"verification"` } type Signature struct { Date time.Time `json:"date"` Email string `json:"email"` Name string `json:"name"` } type CommitObjectTree struct { Sha string `json:"sha"` } type CommitObjectVerification struct { Payload string `json:"payload"` Reason string `json:"reason"` Signature string `json:"signature"` Verified bool `json:"verified"` VerifiedAt string `json:"verified_at"` } type CommitParent = CommitObjectTree type TagObjectVerification = CommitObjectVerification type ReleaseList []Release type Release struct { Assets []ReleaseAsset `json:"assets"` Author UserInfo `json:"author"` Body string `json:"body"` CreatedAt time.Time `json:"created_at"` Draft bool `json:"draft"` ID string `json:"id"` IsLatest bool `json:"is_latest"` Name string `json:"name"` Prerelease bool `json:"prerelease"` PublishedAt time.Time `json:"published_at"` TagCommitish string `json:"tag_commitish"` TagName string `json:"tag_name"` UpdatedAt time.Time `json:"updated_at"` } type ReleaseAsset struct { ContentType string `json:"content_type"` CreatedAt time.Time `json:"created_at"` ID string `json:"id"` Name string `json:"name"` Path string `json:"path"` Size int64 `json:"size"` UpdatedAt time.Time `json:"updated_at"` Uploader UserInfo `json:"uploader"` } type ReleaseAssetUploadURL struct { UploadURL string `json:"upload_url"` ExpiresInSec int `json:"expires_in_sec"` VerifyURL string `json:"verify_url"` } ================================================ FILE: drivers/cnb_releases/util.go ================================================ package cnb_releases import ( "encoding/json" "fmt" "net/http" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" log "github.com/sirupsen/logrus" ) // do others that not defined in Driver interface func (d *CnbReleases) Request(method string, path string, callback base.ReqCallback, resp any) error { if d.ref != nil { return d.ref.Request(method, path, callback, resp) } var url string if strings.HasPrefix(path, "http") { url = path } else { url = "https://api.cnb.cool" + path } req := base.RestyClient.R() req.SetHeader("Accept", "application/json") req.SetAuthScheme("Bearer") req.SetAuthToken(d.Token) if callback != nil { callback(req) } res, err := req.Execute(method, url) log.Debugln(res.String()) if err != nil { return err } if res.StatusCode() != http.StatusOK && res.StatusCode() != http.StatusCreated && res.StatusCode() != http.StatusNoContent { return fmt.Errorf("failed to request %s, status code: %d, message: %s", url, res.StatusCode(), res.String()) } if resp != nil { err = json.Unmarshal(res.Body(), resp) if err != nil { return err } } return nil } func (d *CnbReleases) sumAssetsSize(assets []ReleaseAsset) int64 { var size int64 for _, asset := range assets { size += asset.Size } return size } ================================================ FILE: drivers/crypt/driver.go ================================================ package crypt import ( "bytes" "context" "errors" "fmt" "io" stdpath "path" "regexp" "strings" "sync" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" rcCrypt "github.com/rclone/rclone/backend/crypt" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/obscure" log "github.com/sirupsen/logrus" ) type Crypt struct { model.Storage Addition cipher *rcCrypt.Cipher } const obfuscatedPrefix = "___Obfuscated___" func (d *Crypt) Config() driver.Config { return config } func (d *Crypt) GetAddition() driver.Additional { return &d.Addition } func (d *Crypt) Init(ctx context.Context) error { // obfuscate credentials if it's updated or just created err := d.updateObfusParm(&d.Password) if err != nil { return fmt.Errorf("failed to obfuscate password: %w", err) } err = d.updateObfusParm(&d.Salt) if err != nil { return fmt.Errorf("failed to obfuscate salt: %w", err) } isCryptExt := regexp.MustCompile(`^[.][A-Za-z0-9-_]{2,}$`).MatchString if !isCryptExt(d.EncryptedSuffix) { return fmt.Errorf("EncryptedSuffix is Illegal") } d.FileNameEncoding = utils.GetNoneEmpty(d.FileNameEncoding, "base64") d.EncryptedSuffix = utils.GetNoneEmpty(d.EncryptedSuffix, ".bin") d.RemotePath = utils.FixAndCleanPath(d.RemotePath) p, _ := strings.CutPrefix(d.Password, obfuscatedPrefix) p2, _ := strings.CutPrefix(d.Salt, obfuscatedPrefix) config := configmap.Simple{ "password": p, "password2": p2, "filename_encryption": d.FileNameEnc, "directory_name_encryption": d.DirNameEnc, "filename_encoding": d.FileNameEncoding, "suffix": d.EncryptedSuffix, "pass_bad_blocks": "", } c, err := rcCrypt.NewCipher(config) if err != nil { return fmt.Errorf("failed to create Cipher: %w", err) } d.cipher = c return nil } func (d *Crypt) updateObfusParm(str *string) error { temp := *str if !strings.HasPrefix(temp, obfuscatedPrefix) { temp, err := obscure.Obscure(temp) if err != nil { return err } temp = obfuscatedPrefix + temp *str = temp } return nil } func (d *Crypt) Drop(ctx context.Context) error { return nil } func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { remoteFullPath := dir.GetPath() objs, err := fs.List(ctx, remoteFullPath, &fs.ListArgs{NoLog: true, Refresh: args.Refresh}) // the obj must implement the model.SetPath interface // return objs, err if err != nil { return nil, err } result := make([]model.Obj, 0, len(objs)) for _, obj := range objs { size := obj.GetSize() mask := model.GetObjMask(obj) name := obj.GetName() if mask&model.Virtual == 0 { if obj.IsDir() { name, err = d.cipher.DecryptDirName(model.UnwrapObjName(obj).GetName()) if err != nil { // filter illegal files continue } } else { size, err = d.cipher.DecryptedSize(size) if err != nil { // filter illegal files continue } name, err = d.cipher.DecryptFileName(model.UnwrapObjName(obj).GetName()) if err != nil { // filter illegal files continue } } } if !d.ShowHidden && strings.HasPrefix(name, ".") { continue } objRes := &model.Object{ Path: stdpath.Join(remoteFullPath, obj.GetName()), Name: name, Size: size, Modified: obj.ModTime(), IsFolder: obj.IsDir(), Ctime: obj.CreateTime(), Mask: mask &^ model.Temp, // discarding hash as it's encrypted } if !d.Thumbnail || !strings.HasPrefix(args.ReqPath, "/") { result = append(result, objRes) continue } thumbPath := stdpath.Join(args.ReqPath, ".thumbnails", name+".webp") thumb := fmt.Sprintf("%s/d%s?sign=%s", common.GetApiUrl(ctx), utils.EncodePath(thumbPath, true), sign.Sign(thumbPath)) result = append(result, &model.ObjThumb{ Object: *objRes, Thumbnail: model.Thumbnail{ Thumbnail: thumb, }, }) } return result, nil } func (a Addition) GetRootPath() string { return a.RemotePath } func (d *Crypt) Get(ctx context.Context, path string) (model.Obj, error) { firstTryIsFolder, secondTry := guessPath(path) remoteFullPath := stdpath.Join(d.RemotePath, d.encryptPath(path, firstTryIsFolder)) remoteObj, err := fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true}) if err != nil { if errors.Is(err, errs.StorageNotFound) { remoteFullPath = stdpath.Join(d.RemotePath, path) remoteObj, err = fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true}) if err != nil { // 可能是 虚拟路径+开启文件夹加密：返回NotSupport让op.Get去尝试op.List查找 return nil, errs.NotSupport } } else if secondTry && errs.IsObjectNotFound(err) { // try the opposite remoteFullPath = stdpath.Join(d.RemotePath, d.encryptPath(path, !firstTryIsFolder)) remoteObj, err = fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true}) if err != nil { return nil, err } } else { return nil, err } } size := remoteObj.GetSize() name := remoteObj.GetName() mask := model.GetObjMask(remoteObj) &^ model.Temp if mask&model.Virtual == 0 { if !remoteObj.IsDir() { decryptedSize, err := d.cipher.DecryptedSize(size) if err != nil { log.Warnf("DecryptedSize failed for %s ,will use original size, err:%s", path, err) } else { size = decryptedSize } decryptedName, err := d.cipher.DecryptFileName(model.UnwrapObjName(remoteObj).GetName()) if err != nil { log.Warnf("DecryptFileName failed for %s ,will use original name, err:%s", path, err) } else { name = decryptedName } } else { decryptedName, err := d.cipher.DecryptDirName(model.UnwrapObjName(remoteObj).GetName()) if err != nil { log.Warnf("DecryptDirName failed for %s ,will use original name, err:%s", path, err) } else { name = decryptedName } } } return &model.Object{ Path: remoteFullPath, Name: name, Size: size, Modified: remoteObj.ModTime(), IsFolder: remoteObj.IsDir(), Ctime: remoteObj.CreateTime(), Mask: mask, }, nil } // https://github.com/rclone/rclone/blob/v1.67.0/backend/crypt/cipher.go#L37 const fileHeaderSize = 32 func (d *Crypt) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*model.Link, error) { remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(file.GetPath()) if err != nil { return nil, err } remoteLink, remoteFile, err := op.Link(ctx, remoteStorage, remoteActualPath, model.LinkArgs{}) if err != nil { return nil, err } remoteSize := remoteLink.ContentLength if remoteSize <= 0 { remoteSize = remoteFile.GetSize() } rrf, err := stream.GetRangeReaderFromLink(remoteSize, remoteLink) if err != nil { _ = remoteLink.Close() return nil, fmt.Errorf("the remote storage driver need to be enhanced to support encrytion") } mu := &sync.Mutex{} var fileHeader []byte rangeReaderFunc := func(ctx context.Context, offset, limit int64) (io.ReadCloser, error) { length := limit if offset == 0 && limit > 0 { mu.Lock() if limit <= fileHeaderSize { defer mu.Unlock() if fileHeader != nil { return io.NopCloser(bytes.NewReader(fileHeader[:limit])), nil } length = fileHeaderSize } else if fileHeader == nil { defer mu.Unlock() } else { mu.Unlock() } } remoteReader, err := rrf.RangeRead(ctx, http_range.Range{Start: offset, Length: length}) if err != nil { return nil, err } if offset == 0 && limit > 0 { fileHeader = make([]byte, fileHeaderSize) n, err := io.ReadFull(remoteReader, fileHeader) if n != fileHeaderSize { fileHeader = nil return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", fileHeaderSize, n, err) } if limit <= fileHeaderSize { remoteReader.Close() return io.NopCloser(bytes.NewReader(fileHeader[:limit])), nil } else { remoteReader = utils.ReadCloser{ Reader: io.MultiReader(bytes.NewReader(fileHeader), remoteReader), Closer: remoteReader, } } } return remoteReader, nil } return &model.Link{ RangeReader: stream.RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { readSeeker, err := d.cipher.DecryptDataSeek(ctx, rangeReaderFunc, httpRange.Start, httpRange.Length) if err != nil { return nil, err } return readSeeker, nil }), SyncClosers: utils.NewSyncClosers(remoteLink), RequireReference: remoteLink.RequireReference, }, nil } func (d *Crypt) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(parentDir.GetPath()) if err != nil { return err } encryptedName := d.cipher.EncryptDirName(dirName) return op.MakeDir(ctx, remoteStorage, stdpath.Join(remoteActualPath, encryptedName)) } func (d *Crypt) Move(ctx context.Context, srcObj, dstDir model.Obj) error { _, err := fs.Move(ctx, srcObj.GetPath(), dstDir.GetPath()) return err } func (d *Crypt) Rename(ctx context.Context, srcObj model.Obj, newName string) error { remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(srcObj.GetPath()) if err != nil { return err } var newEncryptedName string if srcObj.IsDir() { newEncryptedName = d.cipher.EncryptDirName(newName) } else { newEncryptedName = d.cipher.EncryptFileName(newName) } return op.Rename(ctx, remoteStorage, remoteActualPath, newEncryptedName) } func (d *Crypt) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { _, err := fs.Copy(ctx, srcObj.GetPath(), dstDir.GetPath()) return err } func (d *Crypt) Remove(ctx context.Context, obj model.Obj) error { remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(obj.GetPath()) if err != nil { return err } return op.Remove(ctx, remoteStorage, remoteActualPath) } func (d *Crypt) Put(ctx context.Context, dstDir model.Obj, streamer model.FileStreamer, up driver.UpdateProgress) error { remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(dstDir.GetPath()) if err != nil { return err } // Encrypt the data into wrappedIn wrappedIn, err := d.cipher.EncryptData(streamer) if err != nil { return fmt.Errorf("failed to EncryptData: %w", err) } // doesn't support seekableStream, since rapid-upload is not working for encrypted data streamOut := &stream.FileStream{ Obj: &model.Object{ ID: streamer.GetID(), Path: streamer.GetPath(), Name: d.cipher.EncryptFileName(streamer.GetName()), Size: d.cipher.EncryptedSize(streamer.GetSize()), Modified: streamer.ModTime(), IsFolder: streamer.IsDir(), }, Reader: wrappedIn, Mimetype: "application/octet-stream", ForceStreamUpload: true, Exist: streamer.GetExist(), } return op.Put(ctx, remoteStorage, remoteActualPath, streamOut, up) } func (d *Crypt) GetDetails(ctx context.Context) (*model.StorageDetails, error) { remoteStorage, _, err := op.GetStorageAndActualPath(d.RemotePath) if err != nil { return nil, errs.NotImplement } remoteDetails, err := op.GetStorageDetails(ctx, remoteStorage) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: remoteDetails.DiskUsage, }, nil } var _ driver.Driver = (*Crypt)(nil) ================================================ FILE: drivers/crypt/meta.go ================================================ package crypt import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { FileNameEnc string `json:"filename_encryption" type:"select" required:"true" options:"off,standard,obfuscate" default:"off"` DirNameEnc string `json:"directory_name_encryption" type:"select" required:"true" options:"false,true" default:"false"` RemotePath string `json:"remote_path" required:"true" help:"This is where the encrypted data stores"` Password string `json:"password" required:"true" confidential:"true" help:"the main password"` Salt string `json:"salt" confidential:"true" help:"If you don't know what is salt, treat it as a second password. Optional but recommended"` EncryptedSuffix string `json:"encrypted_suffix" required:"true" default:".bin" help:"for advanced user only! encrypted files will have this suffix"` FileNameEncoding string `json:"filename_encoding" type:"select" required:"true" options:"base64,base32,base32768" default:"base64" help:"for advanced user only!"` Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` } var config = driver.Config{ Name: "Crypt", LocalSort: true, OnlyProxy: true, NoCache: true, DefaultRoot: "/", NoLinkURL: true, CheckStatus: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Crypt{} }) } ================================================ FILE: drivers/crypt/types.go ================================================ package crypt ================================================ FILE: drivers/crypt/util.go ================================================ package crypt import ( stdpath "path" "path/filepath" "strings" ) // will give the best guessing based on the path func guessPath(path string) (isFolder, secondTry bool) { if strings.HasSuffix(path, "/") { //confirmed a folder return true, false } lastSlash := strings.LastIndex(path, "/") if !strings.Contains(path[lastSlash:], ".") { //no dot, try folder then try file return true, true } return false, true } func (d *Crypt) encryptPath(path string, isFolder bool) string { if isFolder { return d.cipher.EncryptDirName(path) } dir, fileName := filepath.Split(path) return stdpath.Join(d.cipher.EncryptDirName(dir), d.cipher.EncryptFileName(fileName)) } ================================================ FILE: drivers/degoo/driver.go ================================================ package degoo import ( "context" "fmt" "net/http" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type Degoo struct { model.Storage Addition client *http.Client } func (d *Degoo) Config() driver.Config { return config } func (d *Degoo) GetAddition() driver.Additional { return &d.Addition } func (d *Degoo) Init(ctx context.Context) error { d.client = base.HttpClient // Ensure we have a valid token (will login if needed or refresh if expired) if err := d.ensureValidToken(ctx); err != nil { return fmt.Errorf("failed to initialize token: %w", err) } return d.getDevices(ctx) } func (d *Degoo) Drop(ctx context.Context) error { return nil } func (d *Degoo) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { items, err := d.getAllFileChildren5(ctx, dir.GetID()) if err != nil { return nil, err } return utils.MustSliceConvert(items, func(s DegooFileItem) model.Obj { isFolder := s.Category == 2 || s.Category == 1 || s.Category == 10 createTime, modTime, _ := humanReadableTimes(s.CreationTime, s.LastModificationTime, s.LastUploadTime) size, err := strconv.ParseInt(s.Size, 10, 64) if err != nil { size = 0 // Default to 0 if size parsing fails } return &model.Object{ ID: s.ID, Path: s.FilePath, Name: s.Name, Size: size, Modified: modTime, Ctime: createTime, IsFolder: isFolder, } }), nil } func (d *Degoo) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { item, err := d.getOverlay4(ctx, file.GetID()) if err != nil { return nil, err } return &model.Link{URL: item.URL}, nil } func (d *Degoo) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { // This is done by calling the setUploadFile3 API with a special checksum and size. const query = `mutation SetUploadFile3($Token: String!, $FileInfos: [FileInfoUpload3]!) { setUploadFile3(Token: $Token, FileInfos: $FileInfos) }` variables := map[string]interface{}{ "Token": d.AccessToken, "FileInfos": []map[string]interface{}{ { "Checksum": folderChecksum, "Name": dirName, "CreationTime": time.Now().UnixMilli(), "ParentID": parentDir.GetID(), "Size": 0, }, }, } _, err := d.apiCall(ctx, "SetUploadFile3", query, variables) if err != nil { return err } return nil } func (d *Degoo) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { const query = `mutation SetMoveFile($Token: String!, $Copy: Boolean, $NewParentID: String!, $FileIDs: [String]!) { setMoveFile(Token: $Token, Copy: $Copy, NewParentID: $NewParentID, FileIDs: $FileIDs) }` variables := map[string]interface{}{ "Token": d.AccessToken, "Copy": false, "NewParentID": dstDir.GetID(), "FileIDs": []string{srcObj.GetID()}, } _, err := d.apiCall(ctx, "SetMoveFile", query, variables) if err != nil { return nil, err } return srcObj, nil } func (d *Degoo) Rename(ctx context.Context, srcObj model.Obj, newName string) error { const query = `mutation SetRenameFile($Token: String!, $FileRenames: [FileRenameInfo]!) { setRenameFile(Token: $Token, FileRenames: $FileRenames) }` variables := map[string]interface{}{ "Token": d.AccessToken, "FileRenames": []DegooFileRenameInfo{ { ID: srcObj.GetID(), NewName: newName, }, }, } _, err := d.apiCall(ctx, "SetRenameFile", query, variables) if err != nil { return err } return nil } func (d *Degoo) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { // Copy is not implemented, Degoo API does not support direct copy. return nil, errs.NotImplement } func (d *Degoo) Remove(ctx context.Context, obj model.Obj) error { // Remove deletes a file or folder (moves to trash). const query = `mutation SetDeleteFile5($Token: String!, $IsInRecycleBin: Boolean!, $IDs: [IDType]!) { setDeleteFile5(Token: $Token, IsInRecycleBin: $IsInRecycleBin, IDs: $IDs) }` variables := map[string]interface{}{ "Token": d.AccessToken, "IsInRecycleBin": false, "IDs": []map[string]string{{"FileID": obj.GetID()}}, } _, err := d.apiCall(ctx, "SetDeleteFile5", query, variables) return err } func (d *Degoo) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { tmpF, err := file.CacheFullAndWriter(&up, nil) if err != nil { return err } parentID := dstDir.GetID() // Calculate the checksum for the file. checksum, err := d.checkSum(tmpF) if err != nil { return err } // 1. Get upload authorization via getBucketWriteAuth4. auths, err := d.getBucketWriteAuth4(ctx, file, parentID, checksum) if err != nil { return err } // 2. Upload file. // support rapid upload if auths.GetBucketWriteAuth4[0].Error != "Already exist!" { err = d.uploadS3(ctx, auths, tmpF, file, checksum) if err != nil { return err } } // 3. Register metadata with setUploadFile3. data, err := d.SetUploadFile3(ctx, file, parentID, checksum) if err != nil { return err } if !data.SetUploadFile3 { return fmt.Errorf("setUploadFile3 failed: %v", data) } return nil } func (d *Degoo) GetDetails(ctx context.Context) (*model.StorageDetails, error) { quota, err := d.getUserInfo(ctx) if err != nil { return nil, err } used, err := strconv.ParseInt(quota.GetUserInfo3.UsedQuota, 10, 64) if err != nil { return nil, fmt.Errorf("failed to parse used quota: %v", err) } total, err := strconv.ParseInt(quota.GetUserInfo3.TotalQuota, 10, 64) if err != nil { return nil, fmt.Errorf("failed to parse total quota: %v", err) } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: used, }, }, nil } ================================================ FILE: drivers/degoo/meta.go ================================================ package degoo import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootID Username string `json:"username" help:"Your Degoo account email"` Password string `json:"password" help:"Your Degoo account password"` RefreshToken string `json:"refresh_token" help:"Refresh token for automatic token renewal, obtained automatically"` AccessToken string `json:"access_token" help:"Access token for Degoo API, obtained automatically"` } var config = driver.Config{ Name: "Degoo", LocalSort: true, DefaultRoot: "0", NoOverwriteUpload: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Degoo{} }) } ================================================ FILE: drivers/degoo/types.go ================================================ package degoo import ( "encoding/json" ) // DegooLoginRequest represents the login request body. type DegooLoginRequest struct { GenerateToken bool `json:"GenerateToken"` Username string `json:"Username"` Password string `json:"Password"` } // DegooLoginResponse represents a successful login response. type DegooLoginResponse struct { Token string `json:"Token"` RefreshToken string `json:"RefreshToken"` } // DegooAccessTokenRequest represents the token refresh request body. type DegooAccessTokenRequest struct { RefreshToken string `json:"RefreshToken"` } // DegooAccessTokenResponse represents the token refresh response. type DegooAccessTokenResponse struct { AccessToken string `json:"AccessToken"` } // DegooFileItem represents a Degoo file or folder. type DegooFileItem struct { ID string `json:"ID"` ParentID string `json:"ParentID"` Name string `json:"Name"` Category int `json:"Category"` Size string `json:"Size"` URL string `json:"URL"` CreationTime string `json:"CreationTime"` LastModificationTime string `json:"LastModificationTime"` LastUploadTime string `json:"LastUploadTime"` MetadataID string `json:"MetadataID"` DeviceID int64 `json:"DeviceID"` FilePath string `json:"FilePath"` IsInRecycleBin bool `json:"IsInRecycleBin"` } type DegooErrors struct { Path []string `json:"path"` Data interface{} `json:"data"` ErrorType string `json:"errorType"` ErrorInfo interface{} `json:"errorInfo"` Message string `json:"message"` } // DegooGraphqlResponse is the common structure for GraphQL API responses. type DegooGraphqlResponse struct { Data json.RawMessage `json:"data"` Errors []DegooErrors `json:"errors,omitempty"` } // DegooGetChildren5Data is the data field for getFileChildren5. type DegooGetChildren5Data struct { GetFileChildren5 struct { Items []DegooFileItem `json:"Items"` NextToken string `json:"NextToken"` } `json:"getFileChildren5"` } // DegooGetOverlay4Data is the data field for getOverlay4. type DegooGetOverlay4Data struct { GetOverlay4 DegooFileItem `json:"getOverlay4"` } // DegooFileRenameInfo represents a file rename operation. type DegooFileRenameInfo struct { ID string `json:"ID"` NewName string `json:"NewName"` } // DegooFileIDs represents a list of file IDs for move operations. type DegooFileIDs struct { FileIDs []string `json:"FileIDs"` } // DegooGetBucketWriteAuth4Data is the data field for GetBucketWriteAuth4. type DegooGetBucketWriteAuth4Data struct { GetBucketWriteAuth4 []struct { AuthData struct { PolicyBase64 string `json:"PolicyBase64"` Signature string `json:"Signature"` BaseURL string `json:"BaseURL"` KeyPrefix string `json:"KeyPrefix"` AccessKey struct { Key string `json:"Key"` Value string `json:"Value"` } `json:"AccessKey"` ACL string `json:"ACL"` AdditionalBody []struct { Key string `json:"Key"` Value string `json:"Value"` } `json:"AdditionalBody"` } `json:"AuthData"` Error interface{} `json:"Error"` } `json:"getBucketWriteAuth4"` } // DegooSetUploadFile3Data is the data field for SetUploadFile3. type DegooSetUploadFile3Data struct { SetUploadFile3 bool `json:"setUploadFile3"` } type DegooGetUserInfo3Data struct { GetUserInfo3 struct { // ID string // FirstName string // LastName string // Email string // AvatarURL string // CountryCode string = CN // LanguageCode string = zh-cn // Phone string // AccountType int UsedQuota string `json:"UsedQuota"` TotalQuota string `json:"TotalQuota"` // OAuth2Provider // GPMigrationStatus int // FeatureNoAds bool // FeatureTopSecret bool // FeatureDownsampling bool // FeatureAutomaticVideoUploads bool // FileSizeLimit string } `json:"getUserInfo3"` } ================================================ FILE: drivers/degoo/upload.go ================================================ package degoo import ( "bytes" "context" "crypto/sha1" "encoding/base64" "encoding/json" "fmt" "io" "mime/multipart" "net/http" "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) func (d *Degoo) getBucketWriteAuth4(ctx context.Context, file model.FileStreamer, parentID string, checksum string) (*DegooGetBucketWriteAuth4Data, error) { const query = `query GetBucketWriteAuth4( $Token: String! $ParentID: String! $StorageUploadInfos: [StorageUploadInfo2] ) { getBucketWriteAuth4( Token: $Token ParentID: $ParentID StorageUploadInfos: $StorageUploadInfos ) { AuthData { PolicyBase64 Signature BaseURL KeyPrefix AccessKey { Key Value } ACL AdditionalBody { Key Value } } Error } }` variables := map[string]interface{}{ "Token": d.AccessToken, "ParentID": parentID, "StorageUploadInfos": []map[string]string{{ "FileName": file.GetName(), "Checksum": checksum, "Size": strconv.FormatInt(file.GetSize(), 10), }}} data, err := d.apiCall(ctx, "GetBucketWriteAuth4", query, variables) if err != nil { return nil, err } var resp DegooGetBucketWriteAuth4Data err = json.Unmarshal(data, &resp) if err != nil { return nil, err } return &resp, nil } // checkSum calculates the SHA1-based checksum for Degoo upload API. func (d *Degoo) checkSum(file io.Reader) (string, error) { seed := []byte{13, 7, 2, 2, 15, 40, 75, 117, 13, 10, 19, 16, 29, 23, 3, 36} hasher := sha1.New() hasher.Write(seed) if _, err := utils.CopyWithBuffer(hasher, file); err != nil { return "", err } cs := hasher.Sum(nil) csBytes := []byte{10, byte(len(cs))} csBytes = append(csBytes, cs...) csBytes = append(csBytes, 16, 0) return strings.ReplaceAll(base64.StdEncoding.EncodeToString(csBytes), "/", "_"), nil } func (d *Degoo) uploadS3(ctx context.Context, auths *DegooGetBucketWriteAuth4Data, tmpF model.File, file model.FileStreamer, checksum string) error { a := auths.GetBucketWriteAuth4[0].AuthData _, err := tmpF.Seek(0, io.SeekStart) if err != nil { return err } ext := utils.Ext(file.GetName()) key := fmt.Sprintf("%s%s/%s.%s", a.KeyPrefix, ext, checksum, ext) var b bytes.Buffer w := multipart.NewWriter(&b) err = w.WriteField("key", key) if err != nil { return err } err = w.WriteField("acl", a.ACL) if err != nil { return err } err = w.WriteField("policy", a.PolicyBase64) if err != nil { return err } err = w.WriteField("signature", a.Signature) if err != nil { return err } err = w.WriteField(a.AccessKey.Key, a.AccessKey.Value) if err != nil { return err } for _, additional := range a.AdditionalBody { err = w.WriteField(additional.Key, additional.Value) if err != nil { return err } } err = w.WriteField("Content-Type", "") if err != nil { return err } _, err = w.CreateFormFile("file", key) if err != nil { return err } headSize := b.Len() err = w.Close() if err != nil { return err } head := bytes.NewReader(b.Bytes()[:headSize]) tail := bytes.NewReader(b.Bytes()[headSize:]) rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, tmpF, tail)) req, err := http.NewRequestWithContext(ctx, http.MethodPost, a.BaseURL, rateLimitedRd) if err != nil { return err } req.Header.Add("ngsw-bypass", "1") req.Header.Add("Content-Type", w.FormDataContentType()) res, err := d.client.Do(req) if err != nil { return err } defer res.Body.Close() if res.StatusCode != http.StatusNoContent { return fmt.Errorf("upload failed with status code %d", res.StatusCode) } return nil } var _ driver.Driver = (*Degoo)(nil) func (d *Degoo) SetUploadFile3(ctx context.Context, file model.FileStreamer, parentID string, checksum string) (*DegooSetUploadFile3Data, error) { const query = `mutation SetUploadFile3($Token: String!, $FileInfos: [FileInfoUpload3]!) { setUploadFile3(Token: $Token, FileInfos: $FileInfos) }` variables := map[string]interface{}{ "Token": d.AccessToken, "FileInfos": []map[string]string{{ "Checksum": checksum, "CreationTime": strconv.FormatInt(file.CreateTime().UnixMilli(), 10), "Name": file.GetName(), "ParentID": parentID, "Size": strconv.FormatInt(file.GetSize(), 10), }}} data, err := d.apiCall(ctx, "SetUploadFile3", query, variables) if err != nil { return nil, err } var resp DegooSetUploadFile3Data err = json.Unmarshal(data, &resp) if err != nil { return nil, err } return &resp, nil } ================================================ FILE: drivers/degoo/util.go ================================================ package degoo import ( "bytes" "context" "encoding/base64" "encoding/json" "fmt" "net/http" "strconv" "strings" "sync" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/op" ) // Thanks to https://github.com/bernd-wechner/Degoo for API research. const ( // API endpoints loginURL = "https://rest-api.degoo.com/login" accessTokenURL = "https://rest-api.degoo.com/access-token/v2" apiURL = "https://production-appsync.degoo.com/graphql" // API configuration apiKey = "da2-vs6twz5vnjdavpqndtbzg3prra" folderChecksum = "CgAQAg" // Token management tokenRefreshThreshold = 5 * time.Minute // Rate limiting minRequestInterval = 1 * time.Second // Error messages errRateLimited = "rate limited (429), please try again later" errUnauthorized = "unauthorized access" ) var ( // Global rate limiting - protects against concurrent API calls lastRequestTime time.Time requestMutex sync.Mutex ) // JWT payload structure for token expiration checking type JWTPayload struct { UserID string `json:"userID"` Exp int64 `json:"exp"` Iat int64 `json:"iat"` } // Rate limiting helper functions // applyRateLimit ensures minimum interval between API requests func applyRateLimit() { requestMutex.Lock() defer requestMutex.Unlock() if !lastRequestTime.IsZero() { if elapsed := time.Since(lastRequestTime); elapsed < minRequestInterval { time.Sleep(minRequestInterval - elapsed) } } lastRequestTime = time.Now() } // HTTP request helper functions // createJSONRequest creates a new HTTP request with JSON body func createJSONRequest(ctx context.Context, method, url string, body interface{}) (*http.Request, error) { jsonBody, err := json.Marshal(body) if err != nil { return nil, fmt.Errorf("failed to marshal request body: %w", err) } req, err := http.NewRequestWithContext(ctx, method, url, bytes.NewBuffer(jsonBody)) if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) } req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", base.UserAgent) return req, nil } // checkHTTPResponse checks for common HTTP error conditions func checkHTTPResponse(resp *http.Response, operation string) error { if resp.StatusCode == http.StatusTooManyRequests { return fmt.Errorf("%s %s", operation, errRateLimited) } if resp.StatusCode != http.StatusOK { return fmt.Errorf("%s failed: %s", operation, resp.Status) } return nil } // isTokenExpired checks if the JWT token is expired or will expire soon func (d *Degoo) isTokenExpired() bool { if d.AccessToken == "" { return true } payload, err := extractJWTPayload(d.AccessToken) if err != nil { return true // Invalid token format } // Check if token expires within the threshold expireTime := time.Unix(payload.Exp, 0) return time.Now().Add(tokenRefreshThreshold).After(expireTime) } // extractJWTPayload extracts and parses JWT payload func extractJWTPayload(token string) (*JWTPayload, error) { parts := strings.Split(token, ".") if len(parts) != 3 { return nil, fmt.Errorf("invalid JWT format") } // Decode the payload (second part) payload, err := base64.RawURLEncoding.DecodeString(parts[1]) if err != nil { return nil, fmt.Errorf("failed to decode JWT payload: %w", err) } var jwtPayload JWTPayload if err := json.Unmarshal(payload, &jwtPayload); err != nil { return nil, fmt.Errorf("failed to parse JWT payload: %w", err) } return &jwtPayload, nil } // refreshToken attempts to refresh the access token using the refresh token func (d *Degoo) refreshToken(ctx context.Context) error { if d.RefreshToken == "" { return fmt.Errorf("no refresh token available") } // Create request tokenReq := DegooAccessTokenRequest{RefreshToken: d.RefreshToken} req, err := createJSONRequest(ctx, "POST", accessTokenURL, tokenReq) if err != nil { return fmt.Errorf("failed to create refresh token request: %w", err) } // Execute request resp, err := d.client.Do(req) if err != nil { return fmt.Errorf("refresh token request failed: %w", err) } defer resp.Body.Close() // Check response if err := checkHTTPResponse(resp, "refresh token"); err != nil { return err } var accessTokenResp DegooAccessTokenResponse if err := json.NewDecoder(resp.Body).Decode(&accessTokenResp); err != nil { return fmt.Errorf("failed to parse access token response: %w", err) } if accessTokenResp.AccessToken == "" { return fmt.Errorf("empty access token received") } d.AccessToken = accessTokenResp.AccessToken // Save the updated token to storage op.MustSaveDriverStorage(d) return nil } // ensureValidToken ensures we have a valid, non-expired token func (d *Degoo) ensureValidToken(ctx context.Context) error { // Check if token is expired or will expire soon if d.isTokenExpired() { // Try to refresh token first if we have a refresh token if d.RefreshToken != "" { if refreshErr := d.refreshToken(ctx); refreshErr == nil { return nil // Successfully refreshed } else { // If refresh failed, fall back to full login fmt.Printf("Token refresh failed, falling back to full login: %v\n", refreshErr) } } // Perform full login if d.Username != "" && d.Password != "" { return d.login(ctx) } } return nil } // login performs the login process and retrieves the access token. func (d *Degoo) login(ctx context.Context) error { if d.Username == "" || d.Password == "" { return fmt.Errorf("username or password not provided") } creds := DegooLoginRequest{ GenerateToken: true, Username: d.Username, Password: d.Password, } jsonCreds, err := json.Marshal(creds) if err != nil { return fmt.Errorf("failed to serialize login credentials: %w", err) } req, err := http.NewRequestWithContext(ctx, "POST", loginURL, bytes.NewBuffer(jsonCreds)) if err != nil { return fmt.Errorf("failed to create login request: %w", err) } req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", base.UserAgent) req.Header.Set("Origin", "https://app.degoo.com") resp, err := d.client.Do(req) if err != nil { return fmt.Errorf("login request failed: %w", err) } defer resp.Body.Close() // Handle rate limiting (429 Too Many Requests) if resp.StatusCode == http.StatusTooManyRequests { return fmt.Errorf("login rate limited (429), please try again later") } if resp.StatusCode != http.StatusOK { return fmt.Errorf("login failed: %s", resp.Status) } var loginResp DegooLoginResponse if err := json.NewDecoder(resp.Body).Decode(&loginResp); err != nil { return fmt.Errorf("failed to parse login response: %w", err) } if loginResp.RefreshToken != "" { tokenReq := DegooAccessTokenRequest{RefreshToken: loginResp.RefreshToken} jsonTokenReq, err := json.Marshal(tokenReq) if err != nil { return fmt.Errorf("failed to serialize access token request: %w", err) } tokenReqHTTP, err := http.NewRequestWithContext(ctx, "POST", accessTokenURL, bytes.NewBuffer(jsonTokenReq)) if err != nil { return fmt.Errorf("failed to create access token request: %w", err) } tokenReqHTTP.Header.Set("User-Agent", base.UserAgent) tokenResp, err := d.client.Do(tokenReqHTTP) if err != nil { return fmt.Errorf("failed to get access token: %w", err) } defer tokenResp.Body.Close() var accessTokenResp DegooAccessTokenResponse if err := json.NewDecoder(tokenResp.Body).Decode(&accessTokenResp); err != nil { return fmt.Errorf("failed to parse access token response: %w", err) } d.AccessToken = accessTokenResp.AccessToken d.RefreshToken = loginResp.RefreshToken // Save refresh token } else if loginResp.Token != "" { d.AccessToken = loginResp.Token d.RefreshToken = "" // Direct token, no refresh token available } else { return fmt.Errorf("login failed, no valid token returned") } // Save the updated tokens to storage op.MustSaveDriverStorage(d) return nil } // apiCall performs a Degoo GraphQL API request. func (d *Degoo) apiCall(ctx context.Context, operationName, query string, variables map[string]interface{}) (json.RawMessage, error) { // Apply rate limiting applyRateLimit() // Ensure we have a valid token before making the API call if err := d.ensureValidToken(ctx); err != nil { return nil, fmt.Errorf("failed to ensure valid token: %w", err) } // Update the Token in variables if it exists (after potential refresh) d.updateTokenInVariables(variables) return d.executeGraphQLRequest(ctx, operationName, query, variables) } // updateTokenInVariables updates the Token field in GraphQL variables func (d *Degoo) updateTokenInVariables(variables map[string]interface{}) { if variables != nil { if _, hasToken := variables["Token"]; hasToken { variables["Token"] = d.AccessToken } } } // executeGraphQLRequest executes a GraphQL request with retry logic func (d *Degoo) executeGraphQLRequest(ctx context.Context, operationName, query string, variables map[string]interface{}) (json.RawMessage, error) { reqBody := map[string]interface{}{ "operationName": operationName, "query": query, "variables": variables, } // Create and configure request req, err := createJSONRequest(ctx, "POST", apiURL, reqBody) if err != nil { return nil, err } // Set Degoo-specific headers req.Header.Set("x-api-key", apiKey) if d.AccessToken != "" { req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", d.AccessToken)) } // Execute request resp, err := d.client.Do(req) if err != nil { return nil, fmt.Errorf("GraphQL API request failed: %w", err) } defer resp.Body.Close() // Check for HTTP errors if err := checkHTTPResponse(resp, "GraphQL API"); err != nil { return nil, err } // Parse GraphQL response var degooResp DegooGraphqlResponse if err := json.NewDecoder(resp.Body).Decode(°ooResp); err != nil { return nil, fmt.Errorf("failed to decode GraphQL response: %w", err) } // Handle GraphQL errors if len(degooResp.Errors) > 0 { return d.handleGraphQLError(ctx, degooResp.Errors[0], operationName, query, variables) } return degooResp.Data, nil } // handleGraphQLError handles GraphQL-level errors with retry logic func (d *Degoo) handleGraphQLError(ctx context.Context, gqlError DegooErrors, operationName, query string, variables map[string]interface{}) (json.RawMessage, error) { if gqlError.ErrorType == "Unauthorized" { // Re-login and retry if err := d.login(ctx); err != nil { return nil, fmt.Errorf("%s, login failed: %w", errUnauthorized, err) } // Update token in variables and retry d.updateTokenInVariables(variables) return d.apiCall(ctx, operationName, query, variables) } return nil, fmt.Errorf("GraphQL API error: %s", gqlError.Message) } // humanReadableTimes converts Degoo timestamps to Go time.Time. func humanReadableTimes(creation, modification, upload string) (cTime, mTime, uTime time.Time) { cTime, _ = time.Parse(time.RFC3339, creation) if modification != "" { modMillis, _ := strconv.ParseInt(modification, 10, 64) mTime = time.Unix(0, modMillis*int64(time.Millisecond)) } if upload != "" { upMillis, _ := strconv.ParseInt(upload, 10, 64) uTime = time.Unix(0, upMillis*int64(time.Millisecond)) } return cTime, mTime, uTime } // getDevices fetches and caches top-level devices and folders. func (d *Degoo) getDevices(ctx context.Context) error { const query = `query GetFileChildren5($Token: String! $ParentID: String $AllParentIDs: [String] $Limit: Int! $Order: Int! $NextToken: String ) { getFileChildren5(Token: $Token ParentID: $ParentID AllParentIDs: $AllParentIDs Limit: $Limit Order: $Order NextToken: $NextToken) { Items { ParentID } NextToken } }` variables := map[string]interface{}{ "Token": d.AccessToken, "ParentID": "0", "Limit": 10, "Order": 3, } data, err := d.apiCall(ctx, "GetFileChildren5", query, variables) if err != nil { return err } var resp DegooGetChildren5Data if err := json.Unmarshal(data, &resp); err != nil { return fmt.Errorf("failed to parse device list: %w", err) } if d.RootFolderID == "0" { if len(resp.GetFileChildren5.Items) > 0 { d.RootFolderID = resp.GetFileChildren5.Items[0].ParentID } op.MustSaveDriverStorage(d) } return nil } // getAllFileChildren5 fetches all children of a directory with pagination. func (d *Degoo) getAllFileChildren5(ctx context.Context, parentID string) ([]DegooFileItem, error) { const query = `query GetFileChildren5($Token: String! $ParentID: String $AllParentIDs: [String] $Limit: Int! $Order: Int! $NextToken: String ) { getFileChildren5(Token: $Token ParentID: $ParentID AllParentIDs: $AllParentIDs Limit: $Limit Order: $Order NextToken: $NextToken) { Items { ID ParentID Name Category Size CreationTime LastModificationTime LastUploadTime FilePath IsInRecycleBin DeviceID MetadataID } NextToken } }` var allItems []DegooFileItem nextToken := "" for { variables := map[string]interface{}{ "Token": d.AccessToken, "ParentID": parentID, "Limit": 1000, "Order": 3, } if nextToken != "" { variables["NextToken"] = nextToken } data, err := d.apiCall(ctx, "GetFileChildren5", query, variables) if err != nil { return nil, err } var resp DegooGetChildren5Data if err := json.Unmarshal(data, &resp); err != nil { return nil, err } allItems = append(allItems, resp.GetFileChildren5.Items...) if resp.GetFileChildren5.NextToken == "" { break } nextToken = resp.GetFileChildren5.NextToken } return allItems, nil } // getOverlay4 fetches metadata for a single item by ID. func (d *Degoo) getOverlay4(ctx context.Context, id string) (DegooFileItem, error) { const query = `query GetOverlay4($Token: String!, $ID: IDType!) { getOverlay4(Token: $Token, ID: $ID) { ID ParentID Name Category Size CreationTime LastModificationTime LastUploadTime URL FilePath IsInRecycleBin DeviceID MetadataID } }` variables := map[string]interface{}{ "Token": d.AccessToken, "ID": map[string]string{ "FileID": id, }, } data, err := d.apiCall(ctx, "GetOverlay4", query, variables) if err != nil { return DegooFileItem{}, err } var resp DegooGetOverlay4Data if err := json.Unmarshal(data, &resp); err != nil { return DegooFileItem{}, fmt.Errorf("failed to parse item metadata: %w", err) } return resp.GetOverlay4, nil } func (d *Degoo) getUserInfo(ctx context.Context) (DegooGetUserInfo3Data, error) { const query = "query GetUserInfo3($Token: String!) { getUserInfo3(Token: $Token) { UsedQuota TotalQuota } }" variables := map[string]interface{}{ "Token": d.AccessToken, } data, err := d.apiCall(ctx, "GetUserInfo3", query, variables) var resp DegooGetUserInfo3Data if err != nil { return resp, err } if err = json.Unmarshal(data, &resp); err != nil { return resp, fmt.Errorf("failed to parse user info: %w", err) } return resp, nil } ================================================ FILE: drivers/doubao/driver.go ================================================ package doubao import ( "context" "errors" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" "github.com/google/uuid" "golang.org/x/time/rate" ) type Doubao struct { model.Storage Addition *UploadToken UserId string uploadThread int limiter *rate.Limiter } func (d *Doubao) Config() driver.Config { return config } func (d *Doubao) GetAddition() driver.Additional { return &d.Addition } func (d *Doubao) Init(ctx context.Context) error { // TODO login / refresh token //op.MustSaveDriverStorage(d) uploadThread, err := strconv.Atoi(d.UploadThread) if err != nil || uploadThread < 1 { d.uploadThread, d.UploadThread = 3, "3" // Set default value } else { d.uploadThread = uploadThread } if d.UserId == "" { userInfo, err := d.getUserInfo() if err != nil { return err } d.UserId = strconv.FormatInt(userInfo.UserID, 10) } if d.UploadToken == nil { uploadToken, err := d.initUploadToken() if err != nil { return err } d.UploadToken = uploadToken } if d.LimitRate > 0 { d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) } return nil } func (d *Doubao) WaitLimit(ctx context.Context) error { if d.limiter != nil { return d.limiter.Wait(ctx) } return nil } func (d *Doubao) Drop(ctx context.Context) error { return nil } func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } var files []model.Obj fileList, err := d.getFiles(dir.GetID(), "") if err != nil { return nil, err } for _, child := range fileList { files = append(files, &Object{ Object: model.Object{ ID: child.ID, Path: child.ParentID, Name: child.Name, Size: child.Size, Modified: time.Unix(child.UpdateTime, 0), Ctime: time.Unix(child.CreateTime, 0), IsFolder: child.NodeType == 1, }, Key: child.Key, NodeType: child.NodeType, }) } return files, nil } func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } var downloadUrl string if u, ok := file.(*Object); ok { switch d.DownloadApi { case "get_download_info": var r GetDownloadInfoResp _, err := d.request("/samantha/aispace/get_download_info", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "requests": []base.Json{{"node_id": file.GetID()}}, }) }, &r) if err != nil { return nil, err } downloadUrl = r.Data.DownloadInfos[0].MainURL case "get_file_url": switch u.NodeType { case VideoType, AudioType: var r GetVideoFileUrlResp _, err := d.request("/samantha/media/get_play_info", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "key": u.Key, "node_id": file.GetID(), }) }, &r) if err != nil { return nil, err } downloadUrl = r.Data.OriginalMediaInfo.MainURL default: var r GetFileUrlResp _, err := d.request("/alice/message/get_file_url", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "uris": []string{u.Key}, "type": FileNodeType[u.NodeType], }) }, &r) if err != nil { return nil, err } downloadUrl = r.Data.FileUrls[0].MainURL } default: return nil, errs.NotImplement } // 生成标准的Content-Disposition contentDisposition := utils.GenerateContentDisposition(u.Name) return &model.Link{ URL: downloadUrl, Header: http.Header{ "User-Agent": []string{UserAgent}, "Content-Disposition": []string{contentDisposition}, }, }, nil } return nil, errors.New("can't convert obj to URL") } func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { if err := d.WaitLimit(ctx); err != nil { return err } var r UploadNodeResp _, err := d.request("/samantha/aispace/upload_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "node_list": []base.Json{ { "local_id": uuid.New().String(), "name": dirName, "parent_id": parentDir.GetID(), "node_type": 1, }, }, }) }, &r) return err } func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error { if err := d.WaitLimit(ctx); err != nil { return err } var r UploadNodeResp _, err := d.request("/samantha/aispace/move_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "node_list": []base.Json{ {"id": srcObj.GetID()}, }, "current_parent_id": srcObj.GetPath(), "target_parent_id": dstDir.GetID(), }) }, &r) return err } func (d *Doubao) Rename(ctx context.Context, srcObj model.Obj, newName string) error { if err := d.WaitLimit(ctx); err != nil { return err } var r BaseResp _, err := d.request("/samantha/aispace/rename_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "node_id": srcObj.GetID(), "node_name": newName, }) }, &r) return err } func (d *Doubao) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { // TODO copy obj, optional return nil, errs.NotImplement } func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error { if err := d.WaitLimit(ctx); err != nil { return err } var r BaseResp _, err := d.request("/samantha/aispace/delete_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{"node_list": []base.Json{{"id": obj.GetID()}}}) }, &r) return err } func (d *Doubao) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err } // 根据MIME类型确定数据类型 mimetype := file.GetMimetype() dataType := FileDataType switch { case strings.HasPrefix(mimetype, "video/"): dataType = VideoDataType case strings.HasPrefix(mimetype, "audio/"): dataType = VideoDataType // 音频与视频使用相同的处理方式 case strings.HasPrefix(mimetype, "image/"): dataType = ImgDataType } // 获取上传配置 uploadConfig := UploadConfig{} if err := d.getUploadConfig(&uploadConfig, dataType, file); err != nil { return nil, err } // 根据文件大小选择上传方式 if file.GetSize() <= 1*utils.MB { // 小于1MB，使用普通模式上传 return d.Upload(ctx, &uploadConfig, dstDir, file, up, dataType) } // 大文件使用分片上传 return d.UploadByMultipart(ctx, &uploadConfig, file.GetSize(), dstDir, file, up, dataType) } func (d *Doubao) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *Doubao) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *Doubao) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *Doubao) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir // return errs.NotImplement to use an internal archive tool return nil, errs.NotImplement } //func (d *Doubao) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*Doubao)(nil) ================================================ FILE: drivers/doubao/meta.go ================================================ package doubao import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { // Usually one of two // driver.RootPath driver.RootID // define other Cookie string `json:"cookie" type:"text"` UploadThread string `json:"upload_thread" default:"3"` DownloadApi string `json:"download_api" type:"select" options:"get_file_url,get_download_info" default:"get_file_url"` LimitRate float64 `json:"limit_rate" type:"float" default:"2" help:"limit all api request rate ([limit]r/1s)"` } var config = driver.Config{ Name: "Doubao", LocalSort: true, DefaultRoot: "0", } func init() { op.RegisterDriver(func() driver.Driver { return &Doubao{ Addition: Addition{ LimitRate: 2, }, } }) } ================================================ FILE: drivers/doubao/types.go ================================================ package doubao import ( "encoding/json" "fmt" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type BaseResp struct { Code int `json:"code"` Msg string `json:"msg"` } type NodeInfoResp struct { BaseResp Data struct { NodeInfo File `json:"node_info"` Children []File `json:"children"` NextCursor string `json:"next_cursor"` HasMore bool `json:"has_more"` } `json:"data"` } type File struct { ID string `json:"id"` Name string `json:"name"` Key string `json:"key"` NodeType int `json:"node_type"` // 0: 文件, 1: 文件夹 Size int64 `json:"size"` Source int `json:"source"` NameReviewStatus int `json:"name_review_status"` ContentReviewStatus int `json:"content_review_status"` RiskReviewStatus int `json:"risk_review_status"` ConversationID string `json:"conversation_id"` ParentID string `json:"parent_id"` CreateTime int64 `json:"create_time"` UpdateTime int64 `json:"update_time"` } type GetDownloadInfoResp struct { BaseResp Data struct { DownloadInfos []struct { NodeID string `json:"node_id"` MainURL string `json:"main_url"` BackupURL string `json:"backup_url"` } `json:"download_infos"` } `json:"data"` } type GetFileUrlResp struct { BaseResp Data struct { FileUrls []struct { URI string `json:"uri"` MainURL string `json:"main_url"` BackURL string `json:"back_url"` } `json:"file_urls"` } `json:"data"` } type GetVideoFileUrlResp struct { BaseResp Data struct { MediaType string `json:"media_type"` MediaInfo []struct { Meta struct { Height string `json:"height"` Width string `json:"width"` Format string `json:"format"` Duration float64 `json:"duration"` CodecType string `json:"codec_type"` Definition string `json:"definition"` } `json:"meta"` MainURL string `json:"main_url"` BackupURL string `json:"backup_url"` } `json:"media_info"` OriginalMediaInfo struct { Meta struct { Height string `json:"height"` Width string `json:"width"` Format string `json:"format"` Duration float64 `json:"duration"` CodecType string `json:"codec_type"` Definition string `json:"definition"` } `json:"meta"` MainURL string `json:"main_url"` BackupURL string `json:"backup_url"` } `json:"original_media_info"` PosterURL string `json:"poster_url"` PlayableStatus int `json:"playable_status"` } `json:"data"` } type UploadNodeResp struct { BaseResp Data struct { NodeList []struct { LocalID string `json:"local_id"` ID string `json:"id"` ParentID string `json:"parent_id"` Name string `json:"name"` Key string `json:"key"` NodeType int `json:"node_type"` // 0: 文件, 1: 文件夹 } `json:"node_list"` } `json:"data"` } type Object struct { model.Object Key string NodeType int } type UserInfoResp struct { Data UserInfo `json:"data"` Message string `json:"message"` } type AppUserInfo struct { BuiAuditInfo string `json:"bui_audit_info"` } type AuditInfo struct { } type Details struct { } type BuiAuditInfo struct { AuditInfo AuditInfo `json:"audit_info"` IsAuditing bool `json:"is_auditing"` AuditStatus int `json:"audit_status"` LastUpdateTime int64 `json:"last_update_time"` UnpassReason string `json:"unpass_reason"` Details Details `json:"details"` } type Connects struct { Platform string `json:"platform"` ProfileImageURL string `json:"profile_image_url"` ExpiredTime int `json:"expired_time"` ExpiresIn int `json:"expires_in"` PlatformScreenName string `json:"platform_screen_name"` UserID int64 `json:"user_id"` PlatformUID string `json:"platform_uid"` SecPlatformUID string `json:"sec_platform_uid"` PlatformAppID int `json:"platform_app_id"` ModifyTime int `json:"modify_time"` AccessToken string `json:"access_token"` OpenID string `json:"open_id"` } type OperStaffRelationInfo struct { HasPassword int `json:"has_password"` Mobile string `json:"mobile"` SecOperStaffUserID string `json:"sec_oper_staff_user_id"` RelationMobileCountryCode int `json:"relation_mobile_country_code"` } type UserInfo struct { AppID int `json:"app_id"` AppUserInfo AppUserInfo `json:"app_user_info"` AvatarURL string `json:"avatar_url"` BgImgURL string `json:"bg_img_url"` BuiAuditInfo BuiAuditInfo `json:"bui_audit_info"` CanBeFoundByPhone int `json:"can_be_found_by_phone"` Connects []Connects `json:"connects"` CountryCode int `json:"country_code"` Description string `json:"description"` DeviceID int `json:"device_id"` Email string `json:"email"` EmailCollected bool `json:"email_collected"` Gender int `json:"gender"` HasPassword int `json:"has_password"` HmRegion int `json:"hm_region"` IsBlocked int `json:"is_blocked"` IsBlocking int `json:"is_blocking"` IsRecommendAllowed int `json:"is_recommend_allowed"` IsVisitorAccount bool `json:"is_visitor_account"` Mobile string `json:"mobile"` Name string `json:"name"` NeedCheckBindStatus bool `json:"need_check_bind_status"` OdinUserType int `json:"odin_user_type"` OperStaffRelationInfo OperStaffRelationInfo `json:"oper_staff_relation_info"` PhoneCollected bool `json:"phone_collected"` RecommendHintMessage string `json:"recommend_hint_message"` ScreenName string `json:"screen_name"` SecUserID string `json:"sec_user_id"` SessionKey string `json:"session_key"` UseHmRegion bool `json:"use_hm_region"` UserCreateTime int64 `json:"user_create_time"` UserID int64 `json:"user_id"` UserIDStr string `json:"user_id_str"` UserVerified bool `json:"user_verified"` VerifiedContent string `json:"verified_content"` } // UploadToken 上传令牌配置 type UploadToken struct { Alice map[string]UploadAuthToken Samantha MediaUploadAuthToken } // UploadAuthToken 多种类型的上传配置：图片/文件 type UploadAuthToken struct { ServiceID string `json:"service_id"` UploadPathPrefix string `json:"upload_path_prefix"` Auth struct { AccessKeyID string `json:"access_key_id"` SecretAccessKey string `json:"secret_access_key"` SessionToken string `json:"session_token"` ExpiredTime time.Time `json:"expired_time"` CurrentTime time.Time `json:"current_time"` } `json:"auth"` UploadHost string `json:"upload_host"` } // MediaUploadAuthToken 媒体上传配置 type MediaUploadAuthToken struct { StsToken struct { AccessKeyID string `json:"access_key_id"` SecretAccessKey string `json:"secret_access_key"` SessionToken string `json:"session_token"` ExpiredTime time.Time `json:"expired_time"` CurrentTime time.Time `json:"current_time"` } `json:"sts_token"` UploadInfo struct { VideoHost string `json:"video_host"` SpaceName string `json:"space_name"` } `json:"upload_info"` } type UploadAuthTokenResp struct { BaseResp Data UploadAuthToken `json:"data"` } type MediaUploadAuthTokenResp struct { BaseResp Data MediaUploadAuthToken `json:"data"` } type ResponseMetadata struct { RequestID string `json:"RequestId"` Action string `json:"Action"` Version string `json:"Version"` Service string `json:"Service"` Region string `json:"Region"` Error struct { CodeN int `json:"CodeN,omitempty"` Code string `json:"Code,omitempty"` Message string `json:"Message,omitempty"` } `json:"Error,omitempty"` } type UploadConfig struct { UploadAddress UploadAddress `json:"UploadAddress"` FallbackUploadAddress FallbackUploadAddress `json:"FallbackUploadAddress"` InnerUploadAddress InnerUploadAddress `json:"InnerUploadAddress"` RequestID string `json:"RequestId"` SDKParam interface{} `json:"SDKParam"` } type UploadConfigResp struct { ResponseMetadata `json:"ResponseMetadata"` Result UploadConfig `json:"Result"` } // StoreInfo 存储信息 type StoreInfo struct { StoreURI string `json:"StoreUri"` Auth string `json:"Auth"` UploadID string `json:"UploadID"` UploadHeader map[string]interface{} `json:"UploadHeader,omitempty"` StorageHeader map[string]interface{} `json:"StorageHeader,omitempty"` } // UploadAddress 上传地址信息 type UploadAddress struct { StoreInfos []StoreInfo `json:"StoreInfos"` UploadHosts []string `json:"UploadHosts"` UploadHeader map[string]interface{} `json:"UploadHeader"` SessionKey string `json:"SessionKey"` Cloud string `json:"Cloud"` } // FallbackUploadAddress 备用上传地址 type FallbackUploadAddress struct { StoreInfos []StoreInfo `json:"StoreInfos"` UploadHosts []string `json:"UploadHosts"` UploadHeader map[string]interface{} `json:"UploadHeader"` SessionKey string `json:"SessionKey"` Cloud string `json:"Cloud"` } // UploadNode 上传节点信息 type UploadNode struct { Vid string `json:"Vid"` Vids []string `json:"Vids"` StoreInfos []StoreInfo `json:"StoreInfos"` UploadHost string `json:"UploadHost"` UploadHeader map[string]interface{} `json:"UploadHeader"` Type string `json:"Type"` Protocol string `json:"Protocol"` SessionKey string `json:"SessionKey"` NodeConfig struct { UploadMode string `json:"UploadMode"` } `json:"NodeConfig"` Cluster string `json:"Cluster"` } // AdvanceOption 高级选项 type AdvanceOption struct { Parallel int `json:"Parallel"` Stream int `json:"Stream"` SliceSize int `json:"SliceSize"` EncryptionKey string `json:"EncryptionKey"` } // InnerUploadAddress 内部上传地址 type InnerUploadAddress struct { UploadNodes []UploadNode `json:"UploadNodes"` AdvanceOption AdvanceOption `json:"AdvanceOption"` } // UploadPart 上传分片信息 type UploadPart struct { UploadId string `json:"uploadid,omitempty"` PartNumber string `json:"part_number,omitempty"` Crc32 string `json:"crc32,omitempty"` Etag string `json:"etag,omitempty"` Mode string `json:"mode,omitempty"` } // UploadResp 上传响应体 type UploadResp struct { Code int `json:"code"` ApiVersion string `json:"apiversion"` Message string `json:"message"` Data UploadPart `json:"data"` } type VideoCommitUpload struct { Vid string `json:"Vid"` VideoMeta struct { URI string `json:"Uri"` Height int `json:"Height"` Width int `json:"Width"` OriginHeight int `json:"OriginHeight"` OriginWidth int `json:"OriginWidth"` Duration float64 `json:"Duration"` Bitrate int `json:"Bitrate"` Md5 string `json:"Md5"` Format string `json:"Format"` Size int `json:"Size"` FileType string `json:"FileType"` Codec string `json:"Codec"` } `json:"VideoMeta"` WorkflowInput struct { TemplateID string `json:"TemplateId"` } `json:"WorkflowInput"` GetPosterMode string `json:"GetPosterMode"` } type VideoCommitUploadResp struct { ResponseMetadata ResponseMetadata `json:"ResponseMetadata"` Result struct { RequestID string `json:"RequestId"` Results []VideoCommitUpload `json:"Results"` } `json:"Result"` } type CommonResp struct { Code int `json:"code"` Msg string `json:"msg,omitempty"` Message string `json:"message,omitempty"` // 错误情况下的消息 Data json.RawMessage `json:"data,omitempty"` // 原始数据,稍后解析 Error *struct { Code int `json:"code"` Message string `json:"message"` Locale string `json:"locale"` } `json:"error,omitempty"` } // IsSuccess 判断响应是否成功 func (r *CommonResp) IsSuccess() bool { return r.Code == 0 } // GetError 获取错误信息 func (r *CommonResp) GetError() error { if r.IsSuccess() { return nil } // 优先使用message字段 errMsg := r.Message if errMsg == "" { errMsg = r.Msg } // 如果error对象存在且有详细消息,则使用error中的信息 if r.Error != nil && r.Error.Message != "" { errMsg = r.Error.Message } return fmt.Errorf("[doubao] API error (code: %d): %s", r.Code, errMsg) } // UnmarshalData 将data字段解析为指定类型 func (r *CommonResp) UnmarshalData(v interface{}) error { if !r.IsSuccess() { return r.GetError() } if len(r.Data) == 0 { return nil } return json.Unmarshal(r.Data, v) } ================================================ FILE: drivers/doubao/util.go ================================================ package doubao import ( "context" "crypto/hmac" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "hash/crc32" "io" "math/rand" "net/http" "net/url" stdpath "path" "sort" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" "github.com/google/uuid" log "github.com/sirupsen/logrus" ) const ( DirectoryType = 1 FileType = 2 LinkType = 3 ImageType = 4 PagesType = 5 VideoType = 6 AudioType = 7 MeetingMinutesType = 8 ) var FileNodeType = map[int]string{ 1: "directory", 2: "file", 3: "link", 4: "image", 5: "pages", 6: "video", 7: "audio", 8: "meeting_minutes", } const ( BaseURL = "https://www.doubao.com" FileDataType = "file" ImgDataType = "image" VideoDataType = "video" DefaultChunkSize = int64(5 * 1024 * 1024) // 5MB MaxRetryAttempts = 3 // 最大重试次数 UserAgent = base.UserAgentNT Region = "cn-north-1" UploadTimeout = 3 * time.Minute ) // do others that not defined in Driver interface func (d *Doubao) request(path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { reqUrl := BaseURL + path req := base.RestyClient.R() req.SetHeader("Cookie", d.Cookie) if callback != nil { callback(req) } var commonResp CommonResp res, err := req.Execute(method, reqUrl) log.Debugln(res.String()) if err != nil { return nil, err } body := res.Body() // 先解析为通用响应 if err = json.Unmarshal(body, &commonResp); err != nil { return nil, err } // 检查响应是否成功 if !commonResp.IsSuccess() { return body, commonResp.GetError() } if resp != nil { if err = json.Unmarshal(body, resp); err != nil { return body, err } } return body, nil } func (d *Doubao) getFiles(dirId, cursor string) (resp []File, err error) { var r NodeInfoResp var body = base.Json{ "node_id": dirId, } // 如果有游标，则设置游标和大小 if cursor != "" { body["cursor"] = cursor body["size"] = 50 } else { body["need_full_path"] = false } _, err = d.request("/samantha/aispace/node_info", http.MethodPost, func(req *resty.Request) { req.SetBody(body) }, &r) if err != nil { return nil, err } if r.Data.Children != nil { resp = r.Data.Children } if r.Data.NextCursor != "-1" { // 递归获取下一页 nextFiles, err := d.getFiles(dirId, r.Data.NextCursor) if err != nil { return nil, err } resp = append(r.Data.Children, nextFiles...) } return resp, err } func (d *Doubao) getUserInfo() (UserInfo, error) { var r UserInfoResp _, err := d.request("/passport/account/info/v2/", http.MethodGet, nil, &r) if err != nil { return UserInfo{}, err } return r.Data, err } // 签名请求 func (d *Doubao) signRequest(req *resty.Request, method, tokenType, uploadUrl string) error { parsedUrl, err := url.Parse(uploadUrl) if err != nil { return fmt.Errorf("invalid URL format: %w", err) } var accessKeyId, secretAccessKey, sessionToken string var serviceName string if tokenType == VideoDataType { accessKeyId = d.UploadToken.Samantha.StsToken.AccessKeyID secretAccessKey = d.UploadToken.Samantha.StsToken.SecretAccessKey sessionToken = d.UploadToken.Samantha.StsToken.SessionToken serviceName = "vod" } else { accessKeyId = d.UploadToken.Alice[tokenType].Auth.AccessKeyID secretAccessKey = d.UploadToken.Alice[tokenType].Auth.SecretAccessKey sessionToken = d.UploadToken.Alice[tokenType].Auth.SessionToken serviceName = "imagex" } // 当前时间，格式为 ISO8601 now := time.Now().UTC() amzDate := now.Format("20060102T150405Z") dateStamp := now.Format("20060102") req.SetHeader("X-Amz-Date", amzDate) if sessionToken != "" { req.SetHeader("X-Amz-Security-Token", sessionToken) } // 计算请求体的SHA256哈希 var bodyHash string if req.Body != nil { bodyBytes, ok := req.Body.([]byte) if !ok { return fmt.Errorf("request body must be []byte") } bodyHash = hashSHA256(string(bodyBytes)) req.SetHeader("X-Amz-Content-Sha256", bodyHash) } else { bodyHash = hashSHA256("") } // 创建规范请求 canonicalURI := parsedUrl.Path if canonicalURI == "" { canonicalURI = "/" } // 查询参数按照字母顺序排序 canonicalQueryString := getCanonicalQueryString(req.QueryParam) // 规范请求头 canonicalHeaders, signedHeaders := getCanonicalHeadersFromMap(req.Header) canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQueryString + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + bodyHash algorithm := "AWS4-HMAC-SHA256" credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", dateStamp, Region, serviceName) stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + hashSHA256(canonicalRequest) // 计算签名密钥 signingKey := getSigningKey(secretAccessKey, dateStamp, Region, serviceName) // 计算签名 signature := hmacSHA256Hex(signingKey, stringToSign) // 构建授权头 authorizationHeader := fmt.Sprintf( "%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", algorithm, accessKeyId, credentialScope, signedHeaders, signature, ) req.SetHeader("Authorization", authorizationHeader) return nil } func (d *Doubao) requestApi(url, method, tokenType string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeaders(map[string]string{ "user-agent": UserAgent, }) if method == http.MethodPost { req.SetHeader("Content-Type", "text/plain;charset=UTF-8") } if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } // 使用自定义AWS SigV4签名 err := d.signRequest(req, method, tokenType, url) if err != nil { return nil, err } res, err := req.Execute(method, url) if err != nil { return nil, err } return res.Body(), nil } func (d *Doubao) initUploadToken() (*UploadToken, error) { uploadToken := &UploadToken{ Alice: make(map[string]UploadAuthToken), Samantha: MediaUploadAuthToken{}, } fileAuthToken, err := d.getUploadAuthToken(FileDataType) if err != nil { return nil, err } imgAuthToken, err := d.getUploadAuthToken(ImgDataType) if err != nil { return nil, err } mediaAuthToken, err := d.getSamantaUploadAuthToken() if err != nil { return nil, err } uploadToken.Alice[FileDataType] = fileAuthToken uploadToken.Alice[ImgDataType] = imgAuthToken uploadToken.Samantha = mediaAuthToken return uploadToken, nil } func (d *Doubao) getUploadAuthToken(dataType string) (ut UploadAuthToken, err error) { var r UploadAuthTokenResp _, err = d.request("/alice/upload/auth_token", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "scene": "bot_chat", "data_type": dataType, }) }, &r) return r.Data, err } func (d *Doubao) getSamantaUploadAuthToken() (mt MediaUploadAuthToken, err error) { var r MediaUploadAuthTokenResp _, err = d.request("/samantha/media/get_upload_token", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{}) }, &r) return r.Data, err } // getUploadConfig 获取上传配置信息 func (d *Doubao) getUploadConfig(upConfig *UploadConfig, dataType string, file model.FileStreamer) error { tokenType := dataType // 配置参数函数 configureParams := func() (string, map[string]string) { var uploadUrl string var params map[string]string // 根据数据类型设置不同的上传参数 switch dataType { case VideoDataType: // 音频/视频类型 - 使用uploadToken.Samantha的配置 uploadUrl = d.UploadToken.Samantha.UploadInfo.VideoHost params = map[string]string{ "Action": "ApplyUploadInner", "Version": "2020-11-19", "SpaceName": d.UploadToken.Samantha.UploadInfo.SpaceName, "FileType": "video", "IsInner": "1", "NeedFallback": "true", "FileSize": strconv.FormatInt(file.GetSize(), 10), "s": randomString(), } case ImgDataType, FileDataType: // 图片或其他文件类型 - 使用uploadToken.Alice对应配置 uploadUrl = "https://" + d.UploadToken.Alice[dataType].UploadHost params = map[string]string{ "Action": "ApplyImageUpload", "Version": "2018-08-01", "ServiceId": d.UploadToken.Alice[dataType].ServiceID, "NeedFallback": "true", "FileSize": strconv.FormatInt(file.GetSize(), 10), "FileExtension": stdpath.Ext(file.GetName()), "s": randomString(), } } return uploadUrl, params } // 获取初始参数 uploadUrl, params := configureParams() tokenRefreshed := false var configResp UploadConfigResp err := d._retryOperation("get upload_config", func() error { configResp = UploadConfigResp{} _, err := d.requestApi(uploadUrl, http.MethodGet, tokenType, func(req *resty.Request) { req.SetQueryParams(params) }, &configResp) if err != nil { return err } if configResp.ResponseMetadata.Error.Code == "" { *upConfig = configResp.Result return nil } // 100028 凭证过期 if configResp.ResponseMetadata.Error.CodeN == 100028 && !tokenRefreshed { log.Debugln("[doubao] Upload token expired, re-fetching...") newToken, err := d.initUploadToken() if err != nil { return fmt.Errorf("failed to refresh token: %w", err) } d.UploadToken = newToken tokenRefreshed = true uploadUrl, params = configureParams() return retry.Error{errors.New("token refreshed, retry needed")} } return fmt.Errorf("get upload_config failed: %s", configResp.ResponseMetadata.Error.Message) }) return err } // uploadNode 上传 文件信息 func (d *Doubao) uploadNode(uploadConfig *UploadConfig, dir model.Obj, file model.FileStreamer, dataType string) (UploadNodeResp, error) { reqUuid := uuid.New().String() var key string var nodeType int mimetype := file.GetMimetype() switch dataType { case VideoDataType: key = uploadConfig.InnerUploadAddress.UploadNodes[0].Vid if strings.HasPrefix(mimetype, "audio/") { nodeType = AudioType // 音频类型 } else { nodeType = VideoType // 视频类型 } case ImgDataType: key = uploadConfig.InnerUploadAddress.UploadNodes[0].StoreInfos[0].StoreURI nodeType = ImageType // 图片类型 default: // FileDataType key = uploadConfig.InnerUploadAddress.UploadNodes[0].StoreInfos[0].StoreURI nodeType = FileType // 文件类型 } var r UploadNodeResp _, err := d.request("/samantha/aispace/upload_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "node_list": []base.Json{ { "local_id": reqUuid, "parent_id": dir.GetID(), "name": file.GetName(), "key": key, "node_content": base.Json{}, "node_type": nodeType, "size": file.GetSize(), }, }, "request_id": reqUuid, }) }, &r) return r, err } // Upload 普通上传实现 func (d *Doubao) Upload(ctx context.Context, config *UploadConfig, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, dataType string) (model.Obj, error) { ss, err := stream.NewStreamSectionReader(file, int(file.GetSize()), &up) if err != nil { return nil, err } reader, err := ss.GetSectionReader(0, file.GetSize()) if err != nil { return nil, err } // 计算CRC32 crc32Hash := crc32.NewIEEE() w, err := utils.CopyWithBuffer(crc32Hash, reader) if w != file.GetSize() { return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", file.GetSize(), w, err) } crc32Value := hex.EncodeToString(crc32Hash.Sum(nil)) // 构建请求路径 uploadNode := config.InnerUploadAddress.UploadNodes[0] storeInfo := uploadNode.StoreInfos[0] uploadUrl := fmt.Sprintf("https://%s/upload/v1/%s", uploadNode.UploadHost, storeInfo.StoreURI) rateLimitedRd := driver.NewLimitedUploadStream(ctx, reader) err = d._retryOperation("Upload", func() error { reader.Seek(0, io.SeekStart) req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl, rateLimitedRd) if err != nil { return err } req.Header = map[string][]string{ "Referer": {BaseURL + "/"}, "Origin": {BaseURL}, "User-Agent": {UserAgent}, "X-Storage-U": {d.UserId}, "Authorization": {storeInfo.Auth}, "Content-Type": {"application/octet-stream"}, "Content-Crc32": {crc32Value}, "Content-Length": {strconv.FormatInt(file.GetSize(), 10)}, "Content-Disposition": {fmt.Sprintf("attachment; filename=%s", url.QueryEscape(storeInfo.StoreURI))}, } res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() bytes, _ := io.ReadAll(res.Body) resp := UploadResp{} utils.Json.Unmarshal(bytes, &resp) if resp.Code != 2000 { return fmt.Errorf("upload part failed: %s", resp.Message) } else if resp.Data.Crc32 != crc32Value { return fmt.Errorf("upload part failed: crc32 mismatch, expected %s, got %s", crc32Value, resp.Data.Crc32) } return nil }) ss.FreeSectionReader(reader) if err != nil { return nil, err } uploadNodeResp, err := d.uploadNode(config, dstDir, file, dataType) if err != nil { return nil, err } return &model.Object{ ID: uploadNodeResp.Data.NodeList[0].ID, Name: uploadNodeResp.Data.NodeList[0].Name, Size: file.GetSize(), IsFolder: false, }, nil } // UploadByMultipart 分片上传 func (d *Doubao) UploadByMultipart(ctx context.Context, config *UploadConfig, fileSize int64, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, dataType string) (model.Obj, error) { // 构建请求路径 uploadNode := config.InnerUploadAddress.UploadNodes[0] storeInfo := uploadNode.StoreInfos[0] uploadUrl := fmt.Sprintf("https://%s/upload/v1/%s", uploadNode.UploadHost, storeInfo.StoreURI) // 初始化分片上传 var uploadID string err := d._retryOperation("Initialize multipart upload", func() error { var err error uploadID, err = d.initMultipartUpload(config, uploadUrl, storeInfo) return err }) if err != nil { return nil, fmt.Errorf("failed to initialize multipart upload: %w", err) } // 准备分片参数 chunkSize := DefaultChunkSize if config.InnerUploadAddress.AdvanceOption.SliceSize > 0 { chunkSize = int64(config.InnerUploadAddress.AdvanceOption.SliceSize) } ss, err := stream.NewStreamSectionReader(file, int(chunkSize), &up) if err != nil { return nil, err } totalParts := (fileSize + chunkSize - 1) / chunkSize // 创建分片信息组 parts := make([]UploadPart, totalParts) up(10.0) // 更新进度 // 设置并行上传 thread := min(int(totalParts), d.uploadThread) threadG, uploadCtx := errgroup.NewOrderedGroupWithContext(ctx, thread, retry.Attempts(MaxRetryAttempts), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay), retry.MaxJitter(200*time.Millisecond), ) // 并行上传所有分片 for partIndex := range totalParts { if utils.IsCanceled(uploadCtx) { break } partNumber := partIndex + 1 // 分片编号从1开始 // 计算此分片的大小和偏移 offset := partIndex * chunkSize size := chunkSize if partIndex == totalParts-1 { size = fileSize - offset } var reader io.ReadSeeker crc32Value := "" threadG.GoWithLifecycle(errgroup.Lifecycle{ Before: func(ctx context.Context) (err error) { reader, err = ss.GetSectionReader(offset, size) return }, Do: func(ctx context.Context) (err error) { reader.Seek(0, io.SeekStart) if crc32Value == "" { // 把耗时的计算放在这里，避免阻塞其他协程 crc32Hash := crc32.NewIEEE() w, err := utils.CopyWithBuffer(crc32Hash, reader) if w != size { return fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", size, w, err) } crc32Value = hex.EncodeToString(crc32Hash.Sum(nil)) reader.Seek(0, io.SeekStart) } req, err := http.NewRequestWithContext( ctx, http.MethodPost, uploadUrl, driver.NewLimitedUploadStream(ctx, reader), ) if err != nil { return err } query := req.URL.Query() query.Add("uploadid", uploadID) query.Add("part_number", strconv.FormatInt(partNumber, 10)) query.Add("phase", "transfer") req.URL.RawQuery = query.Encode() req.Header = map[string][]string{ "Referer": {BaseURL + "/"}, "Origin": {BaseURL}, "User-Agent": {UserAgent}, "X-Storage-U": {d.UserId}, "Authorization": {storeInfo.Auth}, "Content-Type": {"application/octet-stream"}, "Content-Crc32": {crc32Value}, "Content-Length": {strconv.FormatInt(size, 10)}, "Content-Disposition": {fmt.Sprintf("attachment; filename=%s", url.QueryEscape(storeInfo.StoreURI))}, } res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() bytes, _ := io.ReadAll(res.Body) uploadResp := UploadResp{} utils.Json.Unmarshal(bytes, &uploadResp) if uploadResp.Code != 2000 { return fmt.Errorf("upload part failed: %s", uploadResp.Message) } else if uploadResp.Data.Crc32 != crc32Value { return fmt.Errorf("upload part failed: crc32 mismatch, expected %s, got %s", crc32Value, uploadResp.Data.Crc32) } // 记录成功上传的分片 parts[partIndex] = UploadPart{ PartNumber: strconv.FormatInt(partNumber, 10), Etag: uploadResp.Data.Etag, Crc32: crc32Value, } // 更新进度 progress := 95 * float64(threadG.Success()+1) / float64(totalParts) up(progress) return nil }, After: func(err error) { ss.FreeSectionReader(reader) }, }) } if err = threadG.Wait(); err != nil { return nil, err } // 完成上传-分片合并 if err = d._retryOperation("Complete multipart upload", func() error { return d.completeMultipartUpload(config, uploadUrl, uploadID, parts) }); err != nil { return nil, fmt.Errorf("failed to complete multipart upload: %w", err) } // 提交上传 if err = d._retryOperation("Commit upload", func() error { return d.commitMultipartUpload(config) }); err != nil { return nil, fmt.Errorf("failed to commit upload: %w", err) } up(98.0) // 更新到98% // 上传节点信息 var uploadNodeResp UploadNodeResp if err = d._retryOperation("Upload node", func() error { var err error uploadNodeResp, err = d.uploadNode(config, dstDir, file, dataType) return err }); err != nil { return nil, fmt.Errorf("failed to upload node: %w", err) } up(100.0) // 完成上传 return &model.Object{ ID: uploadNodeResp.Data.NodeList[0].ID, Name: uploadNodeResp.Data.NodeList[0].Name, Size: file.GetSize(), IsFolder: false, }, nil } // 统一上传请求方法 func (d *Doubao) uploadRequest(uploadUrl string, method string, storeInfo StoreInfo, callback base.ReqCallback, resp interface{}) ([]byte, error) { client := resty.New() client.SetTransport(&http.Transport{ DisableKeepAlives: true, // 禁用连接复用 ForceAttemptHTTP2: false, // 强制使用HTTP/1.1 }) client.SetTimeout(UploadTimeout) req := client.R() req.SetHeaders(map[string]string{ "Host": strings.Split(uploadUrl, "/")[2], "Referer": BaseURL + "/", "Origin": BaseURL, "User-Agent": UserAgent, "X-Storage-U": d.UserId, "Authorization": storeInfo.Auth, }) if method == http.MethodPost { req.SetHeader("Content-Type", "text/plain;charset=UTF-8") } if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } res, err := req.Execute(method, uploadUrl) if err != nil && err != io.EOF { return nil, fmt.Errorf("upload request failed: %w", err) } return res.Body(), nil } // 初始化分片上传 func (d *Doubao) initMultipartUpload(config *UploadConfig, uploadUrl string, storeInfo StoreInfo) (uploadId string, err error) { uploadResp := UploadResp{} _, err = d.uploadRequest(uploadUrl, http.MethodPost, storeInfo, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "uploadmode": "part", "phase": "init", }) }, &uploadResp) if err != nil { return uploadId, err } if uploadResp.Code != 2000 { return uploadId, fmt.Errorf("init upload failed: %s", uploadResp.Message) } return uploadResp.Data.UploadId, nil } // 完成分片上传 func (d *Doubao) completeMultipartUpload(config *UploadConfig, uploadUrl, uploadID string, parts []UploadPart) error { uploadResp := UploadResp{} storeInfo := config.InnerUploadAddress.UploadNodes[0].StoreInfos[0] body := _convertUploadParts(parts) err := utils.Retry(MaxRetryAttempts, time.Second, func() (err error) { _, err = d.uploadRequest(uploadUrl, http.MethodPost, storeInfo, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "uploadid": uploadID, "phase": "finish", "uploadmode": "part", }) req.SetBody(body) }, &uploadResp) if err != nil { return err } // 检查响应状态码 2000 成功 4024 分片合并中 if uploadResp.Code != 2000 && uploadResp.Code != 4024 { return fmt.Errorf("finish upload failed: %s", uploadResp.Message) } return err }) if err != nil { return fmt.Errorf("failed to complete multipart upload: %w", err) } return nil } func (d *Doubao) commitMultipartUpload(uploadConfig *UploadConfig) error { uploadUrl := d.UploadToken.Samantha.UploadInfo.VideoHost params := map[string]string{ "Action": "CommitUploadInner", "Version": "2020-11-19", "SpaceName": d.UploadToken.Samantha.UploadInfo.SpaceName, } tokenType := VideoDataType videoCommitUploadResp := VideoCommitUploadResp{} jsonBytes, err := json.Marshal(base.Json{ "SessionKey": uploadConfig.InnerUploadAddress.UploadNodes[0].SessionKey, "Functions": []base.Json{}, }) if err != nil { return fmt.Errorf("failed to marshal request data: %w", err) } _, err = d.requestApi(uploadUrl, http.MethodPost, tokenType, func(req *resty.Request) { req.SetHeader("Content-Type", "application/json") req.SetQueryParams(params) req.SetBody(jsonBytes) }, &videoCommitUploadResp) if err != nil { return err } return nil } // _retryOperation 操作重试 func (d *Doubao) _retryOperation(operation string, fn func() error) error { return retry.Do( fn, retry.Attempts(MaxRetryAttempts), retry.Delay(500*time.Millisecond), retry.DelayType(retry.BackOffDelay), retry.MaxJitter(200*time.Millisecond), retry.OnRetry(func(n uint, err error) { log.Debugf("[doubao] %s retry #%d: %v", operation, n+1, err) }), ) } // _convertUploadParts 将分片信息转换为字符串 func _convertUploadParts(parts []UploadPart) string { if len(parts) == 0 { return "" } var result strings.Builder for i, part := range parts { if i > 0 { result.WriteString(",") } result.WriteString(fmt.Sprintf("%s:%s", part.PartNumber, part.Crc32)) } return result.String() } // 获取规范查询字符串 func getCanonicalQueryString(query url.Values) string { if len(query) == 0 { return "" } keys := make([]string, 0, len(query)) for k := range query { keys = append(keys, k) } sort.Strings(keys) parts := make([]string, 0, len(keys)) for _, k := range keys { values := query[k] for _, v := range values { parts = append(parts, urlEncode(k)+"="+urlEncode(v)) } } return strings.Join(parts, "&") } func urlEncode(s string) string { s = url.QueryEscape(s) s = strings.ReplaceAll(s, "+", "%20") return s } // 获取规范头信息和已签名头列表 func getCanonicalHeadersFromMap(headers map[string][]string) (string, string) { // 不可签名的头部列表 unsignableHeaders := map[string]bool{ "authorization": true, "content-type": true, "content-length": true, "user-agent": true, "presigned-expires": true, "expect": true, "x-amzn-trace-id": true, } headerValues := make(map[string]string) var signedHeadersList []string for k, v := range headers { if len(v) == 0 { continue } lowerKey := strings.ToLower(k) // 检查是否可签名 if strings.HasPrefix(lowerKey, "x-amz-") || !unsignableHeaders[lowerKey] { value := strings.TrimSpace(v[0]) value = strings.Join(strings.Fields(value), " ") headerValues[lowerKey] = value signedHeadersList = append(signedHeadersList, lowerKey) } } sort.Strings(signedHeadersList) var canonicalHeadersStr strings.Builder for _, key := range signedHeadersList { canonicalHeadersStr.WriteString(key) canonicalHeadersStr.WriteString(":") canonicalHeadersStr.WriteString(headerValues[key]) canonicalHeadersStr.WriteString("\n") } signedHeaders := strings.Join(signedHeadersList, ";") return canonicalHeadersStr.String(), signedHeaders } // 计算HMAC-SHA256 func hmacSHA256(key []byte, data string) []byte { h := hmac.New(sha256.New, key) h.Write([]byte(data)) return h.Sum(nil) } // 计算HMAC-SHA256并返回十六进制字符串 func hmacSHA256Hex(key []byte, data string) string { return hex.EncodeToString(hmacSHA256(key, data)) } // 计算SHA256哈希并返回十六进制字符串 func hashSHA256(data string) string { h := sha256.New() h.Write([]byte(data)) return hex.EncodeToString(h.Sum(nil)) } // 获取签名密钥 func getSigningKey(secretKey, dateStamp, region, service string) []byte { kDate := hmacSHA256([]byte("AWS4"+secretKey), dateStamp) kRegion := hmacSHA256(kDate, region) kService := hmacSHA256(kRegion, service) kSigning := hmacSHA256(kService, "aws4_request") return kSigning } func randomString() string { const charset = "0123456789abcdefghijklmnopqrstuvwxyz" const length = 11 // 11位随机字符串 var sb strings.Builder sb.Grow(length) for i := 0; i < length; i++ { sb.WriteByte(charset[rand.Intn(len(charset))]) } return sb.String() } ================================================ FILE: drivers/doubao_share/driver.go ================================================ package doubao_share import ( "context" "errors" "net/http" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) type DoubaoShare struct { model.Storage Addition RootFiles []RootFileList } func (d *DoubaoShare) Config() driver.Config { return config } func (d *DoubaoShare) GetAddition() driver.Additional { return &d.Addition } func (d *DoubaoShare) Init(ctx context.Context) error { // 初始化 虚拟分享列表 if err := d.initShareList(); err != nil { return err } return nil } func (d *DoubaoShare) Drop(ctx context.Context) error { return nil } // 潜在bug：配置二级目录时，可能会出问题 func (d *DoubaoShare) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { // 检查是否为根目录 if dir.GetID() == "" && dir.GetPath() == "/" { return d.listRootDirectory(ctx) } // 非根目录，处理不同情况 if fo, ok := dir.(*FileObject); ok { if fo.ShareID == "" { // 虚拟目录，需要列出子目录 return d.listVirtualDirectoryContent(dir) } else { // 具有分享ID的目录，获取此分享下的文件 shareId, relativePath, err := d._findShareAndPath(dir) if err != nil { return nil, err } return d.getFilesInPath(ctx, shareId, dir.GetID(), relativePath) } } // 使用通用方法 shareId, relativePath, err := d._findShareAndPath(dir) if err != nil { return nil, err } // 获取指定路径下的文件 return d.getFilesInPath(ctx, shareId, dir.GetID(), relativePath) } func (d *DoubaoShare) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var downloadUrl string if u, ok := file.(*FileObject); ok { switch u.NodeType { case VideoType, AudioType: var r GetVideoFileUrlResp _, err := d.request("/samantha/media/get_play_info", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "key": u.Key, "share_id": u.ShareID, "node_id": file.GetID(), }) }, &r) if err != nil { return nil, err } downloadUrl = r.Data.OriginalMediaInfo.MainURL default: var r GetDownloadInfoResp _, err := d.request("/samantha/aispace/get_download_info", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "requests": []base.Json{{"node_id": file.GetID()}}, }) }, &r) if err != nil { return nil, err } downloadUrl = r.Data.DownloadInfos[0].MainURL } // 生成标准的Content-Disposition contentDisposition := utils.GenerateContentDisposition(u.Name) return &model.Link{ URL: downloadUrl, Header: http.Header{ "User-Agent": []string{UserAgent}, "Content-Disposition": []string{contentDisposition}, }, }, nil } return nil, errors.New("can't convert obj to URL") } func (d *DoubaoShare) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { // TODO create folder, optional return nil, errs.NotImplement } func (d *DoubaoShare) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { // TODO move obj, optional return nil, errs.NotImplement } func (d *DoubaoShare) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { // TODO rename obj, optional return nil, errs.NotImplement } func (d *DoubaoShare) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { // TODO copy obj, optional return nil, errs.NotImplement } func (d *DoubaoShare) Remove(ctx context.Context, obj model.Obj) error { // TODO remove obj, optional return errs.NotImplement } func (d *DoubaoShare) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { // TODO upload file, optional return nil, errs.NotImplement } func (d *DoubaoShare) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *DoubaoShare) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *DoubaoShare) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *DoubaoShare) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir // return errs.NotImplement to use an internal archive tool return nil, errs.NotImplement } //func (d *DoubaoShare) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*DoubaoShare)(nil) ================================================ FILE: drivers/doubao_share/meta.go ================================================ package doubao_share import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Cookie string `json:"cookie" type:"text"` ShareIds string `json:"share_ids" type:"text" required:"true"` } var config = driver.Config{ Name: "DoubaoShare", LocalSort: true, NoUpload: true, DefaultRoot: "/", } func init() { op.RegisterDriver(func() driver.Driver { return &DoubaoShare{} }) } ================================================ FILE: drivers/doubao_share/types.go ================================================ package doubao_share import ( "encoding/json" "fmt" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type BaseResp struct { Code int `json:"code"` Msg string `json:"msg"` } type NodeInfoData struct { Share ShareInfo `json:"share,omitempty"` Creator CreatorInfo `json:"creator,omitempty"` NodeList []File `json:"node_list,omitempty"` NodeInfo File `json:"node_info,omitempty"` Children []File `json:"children,omitempty"` Path FilePath `json:"path,omitempty"` NextCursor string `json:"next_cursor,omitempty"` HasMore bool `json:"has_more,omitempty"` } type NodeInfoResp struct { BaseResp NodeInfoData `json:"data"` } type RootFileList struct { ShareID string VirtualPath string NodeInfo NodeInfoData Child *[]RootFileList } type File struct { ID string `json:"id"` Name string `json:"name"` Key string `json:"key"` NodeType int `json:"node_type"` Size int64 `json:"size"` Source int `json:"source"` NameReviewStatus int `json:"name_review_status"` ContentReviewStatus int `json:"content_review_status"` RiskReviewStatus int `json:"risk_review_status"` ConversationID string `json:"conversation_id"` ParentID string `json:"parent_id"` CreateTime int64 `json:"create_time"` UpdateTime int64 `json:"update_time"` } type FileObject struct { model.Object ShareID string Key string NodeID string NodeType int } type ShareInfo struct { ShareID string `json:"share_id"` FirstNode struct { ID string `json:"id"` Name string `json:"name"` Key string `json:"key"` NodeType int `json:"node_type"` Size int `json:"size"` Source int `json:"source"` Content struct { LinkFileType string `json:"link_file_type"` ImageWidth int `json:"image_width"` ImageHeight int `json:"image_height"` AiSkillStatus int `json:"ai_skill_status"` } `json:"content"` NameReviewStatus int `json:"name_review_status"` ContentReviewStatus int `json:"content_review_status"` RiskReviewStatus int `json:"risk_review_status"` ConversationID string `json:"conversation_id"` ParentID string `json:"parent_id"` CreateTime int64 `json:"create_time"` UpdateTime int64 `json:"update_time"` } `json:"first_node"` NodeCount int `json:"node_count"` CreateTime int64 `json:"create_time"` Channel string `json:"channel"` InfluencerType int `json:"influencer_type"` } type CreatorInfo struct { EntityID string `json:"entity_id"` UserName string `json:"user_name"` NickName string `json:"nick_name"` Avatar struct { OriginURL string `json:"origin_url"` TinyURL string `json:"tiny_url"` URI string `json:"uri"` } `json:"avatar"` } type FilePath []struct { ID string `json:"id"` Name string `json:"name"` Key string `json:"key"` NodeType int `json:"node_type"` Size int `json:"size"` Source int `json:"source"` NameReviewStatus int `json:"name_review_status"` ContentReviewStatus int `json:"content_review_status"` RiskReviewStatus int `json:"risk_review_status"` ConversationID string `json:"conversation_id"` ParentID string `json:"parent_id"` CreateTime int64 `json:"create_time"` UpdateTime int64 `json:"update_time"` } type GetDownloadInfoResp struct { BaseResp Data struct { DownloadInfos []struct { NodeID string `json:"node_id"` MainURL string `json:"main_url"` BackupURL string `json:"backup_url"` } `json:"download_infos"` } `json:"data"` } type GetVideoFileUrlResp struct { BaseResp Data struct { MediaType string `json:"media_type"` MediaInfo []struct { Meta struct { Height string `json:"height"` Width string `json:"width"` Format string `json:"format"` Duration float64 `json:"duration"` CodecType string `json:"codec_type"` Definition string `json:"definition"` } `json:"meta"` MainURL string `json:"main_url"` BackupURL string `json:"backup_url"` } `json:"media_info"` OriginalMediaInfo struct { Meta struct { Height string `json:"height"` Width string `json:"width"` Format string `json:"format"` Duration float64 `json:"duration"` CodecType string `json:"codec_type"` Definition string `json:"definition"` } `json:"meta"` MainURL string `json:"main_url"` BackupURL string `json:"backup_url"` } `json:"original_media_info"` PosterURL string `json:"poster_url"` PlayableStatus int `json:"playable_status"` } `json:"data"` } type CommonResp struct { Code int `json:"code"` Msg string `json:"msg,omitempty"` Message string `json:"message,omitempty"` // 错误情况下的消息 Data json.RawMessage `json:"data,omitempty"` // 原始数据,稍后解析 Error *struct { Code int `json:"code"` Message string `json:"message"` Locale string `json:"locale"` } `json:"error,omitempty"` } // IsSuccess 判断响应是否成功 func (r *CommonResp) IsSuccess() bool { return r.Code == 0 } // GetError 获取错误信息 func (r *CommonResp) GetError() error { if r.IsSuccess() { return nil } // 优先使用message字段 errMsg := r.Message if errMsg == "" { errMsg = r.Msg } // 如果error对象存在且有详细消息,则使用error中的信息 if r.Error != nil && r.Error.Message != "" { errMsg = r.Error.Message } return fmt.Errorf("[doubao] API error (code: %d): %s", r.Code, errMsg) } // UnmarshalData 将data字段解析为指定类型 func (r *CommonResp) UnmarshalData(v interface{}) error { if !r.IsSuccess() { return r.GetError() } if len(r.Data) == 0 { return nil } return json.Unmarshal(r.Data, v) } ================================================ FILE: drivers/doubao_share/util.go ================================================ package doubao_share import ( "context" "encoding/json" "fmt" "net/http" "path" "regexp" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) const ( DirectoryType = 1 FileType = 2 LinkType = 3 ImageType = 4 PagesType = 5 VideoType = 6 AudioType = 7 MeetingMinutesType = 8 ) var FileNodeType = map[int]string{ 1: "directory", 2: "file", 3: "link", 4: "image", 5: "pages", 6: "video", 7: "audio", 8: "meeting_minutes", } const ( BaseURL = "https://www.doubao.com" FileDataType = "file" ImgDataType = "image" VideoDataType = "video" UserAgent = base.UserAgentNT ) func (d *DoubaoShare) request(path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { reqUrl := BaseURL + path req := base.RestyClient.R() req.SetHeaders(map[string]string{ "Cookie": d.Cookie, "User-Agent": UserAgent, }) req.SetQueryParams(map[string]string{ "version_code": "20800", "device_platform": "web", }) if callback != nil { callback(req) } var commonResp CommonResp res, err := req.Execute(method, reqUrl) log.Debugln(res.String()) if err != nil { return nil, err } body := res.Body() // 先解析为通用响应 if err = json.Unmarshal(body, &commonResp); err != nil { return nil, err } // 检查响应是否成功 if !commonResp.IsSuccess() { return body, commonResp.GetError() } if resp != nil { if err = json.Unmarshal(body, resp); err != nil { return body, err } } return body, nil } func (d *DoubaoShare) getFiles(dirId, nodeId, cursor string) (resp []File, err error) { var r NodeInfoResp var body = base.Json{ "share_id": dirId, "node_id": nodeId, } // 如果有游标，则设置游标和大小 if cursor != "" { body["cursor"] = cursor body["size"] = 50 } else { body["need_full_path"] = false } _, err = d.request("/samantha/aispace/share/node_info", http.MethodPost, func(req *resty.Request) { req.SetBody(body) }, &r) if err != nil { return nil, err } if r.NodeInfoData.Children != nil { resp = r.NodeInfoData.Children } if r.NodeInfoData.NextCursor != "-1" { // 递归获取下一页 nextFiles, err := d.getFiles(dirId, nodeId, r.NodeInfoData.NextCursor) if err != nil { return nil, err } resp = append(r.NodeInfoData.Children, nextFiles...) } return resp, err } func (d *DoubaoShare) getShareOverview(shareId, cursor string) (resp []File, err error) { return d.getShareOverviewWithHistory(shareId, cursor, make(map[string]bool)) } func (d *DoubaoShare) getShareOverviewWithHistory(shareId, cursor string, cursorHistory map[string]bool) (resp []File, err error) { var r NodeInfoResp var body = base.Json{ "share_id": shareId, } // 如果有游标，则设置游标和大小 if cursor != "" { body["cursor"] = cursor body["size"] = 50 } else { body["need_full_path"] = false } _, err = d.request("/samantha/aispace/share/overview", http.MethodPost, func(req *resty.Request) { req.SetBody(body) }, &r) if err != nil { return nil, err } if r.NodeInfoData.NodeList != nil { resp = r.NodeInfoData.NodeList } if r.NodeInfoData.NextCursor != "-1" { // 检查游标是否重复出现，防止无限循环 if cursorHistory[r.NodeInfoData.NextCursor] { return resp, nil } // 记录当前游标 cursorHistory[r.NodeInfoData.NextCursor] = true // 递归获取下一页 nextFiles, err := d.getShareOverviewWithHistory(shareId, r.NodeInfoData.NextCursor, cursorHistory) if err != nil { return nil, err } resp = append(resp, nextFiles...) } return resp, nil } func (d *DoubaoShare) initShareList() error { if d.Addition.ShareIds == "" { return fmt.Errorf("share_ids is empty") } // 解析分享配置 shareConfigs, rootShares, err := d._parseShareConfigs() if err != nil { return err } // 检查路径冲突 if err := d._detectPathConflicts(shareConfigs); err != nil { return err } // 构建树形结构 rootMap := d._buildTreeStructure(shareConfigs, rootShares) // 提取顶级节点 topLevelNodes := d._extractTopLevelNodes(rootMap, rootShares) if len(topLevelNodes) == 0 { return fmt.Errorf("no valid share_ids found") } // 存储结果 d.RootFiles = topLevelNodes return nil } // 从配置中解析分享ID和路径 func (d *DoubaoShare) _parseShareConfigs() (map[string]string, []string, error) { shareConfigs := make(map[string]string) // 路径 -> 分享ID rootShares := make([]string, 0) // 根目录显示的分享ID lines := strings.Split(strings.TrimSpace(d.Addition.ShareIds), "\n") if len(lines) == 0 { return nil, nil, fmt.Errorf("no share_ids found") } for _, line := range lines { line = strings.TrimSpace(line) if line == "" { continue } // 解析分享ID和路径 parts := strings.Split(line, "|") var shareId, sharePath string if len(parts) == 1 { // 无路径分享，直接在根目录显示 shareId = _extractShareId(parts[0]) if shareId != "" { rootShares = append(rootShares, shareId) } continue } else if len(parts) >= 2 { shareId = _extractShareId(parts[0]) sharePath = strings.Trim(parts[1], "/") } if shareId == "" { log.Warnf("[doubao_share] Invalid Share_id Format: %s", line) continue } // 空路径也加入根目录显示 if sharePath == "" { rootShares = append(rootShares, shareId) continue } // 添加到路径映射 shareConfigs[sharePath] = shareId } return shareConfigs, rootShares, nil } // 检测路径冲突 func (d *DoubaoShare) _detectPathConflicts(shareConfigs map[string]string) error { // 检查直接路径冲突 pathToShareIds := make(map[string][]string) for sharePath, id := range shareConfigs { pathToShareIds[sharePath] = append(pathToShareIds[sharePath], id) } for sharePath, ids := range pathToShareIds { if len(ids) > 1 { return fmt.Errorf("路径冲突: 路径 '%s' 被多个不同的分享ID使用: %s", sharePath, strings.Join(ids, ", ")) } } // 检查层次冲突 for path1, id1 := range shareConfigs { for path2, id2 := range shareConfigs { if path1 == path2 || id1 == id2 { continue } // 检查前缀冲突 if strings.HasPrefix(path2, path1+"/") || strings.HasPrefix(path1, path2+"/") { return fmt.Errorf("路径冲突: 路径 '%s' (ID: %s) 与路径 '%s' (ID: %s) 存在层次冲突", path1, id1, path2, id2) } } } return nil } // 构建树形结构 func (d *DoubaoShare) _buildTreeStructure(shareConfigs map[string]string, rootShares []string) map[string]*RootFileList { rootMap := make(map[string]*RootFileList) // 添加所有分享节点 for sharePath, shareId := range shareConfigs { children := make([]RootFileList, 0) rootMap[sharePath] = &RootFileList{ ShareID: shareId, VirtualPath: sharePath, NodeInfo: NodeInfoData{}, Child: &children, } } // 构建父子关系 for sharePath, node := range rootMap { if sharePath == "" { continue } pathParts := strings.Split(sharePath, "/") if len(pathParts) > 1 { parentPath := strings.Join(pathParts[:len(pathParts)-1], "/") // 确保所有父级路径都已创建 _ensurePathExists(rootMap, parentPath) // 添加当前节点到父节点 if parent, exists := rootMap[parentPath]; exists { *parent.Child = append(*parent.Child, *node) } } } return rootMap } // 提取顶级节点 func (d *DoubaoShare) _extractTopLevelNodes(rootMap map[string]*RootFileList, rootShares []string) []RootFileList { var topLevelNodes []RootFileList // 添加根目录分享 for _, shareId := range rootShares { children := make([]RootFileList, 0) topLevelNodes = append(topLevelNodes, RootFileList{ ShareID: shareId, VirtualPath: "", NodeInfo: NodeInfoData{}, Child: &children, }) } // 添加顶级目录 for rootPath, node := range rootMap { if rootPath == "" { continue } isTopLevel := true pathParts := strings.Split(rootPath, "/") if len(pathParts) > 1 { parentPath := strings.Join(pathParts[:len(pathParts)-1], "/") if _, exists := rootMap[parentPath]; exists { isTopLevel = false } } if isTopLevel { topLevelNodes = append(topLevelNodes, *node) } } return topLevelNodes } // 确保路径存在，创建所有必要的中间节点 func _ensurePathExists(rootMap map[string]*RootFileList, path string) { if path == "" { return } // 如果路径已存在，不需要再处理 if _, exists := rootMap[path]; exists { return } // 创建当前路径节点 children := make([]RootFileList, 0) rootMap[path] = &RootFileList{ ShareID: "", VirtualPath: path, NodeInfo: NodeInfoData{}, Child: &children, } // 处理父路径 pathParts := strings.Split(path, "/") if len(pathParts) > 1 { parentPath := strings.Join(pathParts[:len(pathParts)-1], "/") // 确保父路径存在 _ensurePathExists(rootMap, parentPath) // 将当前节点添加为父节点的子节点 if parent, exists := rootMap[parentPath]; exists { *parent.Child = append(*parent.Child, *rootMap[path]) } } } // _extractShareId 从URL或直接ID中提取分享ID func _extractShareId(input string) string { input = strings.TrimSpace(input) if strings.HasPrefix(input, "http") { regex := regexp.MustCompile(`/drive/s/([a-zA-Z0-9]+)`) if matches := regex.FindStringSubmatch(input); len(matches) > 1 { return matches[1] } return "" } return input // 直接返回ID } // _findRootFileByShareID 查找指定ShareID的配置 func _findRootFileByShareID(rootFiles []RootFileList, shareID string) *RootFileList { for i, rf := range rootFiles { if rf.ShareID == shareID { return &rootFiles[i] } if rf.Child != nil && len(*rf.Child) > 0 { if found := _findRootFileByShareID(*rf.Child, shareID); found != nil { return found } } } return nil } // _findNodeByPath 查找指定路径的节点 func _findNodeByPath(rootFiles []RootFileList, path string) *RootFileList { for i, rf := range rootFiles { if rf.VirtualPath == path { return &rootFiles[i] } if rf.Child != nil && len(*rf.Child) > 0 { if found := _findNodeByPath(*rf.Child, path); found != nil { return found } } } return nil } // _findShareByPath 根据路径查找分享和相对路径 func _findShareByPath(rootFiles []RootFileList, path string) (*RootFileList, string) { // 完全匹配或子路径匹配 for i, rf := range rootFiles { if rf.VirtualPath == path { return &rootFiles[i], "" } if rf.VirtualPath != "" && strings.HasPrefix(path, rf.VirtualPath+"/") { relPath := strings.TrimPrefix(path, rf.VirtualPath+"/") // 先检查子节点 if rf.Child != nil && len(*rf.Child) > 0 { if child, childPath := _findShareByPath(*rf.Child, path); child != nil { return child, childPath } } return &rootFiles[i], relPath } // 递归检查子节点 if rf.Child != nil && len(*rf.Child) > 0 { if child, childPath := _findShareByPath(*rf.Child, path); child != nil { return child, childPath } } } // 检查根目录分享 for i, rf := range rootFiles { if rf.VirtualPath == "" && rf.ShareID != "" { parts := strings.SplitN(path, "/", 2) if len(parts) > 0 && parts[0] == rf.ShareID { if len(parts) > 1 { return &rootFiles[i], parts[1] } return &rootFiles[i], "" } } } return nil, "" } // _findShareAndPath 根据给定路径查找对应的ShareID和相对路径 func (d *DoubaoShare) _findShareAndPath(dir model.Obj) (string, string, error) { dirPath := dir.GetPath() // 如果是根目录，返回空值表示需要列出所有分享 if dirPath == "/" || dirPath == "" { return "", "", nil } // 检查是否是 FileObject 类型，并获取 ShareID if fo, ok := dir.(*FileObject); ok && fo.ShareID != "" { // 直接使用对象中存储的 ShareID // 计算相对路径（移除前导斜杠） relativePath := strings.TrimPrefix(dirPath, "/") // 递归查找对应的 RootFile found := _findRootFileByShareID(d.RootFiles, fo.ShareID) if found != nil { if found.VirtualPath != "" { // 如果此分享配置了路径前缀，需要考虑相对路径的计算 if strings.HasPrefix(relativePath, found.VirtualPath) { return fo.ShareID, strings.TrimPrefix(relativePath, found.VirtualPath+"/"), nil } } return fo.ShareID, relativePath, nil } // 如果找不到对应的 RootFile 配置，仍然使用对象中的 ShareID return fo.ShareID, relativePath, nil } // 移除开头的斜杠 cleanPath := strings.TrimPrefix(dirPath, "/") // 先检查是否有直接匹配的根目录分享 for _, rootFile := range d.RootFiles { if rootFile.VirtualPath == "" && rootFile.ShareID != "" { // 检查是否匹配当前路径的第一部分 parts := strings.SplitN(cleanPath, "/", 2) if len(parts) > 0 && parts[0] == rootFile.ShareID { if len(parts) > 1 { return rootFile.ShareID, parts[1], nil } return rootFile.ShareID, "", nil } } } // 查找匹配此路径的分享或虚拟目录 share, relPath := _findShareByPath(d.RootFiles, cleanPath) if share != nil { return share.ShareID, relPath, nil } log.Warnf("[doubao_share] No matching share path found: %s", dirPath) return "", "", fmt.Errorf("no matching share path found: %s", dirPath) } // convertToFileObject 将File转换为FileObject func (d *DoubaoShare) convertToFileObject(file File, shareId string, relativePath string) *FileObject { // 构建文件对象 obj := &FileObject{ Object: model.Object{ ID: file.ID, Name: file.Name, Size: file.Size, Modified: time.Unix(file.UpdateTime, 0), Ctime: time.Unix(file.CreateTime, 0), IsFolder: file.NodeType == DirectoryType, Path: path.Join(relativePath, file.Name), }, ShareID: shareId, Key: file.Key, NodeID: file.ID, NodeType: file.NodeType, } return obj } // getFilesInPath 获取指定分享和路径下的文件 func (d *DoubaoShare) getFilesInPath(ctx context.Context, shareId, nodeId, relativePath string) ([]model.Obj, error) { var ( files []File err error ) // 调用overview接口获取分享链接信息 nodeId if nodeId == "" { files, err = d.getShareOverview(shareId, "") if err != nil { return nil, fmt.Errorf("failed to get share link information: %w", err) } result := make([]model.Obj, 0, len(files)) for _, file := range files { result = append(result, d.convertToFileObject(file, shareId, "/")) } return result, nil } else { files, err = d.getFiles(shareId, nodeId, "") if err != nil { return nil, fmt.Errorf("failed to get share file: %w", err) } result := make([]model.Obj, 0, len(files)) for _, file := range files { result = append(result, d.convertToFileObject(file, shareId, path.Join("/", relativePath))) } return result, nil } } // listRootDirectory 处理根目录的内容展示 func (d *DoubaoShare) listRootDirectory(ctx context.Context) ([]model.Obj, error) { objects := make([]model.Obj, 0) // 分组处理：直接显示的分享内容 vs 虚拟目录 var directShareIDs []string addedDirs := make(map[string]bool) // 处理所有根节点 for _, rootFile := range d.RootFiles { if rootFile.VirtualPath == "" && rootFile.ShareID != "" { // 无路径分享，记录ShareID以便后续获取内容 directShareIDs = append(directShareIDs, rootFile.ShareID) } else { // 有路径的分享，显示第一级目录 parts := strings.SplitN(rootFile.VirtualPath, "/", 2) firstLevel := parts[0] // 避免重复添加同名目录 if _, exists := addedDirs[firstLevel]; exists { continue } // 创建虚拟目录对象 obj := &FileObject{ Object: model.Object{ ID: "", Name: firstLevel, Modified: time.Now(), Ctime: time.Now(), IsFolder: true, Path: path.Join("/", firstLevel), }, ShareID: rootFile.ShareID, Key: "", NodeID: "", NodeType: DirectoryType, } objects = append(objects, obj) addedDirs[firstLevel] = true } } // 处理直接显示的分享内容 for _, shareID := range directShareIDs { shareFiles, err := d.getFilesInPath(ctx, shareID, "", "") if err != nil { log.Warnf("[doubao_share] Failed to get list of files in share %s: %s", shareID, err) continue } objects = append(objects, shareFiles...) } return objects, nil } // listVirtualDirectoryContent 列出虚拟目录的内容 func (d *DoubaoShare) listVirtualDirectoryContent(dir model.Obj) ([]model.Obj, error) { dirPath := strings.TrimPrefix(dir.GetPath(), "/") objects := make([]model.Obj, 0) // 递归查找此路径的节点 node := _findNodeByPath(d.RootFiles, dirPath) if node != nil && node.Child != nil { // 显示此节点的所有子节点 for _, child := range *node.Child { // 计算显示名称（取路径的最后一部分） displayName := child.VirtualPath if child.VirtualPath != "" { parts := strings.Split(child.VirtualPath, "/") displayName = parts[len(parts)-1] } else if child.ShareID != "" { displayName = child.ShareID } obj := &FileObject{ Object: model.Object{ ID: "", Name: displayName, Modified: time.Now(), Ctime: time.Now(), IsFolder: true, Path: path.Join("/", child.VirtualPath), }, ShareID: child.ShareID, Key: "", NodeID: "", NodeType: DirectoryType, } objects = append(objects, obj) } } return objects, nil } ================================================ FILE: drivers/dropbox/driver.go ================================================ package dropbox import ( "context" "fmt" "io" "math" "net/http" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) type Dropbox struct { model.Storage Addition base string contentBase string } func (d *Dropbox) Config() driver.Config { return config } func (d *Dropbox) GetAddition() driver.Additional { return &d.Addition } func (d *Dropbox) Init(ctx context.Context) error { query := "foo" res, err := d.request("/2/check/user", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "query": query, }) }) if err != nil { return err } result := utils.Json.Get(res, "result").ToString() if result != query { return fmt.Errorf("failed to check user: %s", string(res)) } d.RootNamespaceId, err = d.GetRootNamespaceId(ctx) return err } func (d *Dropbox) GetRootNamespaceId(ctx context.Context) (string, error) { res, err := d.request("/2/users/get_current_account", http.MethodPost, func(req *resty.Request) { req.SetBody(nil) }) if err != nil { return "", err } var currentAccountResp CurrentAccountResp err = utils.Json.Unmarshal(res, ¤tAccountResp) if err != nil { return "", err } rootNamespaceId := currentAccountResp.RootInfo.RootNamespaceId return rootNamespaceId, nil } func (d *Dropbox) Drop(ctx context.Context) error { return nil } func (d *Dropbox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(ctx, dir.GetPath()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return fileToObj(src), nil }) } func (d *Dropbox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { res, err := d.request("/2/files/get_temporary_link", http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "path": file.GetPath(), }) }) if err != nil { return nil, err } url := utils.Json.Get(res, "link").ToString() exp := time.Hour return &model.Link{ URL: url, Expiration: &exp, }, nil } func (d *Dropbox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { _, err := d.request("/2/files/create_folder_v2", http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "autorename": false, "path": parentDir.GetPath() + "/" + dirName, }) }) return err } func (d *Dropbox) Move(ctx context.Context, srcObj, dstDir model.Obj) error { toPath := dstDir.GetPath() + "/" + srcObj.GetName() _, err := d.request("/2/files/move_v2", http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "allow_ownership_transfer": false, "allow_shared_folder": false, "autorename": false, "from_path": srcObj.GetID(), "to_path": toPath, }) }) return err } func (d *Dropbox) Rename(ctx context.Context, srcObj model.Obj, newName string) error { path := srcObj.GetPath() fileName := srcObj.GetName() toPath := path[:len(path)-len(fileName)] + newName _, err := d.request("/2/files/move_v2", http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "allow_ownership_transfer": false, "allow_shared_folder": false, "autorename": false, "from_path": srcObj.GetID(), "to_path": toPath, }) }) return err } func (d *Dropbox) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { toPath := dstDir.GetPath() + "/" + srcObj.GetName() _, err := d.request("/2/files/copy_v2", http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "allow_ownership_transfer": false, "allow_shared_folder": false, "autorename": false, "from_path": srcObj.GetID(), "to_path": toPath, }) }) return err } func (d *Dropbox) Remove(ctx context.Context, obj model.Obj) error { uri := "/2/files/delete_v2" _, err := d.request(uri, http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "path": obj.GetID(), }) }) return err } func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { // 1. start sessionId, err := d.startUploadSession(ctx) if err != nil { return err } // 2.append // A single request should not upload more than 150 MB, and each call must be multiple of 4MB (except for last call) const PartSize = 20971520 count := 1 if stream.GetSize() > PartSize { count = int(math.Ceil(float64(stream.GetSize()) / float64(PartSize))) } offset := int64(0) for i := 0; i < count; i++ { if utils.IsCanceled(ctx) { return ctx.Err() } start := i * PartSize byteSize := stream.GetSize() - int64(start) if byteSize > PartSize { byteSize = PartSize } url := d.contentBase + "/2/files/upload_session/append_v2" reader := driver.NewLimitedUploadStream(ctx, io.LimitReader(stream, PartSize)) req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, reader) if err != nil { log.Errorf("failed to update file when append to upload session, err: %+v", err) return err } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Authorization", "Bearer "+d.AccessToken) args := UploadAppendArgs{ Close: false, Cursor: UploadCursor{ Offset: offset, SessionID: sessionId, }, } argsJson, err := utils.Json.MarshalToString(args) if err != nil { return err } req.Header.Set("Dropbox-API-Arg", argsJson) res, err := base.HttpClient.Do(req) if err != nil { return err } _ = res.Body.Close() up(float64(i+1) * 100 / float64(count)) offset += byteSize } // 3.finish toPath := dstDir.GetPath() + "/" + stream.GetName() err2 := d.finishUploadSession(ctx, toPath, offset, sessionId) if err2 != nil { return err2 } return err } var _ driver.Driver = (*Dropbox)(nil) ================================================ FILE: drivers/dropbox/meta.go ================================================ package dropbox import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath UseOnlineAPI bool `json:"use_online_api" default:"false"` APIAddress string `json:"api_url_address" default:"https://api.oplist.org/dropboxs/renewapi"` ClientID string `json:"client_id" required:"false" help:"Keep it empty if you don't have one"` ClientSecret string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"` AccessToken string RefreshToken string `json:"refresh_token" required:"true"` RootNamespaceId string `json:"RootNamespaceId" required:"false"` } var config = driver.Config{ Name: "Dropbox", NoOverwriteUpload: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Dropbox{ base: "https://api.dropboxapi.com", contentBase: "https://content.dropboxapi.com", } }) } ================================================ FILE: drivers/dropbox/types.go ================================================ package dropbox import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type TokenResp struct { AccessToken string `json:"access_token"` TokenType string `json:"token_type"` ExpiresIn int `json:"expires_in"` } type ErrorResp struct { Error struct { Tag string `json:".tag"` } `json:"error"` ErrorSummary string `json:"error_summary"` } type RefreshTokenErrorResp struct { Error string `json:"error"` ErrorDescription string `json:"error_description"` } type CurrentAccountResp struct { RootInfo struct { RootNamespaceId string `json:"root_namespace_id"` HomeNamespaceId string `json:"home_namespace_id"` } `json:"root_info"` } type File struct { Tag string `json:".tag"` Name string `json:"name"` PathLower string `json:"path_lower"` PathDisplay string `json:"path_display"` ID string `json:"id"` ClientModified time.Time `json:"client_modified"` ServerModified time.Time `json:"server_modified"` Rev string `json:"rev"` Size int `json:"size"` IsDownloadable bool `json:"is_downloadable"` ContentHash string `json:"content_hash"` } type ListResp struct { Entries []File `json:"entries"` Cursor string `json:"cursor"` HasMore bool `json:"has_more"` } type UploadCursor struct { Offset int64 `json:"offset"` SessionID string `json:"session_id"` } type UploadAppendArgs struct { Close bool `json:"close"` Cursor UploadCursor `json:"cursor"` } type UploadFinishArgs struct { Commit struct { Autorename bool `json:"autorename"` Mode string `json:"mode"` Mute bool `json:"mute"` Path string `json:"path"` StrictConflict bool `json:"strict_conflict"` } `json:"commit"` Cursor UploadCursor `json:"cursor"` } func fileToObj(f File) *model.ObjThumb { return &model.ObjThumb{ Object: model.Object{ ID: f.ID, Path: f.PathDisplay, Name: f.Name, Size: int64(f.Size), Modified: f.ServerModified, IsFolder: f.Tag == "folder", }, Thumbnail: model.Thumbnail{}, } } ================================================ FILE: drivers/dropbox/util.go ================================================ package dropbox import ( "context" "fmt" "io" "net/http" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) func (d *Dropbox) refreshToken() error { // 使用在线API刷新Token，无需ClientID和ClientSecret if d.UseOnlineAPI && len(d.APIAddress) > 0 { u := d.APIAddress var resp struct { RefreshToken string `json:"refresh_token"` AccessToken string `json:"access_token"` ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, "server_use": "true", "driver_txt": "dropboxs_go", }). Get(u) if err != nil { return err } if resp.RefreshToken == "" || resp.AccessToken == "" { if resp.ErrorMessage != "" { return fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) } return fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") } d.AccessToken = resp.AccessToken d.RefreshToken = resp.RefreshToken op.MustSaveDriverStorage(d) return nil } url := d.base + "/oauth2/token" var tokenResp TokenResp resp, err := base.RestyClient.R(). //ForceContentType("application/x-www-form-urlencoded"). //SetBasicAuth(d.ClientID, d.ClientSecret). SetFormData(map[string]string{ "grant_type": "refresh_token", "refresh_token": d.RefreshToken, "client_id": d.ClientID, "client_secret": d.ClientSecret, }). Post(url) if err != nil { return err } log.Debugf("[dropbox] refresh token response: %s", resp.String()) if resp.StatusCode() != 200 { return fmt.Errorf("failed to refresh token: %s", resp.String()) } _ = utils.Json.UnmarshalFromString(resp.String(), &tokenResp) d.AccessToken = tokenResp.AccessToken op.MustSaveDriverStorage(d) return nil } func (d *Dropbox) request(uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) if d.RootNamespaceId != "" { apiPathRootJson, err := utils.Json.MarshalToString(map[string]interface{}{ ".tag": "root", "root": d.RootNamespaceId, }) if err != nil { return nil, err } req.SetHeader("Dropbox-API-Path-Root", apiPathRootJson) } if callback != nil { callback(req) } if method == http.MethodPost && req.Body != nil { req.SetHeader("Content-Type", "application/json") } var e ErrorResp req.SetError(&e) res, err := req.Execute(method, d.base+uri) if err != nil { return nil, err } log.Debugf("[dropbox] request (%s) response: %s", uri, res.String()) isRetry := len(retry) > 0 && retry[0] if res.StatusCode() != 200 { body := res.String() if !isRetry && (utils.SliceMeet([]string{"expired_access_token", "invalid_access_token", "authorization"}, body, func(item string, v string) bool { return strings.Contains(v, item) }) || d.AccessToken == "") { err = d.refreshToken() if err != nil { return nil, err } return d.request(uri, method, callback, true) } return nil, fmt.Errorf("%s:%s", e.Error, e.ErrorSummary) } return res.Body(), nil } func (d *Dropbox) list(ctx context.Context, data base.Json, isContinue bool) (*ListResp, error) { var resp ListResp uri := "/2/files/list_folder" if isContinue { uri += "/continue" } _, err := d.request(uri, http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(data).SetResult(&resp) }) if err != nil { return nil, err } return &resp, nil } func (d *Dropbox) getFiles(ctx context.Context, path string) ([]File, error) { hasMore := true var marker string res := make([]File, 0) data := base.Json{ "include_deleted": false, "include_has_explicit_shared_members": false, "include_mounted_folders": false, "include_non_downloadable_files": false, "limit": 2000, "path": path, "recursive": false, } resp, err := d.list(ctx, data, false) if err != nil { return nil, err } marker = resp.Cursor hasMore = resp.HasMore res = append(res, resp.Entries...) for hasMore { data := base.Json{ "cursor": marker, } resp, err := d.list(ctx, data, true) if err != nil { return nil, err } marker = resp.Cursor hasMore = resp.HasMore res = append(res, resp.Entries...) } return res, nil } func (d *Dropbox) finishUploadSession(ctx context.Context, toPath string, offset int64, sessionId string) error { url := d.contentBase + "/2/files/upload_session/finish" req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) if err != nil { return err } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Authorization", "Bearer "+d.AccessToken) if d.RootNamespaceId != "" { apiPathRootJson, err := d.buildPathRootHeader() if err != nil { return err } req.Header.Set("Dropbox-API-Path-Root", apiPathRootJson) } uploadFinishArgs := UploadFinishArgs{ Commit: struct { Autorename bool `json:"autorename"` Mode string `json:"mode"` Mute bool `json:"mute"` Path string `json:"path"` StrictConflict bool `json:"strict_conflict"` }{ Autorename: true, Mode: "add", Mute: false, Path: toPath, StrictConflict: false, }, Cursor: UploadCursor{ Offset: offset, SessionID: sessionId, }, } argsJson, err := utils.Json.MarshalToString(uploadFinishArgs) if err != nil { return err } req.Header.Set("Dropbox-API-Arg", argsJson) res, err := base.HttpClient.Do(req) if err != nil { log.Errorf("failed to update file when finish session, err: %+v", err) return err } _ = res.Body.Close() return nil } func (d *Dropbox) startUploadSession(ctx context.Context) (string, error) { url := d.contentBase + "/2/files/upload_session/start" req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) if err != nil { return "", err } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Authorization", "Bearer "+d.AccessToken) if d.RootNamespaceId != "" { apiPathRootJson, err := d.buildPathRootHeader() if err != nil { return "", err } req.Header.Set("Dropbox-API-Path-Root", apiPathRootJson) } req.Header.Set("Dropbox-API-Arg", "{\"close\":false}") res, err := base.HttpClient.Do(req) if err != nil { log.Errorf("failed to update file when start session, err: %+v", err) return "", err } body, err := io.ReadAll(res.Body) sessionId := utils.Json.Get(body, "session_id").ToString() _ = res.Body.Close() return sessionId, nil } func (d *Dropbox) buildPathRootHeader() (string, error) { return utils.Json.MarshalToString(map[string]interface{}{ ".tag": "root", "root": d.RootNamespaceId, }) } ================================================ FILE: drivers/febbox/driver.go ================================================ package febbox import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "golang.org/x/oauth2" "golang.org/x/oauth2/clientcredentials" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type FebBox struct { model.Storage Addition accessToken string oauth2Token oauth2.TokenSource } func (d *FebBox) Config() driver.Config { return config } func (d *FebBox) GetAddition() driver.Additional { return &d.Addition } func (d *FebBox) Init(ctx context.Context) error { // 初始化 oauth2Config oauth2Config := &clientcredentials.Config{ ClientID: d.ClientID, ClientSecret: d.ClientSecret, AuthStyle: oauth2.AuthStyleInParams, TokenURL: "https://api.febbox.com/oauth/token", } d.initializeOAuth2Token(ctx, oauth2Config, d.Addition.RefreshToken) token, err := d.oauth2Token.Token() if err != nil { return err } d.accessToken = token.AccessToken d.Addition.RefreshToken = token.RefreshToken op.MustSaveDriverStorage(d) return nil } func (d *FebBox) Drop(ctx context.Context) error { return nil } func (d *FebBox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFilesList(dir.GetID()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return fileToObj(src), nil }) } func (d *FebBox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var ip string if d.Addition.UserIP != "" { ip = d.Addition.UserIP } else { ip = args.IP } url, err := d.getDownloadLink(file.GetID(), ip) if err != nil { return nil, err } return &model.Link{ URL: url, }, nil } func (d *FebBox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { err := d.makeDir(parentDir.GetID(), dirName) if err != nil { return nil, err } return nil, nil } func (d *FebBox) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { err := d.move(srcObj.GetID(), dstDir.GetID()) if err != nil { return nil, err } return nil, nil } func (d *FebBox) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { err := d.rename(srcObj.GetID(), newName) if err != nil { return nil, err } return nil, nil } func (d *FebBox) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { err := d.copy(srcObj.GetID(), dstDir.GetID()) if err != nil { return nil, err } return nil, nil } func (d *FebBox) Remove(ctx context.Context, obj model.Obj) error { err := d.remove(obj.GetID()) if err != nil { return err } return nil } func (d *FebBox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { return nil, errs.NotImplement } var _ driver.Driver = (*FebBox)(nil) ================================================ FILE: drivers/febbox/meta.go ================================================ package febbox import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootID ClientID string `json:"client_id" required:"true" default:""` ClientSecret string `json:"client_secret" required:"true" default:""` RefreshToken string SortRule string `json:"sort_rule" required:"true" type:"select" options:"size_asc,size_desc,name_asc,name_desc,update_asc,update_desc,ext_asc,ext_desc" default:"name_asc"` PageSize int64 `json:"page_size" required:"true" type:"number" default:"100" help:"list api per page size of FebBox driver"` UserIP string `json:"user_ip" default:"" help:"user ip address for download link which can speed up the download"` } var config = driver.Config{ Name: "FebBox", NoUpload: true, DefaultRoot: "0", LinkCacheMode: driver.LinkCacheIP, } func init() { op.RegisterDriver(func() driver.Driver { return &FebBox{} }) } ================================================ FILE: drivers/febbox/oauth2.go ================================================ package febbox import ( "context" "encoding/json" "errors" "net/http" "net/url" "strings" "time" "golang.org/x/oauth2" "golang.org/x/oauth2/clientcredentials" ) type customTokenSource struct { config *clientcredentials.Config ctx context.Context refreshToken string } func (c *customTokenSource) Token() (*oauth2.Token, error) { v := url.Values{} if c.refreshToken != "" { v.Set("grant_type", "refresh_token") v.Set("refresh_token", c.refreshToken) } else { v.Set("grant_type", "client_credentials") } v.Set("client_id", c.config.ClientID) v.Set("client_secret", c.config.ClientSecret) req, err := http.NewRequestWithContext(c.ctx, http.MethodPost, c.config.TokenURL, strings.NewReader(v.Encode())) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, errors.New("oauth2: cannot fetch token") } var tokenResp struct { Code int `json:"code"` Msg string `json:"msg"` Data struct { AccessToken string `json:"access_token"` ExpiresIn int64 `json:"expires_in"` TokenType string `json:"token_type"` Scope string `json:"scope"` RefreshToken string `json:"refresh_token"` } `json:"data"` } if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil { return nil, err } if tokenResp.Code != 1 { return nil, errors.New("oauth2: server response error") } c.refreshToken = tokenResp.Data.RefreshToken token := &oauth2.Token{ AccessToken: tokenResp.Data.AccessToken, TokenType: tokenResp.Data.TokenType, RefreshToken: tokenResp.Data.RefreshToken, Expiry: time.Now().Add(time.Duration(tokenResp.Data.ExpiresIn) * time.Second), } return token, nil } func (d *FebBox) initializeOAuth2Token(ctx context.Context, oauth2Config *clientcredentials.Config, refreshToken string) { d.oauth2Token = oauth2.ReuseTokenSource(nil, &customTokenSource{ config: oauth2Config, ctx: ctx, refreshToken: refreshToken, }) } ================================================ FILE: drivers/febbox/types.go ================================================ package febbox import ( "fmt" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" ) type ErrResp struct { ErrorCode int64 `json:"code"` ErrorMsg string `json:"msg"` ServerRunTime float64 `json:"server_runtime"` ServerName string `json:"server_name"` } func (e *ErrResp) IsError() bool { return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ServerRunTime != 0 || e.ServerName != "" } func (e *ErrResp) Error() string { return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ServerRunTime: %f ,ServerName: %s", e.ErrorCode, e.ErrorMsg, e.ServerRunTime, e.ServerName) } type FileListResp struct { Code int `json:"code"` Msg string `json:"msg"` Data struct { FileList []File `json:"file_list"` ShowType string `json:"show_type"` } `json:"data"` } type Rules struct { AllowCopy int64 `json:"allow_copy"` AllowDelete int64 `json:"allow_delete"` AllowDownload int64 `json:"allow_download"` AllowComment int64 `json:"allow_comment"` HideLocation int64 `json:"hide_location"` } type File struct { Fid int64 `json:"fid"` UID int64 `json:"uid"` FileSize int64 `json:"file_size"` Path string `json:"path"` FileName string `json:"file_name"` Ext string `json:"ext"` AddTime int64 `json:"add_time"` FileCreateTime int64 `json:"file_create_time"` FileUpdateTime int64 `json:"file_update_time"` ParentID int64 `json:"parent_id"` UpdateTime int64 `json:"update_time"` LastOpenTime int64 `json:"last_open_time"` IsDir int64 `json:"is_dir"` Epub int64 `json:"epub"` IsMusicList int64 `json:"is_music_list"` OssFid int64 `json:"oss_fid"` Faststart int64 `json:"faststart"` HasVideoQuality int64 `json:"has_video_quality"` TotalDownload int64 `json:"total_download"` Status int64 `json:"status"` Remark string `json:"remark"` OldHash string `json:"old_hash"` Hash string `json:"hash"` HashType string `json:"hash_type"` FromUID int64 `json:"from_uid"` FidOrg int64 `json:"fid_org"` ShareID int64 `json:"share_id"` InvitePermission int64 `json:"invite_permission"` ThumbSmall string `json:"thumb_small"` ThumbSmallWidth int64 `json:"thumb_small_width"` ThumbSmallHeight int64 `json:"thumb_small_height"` Thumb string `json:"thumb"` ThumbWidth int64 `json:"thumb_width"` ThumbHeight int64 `json:"thumb_height"` ThumbBig string `json:"thumb_big"` ThumbBigWidth int64 `json:"thumb_big_width"` ThumbBigHeight int64 `json:"thumb_big_height"` IsCustomThumb int64 `json:"is_custom_thumb"` Photos int64 `json:"photos"` IsAlbum int64 `json:"is_album"` ReadOnly int64 `json:"read_only"` Rules Rules `json:"rules"` IsShared int64 `json:"is_shared"` } func fileToObj(f File) *model.ObjThumb { return &model.ObjThumb{ Object: model.Object{ ID: strconv.FormatInt(f.Fid, 10), Name: f.FileName, Size: f.FileSize, Ctime: time.Unix(f.FileCreateTime, 0), Modified: time.Unix(f.FileUpdateTime, 0), IsFolder: f.IsDir == 1, HashInfo: utils.NewHashInfo(hash_extend.GCID, f.Hash), }, Thumbnail: model.Thumbnail{ Thumbnail: f.Thumb, }, } } type FileDownloadResp struct { Code int `json:"code"` Msg string `json:"msg"` Data []struct { Error int `json:"error"` DownloadURL string `json:"download_url"` Hash string `json:"hash"` HashType string `json:"hash_type"` Fid int `json:"fid"` FileName string `json:"file_name"` ParentID int `json:"parent_id"` FileSize int `json:"file_size"` Ext string `json:"ext"` Thumb string `json:"thumb"` VipLink int `json:"vip_link"` } `json:"data"` } ================================================ FILE: drivers/febbox/util.go ================================================ package febbox import ( "encoding/json" "errors" "fmt" "net/http" "strconv" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/go-resty/resty/v2" ) func (d *FebBox) refreshTokenByOAuth2() error { token, err := d.oauth2Token.Token() if err != nil { return err } d.Status = "work" d.accessToken = token.AccessToken d.Addition.RefreshToken = token.RefreshToken op.MustSaveDriverStorage(d) return nil } func (d *FebBox) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() // 使用oauth2 获取 access_token token, err := d.oauth2Token.Token() if err != nil { return nil, err } req.SetAuthScheme(token.TokenType).SetAuthToken(token.AccessToken) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e ErrResp req.SetError(&e) res, err := req.Execute(method, url) if err != nil { return nil, err } switch e.ErrorCode { case 0: return res.Body(), nil case 1: return res.Body(), nil case -10001: if e.ServerName != "" { // access_token 过期 if err = d.refreshTokenByOAuth2(); err != nil { return nil, err } return d.request(url, method, callback, resp) } else { return nil, errors.New(e.Error()) } default: return nil, errors.New(e.Error()) } } func (d *FebBox) getFilesList(id string) ([]File, error) { if d.PageSize <= 0 { d.PageSize = 100 } res, err := d.listWithLimit(id, d.PageSize) if err != nil { return nil, err } return *res, nil } func (d *FebBox) listWithLimit(dirID string, pageLimit int64) (*[]File, error) { var files []File page := int64(1) for { result, err := d.getFiles(dirID, page, pageLimit) if err != nil { return nil, err } files = append(files, *result...) if int64(len(*result)) < pageLimit { break } else { page++ } } return &files, nil } func (d *FebBox) getFiles(dirID string, page, pageLimit int64) (*[]File, error) { var fileList FileListResp queryParams := map[string]string{ "module": "file_list", "parent_id": dirID, "page": strconv.FormatInt(page, 10), "pagelimit": strconv.FormatInt(pageLimit, 10), "order": d.Addition.SortRule, } res, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { req.SetMultipartFormData(queryParams) }, &fileList) if err != nil { return nil, err } if err = json.Unmarshal(res, &fileList); err != nil { return nil, err } return &fileList.Data.FileList, nil } func (d *FebBox) getDownloadLink(id string, ip string) (string, error) { var fileDownloadResp FileDownloadResp queryParams := map[string]string{ "module": "file_get_download_url", "fids[]": id, "ip": ip, } res, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { req.SetMultipartFormData(queryParams) }, &fileDownloadResp) if err != nil { return "", err } if err = json.Unmarshal(res, &fileDownloadResp); err != nil { return "", err } if len(fileDownloadResp.Data) == 0 { return "", fmt.Errorf("can not get download link, code:%d, msg:%s", fileDownloadResp.Code, fileDownloadResp.Msg) } return fileDownloadResp.Data[0].DownloadURL, nil } func (d *FebBox) makeDir(id string, name string) error { queryParams := map[string]string{ "module": "create_dir", "parent_id": id, "name": name, } _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { req.SetMultipartFormData(queryParams) }, nil) if err != nil { return err } return nil } func (d *FebBox) move(id string, id2 string) error { queryParams := map[string]string{ "module": "file_move", "fids[]": id, "to": id2, } _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { req.SetMultipartFormData(queryParams) }, nil) if err != nil { return err } return nil } func (d *FebBox) rename(id string, name string) error { queryParams := map[string]string{ "module": "file_rename", "fid": id, "name": name, } _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { req.SetMultipartFormData(queryParams) }, nil) if err != nil { return err } return nil } func (d *FebBox) copy(id string, id2 string) error { queryParams := map[string]string{ "module": "file_copy", "fids[]": id, "to": id2, } _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { req.SetMultipartFormData(queryParams) }, nil) if err != nil { return err } return nil } func (d *FebBox) remove(id string) error { queryParams := map[string]string{ "module": "file_delete", "fids[]": id, } _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { req.SetMultipartFormData(queryParams) }, nil) if err != nil { return err } return nil } ================================================ FILE: drivers/ftp/driver.go ================================================ package ftp import ( "context" "errors" "io" stdpath "path" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/jlaffaye/ftp" ) type FTP struct { model.Storage Addition conn *ftp.ServerConn ctx context.Context cancel context.CancelFunc } func (d *FTP) Config() driver.Config { return config } func (d *FTP) GetAddition() driver.Additional { return &d.Addition } func (d *FTP) Init(ctx context.Context) error { d.ctx, d.cancel = context.WithCancel(context.Background()) var err error d.conn, err = d._login(ctx) return err } func (d *FTP) Drop(ctx context.Context) error { if d.conn != nil { _ = d.conn.Quit() d.cancel() } return nil } func (d *FTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { if err := d.login(); err != nil { return nil, err } entries, err := d.conn.List(encode(dir.GetPath(), d.Encoding)) if err != nil { return nil, err } res := make([]model.Obj, 0) for _, entry := range entries { if entry.Name == "." || entry.Name == ".." { continue } name := decode(entry.Name, d.Encoding) f := model.Object{ Name: name, Size: int64(entry.Size), Modified: entry.Time, IsFolder: entry.Type == ftp.EntryTypeFolder, Path: stdpath.Join(dir.GetPath(), name), } res = append(res, &f) } return res, nil } func (d *FTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { conn, err := d._login(ctx) if err != nil { return nil, err } path := encode(file.GetPath(), d.Encoding) size := file.GetSize() resultRangeReader := func(context context.Context, httpRange http_range.Range) (io.ReadCloser, error) { length := httpRange.Length if length < 0 || httpRange.Start+length > size { length = size - httpRange.Start } var c *ftp.ServerConn if ctx == context { c = conn } else { var err error c, err = d._login(context) if err != nil { return nil, err } } resp, err := c.RetrFrom(path, uint64(httpRange.Start)) if err != nil { return nil, err } var close utils.CloseFunc if context == ctx { close = resp.Close } else { close = func() error { return errors.Join(resp.Close(), c.Quit()) } } return utils.ReadCloser{ Reader: io.LimitReader(resp, length), Closer: close, }, nil } return &model.Link{ RangeReader: stream.RateLimitRangeReaderFunc(resultRangeReader), SyncClosers: utils.NewSyncClosers(utils.CloseFunc(conn.Quit)), }, nil } func (d *FTP) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { if err := d.login(); err != nil { return err } return d.conn.MakeDir(encode(stdpath.Join(parentDir.GetPath(), dirName), d.Encoding)) } func (d *FTP) Move(ctx context.Context, srcObj, dstDir model.Obj) error { if err := d.login(); err != nil { return err } return d.conn.Rename( encode(srcObj.GetPath(), d.Encoding), encode(stdpath.Join(dstDir.GetPath(), srcObj.GetName()), d.Encoding), ) } func (d *FTP) Rename(ctx context.Context, srcObj model.Obj, newName string) error { if err := d.login(); err != nil { return err } return d.conn.Rename( encode(srcObj.GetPath(), d.Encoding), encode(stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), d.Encoding), ) } func (d *FTP) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return errs.NotSupport } func (d *FTP) Remove(ctx context.Context, obj model.Obj) error { if err := d.login(); err != nil { return err } path := encode(obj.GetPath(), d.Encoding) if obj.IsDir() { return d.conn.RemoveDirRecur(path) } else { return d.conn.Delete(path) } } func (d *FTP) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { if err := d.login(); err != nil { return err } path := stdpath.Join(dstDir.GetPath(), s.GetName()) return d.conn.Stor(encode(path, d.Encoding), driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, })) } var _ driver.Driver = (*FTP)(nil) ================================================ FILE: drivers/ftp/meta.go ================================================ package ftp import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/axgle/mahonia" ) func encode(str string, encoding string) string { if encoding == "" { return str } encoder := mahonia.NewEncoder(encoding) return encoder.ConvertString(str) } func decode(str string, encoding string) string { if encoding == "" { return str } decoder := mahonia.NewDecoder(encoding) return decoder.ConvertString(str) } type Addition struct { Address string `json:"address" required:"true"` Encoding string `json:"encoding" required:"true"` Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` driver.RootPath } var config = driver.Config{ Name: "FTP", LocalSort: true, OnlyProxy: true, DefaultRoot: "/", NoLinkURL: true, } func init() { op.RegisterDriver(func() driver.Driver { return &FTP{} }) } ================================================ FILE: drivers/ftp/types.go ================================================ package ftp ================================================ FILE: drivers/ftp/util.go ================================================ package ftp import ( "context" "fmt" "time" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/jlaffaye/ftp" ) // do others that not defined in Driver interface func (d *FTP) login() error { _, err, _ := singleflight.AnyGroup.Do(fmt.Sprintf("FTP.login:%p", d), func() (any, error) { var err error if d.conn != nil { err = d.conn.NoOp() if err != nil { d.conn.Quit() d.conn = nil } } if d.conn == nil { d.conn, err = d._login(d.ctx) } return nil, err }) return err } func (d *FTP) _login(ctx context.Context) (*ftp.ServerConn, error) { conn, err := ftp.Dial(d.Address, ftp.DialWithShutTimeout(10*time.Second), ftp.DialWithContext(ctx)) if err != nil { return nil, err } err = conn.Login(d.Username, d.Password) if err != nil { conn.Quit() return nil, err } return conn, nil } ================================================ FILE: drivers/github/driver.go ================================================ package github import ( "context" "encoding/base64" "fmt" "io" "net/http" stdpath "path" "strings" "sync" "text/template" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/ProtonMail/go-crypto/openpgp" "github.com/go-resty/resty/v2" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) type Github struct { model.Storage Addition client *resty.Client mkdirMsgTmpl *template.Template deleteMsgTmpl *template.Template putMsgTmpl *template.Template renameMsgTmpl *template.Template copyMsgTmpl *template.Template moveMsgTmpl *template.Template isOnBranch bool commitMutex sync.Mutex pgpEntity *openpgp.Entity } func (d *Github) Config() driver.Config { return config } func (d *Github) GetAddition() driver.Additional { return &d.Addition } func (d *Github) Init(ctx context.Context) error { d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) if d.CommitterName != "" && d.CommitterEmail == "" { return errors.New("committer email is required") } if d.CommitterName == "" && d.CommitterEmail != "" { return errors.New("committer name is required") } if d.AuthorName != "" && d.AuthorEmail == "" { return errors.New("author email is required") } if d.AuthorName == "" && d.AuthorEmail != "" { return errors.New("author name is required") } var err error d.mkdirMsgTmpl, err = template.New("mkdirCommitMsgTemplate").Parse(d.MkdirCommitMsg) if err != nil { return err } d.deleteMsgTmpl, err = template.New("deleteCommitMsgTemplate").Parse(d.DeleteCommitMsg) if err != nil { return err } d.putMsgTmpl, err = template.New("putCommitMsgTemplate").Parse(d.PutCommitMsg) if err != nil { return err } d.renameMsgTmpl, err = template.New("renameCommitMsgTemplate").Parse(d.RenameCommitMsg) if err != nil { return err } d.copyMsgTmpl, err = template.New("copyCommitMsgTemplate").Parse(d.CopyCommitMsg) if err != nil { return err } d.moveMsgTmpl, err = template.New("moveCommitMsgTemplate").Parse(d.MoveCommitMsg) if err != nil { return err } d.client = base.NewRestyClient(). SetHeader("Accept", "application/vnd.github.object+json"). SetHeader("X-GitHub-Api-Version", "2022-11-28"). SetLogger(log.StandardLogger()). SetDebug(false) token := strings.TrimSpace(d.Token) if token != "" { d.client = d.client.SetHeader("Authorization", "Bearer "+token) } if d.Ref == "" { repo, err := d.getRepo() if err != nil { return err } d.Ref = repo.DefaultBranch d.isOnBranch = true } else { _, err = d.getBranchHead() d.isOnBranch = err == nil } if d.GPGPrivateKey != "" { if d.CommitterName == "" || d.AuthorName == "" { user, e := d.getAuthenticatedUser() if e != nil { return e } if d.CommitterName == "" { d.CommitterName = user.Name d.CommitterEmail = user.Email } if d.AuthorName == "" { d.AuthorName = user.Name d.AuthorEmail = user.Email } } d.pgpEntity, err = loadPrivateKey(d.GPGPrivateKey, d.GPGKeyPassphrase) if err != nil { return err } } return nil } func (d *Github) Drop(ctx context.Context) error { return nil } func (d *Github) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { obj, err := d.get(dir.GetPath()) if err != nil { return nil, err } if obj.Entries == nil { return nil, errs.NotFolder } if len(obj.Entries) >= 1000 { tree, err := d.getTree(obj.Sha) if err != nil { return nil, err } if tree.Truncated { return nil, fmt.Errorf("tree %s is truncated", dir.GetPath()) } ret := make([]model.Obj, 0, len(tree.Trees)) for _, t := range tree.Trees { if t.Path != ".gitkeep" { ret = append(ret, t.toModelObj()) } } return ret, nil } else { ret := make([]model.Obj, 0, len(obj.Entries)) for _, entry := range obj.Entries { if entry.Name != ".gitkeep" { ret = append(ret, entry.toModelObj()) } } return ret, nil } } func (d *Github) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { obj, err := d.get(file.GetPath()) if err != nil { return nil, err } if obj.Type == "submodule" { return nil, errors.New("cannot download a submodule") } url := obj.DownloadURL ghProxy := strings.TrimSpace(d.Addition.GitHubProxy) if ghProxy != "" { url = strings.Replace(url, "https://raw.githubusercontent.com", ghProxy, 1) } return &model.Link{ URL: url, }, nil } func (d *Github) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { if !d.isOnBranch { return errors.New("cannot write to non-branch reference") } d.commitMutex.Lock() defer d.commitMutex.Unlock() parent, err := d.get(parentDir.GetPath()) if err != nil { return err } if parent.Entries == nil { return errs.NotFolder } subDirSha, err := d.newTree("", []interface{}{ map[string]string{ "path": ".gitkeep", "mode": "100644", "type": "blob", "content": "", }, }) if err != nil { return err } newTree := make([]interface{}, 0, 2) newTree = append(newTree, TreeObjReq{ Path: dirName, Mode: "040000", Type: "tree", Sha: subDirSha, }) if len(parent.Entries) == 1 && parent.Entries[0].Name == ".gitkeep" { newTree = append(newTree, TreeObjReq{ Path: ".gitkeep", Mode: "100644", Type: "blob", Sha: nil, }) } newSha, err := d.newTree(parent.Sha, newTree) if err != nil { return err } rootSha, err := d.renewParentTrees(parentDir.GetPath(), parent.Sha, newSha, "/") if err != nil { return err } commitMessage, err := getMessage(d.mkdirMsgTmpl, &MessageTemplateVars{ UserName: getUsername(ctx), ObjName: dirName, ObjPath: stdpath.Join(parentDir.GetPath(), dirName), ParentName: parentDir.GetName(), ParentPath: parentDir.GetPath(), }, "mkdir") if err != nil { return err } return d.commit(commitMessage, rootSha) } func (d *Github) Move(ctx context.Context, srcObj, dstDir model.Obj) error { if !d.isOnBranch { return errors.New("cannot write to non-branch reference") } if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { return errors.New("cannot move parent dir to child") } d.commitMutex.Lock() defer d.commitMutex.Unlock() var rootSha string if strings.HasPrefix(dstDir.GetPath(), stdpath.Dir(srcObj.GetPath())) { // /aa/1 -> /aa/bb/ dstOldSha, dstNewSha, ancestorOldSha, srcParentTree, err := d.copyWithoutRenewTree(srcObj, dstDir) if err != nil { return err } srcParentPath := stdpath.Dir(srcObj.GetPath()) dstRest := dstDir.GetPath()[len(srcParentPath):] if dstRest[0] == '/' { dstRest = dstRest[1:] } dstNextName, _, _ := strings.Cut(dstRest, "/") dstNextPath := stdpath.Join(srcParentPath, dstNextName) dstNextTreeSha, err := d.renewParentTrees(dstDir.GetPath(), dstOldSha, dstNewSha, dstNextPath) if err != nil { return err } var delSrc, dstNextTree *TreeObjReq = nil, nil for _, t := range srcParentTree.Trees { if t.Path == dstNextName { dstNextTree = &t.TreeObjReq dstNextTree.Sha = dstNextTreeSha } if t.Path == srcObj.GetName() { delSrc = &t.TreeObjReq delSrc.Sha = nil } if delSrc != nil && dstNextTree != nil { break } } if delSrc == nil || dstNextTree == nil { return errs.ObjectNotFound } ancestorNewSha, err := d.newTree(ancestorOldSha, []interface{}{*delSrc, *dstNextTree}) if err != nil { return err } rootSha, err = d.renewParentTrees(srcParentPath, ancestorOldSha, ancestorNewSha, "/") if err != nil { return err } } else if strings.HasPrefix(srcObj.GetPath(), dstDir.GetPath()) { // /aa/bb/1 -> /aa/ srcParentPath := stdpath.Dir(srcObj.GetPath()) srcParentTree, srcParentOldSha, err := d.getTreeDirectly(srcParentPath) if err != nil { return err } var src *TreeObjReq = nil for _, t := range srcParentTree.Trees { if t.Path == srcObj.GetName() { if t.Type == "commit" { return errors.New("cannot move a submodule") } src = &t.TreeObjReq break } } if src == nil { return errs.ObjectNotFound } delSrc := *src delSrc.Sha = nil delSrcTree := make([]interface{}, 0, 2) delSrcTree = append(delSrcTree, delSrc) if len(srcParentTree.Trees) == 1 { delSrcTree = append(delSrcTree, map[string]string{ "path": ".gitkeep", "mode": "100644", "type": "blob", "content": "", }) } srcParentNewSha, err := d.newTree(srcParentOldSha, delSrcTree) if err != nil { return err } srcRest := srcObj.GetPath()[len(dstDir.GetPath()):] if srcRest[0] == '/' { srcRest = srcRest[1:] } srcNextName, _, ok := strings.Cut(srcRest, "/") if !ok { // /aa/1 -> /aa/ return errors.New("cannot move in place") } srcNextPath := stdpath.Join(dstDir.GetPath(), srcNextName) srcNextTreeSha, err := d.renewParentTrees(srcParentPath, srcParentOldSha, srcParentNewSha, srcNextPath) if err != nil { return err } ancestorTree, ancestorOldSha, err := d.getTreeDirectly(dstDir.GetPath()) if err != nil { return err } var srcNextTree *TreeObjReq = nil for _, t := range ancestorTree.Trees { if t.Path == srcNextName { srcNextTree = &t.TreeObjReq srcNextTree.Sha = srcNextTreeSha break } } if srcNextTree == nil { return errs.ObjectNotFound } ancestorNewSha, err := d.newTree(ancestorOldSha, []interface{}{*srcNextTree, *src}) if err != nil { return err } rootSha, err = d.renewParentTrees(dstDir.GetPath(), ancestorOldSha, ancestorNewSha, "/") if err != nil { return err } } else { // /aa/1 -> /bb/ // do copy dstOldSha, dstNewSha, srcParentOldSha, srcParentTree, err := d.copyWithoutRenewTree(srcObj, dstDir) if err != nil { return err } // delete src object and create new tree var srcNewTree *TreeObjReq = nil for _, t := range srcParentTree.Trees { if t.Path == srcObj.GetName() { srcNewTree = &t.TreeObjReq srcNewTree.Sha = nil break } } if srcNewTree == nil { return errs.ObjectNotFound } delSrcTree := make([]interface{}, 0, 2) delSrcTree = append(delSrcTree, *srcNewTree) if len(srcParentTree.Trees) == 1 { delSrcTree = append(delSrcTree, map[string]string{ "path": ".gitkeep", "mode": "100644", "type": "blob", "content": "", }) } srcParentNewSha, err := d.newTree(srcParentOldSha, delSrcTree) if err != nil { return err } // renew but the common ancestor of srcPath and dstPath ancestor, srcChildName, dstChildName, _, _ := getPathCommonAncestor(srcObj.GetPath(), dstDir.GetPath()) dstNextTreeSha, err := d.renewParentTrees(dstDir.GetPath(), dstOldSha, dstNewSha, stdpath.Join(ancestor, dstChildName)) if err != nil { return err } srcNextTreeSha, err := d.renewParentTrees(stdpath.Dir(srcObj.GetPath()), srcParentOldSha, srcParentNewSha, stdpath.Join(ancestor, srcChildName)) if err != nil { return err } // renew the tree of the last common ancestor ancestorTree, ancestorOldSha, err := d.getTreeDirectly(ancestor) if err != nil { return err } newTree := make([]interface{}, 2) srcBind := false dstBind := false for _, t := range ancestorTree.Trees { if t.Path == srcChildName { t.Sha = srcNextTreeSha newTree[0] = t.TreeObjReq srcBind = true } if t.Path == dstChildName { t.Sha = dstNextTreeSha newTree[1] = t.TreeObjReq dstBind = true } if srcBind && dstBind { break } } if !srcBind || !dstBind { return errs.ObjectNotFound } ancestorNewSha, err := d.newTree(ancestorOldSha, newTree) if err != nil { return err } // renew until root rootSha, err = d.renewParentTrees(ancestor, ancestorOldSha, ancestorNewSha, "/") if err != nil { return err } } // commit message, err := getMessage(d.moveMsgTmpl, &MessageTemplateVars{ UserName: getUsername(ctx), ObjName: srcObj.GetName(), ObjPath: srcObj.GetPath(), ParentName: stdpath.Base(stdpath.Dir(srcObj.GetPath())), ParentPath: stdpath.Dir(srcObj.GetPath()), TargetName: stdpath.Base(dstDir.GetPath()), TargetPath: dstDir.GetPath(), }, "move") if err != nil { return err } return d.commit(message, rootSha) } func (d *Github) Rename(ctx context.Context, srcObj model.Obj, newName string) error { if !d.isOnBranch { return errors.New("cannot write to non-branch reference") } d.commitMutex.Lock() defer d.commitMutex.Unlock() parentDir := stdpath.Dir(srcObj.GetPath()) tree, _, err := d.getTreeDirectly(parentDir) if err != nil { return err } newTree := make([]interface{}, 2) operated := false for _, t := range tree.Trees { if t.Path == srcObj.GetName() { if t.Type == "commit" { return errors.New("cannot rename a submodule") } delCopy := t.TreeObjReq delCopy.Sha = nil newTree[0] = delCopy t.Path = newName newTree[1] = t.TreeObjReq operated = true break } } if !operated { return errs.ObjectNotFound } newSha, err := d.newTree(tree.Sha, newTree) if err != nil { return err } rootSha, err := d.renewParentTrees(parentDir, tree.Sha, newSha, "/") if err != nil { return err } message, err := getMessage(d.renameMsgTmpl, &MessageTemplateVars{ UserName: getUsername(ctx), ObjName: srcObj.GetName(), ObjPath: srcObj.GetPath(), ParentName: stdpath.Base(parentDir), ParentPath: parentDir, TargetName: newName, TargetPath: stdpath.Join(parentDir, newName), }, "rename") if err != nil { return err } return d.commit(message, rootSha) } func (d *Github) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { if !d.isOnBranch { return errors.New("cannot write to non-branch reference") } if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { return errors.New("cannot copy parent dir to child") } d.commitMutex.Lock() defer d.commitMutex.Unlock() dstSha, newSha, _, _, err := d.copyWithoutRenewTree(srcObj, dstDir) if err != nil { return err } rootSha, err := d.renewParentTrees(dstDir.GetPath(), dstSha, newSha, "/") if err != nil { return err } message, err := getMessage(d.copyMsgTmpl, &MessageTemplateVars{ UserName: getUsername(ctx), ObjName: srcObj.GetName(), ObjPath: srcObj.GetPath(), ParentName: stdpath.Base(stdpath.Dir(srcObj.GetPath())), ParentPath: stdpath.Dir(srcObj.GetPath()), TargetName: stdpath.Base(dstDir.GetPath()), TargetPath: dstDir.GetPath(), }, "copy") if err != nil { return err } return d.commit(message, rootSha) } func (d *Github) Remove(ctx context.Context, obj model.Obj) error { if !d.isOnBranch { return errors.New("cannot write to non-branch reference") } d.commitMutex.Lock() defer d.commitMutex.Unlock() parentDir := stdpath.Dir(obj.GetPath()) tree, treeSha, err := d.getTreeDirectly(parentDir) if err != nil { return err } var del *TreeObjReq = nil for _, t := range tree.Trees { if t.Path == obj.GetName() { if t.Type == "commit" { return errors.New("cannot remove a submodule") } del = &t.TreeObjReq del.Sha = nil break } } if del == nil { return errs.ObjectNotFound } newTree := make([]interface{}, 0, 2) newTree = append(newTree, *del) if len(tree.Trees) == 1 { // completely emptying the repository will get a 404 newTree = append(newTree, map[string]string{ "path": ".gitkeep", "mode": "100644", "type": "blob", "content": "", }) } newSha, err := d.newTree(treeSha, newTree) if err != nil { return err } rootSha, err := d.renewParentTrees(parentDir, treeSha, newSha, "/") if err != nil { return err } commitMessage, err := getMessage(d.deleteMsgTmpl, &MessageTemplateVars{ UserName: getUsername(ctx), ObjName: obj.GetName(), ObjPath: obj.GetPath(), ParentName: stdpath.Base(parentDir), ParentPath: parentDir, }, "remove") if err != nil { return err } return d.commit(commitMessage, rootSha) } func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { if !d.isOnBranch { return errors.New("cannot write to non-branch reference") } blob, err := d.putBlob(ctx, stream, up) if err != nil { return err } d.commitMutex.Lock() defer d.commitMutex.Unlock() parent, err := d.get(dstDir.GetPath()) if err != nil { return err } if parent.Entries == nil { return errs.NotFolder } newTree := make([]interface{}, 0, 2) newTree = append(newTree, TreeObjReq{ Path: stream.GetName(), Mode: "100644", Type: "blob", Sha: blob, }) if len(parent.Entries) == 1 && parent.Entries[0].Name == ".gitkeep" { newTree = append(newTree, TreeObjReq{ Path: ".gitkeep", Mode: "100644", Type: "blob", Sha: nil, }) } newSha, err := d.newTree(parent.Sha, newTree) if err != nil { return err } rootSha, err := d.renewParentTrees(dstDir.GetPath(), parent.Sha, newSha, "/") if err != nil { return err } commitMessage, err := getMessage(d.putMsgTmpl, &MessageTemplateVars{ UserName: getUsername(ctx), ObjName: stream.GetName(), ObjPath: stdpath.Join(dstDir.GetPath(), stream.GetName()), ParentName: dstDir.GetName(), ParentPath: dstDir.GetPath(), }, "upload") if err != nil { return err } return d.commit(commitMessage, rootSha) } var _ driver.Driver = (*Github)(nil) func (d *Github) getContentApiUrl(path string) string { path = utils.FixAndCleanPath(path) return fmt.Sprintf("https://api.github.com/repos/%s/%s/contents%s", d.Owner, d.Repo, path) } func (d *Github) get(path string) (*Object, error) { res, err := d.client.R().SetQueryParam("ref", d.Ref).Get(d.getContentApiUrl(path)) if err != nil { return nil, err } if res.StatusCode() != 200 { return nil, toErr(res) } var resp Object err = utils.Json.Unmarshal(res.Body(), &resp) return &resp, err } func (d *Github) putBlob(ctx context.Context, s model.FileStreamer, up driver.UpdateProgress) (string, error) { beforeContent := "{\"encoding\":\"base64\",\"content\":\"" afterContent := "\"}" length := int64(len(beforeContent)) + calculateBase64Length(s.GetSize()) + int64(len(afterContent)) beforeContentReader := strings.NewReader(beforeContent) contentReader, contentWriter := io.Pipe() go func() { encoder := base64.NewEncoder(base64.StdEncoding, contentWriter) if _, err := utils.CopyWithBuffer(encoder, s); err != nil { _ = contentWriter.CloseWithError(err) return } _ = encoder.Close() _ = contentWriter.Close() }() afterContentReader := strings.NewReader(afterContent) req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("https://api.github.com/repos/%s/%s/git/blobs", d.Owner, d.Repo), driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: &driver.SimpleReaderWithSize{ Reader: io.MultiReader(beforeContentReader, contentReader, afterContentReader), Size: length, }, UpdateProgress: up, })) if err != nil { return "", err } req.Header.Set("Accept", "application/vnd.github+json") req.Header.Set("X-GitHub-Api-Version", "2022-11-28") token := strings.TrimSpace(d.Token) if token != "" { req.Header.Set("Authorization", "Bearer "+token) } req.ContentLength = length res, err := base.HttpClient.Do(req) if err != nil { return "", err } defer res.Body.Close() resBody, err := io.ReadAll(res.Body) if err != nil { return "", err } if res.StatusCode != 201 { var errMsg ErrResp if err = utils.Json.Unmarshal(resBody, &errMsg); err != nil { return "", errors.New(res.Status) } else { return "", fmt.Errorf("%s: %s", res.Status, errMsg.Message) } } var resp PutBlobResp if err = utils.Json.Unmarshal(resBody, &resp); err != nil { return "", err } return resp.Sha, nil } func (d *Github) renewParentTrees(path, prevSha, curSha, until string) (string, error) { for path != until { path = stdpath.Dir(path) tree, sha, err := d.getTreeDirectly(path) if err != nil { return "", err } var newTree *TreeObjReq = nil for _, t := range tree.Trees { if t.Sha == prevSha { newTree = &t.TreeObjReq newTree.Sha = curSha break } } if newTree == nil { return "", errs.ObjectNotFound } curSha, err = d.newTree(sha, []interface{}{*newTree}) if err != nil { return "", err } prevSha = sha } return curSha, nil } func (d *Github) getTree(sha string) (*TreeResp, error) { res, err := d.client.R().Get(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/trees/%s", d.Owner, d.Repo, sha)) if err != nil { return nil, err } if res.StatusCode() != 200 { return nil, toErr(res) } var resp TreeResp if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { return nil, err } return &resp, nil } func (d *Github) getTreeDirectly(path string) (*TreeResp, string, error) { p, err := d.get(path) if err != nil { return nil, "", err } if p.Entries == nil { return nil, "", fmt.Errorf("%s is not a folder", path) } tree, err := d.getTree(p.Sha) if err != nil { return nil, "", err } if tree.Truncated { return nil, "", fmt.Errorf("tree %s is truncated", path) } return tree, p.Sha, nil } func (d *Github) newTree(baseSha string, tree []interface{}) (string, error) { body := &TreeReq{Trees: tree} if baseSha != "" { body.BaseTree = baseSha } res, err := d.client.R().SetBody(body). Post(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/trees", d.Owner, d.Repo)) if err != nil { return "", err } if res.StatusCode() != 201 { return "", toErr(res) } var resp TreeResp if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { return "", err } return resp.Sha, nil } func (d *Github) commit(message, treeSha string) error { oldCommit, err := d.getBranchHead() body := map[string]interface{}{ "message": message, "tree": treeSha, "parents": []string{oldCommit}, } d.addCommitterAndAuthor(&body) if d.pgpEntity != nil { signature, e := signCommit(&body, d.pgpEntity) if e != nil { return e } body["signature"] = signature } res, err := d.client.R().SetBody(body).Post(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/commits", d.Owner, d.Repo)) if err != nil { return err } if res.StatusCode() != 201 { return toErr(res) } var resp CommitResp if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { return err } // update branch head res, err = d.client.R(). SetBody(&UpdateRefReq{ Sha: resp.Sha, Force: false, }). Patch(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/refs/heads/%s", d.Owner, d.Repo, d.Ref)) if err != nil { return err } if res.StatusCode() != 200 { return toErr(res) } return nil } func (d *Github) getBranchHead() (string, error) { res, err := d.client.R().Get(fmt.Sprintf("https://api.github.com/repos/%s/%s/branches/%s", d.Owner, d.Repo, d.Ref)) if err != nil { return "", err } if res.StatusCode() != 200 { return "", toErr(res) } var resp BranchResp if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { return "", err } return resp.Commit.Sha, nil } func (d *Github) copyWithoutRenewTree(srcObj, dstDir model.Obj) (dstSha, newSha, srcParentSha string, srcParentTree *TreeResp, err error) { dst, err := d.get(dstDir.GetPath()) if err != nil { return "", "", "", nil, err } if dst.Entries == nil { return "", "", "", nil, errs.NotFolder } dstSha = dst.Sha srcParentPath := stdpath.Dir(srcObj.GetPath()) srcParentTree, srcParentSha, err = d.getTreeDirectly(srcParentPath) if err != nil { return "", "", "", nil, err } var src *TreeObjReq = nil for _, t := range srcParentTree.Trees { if t.Path == srcObj.GetName() { if t.Type == "commit" { return "", "", "", nil, errors.New("cannot copy a submodule") } src = &t.TreeObjReq break } } if src == nil { return "", "", "", nil, errs.ObjectNotFound } newTree := make([]interface{}, 0, 2) newTree = append(newTree, *src) if len(dst.Entries) == 1 && dst.Entries[0].Name == ".gitkeep" { newTree = append(newTree, TreeObjReq{ Path: ".gitkeep", Mode: "100644", Type: "blob", Sha: nil, }) } newSha, err = d.newTree(dstSha, newTree) if err != nil { return "", "", "", nil, err } return dstSha, newSha, srcParentSha, srcParentTree, nil } func (d *Github) getRepo() (*RepoResp, error) { res, err := d.client.R().Get(fmt.Sprintf("https://api.github.com/repos/%s/%s", d.Owner, d.Repo)) if err != nil { return nil, err } if res.StatusCode() != 200 { return nil, toErr(res) } var resp RepoResp if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { return nil, err } return &resp, nil } func (d *Github) getAuthenticatedUser() (*UserResp, error) { res, err := d.client.R().Get("https://api.github.com/user") if err != nil { return nil, err } if res.StatusCode() != 200 { return nil, toErr(res) } resp := &UserResp{} if err = utils.Json.Unmarshal(res.Body(), resp); err != nil { return nil, err } return resp, nil } func (d *Github) addCommitterAndAuthor(m *map[string]interface{}) { if d.CommitterName != "" { committer := map[string]string{ "name": d.CommitterName, "email": d.CommitterEmail, } (*m)["committer"] = committer } if d.AuthorName != "" { author := map[string]string{ "name": d.AuthorName, "email": d.AuthorEmail, } (*m)["author"] = author } } ================================================ FILE: drivers/github/meta.go ================================================ package github import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Token string `json:"token" type:"string" required:"true"` Owner string `json:"owner" type:"string" required:"true"` Repo string `json:"repo" type:"string" required:"true"` Ref string `json:"ref" type:"string" help:"A branch, a tag or a commit SHA, main branch by default."` GitHubProxy string `json:"gh_proxy" type:"string" help:"GitHub proxy, e.g. https://ghproxy.net/raw.githubusercontent.com or https://gh-proxy.com/raw.githubusercontent.com"` GPGPrivateKey string `json:"gpg_private_key" type:"text"` GPGKeyPassphrase string `json:"gpg_key_passphrase" type:"string"` CommitterName string `json:"committer_name" type:"string"` CommitterEmail string `json:"committer_email" type:"string"` AuthorName string `json:"author_name" type:"string"` AuthorEmail string `json:"author_email" type:"string"` MkdirCommitMsg string `json:"mkdir_commit_message" type:"text" default:"{{.UserName}} mkdir {{.ObjPath}}"` DeleteCommitMsg string `json:"delete_commit_message" type:"text" default:"{{.UserName}} remove {{.ObjPath}}"` PutCommitMsg string `json:"put_commit_message" type:"text" default:"{{.UserName}} upload {{.ObjPath}}"` RenameCommitMsg string `json:"rename_commit_message" type:"text" default:"{{.UserName}} rename {{.ObjPath}} to {{.TargetName}}"` CopyCommitMsg string `json:"copy_commit_message" type:"text" default:"{{.UserName}} copy {{.ObjPath}} to {{.TargetPath}}"` MoveCommitMsg string `json:"move_commit_message" type:"text" default:"{{.UserName}} move {{.ObjPath}} to {{.TargetPath}}"` } var config = driver.Config{ Name: "GitHub API", LocalSort: true, DefaultRoot: "/", } func init() { op.RegisterDriver(func() driver.Driver { return &Github{} }) } ================================================ FILE: drivers/github/types.go ================================================ package github import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type Links struct { Git string `json:"git"` Html string `json:"html"` Self string `json:"self"` } type Object struct { Type string `json:"type"` Encoding string `json:"encoding" required:"false"` Size int64 `json:"size"` Name string `json:"name"` Path string `json:"path"` Content string `json:"Content" required:"false"` Sha string `json:"sha"` URL string `json:"url"` GitURL string `json:"git_url"` HtmlURL string `json:"html_url"` DownloadURL string `json:"download_url"` Entries []Object `json:"entries" required:"false"` Links Links `json:"_links"` SubmoduleGitURL string `json:"submodule_git_url" required:"false"` Target string `json:"target" required:"false"` } func (o *Object) toModelObj() *model.Object { return &model.Object{ Name: o.Name, Size: o.Size, Modified: time.Unix(0, 0), IsFolder: o.Type == "dir", Path: utils.FixAndCleanPath(o.Path), } } type PutBlobResp struct { URL string `json:"url"` Sha string `json:"sha"` } type ErrResp struct { Message string `json:"message"` DocumentationURL string `json:"documentation_url"` Status string `json:"status"` } type TreeObjReq struct { Path string `json:"path"` Mode string `json:"mode"` Type string `json:"type"` Sha interface{} `json:"sha"` } type TreeObjResp struct { TreeObjReq Size int64 `json:"size" required:"false"` URL string `json:"url"` } func (o *TreeObjResp) toModelObj() *model.Object { return &model.Object{ Name: o.Path, Size: o.Size, Modified: time.Unix(0, 0), IsFolder: o.Type == "tree", Path: utils.FixAndCleanPath(o.Path), } } type TreeResp struct { Sha string `json:"sha"` URL string `json:"url"` Trees []TreeObjResp `json:"tree"` Truncated bool `json:"truncated"` } type TreeReq struct { BaseTree interface{} `json:"base_tree,omitempty"` Trees []interface{} `json:"tree"` } type CommitResp struct { Sha string `json:"sha"` } type BranchResp struct { Name string `json:"name"` Commit CommitResp `json:"commit"` } type UpdateRefReq struct { Sha string `json:"sha"` Force bool `json:"force"` } type RepoResp struct { DefaultBranch string `json:"default_branch"` } type UserResp struct { Name string `json:"name"` Email string `json:"email"` } ================================================ FILE: drivers/github/util.go ================================================ package github import ( "bytes" "context" "errors" "fmt" "strings" "text/template" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/ProtonMail/go-crypto/openpgp" "github.com/ProtonMail/go-crypto/openpgp/armor" "github.com/go-resty/resty/v2" ) type MessageTemplateVars struct { UserName string ObjName string ObjPath string ParentName string ParentPath string TargetName string TargetPath string } func getMessage(tmpl *template.Template, vars *MessageTemplateVars, defaultOpStr string) (string, error) { sb := strings.Builder{} if err := tmpl.Execute(&sb, vars); err != nil { return fmt.Sprintf("%s %s %s", vars.UserName, defaultOpStr, vars.ObjPath), err } return sb.String(), nil } func calculateBase64Length(inputLength int64) int64 { return 4 * ((inputLength + 2) / 3) } func toErr(res *resty.Response) error { var errMsg ErrResp if err := utils.Json.Unmarshal(res.Body(), &errMsg); err != nil { return errors.New(res.Status()) } else { return fmt.Errorf("%s: %s", res.Status(), errMsg.Message) } } // Example input: // a = /aaa/bbb/ccc // b = /aaa/b11/ddd/ccc // // Output: // ancestor = /aaa // aChildName = bbb // bChildName = b11 // aRest = bbb/ccc // bRest = b11/ddd/ccc func getPathCommonAncestor(a, b string) (ancestor, aChildName, bChildName, aRest, bRest string) { a = utils.FixAndCleanPath(a) b = utils.FixAndCleanPath(b) idx := 1 for idx < len(a) && idx < len(b) { if a[idx] != b[idx] { break } idx++ } aNextIdx := idx for aNextIdx < len(a) { if a[aNextIdx] == '/' { break } aNextIdx++ } bNextIdx := idx for bNextIdx < len(b) { if b[bNextIdx] == '/' { break } bNextIdx++ } for idx > 0 { if a[idx] == '/' { break } idx-- } ancestor = utils.FixAndCleanPath(a[:idx]) aChildName = a[idx+1 : aNextIdx] bChildName = b[idx+1 : bNextIdx] aRest = a[idx+1:] bRest = b[idx+1:] return ancestor, aChildName, bChildName, aRest, bRest } func getUsername(ctx context.Context) string { user, ok := ctx.Value(conf.UserKey).(*model.User) if !ok { return "" } return user.Username } func loadPrivateKey(key, passphrase string) (*openpgp.Entity, error) { entityList, err := openpgp.ReadArmoredKeyRing(strings.NewReader(key)) if err != nil { return nil, err } if len(entityList) < 1 { return nil, fmt.Errorf("no keys found in key ring") } entity := entityList[0] pass := []byte(passphrase) if entity.PrivateKey != nil && entity.PrivateKey.Encrypted { if err = entity.PrivateKey.Decrypt(pass); err != nil { return nil, fmt.Errorf("password incorrect: %+v", err) } } for _, subKey := range entity.Subkeys { if subKey.PrivateKey != nil && subKey.PrivateKey.Encrypted { if err = subKey.PrivateKey.Decrypt(pass); err != nil { return nil, fmt.Errorf("password incorrect: %+v", err) } } } return entity, nil } func signCommit(m *map[string]interface{}, entity *openpgp.Entity) (string, error) { var commit strings.Builder commit.WriteString(fmt.Sprintf("tree %s\n", (*m)["tree"].(string))) parents := (*m)["parents"].([]string) for _, p := range parents { commit.WriteString(fmt.Sprintf("parent %s\n", p)) } now := time.Now() _, offset := now.Zone() hour := offset / 3600 author := (*m)["author"].(map[string]string) commit.WriteString(fmt.Sprintf("author %s <%s> %d %+03d00\n", author["name"], author["email"], now.Unix(), hour)) author["date"] = now.Format(time.RFC3339) committer := (*m)["committer"].(map[string]string) commit.WriteString(fmt.Sprintf("committer %s <%s> %d %+03d00\n", committer["name"], committer["email"], now.Unix(), hour)) committer["date"] = now.Format(time.RFC3339) commit.WriteString(fmt.Sprintf("\n%s", (*m)["message"].(string))) data := commit.String() var sigBuffer bytes.Buffer err := openpgp.DetachSign(&sigBuffer, entity, strings.NewReader(data), nil) if err != nil { return "", fmt.Errorf("signing failed: %v", err) } var armoredSig bytes.Buffer armorWriter, err := armor.Encode(&armoredSig, "PGP SIGNATURE", nil) if err != nil { return "", err } if _, err = utils.CopyWithBuffer(armorWriter, &sigBuffer); err != nil { return "", err } _ = armorWriter.Close() return armoredSig.String(), nil } ================================================ FILE: drivers/github_releases/driver.go ================================================ package github_releases import ( "context" "fmt" "net/http" stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type GithubReleases struct { model.Storage Addition points []MountPoint } func (d *GithubReleases) Config() driver.Config { return config } func (d *GithubReleases) GetAddition() driver.Additional { return &d.Addition } func (d *GithubReleases) Init(ctx context.Context) error { d.ParseRepos(d.Addition.RepoStructure) return nil } func (d *GithubReleases) Drop(ctx context.Context) error { return nil } func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files := make([]File, 0) path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/")) for i := range d.points { point := &d.points[i] if !d.Addition.ShowAllVersion { // latest point.RequestRelease(d.GetRequest, args.Refresh) if point.Point == path { // 与仓库路径相同 files = append(files, point.GetLatestRelease()...) if d.Addition.ShowReadme { files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) } if d.Addition.ShowSourceCode { files = append(files, point.GetSourceCode()...) } } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 nextDir := GetNextDir(point.Point, path) if nextDir == "" { continue } hasSameDir := false for index := range files { if files[index].GetName() == nextDir { hasSameDir = true files[index].Size += point.GetLatestSize() break } } if !hasSameDir { files = append(files, File{ Path: stdpath.Join(path, nextDir), FileName: nextDir, Size: point.GetLatestSize(), UpdateAt: point.Release.PublishedAt, CreateAt: point.Release.CreatedAt, Type: "dir", Url: "", }) } } } else { // all version point.RequestReleases(d.GetRequest, args.Refresh) if point.Point == path { // 与仓库路径相同 files = append(files, point.GetAllVersion()...) if d.Addition.ShowReadme { files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) } } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 nextDir := GetNextDir(point.Point, path) if nextDir == "" { continue } hasSameDir := false for index := range files { if files[index].GetName() == nextDir { hasSameDir = true files[index].Size += point.GetAllVersionSize() break } } if !hasSameDir { files = append(files, File{ FileName: nextDir, Path: stdpath.Join(path, nextDir), Size: point.GetAllVersionSize(), UpdateAt: (*point.Releases)[0].PublishedAt, CreateAt: (*point.Releases)[0].CreatedAt, Type: "dir", Url: "", }) } } else if strings.HasPrefix(path, point.Point) { // 仓库目录的子目录 tagName := GetNextDir(path, point.Point) if tagName == "" { continue } files = append(files, point.GetReleaseByTagName(tagName)...) if d.Addition.ShowSourceCode { files = append(files, point.GetSourceCodeByTagName(tagName)...) } } } } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return src, nil }) } func (d *GithubReleases) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { url := file.GetID() gh_proxy := strings.TrimSpace(d.Addition.GitHubProxy) if gh_proxy != "" { url = strings.Replace(url, "https://github.com", gh_proxy, 1) } link := model.Link{ URL: url, Header: http.Header{}, } return &link, nil } func (d *GithubReleases) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { // TODO create folder, optional return nil, errs.NotImplement } func (d *GithubReleases) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { // TODO move obj, optional return nil, errs.NotImplement } func (d *GithubReleases) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { // TODO rename obj, optional return nil, errs.NotImplement } func (d *GithubReleases) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { // TODO copy obj, optional return nil, errs.NotImplement } func (d *GithubReleases) Remove(ctx context.Context, obj model.Obj) error { // TODO remove obj, optional return errs.NotImplement } ================================================ FILE: drivers/github_releases/meta.go ================================================ package github_releases import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"OpenListTeam/OpenList" help:"structure:[path:]org/repo"` ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"` Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` ShowSourceCode bool `json:"show_source_code" type:"bool" default:"false" help:"show Source code (zip/tar.gz)"` ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "` } var config = driver.Config{ Name: "GitHub Releases", NoUpload: true, } func init() { op.RegisterDriver(func() driver.Driver { return &GithubReleases{} }) } ================================================ FILE: drivers/github_releases/models.go ================================================ package github_releases type Release struct { Url string `json:"url"` AssetsUrl string `json:"assets_url"` UploadUrl string `json:"upload_url"` HtmlUrl string `json:"html_url"` Id int `json:"id"` Author User `json:"author"` NodeId string `json:"node_id"` TagName string `json:"tag_name"` TargetCommitish string `json:"target_commitish"` Name string `json:"name"` Draft bool `json:"draft"` Prerelease bool `json:"prerelease"` CreatedAt string `json:"created_at"` PublishedAt string `json:"published_at"` Assets []Asset `json:"assets"` TarballUrl string `json:"tarball_url"` ZipballUrl string `json:"zipball_url"` Body string `json:"body"` Reactions Reactions `json:"reactions"` } type User struct { Login string `json:"login"` Id int `json:"id"` NodeId string `json:"node_id"` AvatarUrl string `json:"avatar_url"` GravatarId string `json:"gravatar_id"` Url string `json:"url"` HtmlUrl string `json:"html_url"` FollowersUrl string `json:"followers_url"` FollowingUrl string `json:"following_url"` GistsUrl string `json:"gists_url"` StarredUrl string `json:"starred_url"` SubscriptionsUrl string `json:"subscriptions_url"` OrganizationsUrl string `json:"organizations_url"` ReposUrl string `json:"repos_url"` EventsUrl string `json:"events_url"` ReceivedEventsUrl string `json:"received_events_url"` Type string `json:"type"` UserViewType string `json:"user_view_type"` SiteAdmin bool `json:"site_admin"` } type Asset struct { Url string `json:"url"` Id int `json:"id"` NodeId string `json:"node_id"` Name string `json:"name"` Label string `json:"label"` Uploader User `json:"uploader"` ContentType string `json:"content_type"` State string `json:"state"` Size int64 `json:"size"` DownloadCount int `json:"download_count"` CreatedAt string `json:"created_at"` UpdatedAt string `json:"updated_at"` BrowserDownloadUrl string `json:"browser_download_url"` } type Reactions struct { Url string `json:"url"` TotalCount int `json:"total_count"` PlusOne int `json:"+1"` MinusOne int `json:"-1"` Laugh int `json:"laugh"` Hooray int `json:"hooray"` Confused int `json:"confused"` Heart int `json:"heart"` Rocket int `json:"rocket"` Eyes int `json:"eyes"` } type FileInfo struct { Name string `json:"name"` Path string `json:"path"` Sha string `json:"sha"` Size int64 `json:"size"` Url string `json:"url"` HtmlUrl string `json:"html_url"` GitUrl string `json:"git_url"` DownloadUrl string `json:"download_url"` Type string `json:"type"` } ================================================ FILE: drivers/github_releases/types.go ================================================ package github_releases import ( "encoding/json" "path" "strings" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) type MountPoint struct { Point string // 挂载点 Repo string // 仓库名 owner/repo Release *Release // Release 指针 latest Releases *[]Release // []Release 指针 OtherFile *[]FileInfo // 仓库根目录下的其他文件 } // 请求最新版本 func (m *MountPoint) RequestRelease(get func(url string) (*resty.Response, error), refresh bool) { if m.Repo == "" { return } if m.Release == nil || refresh { resp, _ := get("https://api.github.com/repos/" + m.Repo + "/releases/latest") m.Release = new(Release) json.Unmarshal(resp.Body(), m.Release) } } // 请求所有版本 func (m *MountPoint) RequestReleases(get func(url string) (*resty.Response, error), refresh bool) { if m.Repo == "" { return } if m.Releases == nil || refresh { resp, _ := get("https://api.github.com/repos/" + m.Repo + "/releases") m.Releases = new([]Release) json.Unmarshal(resp.Body(), m.Releases) } } // 获取最新版本 func (m *MountPoint) GetLatestRelease() []File { files := make([]File, 0, len(m.Release.Assets)) for _, asset := range m.Release.Assets { files = append(files, File{ Path: path.Join(m.Point, asset.Name), FileName: asset.Name, Size: asset.Size, Type: "file", UpdateAt: asset.UpdatedAt, CreateAt: asset.CreatedAt, Url: asset.BrowserDownloadUrl, }) } return files } // 获取最新版本大小 func (m *MountPoint) GetLatestSize() int64 { size := int64(0) for _, asset := range m.Release.Assets { size += asset.Size } return size } // 获取所有版本 func (m *MountPoint) GetAllVersion() []File { files := make([]File, 0) for _, release := range *m.Releases { file := File{ Path: path.Join(m.Point, release.TagName), FileName: release.TagName, Size: m.GetSizeByTagName(release.TagName), Type: "dir", UpdateAt: release.PublishedAt, CreateAt: release.CreatedAt, Url: release.HtmlUrl, } for _, asset := range release.Assets { file.Size += asset.Size } files = append(files, file) } return files } // 根据版本号获取版本 func (m *MountPoint) GetReleaseByTagName(tagName string) []File { for _, item := range *m.Releases { if item.TagName == tagName { files := make([]File, 0) for _, asset := range item.Assets { files = append(files, File{ Path: path.Join(m.Point, tagName, asset.Name), FileName: asset.Name, Size: asset.Size, Type: "file", UpdateAt: asset.UpdatedAt, CreateAt: asset.CreatedAt, Url: asset.BrowserDownloadUrl, }) } return files } } return nil } // 根据版本号获取版本大小 func (m *MountPoint) GetSizeByTagName(tagName string) int64 { if m.Releases == nil { return 0 } for _, item := range *m.Releases { if item.TagName == tagName { size := int64(0) for _, asset := range item.Assets { size += asset.Size } return size } } return 0 } // 获取所有版本大小 func (m *MountPoint) GetAllVersionSize() int64 { if m.Releases == nil { return 0 } size := int64(0) for _, release := range *m.Releases { for _, asset := range release.Assets { size += asset.Size } } return size } func (m *MountPoint) GetSourceCode() []File { files := make([]File, 0) // 无法获取文件大小，此处设为 1 files = append(files, File{ Path: path.Join(m.Point, "Source code (zip)"), FileName: "Source code (zip)", Size: 1, Type: "file", UpdateAt: m.Release.CreatedAt, CreateAt: m.Release.CreatedAt, Url: m.Release.ZipballUrl, }) files = append(files, File{ Path: path.Join(m.Point, "Source code (tar.gz)"), FileName: "Source code (tar.gz)", Size: 1, Type: "file", UpdateAt: m.Release.CreatedAt, CreateAt: m.Release.CreatedAt, Url: m.Release.TarballUrl, }) return files } func (m *MountPoint) GetSourceCodeByTagName(tagName string) []File { for _, item := range *m.Releases { if item.TagName == tagName { files := make([]File, 0) files = append(files, File{ Path: path.Join(m.Point, "Source code (zip)"), FileName: "Source code (zip)", Size: 1, Type: "file", UpdateAt: item.CreatedAt, CreateAt: item.CreatedAt, Url: item.ZipballUrl, }) files = append(files, File{ Path: path.Join(m.Point, "Source code (tar.gz)"), FileName: "Source code (tar.gz)", Size: 1, Type: "file", UpdateAt: item.CreatedAt, CreateAt: item.CreatedAt, Url: item.TarballUrl, }) return files } } return nil } func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), refresh bool) []File { if m.OtherFile == nil || refresh { resp, _ := get("https://api.github.com/repos/" + m.Repo + "/contents") m.OtherFile = new([]FileInfo) json.Unmarshal(resp.Body(), m.OtherFile) } files := make([]File, 0) defaultTime := "1970-01-01T00:00:00Z" for _, file := range *m.OtherFile { if strings.HasSuffix(file.Name, ".md") || strings.HasPrefix(file.Name, "LICENSE") { files = append(files, File{ Path: path.Join(m.Point, file.Name), FileName: file.Name, Size: file.Size, Type: "file", UpdateAt: defaultTime, CreateAt: defaultTime, Url: file.DownloadUrl, }) } } return files } type File struct { Path string // 文件路径 FileName string // 文件名 Size int64 // 文件大小 Type string // 文件类型 UpdateAt string // 更新时间 eg:"2025-01-27T16:10:16Z" CreateAt string // 创建时间 Url string // 下载链接 } func (f File) GetHash() utils.HashInfo { return utils.HashInfo{} } func (f File) GetPath() string { return f.Path } func (f File) GetSize() int64 { return f.Size } func (f File) GetName() string { return f.FileName } func (f File) ModTime() time.Time { t, _ := time.Parse(time.RFC3339, f.CreateAt) return t } func (f File) CreateTime() time.Time { t, _ := time.Parse(time.RFC3339, f.CreateAt) return t } func (f File) IsDir() bool { return f.Type == "dir" } func (f File) GetID() string { return f.Url } ================================================ FILE: drivers/github_releases/util.go ================================================ package github_releases import ( "fmt" "path/filepath" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) // 发送 GET 请求 func (d *GithubReleases) GetRequest(url string) (*resty.Response, error) { req := base.RestyClient.R() req.SetHeader("Accept", "application/vnd.github+json") req.SetHeader("X-GitHub-Api-Version", "2022-11-28") if d.Addition.Token != "" { req.SetHeader("Authorization", fmt.Sprintf("Bearer %s", d.Addition.Token)) } res, err := req.Get(url) if err != nil { return nil, err } if res.StatusCode() != 200 { log.Warn("failed to get request: ", res.StatusCode(), res.String()) } return res, nil } // 解析挂载结构 func (d *GithubReleases) ParseRepos(text string) ([]MountPoint, error) { lines := strings.Split(text, "\n") points := make([]MountPoint, 0) for _, line := range lines { line = strings.TrimSpace(line) if line == "" { continue } parts := strings.Split(line, ":") path, repo := "", "" if len(parts) == 1 { path = "/" repo = parts[0] } else if len(parts) == 2 { path = fmt.Sprintf("/%s", strings.Trim(parts[0], "/")) repo = parts[1] } else { return nil, fmt.Errorf("invalid format: %s", line) } points = append(points, MountPoint{ Point: path, Repo: repo, Release: nil, Releases: nil, }) } d.points = points return points, nil } // 获取下一级目录 func GetNextDir(wholePath string, basePath string) string { basePath = fmt.Sprintf("%s/", strings.TrimRight(basePath, "/")) if !strings.HasPrefix(wholePath, basePath) { return "" } remainingPath := strings.TrimLeft(strings.TrimPrefix(wholePath, basePath), "/") if remainingPath != "" { parts := strings.Split(remainingPath, "/") nextDir := parts[0] if strings.HasPrefix(wholePath, strings.TrimRight(basePath, "/")+"/"+nextDir) { return nextDir } } return "" } // 判断当前目录是否是目标目录的祖先目录 func IsAncestorDir(parentDir string, targetDir string) bool { absTargetDir, _ := filepath.Abs(targetDir) absParentDir, _ := filepath.Abs(parentDir) return strings.HasPrefix(absTargetDir, absParentDir) } ================================================ FILE: drivers/google_drive/driver.go ================================================ package google_drive import ( "context" "fmt" "net/http" "strconv" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) type GoogleDrive struct { model.Storage Addition AccessToken string ServiceAccountFile int ServiceAccountFileList []string } func (d *GoogleDrive) Config() driver.Config { return config } func (d *GoogleDrive) GetAddition() driver.Additional { return &d.Addition } func (d *GoogleDrive) Init(ctx context.Context) error { if d.ChunkSize == 0 { d.ChunkSize = 5 } return d.refreshToken() } func (d *GoogleDrive) Drop(ctx context.Context) error { return nil } func (d *GoogleDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(dir.GetID()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return fileToObj(src), nil }) } func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { url := fmt.Sprintf("https://www.googleapis.com/drive/v3/files/%s?includeItemsFromAllDrives=true&supportsAllDrives=true", file.GetID()) _, err := d.request(url, http.MethodGet, nil, nil) if err != nil { return nil, err } link := model.Link{ URL: url + "&alt=media&acknowledgeAbuse=true", Header: http.Header{ "Authorization": []string{"Bearer " + d.AccessToken}, }, } return &link, nil } func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { data := base.Json{ "name": dirName, "parents": []string{parentDir.GetID()}, "mimeType": "application/vnd.google-apps.folder", } _, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *GoogleDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { query := map[string]string{ "addParents": dstDir.GetID(), "removeParents": "root", } url := "https://www.googleapis.com/drive/v3/files/" + srcObj.GetID() _, err := d.request(url, http.MethodPatch, func(req *resty.Request) { req.SetQueryParams(query) }, nil) return err } func (d *GoogleDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) error { data := base.Json{ "name": newName, } url := "https://www.googleapis.com/drive/v3/files/" + srcObj.GetID() _, err := d.request(url, http.MethodPatch, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *GoogleDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return errs.NotSupport } func (d *GoogleDrive) Remove(ctx context.Context, obj model.Obj) error { url := "https://www.googleapis.com/drive/v3/files/" + obj.GetID() _, err := d.request(url, http.MethodDelete, nil, nil) return err } func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { obj := stream.GetExist() var ( e Error url string data base.Json res *resty.Response err error ) if obj != nil { url = fmt.Sprintf("https://www.googleapis.com/upload/drive/v3/files/%s?uploadType=resumable&supportsAllDrives=true", obj.GetID()) data = base.Json{} } else { data = base.Json{ "name": stream.GetName(), "parents": []string{dstDir.GetID()}, } url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true" } req := base.NoRedirectClient.R(). SetHeaders(map[string]string{ "Authorization": "Bearer " + d.AccessToken, "X-Upload-Content-Type": stream.GetMimetype(), "X-Upload-Content-Length": strconv.FormatInt(stream.GetSize(), 10), }). SetError(&e).SetBody(data).SetContext(ctx) if obj != nil { res, err = req.Patch(url) } else { res, err = req.Post(url) } if err != nil { return err } if e.Error.Code != 0 { if e.Error.Code == 401 { err = d.refreshToken() if err != nil { return err } return d.Put(ctx, dstDir, stream, up) } return fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) } putUrl := res.Header().Get("location") if stream.GetSize() < d.ChunkSize*1024*1024 { _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { req.SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)). SetBody(driver.NewLimitedUploadStream(ctx, stream)) }, nil) } else { err = d.chunkUpload(ctx, stream, putUrl, up) } return err } func (d *GoogleDrive) GetDetails(ctx context.Context) (*model.StorageDetails, error) { if d.DisableDiskUsage { return nil, errs.NotImplement } about, err := d.getAbout(ctx) if err != nil { return nil, err } var total, used int64 if about.StorageQuota.Limit == nil { total = 0 } else { total, err = strconv.ParseInt(*about.StorageQuota.Limit, 10, 64) if err != nil { return nil, err } } used, err = strconv.ParseInt(about.StorageQuota.Usage, 10, 64) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: used, }, }, nil } var _ driver.Driver = (*GoogleDrive)(nil) ================================================ FILE: drivers/google_drive/meta.go ================================================ package google_drive import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootID RefreshToken string `json:"refresh_token" required:"true"` OrderBy string `json:"order_by" type:"string" help:"such as: folder,name,modifiedTime"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` UseOnlineAPI bool `json:"use_online_api" default:"true"` APIAddress string `json:"api_url_address" default:"https://api.oplist.org/googleui/renewapi"` ClientID string `json:"client_id"` ClientSecret string `json:"client_secret"` ChunkSize int64 `json:"chunk_size" type:"number" default:"5" help:"chunk size while uploading (unit: MB)"` DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` } var config = driver.Config{ Name: "GoogleDrive", OnlyProxy: true, DefaultRoot: "root", } func init() { op.RegisterDriver(func() driver.Driver { return &GoogleDrive{} }) } ================================================ FILE: drivers/google_drive/types.go ================================================ package google_drive import ( "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" ) type TokenError struct { Error string `json:"error"` ErrorDescription string `json:"error_description"` } type Files struct { NextPageToken string `json:"nextPageToken"` Files []File `json:"files"` } type File struct { Id string `json:"id"` Name string `json:"name"` MimeType string `json:"mimeType"` ModifiedTime time.Time `json:"modifiedTime"` CreatedTime time.Time `json:"createdTime"` Size string `json:"size"` ThumbnailLink string `json:"thumbnailLink"` ShortcutDetails struct { TargetId string `json:"targetId"` TargetMimeType string `json:"targetMimeType"` } `json:"shortcutDetails"` MD5Checksum string `json:"md5Checksum"` SHA1Checksum string `json:"sha1Checksum"` SHA256Checksum string `json:"sha256Checksum"` } func fileToObj(f File) *model.ObjThumb { log.Debugf("google file: %+v", f) size, _ := strconv.ParseInt(f.Size, 10, 64) obj := &model.ObjThumb{ Object: model.Object{ ID: f.Id, Name: f.Name, Size: size, Ctime: f.CreatedTime, Modified: f.ModifiedTime, IsFolder: f.MimeType == "application/vnd.google-apps.folder", HashInfo: utils.NewHashInfoByMap(map[*utils.HashType]string{ utils.MD5: f.MD5Checksum, utils.SHA1: f.SHA1Checksum, utils.SHA256: f.SHA256Checksum, }), }, Thumbnail: model.Thumbnail{ Thumbnail: f.ThumbnailLink, }, } if f.MimeType == "application/vnd.google-apps.shortcut" { obj.ID = f.ShortcutDetails.TargetId obj.IsFolder = f.ShortcutDetails.TargetMimeType == "application/vnd.google-apps.folder" } return obj } type Error struct { Error struct { Errors []struct { Domain string `json:"domain"` Reason string `json:"reason"` Message string `json:"message"` LocationType string `json:"location_type"` Location string `json:"location"` } Code int `json:"code"` Message string `json:"message"` } `json:"error"` } type AboutResp struct { StorageQuota struct { Limit *string `json:"limit"` Usage string `json:"usage"` UsageInDrive string `json:"usageInDrive"` UsageInDriveTrash string `json:"usageInDriveTrash"` } } ================================================ FILE: drivers/google_drive/util.go ================================================ package google_drive import ( "context" "crypto/x509" "encoding/pem" "fmt" "io" "net/http" "os" "regexp" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/avast/retry-go" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" "github.com/golang-jwt/jwt/v4" log "github.com/sirupsen/logrus" ) // do others that not defined in Driver interface // Google Drive API field constants const ( // File list query fields FilesListFields = "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken" // Single file query fields FileInfoFields = "id,name,mimeType,size,md5Checksum,sha1Checksum,sha256Checksum" ) type googleDriveServiceAccount struct { // Type string `json:"type"` // ProjectID string `json:"project_id"` // PrivateKeyID string `json:"private_key_id"` PrivateKey string `json:"private_key"` ClientEMail string `json:"client_email"` // ClientID string `json:"client_id"` // AuthURI string `json:"auth_uri"` TokenURI string `json:"token_uri"` // AuthProviderX509CertURL string `json:"auth_provider_x509_cert_url"` // ClientX509CertURL string `json:"client_x509_cert_url"` } func (d *GoogleDrive) refreshToken() error { // 使用在线API刷新Token，无需ClientID和ClientSecret if d.UseOnlineAPI && len(d.APIAddress) > 0 { u := d.APIAddress var resp struct { RefreshToken string `json:"refresh_token"` AccessToken string `json:"access_token"` ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, "server_use": "true", "driver_txt": "googleui_go", }). Get(u) if err != nil { return err } if resp.RefreshToken == "" || resp.AccessToken == "" { if resp.ErrorMessage != "" { return fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) } return fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") } d.AccessToken = resp.AccessToken d.RefreshToken = resp.RefreshToken op.MustSaveDriverStorage(d) return nil } // 使用本地客户端的情况下检查是否为空 if d.ClientID == "" || d.ClientSecret == "" { return fmt.Errorf("empty ClientID or ClientSecret") } // 走原有的刷新逻辑 // googleDriveServiceAccountFile gdsaFile gdsaFile, gdsaFileErr := os.Stat(d.RefreshToken) if gdsaFileErr == nil { gdsaFileThis := d.RefreshToken if gdsaFile.IsDir() { if len(d.ServiceAccountFileList) <= 0 { gdsaReadDir, gdsaDirErr := os.ReadDir(d.RefreshToken) if gdsaDirErr != nil { log.Error("read dir fail") return gdsaDirErr } var gdsaFileList []string for _, fi := range gdsaReadDir { if !fi.IsDir() { match, _ := regexp.MatchString("^.*\\.json$", fi.Name()) if !match { continue } gdsaDirText := d.RefreshToken if d.RefreshToken[len(d.RefreshToken)-1:] != "/" { gdsaDirText = d.RefreshToken + "/" } gdsaFileList = append(gdsaFileList, gdsaDirText+fi.Name()) } } d.ServiceAccountFileList = gdsaFileList gdsaFileThis = d.ServiceAccountFileList[d.ServiceAccountFile] d.ServiceAccountFile++ } else { if d.ServiceAccountFile < len(d.ServiceAccountFileList) { d.ServiceAccountFile++ } else { d.ServiceAccountFile = 0 } gdsaFileThis = d.ServiceAccountFileList[d.ServiceAccountFile] } } gdsaFileThisContent, err := os.ReadFile(gdsaFileThis) if err != nil { return err } // Now let's unmarshal the data into `payload` var jsonData googleDriveServiceAccount err = utils.Json.Unmarshal(gdsaFileThisContent, &jsonData) if err != nil { return err } gdsaScope := "https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/drive.appdata https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/drive.metadata https://www.googleapis.com/auth/drive.metadata.readonly https://www.googleapis.com/auth/drive.readonly https://www.googleapis.com/auth/drive.scripts" timeNow := time.Now() var timeStart int64 = timeNow.Unix() var timeEnd int64 = timeNow.Add(time.Minute * 60).Unix() // load private key from string privateKeyPem, _ := pem.Decode([]byte(jsonData.PrivateKey)) privateKey, _ := x509.ParsePKCS8PrivateKey(privateKeyPem.Bytes) jwtToken := jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{ "iss": jsonData.ClientEMail, "scope": gdsaScope, "aud": jsonData.TokenURI, "exp": timeEnd, "iat": timeStart, }) assertion, err := jwtToken.SignedString(privateKey) if err != nil { return err } var resp base.TokenResp var e TokenError res, err := base.RestyClient.R().SetResult(&resp).SetError(&e). SetFormData(map[string]string{ "assertion": assertion, "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", }).Post(jsonData.TokenURI) if err != nil { return err } log.Debug(res.String()) if e.Error != "" { return fmt.Errorf(e.Error) } d.AccessToken = resp.AccessToken return nil } else if os.IsExist(gdsaFileErr) { return gdsaFileErr } url := "https://www.googleapis.com/oauth2/v4/token" var resp base.TokenResp var e TokenError res, err := base.RestyClient.R().SetResult(&resp).SetError(&e). SetFormData(map[string]string{ "client_id": d.ClientID, "client_secret": d.ClientSecret, "refresh_token": d.RefreshToken, "grant_type": "refresh_token", }).Post(url) if err != nil { return err } log.Debug(res.String()) if e.Error != "" { return fmt.Errorf(e.Error) } d.AccessToken = resp.AccessToken return nil } func (d *GoogleDrive) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) req.SetQueryParam("includeItemsFromAllDrives", "true") req.SetQueryParam("supportsAllDrives", "true") if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e Error req.SetError(&e) res, err := req.Execute(method, url) if err != nil { return nil, err } if e.Error.Code != 0 { if e.Error.Code == 401 { err = d.refreshToken() if err != nil { return nil, err } return d.request(url, method, callback, resp) } return nil, fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) } return res.Body(), nil } func (d *GoogleDrive) getFiles(id string) ([]File, error) { pageToken := "first" res := make([]File, 0) for pageToken != "" { if pageToken == "first" { pageToken = "" } var resp Files orderBy := "folder,name,modifiedTime desc" if d.OrderBy != "" { orderBy = d.OrderBy + " " + d.OrderDirection } query := map[string]string{ "orderBy": orderBy, "fields": FilesListFields, "pageSize": "1000", "q": fmt.Sprintf("'%s' in parents and trashed = false", id), //"includeItemsFromAllDrives": "true", //"supportsAllDrives": "true", "pageToken": pageToken, } _, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return nil, err } pageToken = resp.NextPageToken // Batch process shortcuts, API calls only for file shortcuts shortcutTargetIds := make([]string, 0) shortcutIndices := make([]int, 0) // Collect target IDs of all file shortcuts (skip folder shortcuts) for i := range resp.Files { if resp.Files[i].MimeType == "application/vnd.google-apps.shortcut" && resp.Files[i].ShortcutDetails.TargetId != "" && resp.Files[i].ShortcutDetails.TargetMimeType != "application/vnd.google-apps.folder" { shortcutTargetIds = append(shortcutTargetIds, resp.Files[i].ShortcutDetails.TargetId) shortcutIndices = append(shortcutIndices, i) } } // Batch get target file info (only for file shortcuts) if len(shortcutTargetIds) > 0 { targetFiles := d.batchGetTargetFilesInfo(shortcutTargetIds) // Update shortcut file info for j, targetId := range shortcutTargetIds { if targetFile, exists := targetFiles[targetId]; exists { fileIndex := shortcutIndices[j] if targetFile.Size != "" { resp.Files[fileIndex].Size = targetFile.Size } if targetFile.MD5Checksum != "" { resp.Files[fileIndex].MD5Checksum = targetFile.MD5Checksum } if targetFile.SHA1Checksum != "" { resp.Files[fileIndex].SHA1Checksum = targetFile.SHA1Checksum } if targetFile.SHA256Checksum != "" { resp.Files[fileIndex].SHA256Checksum = targetFile.SHA256Checksum } } } } res = append(res, resp.Files...) } return res, nil } // getTargetFileInfo gets target file details for shortcuts func (d *GoogleDrive) getTargetFileInfo(targetId string) (File, error) { var targetFile File url := fmt.Sprintf("https://www.googleapis.com/drive/v3/files/%s", targetId) query := map[string]string{ "fields": FileInfoFields, } _, err := d.request(url, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &targetFile) if err != nil { return File{}, err } return targetFile, nil } // batchGetTargetFilesInfo batch gets target file info, sequential processing to avoid concurrency complexity func (d *GoogleDrive) batchGetTargetFilesInfo(targetIds []string) map[string]File { if len(targetIds) == 0 { return make(map[string]File) } result := make(map[string]File) // Sequential processing to avoid concurrency complexity for _, targetId := range targetIds { file, err := d.getTargetFileInfo(targetId) if err == nil { result[targetId] = file } } return result } func (d *GoogleDrive) chunkUpload(ctx context.Context, file model.FileStreamer, url string, up driver.UpdateProgress) error { defaultChunkSize := d.ChunkSize * 1024 * 1024 ss, err := stream.NewStreamSectionReader(file, int(defaultChunkSize), &up) if err != nil { return err } var offset int64 = 0 url += "?includeItemsFromAllDrives=true&supportsAllDrives=true" for offset < file.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } chunkSize := min(file.GetSize()-offset, defaultChunkSize) reader, err := ss.GetSectionReader(offset, chunkSize) if err != nil { return err } limitedReader := driver.NewLimitedUploadStream(ctx, reader) err = retry.Do(func() error { reader.Seek(0, io.SeekStart) req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, limitedReader) if err != nil { return err } req.Header = map[string][]string{ "Authorization": {"Bearer " + d.AccessToken}, "Content-Length": {strconv.FormatInt(chunkSize, 10)}, "Content-Range": {fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, file.GetSize())}, } res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() bytes, _ := io.ReadAll(res.Body) var e Error utils.Json.Unmarshal(bytes, &e) if e.Error.Code != 0 { if e.Error.Code == 401 { err = d.refreshToken() if err != nil { return err } } return fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) } up(float64(offset+chunkSize) / float64(file.GetSize()) * 100) return nil }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) ss.FreeSectionReader(reader) if err != nil { return err } offset += chunkSize } return nil } func (d *GoogleDrive) getAbout(ctx context.Context) (*AboutResp, error) { query := map[string]string{ "fields": "storageQuota", } var resp AboutResp _, err := d.request("https://www.googleapis.com/drive/v3/about", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) req.SetContext(ctx) }, &resp) if err != nil { return nil, err } return &resp, nil } ================================================ FILE: drivers/google_photo/driver.go ================================================ package google_photo import ( "context" "fmt" "net/http" "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) type GooglePhoto struct { model.Storage Addition AccessToken string } func (d *GooglePhoto) Config() driver.Config { return config } func (d *GooglePhoto) GetAddition() driver.Additional { return &d.Addition } func (d *GooglePhoto) Init(ctx context.Context) error { return d.refreshToken() } func (d *GooglePhoto) Drop(ctx context.Context) error { return nil } func (d *GooglePhoto) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(dir.GetID()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src MediaItem) (model.Obj, error) { return fileToObj(src), nil }) } func (d *GooglePhoto) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { f, err := d.getMedia(file.GetID()) if err != nil { return nil, err } if strings.Contains(f.MimeType, "image/") { return &model.Link{ URL: f.BaseURL + "=d", }, nil } else if strings.Contains(f.MimeType, "video/") { return &model.Link{ URL: f.BaseURL + "=dv", }, nil } return &model.Link{}, nil } func (d *GooglePhoto) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { return errs.NotSupport } func (d *GooglePhoto) Move(ctx context.Context, srcObj, dstDir model.Obj) error { return errs.NotSupport } func (d *GooglePhoto) Rename(ctx context.Context, srcObj model.Obj, newName string) error { return errs.NotSupport } func (d *GooglePhoto) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return errs.NotSupport } func (d *GooglePhoto) Remove(ctx context.Context, obj model.Obj) error { return errs.NotSupport } func (d *GooglePhoto) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { var e Error // Create resumable upload url postHeaders := map[string]string{ "Authorization": "Bearer " + d.AccessToken, "Content-type": "application/octet-stream", "X-Goog-Upload-Command": "start", "X-Goog-Upload-Content-Type": stream.GetMimetype(), "X-Goog-Upload-Protocol": "resumable", "X-Goog-Upload-Raw-Size": strconv.FormatInt(stream.GetSize(), 10), } url := "https://photoslibrary.googleapis.com/v1/uploads" res, err := base.NoRedirectClient.R().SetHeaders(postHeaders). SetError(&e). Post(url) if err != nil { return err } if e.Error.Code != 0 { if e.Error.Code == 401 { err = d.refreshToken() if err != nil { return err } return d.Put(ctx, dstDir, stream, up) } return fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) } //Upload to the Google Photo postUrl := res.Header().Get("X-Goog-Upload-URL") //chunkSize := res.Header().Get("X-Goog-Upload-Chunk-Granularity") postHeaders = map[string]string{ "X-Goog-Upload-Command": "upload, finalize", "X-Goog-Upload-Offset": "0", } resp, err := d.request(postUrl, http.MethodPost, func(req *resty.Request) { req.SetBody(driver.NewLimitedUploadStream(ctx, stream)).SetContext(ctx) }, nil, postHeaders) if err != nil { return err } //Create MediaItem createItemUrl := "https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate" postHeaders = map[string]string{ "X-Goog-Upload-Command": "upload, finalize", "X-Goog-Upload-Offset": "0", } data := base.Json{ "newMediaItems": []base.Json{ { "description": "item-description", "simpleMediaItem": base.Json{ "fileName": stream.GetName(), "uploadToken": string(resp), }, }, }, } _, err = d.request(createItemUrl, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil, postHeaders) return err } var _ driver.Driver = (*GooglePhoto)(nil) ================================================ FILE: drivers/google_photo/meta.go ================================================ package google_photo import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootID RefreshToken string `json:"refresh_token" required:"true"` ClientID string `json:"client_id" required:"true" default:"202264815644.apps.googleusercontent.com"` ClientSecret string `json:"client_secret" required:"true" default:"X4Z3ca8xfWDb1Voo-F9a7ZxJ"` ShowArchive bool `json:"show_archive"` } var config = driver.Config{ Name: "GooglePhoto", OnlyProxy: true, DefaultRoot: "root", NoUpload: true, LocalSort: true, } func init() { op.RegisterDriver(func() driver.Driver { return &GooglePhoto{} }) } ================================================ FILE: drivers/google_photo/types.go ================================================ package google_photo import ( "reflect" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type TokenError struct { Error string `json:"error"` ErrorDescription string `json:"error_description"` } type Items struct { NextPageToken string `json:"nextPageToken"` MediaItems []MediaItem `json:"mediaItems,omitempty"` Albums []MediaItem `json:"albums,omitempty"` SharedAlbums []MediaItem `json:"sharedAlbums,omitempty"` } type MediaItem struct { Id string `json:"id"` Title string `json:"title,omitempty"` BaseURL string `json:"baseUrl,omitempty"` CoverPhotoBaseUrl string `json:"coverPhotoBaseUrl,omitempty"` MimeType string `json:"mimeType,omitempty"` FileName string `json:"filename,omitempty"` MediaMetadata MediaMetadata `json:"mediaMetadata,omitempty"` } type MediaMetadata struct { CreationTime time.Time `json:"creationTime"` Width string `json:"width"` Height string `json:"height"` Photo Photo `json:"photo,omitempty"` Video Video `json:"video,omitempty"` } type Photo struct { } type Video struct { } func fileToObj(f MediaItem) *model.ObjThumb { if !reflect.DeepEqual(f.MediaMetadata, MediaMetadata{}) { return &model.ObjThumb{ Object: model.Object{ ID: f.Id, Name: f.FileName, Size: 0, Modified: f.MediaMetadata.CreationTime, IsFolder: false, }, Thumbnail: model.Thumbnail{ Thumbnail: f.BaseURL + "=w100-h100-c", }, } } return &model.ObjThumb{ Object: model.Object{ ID: f.Id, Name: f.Title, Size: 0, Modified: time.Time{}, IsFolder: true, }, Thumbnail: model.Thumbnail{}, } } type Error struct { Error struct { Errors []struct { Domain string `json:"domain"` Reason string `json:"reason"` Message string `json:"message"` LocationType string `json:"location_type"` Location string `json:"location"` } Code int `json:"code"` Message string `json:"message"` } `json:"error"` } ================================================ FILE: drivers/google_photo/util.go ================================================ package google_photo import ( "fmt" "net/http" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/go-resty/resty/v2" ) // do others that not defined in Driver interface const ( FETCH_ALL = "all" FETCH_ALBUMS = "albums" FETCH_ROOT = "root" FETCH_SHARE_ALBUMS = "share_albums" ) func (d *GooglePhoto) refreshToken() error { url := "https://www.googleapis.com/oauth2/v4/token" var resp base.TokenResp var e TokenError _, err := base.RestyClient.R().SetResult(&resp).SetError(&e). SetFormData(map[string]string{ "client_id": d.ClientID, "client_secret": d.ClientSecret, "refresh_token": d.RefreshToken, "grant_type": "refresh_token", }).Post(url) if err != nil { return err } if e.Error != "" { return fmt.Errorf(e.Error) } d.AccessToken = resp.AccessToken return nil } func (d *GooglePhoto) request(url string, method string, callback base.ReqCallback, resp interface{}, headers map[string]string) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) req.SetHeader("Accept-Encoding", "gzip") if headers != nil { req.SetHeaders(headers) } if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e Error req.SetError(&e) res, err := req.Execute(method, url) if err != nil { return nil, err } if e.Error.Code != 0 { if e.Error.Code == 401 { err = d.refreshToken() if err != nil { return nil, err } return d.request(url, method, callback, resp, headers) } return nil, fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) } return res.Body(), nil } func (d *GooglePhoto) getFiles(id string) ([]MediaItem, error) { switch id { case FETCH_ALL: return d.getAllMedias() case FETCH_ALBUMS: return d.getAlbums() case FETCH_SHARE_ALBUMS: return d.getShareAlbums() case FETCH_ROOT: return d.getFakeRoot() default: return d.getMedias(id) } } func (d *GooglePhoto) getFakeRoot() ([]MediaItem, error) { return []MediaItem{ { Id: FETCH_ALL, Title: FETCH_ALL, }, { Id: FETCH_ALBUMS, Title: FETCH_ALBUMS, }, { Id: FETCH_SHARE_ALBUMS, Title: FETCH_SHARE_ALBUMS, }, }, nil } func (d *GooglePhoto) getAlbums() ([]MediaItem, error) { return d.fetchItems( "https://photoslibrary.googleapis.com/v1/albums", map[string]string{ "fields": "albums(id,title,coverPhotoBaseUrl),nextPageToken", "pageSize": "50", "pageToken": "first", }, http.MethodGet) } func (d *GooglePhoto) getShareAlbums() ([]MediaItem, error) { return d.fetchItems( "https://photoslibrary.googleapis.com/v1/sharedAlbums", map[string]string{ "fields": "sharedAlbums(id,title,coverPhotoBaseUrl),nextPageToken", "pageSize": "50", "pageToken": "first", }, http.MethodGet) } func (d *GooglePhoto) getMedias(albumId string) ([]MediaItem, error) { return d.fetchItems( "https://photoslibrary.googleapis.com/v1/mediaItems:search", map[string]string{ "fields": "mediaItems(id,baseUrl,mimeType,mediaMetadata,filename),nextPageToken", "pageSize": "100", "albumId": albumId, "pageToken": "first", }, http.MethodPost) } func (d *GooglePhoto) getAllMedias() ([]MediaItem, error) { return d.fetchItems( "https://photoslibrary.googleapis.com/v1/mediaItems", map[string]string{ "fields": "mediaItems(id,baseUrl,mimeType,mediaMetadata,filename),nextPageToken", "pageSize": "100", "pageToken": "first", }, http.MethodGet) } func (d *GooglePhoto) getMedia(id string) (MediaItem, error) { var resp MediaItem query := map[string]string{ "fields": "mediaMetadata,baseUrl,mimeType", } _, err := d.request(fmt.Sprintf("https://photoslibrary.googleapis.com/v1/mediaItems/%s", id), http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp, nil) if err != nil { return resp, err } return resp, nil } func (d *GooglePhoto) fetchItems(url string, query map[string]string, method string) ([]MediaItem, error) { res := make([]MediaItem, 0) for query["pageToken"] != "" { if query["pageToken"] == "first" { query["pageToken"] = "" } var resp Items _, err := d.request(url, method, func(req *resty.Request) { req.SetQueryParams(query) }, &resp, nil) if err != nil { return nil, err } query["pageToken"] = resp.NextPageToken res = append(res, resp.MediaItems...) res = append(res, resp.Albums...) res = append(res, resp.SharedAlbums...) } return res, nil } ================================================ FILE: drivers/halalcloud/driver.go ================================================ package halalcloud import ( "context" "crypto/sha1" "fmt" "io" "net/url" "path" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/city404/v6-public-rpc-proto/go/v6/common" pbPublicUser "github.com/city404/v6-public-rpc-proto/go/v6/user" pubUserFile "github.com/city404/v6-public-rpc-proto/go/v6/userfile" "github.com/rclone/rclone/lib/readers" "github.com/zzzhr1990/go-common-entity/userfile" ) type HalalCloud struct { *HalalCommon model.Storage Addition uploadThread int } func (d *HalalCloud) Config() driver.Config { return config } func (d *HalalCloud) GetAddition() driver.Additional { return &d.Addition } func (d *HalalCloud) Init(ctx context.Context) error { d.uploadThread, _ = strconv.Atoi(d.UploadThread) if d.uploadThread < 1 || d.uploadThread > 32 { d.uploadThread, d.UploadThread = 3, "3" } if d.HalalCommon == nil { d.HalalCommon = &HalalCommon{ Common: &Common{}, AuthService: &AuthService{ appID: func() string { if d.Addition.AppID != "" { return d.Addition.AppID } return AppID }(), appVersion: func() string { if d.Addition.AppVersion != "" { return d.Addition.AppVersion } return AppVersion }(), appSecret: func() string { if d.Addition.AppSecret != "" { return d.Addition.AppSecret } return AppSecret }(), tr: &TokenResp{ RefreshToken: d.Addition.RefreshToken, }, }, UserInfo: &UserInfo{}, refreshTokenFunc: func(token string) error { d.Addition.RefreshToken = token op.MustSaveDriverStorage(d) return nil }, } } // 防止重复登录 if d.Addition.RefreshToken == "" || !d.IsLogin() { as, err := d.NewAuthServiceWithOauth() if err != nil { d.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) return err } d.HalalCommon.AuthService = as d.SetTokenResp(as.tr) op.MustSaveDriverStorage(d) } var err error d.HalalCommon.serv, err = d.NewAuthService(d.Addition.RefreshToken) if err != nil { return err } return nil } func (d *HalalCloud) Drop(ctx context.Context) error { return nil } func (d *HalalCloud) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { return d.getFiles(ctx, dir) } func (d *HalalCloud) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { return d.getLink(ctx, file, args) } func (d *HalalCloud) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { return d.makeDir(ctx, parentDir, dirName) } func (d *HalalCloud) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return d.move(ctx, srcObj, dstDir) } func (d *HalalCloud) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { return d.rename(ctx, srcObj, newName) } func (d *HalalCloud) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return d.copy(ctx, srcObj, dstDir) } func (d *HalalCloud) Remove(ctx context.Context, obj model.Obj) error { return d.remove(ctx, obj) } func (d *HalalCloud) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { return d.put(ctx, dstDir, stream, up) } func (d *HalalCloud) IsLogin() bool { if d.AuthService.tr == nil { return false } serv, err := d.NewAuthService(d.Addition.RefreshToken) if err != nil { return false } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() result, err := pbPublicUser.NewPubUserClient(serv.GetGrpcConnection()).Get(ctx, &pbPublicUser.User{ Identity: "", }) if result == nil || err != nil { return false } d.UserInfo.Identity = result.Identity d.UserInfo.CreateTs = result.CreateTs d.UserInfo.Name = result.Name d.UserInfo.UpdateTs = result.UpdateTs return true } type HalalCommon struct { *Common *AuthService // 登录信息 *UserInfo // 用户信息 refreshTokenFunc func(token string) error serv *AuthService } func (d *HalalCloud) SetTokenResp(tr *TokenResp) { d.Addition.RefreshToken = tr.RefreshToken } func (d *HalalCloud) getFiles(ctx context.Context, dir model.Obj) ([]model.Obj, error) { files := make([]model.Obj, 0) limit := int64(100) token := "" client := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()) opDir := d.GetCurrentDir(dir) for { result, err := client.List(ctx, &pubUserFile.FileListRequest{ Parent: &pubUserFile.File{Path: opDir}, ListInfo: &common.ScanListRequest{ Limit: limit, Token: token, }, }) if err != nil { return nil, err } for i := 0; len(result.Files) > i; i++ { files = append(files, (*Files)(result.Files[i])) } if result.ListInfo == nil || result.ListInfo.Token == "" { break } token = result.ListInfo.Token } return files, nil } func (d *HalalCloud) getLink(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { client := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()) ctx1, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() result, err := client.ParseFileSlice(ctx1, (*pubUserFile.File)(file.(*Files))) if err != nil { return nil, err } fileAddrs := []*pubUserFile.SliceDownloadInfo{} var addressDuration int64 nodesNumber := len(result.RawNodes) nodesIndex := nodesNumber - 1 startIndex, endIndex := 0, nodesIndex for nodesIndex >= 0 { if nodesIndex >= 200 { endIndex = 200 } else { endIndex = nodesNumber } for ; endIndex <= nodesNumber; endIndex += 200 { if endIndex == 0 { endIndex = 1 } sliceAddress, err := client.GetSliceDownloadAddress(ctx, &pubUserFile.SliceDownloadAddressRequest{ Identity: result.RawNodes[startIndex:endIndex], Version: 1, }) if err != nil { return nil, err } addressDuration = sliceAddress.ExpireAt fileAddrs = append(fileAddrs, sliceAddress.Addresses...) startIndex = endIndex nodesIndex -= 200 } } size := result.FileSize chunks := getChunkSizes(result.Sizes) resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { length := httpRange.Length if httpRange.Length < 0 || httpRange.Start+httpRange.Length >= size { length = size - httpRange.Start } oo := &openObject{ ctx: ctx, d: fileAddrs, chunk: &[]byte{}, chunks: &chunks, skip: httpRange.Start, sha: result.Sha1, shaTemp: sha1.New(), } return readers.NewLimitedReadCloser(oo, length), nil } var duration time.Duration if addressDuration != 0 { duration = time.Until(time.UnixMilli(addressDuration)) } else { duration = time.Until(time.Now().Add(time.Hour)) } return &model.Link{ RangeReader: stream.RateLimitRangeReaderFunc(resultRangeReader), Expiration: &duration, }, nil } func (d *HalalCloud) makeDir(ctx context.Context, dir model.Obj, name string) (model.Obj, error) { newDir := userfile.NewFormattedPath(d.GetCurrentOpDir(dir, []string{name}, 0)).GetPath() _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Create(ctx, &pubUserFile.File{ Path: newDir, }) return nil, err } func (d *HalalCloud) move(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) { oldDir := userfile.NewFormattedPath(d.GetCurrentDir(obj)).GetPath() newDir := userfile.NewFormattedPath(d.GetCurrentDir(dir)).GetPath() _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Move(ctx, &pubUserFile.BatchOperationRequest{ Source: []*pubUserFile.File{ { Identity: obj.GetID(), Path: oldDir, }, }, Dest: &pubUserFile.File{ Identity: dir.GetID(), Path: newDir, }, }) return nil, err } func (d *HalalCloud) rename(ctx context.Context, obj model.Obj, name string) (model.Obj, error) { id := obj.GetID() newPath := userfile.NewFormattedPath(d.GetCurrentOpDir(obj, []string{name}, 0)).GetPath() _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Rename(ctx, &pubUserFile.File{ Path: newPath, Identity: id, Name: name, }) return nil, err } func (d *HalalCloud) copy(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) { id := obj.GetID() sourcePath := userfile.NewFormattedPath(d.GetCurrentDir(obj)).GetPath() if len(id) > 0 { sourcePath = "" } dest := &pubUserFile.File{ Identity: dir.GetID(), Path: userfile.NewFormattedPath(d.GetCurrentDir(dir)).GetPath(), } _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Copy(ctx, &pubUserFile.BatchOperationRequest{ Source: []*pubUserFile.File{ { Path: sourcePath, Identity: id, }, }, Dest: dest, }) return nil, err } func (d *HalalCloud) remove(ctx context.Context, obj model.Obj) error { id := obj.GetID() newPath := userfile.NewFormattedPath(d.GetCurrentDir(obj)).GetPath() //if len(id) > 0 { // newPath = "" //} _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Delete(ctx, &pubUserFile.BatchOperationRequest{ Source: []*pubUserFile.File{ { Path: newPath, Identity: id, }, }, }) return err } func (d *HalalCloud) put(ctx context.Context, dstDir model.Obj, fileStream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { newDir := path.Join(dstDir.GetPath(), fileStream.GetName()) result, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).CreateUploadToken(ctx, &pubUserFile.File{ Path: newDir, }) if err != nil { return nil, err } u, _ := url.Parse(result.Endpoint) u.Host = "s3." + u.Host result.Endpoint = u.String() s, err := session.NewSession(&aws.Config{ HTTPClient: base.HttpClient, Credentials: credentials.NewStaticCredentials(result.AccessKey, result.SecretKey, result.Token), Region: aws.String(result.Region), Endpoint: aws.String(result.Endpoint), S3ForcePathStyle: aws.Bool(true), }) if err != nil { return nil, err } uploader := s3manager.NewUploader(s, func(u *s3manager.Uploader) { u.Concurrency = d.uploadThread }) if fileStream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { uploader.PartSize = fileStream.GetSize() / (s3manager.MaxUploadParts - 1) } reader := driver.NewLimitedUploadStream(ctx, fileStream) _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ Bucket: aws.String(result.Bucket), Key: aws.String(result.Key), Body: io.TeeReader(reader, driver.NewProgress(fileStream.GetSize(), up)), }) return nil, err } var _ driver.Driver = (*HalalCloud)(nil) ================================================ FILE: drivers/halalcloud/meta.go ================================================ package halalcloud import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { // Usually one of two driver.RootPath // define other RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` UploadThread string `json:"upload_thread" default:"3" help:"1 <= thread <= 32"` AppID string `json:"app_id" required:"true" default:"openlist/10001"` AppVersion string `json:"app_version" required:"true" default:"1.0.0"` AppSecret string `json:"app_secret" required:"true" default:"bR4SJwOkvnG5WvVJ"` } var config = driver.Config{ Name: "HalalCloud", OnlyProxy: true, DefaultRoot: "/", NoLinkURL: true, } func init() { op.RegisterDriver(func() driver.Driver { return &HalalCloud{} }) } ================================================ FILE: drivers/halalcloud/options.go ================================================ package halalcloud import "google.golang.org/grpc" func defaultOptions() halalOptions { return halalOptions{ // onRefreshTokenRefreshed: func(string) {}, grpcOptions: []grpc.DialOption{ grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(1024 * 1024 * 32)), // grpc.WithMaxMsgSize(1024 * 1024 * 1024), }, } } type HalalOption interface { apply(*halalOptions) } // halalOptions configure a RPC call. halalOptions are set by the HalalOption // values passed to Dial. type halalOptions struct { onTokenRefreshed func(accessToken string, accessTokenExpiredAt int64, refreshToken string, refreshTokenExpiredAt int64) grpcOptions []grpc.DialOption } // funcDialOption wraps a function that modifies halalOptions into an // implementation of the DialOption interface. type funcDialOption struct { f func(*halalOptions) } func (fdo *funcDialOption) apply(do *halalOptions) { fdo.f(do) } func newFuncDialOption(f func(*halalOptions)) *funcDialOption { return &funcDialOption{ f: f, } } func WithRefreshTokenRefreshedCallback(s func(accessToken string, accessTokenExpiredAt int64, refreshToken string, refreshTokenExpiredAt int64)) HalalOption { return newFuncDialOption(func(o *halalOptions) { o.onTokenRefreshed = s }) } func WithGrpcDialOptions(opts ...grpc.DialOption) HalalOption { return newFuncDialOption(func(o *halalOptions) { o.grpcOptions = opts }) } ================================================ FILE: drivers/halalcloud/types.go ================================================ package halalcloud import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/city404/v6-public-rpc-proto/go/v6/common" pubUserFile "github.com/city404/v6-public-rpc-proto/go/v6/userfile" "google.golang.org/grpc" ) type AuthService struct { appID string appVersion string appSecret string grpcConnection *grpc.ClientConn dopts halalOptions tr *TokenResp } type TokenResp struct { AccessToken string `json:"accessToken,omitempty"` AccessTokenExpiredAt int64 `json:"accessTokenExpiredAt,omitempty"` RefreshToken string `json:"refreshToken,omitempty"` RefreshTokenExpiredAt int64 `json:"refreshTokenExpiredAt,omitempty"` } type UserInfo struct { Identity string `json:"identity,omitempty"` UpdateTs int64 `json:"updateTs,omitempty"` Name string `json:"name,omitempty"` CreateTs int64 `json:"createTs,omitempty"` } type OrderByInfo struct { Field string `json:"field,omitempty"` Asc bool `json:"asc,omitempty"` } type ListInfo struct { Token string `json:"token,omitempty"` Limit int64 `json:"limit,omitempty"` OrderBy []*OrderByInfo `json:"order_by,omitempty"` Version int32 `json:"version,omitempty"` } type FilesList struct { Files []*Files `json:"files,omitempty"` ListInfo *common.ScanListRequest `json:"list_info,omitempty"` } var _ model.Obj = (*Files)(nil) type Files pubUserFile.File func (f *Files) GetSize() int64 { return f.Size } func (f *Files) GetName() string { return f.Name } func (f *Files) ModTime() time.Time { return time.UnixMilli(f.UpdateTs) } func (f *Files) CreateTime() time.Time { return time.UnixMilli(f.UpdateTs) } func (f *Files) IsDir() bool { return f.Dir } func (f *Files) GetHash() utils.HashInfo { return utils.HashInfo{} } func (f *Files) GetID() string { if len(f.Identity) == 0 { f.Identity = "/" } return f.Identity } func (f *Files) GetPath() string { return f.Path } type SteamFile struct { file model.File } func (s *SteamFile) Read(p []byte) (n int, err error) { return s.file.Read(p) } ================================================ FILE: drivers/halalcloud/util.go ================================================ package halalcloud import ( "bytes" "context" "crypto/md5" "crypto/tls" "encoding/hex" "errors" "fmt" "hash" "io" "net/http" "strconv" "strings" "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" pbPublicUser "github.com/city404/v6-public-rpc-proto/go/v6/user" pubUserFile "github.com/city404/v6-public-rpc-proto/go/v6/userfile" "github.com/google/uuid" "github.com/ipfs/go-cid" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" ) const ( AppID = "alist/10001" AppVersion = "1.0.0" AppSecret = "bR4SJwOkvnG5WvVJ" ) const ( grpcServer = "grpcuserapi.2dland.cn:443" grpcServerAuth = "grpcuserapi.2dland.cn" ) func (d *HalalCloud) NewAuthServiceWithOauth(options ...HalalOption) (*AuthService, error) { aService := &AuthService{} err2 := errors.New("") svc := d.HalalCommon.AuthService for _, opt := range options { opt.apply(&svc.dopts) } grpcOptions := svc.dopts.grpcOptions grpcOptions = append(grpcOptions, grpc.WithAuthority(grpcServerAuth), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})), grpc.WithUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { ctxx := svc.signContext(method, ctx) err := invoker(ctxx, method, req, reply, cc, opts...) // invoking RPC method return err })) grpcConnection, err := grpc.NewClient(grpcServer, grpcOptions...) if err != nil { return nil, err } defer grpcConnection.Close() userClient := pbPublicUser.NewPubUserClient(grpcConnection) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() stateString := uuid.New().String() // queryValues.Add("callback", oauthToken.Callback) oauthToken, err := userClient.CreateAuthToken(ctx, &pbPublicUser.LoginRequest{ ReturnType: 2, State: stateString, ReturnUrl: "", }) if err != nil { return nil, err } if len(oauthToken.State) < 1 { oauthToken.State = stateString } if oauthToken.Url != "" { return nil, fmt.Errorf(`need verify: Click Here`, oauthToken.Url) } return aService, err2 } func (d *HalalCloud) NewAuthService(refreshToken string, options ...HalalOption) (*AuthService, error) { svc := d.HalalCommon.AuthService if len(refreshToken) < 1 { refreshToken = d.Addition.RefreshToken } if len(d.tr.AccessToken) > 0 { accessTokenExpiredAt := d.tr.AccessTokenExpiredAt current := time.Now().UnixMilli() if accessTokenExpiredAt < current { // access token expired d.tr.AccessToken = "" d.tr.AccessTokenExpiredAt = 0 } else { svc.tr.AccessTokenExpiredAt = accessTokenExpiredAt svc.tr.AccessToken = d.tr.AccessToken } } for _, opt := range options { opt.apply(&svc.dopts) } grpcOptions := svc.dopts.grpcOptions grpcOptions = append(grpcOptions, grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(10*1024*1024), grpc.MaxCallRecvMsgSize(10*1024*1024)), grpc.WithAuthority(grpcServerAuth), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})), grpc.WithUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { ctxx := svc.signContext(method, ctx) err := invoker(ctxx, method, req, reply, cc, opts...) // invoking RPC method if err != nil { grpcStatus, ok := status.FromError(err) if ok && grpcStatus.Code() == codes.Unauthenticated && strings.Contains(grpcStatus.Err().Error(), "invalid accesstoken") && len(refreshToken) > 0 { // refresh token refreshResponse, err := pbPublicUser.NewPubUserClient(cc).Refresh(ctx, &pbPublicUser.Token{ RefreshToken: refreshToken, }) if err != nil { return err } if len(refreshResponse.AccessToken) > 0 { svc.tr.AccessToken = refreshResponse.AccessToken svc.tr.AccessTokenExpiredAt = refreshResponse.AccessTokenExpireTs svc.OnAccessTokenRefreshed(refreshResponse.AccessToken, refreshResponse.AccessTokenExpireTs, refreshResponse.RefreshToken, refreshResponse.RefreshTokenExpireTs) } // retry ctxx := svc.signContext(method, ctx) err = invoker(ctxx, method, req, reply, cc, opts...) // invoking RPC method if err != nil { return err } else { return nil } } } return err })) grpcConnection, err := grpc.NewClient(grpcServer, grpcOptions...) if err != nil { return nil, err } svc.grpcConnection = grpcConnection return svc, err } func (s *AuthService) OnAccessTokenRefreshed(accessToken string, accessTokenExpiredAt int64, refreshToken string, refreshTokenExpiredAt int64) { s.tr.AccessToken = accessToken s.tr.AccessTokenExpiredAt = accessTokenExpiredAt s.tr.RefreshToken = refreshToken s.tr.RefreshTokenExpiredAt = refreshTokenExpiredAt if s.dopts.onTokenRefreshed != nil { s.dopts.onTokenRefreshed(accessToken, accessTokenExpiredAt, refreshToken, refreshTokenExpiredAt) } } func (s *AuthService) GetGrpcConnection() *grpc.ClientConn { return s.grpcConnection } func (s *AuthService) Close() { _ = s.grpcConnection.Close() } func (s *AuthService) signContext(method string, ctx context.Context) context.Context { var kvString []string currentTimeStamp := strconv.FormatInt(time.Now().UnixMilli(), 10) bufferedString := bytes.NewBufferString(method) kvString = append(kvString, "timestamp", currentTimeStamp) bufferedString.WriteString(currentTimeStamp) kvString = append(kvString, "appid", s.appID) bufferedString.WriteString(s.appID) kvString = append(kvString, "appversion", s.appVersion) bufferedString.WriteString(s.appVersion) if s.tr != nil && len(s.tr.AccessToken) > 0 { authorization := "Bearer " + s.tr.AccessToken kvString = append(kvString, "authorization", authorization) bufferedString.WriteString(authorization) } bufferedString.WriteString(s.appSecret) sign := GetMD5Hash(bufferedString.String()) kvString = append(kvString, "sign", sign) return metadata.AppendToOutgoingContext(ctx, kvString...) } func (d *HalalCloud) GetCurrentOpDir(dir model.Obj, args []string, index int) string { currentDir := dir.GetPath() if len(currentDir) == 0 { currentDir = "/" } opPath := currentDir + "/" + args[index] if strings.HasPrefix(args[index], "/") { opPath = args[index] } return opPath } func (d *HalalCloud) GetCurrentDir(dir model.Obj) string { currentDir := dir.GetPath() if len(currentDir) == 0 { currentDir = "/" } return currentDir } type Common struct { } func getRawFiles(addr *pubUserFile.SliceDownloadInfo) ([]byte, error) { if addr == nil { return nil, errors.New("addr is nil") } client := http.Client{ Timeout: time.Duration(60 * time.Second), // Set timeout to 5 seconds } resp, err := client.Get(addr.DownloadAddress) if err != nil { return nil, err } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("bad status: %s, body: %s", resp.Status, body) } if addr.Encrypt > 0 { cd := uint8(addr.Encrypt) for idx := 0; idx < len(body); idx++ { body[idx] = body[idx] ^ cd } } if addr.StoreType != 10 { sourceCid, err := cid.Decode(addr.Identity) if err != nil { return nil, err } checkCid, err := sourceCid.Prefix().Sum(body) if err != nil { return nil, err } if !checkCid.Equals(sourceCid) { return nil, fmt.Errorf("bad cid: %s, body: %s", checkCid.String(), body) } } return body, nil } type openObject struct { ctx context.Context mu sync.Mutex d []*pubUserFile.SliceDownloadInfo id int skip int64 chunk *[]byte chunks *[]chunkSize closed bool sha string shaTemp hash.Hash } // get the next chunk func (oo *openObject) getChunk(ctx context.Context) (err error) { if oo.id >= len(*oo.chunks) { return io.EOF } var chunk []byte err = utils.Retry(3, time.Second, func() (err error) { chunk, err = getRawFiles(oo.d[oo.id]) return err }) if err != nil { return err } oo.id++ oo.chunk = &chunk return nil } // Read reads up to len(p) bytes into p. func (oo *openObject) Read(p []byte) (n int, err error) { oo.mu.Lock() defer oo.mu.Unlock() if oo.closed { return 0, fmt.Errorf("read on closed file") } // Skip data at the start if requested for oo.skip > 0 { //size := 1024 * 1024 _, size, err := oo.ChunkLocation(oo.id) if err != nil { return 0, err } if oo.skip < int64(size) { break } oo.id++ oo.skip -= int64(size) } if len(*oo.chunk) == 0 { err = oo.getChunk(oo.ctx) if err != nil { return 0, err } if oo.skip > 0 { *oo.chunk = (*oo.chunk)[oo.skip:] oo.skip = 0 } } n = copy(p, *oo.chunk) *oo.chunk = (*oo.chunk)[n:] oo.shaTemp.Write(*oo.chunk) return n, nil } // Close closed the file - MAC errors are reported here func (oo *openObject) Close() (err error) { oo.mu.Lock() defer oo.mu.Unlock() if oo.closed { return nil } // 校验Sha1 if string(oo.shaTemp.Sum(nil)) != oo.sha { return fmt.Errorf("failed to finish download: %w", err) } oo.closed = true return nil } func GetMD5Hash(text string) string { tHash := md5.Sum([]byte(text)) return hex.EncodeToString(tHash[:]) } // chunkSize describes a size and position of chunk type chunkSize struct { position int64 size int } func getChunkSizes(sliceSize []*pubUserFile.SliceSize) (chunks []chunkSize) { chunks = make([]chunkSize, 0) for _, s := range sliceSize { // 对最后一个做特殊处理 if s.EndIndex == 0 { s.EndIndex = s.StartIndex } for j := s.StartIndex; j <= s.EndIndex; j++ { chunks = append(chunks, chunkSize{position: j, size: int(s.Size)}) } } return chunks } func (oo *openObject) ChunkLocation(id int) (position int64, size int, err error) { if id < 0 || id >= len(*oo.chunks) { return 0, 0, errors.New("invalid arguments") } return (*oo.chunks)[id].position, (*oo.chunks)[id].size, nil } ================================================ FILE: drivers/halalcloud_open/common.go ================================================ package halalcloudopen import ( "sync" "time" sdkUser "github.com/halalcloud/golang-sdk-lite/halalcloud/services/user" ) var ( slicePostErrorRetryInterval = time.Second * 120 retryTimes = 5 ) type halalCommon struct { // *AuthService // 登录信息 UserInfo *sdkUser.User // 用户信息 refreshTokenFunc func(token string) error // serv *AuthService configs sync.Map } func (m *halalCommon) GetAccessToken() (string, error) { value, exists := m.configs.Load("access_token") if !exists { return "", nil // 如果不存在，返回空字符串 } return value.(string), nil // 返回配置项的值 } // GetRefreshToken implements ConfigStore. func (m *halalCommon) GetRefreshToken() (string, error) { value, exists := m.configs.Load("refresh_token") if !exists { return "", nil // 如果不存在，返回空字符串 } return value.(string), nil // 返回配置项的值 } // SetAccessToken implements ConfigStore. func (m *halalCommon) SetAccessToken(token string) error { m.configs.Store("access_token", token) return nil } // SetRefreshToken implements ConfigStore. func (m *halalCommon) SetRefreshToken(token string) error { m.configs.Store("refresh_token", token) if m.refreshTokenFunc != nil { return m.refreshTokenFunc(token) } return nil } // SetToken implements ConfigStore. func (m *halalCommon) SetToken(accessToken string, refreshToken string, expiresIn int64) error { m.configs.Store("access_token", accessToken) m.configs.Store("refresh_token", refreshToken) m.configs.Store("expires_in", expiresIn) if m.refreshTokenFunc != nil { return m.refreshTokenFunc(refreshToken) } return nil } // ClearConfigs implements ConfigStore. func (m *halalCommon) ClearConfigs() error { m.configs = sync.Map{} // 清空map return nil } // DeleteConfig implements ConfigStore. func (m *halalCommon) DeleteConfig(key string) error { _, exists := m.configs.Load(key) if !exists { return nil // 如果不存在，直接返回 } m.configs.Delete(key) // 删除指定的配置项 return nil } // GetConfig implements ConfigStore. func (m *halalCommon) GetConfig(key string) (string, error) { value, exists := m.configs.Load(key) if !exists { return "", nil // 如果不存在，返回空字符串 } return value.(string), nil // 返回配置项的值 } // ListConfigs implements ConfigStore. func (m *halalCommon) ListConfigs() (map[string]string, error) { configs := make(map[string]string) m.configs.Range(func(key, value interface{}) bool { configs[key.(string)] = value.(string) // 将每个配置项添加到map中 return true // 继续遍历 }) return configs, nil // 返回所有配置项 } // SetConfig implements ConfigStore. func (m *halalCommon) SetConfig(key string, value string) error { m.configs.Store(key, value) // 使用Store方法设置或更新配置项 return nil // 成功设置配置项后返回nil } func NewHalalCommon() *halalCommon { return &halalCommon{ configs: sync.Map{}, } } ================================================ FILE: drivers/halalcloud_open/driver.go ================================================ package halalcloudopen import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" sdkClient "github.com/halalcloud/golang-sdk-lite/halalcloud/apiclient" sdkUser "github.com/halalcloud/golang-sdk-lite/halalcloud/services/user" sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" ) type HalalCloudOpen struct { *halalCommon model.Storage Addition sdkClient *sdkClient.Client sdkUserFileService *sdkUserFile.UserFileService sdkUserService *sdkUser.UserService uploadThread int } func (d *HalalCloudOpen) Config() driver.Config { return config } func (d *HalalCloudOpen) GetAddition() driver.Additional { return &d.Addition } var _ driver.Driver = (*HalalCloudOpen)(nil) ================================================ FILE: drivers/halalcloud_open/driver_curd_impl.go ================================================ package halalcloudopen import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/model" sdkModel "github.com/halalcloud/golang-sdk-lite/halalcloud/model" sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" ) func (d *HalalCloudOpen) getFiles(ctx context.Context, dir model.Obj) ([]model.Obj, error) { files := make([]model.Obj, 0) limit := int64(100) token := "" for { result, err := d.sdkUserFileService.List(ctx, &sdkUserFile.FileListRequest{ Parent: &sdkUserFile.File{Path: dir.GetPath()}, ListInfo: &sdkModel.ScanListRequest{ Limit: limit, Token: token, }, }) if err != nil { return nil, err } for i := 0; len(result.Files) > i; i++ { files = append(files, NewObjFile(result.Files[i])) } if result.ListInfo == nil || result.ListInfo.Token == "" { break } token = result.ListInfo.Token } return files, nil } func (d *HalalCloudOpen) makeDir(ctx context.Context, dir model.Obj, name string) (model.Obj, error) { _, err := d.sdkUserFileService.Create(ctx, &sdkUserFile.File{ Path: dir.GetPath(), Name: name, }) return nil, err } func (d *HalalCloudOpen) move(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) { oldDir := obj.GetPath() newDir := dir.GetPath() _, err := d.sdkUserFileService.Move(ctx, &sdkUserFile.BatchOperationRequest{ Source: []*sdkUserFile.File{ { Path: oldDir, }, }, Dest: &sdkUserFile.File{ Path: newDir, }, }) return nil, err } func (d *HalalCloudOpen) rename(ctx context.Context, obj model.Obj, name string) (model.Obj, error) { _, err := d.sdkUserFileService.Rename(ctx, &sdkUserFile.File{ Path: obj.GetPath(), Name: name, }) return nil, err } func (d *HalalCloudOpen) copy(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) { id := obj.GetID() sourcePath := obj.GetPath() if len(id) > 0 { sourcePath = "" } destID := dir.GetID() destPath := dir.GetPath() if len(destID) > 0 { destPath = "" } dest := &sdkUserFile.File{ Path: destPath, Identity: destID, } _, err := d.sdkUserFileService.Copy(ctx, &sdkUserFile.BatchOperationRequest{ Source: []*sdkUserFile.File{ { Path: sourcePath, Identity: id, }, }, Dest: dest, }) return nil, err } func (d *HalalCloudOpen) remove(ctx context.Context, obj model.Obj) error { id := obj.GetID() _, err := d.sdkUserFileService.Delete(ctx, &sdkUserFile.BatchOperationRequest{ Source: []*sdkUserFile.File{ { Identity: id, Path: obj.GetPath(), }, }, }) return err } func (d *HalalCloudOpen) details(ctx context.Context) (*model.StorageDetails, error) { ret, err := d.sdkUserService.GetStatisticsAndQuota(ctx) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: ret.DiskStatisticsQuota.BytesQuota, UsedSpace: ret.DiskStatisticsQuota.BytesUsed, }, }, nil } ================================================ FILE: drivers/halalcloud_open/driver_get_link.go ================================================ package halalcloudopen import ( "context" "crypto/sha1" "io" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" "github.com/rclone/rclone/lib/readers" ) func (d *HalalCloudOpen) getLink(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if args.Redirect { // return nil, model.ErrUnsupported fid := file.GetID() fpath := file.GetPath() if fid != "" { fpath = "" } fi, err := d.sdkUserFileService.GetDirectDownloadAddress(ctx, &sdkUserFile.DirectDownloadRequest{ Identity: fid, Path: fpath, }) if err != nil { return nil, err } expireAt := fi.ExpireAt duration := time.Until(time.UnixMilli(expireAt)) return &model.Link{ URL: fi.DownloadAddress, Expiration: &duration, }, nil } result, err := d.sdkUserFileService.ParseFileSlice(ctx, &sdkUserFile.File{ Identity: file.GetID(), Path: file.GetPath(), }) if err != nil { return nil, err } fileAddrs := []*sdkUserFile.SliceDownloadInfo{} var addressDuration int64 nodesNumber := len(result.RawNodes) nodesIndex := nodesNumber - 1 startIndex, endIndex := 0, nodesIndex for nodesIndex >= 0 { if nodesIndex >= 200 { endIndex = 200 } else { endIndex = nodesNumber } for ; endIndex <= nodesNumber; endIndex += 200 { if endIndex == 0 { endIndex = 1 } sliceAddress, err := d.sdkUserFileService.GetSliceDownloadAddress(ctx, &sdkUserFile.SliceDownloadAddressRequest{ Identity: result.RawNodes[startIndex:endIndex], Version: 1, }) if err != nil { return nil, err } addressDuration, _ = strconv.ParseInt(sliceAddress.ExpireAt, 10, 64) fileAddrs = append(fileAddrs, sliceAddress.Addresses...) startIndex = endIndex nodesIndex -= 200 } } size, _ := strconv.ParseInt(result.FileSize, 10, 64) chunks := getChunkSizes(result.Sizes) resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { length := httpRange.Length if httpRange.Length < 0 || httpRange.Start+httpRange.Length >= size { length = size - httpRange.Start } oo := &openObject{ ctx: ctx, d: fileAddrs, chunk: []byte{}, chunks: chunks, skip: httpRange.Start, sha: result.Sha1, shaTemp: sha1.New(), } return readers.NewLimitedReadCloser(oo, length), nil } var duration time.Duration if addressDuration != 0 { duration = time.Until(time.UnixMilli(addressDuration)) } else { duration = time.Until(time.Now().Add(time.Hour)) } return &model.Link{ RangeReader: stream.RateLimitRangeReaderFunc(resultRangeReader), Expiration: &duration, }, nil } ================================================ FILE: drivers/halalcloud_open/driver_init.go ================================================ package halalcloudopen import ( "context" "time" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/halalcloud/golang-sdk-lite/halalcloud/apiclient" sdkUser "github.com/halalcloud/golang-sdk-lite/halalcloud/services/user" sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" ) func (d *HalalCloudOpen) Init(ctx context.Context) error { if d.uploadThread < 1 || d.uploadThread > 32 { d.uploadThread, d.UploadThread = 3, 3 } if d.halalCommon == nil { d.halalCommon = &halalCommon{ UserInfo: &sdkUser.User{}, refreshTokenFunc: func(token string) error { d.Addition.RefreshToken = token op.MustSaveDriverStorage(d) return nil }, } } if d.Addition.RefreshToken != "" { d.halalCommon.SetRefreshToken(d.Addition.RefreshToken) } timeout := d.Addition.TimeOut if timeout <= 0 { timeout = 60 } host := d.Addition.Host if host == "" { host = "openapi.2dland.cn" } client := apiclient.NewClient(nil, host, d.Addition.ClientID, d.Addition.ClientSecret, d.halalCommon, apiclient.WithTimeout(time.Second*time.Duration(timeout))) d.sdkClient = client d.sdkUserFileService = sdkUserFile.NewUserFileService(client) d.sdkUserService = sdkUser.NewUserService(client) userInfo, err := d.sdkUserService.Get(ctx, &sdkUser.User{}) if err != nil { return err } d.halalCommon.UserInfo = userInfo // 能够获取到用户信息，已经检查了 RefreshToken 的有效性，无需再次检查 return nil } ================================================ FILE: drivers/halalcloud_open/driver_interface.go ================================================ package halalcloudopen import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" ) func (d *HalalCloudOpen) Drop(ctx context.Context) error { return nil } func (d *HalalCloudOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { return d.getFiles(ctx, dir) } func (d *HalalCloudOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { return d.getLink(ctx, file, args) } func (d *HalalCloudOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { return d.makeDir(ctx, parentDir, dirName) } func (d *HalalCloudOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return d.move(ctx, srcObj, dstDir) } func (d *HalalCloudOpen) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { return d.rename(ctx, srcObj, newName) } func (d *HalalCloudOpen) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return d.copy(ctx, srcObj, dstDir) } func (d *HalalCloudOpen) Remove(ctx context.Context, obj model.Obj) error { return d.remove(ctx, obj) } func (d *HalalCloudOpen) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { return d.put(ctx, dstDir, stream, up) } func (d *HalalCloudOpen) GetDetails(ctx context.Context) (*model.StorageDetails, error) { return d.details(ctx) } ================================================ FILE: drivers/halalcloud_open/halalcloud_upload.go ================================================ package halalcloudopen import ( "bytes" "context" "encoding/json" "fmt" "io" "net/http" "net/url" "path" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" "github.com/ipfs/go-cid" log "github.com/sirupsen/logrus" ) func (d *HalalCloudOpen) put(ctx context.Context, dstDir model.Obj, fileStream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { newPath := path.Join(dstDir.GetPath(), fileStream.GetName()) uploadTask, err := d.sdkUserFileService.CreateUploadTask(ctx, &sdkUserFile.File{ Path: newPath, Size: fileStream.GetSize(), }) if err != nil { return nil, err } if uploadTask.Created { return nil, nil } slicesList := make([]string, 0) codec := uint64(0x55) if uploadTask.BlockCodec > 0 { codec = uint64(uploadTask.BlockCodec) } blockHashType := uploadTask.BlockHashType mhType := uint64(0x12) if blockHashType > 0 { mhType = uint64(blockHashType) } prefix := cid.Prefix{ Codec: codec, MhLength: -1, MhType: mhType, Version: 1, } blockSize := uploadTask.BlockSize // // Not sure whether FileStream supports concurrent read and write operations, so currently using single-threaded upload to ensure safety. // read file bufferSize := int(blockSize) buffer := make([]byte, bufferSize) offset := 0 teeReader := io.TeeReader(fileStream, driver.NewProgress(fileStream.GetSize(), up)) for { n, err := teeReader.Read(buffer[offset:]) // 这里 len(buf[offset:]) <= 4MB if n > 0 { offset += n if offset == int(blockSize) { uploadCid, err := postFileSlice(ctx, buffer, uploadTask.Task, uploadTask.UploadAddress, prefix, retryTimes) if err != nil { return nil, err } slicesList = append(slicesList, uploadCid.String()) offset = 0 } } if err != nil { if err == io.EOF { if offset > 0 { uploadCid, err := postFileSlice(ctx, buffer[:offset], uploadTask.Task, uploadTask.UploadAddress, prefix, retryTimes) if err != nil { return nil, err } slicesList = append(slicesList, uploadCid.String()) } break } return nil, err } } newFile, err := makeFile(ctx, slicesList, uploadTask.Task, uploadTask.UploadAddress, retryTimes) if err != nil { return nil, err } return NewObjFile(newFile), nil } func makeFile(ctx context.Context, fileSlice []string, taskID string, uploadAddress string, retry int) (*sdkUserFile.File, error) { var lastError error = nil for range retry { newFile, err := doMakeFile(fileSlice, taskID, uploadAddress) if err == nil { return newFile, nil } if ctx.Err() != nil { return nil, err } log.Errorf("make file slice failed, retrying... error: %s", err.Error()) if strings.Contains(err.Error(), "not found") { return nil, err } lastError = err time.Sleep(slicePostErrorRetryInterval) } return nil, fmt.Errorf("mk file slice failed after %d times, error: %s", retry, lastError.Error()) } func doMakeFile(fileSlice []string, taskID string, uploadAddress string) (*sdkUserFile.File, error) { accessUrl := uploadAddress + "/" + taskID getTimeOut := time.Minute * 2 u, err := url.Parse(accessUrl) if err != nil { return nil, err } n, _ := json.Marshal(fileSlice) httpRequest := http.Request{ Method: http.MethodPost, URL: u, Header: map[string][]string{ "Accept": {"application/json"}, "Content-Type": {"application/json"}, //"Content-Length": {strconv.Itoa(len(n))}, }, Body: io.NopCloser(bytes.NewReader(n)), } httpClient := http.Client{ Timeout: getTimeOut, } httpResponse, err := httpClient.Do(&httpRequest) if err != nil { return nil, err } defer httpResponse.Body.Close() if httpResponse.StatusCode != http.StatusOK && httpResponse.StatusCode != http.StatusCreated { b, _ := io.ReadAll(httpResponse.Body) message := string(b) log.Errorf("make file failed, status code: %d, message: %s", httpResponse.StatusCode, message) return nil, fmt.Errorf("mk file slice failed, status code: %d, message: %s", httpResponse.StatusCode, message) } b, _ := io.ReadAll(httpResponse.Body) var result *UploadedFile err = json.Unmarshal(b, &result) if err != nil { log.Errorf("make file failed from response, status code: %d, message: %s", httpResponse.StatusCode, string(b)) return nil, err } return &sdkUserFile.File{ Identity: result.Identity, Path: result.Path, Size: result.Size, ContentIdentity: result.ContentIdentity, }, nil } func postFileSlice(ctx context.Context, fileSlice []byte, taskID string, uploadAddress string, preix cid.Prefix, retry int) (cid.Cid, error) { var lastError error = nil for range retry { newCid, err := doPostFileSlice(fileSlice, taskID, uploadAddress, preix) if err == nil { return newCid, nil } if ctx.Err() != nil { return cid.Undef, err } time.Sleep(slicePostErrorRetryInterval) lastError = err } return cid.Undef, fmt.Errorf("upload file slice failed after %d times, error: %s", retry, lastError.Error()) } func doPostFileSlice(fileSlice []byte, taskID string, uploadAddress string, preix cid.Prefix) (cid.Cid, error) { // 1. sum file slice newCid, err := preix.Sum(fileSlice) if err != nil { return cid.Undef, err } // 2. post file slice sliceCidString := newCid.String() // /{taskID}/{sliceID} accessUrl := uploadAddress + "/" + taskID + "/" + sliceCidString getTimeOut := time.Second * 30 // get {accessUrl} in {getTimeOut} u, err := url.Parse(accessUrl) if err != nil { return cid.Undef, err } // header: accept: application/json // header: content-type: application/octet-stream // header: content-length: {fileSlice.length} // header: x-content-cid: {sliceCidString} // header: x-task-id: {taskID} httpRequest := http.Request{ Method: http.MethodGet, URL: u, Header: map[string][]string{ "Accept": {"application/json"}, }, } httpClient := http.Client{ Timeout: getTimeOut, } httpResponse, err := httpClient.Do(&httpRequest) if err != nil { log.Errorf("access %s failed, method: %s", accessUrl, http.MethodGet) return cid.Undef, err } if httpResponse.StatusCode != http.StatusOK { log.Errorf("access %s failed, method: %s, status code: %d", accessUrl, http.MethodGet, httpResponse.StatusCode) return cid.Undef, fmt.Errorf("upload file slice failed, status code: %d", httpResponse.StatusCode) } var result bool b, err := io.ReadAll(httpResponse.Body) if err != nil { return cid.Undef, err } err = json.Unmarshal(b, &result) if err != nil { return cid.Undef, err } if result { return newCid, nil } httpRequest = http.Request{ Method: http.MethodPost, URL: u, Header: map[string][]string{ "Accept": {"application/json"}, "Content-Type": {"application/octet-stream"}, // "Content-Length": {strconv.Itoa(len(fileSlice))}, }, Body: io.NopCloser(bytes.NewReader(fileSlice)), } httpResponse, err = httpClient.Do(&httpRequest) if err != nil { return cid.Undef, err } defer httpResponse.Body.Close() if httpResponse.StatusCode != http.StatusOK && httpResponse.StatusCode != http.StatusCreated { b, _ := io.ReadAll(httpResponse.Body) message := string(b) log.Errorf("upload file slice failed, status code: %d, message: %s", httpResponse.StatusCode, message) return cid.Undef, fmt.Errorf("upload file slice failed, status code: %d, message: %s", httpResponse.StatusCode, message) } // return newCid, nil } ================================================ FILE: drivers/halalcloud_open/meta.go ================================================ package halalcloudopen import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { // Usually one of two driver.RootPath // define other RefreshToken string `json:"refresh_token" required:"false" help:"If using a personal API approach, the RefreshToken is not required."` UploadThread int `json:"upload_thread" type:"number" default:"3" help:"1 <= thread <= 32"` ClientID string `json:"client_id" required:"true" default:""` ClientSecret string `json:"client_secret" required:"true" default:""` Host string `json:"host" required:"false" default:"openapi.2dland.cn"` TimeOut int `json:"timeout" type:"number" default:"60" help:"timeout in seconds"` } var config = driver.Config{ Name: "HalalCloudOpen", OnlyProxy: false, DefaultRoot: "/", NoLinkURL: false, } func init() { op.RegisterDriver(func() driver.Driver { return &HalalCloudOpen{} }) } type UploadedFile struct { Identity string `json:"identity"` UserIdentity string `json:"user_identity"` Path string `json:"path"` Size int64 `json:"size"` ContentIdentity string `json:"content_identity"` } ================================================ FILE: drivers/halalcloud_open/obj_file.go ================================================ package halalcloudopen import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" ) type ObjFile struct { sdkFile *sdkUserFile.File fileSize int64 modTime time.Time createTime time.Time } func NewObjFile(f *sdkUserFile.File) model.Obj { ofile := &ObjFile{sdkFile: f} ofile.fileSize = f.Size modTimeTs := f.UpdateTs ofile.modTime = time.UnixMilli(modTimeTs) createTimeTs := f.CreateTs ofile.createTime = time.UnixMilli(createTimeTs) return ofile } func (f *ObjFile) GetSize() int64 { return f.fileSize } func (f *ObjFile) GetName() string { return f.sdkFile.Name } func (f *ObjFile) ModTime() time.Time { return f.modTime } func (f *ObjFile) IsDir() bool { return f.sdkFile.Dir } func (f *ObjFile) GetHash() utils.HashInfo { return utils.HashInfo{ // TODO: support more hash types } } func (f *ObjFile) GetID() string { return f.sdkFile.Identity } func (f *ObjFile) GetPath() string { return f.sdkFile.Path } func (f *ObjFile) CreateTime() time.Time { return f.createTime } ================================================ FILE: drivers/halalcloud_open/utils.go ================================================ package halalcloudopen import ( "context" "crypto/md5" "encoding/hex" "errors" "fmt" "hash" "io" "net/http" "sync" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" "github.com/ipfs/go-cid" ) // get the next chunk func (oo *openObject) getChunk(_ context.Context) (err error) { if oo.id >= len(oo.chunks) { return io.EOF } var chunk []byte err = utils.Retry(3, time.Second, func() (err error) { chunk, err = getRawFiles(oo.d[oo.id]) return err }) if err != nil { return err } oo.id++ oo.chunk = chunk return nil } // Read reads up to len(p) bytes into p. func (oo *openObject) Read(p []byte) (n int, err error) { oo.mu.Lock() defer oo.mu.Unlock() if oo.closed { return 0, fmt.Errorf("read on closed file") } // Skip data at the start if requested for oo.skip > 0 { //size := 1024 * 1024 _, size, err := oo.ChunkLocation(oo.id) if err != nil { return 0, err } if oo.skip < int64(size) { break } oo.id++ oo.skip -= int64(size) } if len(oo.chunk) == 0 { err = oo.getChunk(oo.ctx) if err != nil { return 0, err } if oo.skip > 0 { oo.chunk = (oo.chunk)[oo.skip:] oo.skip = 0 } } n = copy(p, oo.chunk) oo.shaTemp.Write(p[:n]) oo.chunk = (oo.chunk)[n:] return n, nil } // Close closed the file - MAC errors are reported here func (oo *openObject) Close() (err error) { oo.mu.Lock() defer oo.mu.Unlock() if oo.closed { return nil } // 校验Sha1 if string(oo.shaTemp.Sum(nil)) != oo.sha { return fmt.Errorf("failed to finish download: SHA mismatch") } oo.closed = true return nil } func GetMD5Hash(text string) string { tHash := md5.Sum([]byte(text)) return hex.EncodeToString(tHash[:]) } type chunkSize struct { position int64 size int } type openObject struct { ctx context.Context mu sync.Mutex d []*sdkUserFile.SliceDownloadInfo id int skip int64 chunk []byte chunks []chunkSize closed bool sha string shaTemp hash.Hash } func getChunkSizes(sliceSize []*sdkUserFile.SliceSize) (chunks []chunkSize) { chunks = make([]chunkSize, 0) for _, s := range sliceSize { // 对最后一个做特殊处理 endIndex := s.EndIndex startIndex := s.StartIndex if endIndex == 0 { endIndex = startIndex } for j := startIndex; j <= endIndex; j++ { size := s.Size chunks = append(chunks, chunkSize{position: j, size: int(size)}) } } return chunks } func (oo *openObject) ChunkLocation(id int) (position int64, size int, err error) { if id < 0 || id >= len(oo.chunks) { return 0, 0, errors.New("invalid arguments") } return (oo.chunks)[id].position, (oo.chunks)[id].size, nil } func getRawFiles(addr *sdkUserFile.SliceDownloadInfo) ([]byte, error) { if addr == nil { return nil, errors.New("addr is nil") } client := http.Client{ Timeout: time.Duration(60 * time.Second), // Set timeout to 60 seconds } resp, err := client.Get(addr.DownloadAddress) if err != nil { return nil, err } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("bad status: %s, body: %s", resp.Status, body) } if addr.Encrypt > 0 { cd := uint8(addr.Encrypt) for idx := 0; idx < len(body); idx++ { body[idx] = body[idx] ^ cd } } storeType := addr.StoreType if storeType != 10 { sourceCid, err := cid.Decode(addr.Identity) if err != nil { return nil, err } checkCid, err := sourceCid.Prefix().Sum(body) if err != nil { return nil, err } if !checkCid.Equals(sourceCid) { return nil, fmt.Errorf("bad cid: %s, body: %s", checkCid.String(), body) } } return body, nil } ================================================ FILE: drivers/ilanzou/driver.go ================================================ package template import ( "context" "encoding/base64" "encoding/hex" "fmt" "io" "net/http" "net/url" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/foxxorcat/mopan-sdk-go" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) type ILanZou struct { model.Storage Addition userID string account string upClient *resty.Client conf Conf config driver.Config } func (d *ILanZou) Config() driver.Config { return d.config } func (d *ILanZou) GetAddition() driver.Additional { return &d.Addition } func (d *ILanZou) Init(ctx context.Context) error { d.upClient = base.NewRestyClient().SetTimeout(time.Minute * 10) if d.UUID == "" { res, err := d.unproved("/getUuid", http.MethodGet, nil) if err != nil { return err } d.UUID = utils.Json.Get(res, "uuid").ToString() } res, err := d.proved("/user/account/map", http.MethodGet, nil) if err != nil { return err } d.userID = utils.Json.Get(res, "map", "userId").ToString() d.account = utils.Json.Get(res, "map", "account").ToString() log.Debugf("[ilanzou] init response: %s", res) return nil } func (d *ILanZou) Drop(ctx context.Context) error { return nil } func (d *ILanZou) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { offset := 1 var res []ListItem for { var resp ListResp _, err := d.proved("/record/file/list", http.MethodGet, func(req *resty.Request) { params := []string{ "offset=" + strconv.Itoa(offset), "limit=60", "folderId=" + dir.GetID(), "type=0", } queryString := strings.Join(params, "&") req.SetQueryString(queryString).SetResult(&resp) }) if err != nil { return nil, err } res = append(res, resp.List...) if resp.Offset < resp.TotalPage { offset++ } else { break } } return utils.SliceConvert(res, func(f ListItem) (model.Obj, error) { updTime, err := time.ParseInLocation("2006-01-02 15:04:05", f.UpdTime, time.Local) if err != nil { return nil, err } obj := model.Object{ ID: strconv.FormatInt(f.FileId, 10), // Path: "", Name: f.FileName, Size: f.FileSize * 1024, Modified: updTime, Ctime: updTime, IsFolder: false, // HashInfo: utils.HashInfo{}, } if f.FileType == 2 { obj.IsFolder = true obj.Size = 0 obj.ID = strconv.FormatInt(f.FolderId, 10) obj.Name = f.FolderName } return &obj, nil }) } func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { u, err := url.Parse(d.conf.base + "/" + d.conf.unproved + "/file/redirect") if err != nil { return nil, err } ts, ts_str, _ := getTimestamp(d.conf.secret) params := []string{ "uuid=" + url.QueryEscape(d.UUID), "devType=6", "devCode=" + url.QueryEscape(d.UUID), "devModel=chrome", "devVersion=" + url.QueryEscape(d.conf.devVersion), "appVersion=", "timestamp=" + ts_str, "appToken=" + url.QueryEscape(d.Token), "enable=0", } downloadId, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%s", file.GetID(), d.userID)), d.conf.secret) if err != nil { return nil, err } params = append(params, "downloadId="+url.QueryEscape(hex.EncodeToString(downloadId))) auth, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%d", file.GetID(), ts)), d.conf.secret) if err != nil { return nil, err } params = append(params, "auth="+url.QueryEscape(hex.EncodeToString(auth))) u.RawQuery = strings.Join(params, "&") realURL := u.String() // get the url after redirect req := base.NoRedirectClient.R() req.SetHeaders(map[string]string{ "Referer": d.conf.site + "/", }) if d.Addition.Ip != "" { req.SetHeader("X-Forwarded-For", d.Addition.Ip) } res, err := req.Get(realURL) if err != nil { return nil, err } if res.StatusCode() == 302 { realURL = res.Header().Get("location") } else { return nil, fmt.Errorf("redirect failed, status: %d, msg: %s", res.StatusCode(), utils.Json.Get(res.Body(), "msg").ToString()) } link := model.Link{URL: realURL} return &link, nil } func (d *ILanZou) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { res, err := d.proved("/file/folder/save", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "folderDesc": "", "folderId": parentDir.GetID(), "folderName": dirName, }) }) if err != nil { return nil, err } return &model.Object{ ID: utils.Json.Get(res, "list", 0, "id").ToString(), // Path: "", Name: dirName, Size: 0, Modified: time.Now(), Ctime: time.Now(), IsFolder: true, // HashInfo: utils.HashInfo{}, }, nil } func (d *ILanZou) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { var fileIds, folderIds []string if srcObj.IsDir() { folderIds = []string{srcObj.GetID()} } else { fileIds = []string{srcObj.GetID()} } _, err := d.proved("/file/folder/move", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "folderIds": strings.Join(folderIds, ","), "fileIds": strings.Join(fileIds, ","), "targetId": dstDir.GetID(), }) }) if err != nil { return nil, err } return srcObj, nil } func (d *ILanZou) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { var err error if srcObj.IsDir() { _, err = d.proved("/file/folder/edit", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "folderDesc": "", "folderId": srcObj.GetID(), "folderName": newName, }) }) } else { _, err = d.proved("/file/edit", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "fileDesc": "", "fileId": srcObj.GetID(), "fileName": newName, }) }) } if err != nil { return nil, err } return &model.Object{ ID: srcObj.GetID(), // Path: "", Name: newName, Size: srcObj.GetSize(), Modified: time.Now(), Ctime: srcObj.CreateTime(), IsFolder: srcObj.IsDir(), }, nil } func (d *ILanZou) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { // TODO copy obj, optional return nil, errs.NotImplement } func (d *ILanZou) Remove(ctx context.Context, obj model.Obj) error { var fileIds, folderIds []string if obj.IsDir() { folderIds = []string{obj.GetID()} } else { fileIds = []string{obj.GetID()} } _, err := d.proved("/file/delete", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "folderIds": strings.Join(folderIds, ","), "fileIds": strings.Join(fileIds, ","), "status": 0, }) }) return err } const DefaultPartSize = 1024 * 1024 * 8 func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { etag := s.GetHash().GetHash(utils.MD5) var err error if len(etag) != utils.MD5.Width { _, etag, err = stream.CacheFullAndHash(s, &up, utils.MD5) if err != nil { return nil, err } } // get upToken res, err := d.proved("/7n/getUpToken", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "fileId": "", "fileName": s.GetName(), "fileSize": s.GetSize()/1024 + 1, "folderId": dstDir.GetID(), "md5": etag, "type": 1, }) }) if err != nil { return nil, err } upToken := utils.Json.Get(res, "upToken").ToString() if upToken == "-1" { // 支持秒传 var resp UploadTokenRapidResp err := utils.Json.Unmarshal(res, &resp) if err != nil { return nil, err } return &model.Object{ ID: strconv.FormatInt(resp.Map.FileID, 10), Name: resp.Map.FileName, Size: s.GetSize(), Modified: s.ModTime(), Ctime: s.CreateTime(), IsFolder: false, HashInfo: utils.NewHashInfo(utils.MD5, etag), }, nil } now := time.Now() key := fmt.Sprintf("disk/%d/%d/%d/%s/%016d", now.Year(), now.Month(), now.Day(), d.account, now.UnixMilli()) reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: &driver.SimpleReaderWithSize{ Reader: s, Size: s.GetSize(), }, UpdateProgress: up, }) var token string if s.GetSize() <= DefaultPartSize { res, err := d.upClient.R().SetContext(ctx).SetMultipartFormData(map[string]string{ "token": upToken, "key": key, "fname": s.GetName(), }).SetMultipartField("file", s.GetName(), s.GetMimetype(), reader). Post("https://upload.qiniup.com/") if err != nil { return nil, err } token = utils.Json.Get(res.Body(), "token").ToString() } else { keyBase64 := base64.URLEncoding.EncodeToString([]byte(key)) res, err := d.upClient.R().SetHeader("Authorization", "UpToken "+upToken).Post(fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads", d.conf.bucket, keyBase64)) if err != nil { return nil, err } uploadId := utils.Json.Get(res.Body(), "uploadId").ToString() parts := make([]Part, 0) partNum := (s.GetSize() + DefaultPartSize - 1) / DefaultPartSize for i := 1; i <= int(partNum); i++ { u := fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s/%d", d.conf.bucket, keyBase64, uploadId, i) res, err = d.upClient.R().SetContext(ctx).SetHeader("Authorization", "UpToken "+upToken).SetBody(io.LimitReader(reader, DefaultPartSize)).Put(u) if err != nil { return nil, err } etag := utils.Json.Get(res.Body(), "etag").ToString() parts = append(parts, Part{ PartNumber: i, ETag: etag, }) } res, err = d.upClient.R().SetHeader("Authorization", "UpToken "+upToken).SetBody(base.Json{ "fnmae": s.GetName(), "parts": parts, }).Post(fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s", d.conf.bucket, keyBase64, uploadId)) if err != nil { return nil, err } token = utils.Json.Get(res.Body(), "token").ToString() } // commit upload var resp UploadResultResp for i := 0; i < 10; i++ { _, err = d.unproved("/7n/results", http.MethodPost, func(req *resty.Request) { params := []string{ "tokenList=" + token, "tokenTime=" + time.Now().Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)"), } queryString := strings.Join(params, "&") req.SetQueryString(queryString).SetResult(&resp) }) if err != nil { return nil, err } if len(resp.List) == 0 { return nil, fmt.Errorf("upload failed, empty response") } if resp.List[0].Status == 1 { break } time.Sleep(time.Second * 1) } file := resp.List[0] if file.Status != 1 { return nil, fmt.Errorf("upload failed, status: %d", resp.List[0].Status) } return &model.Object{ ID: strconv.FormatInt(file.FileId, 10), // Path: , Name: file.FileName, Size: s.GetSize(), Modified: s.ModTime(), Ctime: s.CreateTime(), IsFolder: false, HashInfo: utils.NewHashInfo(utils.MD5, etag), }, nil } func (d *ILanZou) GetDetails(ctx context.Context) (*model.StorageDetails, error) { res, err := d.proved("/user/account/map", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }) if err != nil { return nil, err } vipSize := utils.Json.Get(res, "map", "vipSize").ToInt64() * 1024 totalSize := utils.Json.Get(res, "map", "totalSize").ToInt64() * 1024 rewardSize := utils.Json.Get(res, "map", "rewardSize").ToInt64() * 1024 total := totalSize + rewardSize + vipSize used := utils.Json.Get(res, "map", "usedSize").ToInt64() * 1024 return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: used, }, }, nil } //func (d *ILanZou) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*ILanZou)(nil) ================================================ FILE: drivers/ilanzou/meta.go ================================================ package template import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootID Username string `json:"username" type:"string" required:"true"` Password string `json:"password" type:"string" required:"true"` Ip string `json:"ip" type:"string"` Token string UUID string } type Conf struct { base string secret []byte bucket string unproved string proved string devVersion string site string } func init() { op.RegisterDriver(func() driver.Driver { return &ILanZou{ config: driver.Config{ Name: "ILanZou", DefaultRoot: "0", LocalSort: true, NoOverwriteUpload: true, }, conf: Conf{ base: "https://api.ilanzou.com", secret: []byte("lanZouY-disk-app"), bucket: "wpanstore-lanzou", unproved: "unproved", proved: "proved", devVersion: "125", site: "https://www.ilanzou.com", }, } }) op.RegisterDriver(func() driver.Driver { return &ILanZou{ config: driver.Config{ Name: "FeijiPan", DefaultRoot: "0", LocalSort: true, NoOverwriteUpload: true, }, conf: Conf{ base: "https://api.feijipan.com", secret: []byte("dingHao-disk-app"), bucket: "wpanstore", unproved: "ws", proved: "app", devVersion: "125", site: "https://www.feijipan.com", }, } }) } ================================================ FILE: drivers/ilanzou/types.go ================================================ package template type ListResp struct { Msg string `json:"msg"` Total int `json:"total"` Code int `json:"code"` Offset int `json:"offset"` TotalPage int `json:"totalPage"` Limit int `json:"limit"` List []ListItem `json:"list"` } type ListItem struct { IconId int `json:"iconId"` IsAmt int `json:"isAmt"` FolderDesc string `json:"folderDesc,omitempty"` AddTime string `json:"addTime"` FolderId int64 `json:"folderId"` ParentId int64 `json:"parentId"` ParentName string `json:"parentName"` NoteType int `json:"noteType,omitempty"` UpdTime string `json:"updTime"` IsShare int `json:"isShare"` FolderIcon string `json:"folderIcon,omitempty"` FolderName string `json:"folderName,omitempty"` FileType int `json:"fileType"` Status int `json:"status"` IsFileShare int `json:"isFileShare,omitempty"` FileName string `json:"fileName,omitempty"` FileStars float64 `json:"fileStars,omitempty"` IsFileDownload int `json:"isFileDownload,omitempty"` FileComments int `json:"fileComments,omitempty"` FileSize int64 `json:"fileSize,omitempty"` FileIcon string `json:"fileIcon,omitempty"` FileDownloads int `json:"fileDownloads,omitempty"` FileUrl interface{} `json:"fileUrl"` FileLikes int `json:"fileLikes,omitempty"` FileId int64 `json:"fileId,omitempty"` } type Part struct { PartNumber int `json:"partNumber"` ETag string `json:"etag"` } type UploadTokenRapidResp struct { Msg string `json:"msg"` Code int `json:"code"` UpToken string `json:"upToken"` Map struct { FileIconID int `json:"fileIconId"` FileName string `json:"fileName"` FileIcon string `json:"fileIcon"` FileID int64 `json:"fileId"` } `json:"map"` } type UploadResultResp struct { Msg string `json:"msg"` Code int `json:"code"` List []struct { FileIconId int `json:"fileIconId"` FileName string `json:"fileName"` FileIcon string `json:"fileIcon"` FileId int64 `json:"fileId"` Status int `json:"status"` Token string `json:"token"` } `json:"list"` } ================================================ FILE: drivers/ilanzou/util.go ================================================ package template import ( "encoding/hex" "fmt" "net/http" "net/url" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/foxxorcat/mopan-sdk-go" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) func (d *ILanZou) login() error { res, err := d.unproved("/login", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "loginName": d.Username, "loginPwd": d.Password, }) }) if err != nil { return err } d.Token = utils.Json.Get(res, "data", "appToken").ToString() if d.Token == "" { return fmt.Errorf("failed to login: token is empty, resp: %s", res) } return nil } func getTimestamp(secret []byte) (int64, string, error) { ts := time.Now().UnixMilli() tsStr := strconv.FormatInt(ts, 10) res, err := mopan.AesEncrypt([]byte(tsStr), secret) if err != nil { return 0, "", err } return ts, hex.EncodeToString(res), nil } func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, proved bool, retry ...bool) ([]byte, error) { _, ts_str, err := getTimestamp(d.conf.secret) if err != nil { return nil, err } params := []string{ "uuid=" + url.QueryEscape(d.UUID), "devType=6", "devCode=" + url.QueryEscape(d.UUID), "devModel=chrome", "devVersion=" + url.QueryEscape(d.conf.devVersion), "appVersion=", "timestamp=" + ts_str, } if proved { params = append(params, "appToken="+url.QueryEscape(d.Token)) } params = append(params, "extra=2") queryString := strings.Join(params, "&") req := base.RestyClient.R() req.SetHeaders(map[string]string{ "Origin": d.conf.site, "Referer": d.conf.site + "/", "Accept-Encoding": "gzip", "Accept-Language": "zh-CN,zh;q=0.9,en-US,en;q=0.8", }) if d.Addition.Ip != "" { req.SetHeader("X-Forwarded-For", d.Addition.Ip) } if callback != nil { callback(req) } res, err := req.Execute(method, d.conf.base+pathname+"?"+queryString) if err != nil { if res != nil { log.Errorf("[iLanZou] request error: %s", res.String()) } return nil, err } isRetry := len(retry) > 0 && retry[0] body := res.Body() code := utils.Json.Get(body, "code").ToInt() msg := utils.Json.Get(body, "msg").ToString() if code != 200 { if !isRetry && proved && (utils.SliceContains([]int{-1, -2}, code) || d.Token == "") { err = d.login() if err != nil { return nil, err } return d.request(pathname, method, callback, proved, true) } return nil, fmt.Errorf("%d: %s", code, msg) } return body, nil } func (d *ILanZou) unproved(pathname, method string, callback base.ReqCallback) ([]byte, error) { return d.request("/"+d.conf.unproved+pathname, method, callback, false) } func (d *ILanZou) proved(pathname, method string, callback base.ReqCallback) ([]byte, error) { return d.request("/"+d.conf.proved+pathname, method, callback, true) } ================================================ FILE: drivers/ipfs_api/driver.go ================================================ package ipfs import ( "context" "fmt" "net/url" "path" shell "github.com/ipfs/go-ipfs-api" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type IPFS struct { model.Storage Addition sh *shell.Shell gateURL *url.URL } func (d *IPFS) Config() driver.Config { return config } func (d *IPFS) GetAddition() driver.Additional { return &d.Addition } func (d *IPFS) Init(ctx context.Context) error { d.sh = shell.NewShell(d.Endpoint) gateURL, err := url.Parse(d.Gateway) if err != nil { return err } d.gateURL = gateURL return nil } func (d *IPFS) Drop(ctx context.Context) error { return nil } func (d *IPFS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var ipfsPath string cid := dir.GetID() if cid != "" { ipfsPath = path.Join("/ipfs", cid) } else { // 可能出现ipns dns解析失败的情况，需要重复获取cid，其他情况应该不会出错 ipfsPath = dir.GetPath() switch d.Mode { case "ipfs": ipfsPath = path.Join("/ipfs", ipfsPath) case "ipns": ipfsPath = path.Join("/ipns", ipfsPath) case "mfs": fileStat, err := d.sh.FilesStat(ctx, ipfsPath) if err != nil { return nil, err } ipfsPath = path.Join("/ipfs", fileStat.Hash) default: return nil, fmt.Errorf("mode error") } } dirs, err := d.sh.List(ipfsPath) if err != nil { return nil, err } objlist := []model.Obj{} for _, file := range dirs { objlist = append(objlist, &model.Object{ID: file.Hash, Name: file.Name, Size: int64(file.Size), IsFolder: file.Type == 1}) } return objlist, nil } func (d *IPFS) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { gateurl := d.gateURL.JoinPath("/ipfs/", file.GetID()) gateurl.RawQuery = "filename=" + url.QueryEscape(file.GetName()) return &model.Link{URL: gateurl.String()}, nil } func (d *IPFS) Get(ctx context.Context, rawPath string) (model.Obj, error) { rawPath = path.Join(d.GetRootPath(), rawPath) var ipfsPath string switch d.Mode { case "ipfs": ipfsPath = path.Join("/ipfs", rawPath) case "ipns": ipfsPath = path.Join("/ipns", rawPath) case "mfs": fileStat, err := d.sh.FilesStat(ctx, rawPath) if err != nil { return nil, err } ipfsPath = path.Join("/ipfs", fileStat.Hash) default: return nil, fmt.Errorf("mode error") } file, err := d.sh.FilesStat(ctx, ipfsPath) if err != nil { return nil, err } return &model.Object{ID: file.Hash, Name: path.Base(rawPath), Path: rawPath, Size: int64(file.Size), IsFolder: file.Type == "directory"}, nil } func (d *IPFS) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if d.Mode != "mfs" { return nil, fmt.Errorf("only write in mfs mode") } dirPath := parentDir.GetPath() err := d.sh.FilesMkdir(ctx, path.Join(dirPath, dirName), shell.FilesMkdir.Parents(true)) if err != nil { return nil, err } file, err := d.sh.FilesStat(ctx, path.Join(dirPath, dirName)) if err != nil { return nil, err } return &model.Object{ID: file.Hash, Name: dirName, Path: path.Join(dirPath, dirName), Size: int64(file.Size), IsFolder: true}, nil } func (d *IPFS) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if d.Mode != "mfs" { return nil, fmt.Errorf("only write in mfs mode") } dstPath := path.Join(dstDir.GetPath(), path.Base(srcObj.GetPath())) d.sh.FilesRm(ctx, dstPath, true) return &model.Object{ID: srcObj.GetID(), Name: srcObj.GetName(), Path: dstPath, Size: int64(srcObj.GetSize()), IsFolder: srcObj.IsDir()}, d.sh.FilesMv(ctx, srcObj.GetPath(), dstDir.GetPath()) } func (d *IPFS) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { if d.Mode != "mfs" { return nil, fmt.Errorf("only write in mfs mode") } dstPath := path.Join(path.Dir(srcObj.GetPath()), newName) d.sh.FilesRm(ctx, dstPath, true) return &model.Object{ID: srcObj.GetID(), Name: newName, Path: dstPath, Size: int64(srcObj.GetSize()), IsFolder: srcObj.IsDir()}, d.sh.FilesMv(ctx, srcObj.GetPath(), dstPath) } func (d *IPFS) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if d.Mode != "mfs" { return nil, fmt.Errorf("only write in mfs mode") } dstPath := path.Join(dstDir.GetPath(), path.Base(srcObj.GetPath())) d.sh.FilesRm(ctx, dstPath, true) return &model.Object{ID: srcObj.GetID(), Name: srcObj.GetName(), Path: dstPath, Size: int64(srcObj.GetSize()), IsFolder: srcObj.IsDir()}, d.sh.FilesCp(ctx, path.Join("/ipfs/", srcObj.GetID()), dstPath, shell.FilesCp.Parents(true)) } func (d *IPFS) Remove(ctx context.Context, obj model.Obj) error { if d.Mode != "mfs" { return fmt.Errorf("only write in mfs mode") } return d.sh.FilesRm(ctx, obj.GetPath(), true) } func (d *IPFS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { if d.Mode != "mfs" { return nil, fmt.Errorf("only write in mfs mode") } outHash, err := d.sh.Add(driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, })) if err != nil { return nil, err } dstPath := path.Join(dstDir.GetPath(), s.GetName()) if s.GetExist() != nil { d.sh.FilesRm(ctx, dstPath, true) } err = d.sh.FilesCp(ctx, path.Join("/ipfs/", outHash), dstPath, shell.FilesCp.Parents(true)) gateurl := d.gateURL.JoinPath("/ipfs/", outHash) gateurl.RawQuery = "filename=" + url.QueryEscape(s.GetName()) return &model.Object{ID: outHash, Name: s.GetName(), Path: dstPath, Size: int64(s.GetSize()), IsFolder: s.IsDir()}, err } //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*IPFS)(nil) ================================================ FILE: drivers/ipfs_api/meta.go ================================================ package ipfs import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { // Usually one of two driver.RootPath Mode string `json:"mode" options:"ipfs,ipns,mfs" type:"select" required:"true"` Endpoint string `json:"endpoint" default:"http://127.0.0.1:5001" required:"true"` Gateway string `json:"gateway" default:"http://127.0.0.1:8080" required:"true"` } var config = driver.Config{ Name: "IPFS API", DefaultRoot: "/", LocalSort: true, } func init() { op.RegisterDriver(func() driver.Driver { return &IPFS{} }) } ================================================ FILE: drivers/kodbox/driver.go ================================================ package kodbox import ( "context" "fmt" "net/http" "path/filepath" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) type KodBox struct { model.Storage Addition authorization string } func (d *KodBox) Config() driver.Config { return config } func (d *KodBox) GetAddition() driver.Additional { return &d.Addition } func (d *KodBox) Init(ctx context.Context) error { d.Address = strings.TrimSuffix(d.Address, "/") d.RootFolderPath = strings.TrimPrefix(utils.FixAndCleanPath(d.RootFolderPath), "/") return d.getToken() } func (d *KodBox) Drop(ctx context.Context) error { return nil } func (d *KodBox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var ( resp *CommonResp listPathData *ListPathData ) _, err := d.request(http.MethodPost, "/?explorer/list/path", func(req *resty.Request) { req.SetResult(&resp).SetFormData(map[string]string{ "path": dir.GetPath(), }) }, true) if err != nil { return nil, err } dataBytes, err := utils.Json.Marshal(resp.Data) if err != nil { return nil, err } err = utils.Json.Unmarshal(dataBytes, &listPathData) if err != nil { return nil, err } FolderAndFiles := append(listPathData.FolderList, listPathData.FileList...) return utils.SliceConvert(FolderAndFiles, func(f FolderOrFile) (model.Obj, error) { return &model.ObjThumb{ Object: model.Object{ Path: f.Path, Name: f.Name, Ctime: time.Unix(f.CreateTime, 0), Modified: time.Unix(f.ModifyTime, 0), Size: f.Size, IsFolder: f.Type == "folder", }, //Thumbnail: model.Thumbnail{}, }, nil }) } func (d *KodBox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { path := file.GetPath() return &model.Link{ URL: fmt.Sprintf("%s/?explorer/index/fileOut&path=%s&download=1&accessToken=%s", d.Address, path, d.authorization)}, nil } func (d *KodBox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { var resp *CommonResp newDirPath := filepath.Join(parentDir.GetPath(), dirName) _, err := d.request(http.MethodPost, "/?explorer/index/mkdir", func(req *resty.Request) { req.SetResult(&resp).SetFormData(map[string]string{ "path": newDirPath, }) }) if err != nil { return nil, err } code := resp.Code.(bool) if !code { return nil, fmt.Errorf("%s", resp.Data) } return &model.ObjThumb{ Object: model.Object{ Path: resp.Info.(string), Name: dirName, IsFolder: true, Modified: time.Now(), Ctime: time.Now(), }, }, nil } func (d *KodBox) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { var resp *CommonResp _, err := d.request(http.MethodPost, "/?explorer/index/pathCuteTo", func(req *resty.Request) { req.SetResult(&resp).SetFormData(map[string]string{ "dataArr": fmt.Sprintf("[{\"path\": \"%s\", \"name\": \"%s\"}]", srcObj.GetPath(), srcObj.GetName()), "path": dstDir.GetPath(), }) }, true) if err != nil { return nil, err } code := resp.Code.(bool) if !code { return nil, fmt.Errorf("%s", resp.Data) } return &model.ObjThumb{ Object: model.Object{ Path: srcObj.GetPath(), Name: srcObj.GetName(), IsFolder: srcObj.IsDir(), Modified: srcObj.ModTime(), Ctime: srcObj.CreateTime(), }, }, nil } func (d *KodBox) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { var resp *CommonResp _, err := d.request(http.MethodPost, "/?explorer/index/pathRename", func(req *resty.Request) { req.SetResult(&resp).SetFormData(map[string]string{ "path": srcObj.GetPath(), "newName": newName, }) }, true) if err != nil { return nil, err } code := resp.Code.(bool) if !code { return nil, fmt.Errorf("%s", resp.Data) } return &model.ObjThumb{ Object: model.Object{ Path: srcObj.GetPath(), Name: newName, IsFolder: srcObj.IsDir(), Modified: time.Now(), Ctime: srcObj.CreateTime(), }, }, nil } func (d *KodBox) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { var resp *CommonResp _, err := d.request(http.MethodPost, "/?explorer/index/pathCopyTo", func(req *resty.Request) { req.SetResult(&resp).SetFormData(map[string]string{ "dataArr": fmt.Sprintf("[{\"path\": \"%s\", \"name\": \"%s\"}]", srcObj.GetPath(), srcObj.GetName()), "path": dstDir.GetPath(), }) }) if err != nil { return nil, err } code := resp.Code.(bool) if !code { return nil, fmt.Errorf("%s", resp.Data) } path := resp.Info.([]interface{})[0].(string) objectName, err := d.getFileOrFolderName(ctx, path) if err != nil { return nil, err } return &model.ObjThumb{ Object: model.Object{ Path: path, Name: *objectName, IsFolder: srcObj.IsDir(), Modified: time.Now(), Ctime: time.Now(), }, }, nil } func (d *KodBox) Remove(ctx context.Context, obj model.Obj) error { var resp *CommonResp _, err := d.request(http.MethodPost, "/?explorer/index/pathDelete", func(req *resty.Request) { req.SetResult(&resp).SetFormData(map[string]string{ "dataArr": fmt.Sprintf("[{\"path\": \"%s\", \"name\": \"%s\"}]", obj.GetPath(), obj.GetName()), "shiftDelete": "1", }) }) if err != nil { return err } code := resp.Code.(bool) if !code { return fmt.Errorf("%s", resp.Data) } return nil } func (d *KodBox) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { var resp *CommonResp _, err := d.request(http.MethodPost, "/?explorer/upload/fileUpload", func(req *resty.Request) { r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, }) req.SetFileReader("file", s.GetName(), r). SetResult(&resp). SetFormData(map[string]string{ "path": dstDir.GetPath(), }). SetContext(ctx) }) if err != nil { return nil, err } code := resp.Code.(bool) if !code { return nil, fmt.Errorf("%s", resp.Data) } return &model.ObjThumb{ Object: model.Object{ Path: resp.Info.(string), Name: s.GetName(), Size: s.GetSize(), IsFolder: false, Modified: time.Now(), Ctime: time.Now(), }, }, nil } func (d *KodBox) getFileOrFolderName(ctx context.Context, path string) (*string, error) { var resp *CommonResp _, err := d.request(http.MethodPost, "/?explorer/index/pathInfo", func(req *resty.Request) { req.SetResult(&resp).SetFormData(map[string]string{ "dataArr": fmt.Sprintf("[{\"path\": \"%s\"}]", path)}) }) if err != nil { return nil, err } code := resp.Code.(bool) if !code { return nil, fmt.Errorf("%s", resp.Data) } folderOrFileName := resp.Data.(map[string]any)["name"].(string) return &folderOrFileName, nil } var _ driver.Driver = (*KodBox)(nil) ================================================ FILE: drivers/kodbox/meta.go ================================================ package kodbox import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Address string `json:"address" required:"true"` UserName string `json:"username" required:"false"` Password string `json:"password" required:"false"` } var config = driver.Config{ Name: "KodBox", } func init() { op.RegisterDriver(func() driver.Driver { return &KodBox{} }) } ================================================ FILE: drivers/kodbox/types.go ================================================ package kodbox type CommonResp struct { Code any `json:"code"` TimeUse string `json:"timeUse"` TimeNow string `json:"timeNow"` Data any `json:"data"` Info any `json:"info"` } type ListPathData struct { FolderList []FolderOrFile `json:"folderList"` FileList []FolderOrFile `json:"fileList"` } type FolderOrFile struct { Name string `json:"name"` Path string `json:"path"` Type string `json:"type"` Ext string `json:"ext,omitempty"` // 文件特有字段 Size int64 `json:"size"` CreateTime int64 `json:"createTime"` ModifyTime int64 `json:"modifyTime"` } ================================================ FILE: drivers/kodbox/util.go ================================================ package kodbox import ( "fmt" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) func (d *KodBox) getToken() error { var authResp CommonResp res, err := base.RestyClient.R(). SetResult(&authResp). SetQueryParams(map[string]string{ "name": d.UserName, "password": d.Password, }). Post(d.Address + "/?user/index/loginSubmit") if err != nil { return err } if res.StatusCode() >= 400 { return fmt.Errorf("get token failed: %s", res.String()) } if res.StatusCode() == 200 && authResp.Code.(bool) == false { return fmt.Errorf("get token failed: %s", res.String()) } d.authorization = fmt.Sprintf("%s", authResp.Info) return nil } func (d *KodBox) request(method string, pathname string, callback base.ReqCallback, noRedirect ...bool) ([]byte, error) { full := pathname if !strings.HasPrefix(pathname, "http") { full = d.Address + pathname } req := base.RestyClient.R() if len(noRedirect) > 0 && noRedirect[0] { req = base.NoRedirectClient.R() } req.SetFormData(map[string]string{ "accessToken": d.authorization, }) callback(req) var ( res *resty.Response commonResp *CommonResp err error skip bool ) for i := 0; i < 2; i++ { if skip { break } res, err = req.Execute(method, full) if err != nil { return nil, err } err := utils.Json.Unmarshal(res.Body(), &commonResp) if err != nil { return nil, err } switch commonResp.Code.(type) { case bool: skip = true case string: if commonResp.Code.(string) == "10001" { err = d.getToken() if err != nil { return nil, err } req.SetFormData(map[string]string{"accessToken": d.authorization}) } } } if commonResp.Code.(bool) == false { return nil, fmt.Errorf("request failed: %s", commonResp.Data) } return res.Body(), nil } ================================================ FILE: drivers/lanzou/driver.go ================================================ package lanzou import ( "context" "net/http" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) type LanZou struct { Addition model.Storage uid string vei string flag int32 } func (d *LanZou) Config() driver.Config { return config } func (d *LanZou) GetAddition() driver.Additional { return &d.Addition } func (d *LanZou) Init(ctx context.Context) (err error) { if d.UserAgent == "" { d.UserAgent = base.UserAgentNT } switch d.Type { case "account": _, err := d.Login() if err != nil { return err } fallthrough case "cookie": if d.RootFolderID == "" { d.RootFolderID = "-1" } d.vei, d.uid, err = d.getVeiAndUid() } return } func (d *LanZou) Drop(ctx context.Context) error { d.uid = "" return nil } // 获取的大小和时间不准确 func (d *LanZou) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { if d.IsCookie() || d.IsAccount() { return d.GetAllFiles(dir.GetID()) } else { return d.GetFileOrFolderByShareUrl(dir.GetID(), d.SharePassword) } } func (d *LanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var ( err error dfile *FileOrFolderByShareUrl ) switch file := file.(type) { case *FileOrFolder: // 先获取分享链接 sfile := file.GetShareInfo() if sfile == nil { sfile, err = d.getFileShareUrlByID(file.GetID()) if err != nil { return nil, err } file.SetShareInfo(sfile) } // 然后获取下载链接 dfile, err = d.GetFilesByShareUrl(sfile.FID, sfile.Pwd) if err != nil { return nil, err } // 修复文件大小 if d.RepairFileInfo && !file.repairFlag { size, time := d.getFileRealInfo(dfile.Url) if size != nil { file.size = size file.repairFlag = true } if file.time != nil { file.time = time } } case *FileOrFolderByShareUrl: dfile, err = d.GetFilesByShareUrl(file.GetID(), file.Pwd) if err != nil { return nil, err } // 修复文件大小 if d.RepairFileInfo && !file.repairFlag { size, time := d.getFileRealInfo(dfile.Url) if size != nil { file.size = size file.repairFlag = true } if file.time != nil { file.time = time } } } exp := GetExpirationTime(dfile.Url) return &model.Link{ URL: dfile.Url, Header: http.Header{ "User-Agent": []string{base.UserAgent}, }, Expiration: &exp, }, nil } func (d *LanZou) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if d.IsCookie() || d.IsAccount() { data, err := d.doupload(func(req *resty.Request) { req.SetContext(ctx) req.SetFormData(map[string]string{ "task": "2", "parent_id": parentDir.GetID(), "folder_name": dirName, "folder_description": "", }) }, nil) if err != nil { return nil, err } return &FileOrFolder{ Name: dirName, FolID: utils.Json.Get(data, "text").ToString(), }, nil } return nil, errs.NotSupport } func (d *LanZou) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if d.IsCookie() || d.IsAccount() { if !srcObj.IsDir() { _, err := d.doupload(func(req *resty.Request) { req.SetContext(ctx) req.SetFormData(map[string]string{ "task": "20", "folder_id": dstDir.GetID(), "file_id": srcObj.GetID(), }) }, nil) if err != nil { return nil, err } return srcObj, nil } } return nil, errs.NotSupport } func (d *LanZou) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { if d.IsCookie() || d.IsAccount() { if !srcObj.IsDir() { _, err := d.doupload(func(req *resty.Request) { req.SetContext(ctx) req.SetFormData(map[string]string{ "task": "46", "file_id": srcObj.GetID(), "file_name": newName, "type": "2", }) }, nil) if err != nil { return nil, err } srcObj.(*FileOrFolder).NameAll = newName return srcObj, nil } } return nil, errs.NotSupport } func (d *LanZou) Remove(ctx context.Context, obj model.Obj) error { if d.IsCookie() || d.IsAccount() { _, err := d.doupload(func(req *resty.Request) { req.SetContext(ctx) if obj.IsDir() { req.SetFormData(map[string]string{ "task": "3", "folder_id": obj.GetID(), }) } else { req.SetFormData(map[string]string{ "task": "6", "file_id": obj.GetID(), }) } }, nil) return err } return errs.NotSupport } func (d *LanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { if d.IsCookie() || d.IsAccount() { var resp RespText[[]FileOrFolder] _, err := d._post(d.BaseUrl+"/html5up.php", func(req *resty.Request) { reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, }) req.SetFormData(map[string]string{ "task": "1", "vie": "2", "ve": "2", "id": "WU_FILE_0", "name": s.GetName(), "folder_id_bb_n": dstDir.GetID(), }).SetFileReader("upload_file", s.GetName(), reader).SetContext(ctx) }, &resp, true) if err != nil { return nil, err } return &resp.Text[0], nil } return nil, errs.NotSupport } ================================================ FILE: drivers/lanzou/help.go ================================================ package lanzou import ( "encoding/hex" "errors" "fmt" "net/http" "regexp" "strconv" "strings" "time" "unicode" ) const DAY time.Duration = 84600000000000 // 解析时间 var timeSplitReg = regexp.MustCompile("([0-9.]*)\\s*([\u4e00-\u9fa5]+)") // 如果解析失败,则返回当前时间 func MustParseTime(str string) time.Time { lastOpTime, err := time.ParseInLocation("2006-01-02 -07", str+" +08", time.Local) if err != nil { strs := timeSplitReg.FindStringSubmatch(str) lastOpTime = time.Now() if len(strs) == 3 { i, _ := strconv.ParseInt(strs[1], 10, 64) ti := time.Duration(-i) switch strs[2] { case "秒前": lastOpTime = lastOpTime.Add(time.Second * ti) case "分钟前": lastOpTime = lastOpTime.Add(time.Minute * ti) case "小时前": lastOpTime = lastOpTime.Add(time.Hour * ti) case "天前": lastOpTime = lastOpTime.Add(DAY * ti) case "昨天": lastOpTime = lastOpTime.Add(-DAY) case "前天": lastOpTime = lastOpTime.Add(-DAY * 2) } } } return lastOpTime } // 解析大小 var sizeSplitReg = regexp.MustCompile(`(?i)([0-9.]+)\s*([bkm]+)`) // 解析失败返回0 func SizeStrToInt64(size string) int64 { strs := sizeSplitReg.FindStringSubmatch(size) if len(strs) < 3 { return 0 } s, _ := strconv.ParseFloat(strs[1], 64) switch strings.ToUpper(strs[2]) { case "B": return int64(s) case "K": return int64(s * (1 << 10)) case "M": return int64(s * (1 << 20)) } return 0 } // 移除注释 func RemoveNotes(html string) string { return regexp.MustCompile(`|[^:]//.*|/\*.*?\*/`).ReplaceAllStringFunc(html, func(b string) string { if b[1:3] == "//" { return b[:1] } return "\n" }) } // 清理JS注释 func RemoveJSComment(data string) string { var result strings.Builder inComment := false inSingleLineComment := false for i := 0; i < len(data); i++ { v := data[i] if inSingleLineComment && (v == '\n' || v == '\r') { inSingleLineComment = false result.WriteByte(v) continue } if inComment && v == '*' && i+1 < len(data) && data[i+1] == '/' { inComment = false i++ continue } if inComment || inSingleLineComment { continue } if v == '/' && i+1 < len(data) { nextChar := data[i+1] if nextChar == '*' { inComment = true i++ continue } else if nextChar == '/' { inSingleLineComment = true i++ continue } } result.WriteByte(v) } return result.String() } var findAcwScV2Reg = regexp.MustCompile(`arg1='([0-9A-Z]+)'`) // 在页面被过多访问或其他情况下，有时候会先返回一个加密的页面，其执行计算出一个acw_sc__v2后放入页面后再重新访问页面才能获得正常页面 // 若该页面进行了js加密，则进行解密，计算acw_sc__v2，并加入cookie func CalcAcwScV2(htmlContent string) (string, error) { matches := findAcwScV2Reg.FindStringSubmatch(htmlContent) if len(matches) != 2 { return "", errors.New("无法匹配到 arg1 参数") } arg1 := matches[1] mask := "3000176000856006061501533003690027800375" result, err := hexXor(unbox(arg1), mask) if err != nil { return "", fmt.Errorf("hexXor 操作失败: %w", err) } return result, nil } func unbox(hex string) string { var box = []int{6, 28, 34, 31, 33, 18, 30, 23, 9, 8, 19, 38, 17, 24, 0, 5, 32, 21, 10, 22, 25, 14, 15, 3, 16, 27, 13, 35, 2, 29, 11, 26, 4, 36, 1, 39, 37, 7, 20, 12} var newBox = make([]byte, len(hex)) for i, j := range box { if len(newBox) > j { newBox[j] = hex[i] } } return string(newBox) } func hexXor(hex1, hex2 string) (string, error) { bytes1, err := hex.DecodeString(hex1) if err != nil { return "", fmt.Errorf("解码 hex1 失败: %w", err) } bytes2, err := hex.DecodeString(hex2) if err != nil { return "", fmt.Errorf("解码 hex2 失败: %w", err) } minLength := min(len(bytes2), len(bytes1)) resultBytes := make([]byte, minLength) for i := range minLength { resultBytes[i] = bytes1[i] ^ bytes2[i] } return hex.EncodeToString(resultBytes), nil } var findDataReg = regexp.MustCompile(`data[:\s]+({[^}]+})`) // 查找json var findKVReg = regexp.MustCompile(`'(.+?)':('?([^' },]*)'?)`) // 拆分kv // 根据key查询js变量 func findJSVarFunc(key, data string) string { var values []string if key != "sasign" { values = regexp.MustCompile(`var ` + key + `\s*=\s*['"]?(.+?)['"]?;`).FindStringSubmatch(data) } else { matches := regexp.MustCompile(`var `+key+`\s*=\s*['"]?(.+?)['"]?;`).FindAllStringSubmatch(data, -1) if len(matches) == 3 { values = matches[1] } else { if len(matches) > 0 { values = matches[0] } } } if len(values) == 0 { return "" } return values[1] } var findFunction = regexp.MustCompile(`(?ims)^function[^{]+`) var findFunctionAll = regexp.MustCompile(`(?is)function[^{]+`) // 查找所有方法位置 func findJSFunctionIndex(data string, all bool) [][2]int { findFunction := findFunction if all { findFunction = findFunctionAll } indexs := findFunction.FindAllStringIndex(data, -1) fIndexs := make([][2]int, 0, len(indexs)) for _, index := range indexs { if len(index) != 2 { continue } count, data := 0, data[index[1]:] for ii, v := range data { if v == ' ' && count == 0 { continue } if v == '{' { count++ } if v == '}' { count-- } if count == 0 { fIndexs = append(fIndexs, [2]int{index[0], index[1] + ii + 1}) break } } } return fIndexs } // 删除JS全局方法 func removeJSGlobalFunction(html string) string { indexs := findJSFunctionIndex(html, false) block := make([]string, len(indexs)) for i, next := len(indexs)-1, len(html); i >= 0; i-- { index := indexs[i] block[i] = html[index[1]:next] next = index[0] } return strings.Join(block, "") } // 根据名称获取方法 func getJSFunctionByName(html string, name string) (string, error) { indexs := findJSFunctionIndex(html, true) for _, index := range indexs { data := html[index[0]:index[1]] if regexp.MustCompile(`function\s+` + name + `[()\s]+{`).MatchString(data) { return data, nil } } return "", fmt.Errorf("not find %s function", name) } // 解析html中的JSON,选择最长的数据 func htmlJsonToMap2(html string) (map[string]string, error) { datas := findDataReg.FindAllStringSubmatch(html, -1) var sData string for _, data := range datas { if len(datas) > 0 && len(data[1]) > len(sData) { sData = data[1] } } if sData == "" { return nil, fmt.Errorf("not find data") } return jsonToMap(sData, html), nil } // 解析html中的JSON func htmlJsonToMap(html string) (map[string]string, error) { datas := findDataReg.FindStringSubmatch(html) if len(datas) != 2 { return nil, fmt.Errorf("not find data") } return jsonToMap(datas[1], html), nil } func jsonToMap(data, html string) map[string]string { var param = make(map[string]string) kvs := findKVReg.FindAllStringSubmatch(data, -1) for _, kv := range kvs { k, v := kv[1], kv[3] if v == "" || strings.Contains(kv[2], "'") || IsNumber(kv[2]) { param[k] = v } else { param[k] = findJSVarFunc(v, html) } } return param } func IsNumber(str string) bool { for _, s := range str { if !unicode.IsDigit(s) { return false } } return true } var findFromReg = regexp.MustCompile(`data : '(.+?)'`) // 查找from字符串 // 解析html中的form func htmlFormToMap(html string) (map[string]string, error) { forms := findFromReg.FindStringSubmatch(html) if len(forms) != 2 { return nil, fmt.Errorf("not find file sgin") } return formToMap(forms[1]), nil } func formToMap(from string) map[string]string { var param = make(map[string]string) for _, kv := range strings.Split(from, "&") { kv := strings.SplitN(kv, "=", 2)[:2] param[kv[0]] = kv[1] } return param } var regExpirationTime = regexp.MustCompile(`e=(\d+)`) func GetExpirationTime(url string) (etime time.Duration) { exps := regExpirationTime.FindStringSubmatch(url) if len(exps) < 2 { return } timestamp, err := strconv.ParseInt(exps[1], 10, 64) if err != nil { return } etime = time.Duration(timestamp-time.Now().Unix()) * time.Second return } func CookieToString(cookies []*http.Cookie) string { if cookies == nil { return "" } cookieStrings := make([]string, len(cookies)) for i, cookie := range cookies { cookieStrings[i] = cookie.Name + "=" + cookie.Value } return strings.Join(cookieStrings, ";") } ================================================ FILE: drivers/lanzou/meta.go ================================================ package lanzou import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { Type string `json:"type" type:"select" options:"account,cookie,url" default:"cookie"` Account string `json:"account"` Password string `json:"password"` Cookie string `json:"cookie" help:"about 15 days valid, ignore if shareUrl is used"` driver.RootID SharePassword string `json:"share_password"` BaseUrl string `json:"baseUrl" required:"true" default:"https://pc.woozooo.com" help:"basic URL for file operation"` ShareUrl string `json:"shareUrl" required:"true" default:"https://pan.lanzoui.com" help:"used to get the sharing page"` UserAgent string `json:"user_agent" required:"true" default:"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.39 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.39"` RepairFileInfo bool `json:"repair_file_info" help:"To use webdav, you need to enable it"` } func (a *Addition) IsCookie() bool { return a.Type == "cookie" } func (a *Addition) IsAccount() bool { return a.Type == "account" } var config = driver.Config{ Name: "Lanzou", LocalSort: true, DefaultRoot: "-1", } func init() { op.RegisterDriver(func() driver.Driver { return &LanZou{} }) } ================================================ FILE: drivers/lanzou/types.go ================================================ package lanzou import ( "errors" "fmt" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) var ErrFileShareCancel = errors.New("file sharing cancellation") var ErrFileNotExist = errors.New("file does not exist") var ErrCookieExpiration = errors.New("cookie expiration") type RespText[T any] struct { Text T `json:"text"` } type RespInfo[T any] struct { Info T `json:"info"` } var _ model.Obj = (*FileOrFolder)(nil) var _ model.Obj = (*FileOrFolderByShareUrl)(nil) type FileOrFolder struct { Name string `json:"name"` //Onof string `json:"onof"` // 是否存在提取码 //IsLock string `json:"is_lock"` //IsCopyright int `json:"is_copyright"` // 文件通用 ID string `json:"id"` NameAll string `json:"name_all"` Size string `json:"size"` Time string `json:"time"` //Icon string `json:"icon"` //Downs string `json:"downs"` //Filelock string `json:"filelock"` //IsBakdownload int `json:"is_bakdownload"` //Bakdownload string `json:"bakdownload"` //IsDes int `json:"is_des"` // 是否存在描述 //IsIco int `json:"is_ico"` // 文件夹 FolID string `json:"fol_id"` //Folderlock string `json:"folderlock"` //FolderDes string `json:"folder_des"` // 缓存字段 size *int64 `json:"-"` time *time.Time `json:"-"` repairFlag bool `json:"-"` shareInfo *FileShare `json:"-"` } func (f *FileOrFolder) CreateTime() time.Time { return f.ModTime() } func (f *FileOrFolder) GetHash() utils.HashInfo { return utils.HashInfo{} } func (f *FileOrFolder) GetID() string { if f.IsDir() { return f.FolID } return f.ID } func (f *FileOrFolder) GetName() string { if f.IsDir() { return f.Name } return f.NameAll } func (f *FileOrFolder) GetPath() string { return "" } func (f *FileOrFolder) GetSize() int64 { if f.size == nil { size := SizeStrToInt64(f.Size) f.size = &size } return *f.size } func (f *FileOrFolder) IsDir() bool { return f.FolID != "" } func (f *FileOrFolder) ModTime() time.Time { if f.time == nil { time := MustParseTime(f.Time) f.time = &time } return *f.time } func (f *FileOrFolder) SetShareInfo(fs *FileShare) { f.shareInfo = fs } func (f *FileOrFolder) GetShareInfo() *FileShare { return f.shareInfo } /* 通过ID获取文件/文件夹分享信息 */ type FileShare struct { Pwd string `json:"pwd"` Onof string `json:"onof"` Taoc string `json:"taoc"` IsNewd string `json:"is_newd"` // 文件 FID string `json:"f_id"` // 文件夹 NewUrl string `json:"new_url"` Name string `json:"name"` Des string `json:"des"` } /* 分享类型为文件夹 */ type FileOrFolderByShareUrlResp struct { Text []FileOrFolderByShareUrl `json:"text"` } type FileOrFolderByShareUrl struct { ID string `json:"id"` NameAll string `json:"name_all"` // 文件特有 Duan string `json:"duan"` Size string `json:"size"` Time string `json:"time"` //Icon string `json:"icon"` //PIco int `json:"p_ico"` //T int `json:"t"` // 文件夹特有 IsFloder bool `json:"-"` // Url string `json:"-"` Pwd string `json:"-"` // 缓存字段 size *int64 `json:"-"` time *time.Time `json:"-"` repairFlag bool `json:"-"` } func (f *FileOrFolderByShareUrl) CreateTime() time.Time { return f.ModTime() } func (f *FileOrFolderByShareUrl) GetHash() utils.HashInfo { return utils.HashInfo{} } func (f *FileOrFolderByShareUrl) GetID() string { return f.ID } func (f *FileOrFolderByShareUrl) GetName() string { return f.NameAll } func (f *FileOrFolderByShareUrl) GetPath() string { return "" } func (f *FileOrFolderByShareUrl) GetSize() int64 { if f.size == nil { size := SizeStrToInt64(f.Size) f.size = &size } return *f.size } func (f *FileOrFolderByShareUrl) IsDir() bool { return f.IsFloder } func (f *FileOrFolderByShareUrl) ModTime() time.Time { if f.time == nil { time := MustParseTime(f.Time) f.time = &time } return *f.time } // 获取下载链接的响应 type FileShareInfoAndUrlResp[T string | int] struct { Dom string `json:"dom"` URL string `json:"url"` Inf T `json:"inf"` } func (u *FileShareInfoAndUrlResp[T]) GetBaseUrl() string { return fmt.Sprint(u.Dom, "/file") } func (u *FileShareInfoAndUrlResp[T]) GetDownloadUrl() string { return fmt.Sprint(u.GetBaseUrl(), "/", u.URL) } ================================================ FILE: drivers/lanzou/util.go ================================================ package lanzou import ( "errors" "fmt" "io" "net/http" "regexp" "runtime" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) var upClient *resty.Client var once sync.Once func (d *LanZou) doupload(callback base.ReqCallback, resp interface{}) ([]byte, error) { return d.post(d.BaseUrl+"/doupload.php", func(req *resty.Request) { req.SetQueryParams(map[string]string{ "uid": d.uid, "vei": d.vei, }) if callback != nil { callback(req) } }, resp) } func (d *LanZou) get(url string, callback base.ReqCallback) ([]byte, error) { return d.request(url, http.MethodGet, callback, false) } func (d *LanZou) post(url string, callback base.ReqCallback, resp interface{}) ([]byte, error) { data, err := d._post(url, callback, resp, false) if err == ErrCookieExpiration && d.IsAccount() { if atomic.CompareAndSwapInt32(&d.flag, 0, 1) { _, err2 := d.Login() atomic.SwapInt32(&d.flag, 0) if err2 != nil { err = errors.Join(err, err2) d.Status = err.Error() op.MustSaveDriverStorage(d) return data, err } } for atomic.LoadInt32(&d.flag) != 0 { runtime.Gosched() } return d._post(url, callback, resp, false) } return data, err } func (d *LanZou) _post(url string, callback base.ReqCallback, resp interface{}, up bool) ([]byte, error) { data, err := d.request(url, http.MethodPost, func(req *resty.Request) { req.AddRetryCondition(func(r *resty.Response, err error) bool { if utils.Json.Get(r.Body(), "zt").ToInt() == 4 { time.Sleep(time.Second) return true } return false }) if callback != nil { callback(req) } }, up) if err != nil { return data, err } switch utils.Json.Get(data, "zt").ToInt() { case 1, 2, 4: if resp != nil { // 返回类型不统一,忽略错误 utils.Json.Unmarshal(data, resp) } return data, nil case 9: // 登录过期 return data, ErrCookieExpiration default: info := utils.Json.Get(data, "inf").ToString() if info == "" { info = utils.Json.Get(data, "info").ToString() } return data, fmt.Errorf(info) } } // 修复点：所有请求都自动处理 acw_sc__v2 验证和 down_ip=1 func (d *LanZou) request(url string, method string, callback base.ReqCallback, up bool) ([]byte, error) { var req *resty.Request var vs string for retry := 0; retry < 3; retry++ { if up { once.Do(func() { upClient = base.NewRestyClient().SetTimeout(120 * time.Second) }) req = upClient.R() } else { req = base.RestyClient.R() } req.SetHeaders(map[string]string{ "Referer": "https://pc.woozooo.com", "User-Agent": d.UserAgent, }) // 下载直链时需要加 down_ip=1 if strings.Contains(url, "/file/") { cookie := d.Cookie if cookie != "" { cookie += "; " } cookie += "down_ip=1" if vs != "" { cookie += "; acw_sc__v2=" + vs } req.SetHeader("cookie", cookie) } else if d.Cookie != "" { cookie := d.Cookie if vs != "" { cookie += "; acw_sc__v2=" + vs } req.SetHeader("cookie", cookie) } else if vs != "" { req.SetHeader("cookie", "acw_sc__v2="+vs) } if callback != nil { callback(req) } res, err := req.Execute(method, url) if err != nil { return nil, err } bodyStr := res.String() log.Debugf("lanzou request: url=>%s ,stats=>%d ,body => %s\n", res.Request.URL, res.StatusCode(), bodyStr) if strings.Contains(bodyStr, "acw_sc__v2") { vs, err = CalcAcwScV2(bodyStr) if err != nil { return nil, err } continue } return res.Body(), err } return nil, errors.New("acw_sc__v2 validation error") } func (d *LanZou) Login() ([]*http.Cookie, error) { resp, err := base.NewRestyClient().SetRedirectPolicy(resty.NoRedirectPolicy()). R().SetFormData(map[string]string{ "task": "3", "uid": d.Account, "pwd": d.Password, "setSessionId": "", "setSig": "", "setScene": "", "setTocen": "", "formhash": "", }).Post("https://up.woozooo.com/mlogin.php") if err != nil { return nil, err } if utils.Json.Get(resp.Body(), "zt").ToInt() != 1 { return nil, fmt.Errorf("login err: %s", resp.Body()) } d.Cookie = CookieToString(resp.Cookies()) return resp.Cookies(), nil } /* 通过cookie获取数据 */ // 获取文件和文件夹,获取到的文件大小、更改时间不可信 func (d *LanZou) GetAllFiles(folderID string) ([]model.Obj, error) { folders, err := d.GetFolders(folderID) if err != nil { return nil, err } files, err := d.GetFiles(folderID) if err != nil { return nil, err } return append( utils.MustSliceConvert(folders, func(folder FileOrFolder) model.Obj { return &folder }), utils.MustSliceConvert(files, func(file FileOrFolder) model.Obj { return &file })..., ), nil } // 通过ID获取文件夹 func (d *LanZou) GetFolders(folderID string) ([]FileOrFolder, error) { var resp RespText[[]FileOrFolder] _, err := d.doupload(func(req *resty.Request) { req.SetFormData(map[string]string{ "task": "47", "folder_id": folderID, }) }, &resp) if err != nil { return nil, err } return resp.Text, nil } // 通过ID获取文件 func (d *LanZou) GetFiles(folderID string) ([]FileOrFolder, error) { files := make([]FileOrFolder, 0) for pg := 1; ; pg++ { var resp RespText[[]FileOrFolder] _, err := d.doupload(func(req *resty.Request) { req.SetFormData(map[string]string{ "task": "5", "folder_id": folderID, "pg": strconv.Itoa(pg), }) }, &resp) if err != nil { return nil, err } if len(resp.Text) == 0 { break } files = append(files, resp.Text...) } return files, nil } // 通过ID获取文件夹分享地址 func (d *LanZou) getFolderShareUrlByID(fileID string) (*FileShare, error) { var resp RespInfo[FileShare] _, err := d.doupload(func(req *resty.Request) { req.SetFormData(map[string]string{ "task": "18", "file_id": fileID, }) }, &resp) if err != nil { return nil, err } return &resp.Info, nil } // 通过ID获取文件分享地址 func (d *LanZou) getFileShareUrlByID(fileID string) (*FileShare, error) { var resp RespInfo[FileShare] _, err := d.doupload(func(req *resty.Request) { req.SetFormData(map[string]string{ "task": "22", "file_id": fileID, }) }, &resp) if err != nil { return nil, err } return &resp.Info, nil } /* 通过分享链接获取数据 */ // 判断类容 var isFileReg = regexp.MustCompile(`class="fileinfo"|id="file"|文件描述`) var isFolderReg = regexp.MustCompile(`id="infos"`) // 获取文件文件夹基础信息 // 获取文件名称 var nameFindReg = regexp.MustCompile(`(.+?) - 蓝奏云|id="filenajax">(.+?)|var filename = '(.+?)';|
([^<>]+?)
`) // 获取文件大小 var sizeFindReg = regexp.MustCompile(`(?i)大小\W*([0-9.]+\s*[bkm]+)`) // 获取文件时间 var timeFindReg = regexp.MustCompile(`\d+\s*[秒天分小][钟时]?前|[昨前]天|\d{4}-\d{2}-\d{2}`) // 查找分享文件夹子文件夹ID和名称 var findSubFolderReg = regexp.MustCompile(`(?i)(?:folderlink|mbxfolder).+href="/(.+?)"(?:.+filename")?>(.+?)<`) // 获取下载页面链接 var findDownPageParamReg = regexp.MustCompile(` acw_sc__v2 validation error ,data => %s\n", firstPageDataStr) return "", err } continue } return firstPageDataStr, nil } return "", errors.New("acw_sc__v2 validation error") } // 通过分享链接获取文件或文件夹 func (d *LanZou) GetFileOrFolderByShareUrl(shareID, pwd string) ([]model.Obj, error) { pageData, err := d.getShareUrlHtml(shareID) if err != nil { return nil, err } if !isFileReg.MatchString(pageData) { files, err := d.getFolderByShareUrl(pwd, pageData) if err != nil { return nil, err } return utils.MustSliceConvert(files, func(file FileOrFolderByShareUrl) model.Obj { return &file }), nil } else { file, err := d.getFilesByShareUrl(shareID, pwd, pageData) if err != nil { return nil, err } return []model.Obj{file}, nil } } // 通过分享链接获取文件(下载链接也使用此方法) // FileOrFolderByShareUrl 包含 pwd 和 url 字段 // 参考 https://github.com/zaxtyson/LanZouCloud-API/blob/ab2e9ec715d1919bf432210fc16b91c6775fbb99/lanzou/api/core.py#L440 func (d *LanZou) GetFilesByShareUrl(shareID, pwd string) (file *FileOrFolderByShareUrl, err error) { pageData, err := d.getShareUrlHtml(shareID) if err != nil { return nil, err } return d.getFilesByShareUrl(shareID, pwd, pageData) } func (d *LanZou) getFilesByShareUrl(shareID, pwd string, sharePageData string) (*FileOrFolderByShareUrl, error) { var ( param map[string]string downloadUrl string baseUrl string file FileOrFolderByShareUrl ) // 删除注释 sharePageData = RemoveNotes(sharePageData) sharePageData = RemoveJSComment(sharePageData) // 需要密码 if strings.Contains(sharePageData, "pwdload") || strings.Contains(sharePageData, "passwddiv") { sharePageData, err := getJSFunctionByName(sharePageData, "down_p") if err != nil { return nil, err } param, err := htmlJsonToMap(sharePageData) if err != nil { return nil, err } param["p"] = pwd fileIDs := findFileIDReg.FindStringSubmatch(sharePageData) var fileID string if len(fileIDs) > 1 { fileID = fileIDs[1] } else { return nil, fmt.Errorf("not find file id") } var resp FileShareInfoAndUrlResp[string] _, err = d.post(d.ShareUrl+"/ajaxm.php?file="+fileID, func(req *resty.Request) { req.SetFormData(param) }, &resp) if err != nil { return nil, err } file.NameAll = resp.Inf file.Pwd = pwd baseUrl = resp.GetBaseUrl() downloadUrl = resp.GetDownloadUrl() } else { urlpaths := findDownPageParamReg.FindStringSubmatch(sharePageData) if len(urlpaths) != 2 { log.Errorf("lanzou: err => not find file page param ,data => %s\n", sharePageData) return nil, fmt.Errorf("not find file page param") } data, err := d.get(fmt.Sprint(d.ShareUrl, urlpaths[1]), nil) if err != nil { return nil, err } nextPageData := RemoveNotes(string(data)) param, err = htmlJsonToMap(nextPageData) if err != nil { return nil, err } fileIDs := findFileIDReg.FindStringSubmatch(nextPageData) var fileID string if len(fileIDs) > 1 { fileID = fileIDs[1] } else { return nil, fmt.Errorf("not find file id") } var resp FileShareInfoAndUrlResp[int] _, err = d.post(d.ShareUrl+"/ajaxm.php?file="+fileID, func(req *resty.Request) { req.SetFormData(param) }, &resp) if err != nil { return nil, err } baseUrl = resp.GetBaseUrl() downloadUrl = resp.GetDownloadUrl() names := nameFindReg.FindStringSubmatch(sharePageData) if len(names) > 1 { for _, name := range names[1:] { if name != "" { file.NameAll = name break } } } } sizes := sizeFindReg.FindStringSubmatch(sharePageData) if len(sizes) == 2 { file.Size = sizes[1] } file.ID = shareID file.Time = timeFindReg.FindString(sharePageData) // 重定向获取真实链接 var ( res *resty.Response err error ) var vs string var bodyStr string for i := 0; i < 3; i++ { res, err = base.NoRedirectClient.R().SetHeaders(map[string]string{ "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6", "Referer": baseUrl, }).SetDoNotParseResponse(true). SetCookie(&http.Cookie{ Name: "acw_sc__v2", Value: vs, }).SetHeader("cookie", "down_ip=1").Get(downloadUrl) if err != nil { return nil, err } if res.StatusCode() == 302 { if res.RawBody() != nil { res.RawBody().Close() } break } bodyBytes, err := io.ReadAll(res.RawBody()) if res.RawBody() != nil { res.RawBody().Close() } if err != nil { return nil, fmt.Errorf("读取响应体失败: %w", err) } bodyStr = string(bodyBytes) if strings.Contains(bodyStr, "acw_sc__v2") { if vs, err = CalcAcwScV2(bodyStr); err != nil { log.Errorf("lanzou: err => acw_sc__v2 validation error ,data => %s\n", bodyStr) return nil, err } continue } break } if err != nil { return nil, err } file.Url = res.Header().Get("location") // 触发二次验证，也需要处理一下触发acw_sc__v2的情况 if res.StatusCode() != 302 { param, err = htmlJsonToMap(bodyStr) if err != nil { return nil, err } param["el"] = "2" time.Sleep(time.Second * 2) // 通过验证获取直链 var data []byte for i := 0; i < 3; i++ { data, err = d.post(fmt.Sprint(baseUrl, "/ajax.php"), func(req *resty.Request) { req.SetFormData(param) req.SetHeader("cookie", "down_ip=1") if vs != "" { req.SetCookie(&http.Cookie{ Name: "acw_sc__v2", Value: vs, }) } }, nil) if err != nil { return nil, err } ajaxBodyStr := string(data) if strings.Contains(ajaxBodyStr, "acw_sc__v2") { if vs, err = CalcAcwScV2(ajaxBodyStr); err != nil { log.Errorf("lanzou: err => acw_sc__v2 validation error ,data => %s\n", ajaxBodyStr) return nil, err } time.Sleep(time.Second * 2) continue } break } if err != nil { return nil, err } file.Url = utils.Json.Get(data, "url").ToString() } return &file, nil } // 通过分享链接获取文件夹 // 似乎子目录和文件不会加密 // 参考 https://github.com/zaxtyson/LanZouCloud-API/blob/ab2e9ec715d1919bf432210fc16b91c6775fbb99/lanzou/api/core.py#L1089 func (d *LanZou) GetFolderByShareUrl(shareID, pwd string) ([]FileOrFolderByShareUrl, error) { pageData, err := d.getShareUrlHtml(shareID) if err != nil { return nil, err } return d.getFolderByShareUrl(pwd, pageData) } func (d *LanZou) getFolderByShareUrl(pwd string, sharePageData string) ([]FileOrFolderByShareUrl, error) { from, err := htmlJsonToMap(sharePageData) if err != nil { return nil, err } files := make([]FileOrFolderByShareUrl, 0) // vip获取文件夹 floders := findSubFolderReg.FindAllStringSubmatch(sharePageData, -1) for _, floder := range floders { if len(floder) == 3 { files = append(files, FileOrFolderByShareUrl{ // Pwd: pwd, // 子文件夹不加密 ID: floder[1], NameAll: floder[2], IsFloder: true, }) } } // 获取文件 from["pwd"] = pwd for page := 1; ; page++ { from["pg"] = strconv.Itoa(page) var resp FileOrFolderByShareUrlResp _, err := d.post(d.ShareUrl+"/filemoreajax.php", func(req *resty.Request) { req.SetFormData(from) }, &resp) if err != nil { return nil, err } // 文件夹中的文件加密 for i := 0; i < len(resp.Text); i++ { resp.Text[i].Pwd = pwd } if len(resp.Text) == 0 { break } files = append(files, resp.Text...) time.Sleep(time.Second) } return files, nil } // 通过下载头获取真实文件信息 func (d *LanZou) getFileRealInfo(downURL string) (*int64, *time.Time) { res, _ := base.RestyClient.R().Head(downURL) if res == nil { return nil, nil } time, _ := http.ParseTime(res.Header().Get("Last-Modified")) size, _ := strconv.ParseInt(res.Header().Get("Content-Length"), 10, 64) return &size, &time } func (d *LanZou) getVeiAndUid() (vei string, uid string, err error) { var resp []byte resp, err = d.get("https://pc.woozooo.com/mydisk.php", func(req *resty.Request) { req.SetQueryParams(map[string]string{ "item": "files", "action": "index", }) }) if err != nil { return } // uid uids := regexp.MustCompile(`uid=([^'"&;]+)`).FindStringSubmatch(string(resp)) if len(uids) < 2 { err = fmt.Errorf("uid variable not find") return } uid = uids[1] // vei html := RemoveNotes(string(resp)) data, err := htmlJsonToMap(html) if err != nil { return } vei = data["vei"] return } ================================================ FILE: drivers/lenovonas_share/driver.go ================================================ package LenovoNasShare import ( "context" "fmt" "net/http" "net/url" "strings" "time" "github.com/go-resty/resty/v2" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type LenovoNasShare struct { model.Storage Addition stoken string expireAt int64 } func (d *LenovoNasShare) Config() driver.Config { return config } func (d *LenovoNasShare) GetAddition() driver.Additional { return &d.Addition } func (d *LenovoNasShare) Init(ctx context.Context) error { if err := d.getStoken(); err != nil { return err } if !d.ShowRootFolder && d.RootFolderPath == "" { list, _ := d.List(ctx, File{}, model.ListArgs{}) d.RootFolderPath = list[0].GetPath() } return nil } func (d *LenovoNasShare) Drop(ctx context.Context) error { return nil } func (d *LenovoNasShare) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { d.checkStoken() // 检查stoken是否过期 path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/")) var resp Files query := map[string]string{ "code": d.ShareId, "num": "5000", "stoken": d.stoken, "path": path, } _, err := d.request(d.Host+"/oneproxy/api/share/v1/files", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return nil, err } return utils.SliceConvert(resp.Data.List, func(src File) (model.Obj, error) { if src.IsDir() { return src, nil } return &model.ObjThumb{ Object: model.Object{ Name: src.GetName(), Path: src.GetPath(), Size: src.GetSize(), Modified: src.ModTime(), IsFolder: src.IsDir(), }, Thumbnail: model.Thumbnail{ Thumbnail: func() string { thumbUrl := d.Host + "/oneproxy/api/share/v1/file/thumb?code=" + d.ShareId + "&stoken=" + d.stoken + "&path=" + url.QueryEscape(src.GetPath()) return thumbUrl }(), }, }, nil }) } func (d *LenovoNasShare) checkStoken() { // 检查stoken是否过期 if d.expireAt < time.Now().Unix() { d.getStoken() } } func (d *LenovoNasShare) getStoken() error { // 获取stoken if d.Host == "" { d.Host = "https://siot-share.lenovo.com.cn" } parts := strings.Split(d.ShareId, "/") d.ShareId = parts[len(parts)-1] query := map[string]string{ "code": d.ShareId, "password": d.SharePwd, } resp, err := d.request(d.Host+"/oneproxy/api/share/v1/access", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, nil) if err != nil { return err } d.stoken = utils.Json.Get(resp, "data", "stoken").ToString() d.expireAt = utils.Json.Get(resp, "data", "expires_in").ToInt64() + time.Now().Unix() - 60 return nil } func (d *LenovoNasShare) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { d.checkStoken() // 检查stoken是否过期 query := map[string]string{ "code": d.ShareId, "stoken": d.stoken, "path": file.GetPath(), } resp, err := d.request(d.Host+"/oneproxy/api/share/v1/file/link", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, nil) if err != nil { return nil, err } downloadUrl := d.Host + "/oneproxy/api/share/v1/file/download?code=" + d.ShareId + "&dtoken=" + utils.Json.Get(resp, "data", "param", "dtoken").ToString() link := model.Link{ URL: downloadUrl, Header: http.Header{ "Referer": []string{"https://siot-share.lenovo.com.cn"}, }, } return &link, nil } func (d *LenovoNasShare) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { return nil, errs.NotImplement } func (d *LenovoNasShare) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return nil, errs.NotImplement } func (d *LenovoNasShare) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { return nil, errs.NotImplement } func (d *LenovoNasShare) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return nil, errs.NotImplement } func (d *LenovoNasShare) Remove(ctx context.Context, obj model.Obj) error { return errs.NotImplement } func (d *LenovoNasShare) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { return nil, errs.NotImplement } var _ driver.Driver = (*LenovoNasShare)(nil) ================================================ FILE: drivers/lenovonas_share/meta.go ================================================ package LenovoNasShare import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath ShareId string `json:"share_id" required:"true" help:"The part after the last / in the shared link"` SharePwd string `json:"share_pwd" required:"true" help:"The password of the shared link"` Host string `json:"host" required:"true" default:"https://siot-share.lenovo.com.cn" help:"You can change it to your local area network"` ShowRootFolder bool `json:"show_root_folder" default:"true"` } var config = driver.Config{ Name: "LenovoNasShare", LocalSort: true, NoUpload: true, } func init() { op.RegisterDriver(func() driver.Driver { return &LenovoNasShare{} }) } ================================================ FILE: drivers/lenovonas_share/types.go ================================================ package LenovoNasShare import ( "encoding/json" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" _ "github.com/OpenListTeam/OpenList/v4/internal/model" ) func (f *File) UnmarshalJSON(data []byte) error { type Alias File aux := &struct { CreateAt int64 `json:"time"` UpdateAt int64 `json:"chtime"` *Alias }{ Alias: (*Alias)(f), } if err := json.Unmarshal(data, aux); err != nil { return err } f.CreateAt = time.Unix(aux.CreateAt, 0) f.UpdateAt = time.Unix(aux.UpdateAt, 0) return nil } type File struct { FileName string `json:"name"` Size int64 `json:"size"` CreateAt time.Time `json:"time"` UpdateAt time.Time `json:"chtime"` Path string `json:"path"` Type string `json:"type"` } func (f File) GetHash() utils.HashInfo { return utils.HashInfo{} } func (f File) GetPath() string { return f.Path } func (f File) GetSize() int64 { if f.IsDir() { return 0 } else { return f.Size } } func (f File) GetName() string { return f.FileName } func (f File) ModTime() time.Time { return f.UpdateAt } func (f File) CreateTime() time.Time { return f.CreateAt } func (f File) IsDir() bool { return f.Type == "dir" } func (f File) GetID() string { return f.GetPath() } type Files struct { Data struct { List []File `json:"list"` HasMore bool `json:"has_more"` } `json:"data"` } ================================================ FILE: drivers/lenovonas_share/util.go ================================================ package LenovoNasShare import ( "errors" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/pkg/utils" jsoniter "github.com/json-iterator/go" ) func (d *LenovoNasShare) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeaders(map[string]string{ "origin": "https://siot-share.lenovo.com.cn", "referer": "https://siot-share.lenovo.com.cn/", "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) openlist-client", "platform": "web", "app-version": "3", }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } res, err := req.Execute(method, url) if err != nil { return nil, err } body := res.Body() result := utils.Json.Get(body, "result").ToBool() if !result { return nil, errors.New(jsoniter.Get(body, "error", "msg").ToString()) } return body, nil } ================================================ FILE: drivers/local/benchmark_calculatedirsize_test.go ================================================ package local // TestDirCalculateSize tests the directory size calculation // It should be run with the local driver enabled and directory size calculation set to true import ( "os" "path/filepath" "strconv" "testing" "github.com/OpenListTeam/OpenList/v4/internal/driver" ) func generatedTestDir(dir string, dep, filecount int) { if dep == 0 { return } for i := 0; i < dep; i++ { subDir := dir + "/dir" + strconv.Itoa(i) os.Mkdir(subDir, 0755) generatedTestDir(subDir, dep-1, filecount) generatedFiles(subDir, filecount) } } func generatedFiles(path string, count int) error { for i := 0; i < count; i++ { filePath := filepath.Join(path, "file"+strconv.Itoa(i)+".txt") file, err := os.Create(filePath) if err != nil { return err } // 使用随机ascii字符填充文件 content := make([]byte, 1024) // 1KB file for j := range content { content[j] = byte('a' + j%26) // Fill with 'a' to 'z' } _, err = file.Write(content) if err != nil { return err } file.Close() } return nil } // performance tests for directory size calculation func BenchmarkCalculateDirSize(t *testing.B) { // 初始化t的日志 t.Logf("Starting performance test for directory size calculation") // 确保测试目录存在 if testing.Short() { t.Skip("Skipping performance test in short mode") } // 创建tmp directory for testing testTempDir := t.TempDir() err := os.MkdirAll(testTempDir, 0755) if err != nil { t.Fatalf("Failed to create test directory: %v", err) } defer os.RemoveAll(testTempDir) // Clean up after test // 构建一个深度为5，每层10个文件和10个目录的目录结构 generatedTestDir(testTempDir, 5, 10) // Initialize the local driver with directory size calculation enabled d := &Local{ directoryMap: DirectoryMap{ root: testTempDir, }, Addition: Addition{ DirectorySize: true, RootPath: driver.RootPath{ RootFolderPath: testTempDir, }, }, } //record the start time t.StartTimer() // Calculate the directory size err = d.directoryMap.RecalculateDirSize() if err != nil { t.Fatalf("Failed to calculate directory size: %v", err) } //record the end time t.StopTimer() // Print the size and duration node, ok := d.directoryMap.Get(d.directoryMap.root) if !ok { t.Fatalf("Failed to get root node from directory map") } t.Logf("Directory size: %d bytes", node.fileSum+node.directorySum) t.Logf("Performance test completed successfully") } ================================================ FILE: drivers/local/copy_namedpipes.go ================================================ //go:build !windows && !plan9 && !netbsd && !aix && !illumos && !solaris && !js package local import ( "os" "path/filepath" "syscall" ) func copyNamedPipe(dstPath string, mode os.FileMode, dirMode os.FileMode) error { if err := os.MkdirAll(filepath.Dir(dstPath), dirMode); err != nil { return err } return syscall.Mkfifo(dstPath, uint32(mode)) } ================================================ FILE: drivers/local/copy_namedpipes_x.go ================================================ //go:build windows || plan9 || netbsd || aix || illumos || solaris || js package local import "os" func copyNamedPipe(_ string, _, _ os.FileMode) error { return nil } ================================================ FILE: drivers/local/driver.go ================================================ package local import ( "bytes" "context" "errors" "fmt" "io/fs" "net/http" "os" stdpath "path" "path/filepath" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/times" log "github.com/sirupsen/logrus" _ "golang.org/x/image/webp" ) type Local struct { model.Storage Addition mkdirPerm int32 // directory size data directoryMap DirectoryMap // zero means no limit thumbConcurrency int thumbTokenBucket TokenBucket // video thumb position videoThumbPos float64 videoThumbPosIsPercentage bool } func (d *Local) Config() driver.Config { return config } func (d *Local) Init(ctx context.Context) error { if d.MkdirPerm == "" { d.mkdirPerm = 0o777 } else { v, err := strconv.ParseUint(d.MkdirPerm, 8, 32) if err != nil { return err } d.mkdirPerm = int32(v) } if !utils.Exists(d.GetRootPath()) { return fmt.Errorf("root folder %s not exists", d.GetRootPath()) } if !filepath.IsAbs(d.GetRootPath()) { abs, err := filepath.Abs(d.GetRootPath()) if err != nil { return err } d.Addition.RootFolderPath = abs } if d.DirectorySize { d.directoryMap.root = d.GetRootPath() _, err := d.directoryMap.CalculateDirSize(d.GetRootPath()) if err != nil { return err } } else { d.directoryMap.Clear() } if d.ThumbCacheFolder != "" && !utils.Exists(d.ThumbCacheFolder) { err := os.MkdirAll(d.ThumbCacheFolder, os.FileMode(d.mkdirPerm)) if err != nil { return err } } if d.ThumbConcurrency != "" { v, err := strconv.ParseUint(d.ThumbConcurrency, 10, 32) if err != nil { return err } d.thumbConcurrency = int(v) } if d.thumbConcurrency == 0 { d.thumbTokenBucket = NewNopTokenBucket() } else { d.thumbTokenBucket = NewStaticTokenBucketWithMigration(d.thumbTokenBucket, d.thumbConcurrency) } // Check the VideoThumbPos value if d.VideoThumbPos == "" { d.VideoThumbPos = "20%" } if strings.HasSuffix(d.VideoThumbPos, "%") { percentage := strings.TrimSuffix(d.VideoThumbPos, "%") val, err := strconv.ParseFloat(percentage, 64) if err != nil { return fmt.Errorf("invalid video_thumb_pos value: %s, err: %s", d.VideoThumbPos, err) } if val < 0 || val > 100 { return fmt.Errorf("invalid video_thumb_pos value: %s, the precentage must be a number between 0 and 100", d.VideoThumbPos) } d.videoThumbPosIsPercentage = true d.videoThumbPos = val / 100 } else { val, err := strconv.ParseFloat(d.VideoThumbPos, 64) if err != nil { return fmt.Errorf("invalid video_thumb_pos value: %s, err: %s", d.VideoThumbPos, err) } if val < 0 { return fmt.Errorf("invalid video_thumb_pos value: %s, the time must be a positive number", d.VideoThumbPos) } d.videoThumbPosIsPercentage = false d.videoThumbPos = val } return nil } func (d *Local) Drop(ctx context.Context) error { return nil } func (d *Local) GetAddition() driver.Additional { return &d.Addition } func (d *Local) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { fullPath := dir.GetPath() rawFiles, err := readDir(fullPath) if d.DirectorySize && args.Refresh { d.directoryMap.RecalculateDirSize() } if err != nil { return nil, err } var files []model.Obj for _, f := range rawFiles { if d.ShowHidden || !isHidden(f, fullPath) { files = append(files, d.FileInfoToObj(ctx, f, args.ReqPath, fullPath)) } } return files, nil } func (d *Local) FileInfoToObj(ctx context.Context, f fs.FileInfo, reqPath string, fullPath string) model.Obj { thumb := "" if d.Thumbnail { typeName := utils.GetFileType(f.Name()) if typeName == conf.IMAGE || typeName == conf.VIDEO { thumb = common.GetApiUrl(ctx) + stdpath.Join("/d", reqPath, f.Name()) thumb = utils.EncodePath(thumb, true) thumb += "?type=thumb&sign=" + sign.Sign(stdpath.Join(reqPath, f.Name())) } } isFolder := f.IsDir() || isSymlinkDir(f, fullPath) var size int64 if isFolder { node, ok := d.directoryMap.Get(filepath.Join(fullPath, f.Name())) if ok { size = node.fileSum + node.directorySum } } else { size = f.Size() } var ctime time.Time t, err := times.Stat(stdpath.Join(fullPath, f.Name())) if err == nil { if t.HasBirthTime() { ctime = t.BirthTime() } } file := model.ObjThumb{ Object: model.Object{ Path: filepath.Join(fullPath, f.Name()), Name: f.Name(), Modified: f.ModTime(), Size: size, IsFolder: isFolder, Ctime: ctime, }, Thumbnail: model.Thumbnail{ Thumbnail: thumb, }, } return &file } func (d *Local) Get(ctx context.Context, path string) (model.Obj, error) { path = filepath.Join(d.GetRootPath(), path) f, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { return nil, errs.ObjectNotFound } return nil, err } isFolder := f.IsDir() || isSymlinkDir(f, path) size := f.Size() if isFolder { node, ok := d.directoryMap.Get(path) if ok { size = node.fileSum + node.directorySum } } else { size = f.Size() } var ctime time.Time t, err := times.Stat(path) if err == nil { if t.HasBirthTime() { ctime = t.BirthTime() } } file := model.Object{ Path: path, Name: f.Name(), Modified: f.ModTime(), Ctime: ctime, Size: size, IsFolder: isFolder, } return &file, nil } func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { fullPath := file.GetPath() link := &model.Link{} var MFile model.File if args.Type == "thumb" && utils.Ext(file.GetName()) != "svg" { var buf *bytes.Buffer var thumbPath *string err := d.thumbTokenBucket.Do(ctx, func() error { var err error buf, thumbPath, err = d.getThumb(file) return err }) if err != nil { return nil, err } link.Header = http.Header{ "Content-Type": []string{"image/png"}, } if thumbPath != nil { open, err := os.Open(*thumbPath) if err != nil { return nil, err } // Get thumbnail file size for Content-Length stat, err := open.Stat() if err != nil { open.Close() return nil, err } link.ContentLength = int64(stat.Size()) MFile = open } else { MFile = bytes.NewReader(buf.Bytes()) link.ContentLength = int64(buf.Len()) } } else { open, err := os.Open(fullPath) if err != nil { return nil, err } link.ContentLength = file.GetSize() MFile = open } link.SyncClosers.AddIfCloser(MFile) link.RangeReader = stream.GetRangeReaderFromMFile(link.ContentLength, MFile) link.RequireReference = link.SyncClosers.Length() > 0 return link, nil } func (d *Local) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { fullPath := filepath.Join(parentDir.GetPath(), dirName) err := os.MkdirAll(fullPath, os.FileMode(d.mkdirPerm)) if err != nil { return err } return nil } func (d *Local) Move(ctx context.Context, srcObj, dstDir model.Obj) error { srcPath := srcObj.GetPath() dstPath := filepath.Join(dstDir.GetPath(), srcObj.GetName()) if utils.IsSubPath(srcPath, dstPath) { return fmt.Errorf("the destination folder is a subfolder of the source folder") } err := os.Rename(srcPath, dstPath) if isCrossDeviceError(err) { // 跨设备移动，变更为移动任务 return errs.NotImplement } if err == nil { srcParent := filepath.Dir(srcPath) dstParent := filepath.Dir(dstPath) if d.directoryMap.Has(srcParent) { d.directoryMap.UpdateDirSize(srcParent) d.directoryMap.UpdateDirParents(srcParent) } if d.directoryMap.Has(dstParent) { d.directoryMap.UpdateDirSize(dstParent) d.directoryMap.UpdateDirParents(dstParent) } } return err } func (d *Local) Rename(ctx context.Context, srcObj model.Obj, newName string) error { srcPath := srcObj.GetPath() dstPath := filepath.Join(filepath.Dir(srcPath), newName) err := os.Rename(srcPath, dstPath) if err != nil { return err } if srcObj.IsDir() { if d.directoryMap.Has(srcPath) { d.directoryMap.DeleteDirNode(srcPath) d.directoryMap.CalculateDirSize(dstPath) } } return nil } func (d *Local) Copy(_ context.Context, srcObj, dstDir model.Obj) error { srcPath := srcObj.GetPath() dstPath := filepath.Join(dstDir.GetPath(), srcObj.GetName()) if utils.IsSubPath(srcPath, dstPath) { return fmt.Errorf("the destination folder is a subfolder of the source folder") } info, err := os.Lstat(srcPath) if err != nil { return err } // 复制regular文件会返回errs.NotImplement, 转为复制任务 if err = d.tryCopy(srcPath, dstPath, info); err != nil { return err } if d.directoryMap.Has(filepath.Dir(dstPath)) { d.directoryMap.UpdateDirSize(filepath.Dir(dstPath)) d.directoryMap.UpdateDirParents(filepath.Dir(dstPath)) } return nil } func (d *Local) Remove(ctx context.Context, obj model.Obj) error { var err error if utils.SliceContains([]string{"", "delete permanently"}, d.RecycleBinPath) { if obj.IsDir() { err = os.RemoveAll(obj.GetPath()) } else { err = os.Remove(obj.GetPath()) } } else { objPath := obj.GetPath() objName := obj.GetName() var relPath string relPath, err = filepath.Rel(d.GetRootPath(), filepath.Dir(objPath)) if err != nil { return err } recycleBinPath := filepath.Join(d.RecycleBinPath, relPath) if !utils.Exists(recycleBinPath) { err = os.MkdirAll(recycleBinPath, 0o755) if err != nil { return err } } dstPath := filepath.Join(recycleBinPath, objName) if utils.Exists(dstPath) { dstPath = filepath.Join(recycleBinPath, objName+"_"+time.Now().Format("20060102150405")) } err = os.Rename(objPath, dstPath) } if err != nil { return err } if obj.IsDir() { if d.directoryMap.Has(obj.GetPath()) { d.directoryMap.DeleteDirNode(obj.GetPath()) d.directoryMap.UpdateDirSize(filepath.Dir(obj.GetPath())) d.directoryMap.UpdateDirParents(filepath.Dir(obj.GetPath())) } } else { if d.directoryMap.Has(filepath.Dir(obj.GetPath())) { d.directoryMap.UpdateDirSize(filepath.Dir(obj.GetPath())) d.directoryMap.UpdateDirParents(filepath.Dir(obj.GetPath())) } } return nil } func (d *Local) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { fullPath := filepath.Join(dstDir.GetPath(), stream.GetName()) out, err := os.Create(fullPath) if err != nil { return err } defer func() { _ = out.Close() if errors.Is(err, context.Canceled) { _ = os.Remove(fullPath) } }() err = utils.CopyWithCtx(ctx, out, stream, stream.GetSize(), up) if err != nil { return err } err = os.Chtimes(fullPath, stream.ModTime(), stream.ModTime()) if err != nil { log.Errorf("[local] failed to change time of %s: %s", fullPath, err) } if d.directoryMap.Has(dstDir.GetPath()) { d.directoryMap.UpdateDirSize(dstDir.GetPath()) d.directoryMap.UpdateDirParents(dstDir.GetPath()) } return nil } func (d *Local) GetDetails(ctx context.Context) (*model.StorageDetails, error) { du, err := getDiskUsage(d.RootFolderPath) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: du, }, nil } var _ driver.Driver = (*Local)(nil) ================================================ FILE: drivers/local/meta.go ================================================ package local import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath DirectorySize bool `json:"directory_size" default:"false" help:"This might impact host performance"` Thumbnail bool `json:"thumbnail" required:"true" help:"enable thumbnail"` ThumbCacheFolder string `json:"thumb_cache_folder"` ThumbConcurrency string `json:"thumb_concurrency" default:"16" required:"false" help:"Number of concurrent thumbnail generation goroutines. This controls how many thumbnails can be generated in parallel."` VideoThumbPos string `json:"video_thumb_pos" default:"20%" required:"false" help:"The position of the video thumbnail. If the value is a number (integer ot floating point), it represents the time in seconds. If the value ends with '%', it represents the percentage of the video duration."` ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` MkdirPerm string `json:"mkdir_perm" default:"777"` RecycleBinPath string `json:"recycle_bin_path" default:"delete permanently" help:"path to recycle bin, delete permanently if empty or keep 'delete permanently'"` } var config = driver.Config{ Name: "Local", LocalSort: true, OnlyProxy: true, NoCache: true, DefaultRoot: "/", NoLinkURL: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Local{ directoryMap: DirectoryMap{}, } }) } ================================================ FILE: drivers/local/token_bucket.go ================================================ package local import "context" type TokenBucket interface { Take() <-chan struct{} Put() Do(context.Context, func() error) error } // StaticTokenBucket is a bucket with a fixed number of tokens, // where the retrieval and return of tokens are manually controlled. // In the initial state, the bucket is full. type StaticTokenBucket struct { bucket chan struct{} } func NewStaticTokenBucket(size int) StaticTokenBucket { bucket := make(chan struct{}, size) for range size { bucket <- struct{}{} } return StaticTokenBucket{bucket: bucket} } func NewStaticTokenBucketWithMigration(oldBucket TokenBucket, size int) StaticTokenBucket { if oldBucket != nil { oldStaticBucket, ok := oldBucket.(StaticTokenBucket) if ok { oldSize := cap(oldStaticBucket.bucket) migrateSize := oldSize if size < migrateSize { migrateSize = size } bucket := make(chan struct{}, size) for range size - migrateSize { bucket <- struct{}{} } if migrateSize != 0 { go func() { for range migrateSize { <-oldStaticBucket.bucket bucket <- struct{}{} } close(oldStaticBucket.bucket) }() } return StaticTokenBucket{bucket: bucket} } } return NewStaticTokenBucket(size) } // Take channel maybe closed when local driver is modified. // don't call Put method after the channel is closed. func (b StaticTokenBucket) Take() <-chan struct{} { return b.bucket } func (b StaticTokenBucket) Put() { b.bucket <- struct{}{} } func (b StaticTokenBucket) Do(ctx context.Context, f func() error) error { select { case <-ctx.Done(): return ctx.Err() case _, ok := <-b.Take(): if ok { defer b.Put() } } return f() } // NopTokenBucket all function calls to this bucket will success immediately type NopTokenBucket struct { nop chan struct{} } func NewNopTokenBucket() NopTokenBucket { nop := make(chan struct{}) close(nop) return NopTokenBucket{nop} } func (b NopTokenBucket) Take() <-chan struct{} { return b.nop } func (b NopTokenBucket) Put() {} func (b NopTokenBucket) Do(_ context.Context, f func() error) error { return f() } ================================================ FILE: drivers/local/util.go ================================================ package local import ( "bytes" "encoding/json" "errors" "fmt" "io/fs" "os" "path/filepath" "runtime" "slices" "sort" "strconv" "strings" "sync" "github.com/KarpelesLab/reflink" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/disintegration/imaging" ffmpeg "github.com/u2takey/ffmpeg-go" ) func isSymlinkDir(f fs.FileInfo, path string) bool { if f.Mode()&os.ModeSymlink == os.ModeSymlink || (runtime.GOOS == "windows" && f.Mode()&os.ModeIrregular == os.ModeIrregular) { // os.ModeIrregular is Junction bit in Windows dst, err := os.Readlink(filepath.Join(path, f.Name())) if err != nil { return false } if !filepath.IsAbs(dst) { dst = filepath.Join(path, dst) } stat, err := os.Stat(dst) if err != nil { return false } return stat.IsDir() } return false } // Get the snapshot of the video func (d *Local) GetSnapshot(videoPath string) (imgData *bytes.Buffer, err error) { // Run ffprobe to get the video duration jsonOutput, err := ffmpeg.Probe(videoPath) if err != nil { return nil, err } // get format.duration from the json string type probeFormat struct { Duration string `json:"duration"` } type probeData struct { Format probeFormat `json:"format"` } var probe probeData err = json.Unmarshal([]byte(jsonOutput), &probe) if err != nil { return nil, err } totalDuration, err := strconv.ParseFloat(probe.Format.Duration, 64) if err != nil { return nil, err } var ss string if d.videoThumbPosIsPercentage { ss = fmt.Sprintf("%f", totalDuration*d.videoThumbPos) } else { // If the value is greater than the total duration, use the total duration if d.videoThumbPos > totalDuration { ss = fmt.Sprintf("%f", totalDuration) } else { ss = fmt.Sprintf("%f", d.videoThumbPos) } } // Run ffmpeg to get the snapshot srcBuf := bytes.NewBuffer(nil) // If the remaining time from the seek point to the end of the video is less // than the duration of a single frame, ffmpeg cannot extract any frames // within the specified range and will exit with an error. // The "noaccurate_seek" option prevents this error and would also speed up // the seek process. stream := ffmpeg.Input(videoPath, ffmpeg.KwArgs{"ss": ss, "noaccurate_seek": ""}). Output("pipe:", ffmpeg.KwArgs{"vframes": 1, "format": "image2", "vcodec": "mjpeg"}). GlobalArgs("-loglevel", "error").Silent(true). WithOutput(srcBuf, os.Stdout) if err = stream.Run(); err != nil { return nil, err } return srcBuf, nil } func readDir(dirname string) ([]fs.FileInfo, error) { f, err := os.Open(dirname) if err != nil { return nil, err } list, err := f.Readdir(-1) f.Close() if err != nil { return nil, err } sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() }) return list, nil } func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { fullPath := file.GetPath() thumbPrefix := "openlist_thumb_" thumbName := thumbPrefix + utils.GetMD5EncodeStr(fullPath) + ".png" if d.ThumbCacheFolder != "" { // skip if the file is a thumbnail if strings.HasPrefix(file.GetName(), thumbPrefix) { return nil, &fullPath, nil } thumbPath := filepath.Join(d.ThumbCacheFolder, thumbName) if utils.Exists(thumbPath) { return nil, &thumbPath, nil } } var srcBuf *bytes.Buffer if utils.GetFileType(file.GetName()) == conf.VIDEO { videoBuf, err := d.GetSnapshot(fullPath) if err != nil { return nil, nil, err } srcBuf = videoBuf } else { imgData, err := os.ReadFile(fullPath) if err != nil { return nil, nil, err } imgBuf := bytes.NewBuffer(imgData) srcBuf = imgBuf } image, err := imaging.Decode(srcBuf, imaging.AutoOrientation(true)) if err != nil { return nil, nil, err } thumbImg := imaging.Resize(image, 144, 0, imaging.Lanczos) var buf bytes.Buffer err = imaging.Encode(&buf, thumbImg, imaging.PNG) if err != nil { return nil, nil, err } if d.ThumbCacheFolder != "" { err = os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), buf.Bytes(), 0o666) if err != nil { return nil, nil, err } } return &buf, nil, nil } type DirectoryMap struct { root string data sync.Map } type DirectoryNode struct { fileSum int64 directorySum int64 children []string } type DirectoryTask struct { path string cache *DirectoryTaskCache } type DirectoryTaskCache struct { fileSum int64 children []string } func (m *DirectoryMap) Has(path string) bool { _, ok := m.data.Load(path) return ok } func (m *DirectoryMap) Get(path string) (*DirectoryNode, bool) { value, ok := m.data.Load(path) if !ok { return &DirectoryNode{}, false } node, ok := value.(*DirectoryNode) if !ok { return &DirectoryNode{}, false } return node, true } func (m *DirectoryMap) Set(path string, node *DirectoryNode) { m.data.Store(path, node) } func (m *DirectoryMap) Delete(path string) { m.data.Delete(path) } func (m *DirectoryMap) Clear() { m.data.Clear() } func (m *DirectoryMap) RecalculateDirSize() error { m.Clear() if m.root == "" { return fmt.Errorf("root path is not set") } size, err := m.CalculateDirSize(m.root) if err != nil { return err } if node, ok := m.Get(m.root); ok { node.fileSum = size node.directorySum = size } return nil } func (m *DirectoryMap) CalculateDirSize(dirname string) (int64, error) { stack := []DirectoryTask{ {path: dirname}, } for len(stack) > 0 { task := stack[len(stack)-1] stack = stack[:len(stack)-1] if task.cache != nil { directorySum := int64(0) for _, filename := range task.cache.children { child, ok := m.Get(filepath.Join(task.path, filename)) if !ok { return 0, fmt.Errorf("child node not found") } directorySum += child.fileSum + child.directorySum } m.Set(task.path, &DirectoryNode{ fileSum: task.cache.fileSum, directorySum: directorySum, children: task.cache.children, }) continue } files, err := readDir(task.path) if err != nil { return 0, err } fileSum := int64(0) directorySum := int64(0) children := []string{} queue := []DirectoryTask{} for _, f := range files { fullpath := filepath.Join(task.path, f.Name()) isFolder := f.IsDir() || isSymlinkDir(f, fullpath) if isFolder { if node, ok := m.Get(fullpath); ok { directorySum += node.fileSum + node.directorySum } else { queue = append(queue, DirectoryTask{ path: fullpath, }) } children = append(children, f.Name()) } else { fileSum += f.Size() } } if len(queue) > 0 { stack = append(stack, DirectoryTask{ path: task.path, cache: &DirectoryTaskCache{ fileSum: fileSum, children: children, }, }) stack = append(stack, queue...) continue } m.Set(task.path, &DirectoryNode{ fileSum: fileSum, directorySum: directorySum, children: children, }) } if node, ok := m.Get(dirname); ok { return node.fileSum + node.directorySum, nil } return 0, nil } func (m *DirectoryMap) UpdateDirSize(dirname string) (int64, error) { node, ok := m.Get(dirname) if !ok { return 0, fmt.Errorf("directory node not found") } files, err := readDir(dirname) if err != nil { return 0, err } fileSum := int64(0) directorySum := int64(0) children := []string{} for _, f := range files { fullpath := filepath.Join(dirname, f.Name()) isFolder := f.IsDir() || isSymlinkDir(f, fullpath) if isFolder { if node, ok := m.Get(fullpath); ok { directorySum += node.fileSum + node.directorySum } else { value, err := m.CalculateDirSize(fullpath) if err != nil { return 0, err } directorySum += value } children = append(children, f.Name()) } else { fileSum += f.Size() } } for _, c := range node.children { if !slices.Contains(children, c) { m.DeleteDirNode(filepath.Join(dirname, c)) } } node.fileSum = fileSum node.directorySum = directorySum node.children = children return fileSum + directorySum, nil } func (m *DirectoryMap) UpdateDirParents(dirname string) error { parentPath := filepath.Dir(dirname) for parentPath != m.root && !strings.HasPrefix(m.root, parentPath) { if node, ok := m.Get(parentPath); ok { directorySum := int64(0) for _, c := range node.children { child, ok := m.Get(filepath.Join(parentPath, c)) if !ok { return fmt.Errorf("child node not found") } directorySum += child.fileSum + child.directorySum } node.directorySum = directorySum } parentPath = filepath.Dir(parentPath) } return nil } func (m *DirectoryMap) DeleteDirNode(dirname string) error { stack := []string{dirname} for len(stack) > 0 { current := stack[len(stack)-1] stack = stack[:len(stack)-1] if node, ok := m.Get(current); ok { for _, filename := range node.children { stack = append(stack, filepath.Join(current, filename)) } m.Delete(current) } } return nil } func (d *Local) tryCopy(srcPath, dstPath string, info os.FileInfo) error { if info.Mode()&os.ModeDevice != 0 { return errors.New("cannot copy a device") } else if info.Mode()&os.ModeSymlink != 0 { return d.copySymlink(srcPath, dstPath) } else if info.Mode()&os.ModeNamedPipe != 0 { return copyNamedPipe(dstPath, info.Mode(), os.FileMode(d.mkdirPerm)) } else if info.IsDir() { return d.recurAndTryCopy(srcPath, dstPath) } else { return tryReflinkCopy(srcPath, dstPath) } } func (d *Local) copySymlink(srcPath, dstPath string) error { linkOrig, err := os.Readlink(srcPath) if err != nil { return err } dstDir := filepath.Dir(dstPath) if !filepath.IsAbs(linkOrig) { srcDir := filepath.Dir(srcPath) rel, err := filepath.Rel(dstDir, srcDir) if err != nil { rel, err = filepath.Abs(srcDir) } if err != nil { return err } linkOrig = filepath.Clean(filepath.Join(rel, linkOrig)) } err = os.MkdirAll(dstDir, os.FileMode(d.mkdirPerm)) if err != nil { return err } return os.Symlink(linkOrig, dstPath) } func (d *Local) recurAndTryCopy(srcPath, dstPath string) error { err := os.MkdirAll(dstPath, os.FileMode(d.mkdirPerm)) if err != nil { return err } files, err := readDir(srcPath) if err != nil { return err } for _, f := range files { if !f.IsDir() { sp := filepath.Join(srcPath, f.Name()) dp := filepath.Join(dstPath, f.Name()) if err = d.tryCopy(sp, dp, f); err != nil { return err } } } for _, f := range files { if f.IsDir() { sp := filepath.Join(srcPath, f.Name()) dp := filepath.Join(dstPath, f.Name()) if err = d.recurAndTryCopy(sp, dp); err != nil { return err } } } return nil } func tryReflinkCopy(srcPath, dstPath string) error { err := reflink.Always(srcPath, dstPath) if errors.Is(err, reflink.ErrReflinkUnsupported) || errors.Is(err, reflink.ErrReflinkFailed) || isCrossDeviceError(err) { return errs.NotImplement } return err } ================================================ FILE: drivers/local/util_unix.go ================================================ //go:build !windows package local import ( "errors" "io/fs" "strings" "syscall" "github.com/OpenListTeam/OpenList/v4/internal/model" "golang.org/x/sys/unix" ) func isHidden(f fs.FileInfo, _ string) bool { return strings.HasPrefix(f.Name(), ".") } func getDiskUsage(path string) (model.DiskUsage, error) { var stat syscall.Statfs_t err := syscall.Statfs(path, &stat) if err != nil { return model.DiskUsage{}, err } total := int64(stat.Blocks) * int64(stat.Bsize) free := int64(stat.Bfree) * int64(stat.Bsize) return model.DiskUsage{ TotalSpace: total, UsedSpace: total - free, }, nil } func isCrossDeviceError(err error) bool { return errors.Is(err, unix.EXDEV) } ================================================ FILE: drivers/local/util_windows.go ================================================ //go:build windows package local import ( "errors" "io/fs" "path/filepath" "syscall" "github.com/OpenListTeam/OpenList/v4/internal/model" "golang.org/x/sys/windows" ) func isHidden(f fs.FileInfo, fullPath string) bool { filePath := filepath.Join(fullPath, f.Name()) namePtr, err := syscall.UTF16PtrFromString(filePath) if err != nil { return false } attrs, err := syscall.GetFileAttributes(namePtr) if err != nil { return false } return attrs&syscall.FILE_ATTRIBUTE_HIDDEN != 0 } func getDiskUsage(path string) (model.DiskUsage, error) { abs, err := filepath.Abs(path) if err != nil { return model.DiskUsage{}, err } root := filepath.VolumeName(abs) if len(root) != 2 || root[1] != ':' { return model.DiskUsage{}, errors.New("cannot get disk label") } var freeBytes, totalBytes, totalFreeBytes uint64 err = windows.GetDiskFreeSpaceEx( windows.StringToUTF16Ptr(root), &freeBytes, &totalBytes, &totalFreeBytes, ) if err != nil { return model.DiskUsage{}, err } return model.DiskUsage{ TotalSpace: int64(totalBytes), UsedSpace: int64(totalBytes - freeBytes), }, nil } func isCrossDeviceError(err error) bool { return errors.Is(err, windows.ERROR_NOT_SAME_DEVICE) } ================================================ FILE: drivers/mediafire/driver.go ================================================ package mediafire /* Package mediafire Author: Da3zKi7 Date: 2025-09-11 D@' 3z K!7 - The King Of Cracking Modifications by ILoveScratch2 Date: 2025-09-21 Date: 2025-09-26 Final opts by @Suyunjing @j2rong4cn @KirCute @Da3zKi7 */ import ( "context" "fmt" "math/rand" "net/http" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "golang.org/x/time/rate" ) type Mediafire struct { model.Storage Addition cron *cron.Cron actionToken string limiter *rate.Limiter appBase string apiBase string hostBase string maxRetries int secChUa string secChUaPlatform string userAgent string } func (d *Mediafire) Config() driver.Config { return config } func (d *Mediafire) GetAddition() driver.Additional { return &d.Addition } // Init initializes the MediaFire driver with session token and cookie validation func (d *Mediafire) Init(ctx context.Context) error { if d.Cookie == "" { return fmt.Errorf("Init :: [MediaFire] {critical} missing Cookie") } // If SessionToken is empty, try to get it from cookie if d.SessionToken == "" { if _, err := d.getSessionToken(ctx); err != nil { return fmt.Errorf("Init :: [MediaFire] {critical} failed to get session token from cookie: %w", err) } } // Setup rate limiter if rate limit is configured if d.LimitRate > 0 { d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) } // Validate and refresh session token if needed if _, err := d.getSessionToken(ctx); err != nil { d.renewToken(ctx) // Avoids 10 mins token expiry (6- 9) num := rand.Intn(4) + 6 d.cron = cron.NewCron(time.Minute * time.Duration(num)) d.cron.Do(func() { // Crazy, but working way to refresh session token d.renewToken(ctx) }) } return nil } // Drop cleans up driver resources func (d *Mediafire) Drop(ctx context.Context) error { // Clear cached resources d.actionToken = "" if d.cron != nil { d.cron.Stop() d.cron = nil } return nil } // List retrieves files and folders from the specified directory func (d *Mediafire) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(ctx, dir.GetID()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return d.fileToObj(src), nil }) } // Link generates a direct download link for the specified file func (d *Mediafire) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { downloadUrl, err := d.getDirectDownloadLink(ctx, file.GetID()) if err != nil { return nil, err } res, err := base.NoRedirectClient.R().SetDoNotParseResponse(true).SetContext(ctx).Head(downloadUrl) if err != nil { return nil, err } defer func() { _ = res.RawBody().Close() }() if res.StatusCode() == 302 { downloadUrl = res.Header().Get("location") } return &model.Link{ URL: downloadUrl, Header: http.Header{ "Origin": []string{d.appBase}, "Referer": []string{d.appBase + "/"}, "sec-ch-ua": []string{d.secChUa}, "sec-ch-ua-platform": []string{d.secChUaPlatform}, "User-Agent": []string{d.userAgent}, }, }, nil } // MakeDir creates a new folder in the specified parent directory func (d *Mediafire) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { data := map[string]string{ "session_token": d.SessionToken, "response_format": "json", "parent_key": parentDir.GetID(), "foldername": dirName, } var resp MediafireFolderCreateResponse _, err := d.postForm(ctx, "/folder/create.php", data, &resp) if err != nil { return nil, err } if err := checkAPIResult(resp.Response.Result); err != nil { return nil, err } created, _ := time.Parse("2006-01-02T15:04:05Z", resp.Response.CreatedUTC) return &model.Object{ ID: resp.Response.FolderKey, Name: resp.Response.Name, Size: 0, Modified: created, Ctime: created, IsFolder: true, }, nil } // Move relocates a file or folder to a different parent directory func (d *Mediafire) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { var data map[string]string var endpoint string if srcObj.IsDir() { endpoint = "/folder/move.php" data = map[string]string{ "session_token": d.SessionToken, "response_format": "json", "folder_key_src": srcObj.GetID(), "folder_key_dst": dstDir.GetID(), } } else { endpoint = "/file/move.php" data = map[string]string{ "session_token": d.SessionToken, "response_format": "json", "quick_key": srcObj.GetID(), "folder_key": dstDir.GetID(), } } var resp MediafireMoveResponse _, err := d.postForm(ctx, endpoint, data, &resp) if err != nil { return nil, err } if err := checkAPIResult(resp.Response.Result); err != nil { return nil, err } return srcObj, nil } // Rename changes the name of a file or folder func (d *Mediafire) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { var data map[string]string var endpoint string if srcObj.IsDir() { endpoint = "/folder/update.php" data = map[string]string{ "session_token": d.SessionToken, "response_format": "json", "folder_key": srcObj.GetID(), "foldername": newName, } } else { endpoint = "/file/update.php" data = map[string]string{ "session_token": d.SessionToken, "response_format": "json", "quick_key": srcObj.GetID(), "filename": newName, } } var resp MediafireRenameResponse _, err := d.postForm(ctx, endpoint, data, &resp) if err != nil { return nil, err } if err := checkAPIResult(resp.Response.Result); err != nil { return nil, err } return &model.Object{ ID: srcObj.GetID(), Name: newName, Size: srcObj.GetSize(), Modified: srcObj.ModTime(), Ctime: srcObj.CreateTime(), IsFolder: srcObj.IsDir(), }, nil } // Copy creates a duplicate of a file or folder in the specified destination directory func (d *Mediafire) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { var data map[string]string var endpoint string if srcObj.IsDir() { endpoint = "/folder/copy.php" data = map[string]string{ "session_token": d.SessionToken, "response_format": "json", "folder_key_src": srcObj.GetID(), "folder_key_dst": dstDir.GetID(), } } else { endpoint = "/file/copy.php" data = map[string]string{ "session_token": d.SessionToken, "response_format": "json", "quick_key": srcObj.GetID(), "folder_key": dstDir.GetID(), } } var resp MediafireCopyResponse _, err := d.postForm(ctx, endpoint, data, &resp) if err != nil { return nil, err } if err := checkAPIResult(resp.Response.Result); err != nil { return nil, err } var newID string if srcObj.IsDir() { if len(resp.Response.NewFolderKeys) > 0 { newID = resp.Response.NewFolderKeys[0] } } else { if len(resp.Response.NewQuickKeys) > 0 { newID = resp.Response.NewQuickKeys[0] } } return &model.Object{ ID: newID, Name: srcObj.GetName(), Size: srcObj.GetSize(), Modified: srcObj.ModTime(), Ctime: srcObj.CreateTime(), IsFolder: srcObj.IsDir(), }, nil } // Remove deletes a file or folder permanently func (d *Mediafire) Remove(ctx context.Context, obj model.Obj) error { var data map[string]string var endpoint string if obj.IsDir() { endpoint = "/folder/delete.php" data = map[string]string{ "session_token": d.SessionToken, "response_format": "json", "folder_key": obj.GetID(), } } else { endpoint = "/file/delete.php" data = map[string]string{ "session_token": d.SessionToken, "response_format": "json", "quick_key": obj.GetID(), } } var resp MediafireRemoveResponse _, err := d.postForm(ctx, endpoint, data, &resp) if err != nil { return err } return checkAPIResult(resp.Response.Result) } // Put uploads a file to the specified directory with support for resumable upload and quick upload func (d *Mediafire) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { fileHash := file.GetHash().GetHash(utils.SHA256) var err error // Try to use existing hash first, cache only if necessary if len(fileHash) != utils.SHA256.Width { _, fileHash, err = stream.CacheFullAndHash(file, &up, utils.SHA256) if err != nil { return nil, err } } checkResp, err := d.uploadCheck(ctx, file.GetName(), file.GetSize(), fileHash, dstDir.GetID()) if err != nil { return nil, err } if checkResp.Response.HashExists == "yes" && checkResp.Response.InAccount == "yes" { up(100.0) existingFile, err := d.getExistingFileInfo(ctx, fileHash, file.GetName(), dstDir.GetID()) if err == nil && existingFile != nil { // File exists, return existing file info return &model.Object{ ID: existingFile.GetID(), Name: file.GetName(), Size: file.GetSize(), }, nil } // If getExistingFileInfo fails, log and continue with normal upload // This ensures upload doesn't fail due to search issues } var pollKey string if checkResp.Response.ResumableUpload.AllUnitsReady != "yes" { pollKey, err = d.uploadUnits(ctx, file, checkResp, file.GetName(), fileHash, dstDir.GetID(), up) if err != nil { return nil, err } } else { pollKey = checkResp.Response.ResumableUpload.UploadKey } defer up(100.0) pollResp, err := d.pollUpload(ctx, pollKey) if err != nil { return nil, err } return &model.Object{ ID: pollResp.Response.Doupload.QuickKey, Name: file.GetName(), Size: file.GetSize(), }, nil } func (d *Mediafire) GetDetails(ctx context.Context) (*model.StorageDetails, error) { data := map[string]string{ "session_token": d.SessionToken, "response_format": "json", } var resp MediafireUserInfoResponse _, err := d.postForm(ctx, "/user/get_info.php", data, &resp) if err != nil { return nil, err } used, err := strconv.ParseInt(resp.Response.UserInfo.UsedStorageSize, 10, 64) if err != nil { return nil, err } total, err := strconv.ParseInt(resp.Response.UserInfo.StorageLimit, 10, 64) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: used, }, }, nil } var _ driver.Driver = (*Mediafire)(nil) ================================================ FILE: drivers/mediafire/meta.go ================================================ package mediafire /* Package mediafire Author: Da3zKi7 Date: 2025-09-11 D@' 3z K!7 - The King Of Cracking Modifications by ILoveScratch2 Date: 2025-09-21 Date: 2025-09-26 Final opts by @Suyunjing @j2rong4cn @KirCute @Da3zKi7 */ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath //driver.RootID SessionToken string `json:"session_token" required:"false" type:"string" help:"Optional for MediaFire API, can be auto-acquired from cookie"` Cookie string `json:"cookie" required:"true" type:"string" help:"Required for MediaFire API authentication"` OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` ChunkSize int64 `json:"chunk_size" type:"number" default:"100"` UploadThreads int `json:"upload_threads" type:"number" default:"3" help:"concurrent upload threads"` LimitRate float64 `json:"limit_rate" type:"float" default:"2" help:"limit all api request rate ([limit]r/1s)"` } var config = driver.Config{ Name: "MediaFire", LocalSort: false, OnlyProxy: false, NoCache: false, NoUpload: false, NeedMs: false, DefaultRoot: "/", CheckStatus: false, Alert: "", NoOverwriteUpload: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Mediafire{ appBase: "https://app.mediafire.com", apiBase: "https://www.mediafire.com/api/1.5", hostBase: "https://www.mediafire.com", maxRetries: 3, userAgent: base.UserAgent, } }) } ================================================ FILE: drivers/mediafire/types.go ================================================ package mediafire /* Package mediafire Author: Da3zKi7 Date: 2025-09-11 D@' 3z K!7 - The King Of Cracking */ type MediafireRenewTokenResponse struct { Response struct { Action string `json:"action"` SessionToken string `json:"session_token"` Result string `json:"result"` CurrentAPIVersion string `json:"current_api_version"` } `json:"response"` } type MediafireResponse struct { Response struct { Action string `json:"action"` FolderContent struct { ChunkSize string `json:"chunk_size"` ContentType string `json:"content_type"` ChunkNumber string `json:"chunk_number"` FolderKey string `json:"folderkey"` Folders []MediafireFolder `json:"folders,omitempty"` Files []MediafireFile `json:"files,omitempty"` MoreChunks string `json:"more_chunks"` } `json:"folder_content"` Result string `json:"result"` } `json:"response"` } type MediafireFolder struct { FolderKey string `json:"folderkey"` Name string `json:"name"` Created string `json:"created"` CreatedUTC string `json:"created_utc"` } type MediafireFile struct { QuickKey string `json:"quickkey"` Filename string `json:"filename"` Size string `json:"size"` Created string `json:"created"` CreatedUTC string `json:"created_utc"` MimeType string `json:"mimetype"` } type File struct { ID string Name string Size int64 CreatedUTC string IsFolder bool } type FolderContentResponse struct { Folders []MediafireFolder Files []MediafireFile MoreChunks bool } type MediafireLinksResponse struct { Response struct { Action string `json:"action"` Links []struct { QuickKey string `json:"quickkey"` View string `json:"view"` NormalDownload string `json:"normal_download"` OneTime struct { Download string `json:"download"` View string `json:"view"` } `json:"one_time"` } `json:"links"` OneTimeKeyRequestCount string `json:"one_time_key_request_count"` OneTimeKeyRequestMaxCount string `json:"one_time_key_request_max_count"` Result string `json:"result"` CurrentAPIVersion string `json:"current_api_version"` } `json:"response"` } type MediafireDirectDownloadResponse struct { Response struct { Action string `json:"action"` Links []struct { QuickKey string `json:"quickkey"` DirectDownload string `json:"direct_download"` } `json:"links"` DirectDownloadFreeBandwidth string `json:"direct_download_free_bandwidth"` Result string `json:"result"` CurrentAPIVersion string `json:"current_api_version"` } `json:"response"` } type MediafireFolderCreateResponse struct { Response struct { Action string `json:"action"` FolderKey string `json:"folder_key"` UploadKey string `json:"upload_key"` ParentFolderKey string `json:"parent_folderkey"` Name string `json:"name"` Description string `json:"description"` Created string `json:"created"` CreatedUTC string `json:"created_utc"` Privacy string `json:"privacy"` FileCount string `json:"file_count"` FolderCount string `json:"folder_count"` Revision string `json:"revision"` DropboxEnabled string `json:"dropbox_enabled"` Flag string `json:"flag"` Result string `json:"result"` CurrentAPIVersion string `json:"current_api_version"` NewDeviceRevision int `json:"new_device_revision"` } `json:"response"` } type MediafireMoveResponse struct { Response struct { Action string `json:"action"` Asynchronous string `json:"asynchronous,omitempty"` NewNames []string `json:"new_names"` Result string `json:"result"` CurrentAPIVersion string `json:"current_api_version"` NewDeviceRevision int `json:"new_device_revision"` } `json:"response"` } type MediafireRenameResponse struct { Response struct { Action string `json:"action"` Asynchronous string `json:"asynchronous,omitempty"` Result string `json:"result"` CurrentAPIVersion string `json:"current_api_version"` NewDeviceRevision int `json:"new_device_revision"` } `json:"response"` } type MediafireCopyResponse struct { Response struct { Action string `json:"action"` Asynchronous string `json:"asynchronous,omitempty"` NewQuickKeys []string `json:"new_quickkeys,omitempty"` NewFolderKeys []string `json:"new_folderkeys,omitempty"` SkippedCount string `json:"skipped_count,omitempty"` OtherCount string `json:"other_count,omitempty"` Result string `json:"result"` CurrentAPIVersion string `json:"current_api_version"` NewDeviceRevision int `json:"new_device_revision"` } `json:"response"` } type MediafireRemoveResponse struct { Response struct { Action string `json:"action"` Asynchronous string `json:"asynchronous,omitempty"` Result string `json:"result"` CurrentAPIVersion string `json:"current_api_version"` NewDeviceRevision int `json:"new_device_revision"` } `json:"response"` } type MediafireCheckResponse struct { Response struct { Action string `json:"action"` HashExists string `json:"hash_exists"` InAccount string `json:"in_account"` InFolder string `json:"in_folder"` FileExists string `json:"file_exists"` ResumableUpload struct { AllUnitsReady string `json:"all_units_ready"` NumberOfUnits string `json:"number_of_units"` UnitSize string `json:"unit_size"` Bitmap struct { Count string `json:"count"` Words []string `json:"words"` } `json:"bitmap"` UploadKey string `json:"upload_key"` } `json:"resumable_upload"` AvailableSpace string `json:"available_space"` UsedStorageSize string `json:"used_storage_size"` StorageLimit string `json:"storage_limit"` StorageLimitExceeded string `json:"storage_limit_exceeded"` UploadURL struct { Simple string `json:"simple"` SimpleFallback string `json:"simple_fallback"` Resumable string `json:"resumable"` ResumableFallback string `json:"resumable_fallback"` } `json:"upload_url"` Result string `json:"result"` CurrentAPIVersion string `json:"current_api_version"` } `json:"response"` } type MediafireActionTokenResponse struct { Response struct { Action string `json:"action"` ActionToken string `json:"action_token"` Result string `json:"result"` CurrentAPIVersion string `json:"current_api_version"` } `json:"response"` } type MediafirePollResponse struct { Response struct { Action string `json:"action"` Doupload struct { Result string `json:"result"` Status string `json:"status"` Description string `json:"description"` QuickKey string `json:"quickkey"` Hash string `json:"hash"` Filename string `json:"filename"` Size string `json:"size"` Created string `json:"created"` CreatedUTC string `json:"created_utc"` Revision string `json:"revision"` } `json:"doupload"` Result string `json:"result"` CurrentAPIVersion string `json:"current_api_version"` } `json:"response"` } type MediafireFileSearchResponse struct { Response struct { Action string `json:"action"` FileInfo []File `json:"file_info"` Result string `json:"result"` CurrentAPIVersion string `json:"current_api_version"` } `json:"response"` } type MediafireUserInfoResponse struct { Response struct { Action string `json:"action"` UserInfo struct { Email string `json:"string"` DisplayName string `json:"display_name"` UsedStorageSize string `json:"used_storage_size"` StorageLimit string `json:"storage_limit"` } `json:"user_info"` Result string `json:"result"` CurrentAPIVersion string `json:"current_api_version"` } `json:"response"` } ================================================ FILE: drivers/mediafire/util.go ================================================ package mediafire /* Package mediafire Author: Da3zKi7 Date: 2025-09-11 D@' 3z K!7 - The King Of Cracking Modifications by ILoveScratch2 Date: 2025-09-21 Date: 2025-09-26 Final opts by @Suyunjing @j2rong4cn @KirCute @Da3zKi7 */ import ( "compress/gzip" "context" "encoding/json" "fmt" "io" "net/http" "strconv" "strings" "sync" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" ) // checkAPIResult validates MediaFire API response result and returns error if not successful func checkAPIResult(result string) error { if result != "Success" { return fmt.Errorf("MediaFire API error: %s", result) } return nil } // getSessionToken retrieves and validates session token from MediaFire func (d *Mediafire) getSessionToken(ctx context.Context) (string, error) { if d.limiter != nil { if err := d.limiter.Wait(ctx); err != nil { return "", fmt.Errorf("rate limit wait failed: %w", err) } } tokenURL := d.hostBase + "/application/get_session_token.php" req, err := http.NewRequestWithContext(ctx, http.MethodPost, tokenURL, nil) if err != nil { return "", err } req.Header.Set("Accept", "*/*") req.Header.Set("Accept-Encoding", "gzip") req.Header.Set("Accept-Language", "en-US,en;q=0.9") req.Header.Set("Content-Length", "0") req.Header.Set("Cookie", d.Cookie) req.Header.Set("DNT", "1") req.Header.Set("Origin", d.hostBase) req.Header.Set("Priority", "u=1, i") req.Header.Set("Referer", (d.hostBase + "/")) req.Header.Set("Sec-Ch-Ua", d.secChUa) req.Header.Set("Sec-Ch-Ua-Mobile", "?0") req.Header.Set("Sec-Ch-Ua-Platform", d.secChUaPlatform) req.Header.Set("Sec-Fetch-Dest", "empty") req.Header.Set("Sec-Fetch-Mode", "cors") req.Header.Set("Sec-Fetch-Site", "same-site") req.Header.Set("User-Agent", d.userAgent) // req.Header.Set("Connection", "keep-alive") resp, err := base.HttpClient.Do(req) if err != nil { return "", err } defer resp.Body.Close() var body []byte // Handle gzip decompression if needed if resp.Header.Get("Content-Encoding") == "gzip" { gzipReader, err := gzip.NewReader(resp.Body) if err != nil { return "", fmt.Errorf("failed to create gzip reader: %w", err) } defer gzipReader.Close() body, _ = io.ReadAll(gzipReader) } else { body, err = io.ReadAll(resp.Body) } if err != nil { return "", err } // fmt.Printf("getSessionToken :: Raw response: %s\n", string(body)) // fmt.Printf("getSessionToken :: Parsed response: %+v\n", resp) var tokenResp struct { Response struct { SessionToken string `json:"session_token"` } `json:"response"` } if resp.StatusCode == 200 { if err := json.Unmarshal(body, &tokenResp); err != nil { return "", err } if tokenResp.Response.SessionToken == "" { return "", fmt.Errorf("empty session token received") } cookieMap := make(map[string]string) for _, cookie := range resp.Cookies() { cookieMap[cookie.Name] = cookie.Value } if len(cookieMap) > 0 { var cookies []string for name, value := range cookieMap { cookies = append(cookies, fmt.Sprintf("%s=%s", name, value)) } d.Cookie = strings.Join(cookies, "; ") op.MustSaveDriverStorage(d) // fmt.Printf("getSessionToken :: Captured cookies: %s\n", d.Cookie) } } else { return "", fmt.Errorf("getSessionToken :: failed to get session token, status code: %d", resp.StatusCode) } d.SessionToken = tokenResp.Response.SessionToken // fmt.Printf("Init :: Obtain Session Token %v", d.SessionToken) op.MustSaveDriverStorage(d) return d.SessionToken, nil } // renewToken refreshes the current session token when expired func (d *Mediafire) renewToken(ctx context.Context) error { query := map[string]string{ "session_token": d.SessionToken, "response_format": "json", } var resp MediafireRenewTokenResponse _, err := d.postForm(ctx, "/user/renew_session_token.php", query, &resp) if err != nil { return fmt.Errorf("failed to renew token: %w", err) } // fmt.Printf("getInfo :: Raw response: %s\n", string(body)) // fmt.Printf("getInfo :: Parsed response: %+v\n", resp) if resp.Response.Result != "Success" { return fmt.Errorf("MediaFire token renewal failed: %s", resp.Response.Result) } d.SessionToken = resp.Response.SessionToken // fmt.Printf("Init :: Renew Session Token: %s", resp.Response.Result) op.MustSaveDriverStorage(d) return nil } func (d *Mediafire) getFiles(ctx context.Context, folderKey string) ([]File, error) { // Pre-allocate slice with reasonable capacity to reduce memory allocations files := make([]File, 0, d.ChunkSize*2) // Estimate: ChunkSize for files + folders hasMore := true chunkNumber := 1 for hasMore { resp, err := d.getFolderContent(ctx, folderKey, chunkNumber) if err != nil { return nil, err } // Process folders and files in single loop to improve cache locality totalItems := len(resp.Folders) + len(resp.Files) if cap(files)-len(files) < totalItems { // Grow slice if needed newFiles := make([]File, len(files), len(files)+totalItems+int(d.ChunkSize)) copy(newFiles, files) files = newFiles } for _, folder := range resp.Folders { files = append(files, File{ ID: folder.FolderKey, Name: folder.Name, Size: 0, CreatedUTC: folder.CreatedUTC, IsFolder: true, }) } for _, file := range resp.Files { size, _ := strconv.ParseInt(file.Size, 10, 64) files = append(files, File{ ID: file.QuickKey, Name: file.Filename, Size: size, CreatedUTC: file.CreatedUTC, IsFolder: false, }) } hasMore = resp.MoreChunks chunkNumber++ } return files, nil } func (d *Mediafire) getFolderContent(ctx context.Context, folderKey string, chunkNumber int) (*FolderContentResponse, error) { foldersResp, err := d.getFolderContentByType(ctx, folderKey, "folders", chunkNumber) if err != nil { return nil, err } filesResp, err := d.getFolderContentByType(ctx, folderKey, "files", chunkNumber) if err != nil { return nil, err } return &FolderContentResponse{ Folders: foldersResp.Response.FolderContent.Folders, Files: filesResp.Response.FolderContent.Files, MoreChunks: foldersResp.Response.FolderContent.MoreChunks == "yes" || filesResp.Response.FolderContent.MoreChunks == "yes", }, nil } func (d *Mediafire) getFolderContentByType(ctx context.Context, folderKey, contentType string, chunkNumber int) (*MediafireResponse, error) { data := map[string]string{ "session_token": d.SessionToken, "response_format": "json", "folder_key": folderKey, "content_type": contentType, "chunk": strconv.Itoa(chunkNumber), "chunk_size": strconv.FormatInt(d.ChunkSize, 10), "details": "yes", "order_direction": d.OrderDirection, "order_by": d.OrderBy, "filter": "", } var resp MediafireResponse _, err := d.postForm(ctx, "/folder/get_content.php", data, &resp) if err != nil { return nil, err } if err := checkAPIResult(resp.Response.Result); err != nil { return nil, err } return &resp, nil } // fileToObj converts MediaFire file data to model.ObjThumb with thumbnail support func (d *Mediafire) fileToObj(f File) *model.ObjThumb { created, _ := time.Parse("2006-01-02T15:04:05Z", f.CreatedUTC) var thumbnailURL string if !f.IsFolder && f.ID != "" { thumbnailURL = d.hostBase + "/convkey/acaa/" + f.ID + "3g.jpg" } return &model.ObjThumb{ Object: model.Object{ ID: f.ID, // Path: "", Name: f.Name, Size: f.Size, Modified: created, Ctime: created, IsFolder: f.IsFolder, }, Thumbnail: model.Thumbnail{ Thumbnail: thumbnailURL, }, } } func (d *Mediafire) setCommonHeaders(req *resty.Request) { req.SetHeaders(map[string]string{ "Cookie": d.Cookie, "User-Agent": d.userAgent, "Origin": d.appBase, "Referer": d.appBase + "/", }) } // apiRequest performs HTTP request to MediaFire API with rate limiting and common headers func (d *Mediafire) apiRequest(ctx context.Context, method, endpoint string, queryParams, formData map[string]string, resp interface{}) ([]byte, error) { if d.limiter != nil { if err := d.limiter.Wait(ctx); err != nil { return nil, fmt.Errorf("rate limit wait failed: %w", err) } } req := base.RestyClient.R() req.SetContext(ctx) d.setCommonHeaders(req) // Set query parameters for GET requests if queryParams != nil { req.SetQueryParams(queryParams) } // Set form data for POST requests if formData != nil { req.SetFormData(formData) req.SetHeader("Content-Type", "application/x-www-form-urlencoded") } // Set response object if provided if resp != nil { req.SetResult(resp) } var res *resty.Response var err error // Execute request based on method switch method { case "GET": res, err = req.Get(d.apiBase + endpoint) case "POST": res, err = req.Post(d.apiBase + endpoint) default: return nil, fmt.Errorf("unsupported HTTP method: %s", method) } if err != nil { return nil, err } return res.Body(), nil } func (d *Mediafire) getForm(ctx context.Context, endpoint string, query map[string]string, resp interface{}) ([]byte, error) { return d.apiRequest(ctx, "GET", endpoint, query, nil, resp) } func (d *Mediafire) postForm(ctx context.Context, endpoint string, data map[string]string, resp interface{}) ([]byte, error) { return d.apiRequest(ctx, "POST", endpoint, nil, data, resp) } func (d *Mediafire) getDirectDownloadLink(ctx context.Context, fileID string) (string, error) { data := map[string]string{ "session_token": d.SessionToken, "quick_key": fileID, "link_type": "direct_download", "response_format": "json", } var resp MediafireDirectDownloadResponse _, err := d.getForm(ctx, "/file/get_links.php", data, &resp) if err != nil { return "", err } if err := checkAPIResult(resp.Response.Result); err != nil { return "", err } if len(resp.Response.Links) == 0 { return "", fmt.Errorf("no download links found") } return resp.Response.Links[0].DirectDownload, nil } func (d *Mediafire) uploadCheck(ctx context.Context, filename string, filesize int64, filehash, folderKey string) (*MediafireCheckResponse, error) { actionToken, err := d.getActionToken(ctx) if err != nil { return nil, fmt.Errorf("failed to get action token: %w", err) } query := map[string]string{ "session_token": actionToken, /* d.SessionToken */ "filename": filename, "size": strconv.FormatInt(filesize, 10), "hash": filehash, "folder_key": folderKey, "resumable": "yes", "response_format": "json", } var resp MediafireCheckResponse _, err = d.postForm(ctx, "/upload/check.php", query, &resp) if err != nil { return nil, err } // fmt.Printf("uploadCheck :: Raw response: %s\n", string(body)) // fmt.Printf("uploadCheck :: Parsed response: %+v\n", resp) // fmt.Printf("uploadCheck :: ResumableUpload section: %+v\n", resp.Response.ResumableUpload) // fmt.Printf("uploadCheck :: Upload key specifically: '%s'\n", resp.Response.ResumableUpload.UploadKey) if err := checkAPIResult(resp.Response.Result); err != nil { return nil, err } return &resp, nil } func (d *Mediafire) uploadUnits(ctx context.Context, file model.FileStreamer, checkResp *MediafireCheckResponse, filename, fileHash, folderKey string, up driver.UpdateProgress) (string, error) { unitSize, _ := strconv.ParseInt(checkResp.Response.ResumableUpload.UnitSize, 10, 64) numUnits, _ := strconv.Atoi(checkResp.Response.ResumableUpload.NumberOfUnits) uploadKey := checkResp.Response.ResumableUpload.UploadKey stringWords := checkResp.Response.ResumableUpload.Bitmap.Words intWords := make([]int, 0, len(stringWords)) for _, word := range stringWords { if intWord, err := strconv.Atoi(word); err == nil { intWords = append(intWords, intWord) } } // Intelligent buffer sizing for large files bufferSize := int(unitSize) fileSize := file.GetSize() // Split in chunks if fileSize > d.ChunkSize*1024*1024 { // Large, use ChunkSize (default = 100MB) bufferSize = min(int(fileSize), int(d.ChunkSize)*1024*1024) } else if fileSize > 10*1024*1024 { // Medium, use full file size for concurrent access bufferSize = int(fileSize) } // Create stream section reader for efficient chunking ss, err := stream.NewStreamSectionReader(file, bufferSize, &up) if err != nil { return "", err } // Cal minimal parallel upload threads, allows MediaFire resumable upload to rule it over custom value // If file is big, likely will respect d.UploadThreads instead of MediaFire's suggestion i.e. 5 threads thread := min(numUnits, d.UploadThreads) // Create ordered group for sequential upload processing with retry logic threadG, uploadCtx := errgroup.NewOrderedGroupWithContext(ctx, thread, retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) var finalUploadKey string var keyMutex sync.Mutex fileSize = file.GetSize() for unitID := range numUnits { if utils.IsCanceled(uploadCtx) { break } start := int64(unitID) * unitSize size := unitSize if start+size > fileSize { size = fileSize - start } var reader io.ReadSeeker var unitHash string // Use lifecycle pattern for proper resource management threadG.GoWithLifecycle(errgroup.Lifecycle{ Before: func(ctx context.Context) (err error) { // Skip already uploaded units if d.isUnitUploaded(intWords, unitID) { return ss.DiscardSection(start, size) } reader, err = ss.GetSectionReader(start, size) return }, Do: func(ctx context.Context) (err error) { if reader == nil { return nil // Skip if reader is not initialized (already uploaded) } reader.Seek(0, io.SeekStart) if unitHash == "" { var err error unitHash, err = utils.HashReader(utils.SHA256, reader) if err != nil { return err } reader.Seek(0, io.SeekStart) } // Perform upload actionToken, err := d.getActionToken(ctx) if err != nil { return err } if d.limiter != nil { if err := d.limiter.Wait(ctx); err != nil { return fmt.Errorf("rate limit wait failed: %w", err) } } url := d.apiBase + "/upload/resumable.php" req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, driver.NewLimitedUploadStream(ctx, reader)) if err != nil { return err } q := req.URL.Query() q.Add("folder_key", folderKey) q.Add("response_format", "json") q.Add("session_token", actionToken) q.Add("key", uploadKey) req.URL.RawQuery = q.Encode() req.Header.Set("x-filehash", fileHash) req.Header.Set("x-filesize", strconv.FormatInt(fileSize, 10)) req.Header.Set("x-unit-id", strconv.Itoa(unitID)) req.Header.Set("x-unit-size", strconv.FormatInt(size, 10)) req.Header.Set("x-unit-hash", unitHash) req.Header.Set("x-filename", filename) req.Header.Set("Content-Type", "application/octet-stream") req.ContentLength = size /* fmt.Printf("Debug resumable upload request:\n") fmt.Printf(" URL: %s\n", req.URL.String()) fmt.Printf(" Headers: %+v\n", req.Header) fmt.Printf(" Unit ID: %d\n", unitID) fmt.Printf(" Unit Size: %d\n", len(unitData)) fmt.Printf(" Upload Key: %s\n", uploadKey) fmt.Printf(" Action Token: %s\n", actionToken) */ res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() body, err := io.ReadAll(res.Body) if err != nil { return fmt.Errorf("failed to read response body: %v", err) } // fmt.Printf("MediaFire resumable upload response (status %d): %s\n", res.StatusCode, string(body)) var uploadResp struct { Response struct { Doupload struct { Key string `json:"key"` } `json:"doupload"` Result string `json:"result"` } `json:"response"` } if err := json.Unmarshal(body, &uploadResp); err != nil { return fmt.Errorf("failed to parse response: %v", err) } if res.StatusCode != 200 { return fmt.Errorf("resumable upload failed with status %d", res.StatusCode) } // Thread-safe update of final upload key keyMutex.Lock() finalUploadKey = uploadResp.Response.Doupload.Key keyMutex.Unlock() up(float64(threadG.Success()+1) * 100 / float64(numUnits+1)) return nil }, After: func(err error) { if reader != nil { // Cleanup resources ss.FreeSectionReader(reader) } }, }) } if err := threadG.Wait(); err != nil { return "", err } return finalUploadKey, nil } /*func (d *Mediafire) uploadSingleUnit(ctx context.Context, file model.FileStreamer, unitID int, unitSize int64, fileHash, filename, uploadKey, folderKey string, fileSize int64) (string, error) { start := int64(unitID) * unitSize size := unitSize if start+size > fileSize { size = fileSize - start } unitData := make([]byte, size) _, err := file.Read(unitData) if err != nil { return "", err } return d.resumableUpload(ctx, folderKey, uploadKey, unitData, unitID, fileHash, filename, fileSize) }*/ func (d *Mediafire) getActionToken(ctx context.Context) (string, error) { if d.actionToken != "" { return d.actionToken, nil } data := map[string]string{ "type": "upload", "lifespan": "1440", "response_format": "json", "session_token": d.SessionToken, } var resp MediafireActionTokenResponse _, err := d.postForm(ctx, "/user/get_action_token.php", data, &resp) if err != nil { return "", err } if resp.Response.Result != "Success" { return "", fmt.Errorf("MediaFire action token failed: %s", resp.Response.Result) } return resp.Response.ActionToken, nil } func (d *Mediafire) pollUpload(ctx context.Context, key string) (*MediafirePollResponse, error) { actionToken, err := d.getActionToken(ctx) if err != nil { return nil, fmt.Errorf("failed to get action token: %w", err) } // fmt.Printf("Debug Key: %+v\n", key) query := map[string]string{ "key": key, "response_format": "json", "session_token": actionToken, /* d.SessionToken */ } var resp MediafirePollResponse _, err = d.postForm(ctx, "/upload/poll_upload.php", query, &resp) if err != nil { return nil, err } // fmt.Printf("pollUpload :: Raw response: %s\n", string(body)) // fmt.Printf("pollUpload :: Parsed response: %+v\n", resp) // fmt.Printf("pollUpload :: Debug Result: %+v\n", resp.Response.Result) if err := checkAPIResult(resp.Response.Result); err != nil { return nil, err } return &resp, nil } func (d *Mediafire) isUnitUploaded(words []int, unitID int) bool { wordIndex := unitID / 16 bitIndex := unitID % 16 if wordIndex >= len(words) { return false } return (words[wordIndex]>>bitIndex)&1 == 1 } func (d *Mediafire) getExistingFileInfo(ctx context.Context, fileHash, filename, folderKey string) (*model.ObjThumb, error) { // First try to find by hash directly (most efficient) if fileInfo, err := d.getFileByHash(ctx, fileHash); err == nil && fileInfo != nil { return fileInfo, nil } // If hash search fails, search in the target folder // This is a fallback method in case the file exists but hash search doesn't work files, err := d.getFiles(ctx, folderKey) if err != nil { return nil, err } for _, file := range files { if file.Name == filename && !file.IsFolder { return d.fileToObj(file), nil } } return nil, fmt.Errorf("existing file not found") } func (d *Mediafire) getFileByHash(ctx context.Context, hash string) (*model.ObjThumb, error) { query := map[string]string{ "session_token": d.SessionToken, "response_format": "json", "hash": hash, } var resp MediafireFileSearchResponse _, err := d.postForm(ctx, "/file/get_info.php", query, &resp) if err != nil { return nil, err } if resp.Response.Result != "Success" { return nil, fmt.Errorf("MediaFire file search failed: %s", resp.Response.Result) } if len(resp.Response.FileInfo) == 0 { return nil, fmt.Errorf("file not found by hash") } file := resp.Response.FileInfo[0] return d.fileToObj(file), nil } ================================================ FILE: drivers/mediatrack/driver.go ================================================ package mediatrack import ( "context" "crypto/md5" "encoding/hex" "fmt" "io" "net/http" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" "github.com/google/uuid" log "github.com/sirupsen/logrus" ) type MediaTrack struct { model.Storage Addition } func (d *MediaTrack) Config() driver.Config { return config } func (d *MediaTrack) GetAddition() driver.Additional { return &d.Addition } func (d *MediaTrack) Init(ctx context.Context) error { _, err := d.request("https://kayle.api.mediatrack.cn/users", http.MethodGet, nil, nil) return err } func (d *MediaTrack) Drop(ctx context.Context) error { return nil } func (d *MediaTrack) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(dir.GetID()) if err != nil { return nil, err } return utils.SliceConvert(files, func(f File) (model.Obj, error) { size, _ := strconv.ParseInt(f.Size, 10, 64) thumb := "" if f.File != nil && f.File.Cover != "" { thumb = "https://nano.mtres.cn/" + f.File.Cover } return &Object{ Object: model.Object{ ID: f.ID, Name: f.Title, Modified: f.UpdatedAt, IsFolder: f.File == nil, Size: size, }, Thumbnail: model.Thumbnail{Thumbnail: thumb}, ParentID: dir.GetID(), }, nil }) } func (d *MediaTrack) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { url := fmt.Sprintf("https://kayn.api.mediatrack.cn/v1/download_token/asset?asset_id=%s&source_type=project&password=&source_id=%s", file.GetID(), d.ProjectID) log.Debugf("media track url: %s", url) body, err := d.request(url, http.MethodGet, nil, nil) if err != nil { return nil, err } token := utils.Json.Get(body, "data", "token").ToString() url = "https://kayn.api.mediatrack.cn/v1/download/redirect?token=" + token res, err := base.NoRedirectClient.R().Get(url) if err != nil { return nil, err } log.Debug(res.String()) link := model.Link{ URL: url, } log.Debugln("res code: ", res.StatusCode()) if res.StatusCode() == 302 { link.URL = res.Header().Get("location") expired := time.Duration(60) * time.Second link.Expiration = &expired } return &link, nil } func (d *MediaTrack) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { url := fmt.Sprintf("https://jayce.api.mediatrack.cn/v3/assets/%s/children", parentDir.GetID()) _, err := d.request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "type": 1, "title": dirName, }) }, nil) return err } func (d *MediaTrack) Move(ctx context.Context, srcObj, dstDir model.Obj) error { data := base.Json{ "parent_id": dstDir.GetID(), "ids": []string{srcObj.GetID()}, } url := "https://jayce.api.mediatrack.cn/v4/assets/batch/move" _, err := d.request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *MediaTrack) Rename(ctx context.Context, srcObj model.Obj, newName string) error { url := "https://jayce.api.mediatrack.cn/v3/assets/" + srcObj.GetID() data := base.Json{ "title": newName, } _, err := d.request(url, http.MethodPut, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *MediaTrack) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { data := base.Json{ "parent_id": dstDir.GetID(), "ids": []string{srcObj.GetID()}, } url := "https://jayce.api.mediatrack.cn/v4/assets/batch/clone" _, err := d.request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *MediaTrack) Remove(ctx context.Context, obj model.Obj) error { var parentID string if o, ok := obj.(*Object); ok { parentID = o.ParentID } else { return fmt.Errorf("obj is not local Object") } data := base.Json{ "origin_id": parentID, "ids": []string{obj.GetID()}, } url := "https://jayce.api.mediatrack.cn/v4/assets/batch/delete" _, err := d.request(url, http.MethodDelete, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { src := "assets/" + uuid.New().String() var resp UploadResp _, err := d.request("https://jayce.api.mediatrack.cn/v3/storage/tokens/asset", http.MethodGet, func(req *resty.Request) { req.SetQueryParam("src", src) }, &resp) if err != nil { return err } credential := resp.Data.Credentials cfg := &aws.Config{ Credentials: credentials.NewStaticCredentials(credential.TmpSecretID, credential.TmpSecretKey, credential.Token), Region: &resp.Data.Region, Endpoint: aws.String("cos.accelerate.myqcloud.com"), } s, err := session.NewSession(cfg) if err != nil { return err } tempFile, err := file.CacheFullAndWriter(&up, nil) if err != nil { return err } uploader := s3manager.NewUploader(s) if file.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { uploader.PartSize = file.GetSize() / (s3manager.MaxUploadParts - 1) } input := &s3manager.UploadInput{ Bucket: &resp.Data.Bucket, Key: &resp.Data.Object, Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: &driver.SimpleReaderWithSize{ Reader: tempFile, Size: file.GetSize(), }, UpdateProgress: up, }), } _, err = uploader.UploadWithContext(ctx, input) if err != nil { return err } url := fmt.Sprintf("https://jayce.api.mediatrack.cn/v3/assets/%s/children", dstDir.GetID()) _, err = tempFile.Seek(0, io.SeekStart) if err != nil { return err } h := md5.New() _, err = utils.CopyWithBuffer(h, tempFile) if err != nil { return err } hash := hex.EncodeToString(h.Sum(nil)) data := base.Json{ "category": 0, "description": file.GetName(), "hash": hash, "mime": file.GetMimetype(), "size": file.GetSize(), "src": src, "title": file.GetName(), "type": 0, } _, err = d.request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } var _ driver.Driver = (*MediaTrack)(nil) ================================================ FILE: drivers/mediatrack/meta.go ================================================ package mediatrack import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { AccessToken string `json:"access_token" required:"true"` ProjectID string `json:"project_id"` driver.RootID OrderBy string `json:"order_by" type:"select" options:"updated_at,title,size" default:"title"` OrderDesc bool `json:"order_desc"` } var config = driver.Config{ Name: "MediaTrack", } func init() { op.RegisterDriver(func() driver.Driver { return &MediaTrack{} }) } ================================================ FILE: drivers/mediatrack/types.go ================================================ package mediatrack import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type BaseResp struct { Status string `json:"status"` Message string `json:"message"` } type File struct { Category int `json:"category"` ChildAssets []interface{} `json:"childAssets"` CommentCount int `json:"comment_count"` CoverAsset interface{} `json:"cover_asset"` CoverAssetID string `json:"cover_asset_id"` CreatedAt time.Time `json:"created_at"` DeletedAt string `json:"deleted_at"` Description string `json:"description"` File *struct { Cover string `json:"cover"` Src string `json:"src"` } `json:"file"` //FileID string `json:"file_id"` ID string `json:"id"` Size string `json:"size"` Thumbnails []interface{} `json:"thumbnails"` Title string `json:"title"` UpdatedAt time.Time `json:"updated_at"` } type ChildrenResp struct { Status string `json:"status"` Data struct { Total int `json:"total"` Assets []File `json:"assets"` } `json:"data"` Path string `json:"path"` TraceID string `json:"trace_id"` RequestID string `json:"requestId"` } type UploadResp struct { Status string `json:"status"` Data struct { Credentials struct { TmpSecretID string `json:"TmpSecretId"` TmpSecretKey string `json:"TmpSecretKey"` Token string `json:"Token"` ExpiredTime int `json:"ExpiredTime"` Expiration time.Time `json:"Expiration"` StartTime int `json:"StartTime"` } `json:"credentials"` Object string `json:"object"` Bucket string `json:"bucket"` Region string `json:"region"` URL string `json:"url"` Size string `json:"size"` } `json:"data"` Path string `json:"path"` TraceID string `json:"trace_id"` RequestID string `json:"requestId"` } type Object struct { model.Object model.Thumbnail ParentID string } ================================================ FILE: drivers/mediatrack/util.go ================================================ package mediatrack import ( "errors" "fmt" "net/http" "strconv" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) // do others that not defined in Driver interface func (d *MediaTrack) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) if callback != nil { callback(req) } var e BaseResp req.SetResult(&e) res, err := req.Execute(method, url) if err != nil { return nil, err } log.Debugln(res.String()) if e.Status != "SUCCESS" { return nil, errors.New(e.Message) } if resp != nil { err = utils.Json.Unmarshal(res.Body(), resp) } return res.Body(), err } func (d *MediaTrack) getFiles(parentId string) ([]File, error) { files := make([]File, 0) url := fmt.Sprintf("https://jayce.api.mediatrack.cn/v4/assets/%s/children", parentId) sort := "" if d.OrderBy != "" { if d.OrderDesc { sort = "-" } sort += d.OrderBy } page := 1 for { var resp ChildrenResp _, err := d.request(url, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "page": strconv.Itoa(page), "size": "50", "sort": sort, }) }, &resp) if err != nil { return nil, err } if len(resp.Data.Assets) == 0 { break } page++ files = append(files, resp.Data.Assets...) } return files, nil } ================================================ FILE: drivers/mega/driver.go ================================================ package mega import ( "context" "errors" "fmt" "io" "time" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/pquerna/otp/totp" "github.com/rclone/rclone/lib/readers" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" "github.com/t3rm1n4l/go-mega" ) type Mega struct { model.Storage Addition c *mega.Mega } func (d *Mega) Config() driver.Config { return config } func (d *Mega) GetAddition() driver.Additional { return &d.Addition } func (d *Mega) Init(ctx context.Context) error { var twoFACode = d.TwoFACode d.c = mega.New() if d.TwoFASecret != "" { code, err := totp.GenerateCode(d.TwoFASecret, time.Now()) if err != nil { return fmt.Errorf("generate totp code failed: %w", err) } twoFACode = code } return d.c.MultiFactorLogin(d.Email, d.Password, twoFACode) } func (d *Mega) Drop(ctx context.Context) error { return nil } func (d *Mega) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { if node, ok := dir.(*MegaNode); ok { nodes, err := d.c.FS.GetChildren(node.n) if err != nil { return nil, err } fn := make(map[string]model.Obj) for i := range nodes { n := nodes[i] if n.GetType() != mega.FILE && n.GetType() != mega.FOLDER { continue } if _, ok := fn[n.GetName()]; !ok { fn[n.GetName()] = &MegaNode{n} } else if sameNameObj := fn[n.GetName()]; (&MegaNode{n}).ModTime().After(sameNameObj.ModTime()) { fn[n.GetName()] = &MegaNode{n} } } res := make([]model.Obj, 0) for _, v := range fn { res = append(res, v) } return res, nil } log.Errorf("can't convert: %+v", dir) return nil, fmt.Errorf("unable to convert dir to mega n") } func (d *Mega) GetRoot(ctx context.Context) (model.Obj, error) { n := d.c.FS.GetRoot() log.Debugf("mega root: %+v", *n) return &MegaNode{n}, nil } func (d *Mega) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if node, ok := file.(*MegaNode); ok { //down, err := d.c.NewDownload(n.Node) //if err != nil { // return nil, fmt.Errorf("open download file failed: %w", err) //} size := file.GetSize() resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { length := httpRange.Length if httpRange.Length < 0 || httpRange.Start+httpRange.Length >= size { length = size - httpRange.Start } var down *mega.Download err := utils.Retry(3, time.Second, func() (err error) { down, err = d.c.NewDownload(node.n) return err }) if err != nil { return nil, fmt.Errorf("open download file failed: %w", err) } oo := &openObject{ ctx: ctx, d: down, skip: httpRange.Start, } return readers.NewLimitedReadCloser(oo, length), nil } return &model.Link{ RangeReader: stream.RateLimitRangeReaderFunc(resultRangeReader), }, nil } return nil, fmt.Errorf("unable to convert dir to mega n") } func (d *Mega) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { if parentNode, ok := parentDir.(*MegaNode); ok { _, err := d.c.CreateDir(dirName, parentNode.n) return err } return fmt.Errorf("unable to convert dir to mega n") } func (d *Mega) Move(ctx context.Context, srcObj, dstDir model.Obj) error { if srcNode, ok := srcObj.(*MegaNode); ok { if dstNode, ok := dstDir.(*MegaNode); ok { return d.c.Move(srcNode.n, dstNode.n) } } return fmt.Errorf("unable to convert dir to mega n") } func (d *Mega) Rename(ctx context.Context, srcObj model.Obj, newName string) error { if srcNode, ok := srcObj.(*MegaNode); ok { return d.c.Rename(srcNode.n, newName) } return fmt.Errorf("unable to convert dir to mega n") } func (d *Mega) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return errs.NotImplement } func (d *Mega) Remove(ctx context.Context, obj model.Obj) error { if node, ok := obj.(*MegaNode); ok { return d.c.Delete(node.n, !d.MoveToTrash) } return fmt.Errorf("unable to convert dir to mega n") } func (d *Mega) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { if dstNode, ok := dstDir.(*MegaNode); ok { u, err := d.c.NewUpload(dstNode.n, stream.GetName(), stream.GetSize()) if err != nil { return err } reader := driver.NewLimitedUploadStream(ctx, stream) for id := 0; id < u.Chunks(); id++ { if utils.IsCanceled(ctx) { return ctx.Err() } _, chkSize, err := u.ChunkLocation(id) if err != nil { return err } chunk := make([]byte, chkSize) n, err := io.ReadFull(reader, chunk) if err != nil && err != io.EOF { return err } if n != len(chunk) { return errors.New("chunk too short") } err = u.UploadChunk(id, chunk) if err != nil { return err } up(float64(id) * 100 / float64(u.Chunks())) } _, err = u.Finish() return err } return fmt.Errorf("unable to convert dir to mega n") } func (d *Mega) GetDetails(ctx context.Context) (*model.StorageDetails, error) { quota, err := d.c.GetQuota() if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: int64(quota.Mstrg), UsedSpace: int64(quota.Cstrg), }, }, nil } //func (d *Mega) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*Mega)(nil) ================================================ FILE: drivers/mega/meta.go ================================================ package mega import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { // Usually one of two //driver.RootPath //driver.RootID Email string `json:"email" required:"true"` Password string `json:"password" required:"true"` TwoFACode string `json:"two_fa_code" required:"false" help:"2FA 6-digit code, filling in the 2FA code alone will not support reloading driver"` TwoFASecret string `json:"two_fa_secret" required:"false" help:"2FA secret"` MoveToTrash bool `json:"move_to_trash" default:"true" help:"move to trash when deleting files"` } var config = driver.Config{ Name: "Mega_nz", LocalSort: true, OnlyProxy: true, NoLinkURL: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Mega{} }) } ================================================ FILE: drivers/mega/types.go ================================================ package mega import ( "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/t3rm1n4l/go-mega" ) type MegaNode struct { n *mega.Node } func (m *MegaNode) GetSize() int64 { return m.n.GetSize() } func (m *MegaNode) GetName() string { return m.n.GetName() } func (m *MegaNode) CreateTime() time.Time { return m.n.GetTimeStamp() } func (m *MegaNode) GetHash() utils.HashInfo { //Meganz use md5, but can't get the original file hash, due to it's encrypted in the cloud return utils.HashInfo{} } func (m *MegaNode) ModTime() time.Time { return m.n.GetTimeStamp() } func (m *MegaNode) IsDir() bool { return m.n.GetType() == mega.FOLDER || m.n.GetType() == mega.ROOT } func (m *MegaNode) GetID() string { return m.n.GetHash() } func (m *MegaNode) GetPath() string { return "" } var _ model.Obj = (*MegaNode)(nil) ================================================ FILE: drivers/mega/util.go ================================================ package mega import ( "context" "fmt" "io" "sync" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/t3rm1n4l/go-mega" ) // do others that not defined in Driver interface // openObject represents a download in progress type openObject struct { ctx context.Context mu sync.Mutex d *mega.Download id int skip int64 chunk []byte closed bool } // get the next chunk func (oo *openObject) getChunk(ctx context.Context) (err error) { if oo.id >= oo.d.Chunks() { return io.EOF } var chunk []byte err = utils.Retry(3, time.Second, func() (err error) { chunk, err = oo.d.DownloadChunk(oo.id) return err }) if err != nil { return err } oo.id++ oo.chunk = chunk return nil } // Read reads up to len(p) bytes into p. func (oo *openObject) Read(p []byte) (n int, err error) { oo.mu.Lock() defer oo.mu.Unlock() if oo.closed { return 0, fmt.Errorf("read on closed file") } // Skip data at the start if requested for oo.skip > 0 { _, size, err := oo.d.ChunkLocation(oo.id) if err != nil { return 0, err } if oo.skip < int64(size) { break } oo.id++ oo.skip -= int64(size) } if len(oo.chunk) == 0 { err = oo.getChunk(oo.ctx) if err != nil { return 0, err } if oo.skip > 0 { oo.chunk = oo.chunk[oo.skip:] oo.skip = 0 } } n = copy(p, oo.chunk) oo.chunk = oo.chunk[n:] return n, nil } // Close closed the file - MAC errors are reported here func (oo *openObject) Close() (err error) { oo.mu.Lock() defer oo.mu.Unlock() if oo.closed { return nil } err = utils.Retry(3, 500*time.Millisecond, func() (err error) { return oo.d.Finish() }) if err != nil { return fmt.Errorf("failed to finish download: %w", err) } oo.closed = true return nil } ================================================ FILE: drivers/misskey/driver.go ================================================ package misskey import ( "context" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type Misskey struct { model.Storage Addition } func (d *Misskey) Config() driver.Config { return config } func (d *Misskey) GetAddition() driver.Additional { return &d.Addition } func (d *Misskey) Init(ctx context.Context) error { d.Endpoint = strings.TrimSuffix(d.Endpoint, "/") if d.Endpoint == "" || d.AccessToken == "" { return errs.EmptyToken } else { return nil } } func (d *Misskey) Drop(ctx context.Context) error { return nil } func (d *Misskey) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { return d.list(dir) } func (d *Misskey) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { return d.link(file) } func (d *Misskey) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { return d.makeDir(parentDir, dirName) } func (d *Misskey) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return d.move(srcObj, dstDir) } func (d *Misskey) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { return d.rename(srcObj, newName) } func (d *Misskey) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return d.copy(srcObj, dstDir) } func (d *Misskey) Remove(ctx context.Context, obj model.Obj) error { return d.remove(obj) } func (d *Misskey) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { return d.put(ctx, dstDir, stream, up) } //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*Misskey)(nil) ================================================ FILE: drivers/misskey/meta.go ================================================ package misskey import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { // Usually one of two driver.RootPath // define other // Field string `json:"field" type:"select" required:"true" options:"a,b,c" default:"a"` Endpoint string `json:"endpoint" required:"true" default:"https://misskey.io"` AccessToken string `json:"access_token" required:"true"` } var config = driver.Config{ Name: "Misskey", DefaultRoot: "/", } func init() { op.RegisterDriver(func() driver.Driver { return &Misskey{} }) } ================================================ FILE: drivers/misskey/types.go ================================================ package misskey type Resp struct { Code int Raw []byte } type Properties struct { Width int `json:"width"` Height int `json:"height"` } type MFile struct { ID string `json:"id"` CreatedAt string `json:"createdAt"` Name string `json:"name"` Type string `json:"type"` MD5 string `json:"md5"` Size int64 `json:"size"` IsSensitive bool `json:"isSensitive"` Blurhash string `json:"blurhash"` Properties Properties `json:"properties"` URL string `json:"url"` ThumbnailURL string `json:"thumbnailUrl"` Comment *string `json:"comment"` FolderID *string `json:"folderId"` Folder MFolder `json:"folder"` } type MFolder struct { ID string `json:"id"` CreatedAt string `json:"createdAt"` Name string `json:"name"` ParentID *string `json:"parentId"` } ================================================ FILE: drivers/misskey/util.go ================================================ package misskey import ( "context" "errors" "io" "net/http" "time" "github.com/go-resty/resty/v2" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) // Base layer methods func (d *Misskey) request(path, method string, callback base.ReqCallback, resp interface{}) error { url := d.Endpoint + "/api/drive" + path req := base.RestyClient.R() req.SetAuthToken(d.AccessToken).SetHeader("Content-Type", "application/json") if callback != nil { callback(req) } else { req.SetBody("{}") } req.SetResult(resp) // 启用调试模式 req.EnableTrace() response, err := req.Execute(method, url) if err != nil { return err } if !response.IsSuccess() { return errors.New(response.String()) } return nil } func (d *Misskey) getThumb(ctx context.Context, obj model.Obj) (io.Reader, error) { // TODO return the thumb of obj, optional return nil, errs.NotImplement } func setBody(body interface{}) base.ReqCallback { return func(req *resty.Request) { req.SetBody(body) } } func handleFolderId(dir model.Obj) interface{} { if isRootFolder(dir) { return nil // Root folder doesn't need folderId } return dir.GetID() } func isRootFolder(dir model.Obj) bool { return dir.GetID() == "" } // API layer methods func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) { var files []MFile var body map[string]string if !isRootFolder(dir) { body = map[string]string{"folderId": dir.GetID()} } else { body = map[string]string{} } err := d.request("/files", http.MethodPost, setBody(body), &files) if err != nil { return []model.Obj{}, err } return utils.SliceConvert(files, func(src MFile) (model.Obj, error) { return mFile2Object(src), nil }) } func (d *Misskey) getFolders(dir model.Obj) ([]model.Obj, error) { var folders []MFolder var body map[string]string if !isRootFolder(dir) { body = map[string]string{"folderId": dir.GetID()} } else { body = map[string]string{} } err := d.request("/folders", http.MethodPost, setBody(body), &folders) if err != nil { return []model.Obj{}, err } return utils.SliceConvert(folders, func(src MFolder) (model.Obj, error) { return mFolder2Object(src), nil }) } func (d *Misskey) list(dir model.Obj) ([]model.Obj, error) { files, _ := d.getFiles(dir) folders, _ := d.getFolders(dir) return append(files, folders...), nil } func (d *Misskey) link(file model.Obj) (*model.Link, error) { var mFile MFile err := d.request("/files/show", http.MethodPost, setBody(map[string]string{"fileId": file.GetID()}), &mFile) if err != nil { return nil, err } return &model.Link{ URL: mFile.URL, }, nil } func (d *Misskey) makeDir(parentDir model.Obj, dirName string) (model.Obj, error) { var folder MFolder err := d.request("/folders/create", http.MethodPost, setBody(map[string]interface{}{"parentId": handleFolderId(parentDir), "name": dirName}), &folder) if err != nil { return nil, err } return mFolder2Object(folder), nil } func (d *Misskey) move(srcObj, dstDir model.Obj) (model.Obj, error) { if srcObj.IsDir() { var folder MFolder err := d.request("/folders/update", http.MethodPost, setBody(map[string]interface{}{"folderId": srcObj.GetID(), "parentId": handleFolderId(dstDir)}), &folder) return mFolder2Object(folder), err } else { var file MFile err := d.request("/files/update", http.MethodPost, setBody(map[string]interface{}{"fileId": srcObj.GetID(), "folderId": handleFolderId(dstDir)}), &file) return mFile2Object(file), err } } func (d *Misskey) rename(srcObj model.Obj, newName string) (model.Obj, error) { if srcObj.IsDir() { var folder MFolder err := d.request("/folders/update", http.MethodPost, setBody(map[string]string{"folderId": srcObj.GetID(), "name": newName}), &folder) return mFolder2Object(folder), err } else { var file MFile err := d.request("/files/update", http.MethodPost, setBody(map[string]string{"fileId": srcObj.GetID(), "name": newName}), &file) return mFile2Object(file), err } } func (d *Misskey) copy(srcObj, dstDir model.Obj) (model.Obj, error) { if srcObj.IsDir() { folder, err := d.makeDir(dstDir, srcObj.GetName()) if err != nil { return nil, err } list, err := d.list(srcObj) if err != nil { return nil, err } for _, obj := range list { _, err := d.copy(obj, folder) if err != nil { return nil, err } } return folder, nil } else { var file MFile url, err := d.link(srcObj) if err != nil { return nil, err } err = d.request("/files/upload-from-url", http.MethodPost, setBody(map[string]interface{}{"url": url.URL, "folderId": handleFolderId(dstDir)}), &file) if err != nil { return nil, err } return mFile2Object(file), nil } } func (d *Misskey) remove(obj model.Obj) error { if obj.IsDir() { err := d.request("/folders/delete", http.MethodPost, setBody(map[string]string{"folderId": obj.GetID()}), nil) return err } else { err := d.request("/files/delete", http.MethodPost, setBody(map[string]string{"fileId": obj.GetID()}), nil) return err } } func (d *Misskey) put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { var file MFile reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: stream, UpdateProgress: up, }) // Build form data, only add folderId if not root folder formData := map[string]string{ "name": stream.GetName(), "comment": "", "isSensitive": "false", "force": "false", } folderId := handleFolderId(dstDir) if folderId != nil { formData["folderId"] = folderId.(string) } req := base.RestyClient.R(). SetContext(ctx). SetFileReader("file", stream.GetName(), reader). SetFormData(formData). SetResult(&file). SetAuthToken(d.AccessToken) resp, err := req.Post(d.Endpoint + "/api/drive/files/create") if err != nil { return nil, err } if !resp.IsSuccess() { return nil, errors.New(resp.String()) } return mFile2Object(file), nil } func mFile2Object(file MFile) *model.ObjThumbURL { ctime, err := time.Parse(time.RFC3339, file.CreatedAt) if err != nil { ctime = time.Time{} } return &model.ObjThumbURL{ Object: model.Object{ ID: file.ID, Name: file.Name, Ctime: ctime, IsFolder: false, Size: file.Size, }, Thumbnail: model.Thumbnail{ Thumbnail: file.ThumbnailURL, }, Url: model.Url{ Url: file.URL, }, } } func mFolder2Object(folder MFolder) *model.Object { ctime, err := time.Parse(time.RFC3339, folder.CreatedAt) if err != nil { ctime = time.Time{} } return &model.Object{ ID: folder.ID, Name: folder.Name, Ctime: ctime, IsFolder: true, } } ================================================ FILE: drivers/mopan/driver.go ================================================ package mopan import ( "context" "errors" "fmt" "io" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/foxxorcat/mopan-sdk-go" log "github.com/sirupsen/logrus" ) type MoPan struct { model.Storage Addition client *mopan.MoClient userID string uploadThread int } func (d *MoPan) Config() driver.Config { return config } func (d *MoPan) GetAddition() driver.Additional { return &d.Addition } func (d *MoPan) Init(ctx context.Context) error { d.uploadThread, _ = strconv.Atoi(d.UploadThread) if d.uploadThread < 1 || d.uploadThread > 32 { d.uploadThread, d.UploadThread = 3, "3" } defer func() { d.SMSCode = "" }() login := func() (err error) { var loginData *mopan.LoginResp if d.SMSCode != "" { loginData, err = d.client.LoginBySmsStep2(d.Phone, d.SMSCode) } else { loginData, err = d.client.Login(d.Phone, d.Password) } if err != nil { return err } d.client.SetAuthorization(loginData.Token) info, err := d.client.GetUserInfo() if err != nil { return err } d.userID = info.UserID log.Debugf("[mopan] Phone: %s UserCloudStorageRelations: %+v", d.Phone, loginData.UserCloudStorageRelations) cloudCircleApp, _ := d.client.QueryAllCloudCircleApp() log.Debugf("[mopan] Phone: %s CloudCircleApp: %+v", d.Phone, cloudCircleApp) if d.RootFolderID == "" { for _, userCloudStorage := range loginData.UserCloudStorageRelations { if userCloudStorage.Path == "/文件" { d.RootFolderID = userCloudStorage.FolderID } } } return nil } d.client = mopan.NewMoClientWithRestyClient(base.NewRestyClient()). SetRestyClient(base.RestyClient). SetOnAuthorizationExpired(func(_ error) error { err := login() if err != nil { d.Status = err.Error() op.MustSaveDriverStorage(d) } return err }) var deviceInfo mopan.DeviceInfo if strings.TrimSpace(d.DeviceInfo) != "" && utils.Json.UnmarshalFromString(d.DeviceInfo, &deviceInfo) == nil { d.client.SetDeviceInfo(&deviceInfo) } d.DeviceInfo, _ = utils.Json.MarshalToString(d.client.GetDeviceInfo()) if strings.Contains(d.SMSCode, "send") { if _, err := d.client.LoginBySms(d.Phone); err != nil { return err } return errors.New("please enter the SMS code") } return login() } func (d *MoPan) Drop(ctx context.Context) error { d.client = nil d.userID = "" return nil } func (d *MoPan) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var files []model.Obj for page := 1; ; page++ { data, err := d.client.QueryFiles(dir.GetID(), page, mopan.WarpParamOption( func(j mopan.Json) { j["orderBy"] = d.OrderBy j["descending"] = d.OrderDirection == "desc" }, mopan.ParamOptionShareFile(d.CloudID), )) if err != nil { return nil, err } if len(data.FileListAO.FileList)+len(data.FileListAO.FolderList) == 0 { break } log.Debugf("[mopan] Phone: %s folder: %+v", d.Phone, data.FileListAO.FolderList) files = append(files, utils.MustSliceConvert(data.FileListAO.FolderList, folderToObj)...) files = append(files, utils.MustSliceConvert(data.FileListAO.FileList, fileToObj)...) } return files, nil } func (d *MoPan) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { data, err := d.client.GetFileDownloadUrl(file.GetID(), mopan.WarpParamOption(mopan.ParamOptionShareFile(d.CloudID))) if err != nil { return nil, err } data.DownloadUrl = strings.Replace(strings.ReplaceAll(data.DownloadUrl, "&", "&"), "http://", "https://", 1) res, err := base.NoRedirectClient.R().SetDoNotParseResponse(true).SetContext(ctx).Get(data.DownloadUrl) if err != nil { return nil, err } defer func() { _ = res.RawBody().Close() }() if res.StatusCode() == 302 { data.DownloadUrl = res.Header().Get("location") } return &model.Link{ URL: data.DownloadUrl, }, nil } func (d *MoPan) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { f, err := d.client.CreateFolder(dirName, parentDir.GetID(), mopan.WarpParamOption( mopan.ParamOptionShareFile(d.CloudID), )) if err != nil { return nil, err } return folderToObj(*f), nil } func (d *MoPan) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return d.newTask(srcObj, dstDir, mopan.TASK_MOVE) } func (d *MoPan) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { if srcObj.IsDir() { _, err := d.client.RenameFolder(srcObj.GetID(), newName, mopan.WarpParamOption( mopan.ParamOptionShareFile(d.CloudID), )) if err != nil { return nil, err } } else { _, err := d.client.RenameFile(srcObj.GetID(), newName, mopan.WarpParamOption( mopan.ParamOptionShareFile(d.CloudID), )) if err != nil { return nil, err } } return CloneObj(srcObj, srcObj.GetID(), newName), nil } func (d *MoPan) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return d.newTask(srcObj, dstDir, mopan.TASK_COPY) } func (d *MoPan) newTask(srcObj, dstDir model.Obj, taskType mopan.TaskType) (model.Obj, error) { param := mopan.TaskParam{ UserOrCloudID: d.userID, Source: 1, TaskType: taskType, TargetSource: 1, TargetUserOrCloudID: d.userID, TargetType: 1, TargetFolderID: dstDir.GetID(), TaskStatusDetailDTOList: []mopan.TaskFileParam{ { FileID: srcObj.GetID(), IsFolder: srcObj.IsDir(), FileName: srcObj.GetName(), }, }, } if d.CloudID != "" { param.UserOrCloudID = d.CloudID param.Source = 2 param.TargetSource = 2 param.TargetUserOrCloudID = d.CloudID } task, err := d.client.AddBatchTask(param) if err != nil { return nil, err } for count := 0; count < 5; count++ { stat, err := d.client.CheckBatchTask(mopan.TaskCheckParam{ TaskId: task.TaskIDList[0], TaskType: task.TaskType, TargetType: 1, TargetFolderID: task.TargetFolderID, TargetSource: param.TargetSource, TargetUserOrCloudID: param.TargetUserOrCloudID, }) if err != nil { return nil, err } switch stat.TaskStatus { case 2: if err := d.client.CancelBatchTask(stat.TaskID, task.TaskType); err != nil { return nil, err } return nil, errors.New("file name conflict") case 4: if task.TaskType == mopan.TASK_MOVE { return CloneObj(srcObj, srcObj.GetID(), srcObj.GetName()), nil } return CloneObj(srcObj, stat.SuccessedFileIDList[0], srcObj.GetName()), nil } time.Sleep(time.Second) } return nil, nil } func (d *MoPan) Remove(ctx context.Context, obj model.Obj) error { _, err := d.client.DeleteToRecycle([]mopan.TaskFileParam{ { FileID: obj.GetID(), IsFolder: obj.IsDir(), FileName: obj.GetName(), }, }, mopan.WarpParamOption(mopan.ParamOptionShareFile(d.CloudID))) return err } func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { file, err := stream.CacheFullAndWriter(&up, nil) if err != nil { return nil, err } // step.1 uploadPartData, err := mopan.InitUploadPartData(ctx, mopan.UpdloadFileParam{ ParentFolderId: dstDir.GetID(), FileName: stream.GetName(), FileSize: stream.GetSize(), File: file, }) if err != nil { return nil, err } // 尝试恢复进度 initUpdload, ok := base.GetUploadProgress[*mopan.InitMultiUploadData](d, d.client.Authorization, uploadPartData.FileMd5) if !ok { // step.2 initUpdload, err = d.client.InitMultiUpload(ctx, *uploadPartData, mopan.WarpParamOption( mopan.ParamOptionShareFile(d.CloudID), )) if err != nil { return nil, err } } if !initUpdload.FileDataExists { // utils.Log.Error(d.client.CloudDiskStartBusiness()) threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) // step.3 parts, err := d.client.GetAllMultiUploadUrls(initUpdload.UploadFileID, initUpdload.PartInfos) if err != nil { return nil, err } for i, part := range parts { if utils.IsCanceled(upCtx) { break } i, part, byteSize := i, part, initUpdload.PartSize if part.PartNumber == uploadPartData.PartTotal { byteSize = initUpdload.LastPartSize } // step.4 threadG.Go(func(ctx context.Context) error { reader := io.NewSectionReader(file, int64(part.PartNumber-1)*initUpdload.PartSize, byteSize) req, err := part.NewRequest(ctx, driver.NewLimitedUploadStream(ctx, reader)) if err != nil { return err } req.ContentLength = byteSize resp, err := base.HttpClient.Do(req) if err != nil { return err } _ = resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("upload err,code=%d", resp.StatusCode) } up(100 * float64(threadG.Success()+1) / float64(len(parts)+1)) initUpdload.PartInfos[i] = "" return nil }) } if err = threadG.Wait(); err != nil { if errors.Is(err, context.Canceled) { initUpdload.PartInfos = utils.SliceFilter(initUpdload.PartInfos, func(s string) bool { return s != "" }) base.SaveUploadProgress(d, initUpdload, d.client.Authorization, uploadPartData.FileMd5) } return nil, err } } //step.5 uFile, err := d.client.CommitMultiUploadFile(initUpdload.UploadFileID, nil) if err != nil { return nil, err } return &model.Object{ ID: uFile.UserFileID, Name: uFile.FileName, Size: int64(uFile.FileSize), Modified: time.Time(uFile.CreateDate), }, nil } func (d *MoPan) GetDetails(ctx context.Context) (*model.StorageDetails, error) { quota, err := d.client.UsedSpace() if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: int64(quota.Capacity), UsedSpace: int64(quota.Used), }, }, nil } var _ driver.Driver = (*MoPan)(nil) var _ driver.MkdirResult = (*MoPan)(nil) var _ driver.MoveResult = (*MoPan)(nil) var _ driver.RenameResult = (*MoPan)(nil) var _ driver.Remove = (*MoPan)(nil) var _ driver.CopyResult = (*MoPan)(nil) var _ driver.PutResult = (*MoPan)(nil) ================================================ FILE: drivers/mopan/meta.go ================================================ package mopan import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { Phone string `json:"phone" required:"true"` Password string `json:"password" required:"true"` SMSCode string `json:"sms_code" help:"input 'send' send sms "` RootFolderID string `json:"root_folder_id" default:""` CloudID string `json:"cloud_id"` OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` DeviceInfo string `json:"device_info"` UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` } func (a *Addition) GetRootId() string { return a.RootFolderID } var config = driver.Config{ Name: "MoPan", CheckStatus: true, Alert: "warning|This network disk may store your password in clear text. Please set your password carefully", } func init() { op.RegisterDriver(func() driver.Driver { return &MoPan{} }) } ================================================ FILE: drivers/mopan/types.go ================================================ package mopan ================================================ FILE: drivers/mopan/util.go ================================================ package mopan import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/foxxorcat/mopan-sdk-go" ) func fileToObj(f mopan.File) model.Obj { return &model.ObjThumb{ Object: model.Object{ ID: string(f.ID), Name: f.Name, Size: int64(f.Size), Modified: time.Time(f.LastOpTime), Ctime: time.Time(f.CreateDate), HashInfo: utils.NewHashInfo(utils.MD5, f.Md5), }, Thumbnail: model.Thumbnail{ Thumbnail: f.Icon.SmallURL, }, } } func folderToObj(f mopan.Folder) model.Obj { return &model.Object{ ID: string(f.ID), Name: f.Name, Modified: time.Time(f.LastOpTime), Ctime: time.Time(f.CreateDate), IsFolder: true, } } func CloneObj(o model.Obj, newID, newName string) model.Obj { if o.IsDir() { return &model.Object{ ID: newID, Name: newName, IsFolder: true, Modified: o.ModTime(), Ctime: o.CreateTime(), } } thumb := "" if o, ok := o.(model.Thumb); ok { thumb = o.Thumb() } return &model.ObjThumb{ Object: model.Object{ ID: newID, Name: newName, Size: o.GetSize(), Modified: o.ModTime(), Ctime: o.CreateTime(), HashInfo: o.GetHash(), }, Thumbnail: model.Thumbnail{ Thumbnail: thumb, }, } } ================================================ FILE: drivers/netease_music/crypto.go ================================================ package netease_music import ( "bytes" "crypto/aes" "crypto/cipher" "crypto/md5" "crypto/rsa" "crypto/x509" "encoding/base64" "encoding/hex" "encoding/pem" "math/big" "strings" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" ) var ( linuxapiKey = []byte("rFgB&h#%2?^eDg:Q") eapiKey = []byte("e82ckenh8dichen8") iv = []byte("0102030405060708") presetKey = []byte("0CoJUm6Qyw8W8jud") publicKey = []byte("-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgtQn2JZ34ZC28NWYpAUd98iZ37BUrX/aKzmFbt7clFSs6sXqHauqKWqdtLkF2KexO40H1YTX8z2lSgBBOAxLsvaklV8k4cBFK9snQXE9/DDaFt6Rr7iVZMldczhC0JNgTz+SHXT6CBHuX3e9SdB1Ua44oncaTWz7OBGLbCiK45wIDAQAB\n-----END PUBLIC KEY-----") stdChars = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") ) func aesKeyPending(key []byte) []byte { k := len(key) count := 0 switch true { case k <= 16: count = 16 - k case k <= 24: count = 24 - k case k <= 32: count = 32 - k default: return key[:32] } if count == 0 { return key } return append(key, bytes.Repeat([]byte{0}, count)...) } func pkcs7Padding(src []byte, blockSize int) []byte { padding := blockSize - len(src)%blockSize padtext := bytes.Repeat([]byte{byte(padding)}, padding) return append(src, padtext...) } func aesCBCEncrypt(src, key, iv []byte) []byte { block, _ := aes.NewCipher(aesKeyPending(key)) src = pkcs7Padding(src, block.BlockSize()) dst := make([]byte, len(src)) mode := cipher.NewCBCEncrypter(block, iv) mode.CryptBlocks(dst, src) return dst } func aesECBEncrypt(src, key []byte) []byte { block, _ := aes.NewCipher(aesKeyPending(key)) src = pkcs7Padding(src, block.BlockSize()) dst := make([]byte, len(src)) ecbCryptBlocks(block, dst, src) return dst } func ecbCryptBlocks(block cipher.Block, dst, src []byte) { bs := block.BlockSize() for len(src) > 0 { block.Encrypt(dst, src[:bs]) src = src[bs:] dst = dst[bs:] } } func rsaEncrypt(buffer, key []byte) []byte { buffers := make([]byte, 128-16, 128) buffers = append(buffers, buffer...) block, _ := pem.Decode(key) pubInterface, _ := x509.ParsePKIXPublicKey(block.Bytes) pub := pubInterface.(*rsa.PublicKey) c := new(big.Int).SetBytes([]byte(buffers)) return c.Exp(c, big.NewInt(int64(pub.E)), pub.N).Bytes() } func getSecretKey() ([]byte, []byte) { key := make([]byte, 16) reversed := make([]byte, 16) for i := 0; i < 16; i++ { result := stdChars[random.RangeInt64(0, 62)] key[i] = result reversed[15-i] = result } return key, reversed } func weapi(data map[string]string) map[string]string { text, _ := utils.Json.Marshal(data) secretKey, reversedKey := getSecretKey() params := []byte(base64.StdEncoding.EncodeToString(aesCBCEncrypt(text, presetKey, iv))) return map[string]string{ "params": base64.StdEncoding.EncodeToString(aesCBCEncrypt(params, reversedKey, iv)), "encSecKey": hex.EncodeToString(rsaEncrypt(secretKey, publicKey)), } } func eapi(url string, data map[string]interface{}) map[string]string { text, _ := utils.Json.Marshal(data) msg := "nobody" + url + "use" + string(text) + "md5forencrypt" h := md5.New() h.Write([]byte(msg)) digest := hex.EncodeToString(h.Sum(nil)) params := []byte(url + "-36cd479b6b5-" + string(text) + "-36cd479b6b5-" + digest) return map[string]string{ "params": hex.EncodeToString(aesECBEncrypt(params, eapiKey)), } } func linuxapi(data map[string]interface{}) map[string]string { text, _ := utils.Json.Marshal(data) return map[string]string{ "eparams": strings.ToUpper(hex.EncodeToString(aesECBEncrypt(text, linuxapiKey))), } } ================================================ FILE: drivers/netease_music/driver.go ================================================ package netease_music import ( "context" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" _ "golang.org/x/image/webp" ) type NeteaseMusic struct { model.Storage Addition csrfToken string musicU string fileMapByName map[string]model.Obj } func (d *NeteaseMusic) Config() driver.Config { return config } func (d *NeteaseMusic) GetAddition() driver.Additional { return &d.Addition } func (d *NeteaseMusic) Init(ctx context.Context) error { d.csrfToken = d.Addition.getCookie("__csrf") d.musicU = d.Addition.getCookie("MUSIC_U") if d.csrfToken == "" || d.musicU == "" { return errs.EmptyToken } return nil } func (d *NeteaseMusic) Drop(ctx context.Context) error { return nil } func (Addition) GetRootPath() string { return "/" } func (d *NeteaseMusic) Get(ctx context.Context, path string) (model.Obj, error) { fragments := strings.Split(path, "/") if len(fragments) > 1 { fileName := fragments[1] if strings.HasSuffix(fileName, ".lrc") { lrc := d.fileMapByName[fileName] return d.getLyricObj(lrc) } if song, ok := d.fileMapByName[fileName]; ok { return song, nil } else { return nil, errs.ObjectNotFound } } return nil, errs.ObjectNotFound } func (d *NeteaseMusic) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { return d.getSongObjs(args) } func (d *NeteaseMusic) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if lrc, ok := file.(*LyricObj); ok { if args.Type == "parsed" && !args.Redirect { return lrc.getLyricLink(), nil } else { return lrc.getProxyLink(ctx), nil } } return d.getSongLink(file) } func (d *NeteaseMusic) Remove(ctx context.Context, obj model.Obj) error { return d.removeSongObj(obj) } func (d *NeteaseMusic) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { return d.putSongStream(ctx, stream, up) } func (d *NeteaseMusic) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return errs.NotSupport } func (d *NeteaseMusic) Move(ctx context.Context, srcObj, dstDir model.Obj) error { return errs.NotSupport } func (d *NeteaseMusic) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { return errs.NotSupport } func (d *NeteaseMusic) Rename(ctx context.Context, srcObj model.Obj, newName string) error { return errs.NotSupport } var _ driver.Driver = (*NeteaseMusic)(nil) ================================================ FILE: drivers/netease_music/meta.go ================================================ package netease_music import ( "regexp" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { Cookie string `json:"cookie" type:"text" required:"true" help:""` SongLimit uint64 `json:"song_limit" default:"200" type:"number" help:"only get 200 songs by default"` } func (ad *Addition) getCookie(name string) string { re := regexp.MustCompile(name + "=([^(;|$)]+)") matches := re.FindStringSubmatch(ad.Cookie) if len(matches) < 2 { return "" } return matches[1] } var config = driver.Config{ Name: "NeteaseMusic", } func init() { op.RegisterDriver(func() driver.Driver { return &NeteaseMusic{} }) } ================================================ FILE: drivers/netease_music/types.go ================================================ package netease_music import ( "context" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/OpenListTeam/OpenList/v4/server/common" ) type HostsResp struct { Upload []string `json:"upload"` } type SongResp struct { Data []struct { Url string `json:"url"` } `json:"data"` } type ListResp struct { Size int64 `json:"size"` MaxSize int64 `json:"maxSize"` Data []struct { AddTime int64 `json:"addTime"` FileName string `json:"fileName"` FileSize int64 `json:"fileSize"` SongId int64 `json:"songId"` SimpleSong struct { Al struct { PicUrl string `json:"picUrl"` } `json:"al"` } `json:"simpleSong"` } `json:"data"` } type LyricObj struct { model.Object lyric string } func (lrc *LyricObj) getProxyLink(ctx context.Context) *model.Link { rawURL := common.GetApiUrl(ctx) + "/p" + lrc.Path rawURL = utils.EncodePath(rawURL, true) + "?type=parsed&sign=" + sign.Sign(lrc.Path) return &model.Link{URL: rawURL} } func (lrc *LyricObj) getLyricLink() *model.Link { return &model.Link{ RangeReader: stream.GetRangeReaderFromMFile(int64(len(lrc.lyric)), strings.NewReader(lrc.lyric)), } } type ReqOption struct { crypto string stream model.FileStreamer up driver.UpdateProgress ctx context.Context data map[string]string headers map[string]string cookies []*http.Cookie url string } type Characteristic map[string]string func (ch *Characteristic) fromDriver(d *NeteaseMusic) *Characteristic { *ch = map[string]string{ "osver": "", "deviceId": "", "mobilename": "", "appver": "6.1.1", "versioncode": "140", "buildver": strconv.FormatInt(time.Now().Unix(), 10), "resolution": "1920x1080", "os": "android", "channel": "", "requestId": strconv.FormatInt(time.Now().Unix()*1000, 10) + strconv.Itoa(int(random.RangeInt64(0, 1000))), "MUSIC_U": d.musicU, } return ch } func (ch Characteristic) toCookies() []*http.Cookie { cookies := make([]*http.Cookie, 0) for k, v := range ch { cookies = append(cookies, &http.Cookie{Name: k, Value: v}) } return cookies } func (ch *Characteristic) merge(data map[string]string) map[string]interface{} { body := map[string]interface{}{ "header": ch, } for k, v := range data { body[k] = v } return body } ================================================ FILE: drivers/netease_music/upload.go ================================================ package netease_music import ( "context" "crypto/md5" "encoding/hex" "io" "net/http" "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/dhowden/tag" ) type token struct { resourceId string objectKey string token string } type songmeta struct { needUpload bool songId string name string artist string album string } type uploader struct { driver *NeteaseMusic file model.File meta songmeta md5 string ext string size string filename string } func (u *uploader) init(stream model.FileStreamer) error { u.filename = stream.GetName() u.size = strconv.FormatInt(stream.GetSize(), 10) u.ext = "mp3" if strings.HasSuffix(stream.GetMimetype(), "flac") { u.ext = "flac" } h := md5.New() _, err := utils.CopyWithBuffer(h, stream) if err != nil { return err } u.md5 = hex.EncodeToString(h.Sum(nil)) _, err = u.file.Seek(0, io.SeekStart) if err != nil { return err } if m, err := tag.ReadFrom(u.file); err != nil { u.meta = songmeta{} } else { u.meta = songmeta{ name: m.Title(), artist: m.Artist(), album: m.Album(), } } if u.meta.name == "" { u.meta.name = u.filename } if u.meta.album == "" { u.meta.album = "未知专辑" } if u.meta.artist == "" { u.meta.artist = "未知艺术家" } _, err = u.file.Seek(0, io.SeekStart) if err != nil { return err } return nil } func (u *uploader) checkIfExisted() error { body, err := u.driver.request("https://interface.music.163.com/api/cloud/upload/check", http.MethodPost, ReqOption{ crypto: "weapi", data: map[string]string{ "ext": "", "songId": "0", "version": "1", "bitrate": "999000", "length": u.size, "md5": u.md5, }, cookies: []*http.Cookie{ {Name: "os", Value: "pc"}, {Name: "appver", Value: "2.9.7"}, }, }, ) if err != nil { return err } u.meta.songId = utils.Json.Get(body, "songId").ToString() u.meta.needUpload = utils.Json.Get(body, "needUpload").ToBool() return nil } func (u *uploader) allocToken(bucket ...string) (token, error) { if len(bucket) == 0 { bucket = []string{""} } body, err := u.driver.request("https://music.163.com/weapi/nos/token/alloc", http.MethodPost, ReqOption{ crypto: "weapi", data: map[string]string{ "bucket": bucket[0], "local": "false", "type": "audio", "nos_product": "3", "filename": u.filename, "md5": u.md5, "ext": u.ext, }, }) if err != nil { return token{}, err } return token{ resourceId: utils.Json.Get(body, "result", "resourceId").ToString(), objectKey: utils.Json.Get(body, "result", "objectKey").ToString(), token: utils.Json.Get(body, "result", "token").ToString(), }, nil } func (u *uploader) publishInfo(resourceId string) error { body, err := u.driver.request("https://music.163.com/api/upload/cloud/info/v2", http.MethodPost, ReqOption{ crypto: "weapi", data: map[string]string{ "md5": u.md5, "filename": u.filename, "song": u.meta.name, "album": u.meta.album, "artist": u.meta.artist, "songid": u.meta.songId, "resourceId": resourceId, "bitrate": "999000", }, }) if err != nil { return err } _, err = u.driver.request("https://interface.music.163.com/api/cloud/pub/v2", http.MethodPost, ReqOption{ crypto: "weapi", data: map[string]string{ "songid": utils.Json.Get(body, "songId").ToString(), }, }) if err != nil { return err } return nil } func (u *uploader) upload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress) error { bucket := "jd-musicrep-privatecloud-audio-public" token, err := u.allocToken(bucket) if err != nil { return err } body, err := u.driver.request("https://wanproxy.127.net/lbs?version=1.0&bucketname="+bucket, http.MethodGet, ReqOption{}, ) if err != nil { return err } var resp HostsResp err = utils.Json.Unmarshal(body, &resp) if err != nil { return err } objectKey := strings.ReplaceAll(token.objectKey, "/", "%2F") _, err = u.driver.request( resp.Upload[0]+"/"+bucket+"/"+objectKey+"?offset=0&complete=true&version=1.0", http.MethodPost, ReqOption{ stream: stream, up: up, ctx: ctx, headers: map[string]string{ "x-nos-token": token.token, "Content-Type": "audio/mpeg", "Content-Length": u.size, "Content-MD5": u.md5, }, }, ) if err != nil { return err } return nil } ================================================ FILE: drivers/netease_music/util.go ================================================ package netease_music import ( "context" "net/http" "path" "regexp" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) func (d *NeteaseMusic) request(url, method string, opt ReqOption) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Cookie", d.Addition.Cookie) if strings.Contains(url, "music.163.com") { req.SetHeader("Referer", "https://music.163.com") } if opt.cookies != nil { for _, cookie := range opt.cookies { req.SetCookie(cookie) } } if opt.headers != nil { for header, value := range opt.headers { req.SetHeader(header, value) } } data := opt.data if opt.crypto == "weapi" { data = weapi(data) re, _ := regexp.Compile(`/\w*api/`) url = re.ReplaceAllString(url, "/weapi/") } else if opt.crypto == "eapi" { ch := new(Characteristic).fromDriver(d) req.SetCookies(ch.toCookies()) data = eapi(opt.url, ch.merge(data)) re, _ := regexp.Compile(`/\w*api/`) url = re.ReplaceAllString(url, "/eapi/") } else if opt.crypto == "linuxapi" { re, _ := regexp.Compile(`/\w*api/`) data = linuxapi(map[string]interface{}{ "url": re.ReplaceAllString(url, "/api/"), "method": method, "params": data, }) req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36") url = "https://music.163.com/api/linux/forward" } if opt.ctx != nil { req.SetContext(opt.ctx) } if method == http.MethodPost { if opt.stream != nil { if opt.up == nil { opt.up = func(_ float64) {} } req.SetContentLength(true) req.SetBody(driver.NewLimitedUploadStream(opt.ctx, &driver.ReaderUpdatingProgress{ Reader: opt.stream, UpdateProgress: opt.up, })) } else { req.SetFormData(data) } res, err := req.Post(url) if err != nil { return nil, err } return res.Body(), nil } if method == http.MethodGet { res, err := req.Get(url) if err != nil { return nil, err } return res.Body(), nil } return nil, errs.NotImplement } func (d *NeteaseMusic) getSongObjs(args model.ListArgs) ([]model.Obj, error) { body, err := d.request("https://music.163.com/weapi/v1/cloud/get", http.MethodPost, ReqOption{ crypto: "weapi", data: map[string]string{ "limit": strconv.FormatUint(d.Addition.SongLimit, 10), "offset": "0", }, cookies: []*http.Cookie{ {Name: "os", Value: "pc"}, }, }) if err != nil { return nil, err } var resp ListResp err = utils.Json.Unmarshal(body, &resp) if err != nil { return nil, err } d.fileMapByName = make(map[string]model.Obj) files := make([]model.Obj, 0, len(resp.Data)) for _, f := range resp.Data { song := &model.ObjThumb{ Object: model.Object{ IsFolder: false, Size: f.FileSize, Name: f.FileName, Modified: time.UnixMilli(f.AddTime), ID: strconv.FormatInt(f.SongId, 10), }, Thumbnail: model.Thumbnail{Thumbnail: f.SimpleSong.Al.PicUrl}, } d.fileMapByName[song.Name] = song files = append(files, song) // map song id for lyric lrcName := strings.Split(f.FileName, ".")[0] + ".lrc" lrc := &model.Object{ IsFolder: false, Name: lrcName, Path: path.Join(args.ReqPath, lrcName), ID: strconv.FormatInt(f.SongId, 10), } d.fileMapByName[lrc.Name] = lrc } return files, nil } func (d *NeteaseMusic) getSongLink(file model.Obj) (*model.Link, error) { body, err := d.request( "https://music.163.com/api/song/enhance/player/url", http.MethodPost, ReqOption{ crypto: "linuxapi", data: map[string]string{ "ids": "[" + file.GetID() + "]", "br": "999000", }, cookies: []*http.Cookie{ {Name: "os", Value: "pc"}, }, }, ) if err != nil { return nil, err } var resp SongResp err = utils.Json.Unmarshal(body, &resp) if err != nil { return nil, err } if len(resp.Data) < 1 { return nil, errs.ObjectNotFound } return &model.Link{URL: resp.Data[0].Url}, nil } func (d *NeteaseMusic) getLyricObj(file model.Obj) (model.Obj, error) { if lrc, ok := file.(*LyricObj); ok { return lrc, nil } body, err := d.request( "https://music.163.com/api/song/lyric?_nmclfl=1", http.MethodPost, ReqOption{ data: map[string]string{ "id": file.GetID(), "tv": "-1", "lv": "-1", "rv": "-1", "kv": "-1", }, cookies: []*http.Cookie{ {Name: "os", Value: "ios"}, }, }, ) if err != nil { return nil, err } lyric := utils.Json.Get(body, "lrc", "lyric").ToString() return &LyricObj{ lyric: lyric, Object: model.Object{ IsFolder: false, ID: file.GetID(), Name: file.GetName(), Path: file.GetPath(), Size: int64(len(lyric)), }, }, nil } func (d *NeteaseMusic) removeSongObj(file model.Obj) error { _, err := d.request("http://music.163.com/weapi/cloud/del", http.MethodPost, ReqOption{ crypto: "weapi", data: map[string]string{ "songIds": "[" + file.GetID() + "]", }, }) return err } func (d *NeteaseMusic) putSongStream(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress) error { tmp, err := stream.CacheFullAndWriter(&up, nil) if err != nil { return err } u := uploader{driver: d, file: tmp} err = u.init(stream) if err != nil { return err } err = u.checkIfExisted() if err != nil { return err } token, err := u.allocToken() if err != nil { return err } if u.meta.needUpload { err = u.upload(ctx, stream, up) if err != nil { return err } } err = u.publishInfo(token.resourceId) if err != nil { return err } return nil } ================================================ FILE: drivers/onedrive/driver.go ================================================ package onedrive import ( "context" "fmt" "net/http" "net/url" "path" "sync" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) type Onedrive struct { model.Storage Addition AccessToken string root *Object mutex sync.Mutex ref *Onedrive } func (d *Onedrive) Config() driver.Config { return config } func (d *Onedrive) GetAddition() driver.Additional { return &d.Addition } func (d *Onedrive) Init(ctx context.Context) error { if d.ChunkSize < 1 { d.ChunkSize = 5 } if d.ref != nil { return nil } return d.refreshToken() } func (d *Onedrive) InitReference(refStorage driver.Driver) error { if ref, ok := refStorage.(*Onedrive); ok { d.ref = ref return nil } return errs.NotSupport } func (d *Onedrive) Drop(ctx context.Context) error { d.ref = nil return nil } func (d *Onedrive) GetRoot(ctx context.Context) (model.Obj, error) { if d.root != nil { return d.root, nil } d.mutex.Lock() defer d.mutex.Unlock() root := &Object{ ObjThumb: model.ObjThumb{ Object: model.Object{ ID: "root", Path: d.RootFolderPath, Name: "root", Size: 0, Modified: d.Modified, Ctime: d.Modified, IsFolder: true, }, }, ParentID: "", } if !utils.PathEqual(d.RootFolderPath, "/") { // get root folder id url := d.GetMetaUrl(false, d.RootFolderPath) var resp struct { Id string `json:"id"` } _, err := d.Request(url, http.MethodGet, nil, &resp) if err != nil { return nil, err } root.ID = resp.Id } d.root = root return d.root, nil } func (d *Onedrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(dir.GetPath()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { obj := fileToObj(src, dir.GetID()) obj.Path = path.Join(dir.GetPath(), obj.GetName()) return obj, nil }) } func (d *Onedrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { f, err := d.GetFile(file.GetPath()) if err != nil { return nil, err } if f.File == nil { return nil, errs.NotFile } u := f.Url if d.CustomHost != "" { _u, err := url.Parse(f.Url) if err != nil { return nil, err } _u.Host = d.CustomHost u = _u.String() } return &model.Link{ URL: u, }, nil } func (d *Onedrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { url := d.GetMetaUrl(false, parentDir.GetPath()) + "/children" data := base.Json{ "name": dirName, "folder": base.Json{}, "@microsoft.graph.conflictBehavior": "rename", } // todo 修复文件夹 ctime/mtime, onedrive 可在 data 里设置 fileSystemInfo 字段, 但是此接口未提供 ctime/mtime _, err := d.Request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *Onedrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { parentPath := "" if dstDir.GetID() == "" { parentPath = dstDir.GetPath() if utils.PathEqual(parentPath, "/") { parentPath = path.Join("/drive/root", parentPath) } else { parentPath = path.Join("/drive/root:/", parentPath) } } data := base.Json{ "parentReference": base.Json{ "id": dstDir.GetID(), "path": parentPath, }, "name": srcObj.GetName(), } url := d.GetMetaUrl(false, srcObj.GetPath()) _, err := d.Request(url, http.MethodPatch, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *Onedrive) Rename(ctx context.Context, srcObj model.Obj, newName string) error { var parentID string if o, ok := srcObj.(*Object); ok { parentID = o.ParentID } else { return fmt.Errorf("srcObj is not Object") } if parentID == "" { parentID = "root" } data := base.Json{ "parentReference": base.Json{ "id": parentID, }, "name": newName, } url := d.GetMetaUrl(false, srcObj.GetPath()) _, err := d.Request(url, http.MethodPatch, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *Onedrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { dst, err := d.GetFile(dstDir.GetPath()) if err != nil { return err } data := base.Json{ "parentReference": base.Json{ "driveId": dst.ParentReference.DriveId, "id": dst.Id, }, "name": srcObj.GetName(), } url := d.GetMetaUrl(false, srcObj.GetPath()) + "/copy" _, err = d.Request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *Onedrive) Remove(ctx context.Context, obj model.Obj) error { url := d.GetMetaUrl(false, obj.GetPath()) _, err := d.Request(url, http.MethodDelete, nil, nil) return err } func (d *Onedrive) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { var err error if stream.GetSize() <= 4*1024*1024 { err = d.upSmall(ctx, dstDir, stream) } else { err = d.upBig(ctx, dstDir, stream, up) } return err } func (d *Onedrive) GetDetails(ctx context.Context) (*model.StorageDetails, error) { if d.DisableDiskUsage { return nil, errs.NotImplement } drive, err := d.getDrive(ctx) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: drive.Quota.Total, UsedSpace: drive.Quota.Used, }, }, nil } func (d *Onedrive) GetDirectUploadTools() []string { if !d.EnableDirectUpload { return nil } return []string{"HttpDirect"} } // GetDirectUploadInfo returns the direct upload info for OneDrive func (d *Onedrive) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) { if !d.EnableDirectUpload { return nil, errs.NotImplement } return d.getDirectUploadInfo(ctx, path.Join(dstDir.GetPath(), fileName)) } var _ driver.Driver = (*Onedrive)(nil) ================================================ FILE: drivers/onedrive/meta.go ================================================ package onedrive import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Region string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"` IsSharepoint bool `json:"is_sharepoint"` UseOnlineAPI bool `json:"use_online_api" default:"true"` APIAddress string `json:"api_url_address" default:"https://api.oplist.org/onedrive/renewapi"` ClientID string `json:"client_id"` ClientSecret string `json:"client_secret"` RedirectUri string `json:"redirect_uri" required:"true" default:"https://api.oplist.org/onedrive/callback"` RefreshToken string `json:"refresh_token" required:"true"` SiteId string `json:"site_id"` ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"` DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` EnableDirectUpload bool `json:"enable_direct_upload" default:"false" help:"Enable direct upload from client to OneDrive"` } var config = driver.Config{ Name: "Onedrive", LocalSort: true, DefaultRoot: "/", } func init() { op.RegisterDriver(func() driver.Driver { return &Onedrive{} }) } ================================================ FILE: drivers/onedrive/types.go ================================================ package onedrive import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type Host struct { Oauth string Api string } type TokenErr struct { Error string `json:"error"` ErrorDescription string `json:"error_description"` } type RespErr struct { Error struct { Code string `json:"code"` Message string `json:"message"` } `json:"error"` } type File struct { Id string `json:"id"` Name string `json:"name"` Size int64 `json:"size"` FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo"` Url string `json:"@microsoft.graph.downloadUrl"` File *struct { MimeType string `json:"mimeType"` } `json:"file"` Thumbnails []struct { Medium struct { Url string `json:"url"` } `json:"medium"` } `json:"thumbnails"` ParentReference struct { DriveId string `json:"driveId"` } `json:"parentReference"` } type Object struct { model.ObjThumb ParentID string } func fileToObj(f File, parentID string) *Object { thumb := "" if len(f.Thumbnails) > 0 { thumb = f.Thumbnails[0].Medium.Url } return &Object{ ObjThumb: model.ObjThumb{ Object: model.Object{ ID: f.Id, Name: f.Name, Size: f.Size, Modified: f.FileSystemInfo.LastModifiedDateTime, IsFolder: f.File == nil, }, Thumbnail: model.Thumbnail{Thumbnail: thumb}, //Url: model.Url{Url: f.Url}, }, ParentID: parentID, } } type Files struct { Value []File `json:"value"` NextLink string `json:"@odata.nextLink"` } // Metadata represents a request to update Metadata. // It includes only the writeable properties. // omitempty is intentionally included for all, per https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_update?view=odsp-graph-online#request-body type Metadata struct { Description string `json:"description,omitempty"` // Provides a user-visible description of the item. Read-write. Only on OneDrive Personal. Undocumented limit of 1024 characters. FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo,omitempty"` // File system information on client. Read-write. } // FileSystemInfoFacet contains properties that are reported by the // device's local file system for the local version of an item. This // facet can be used to specify the last modified date or created date // of the item as it was on the local device. type FileSystemInfoFacet struct { CreatedDateTime time.Time `json:"createdDateTime,omitempty"` // The UTC date and time the file was created on a client. LastModifiedDateTime time.Time `json:"lastModifiedDateTime,omitempty"` // The UTC date and time the file was last modified on a client. } type DriveResp struct { ID string `json:"id"` DriveType string `json:"driveType"` Quota struct { Deleted uint64 `json:"deleted"` Remaining int64 `json:"remaining"` State string `json:"state"` Total int64 `json:"total"` Used int64 `json:"used"` } `json:"quota"` } ================================================ FILE: drivers/onedrive/util.go ================================================ package onedrive import ( "context" "errors" "fmt" "io" "net/http" stdpath "path" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" ) var onedriveHostMap = map[string]Host{ "global": { Oauth: "https://login.microsoftonline.com", Api: "https://graph.microsoft.com", }, "cn": { Oauth: "https://login.chinacloudapi.cn", Api: "https://microsoftgraph.chinacloudapi.cn", }, "us": { Oauth: "https://login.microsoftonline.us", Api: "https://graph.microsoft.us", }, "de": { Oauth: "https://login.microsoftonline.de", Api: "https://graph.microsoft.de", }, } func (d *Onedrive) GetMetaUrl(auth bool, path string) string { host, _ := onedriveHostMap[d.Region] path = utils.EncodePath(path, true) if auth { return host.Oauth } if d.IsSharepoint { if path == "/" || path == "\\" { return fmt.Sprintf("%s/v1.0/sites/%s/drive/root", host.Api, d.SiteId) } else { return fmt.Sprintf("%s/v1.0/sites/%s/drive/root:%s:", host.Api, d.SiteId, path) } } else { if path == "/" || path == "\\" { return fmt.Sprintf("%s/v1.0/me/drive/root", host.Api) } else { return fmt.Sprintf("%s/v1.0/me/drive/root:%s:", host.Api, path) } } } func (d *Onedrive) refreshToken() error { var err error for i := 0; i < 3; i++ { err = d._refreshToken() if err == nil { break } } return err } func (d *Onedrive) _refreshToken() error { // 使用在线API刷新Token，无需ClientID和ClientSecret if d.UseOnlineAPI && len(d.APIAddress) > 0 { u := d.APIAddress var resp struct { RefreshToken string `json:"refresh_token"` AccessToken string `json:"access_token"` ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, "server_use": "true", "driver_txt": "onedrive_pr", }). Get(u) if err != nil { return err } if resp.RefreshToken == "" || resp.AccessToken == "" { if resp.ErrorMessage != "" { return fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) } return fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") } d.AccessToken = resp.AccessToken d.RefreshToken = resp.RefreshToken op.MustSaveDriverStorage(d) return nil } // 使用本地客户端的情况下检查是否为空 if d.ClientID == "" || d.ClientSecret == "" { return fmt.Errorf("empty ClientID or ClientSecret") } // 走原有的刷新逻辑 url := d.GetMetaUrl(true, "") + "/common/oauth2/v2.0/token" var resp base.TokenResp var e TokenErr _, err := base.RestyClient.R().SetResult(&resp).SetError(&e).SetFormData(map[string]string{ "grant_type": "refresh_token", "client_id": d.ClientID, "client_secret": d.ClientSecret, "redirect_uri": d.RedirectUri, "refresh_token": d.RefreshToken, }).Post(url) if err != nil { return err } if e.Error != "" { return fmt.Errorf("%s", e.ErrorDescription) } if resp.RefreshToken == "" { return errs.EmptyToken } d.RefreshToken, d.AccessToken = resp.RefreshToken, resp.AccessToken op.MustSaveDriverStorage(d) return nil } func (d *Onedrive) Request(url string, method string, callback base.ReqCallback, resp interface{}, noRetry ...bool) ([]byte, error) { if d.ref != nil { return d.ref.Request(url, method, callback, resp) } req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e RespErr req.SetError(&e) res, err := req.Execute(method, url) if err != nil { return nil, err } if e.Error.Code != "" { if e.Error.Code == "InvalidAuthenticationToken" && !utils.IsBool(noRetry...) { err = d.refreshToken() if err != nil { return nil, err } return d.Request(url, method, callback, resp) } return nil, errors.New(e.Error.Message) } return res.Body(), nil } func (d *Onedrive) getFiles(path string) ([]File, error) { var res []File nextLink := d.GetMetaUrl(false, path) + "/children?$top=1000&$expand=thumbnails($select=medium)&$select=id,name,size,fileSystemInfo,content.downloadUrl,file,parentReference" for nextLink != "" { var files Files _, err := d.Request(nextLink, http.MethodGet, nil, &files) if err != nil { return nil, err } res = append(res, files.Value...) nextLink = files.NextLink } return res, nil } func (d *Onedrive) GetFile(path string) (*File, error) { var file File u := d.GetMetaUrl(false, path) _, err := d.Request(u, http.MethodGet, nil, &file) return &file, err } func (d *Onedrive) upSmall(ctx context.Context, dstDir model.Obj, stream model.FileStreamer) error { filepath := stdpath.Join(dstDir.GetPath(), stream.GetName()) // 1. upload new file // ApiDoc: https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online url := d.GetMetaUrl(false, filepath) + "/content" _, err := d.Request(url, http.MethodPut, func(req *resty.Request) { req.SetBody(driver.NewLimitedUploadStream(ctx, stream)).SetContext(ctx) }, nil) if err != nil { return fmt.Errorf("onedrive: Failed to upload new file(path=%v): %w", filepath, err) } // 2. update metadata err = d.updateMetadata(ctx, stream, filepath) if err != nil { return fmt.Errorf("onedrive: Failed to update file(path=%v) metadata: %w", filepath, err) } return nil } func (d *Onedrive) updateMetadata(ctx context.Context, stream model.FileStreamer, filepath string) error { url := d.GetMetaUrl(false, filepath) metadata := toAPIMetadata(stream) // ApiDoc: https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_update?view=odsp-graph-online _, err := d.Request(url, http.MethodPatch, func(req *resty.Request) { req.SetBody(metadata).SetContext(ctx) }, nil) return err } func toAPIMetadata(stream model.FileStreamer) Metadata { metadata := Metadata{ FileSystemInfo: &FileSystemInfoFacet{}, } if !stream.ModTime().IsZero() { metadata.FileSystemInfo.LastModifiedDateTime = stream.ModTime() } if !stream.CreateTime().IsZero() { metadata.FileSystemInfo.CreatedDateTime = stream.CreateTime() } if stream.CreateTime().IsZero() && !stream.ModTime().IsZero() { metadata.FileSystemInfo.CreatedDateTime = stream.CreateTime() } return metadata } func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { url := d.GetMetaUrl(false, stdpath.Join(dstDir.GetPath(), stream.GetName())) + "/createUploadSession" metadata := map[string]any{"item": toAPIMetadata(stream)} res, err := d.Request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(metadata).SetContext(ctx) }, nil) if err != nil { return err } DEFAULT := d.ChunkSize * 1024 * 1024 ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT), &up) if err != nil { return err } uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() var finish int64 = 0 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } left := stream.GetSize() - finish byteSize := min(left, DEFAULT) utils.Log.Debugf("[Onedrive] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) rd, err := ss.GetSectionReader(finish, byteSize) if err != nil { return err } err = retry.Do( func() error { rd.Seek(0, io.SeekStart) req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, driver.NewLimitedUploadStream(ctx, rd)) if err != nil { return err } req.ContentLength = byteSize req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession switch { case res.StatusCode >= 500 && res.StatusCode <= 504: return fmt.Errorf("server error: %d", res.StatusCode) case res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200: data, _ := io.ReadAll(res.Body) return errors.New(string(data)) default: return nil } }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) ss.FreeSectionReader(rd) if err != nil { return err } finish += byteSize up(float64(finish) * 100 / float64(stream.GetSize())) } return nil } func (d *Onedrive) getDrive(ctx context.Context) (*DriveResp, error) { var api string host, _ := onedriveHostMap[d.Region] if d.IsSharepoint { api = fmt.Sprintf("%s/v1.0/sites/%s/drive", host.Api, d.SiteId) } else { api = fmt.Sprintf("%s/v1.0/me/drive", host.Api) } var resp DriveResp _, err := d.Request(api, http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }, &resp, true) if err != nil { return nil, err } return &resp, nil } func (d *Onedrive) getDirectUploadInfo(ctx context.Context, path string) (*model.HttpDirectUploadInfo, error) { // Create upload session url := d.GetMetaUrl(false, path) + "/createUploadSession" metadata := map[string]any{ "item": map[string]any{ "@microsoft.graph.conflictBehavior": "rename", }, } res, err := d.Request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(metadata).SetContext(ctx) }, nil) if err != nil { return nil, err } uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() if uploadUrl == "" { return nil, fmt.Errorf("failed to get upload URL from response") } return &model.HttpDirectUploadInfo{ UploadURL: uploadUrl, ChunkSize: d.ChunkSize * 1024 * 1024, // Convert MB to bytes Method: "PUT", }, nil } ================================================ FILE: drivers/onedrive_app/driver.go ================================================ package onedrive_app import ( "context" "fmt" "net/http" "net/url" "path" "sync" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) type OnedriveAPP struct { model.Storage Addition AccessToken string root *Object mutex sync.Mutex } func (d *OnedriveAPP) Config() driver.Config { return config } func (d *OnedriveAPP) GetAddition() driver.Additional { return &d.Addition } func (d *OnedriveAPP) Init(ctx context.Context) error { if d.ChunkSize < 1 { d.ChunkSize = 5 } return d.accessToken() } func (d *OnedriveAPP) Drop(ctx context.Context) error { return nil } func (d *OnedriveAPP) GetRoot(ctx context.Context) (model.Obj, error) { if d.root != nil { return d.root, nil } d.mutex.Lock() defer d.mutex.Unlock() root := &Object{ ObjThumb: model.ObjThumb{ Object: model.Object{ ID: "root", Path: d.RootFolderPath, Name: "root", Size: 0, Modified: d.Modified, Ctime: d.Modified, IsFolder: true, }, }, ParentID: "", } if !utils.PathEqual(d.RootFolderPath, "/") { // get root folder id url := d.GetMetaUrl(false, d.RootFolderPath) var resp struct { Id string `json:"id"` } _, err := d.Request(url, http.MethodGet, nil, &resp) if err != nil { return nil, err } root.ID = resp.Id } d.root = root return d.root, nil } func (d *OnedriveAPP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(dir.GetPath()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { obj := fileToObj(src, dir.GetID()) obj.Path = path.Join(dir.GetPath(), obj.GetName()) return obj, nil }) } func (d *OnedriveAPP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { f, err := d.GetFile(file.GetPath()) if err != nil { return nil, err } if f.File == nil { return nil, errs.NotFile } u := f.Url if d.CustomHost != "" { _u, err := url.Parse(f.Url) if err != nil { return nil, err } _u.Host = d.CustomHost u = _u.String() } return &model.Link{ URL: u, }, nil } func (d *OnedriveAPP) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { url := d.GetMetaUrl(false, parentDir.GetPath()) + "/children" data := base.Json{ "name": dirName, "folder": base.Json{}, "@microsoft.graph.conflictBehavior": "rename", } _, err := d.Request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *OnedriveAPP) Move(ctx context.Context, srcObj, dstDir model.Obj) error { parentPath := "" if dstDir.GetID() == "" { parentPath = dstDir.GetPath() if utils.PathEqual(parentPath, "/") { parentPath = path.Join("/drive/root", parentPath) } else { parentPath = path.Join("/drive/root:/", parentPath) } } data := base.Json{ "parentReference": base.Json{ "id": dstDir.GetID(), "path": parentPath, }, "name": srcObj.GetName(), } url := d.GetMetaUrl(false, srcObj.GetPath()) _, err := d.Request(url, http.MethodPatch, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *OnedriveAPP) Rename(ctx context.Context, srcObj model.Obj, newName string) error { var parentID string if o, ok := srcObj.(*Object); ok { parentID = o.ParentID } else { return fmt.Errorf("srcObj is not Object") } if parentID == "" { parentID = "root" } data := base.Json{ "parentReference": base.Json{ "id": parentID, }, "name": newName, } url := d.GetMetaUrl(false, srcObj.GetPath()) _, err := d.Request(url, http.MethodPatch, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *OnedriveAPP) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { dst, err := d.GetFile(dstDir.GetPath()) if err != nil { return err } data := base.Json{ "parentReference": base.Json{ "driveId": dst.ParentReference.DriveId, "id": dst.Id, }, "name": srcObj.GetName(), } url := d.GetMetaUrl(false, srcObj.GetPath()) + "/copy" _, err = d.Request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *OnedriveAPP) Remove(ctx context.Context, obj model.Obj) error { url := d.GetMetaUrl(false, obj.GetPath()) _, err := d.Request(url, http.MethodDelete, nil, nil) return err } func (d *OnedriveAPP) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { var err error if stream.GetSize() <= 4*1024*1024 { err = d.upSmall(ctx, dstDir, stream) } else { err = d.upBig(ctx, dstDir, stream, up) } return err } func (d *OnedriveAPP) GetDetails(ctx context.Context) (*model.StorageDetails, error) { if d.DisableDiskUsage { return nil, errs.NotImplement } drive, err := d.getDrive(ctx) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: drive.Quota.Total, UsedSpace: drive.Quota.Used, }, }, nil } func (d *OnedriveAPP) GetDirectUploadTools() []string { if !d.EnableDirectUpload { return nil } return []string{"HttpDirect"} } func (d *OnedriveAPP) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) { if !d.EnableDirectUpload { return nil, errs.NotImplement } return d.getDirectUploadInfo(ctx, path.Join(dstDir.GetPath(), fileName)) } var _ driver.Driver = (*OnedriveAPP)(nil) ================================================ FILE: drivers/onedrive_app/meta.go ================================================ package onedrive_app import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Region string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"` ClientID string `json:"client_id" required:"true"` ClientSecret string `json:"client_secret" required:"true"` TenantID string `json:"tenant_id"` Email string `json:"email"` ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"` DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` EnableDirectUpload bool `json:"enable_direct_upload" default:"false" help:"Enable direct upload from client to OneDrive"` } var config = driver.Config{ Name: "OnedriveAPP", LocalSort: true, DefaultRoot: "/", } func init() { op.RegisterDriver(func() driver.Driver { return &OnedriveAPP{} }) } ================================================ FILE: drivers/onedrive_app/types.go ================================================ package onedrive_app import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type Host struct { Oauth string Api string } type TokenErr struct { Error string `json:"error"` ErrorDescription string `json:"error_description"` } type RespErr struct { Error struct { Code string `json:"code"` Message string `json:"message"` } `json:"error"` } type File struct { Id string `json:"id"` Name string `json:"name"` Size int64 `json:"size"` LastModifiedDateTime time.Time `json:"lastModifiedDateTime"` Url string `json:"@microsoft.graph.downloadUrl"` File *struct { MimeType string `json:"mimeType"` } `json:"file"` Thumbnails []struct { Medium struct { Url string `json:"url"` } `json:"medium"` } `json:"thumbnails"` ParentReference struct { DriveId string `json:"driveId"` } `json:"parentReference"` } type Object struct { model.ObjThumb ParentID string } func fileToObj(f File, parentID string) *Object { thumb := "" if len(f.Thumbnails) > 0 { thumb = f.Thumbnails[0].Medium.Url } return &Object{ ObjThumb: model.ObjThumb{ Object: model.Object{ ID: f.Id, Name: f.Name, Size: f.Size, Modified: f.LastModifiedDateTime, IsFolder: f.File == nil, }, Thumbnail: model.Thumbnail{Thumbnail: thumb}, //Url: model.Url{Url: f.Url}, }, ParentID: parentID, } } type Files struct { Value []File `json:"value"` NextLink string `json:"@odata.nextLink"` } type DriveResp struct { ID string `json:"id"` DriveType string `json:"driveType"` Quota struct { Deleted uint64 `json:"deleted"` Remaining int64 `json:"remaining"` State string `json:"state"` Total int64 `json:"total"` Used int64 `json:"used"` } `json:"quota"` } ================================================ FILE: drivers/onedrive_app/util.go ================================================ package onedrive_app import ( "context" "errors" "fmt" "io" "net/http" stdpath "path" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" ) var onedriveHostMap = map[string]Host{ "global": { Oauth: "https://login.microsoftonline.com", Api: "https://graph.microsoft.com", }, "cn": { Oauth: "https://login.chinacloudapi.cn", Api: "https://microsoftgraph.chinacloudapi.cn", }, "us": { Oauth: "https://login.microsoftonline.us", Api: "https://graph.microsoft.us", }, "de": { Oauth: "https://login.microsoftonline.de", Api: "https://graph.microsoft.de", }, } func (d *OnedriveAPP) GetMetaUrl(auth bool, path string) string { host := onedriveHostMap[d.Region] path = utils.EncodePath(path, true) if auth { return host.Oauth } if path == "/" || path == "\\" { return fmt.Sprintf("%s/v1.0/users/%s/drive/root", host.Api, d.Email) } return fmt.Sprintf("%s/v1.0/users/%s/drive/root:%s:", host.Api, d.Email, path) } func (d *OnedriveAPP) accessToken() error { var err error for i := 0; i < 3; i++ { err = d._accessToken() if err == nil { break } } return err } func (d *OnedriveAPP) _accessToken() error { url := d.GetMetaUrl(true, "") + "/" + d.TenantID + "/oauth2/token" var resp base.TokenResp var e TokenErr _, err := base.RestyClient.R().SetResult(&resp).SetError(&e).SetFormData(map[string]string{ "grant_type": "client_credentials", "client_id": d.ClientID, "client_secret": d.ClientSecret, "resource": onedriveHostMap[d.Region].Api + "/", "scope": onedriveHostMap[d.Region].Api + "/.default", }).Post(url) if err != nil { return err } if e.Error != "" { return fmt.Errorf("%s", e.ErrorDescription) } if resp.AccessToken == "" { return errs.EmptyToken } d.AccessToken = resp.AccessToken op.MustSaveDriverStorage(d) return nil } func (d *OnedriveAPP) Request(url string, method string, callback base.ReqCallback, resp interface{}, noRetry ...bool) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e RespErr req.SetError(&e) res, err := req.Execute(method, url) if err != nil { return nil, err } if e.Error.Code != "" { if e.Error.Code == "InvalidAuthenticationToken" && !utils.IsBool(noRetry...) { err = d.accessToken() if err != nil { return nil, err } return d.Request(url, method, callback, resp) } return nil, errors.New(e.Error.Message) } return res.Body(), nil } func (d *OnedriveAPP) getFiles(path string) ([]File, error) { var res []File nextLink := d.GetMetaUrl(false, path) + "/children?$top=1000&$expand=thumbnails($select=medium)&$select=id,name,size,lastModifiedDateTime,content.downloadUrl,file,parentReference" for nextLink != "" { var files Files _, err := d.Request(nextLink, http.MethodGet, nil, &files) if err != nil { return nil, err } res = append(res, files.Value...) nextLink = files.NextLink } return res, nil } func (d *OnedriveAPP) GetFile(path string) (*File, error) { var file File u := d.GetMetaUrl(false, path) _, err := d.Request(u, http.MethodGet, nil, &file) return &file, err } func (d *OnedriveAPP) upSmall(ctx context.Context, dstDir model.Obj, stream model.FileStreamer) error { url := d.GetMetaUrl(false, stdpath.Join(dstDir.GetPath(), stream.GetName())) + "/content" _, err := d.Request(url, http.MethodPut, func(req *resty.Request) { req.SetBody(driver.NewLimitedUploadStream(ctx, stream)).SetContext(ctx) }, nil) return err } func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { url := d.GetMetaUrl(false, stdpath.Join(dstDir.GetPath(), stream.GetName())) + "/createUploadSession" res, err := d.Request(url, http.MethodPost, nil, nil) if err != nil { return err } DEFAULT := d.ChunkSize * 1024 * 1024 ss, err := streamPkg.NewStreamSectionReader(stream, int(DEFAULT), &up) if err != nil { return err } uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() var finish int64 = 0 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } left := stream.GetSize() - finish byteSize := min(left, DEFAULT) utils.Log.Debugf("[OnedriveAPP] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) rd, err := ss.GetSectionReader(finish, byteSize) if err != nil { return err } err = retry.Do( func() error { rd.Seek(0, io.SeekStart) req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, driver.NewLimitedUploadStream(ctx, rd)) if err != nil { return err } req.ContentLength = byteSize req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) res, err := base.HttpClient.Do(req) if err != nil { return err } defer res.Body.Close() // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession switch { case res.StatusCode >= 500 && res.StatusCode <= 504: return fmt.Errorf("server error: %d", res.StatusCode) case res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200: data, _ := io.ReadAll(res.Body) return errors.New(string(data)) default: return nil } }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) ss.FreeSectionReader(rd) if err != nil { return err } finish += byteSize up(float64(finish) * 100 / float64(stream.GetSize())) } return nil } func (d *OnedriveAPP) getDrive(ctx context.Context) (*DriveResp, error) { host, _ := onedriveHostMap[d.Region] api := fmt.Sprintf("%s/v1.0/users/%s/drive", host.Api, d.Email) var resp DriveResp _, err := d.Request(api, http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }, &resp, true) if err != nil { return nil, err } return &resp, nil } func (d *OnedriveAPP) getDirectUploadInfo(ctx context.Context, path string) (*model.HttpDirectUploadInfo, error) { // Create upload session url := d.GetMetaUrl(false, path) + "/createUploadSession" metadata := map[string]any{ "item": map[string]any{ "@microsoft.graph.conflictBehavior": "rename", }, } res, err := d.Request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(metadata).SetContext(ctx) }, nil) if err != nil { return nil, err } uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() if uploadUrl == "" { return nil, fmt.Errorf("failed to get upload URL from response") } return &model.HttpDirectUploadInfo{ UploadURL: uploadUrl, ChunkSize: d.ChunkSize * 1024 * 1024, // Convert MB to bytes Method: "PUT", }, nil } ================================================ FILE: drivers/onedrive_sharelink/driver.go ================================================ package onedrive_sharelink import ( "context" "fmt" "io" "net/http" stdpath "path" "strings" "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" ) const headerTTL = 25 * time.Minute type OnedriveSharelink struct { model.Storage cron *cron.Cron Addition headerMu sync.RWMutex sg singleflight.Group[http.Header] } func (d *OnedriveSharelink) Config() driver.Config { return config } func (d *OnedriveSharelink) GetAddition() driver.Additional { return &d.Addition } func (d *OnedriveSharelink) Init(ctx context.Context) error { // Initialize error variable var err error // If there is "-my" in the URL, it is NOT a SharePoint link d.IsSharepoint = !strings.Contains(d.ShareLinkURL, "-my") // Initialize cron job to run every hour d.cron = cron.NewCron(time.Hour * 1) d.cron.Do(func() { var err error h, err := d.getHeaders(ctx) if err != nil { log.Errorf("%+v", err) return } d.storeHeaders(h) }) // Get initial headers h, err := d.getHeaders(ctx) if err != nil { return err } d.storeHeaders(h) return nil } func (d *OnedriveSharelink) Drop(ctx context.Context) error { return nil } func (d *OnedriveSharelink) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(ctx, dir.GetPath()) if err != nil { return nil, err } // Convert the slice of files to the required model.Obj format return utils.SliceConvert(files, func(src Item) (model.Obj, error) { obj := fileToObj(src) obj.Path = stdpath.Join(dir.GetPath(), obj.GetName()) return obj, nil }) } func (d *OnedriveSharelink) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { // Get the unique ID of the file uniqueId := file.GetID() // Cut the first char and the last char uniqueId = uniqueId[1 : len(uniqueId)-1] url := d.downloadLinkPrefix + uniqueId header, err := d.getValidHeaders(ctx) if err != nil { return nil, err } return &model.Link{ URL: url, Header: header, RangeReader: rangeReaderFunc(func(ctx context.Context, hr http_range.Range) (io.ReadCloser, error) { return d.rangeReadWithRefresh(ctx, url, hr) }), }, nil } func (d *OnedriveSharelink) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { // TODO create folder, optional return errs.NotImplement } func (d *OnedriveSharelink) Move(ctx context.Context, srcObj, dstDir model.Obj) error { // TODO move obj, optional return errs.NotImplement } func (d *OnedriveSharelink) Rename(ctx context.Context, srcObj model.Obj, newName string) error { // TODO rename obj, optional return errs.NotImplement } func (d *OnedriveSharelink) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { // TODO copy obj, optional return errs.NotImplement } func (d *OnedriveSharelink) Remove(ctx context.Context, obj model.Obj) error { // TODO remove obj, optional return errs.NotImplement } func (d *OnedriveSharelink) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { // TODO upload file, optional return errs.NotImplement } //func (d *OnedriveSharelink) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*OnedriveSharelink)(nil) // rangeReadWithRefresh tries once with current headers, and if the response // looks invalid (error status or html login page), it refreshes headers and retries. func (d *OnedriveSharelink) rangeReadWithRefresh(ctx context.Context, url string, hr http_range.Range) (io.ReadCloser, error) { tryOnce := func(header http.Header) (io.ReadCloser, error) { h := cloneHeader(header) if h == nil { h = http.Header{} } h = http_range.ApplyRangeToHttpHeader(hr, h) resp, err := net.RequestHttp(ctx, http.MethodGet, h, url) if err != nil { return nil, err } ct := strings.ToLower(resp.Header.Get("Content-Type")) if strings.Contains(ct, "text/html") { _ = resp.Body.Close() return nil, fmt.Errorf("unexpected html response") } return resp.Body, nil } header, err := d.getValidHeaders(ctx) if err != nil { return nil, err } if body, err := tryOnce(header); err == nil { return body, nil } // refresh and retry once header, err = d.refreshHeaders(ctx) if err != nil { return nil, err } return tryOnce(header) } type rangeReaderFunc func(ctx context.Context, hr http_range.Range) (io.ReadCloser, error) func (f rangeReaderFunc) RangeRead(ctx context.Context, hr http_range.Range) (io.ReadCloser, error) { return f(ctx, hr) } func cloneHeader(header http.Header) http.Header { if header == nil { return nil } return header.Clone() } func (d *OnedriveSharelink) headerSnapshot() http.Header { d.headerMu.RLock() defer d.headerMu.RUnlock() return cloneHeader(d.Headers) } func (d *OnedriveSharelink) storeHeaders(header http.Header) { if header == nil { return } d.headerMu.Lock() d.Headers = header d.HeaderTime = time.Now().Unix() d.headerMu.Unlock() } func (d *OnedriveSharelink) headersExpired() bool { d.headerMu.RLock() defer d.headerMu.RUnlock() return time.Since(time.Unix(d.HeaderTime, 0)) > headerTTL } func (d *OnedriveSharelink) refreshHeaders(ctx context.Context) (http.Header, error) { header, err, _ := d.sg.Do("refresh", func() (http.Header, error) { h, e := d.getHeaders(ctx) if e != nil { return nil, e } d.storeHeaders(h) return h, nil }) return header, err } func (d *OnedriveSharelink) getValidHeaders(ctx context.Context) (http.Header, error) { if h := d.headerSnapshot(); h != nil && !d.headersExpired() { return h, nil } h, err := d.refreshHeaders(ctx) if err != nil { if h2 := d.headerSnapshot(); h2 != nil { log.Warnf("onedrive_sharelink: use cached headers after refresh failure: %+v", err) return h2, nil } return nil, err } return h, nil } ================================================ FILE: drivers/onedrive_sharelink/meta.go ================================================ package onedrive_sharelink import ( "net/http" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath ShareLinkURL string `json:"url" required:"true"` ShareLinkPassword string `json:"password"` IsSharepoint bool downloadLinkPrefix string Headers http.Header HeaderTime int64 } var config = driver.Config{ Name: "Onedrive Sharelink", OnlyProxy: true, NoUpload: true, DefaultRoot: "/", } func init() { op.RegisterDriver(func() driver.Driver { return &OnedriveSharelink{} }) } ================================================ FILE: drivers/onedrive_sharelink/types.go ================================================ package onedrive_sharelink import ( "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) // FolderResp represents the structure of the folder response from the OneDrive API. type FolderResp struct { // Data holds the nested structure of the response. Data struct { Legacy struct { RenderListData struct { ListData struct { Items []Item `json:"Row"` // Items contains the list of items in the folder. } `json:"ListData"` } `json:"renderListDataAsStream"` } `json:"legacy"` } `json:"data"` } // Item represents an individual item in the folder. type Item struct { ObjType string `json:"FSObjType"` // ObjType indicates if the item is a file or folder. Name string `json:"FileLeafRef"` // Name is the name of the item. ModifiedTime time.Time `json:"Modified."` // ModifiedTime is the last modified time of the item. Size string `json:"File_x0020_Size"` // Size is the size of the item in string format. Id string `json:"UniqueId"` // Id is the unique identifier of the item. } // fileToObj converts an Item to an ObjThumb. func fileToObj(f Item) *model.ObjThumb { // Convert Size from string to int64. size, _ := strconv.ParseInt(f.Size, 10, 64) // Convert ObjType from string to int. objtype, _ := strconv.Atoi(f.ObjType) // Create a new ObjThumb with the converted values. file := &model.ObjThumb{ Object: model.Object{ Name: f.Name, Modified: f.ModifiedTime, Size: size, IsFolder: objtype == 1, // Check if the item is a folder. ID: f.Id, }, Thumbnail: model.Thumbnail{}, } return file } // GraphQLNEWRequest represents the structure of a new GraphQL request. type GraphQLNEWRequest struct { ListData struct { NextHref string `json:"NextHref"` // NextHref is the link to the next set of data. Row []Item `json:"Row"` // Row contains the list of items. } `json:"ListData"` } // GraphQLRequest represents the structure of a GraphQL request. type GraphQLRequest struct { Data struct { Legacy struct { RenderListDataAsStream struct { ListData struct { NextHref string `json:"NextHref"` // NextHref is the link to the next set of data. Row []Item `json:"Row"` // Row contains the list of items. } `json:"ListData"` ViewMetadata struct { ListViewXml string `json:"ListViewXml"` // ListViewXml contains the XML of the list view. } `json:"ViewMetadata"` } `json:"renderListDataAsStream"` } `json:"legacy"` } `json:"data"` } ================================================ FILE: drivers/onedrive_sharelink/util.go ================================================ package onedrive_sharelink import ( "context" "crypto/tls" "encoding/json" "fmt" "io" "net/http" "net/url" "regexp" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" log "github.com/sirupsen/logrus" "golang.org/x/net/html" ) // NewNoRedirectClient creates an HTTP client that doesn't follow redirects func NewNoRedirectCLient() *http.Client { return &http.Client{ Timeout: time.Hour * 48, Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, }, // Prevent following redirects CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } } // getCookiesWithPassword fetches cookies required for authenticated access using the provided password func getCookiesWithPassword(link, password string) (string, error) { // Send GET request resp, err := http.Get(link) if err != nil { return "", err } defer resp.Body.Close() // Parse the HTML response doc, err := html.Parse(resp.Body) if err != nil { return "", err } // Initialize variables to store form data var viewstate, eventvalidation, postAction string // Recursive function to find input fields by their IDs var findInputFields func(*html.Node) findInputFields = func(n *html.Node) { if n.Type == html.ElementNode && n.Data == "input" { for _, attr := range n.Attr { if attr.Key == "id" { switch attr.Val { case "__VIEWSTATE": viewstate = getAttrValue(n, "value") case "__EVENTVALIDATION": eventvalidation = getAttrValue(n, "value") } } } } if n.Type == html.ElementNode && n.Data == "form" { for _, attr := range n.Attr { if attr.Key == "id" && attr.Val == "inputForm" { postAction = getAttrValue(n, "action") } } } for c := n.FirstChild; c != nil; c = c.NextSibling { findInputFields(c) } } findInputFields(doc) // Prepare the new URL for the POST request linkParts, err := url.Parse(link) if err != nil { return "", err } newURL := fmt.Sprintf("%s://%s%s", linkParts.Scheme, linkParts.Host, postAction) // Prepare the request body data := url.Values{ "txtPassword": []string{password}, "__EVENTVALIDATION": []string{eventvalidation}, "__VIEWSTATE": []string{viewstate}, "__VIEWSTATEENCRYPTED": []string{""}, } client := &http.Client{ CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } // Send the POST request, preventing redirects resp, err = client.PostForm(newURL, data) if err != nil { return "", err } // Extract the desired cookie value cookie := resp.Cookies() var fedAuthCookie string for _, c := range cookie { if c.Name == "FedAuth" { fedAuthCookie = c.Value break } } if fedAuthCookie == "" { return "", fmt.Errorf("wrong password") } return fmt.Sprintf("FedAuth=%s;", fedAuthCookie), nil } // getAttrValue retrieves the value of the specified attribute from an HTML node func getAttrValue(n *html.Node, key string) string { for _, attr := range n.Attr { if attr.Key == key { return attr.Val } } return "" } // getHeaders constructs and returns the necessary HTTP headers for accessing the OneDrive share link func (d *OnedriveSharelink) getHeaders(ctx context.Context) (http.Header, error) { header := http.Header{} header.Set("User-Agent", base.UserAgent) header.Set("accept-language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6") // Save current timestamp to d.HeaderTime d.HeaderTime = time.Now().Unix() if d.ShareLinkPassword == "" { // Create a no-redirect client clientNoDirect := NewNoRedirectCLient() req, err := http.NewRequestWithContext(ctx, http.MethodGet, d.ShareLinkURL, nil) if err != nil { return nil, err } // Set headers for the request req.Header = header answerNoRedirect, err := clientNoDirect.Do(req) if err != nil { return nil, err } redirectUrl := answerNoRedirect.Header.Get("Location") log.Debugln("redirectUrl:", redirectUrl) if redirectUrl == "" { return nil, fmt.Errorf("password protected link. Please provide password") } header.Set("Cookie", answerNoRedirect.Header.Get("Set-Cookie")) header.Set("Referer", redirectUrl) // Extract the host part of the redirect URL and set it as the authority u, err := url.Parse(redirectUrl) if err != nil { return nil, err } header.Set("authority", u.Host) return header, nil } else { cookie, err := getCookiesWithPassword(d.ShareLinkURL, d.ShareLinkPassword) if err != nil { return nil, err } header.Set("Cookie", cookie) header.Set("Referer", d.ShareLinkURL) header.Set("authority", strings.Split(strings.Split(d.ShareLinkURL, "//")[1], "/")[0]) return header, nil } } // getFiles retrieves the files from the OneDrive share link at the specified path func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, error) { clientNoDirect := NewNoRedirectCLient() req, err := http.NewRequestWithContext(ctx, http.MethodGet, d.ShareLinkURL, nil) if err != nil { return nil, err } header := req.Header redirectUrl := "" if d.ShareLinkPassword == "" { header.Set("User-Agent", base.UserAgent) header.Set("accept-language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6") req.Header = header answerNoRedirect, err := clientNoDirect.Do(req) if err != nil { return nil, err } redirectUrl = answerNoRedirect.Header.Get("Location") } else { header = d.Headers req.Header = header answerNoRedirect, err := clientNoDirect.Do(req) if err != nil { return nil, err } redirectUrl = answerNoRedirect.Header.Get("Location") } redirectSplitURL := strings.Split(redirectUrl, "/") req.Header = d.Headers downloadLinkPrefix := "" rootFolderPre := "" // Determine the appropriate URL and root folder based on whether the link is SharePoint if d.IsSharepoint { // update req url req.URL, err = url.Parse(redirectUrl) if err != nil { return nil, err } // Get redirectUrl answer, err := clientNoDirect.Do(req) if err != nil { d.Headers, err = d.getHeaders(ctx) if err != nil { return nil, err } return d.getFiles(ctx, path) } defer answer.Body.Close() re := regexp.MustCompile(`templateUrl":"(.*?)"`) body, err := io.ReadAll(answer.Body) if err != nil { return nil, err } template := re.FindString(string(body)) template = template[strings.Index(template, "templateUrl\":\"")+len("templateUrl\":\""):] template = template[:strings.Index(template, "?id=")] template = template[:strings.LastIndex(template, "/")] downloadLinkPrefix = template + "/download.aspx?UniqueId=" params, err := url.ParseQuery(redirectUrl[strings.Index(redirectUrl, "?")+1:]) if err != nil { return nil, err } rootFolderPre = params.Get("id") } else { redirectUrlCut := redirectUrl[:strings.LastIndex(redirectUrl, "/")] downloadLinkPrefix = redirectUrlCut + "/download.aspx?UniqueId=" params, err := url.ParseQuery(redirectUrl[strings.Index(redirectUrl, "?")+1:]) if err != nil { return nil, err } rootFolderPre = params.Get("id") } d.downloadLinkPrefix = downloadLinkPrefix rootFolder, err := url.QueryUnescape(rootFolderPre) if err != nil { return nil, err } log.Debugln("rootFolder:", rootFolder) // Extract the relative path up to and including "Documents" relativePath := strings.Split(rootFolder, "Documents")[0] + "Documents" // URL encode the relative path relativeUrl := url.QueryEscape(relativePath) // Replace underscores and hyphens in the encoded relative path relativeUrl = strings.Replace(relativeUrl, "_", "%5F", -1) relativeUrl = strings.Replace(relativeUrl, "-", "%2D", -1) // If the path is not the root, append the path to the root folder if path != "/" { rootFolder = rootFolder + path } // URL encode the full root folder path rootFolderUrl := url.QueryEscape(rootFolder) // Replace underscores and hyphens in the encoded root folder URL rootFolderUrl = strings.Replace(rootFolderUrl, "_", "%5F", -1) rootFolderUrl = strings.Replace(rootFolderUrl, "-", "%2D", -1) log.Debugln("relativePath:", relativePath, "relativeUrl:", relativeUrl, "rootFolder:", rootFolder, "rootFolderUrl:", rootFolderUrl) // Construct the GraphQL query with the encoded paths graphqlVar := fmt.Sprintf(`{"query":"query (\n $listServerRelativeUrl: String!,$renderListDataAsStreamParameters: RenderListDataAsStreamParameters!,$renderListDataAsStreamQueryString: String!\n )\n {\n \n legacy {\n \n renderListDataAsStream(\n listServerRelativeUrl: $listServerRelativeUrl,\n parameters: $renderListDataAsStreamParameters,\n queryString: $renderListDataAsStreamQueryString\n )\n }\n \n \n perf {\n executionTime\n overheadTime\n parsingTime\n queryCount\n validationTime\n resolvers {\n name\n queryCount\n resolveTime\n waitTime\n }\n }\n }","variables":{"listServerRelativeUrl":"%s","renderListDataAsStreamParameters":{"renderOptions":5707527,"allowMultipleValueFilterForTaxonomyFields":true,"addRequiredFields":true,"folderServerRelativeUrl":"%s"},"renderListDataAsStreamQueryString":"@a1=\'%s\'&RootFolder=%s&TryNewExperienceSingle=TRUE"}}`, relativePath, rootFolder, relativeUrl, rootFolderUrl) tempHeader := make(http.Header) for k, v := range d.Headers { tempHeader[k] = v } tempHeader["Content-Type"] = []string{"application/json;odata=verbose"} client := &http.Client{} postUrl := strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/v2.1/graphql" req, err = http.NewRequest(http.MethodPost, postUrl, strings.NewReader(graphqlVar)) if err != nil { return nil, err } req.Header = tempHeader resp, err := client.Do(req) if err != nil { d.Headers, err = d.getHeaders(ctx) if err != nil { return nil, err } return d.getFiles(ctx, path) } defer resp.Body.Close() var graphqlReq GraphQLRequest json.NewDecoder(resp.Body).Decode(&graphqlReq) log.Debugln("graphqlReq:", graphqlReq) filesData := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row if graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref != "" { nextHref := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE" nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1) log.Debugln("nextHref:", nextHref) filesData = append(filesData, graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row...) listViewXml := graphqlReq.Data.Legacy.RenderListDataAsStream.ViewMetadata.ListViewXml log.Debugln("listViewXml:", listViewXml) renderListDataAsStreamVar := `{"parameters":{"__metadata":{"type":"SP.RenderListDataParameters"},"RenderOptions":1216519,"ViewXml":"REPLACEME","AllowMultipleValueFilterForTaxonomyFields":true,"AddRequiredFields":true}}` listViewXml = strings.Replace(listViewXml, `"`, `\"`, -1) renderListDataAsStreamVar = strings.Replace(renderListDataAsStreamVar, "REPLACEME", listViewXml, -1) graphqlReqNEW := GraphQLNEWRequest{} postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref req, _ = http.NewRequest(http.MethodPost, postUrl, strings.NewReader(renderListDataAsStreamVar)) req.Header = tempHeader resp, err := client.Do(req) if err != nil { d.Headers, err = d.getHeaders(ctx) if err != nil { return nil, err } return d.getFiles(ctx, path) } defer resp.Body.Close() json.NewDecoder(resp.Body).Decode(&graphqlReqNEW) for graphqlReqNEW.ListData.NextHref != "" { graphqlReqNEW = GraphQLNEWRequest{} postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref req, _ = http.NewRequest(http.MethodPost, postUrl, strings.NewReader(renderListDataAsStreamVar)) req.Header = tempHeader resp, err := client.Do(req) if err != nil { d.Headers, err = d.getHeaders(ctx) if err != nil { return nil, err } return d.getFiles(ctx, path) } defer resp.Body.Close() json.NewDecoder(resp.Body).Decode(&graphqlReqNEW) nextHref = graphqlReqNEW.ListData.NextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE" nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1) filesData = append(filesData, graphqlReqNEW.ListData.Row...) } filesData = append(filesData, graphqlReqNEW.ListData.Row...) } else { filesData = append(filesData, graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row...) } return filesData, nil } ================================================ FILE: drivers/openlist/driver.go ================================================ package openlist import ( "context" "fmt" "io" "net/http" "net/url" "path" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) type OpenList struct { model.Storage Addition } func (d *OpenList) Config() driver.Config { return config } func (d *OpenList) GetAddition() driver.Additional { return &d.Addition } func (d *OpenList) Init(ctx context.Context) error { d.Addition.Address = strings.TrimSuffix(d.Addition.Address, "/") var resp common.Resp[MeResp] _, _, err := d.request("/me", http.MethodGet, func(req *resty.Request) { req.SetResult(&resp) }) if err != nil { return err } // if the username is not empty and the username is not the same as the current username, then login again if d.Username != resp.Data.Username { err = d.login() if err != nil { return err } } // re-get the user info _, _, err = d.request("/me", http.MethodGet, func(req *resty.Request) { req.SetResult(&resp) }) if err != nil { return err } if resp.Data.Role == model.GUEST { u := d.Address + "/api/public/settings" res, err := base.RestyClient.R().Get(u) if err != nil { return err } allowMounted := utils.Json.Get(res.Body(), "data", conf.AllowMounted).ToString() == "true" if !allowMounted { return fmt.Errorf("the site does not allow mounted") } } return err } func (d *OpenList) Drop(ctx context.Context) error { return nil } func (d *OpenList) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var resp common.Resp[FsListResp] _, _, err := d.request("/fs/list", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(ListReq{ PageReq: model.PageReq{ Page: 1, PerPage: 0, }, Path: dir.GetPath(), Password: d.MetaPassword, Refresh: false, }) }) if err != nil { return nil, err } var files []model.Obj for _, f := range resp.Data.Content { file := model.ObjThumb{ Object: model.Object{ Name: f.Name, Path: path.Join(dir.GetPath(), f.Name), Modified: f.Modified, Ctime: f.Created, Size: f.Size, IsFolder: f.IsDir, HashInfo: utils.FromString(f.HashInfo), }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumb}, } files = append(files, &file) } return files, nil } func (d *OpenList) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp common.Resp[FsGetResp] headers := map[string]string{ "User-Agent": base.UserAgent, } // if PassUAToUpsteam is true, then pass the user-agent to the upstream if d.PassUAToUpsteam { userAgent := args.Header.Get("user-agent") if userAgent != "" { headers["User-Agent"] = userAgent } } // if PassIPToUpsteam is true, then pass the ip address to the upstream if d.PassIPToUpsteam { ip := args.IP if ip != "" { headers["X-Forwarded-For"] = ip headers["X-Real-Ip"] = ip } } _, _, err := d.request("/fs/get", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(FsGetReq{ Path: file.GetPath(), Password: d.MetaPassword, }).SetHeaders(headers) }) if err != nil { return nil, err } return &model.Link{ URL: resp.Data.RawURL, }, nil } func (d *OpenList) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { _, _, err := d.request("/fs/mkdir", http.MethodPost, func(req *resty.Request) { req.SetBody(MkdirOrLinkReq{ Path: path.Join(parentDir.GetPath(), dirName), }) }) return err } func (d *OpenList) Move(ctx context.Context, srcObj, dstDir model.Obj) error { _, _, err := d.request("/fs/move", http.MethodPost, func(req *resty.Request) { req.SetBody(MoveCopyReq{ SrcDir: path.Dir(srcObj.GetPath()), DstDir: dstDir.GetPath(), Names: []string{srcObj.GetName()}, }) }) return err } func (d *OpenList) Rename(ctx context.Context, srcObj model.Obj, newName string) error { _, _, err := d.request("/fs/rename", http.MethodPost, func(req *resty.Request) { req.SetBody(RenameReq{ Path: srcObj.GetPath(), Name: newName, }) }) return err } func (d *OpenList) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { _, _, err := d.request("/fs/copy", http.MethodPost, func(req *resty.Request) { req.SetBody(MoveCopyReq{ SrcDir: path.Dir(srcObj.GetPath()), DstDir: dstDir.GetPath(), Names: []string{srcObj.GetName()}, }) }) return err } func (d *OpenList) Remove(ctx context.Context, obj model.Obj) error { _, _, err := d.request("/fs/remove", http.MethodPost, func(req *resty.Request) { req.SetBody(RemoveReq{ Dir: path.Dir(obj.GetPath()), Names: []string{obj.GetName()}, }) }) return err } func (d *OpenList) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, }) req, err := http.NewRequestWithContext(ctx, http.MethodPut, d.Address+"/api/fs/put", reader) if err != nil { return err } req.Header.Set("Authorization", d.Token) req.Header.Set("File-Path", path.Join(dstDir.GetPath(), s.GetName())) req.Header.Set("Password", d.MetaPassword) if md5 := s.GetHash().GetHash(utils.MD5); len(md5) > 0 { req.Header.Set("X-File-Md5", md5) } if sha1 := s.GetHash().GetHash(utils.SHA1); len(sha1) > 0 { req.Header.Set("X-File-Sha1", sha1) } if sha256 := s.GetHash().GetHash(utils.SHA256); len(sha256) > 0 { req.Header.Set("X-File-Sha256", sha256) } req.ContentLength = s.GetSize() // client := base.NewHttpClient() // client.Timeout = time.Hour * 6 res, err := base.HttpClient.Do(req) if err != nil { return err } bytes, err := io.ReadAll(res.Body) if err != nil { return err } log.Debugf("[openlist] response body: %s", string(bytes)) if res.StatusCode >= 400 { return fmt.Errorf("request failed, status: %s", res.Status) } code := utils.Json.Get(bytes, "code").ToInt() if code != 200 { if code == 401 || code == 403 { err = d.login() if err != nil { return err } } return fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(bytes, "message").ToString()) } return nil } func (d *OpenList) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { if !d.ForwardArchiveReq { return nil, errs.NotImplement } var resp common.Resp[ArchiveMetaResp] _, code, err := d.request("/fs/archive/meta", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(ArchiveMetaReq{ ArchivePass: args.Password, Password: d.MetaPassword, Path: obj.GetPath(), Refresh: false, }) }) if code == 202 { return nil, errs.WrongArchivePassword } if err != nil { return nil, err } var tree []model.ObjTree if resp.Data.Content != nil { tree = make([]model.ObjTree, 0, len(resp.Data.Content)) for _, content := range resp.Data.Content { tree = append(tree, &content) } } return &model.ArchiveMetaInfo{ Comment: resp.Data.Comment, Encrypted: resp.Data.Encrypted, Tree: tree, }, nil } func (d *OpenList) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { if !d.ForwardArchiveReq { return nil, errs.NotImplement } var resp common.Resp[ArchiveListResp] _, code, err := d.request("/fs/archive/list", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(ArchiveListReq{ ArchiveMetaReq: ArchiveMetaReq{ ArchivePass: args.Password, Password: d.MetaPassword, Path: obj.GetPath(), Refresh: false, }, PageReq: model.PageReq{ Page: 1, PerPage: 0, }, InnerPath: args.InnerPath, }) }) if code == 202 { return nil, errs.WrongArchivePassword } if err != nil { return nil, err } var files []model.Obj for _, f := range resp.Data.Content { file := model.ObjThumb{ Object: model.Object{ Name: f.Name, Modified: f.Modified, Ctime: f.Created, Size: f.Size, IsFolder: f.IsDir, HashInfo: utils.FromString(f.HashInfo), }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumb}, } files = append(files, &file) } return files, nil } func (d *OpenList) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { if !d.ForwardArchiveReq { return nil, errs.NotSupport } var resp common.Resp[ArchiveMetaResp] _, _, err := d.request("/fs/archive/meta", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(ArchiveMetaReq{ ArchivePass: args.Password, Password: d.MetaPassword, Path: obj.GetPath(), Refresh: false, }) }) if err != nil { return nil, err } return &model.Link{ URL: fmt.Sprintf("%s?inner=%s&pass=%s&sign=%s", resp.Data.RawURL, utils.EncodePath(args.InnerPath, true), url.QueryEscape(args.Password), resp.Data.Sign), }, nil } func (d *OpenList) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { if !d.ForwardArchiveReq { return errs.NotImplement } dir, name := path.Split(srcObj.GetPath()) _, _, err := d.request("/fs/archive/decompress", http.MethodPost, func(req *resty.Request) { req.SetBody(DecompressReq{ ArchivePass: args.Password, CacheFull: args.CacheFull, DstDir: dstDir.GetPath(), InnerPath: args.InnerPath, Name: []string{name}, PutIntoNewDir: args.PutIntoNewDir, SrcDir: dir, Overwrite: args.Overwrite, }) }) return err } func (d *OpenList) ResolveLinkCacheMode(_ string) driver.LinkCacheMode { var mode driver.LinkCacheMode if d.PassIPToUpsteam { mode |= driver.LinkCacheIP } if d.PassUAToUpsteam { mode |= driver.LinkCacheUA } return mode } var _ driver.Driver = (*OpenList)(nil) ================================================ FILE: drivers/openlist/meta.go ================================================ package openlist import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Address string `json:"url" required:"true"` MetaPassword string `json:"meta_password"` Username string `json:"username"` Password string `json:"password"` Token string `json:"token"` PassIPToUpsteam bool `json:"pass_ip_to_upsteam" default:"true"` PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"` ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"` } var config = driver.Config{ Name: "OpenList", LocalSort: true, DefaultRoot: "/", ProxyRangeOption: true, LinkCacheMode: driver.LinkCacheAuto, } func init() { op.RegisterDriver(func() driver.Driver { return &OpenList{} }) } ================================================ FILE: drivers/openlist/types.go ================================================ package openlist import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type ListReq struct { model.PageReq Path string `json:"path" form:"path"` Password string `json:"password" form:"password"` Refresh bool `json:"refresh"` } type ObjResp struct { Name string `json:"name"` Size int64 `json:"size"` IsDir bool `json:"is_dir"` Modified time.Time `json:"modified"` Created time.Time `json:"created"` Sign string `json:"sign"` Thumb string `json:"thumb"` Type int `json:"type"` HashInfo string `json:"hashinfo"` } type FsListResp struct { Content []ObjResp `json:"content"` Total int64 `json:"total"` Readme string `json:"readme"` Write bool `json:"write"` Provider string `json:"provider"` } type FsGetReq struct { Path string `json:"path" form:"path"` Password string `json:"password" form:"password"` } type FsGetResp struct { ObjResp RawURL string `json:"raw_url"` Readme string `json:"readme"` Provider string `json:"provider"` Related []ObjResp `json:"related"` } type MkdirOrLinkReq struct { Path string `json:"path" form:"path"` } type MoveCopyReq struct { SrcDir string `json:"src_dir"` DstDir string `json:"dst_dir"` Names []string `json:"names"` } type RenameReq struct { Path string `json:"path"` Name string `json:"name"` } type RemoveReq struct { Dir string `json:"dir"` Names []string `json:"names"` } type LoginResp struct { Token string `json:"token"` } type MeResp struct { Id int `json:"id"` Username string `json:"username"` Password string `json:"password"` BasePath string `json:"base_path"` Role int `json:"role"` Disabled bool `json:"disabled"` Permission int `json:"permission"` SsoId string `json:"sso_id"` Otp bool `json:"otp"` } type ArchiveMetaReq struct { ArchivePass string `json:"archive_pass"` Password string `json:"password"` Path string `json:"path"` Refresh bool `json:"refresh"` } type TreeResp struct { ObjResp Children []TreeResp `json:"children"` hashCache *utils.HashInfo } func (t *TreeResp) GetSize() int64 { return t.Size } func (t *TreeResp) GetName() string { return t.Name } func (t *TreeResp) ModTime() time.Time { return t.Modified } func (t *TreeResp) CreateTime() time.Time { return t.Created } func (t *TreeResp) IsDir() bool { return t.ObjResp.IsDir } func (t *TreeResp) GetHash() utils.HashInfo { return utils.FromString(t.HashInfo) } func (t *TreeResp) GetID() string { return "" } func (t *TreeResp) GetPath() string { return "" } func (t *TreeResp) GetChildren() []model.ObjTree { ret := make([]model.ObjTree, 0, len(t.Children)) for _, child := range t.Children { ret = append(ret, &child) } return ret } func (t *TreeResp) Thumb() string { return t.ObjResp.Thumb } type ArchiveMetaResp struct { Comment string `json:"comment"` Encrypted bool `json:"encrypted"` Content []TreeResp `json:"content"` RawURL string `json:"raw_url"` Sign string `json:"sign"` } type ArchiveListReq struct { model.PageReq ArchiveMetaReq InnerPath string `json:"inner_path"` } type ArchiveListResp struct { Content []ObjResp `json:"content"` Total int64 `json:"total"` } type DecompressReq struct { ArchivePass string `json:"archive_pass"` CacheFull bool `json:"cache_full"` DstDir string `json:"dst_dir"` InnerPath string `json:"inner_path"` Name []string `json:"name"` PutIntoNewDir bool `json:"put_into_new_dir"` SrcDir string `json:"src_dir"` Overwrite bool `json:"overwrite"` } ================================================ FILE: drivers/openlist/util.go ================================================ package openlist import ( "fmt" "net/http" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) func (d *OpenList) login() error { if d.Username == "" { return nil } var resp common.Resp[LoginResp] _, _, err := d.request("/auth/login", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(base.Json{ "username": d.Username, "password": d.Password, }) }) if err != nil { return err } d.Token = resp.Data.Token op.MustSaveDriverStorage(d) return nil } func (d *OpenList) request(api, method string, callback base.ReqCallback, retry ...bool) ([]byte, int, error) { url := d.Address + "/api" + api req := base.RestyClient.R() req.SetHeader("Authorization", d.Token) if callback != nil { callback(req) } res, err := req.Execute(method, url) if err != nil { code := 0 if res != nil { code = res.StatusCode() } return nil, code, err } log.Debugf("[openlist] response body: %s", res.String()) if res.StatusCode() >= 400 { return nil, res.StatusCode(), fmt.Errorf("request failed, status: %s", res.Status()) } code := utils.Json.Get(res.Body(), "code").ToInt() if code != 200 { if (code == 401 || code == 403) && !utils.IsBool(retry...) { err = d.login() if err != nil { return nil, code, err } return d.request(api, method, callback, true) } return nil, code, fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(res.Body(), "message").ToString()) } return res.Body(), 200, nil } ================================================ FILE: drivers/openlist_share/driver.go ================================================ package openlist_share import ( "context" "fmt" "net/http" "net/url" stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/go-resty/resty/v2" ) type OpenListShare struct { model.Storage Addition serverArchivePreview bool } func (d *OpenListShare) Config() driver.Config { return config } func (d *OpenListShare) GetAddition() driver.Additional { return &d.Addition } func (d *OpenListShare) Init(ctx context.Context) error { d.Addition.Address = strings.TrimSuffix(d.Addition.Address, "/") var settings common.Resp[map[string]string] _, _, err := d.request("/public/settings", http.MethodGet, func(req *resty.Request) { req.SetResult(&settings) }) if err != nil { return err } d.serverArchivePreview = settings.Data["share_archive_preview"] == "true" return nil } func (d *OpenListShare) Drop(ctx context.Context) error { return nil } func (d *OpenListShare) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var resp common.Resp[FsListResp] _, _, err := d.request("/fs/list", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(ListReq{ PageReq: model.PageReq{ Page: 1, PerPage: 0, }, Path: stdpath.Join(fmt.Sprintf("/@s/%s", d.ShareId), dir.GetPath()), Password: d.Pwd, Refresh: false, }) }) if err != nil { return nil, err } var files []model.Obj for _, f := range resp.Data.Content { file := model.ObjThumb{ Object: model.Object{ Name: f.Name, Path: stdpath.Join(dir.GetPath(), f.Name), Modified: f.Modified, Ctime: f.Created, Size: f.Size, IsFolder: f.IsDir, HashInfo: utils.FromString(f.HashInfo), }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumb}, } files = append(files, &file) } return files, nil } func (d *OpenListShare) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { path := utils.FixAndCleanPath(stdpath.Join(d.ShareId, file.GetPath())) u := fmt.Sprintf("%s/sd%s?pwd=%s", d.Address, path, d.Pwd) return &model.Link{URL: u}, nil } func (d *OpenListShare) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { if !d.serverArchivePreview || !d.ForwardArchiveReq { return nil, errs.NotImplement } var resp common.Resp[ArchiveMetaResp] _, code, err := d.request("/fs/archive/meta", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(ArchiveMetaReq{ ArchivePass: args.Password, Path: stdpath.Join(fmt.Sprintf("/@s/%s", d.ShareId), obj.GetPath()), Password: d.Pwd, Refresh: false, }) }) if code == 202 { return nil, errs.WrongArchivePassword } if err != nil { return nil, err } var tree []model.ObjTree if resp.Data.Content != nil { tree = make([]model.ObjTree, 0, len(resp.Data.Content)) for _, content := range resp.Data.Content { tree = append(tree, &content) } } return &model.ArchiveMetaInfo{ Comment: resp.Data.Comment, Encrypted: resp.Data.Encrypted, Tree: tree, }, nil } func (d *OpenListShare) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { if !d.serverArchivePreview || !d.ForwardArchiveReq { return nil, errs.NotImplement } var resp common.Resp[ArchiveListResp] _, code, err := d.request("/fs/archive/list", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(ArchiveListReq{ ArchiveMetaReq: ArchiveMetaReq{ ArchivePass: args.Password, Path: stdpath.Join(fmt.Sprintf("/@s/%s", d.ShareId), obj.GetPath()), Password: d.Pwd, Refresh: false, }, PageReq: model.PageReq{ Page: 1, PerPage: 0, }, InnerPath: args.InnerPath, }) }) if code == 202 { return nil, errs.WrongArchivePassword } if err != nil { return nil, err } var files []model.Obj for _, f := range resp.Data.Content { file := model.ObjThumb{ Object: model.Object{ Name: f.Name, Modified: f.Modified, Ctime: f.Created, Size: f.Size, IsFolder: f.IsDir, HashInfo: utils.FromString(f.HashInfo), }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumb}, } files = append(files, &file) } return files, nil } func (d *OpenListShare) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { if !d.serverArchivePreview || !d.ForwardArchiveReq { return nil, errs.NotSupport } path := utils.FixAndCleanPath(stdpath.Join(d.ShareId, obj.GetPath())) u := fmt.Sprintf("%s/sad%s?pwd=%s&inner=%s&pass=%s", d.Address, path, d.Pwd, utils.EncodePath(args.InnerPath, true), url.QueryEscape(args.Password)) return &model.Link{URL: u}, nil } var _ driver.Driver = (*OpenListShare)(nil) ================================================ FILE: drivers/openlist_share/meta.go ================================================ package openlist_share import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Address string `json:"url" required:"true"` ShareId string `json:"sid" required:"true"` Pwd string `json:"pwd"` ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"` } var config = driver.Config{ Name: "OpenListShare", LocalSort: true, NoUpload: true, DefaultRoot: "/", } func init() { op.RegisterDriver(func() driver.Driver { return &OpenListShare{} }) } ================================================ FILE: drivers/openlist_share/types.go ================================================ package openlist_share import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type ListReq struct { model.PageReq Path string `json:"path" form:"path"` Password string `json:"password" form:"password"` Refresh bool `json:"refresh"` } type ObjResp struct { Name string `json:"name"` Size int64 `json:"size"` IsDir bool `json:"is_dir"` Modified time.Time `json:"modified"` Created time.Time `json:"created"` Sign string `json:"sign"` Thumb string `json:"thumb"` Type int `json:"type"` HashInfo string `json:"hashinfo"` } type FsListResp struct { Content []ObjResp `json:"content"` Total int64 `json:"total"` Readme string `json:"readme"` Write bool `json:"write"` Provider string `json:"provider"` } type ArchiveMetaReq struct { ArchivePass string `json:"archive_pass"` Password string `json:"password"` Path string `json:"path"` Refresh bool `json:"refresh"` } type TreeResp struct { ObjResp Children []TreeResp `json:"children"` hashCache *utils.HashInfo } func (t *TreeResp) GetSize() int64 { return t.Size } func (t *TreeResp) GetName() string { return t.Name } func (t *TreeResp) ModTime() time.Time { return t.Modified } func (t *TreeResp) CreateTime() time.Time { return t.Created } func (t *TreeResp) IsDir() bool { return t.ObjResp.IsDir } func (t *TreeResp) GetHash() utils.HashInfo { return utils.FromString(t.HashInfo) } func (t *TreeResp) GetID() string { return "" } func (t *TreeResp) GetPath() string { return "" } func (t *TreeResp) GetChildren() []model.ObjTree { ret := make([]model.ObjTree, 0, len(t.Children)) for _, child := range t.Children { ret = append(ret, &child) } return ret } func (t *TreeResp) Thumb() string { return t.ObjResp.Thumb } type ArchiveMetaResp struct { Comment string `json:"comment"` Encrypted bool `json:"encrypted"` Content []TreeResp `json:"content"` RawURL string `json:"raw_url"` Sign string `json:"sign"` } type ArchiveListReq struct { model.PageReq ArchiveMetaReq InnerPath string `json:"inner_path"` } type ArchiveListResp struct { Content []ObjResp `json:"content"` Total int64 `json:"total"` } ================================================ FILE: drivers/openlist_share/util.go ================================================ package openlist_share import ( "fmt" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) func (d *OpenListShare) request(api, method string, callback base.ReqCallback) ([]byte, int, error) { url := d.Address + "/api" + api req := base.RestyClient.R() if callback != nil { callback(req) } res, err := req.Execute(method, url) if err != nil { code := 0 if res != nil { code = res.StatusCode() } return nil, code, err } if res.StatusCode() >= 400 { return nil, res.StatusCode(), fmt.Errorf("request failed, status: %s", res.Status()) } code := utils.Json.Get(res.Body(), "code").ToInt() if code != 200 { return nil, code, fmt.Errorf("request failed, code: %d, message: %s", code, utils.Json.Get(res.Body(), "message").ToString()) } return res.Body(), 200, nil } ================================================ FILE: drivers/pikpak/driver.go ================================================ package pikpak import ( "context" "encoding/json" "fmt" "net/http" "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) type PikPak struct { model.Storage Addition *Common RefreshToken string AccessToken string } func (d *PikPak) Config() driver.Config { return config } func (d *PikPak) GetAddition() driver.Additional { return &d.Addition } func (d *PikPak) Init(ctx context.Context) (err error) { if d.Common == nil { d.Common = &Common{ client: base.NewRestyClient(), CaptchaToken: "", UserID: "", DeviceID: utils.GetMD5EncodeStr(d.Username + d.Password), UserAgent: "", RefreshCTokenCk: func(token string) { d.Common.CaptchaToken = token op.MustSaveDriverStorage(d) }, } } if d.Platform == "android" { d.ClientID = AndroidClientID d.ClientSecret = AndroidClientSecret d.ClientVersion = AndroidClientVersion d.PackageName = AndroidPackageName d.Algorithms = AndroidAlgorithms d.UserAgent = BuildCustomUserAgent(utils.GetMD5EncodeStr(d.Username+d.Password), AndroidClientID, AndroidPackageName, AndroidSdkVersion, AndroidClientVersion, AndroidPackageName, "") } else if d.Platform == "web" { d.ClientID = WebClientID d.ClientSecret = WebClientSecret d.ClientVersion = WebClientVersion d.PackageName = WebPackageName d.Algorithms = WebAlgorithms d.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36" } else if d.Platform == "pc" { d.ClientID = PCClientID d.ClientSecret = PCClientSecret d.ClientVersion = PCClientVersion d.PackageName = PCPackageName d.Algorithms = PCAlgorithms d.UserAgent = "MainWindow Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) PikPak/2.6.11.4955 Chrome/100.0.4896.160 Electron/18.3.15 Safari/537.36" } if d.Addition.CaptchaToken != "" && d.Addition.RefreshToken == "" { d.SetCaptchaToken(d.Addition.CaptchaToken) } if d.Addition.DeviceID != "" { d.SetDeviceID(d.Addition.DeviceID) } else { d.Addition.DeviceID = d.Common.DeviceID op.MustSaveDriverStorage(d) } // 如果已经有RefreshToken，直接获取AccessToken if d.Addition.RefreshToken != "" { if err = d.refreshToken(d.Addition.RefreshToken); err != nil { return err } } else { // 如果没有填写RefreshToken，尝试登录 获取 refreshToken if err = d.login(); err != nil { return err } } // 获取CaptchaToken err = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.net/drive/v1/files"), d.Common.GetUserID()) if err != nil { return err } // 更新UserAgent if d.Platform == "android" { d.Common.UserAgent = BuildCustomUserAgent(utils.GetMD5EncodeStr(d.Username+d.Password), AndroidClientID, AndroidPackageName, AndroidSdkVersion, AndroidClientVersion, AndroidPackageName, d.Common.UserID) } // 保存 有效的 RefreshToken d.Addition.RefreshToken = d.RefreshToken op.MustSaveDriverStorage(d) return nil } func (d *PikPak) Drop(ctx context.Context) error { return nil } func (d *PikPak) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(dir.GetID()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return fileToObj(src), nil }) } func (d *PikPak) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp File var url string queryParams := map[string]string{ "_magic": "2021", "usage": "FETCH", "thumbnail_size": "SIZE_LARGE", } if !d.DisableMediaLink { queryParams["usage"] = "CACHE" } _, err := d.request(fmt.Sprintf("https://api-drive.mypikpak.net/drive/v1/files/%s", file.GetID()), http.MethodGet, func(req *resty.Request) { req.SetContext(ctx). SetQueryParams(queryParams) }, &resp) if err != nil { return nil, err } url = resp.WebContentLink if !d.DisableMediaLink && len(resp.Medias) > 0 && resp.Medias[0].Link.Url != "" { log.Debugln("use media link") url = resp.Medias[0].Link.Url } return &model.Link{ URL: url, }, nil } func (d *PikPak) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "kind": "drive#folder", "parent_id": parentDir.GetID(), "name": dirName, }) }, nil) return err } func (d *PikPak) Move(ctx context.Context, srcObj, dstDir model.Obj) error { _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files:batchMove", http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "ids": []string{srcObj.GetID()}, "to": base.Json{ "parent_id": dstDir.GetID(), }, }) }, nil) return err } func (d *PikPak) Rename(ctx context.Context, srcObj model.Obj, newName string) error { _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files/"+srcObj.GetID(), http.MethodPatch, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "name": newName, }) }, nil) return err } func (d *PikPak) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files:batchCopy", http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "ids": []string{srcObj.GetID()}, "to": base.Json{ "parent_id": dstDir.GetID(), }, }) }, nil) return err } func (d *PikPak) Remove(ctx context.Context, obj model.Obj) error { _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files:batchTrash", http.MethodPost, func(req *resty.Request) { req.SetContext(ctx).SetBody(base.Json{ "ids": []string{obj.GetID()}, }) }, nil) return err } func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { sha1Str := stream.GetHash().GetHash(hash_extend.GCID) if len(sha1Str) < hash_extend.GCID.Width { var err error _, sha1Str, err = streamPkg.CacheFullAndHash(stream, &up, hash_extend.GCID, stream.GetSize()) if err != nil { return err } } var resp UploadTaskData res, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "kind": "drive#file", "name": stream.GetName(), "size": stream.GetSize(), "hash": strings.ToUpper(sha1Str), "upload_type": "UPLOAD_TYPE_RESUMABLE", "objProvider": base.Json{"provider": "UPLOAD_TYPE_UNKNOWN"}, "parent_id": dstDir.GetID(), "folder_type": "NORMAL", }) }, &resp) if err != nil { return err } // 秒传成功 if resp.Resumable == nil { log.Debugln(string(res)) return nil } params := resp.Resumable.Params // endpoint := strings.Join(strings.Split(params.Endpoint, ".")[1:], ".") // web 端上传 返回的endpoint 为 `mypikpak.net` | android 端上传 返回的endpoint 为 `vip-lixian-07.mypikpak.net`· if d.Addition.Platform == "android" { params.Endpoint = "mypikpak.net" } if stream.GetSize() <= 10*utils.MB { // 文件大小 小于10MB，改用普通模式上传 return d.UploadByOSS(ctx, ¶ms, stream, up) } // 分片上传 return d.UploadByMultipart(ctx, ¶ms, stream.GetSize(), stream, up) } func (d *PikPak) GetDetails(ctx context.Context) (*model.StorageDetails, error) { var about AboutResponse _, err := d.request("https://api-drive.mypikpak.com/drive/v1/about", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }, &about) if err != nil { return nil, err } total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) if err != nil { return nil, err } used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: used, }, }, nil } // 离线下载文件 func (d *PikPak) OfflineDownload(ctx context.Context, fileUrl string, parentDir model.Obj, fileName string) (*OfflineTask, error) { requestBody := base.Json{ "kind": "drive#file", "name": fileName, "upload_type": "UPLOAD_TYPE_URL", "url": base.Json{ "url": fileUrl, }, "parent_id": parentDir.GetID(), "folder_type": "", } var resp OfflineDownloadResp _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(req *resty.Request) { req.SetContext(ctx). SetBody(requestBody) }, &resp) if err != nil { return nil, err } return &resp.Task, err } /* 获取离线下载任务列表 phase 可能的取值： PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING */ func (d *PikPak) OfflineList(ctx context.Context, nextPageToken string, phase []string) ([]OfflineTask, error) { res := make([]OfflineTask, 0) url := "https://api-drive.mypikpak.net/drive/v1/tasks" if len(phase) == 0 { phase = []string{"PHASE_TYPE_RUNNING", "PHASE_TYPE_ERROR", "PHASE_TYPE_COMPLETE", "PHASE_TYPE_PENDING"} } params := map[string]string{ "type": "offline", "thumbnail_size": "SIZE_SMALL", "limit": "10000", "page_token": nextPageToken, "with": "reference_resource", } // 处理 phase 参数 if len(phase) > 0 { filters := base.Json{ "phase": map[string]string{ "in": strings.Join(phase, ","), }, } filtersJSON, err := json.Marshal(filters) if err != nil { return nil, fmt.Errorf("failed to marshal filters: %w", err) } params["filters"] = string(filtersJSON) } var resp OfflineListResp _, err := d.request(url, http.MethodGet, func(req *resty.Request) { req.SetContext(ctx). SetQueryParams(params) }, &resp) if err != nil { return nil, fmt.Errorf("failed to get offline list: %w", err) } res = append(res, resp.Tasks...) return res, nil } func (d *PikPak) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error { url := "https://api-drive.mypikpak.net/drive/v1/tasks" params := map[string]string{ "task_ids": strings.Join(taskIDs, ","), "delete_files": strconv.FormatBool(deleteFiles), } _, err := d.request(url, http.MethodDelete, func(req *resty.Request) { req.SetContext(ctx). SetQueryParams(params) }, nil) if err != nil { return fmt.Errorf("failed to delete tasks %v: %w", taskIDs, err) } return nil } var _ driver.Driver = (*PikPak)(nil) ================================================ FILE: drivers/pikpak/meta.go ================================================ package pikpak import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootID Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` Platform string `json:"platform" required:"true" default:"web" type:"select" options:"android,web,pc"` RefreshToken string `json:"refresh_token" required:"true" default:""` CaptchaToken string `json:"captcha_token" default:""` DeviceID string `json:"device_id" required:"false" default:""` DisableMediaLink bool `json:"disable_media_link" default:"true"` } var config = driver.Config{ Name: "PikPak", LocalSort: true, } func init() { op.RegisterDriver(func() driver.Driver { return &PikPak{} }) } ================================================ FILE: drivers/pikpak/types.go ================================================ package pikpak import ( "fmt" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" ) type Files struct { Files []File `json:"files"` NextPageToken string `json:"next_page_token"` } type File struct { Id string `json:"id"` Kind string `json:"kind"` Name string `json:"name"` CreatedTime time.Time `json:"created_time"` ModifiedTime time.Time `json:"modified_time"` Hash string `json:"hash"` Size string `json:"size"` ThumbnailLink string `json:"thumbnail_link"` WebContentLink string `json:"web_content_link"` Medias []Media `json:"medias"` } func fileToObj(f File) *model.ObjThumb { size, _ := strconv.ParseInt(f.Size, 10, 64) return &model.ObjThumb{ Object: model.Object{ ID: f.Id, Name: f.Name, Size: size, Ctime: f.CreatedTime, Modified: f.ModifiedTime, IsFolder: f.Kind == "drive#folder", HashInfo: utils.NewHashInfo(hash_extend.GCID, f.Hash), }, Thumbnail: model.Thumbnail{ Thumbnail: f.ThumbnailLink, }, } } type Media struct { MediaId string `json:"media_id"` MediaName string `json:"media_name"` Video struct { Height int `json:"height"` Width int `json:"width"` Duration int `json:"duration"` BitRate int `json:"bit_rate"` FrameRate int `json:"frame_rate"` VideoCodec string `json:"video_codec"` AudioCodec string `json:"audio_codec"` VideoType string `json:"video_type"` } `json:"video"` Link struct { Url string `json:"url"` Token string `json:"token"` Expire time.Time `json:"expire"` } `json:"link"` NeedMoreQuota bool `json:"need_more_quota"` VipTypes []interface{} `json:"vip_types"` RedirectLink string `json:"redirect_link"` IconLink string `json:"icon_link"` IsDefault bool `json:"is_default"` Priority int `json:"priority"` IsOrigin bool `json:"is_origin"` ResolutionName string `json:"resolution_name"` IsVisible bool `json:"is_visible"` Category string `json:"category"` } type UploadTaskData struct { UploadType string `json:"upload_type"` // UPLOAD_TYPE_RESUMABLE Resumable *struct { Kind string `json:"kind"` Params S3Params `json:"params"` Provider string `json:"provider"` } `json:"resumable"` File File `json:"file"` } type S3Params struct { AccessKeyID string `json:"access_key_id"` AccessKeySecret string `json:"access_key_secret"` Bucket string `json:"bucket"` Endpoint string `json:"endpoint"` Expiration time.Time `json:"expiration"` Key string `json:"key"` SecurityToken string `json:"security_token"` } // 添加离线下载响应 type OfflineDownloadResp struct { File *string `json:"file"` Task OfflineTask `json:"task"` UploadType string `json:"upload_type"` URL struct { Kind string `json:"kind"` } `json:"url"` } // 离线下载列表 type OfflineListResp struct { ExpiresIn int64 `json:"expires_in"` NextPageToken string `json:"next_page_token"` Tasks []OfflineTask `json:"tasks"` } // offlineTask type OfflineTask struct { Callback string `json:"callback"` CreatedTime string `json:"created_time"` FileID string `json:"file_id"` FileName string `json:"file_name"` FileSize string `json:"file_size"` IconLink string `json:"icon_link"` ID string `json:"id"` Kind string `json:"kind"` Message string `json:"message"` Name string `json:"name"` Params Params `json:"params"` Phase string `json:"phase"` // PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING Progress int64 `json:"progress"` ReferenceResource ReferenceResource `json:"reference_resource"` Space string `json:"space"` StatusSize int64 `json:"status_size"` Statuses []string `json:"statuses"` ThirdTaskID string `json:"third_task_id"` Type string `json:"type"` UpdatedTime string `json:"updated_time"` UserID string `json:"user_id"` } type Params struct { Age string `json:"age"` MIMEType *string `json:"mime_type,omitempty"` PredictType string `json:"predict_type"` URL string `json:"url"` } type ReferenceResource struct { Type string `json:"@type"` Audit interface{} `json:"audit"` Hash string `json:"hash"` IconLink string `json:"icon_link"` ID string `json:"id"` Kind string `json:"kind"` Medias []Media `json:"medias"` MIMEType string `json:"mime_type"` Name string `json:"name"` Params map[string]interface{} `json:"params"` ParentID string `json:"parent_id"` Phase string `json:"phase"` Size string `json:"size"` Space string `json:"space"` Starred bool `json:"starred"` Tags []string `json:"tags"` ThumbnailLink string `json:"thumbnail_link"` } type ErrResp struct { ErrorCode int64 `json:"error_code"` ErrorMsg string `json:"error"` ErrorDescription string `json:"error_description"` } func (e *ErrResp) IsError() bool { return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" } func (e *ErrResp) Error() string { return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ErrorDescription: %s ", e.ErrorCode, e.ErrorMsg, e.ErrorDescription) } type CaptchaTokenRequest struct { Action string `json:"action"` CaptchaToken string `json:"captcha_token"` ClientID string `json:"client_id"` DeviceID string `json:"device_id"` Meta map[string]string `json:"meta"` RedirectUri string `json:"redirect_uri"` } type CaptchaTokenResponse struct { CaptchaToken string `json:"captcha_token"` ExpiresIn int64 `json:"expires_in"` Url string `json:"url"` } type AboutResponse struct { Quota struct { Limit string `json:"limit"` Usage string `json:"usage"` UsageInTrash string `json:"usage_in_trash"` IsUnlimited bool `json:"is_unlimited"` Complimentary string `json:"complimentary"` } `json:"quota"` ExpiresAt string `json:"expires_at"` UserType int `json:"user_type"` } ================================================ FILE: drivers/pikpak/util.go ================================================ package pikpak import ( "bytes" "context" "crypto/md5" "crypto/sha1" "encoding/hex" "fmt" "io" "net/http" "path/filepath" "regexp" "strings" "sync" "sync/atomic" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" netutil "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" ) var AndroidAlgorithms = []string{ "SOP04dGzk0TNO7t7t9ekDbAmx+eq0OI1ovEx", "nVBjhYiND4hZ2NCGyV5beamIr7k6ifAsAbl", "Ddjpt5B/Cit6EDq2a6cXgxY9lkEIOw4yC1GDF28KrA", "VVCogcmSNIVvgV6U+AochorydiSymi68YVNGiz", "u5ujk5sM62gpJOsB/1Gu/zsfgfZO", "dXYIiBOAHZgzSruaQ2Nhrqc2im", "z5jUTBSIpBN9g4qSJGlidNAutX6", "KJE2oveZ34du/g1tiimm", } var WebAlgorithms = []string{ "C9qPpZLN8ucRTaTiUMWYS9cQvWOE", "+r6CQVxjzJV6LCV", "F", "pFJRC", "9WXYIDGrwTCz2OiVlgZa90qpECPD6olt", "/750aCr4lm/Sly/c", "RB+DT/gZCrbV", "", "CyLsf7hdkIRxRm215hl", "7xHvLi2tOYP0Y92b", "ZGTXXxu8E/MIWaEDB+Sm/", "1UI3", "E7fP5Pfijd+7K+t6Tg/NhuLq0eEUVChpJSkrKxpO", "ihtqpG6FMt65+Xk+tWUH2", "NhXXU9rg4XXdzo7u5o", } var PCAlgorithms = []string{ "KHBJ07an7ROXDoK7Db", "G6n399rSWkl7WcQmw5rpQInurc1DkLmLJqE", "JZD1A3M4x+jBFN62hkr7VDhkkZxb9g3rWqRZqFAAb", "fQnw/AmSlbbI91Ik15gpddGgyU7U", "/Dv9JdPYSj3sHiWjouR95NTQff", "yGx2zuTjbWENZqecNI+edrQgqmZKP", "ljrbSzdHLwbqcRn", "lSHAsqCkGDGxQqqwrVu", "TsWXI81fD1", "vk7hBjawK/rOSrSWajtbMk95nfgf3", } const ( OSSUserAgent = "aliyun-sdk-android/2.9.13(Linux/Android 14/M2004j7ac;UKQ1.231108.001)" OssSecurityTokenHeaderName = "X-OSS-Security-Token" ThreadsNum = 10 ) const ( AndroidClientID = "YNxT9w7GMdWvEOKa" AndroidClientSecret = "dbw2OtmVEeuUvIptb1Coyg" AndroidClientVersion = "1.53.2" AndroidPackageName = "com.pikcloud.pikpak" AndroidSdkVersion = "2.0.6.206003" WebClientID = "YUMx5nI8ZU8Ap8pm" WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" WebClientVersion = "2.0.0" WebPackageName = "mypikpak.com" WebSdkVersion = "8.0.3" PCClientID = "YvtoWO6GNHiuCl7x" PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" PCClientVersion = "undefined" // 2.6.11.4955 PCPackageName = "mypikpak.com" PCSdkVersion = "8.0.3" ) func (d *PikPak) login() error { // 检查用户名和密码是否为空 if d.Addition.Username == "" || d.Addition.Password == "" { return errors.New("username or password is empty") } url := "https://user.mypikpak.net/v1/auth/signin" // 使用 用户填写的 CaptchaToken —————— (验证后的captcha_token) if d.GetCaptchaToken() == "" { if err := d.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), d.Username); err != nil { return err } } var e ErrResp res, err := base.RestyClient.SetRetryCount(1).R().SetError(&e).SetBody(base.Json{ "captcha_token": d.GetCaptchaToken(), "client_id": d.ClientID, "client_secret": d.ClientSecret, "username": d.Username, "password": d.Password, }).SetQueryParam("client_id", d.ClientID).Post(url) if err != nil { return err } if e.ErrorCode != 0 { return &e } data := res.Body() d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() d.AccessToken = jsoniter.Get(data, "access_token").ToString() d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) return nil } func (d *PikPak) refreshToken(refreshToken string) error { url := "https://user.mypikpak.net/v1/auth/token" var e ErrResp res, err := base.RestyClient.SetRetryCount(1).R().SetError(&e). SetHeader("user-agent", "").SetBody(base.Json{ "client_id": d.ClientID, "client_secret": d.ClientSecret, "grant_type": "refresh_token", "refresh_token": refreshToken, }).SetQueryParam("client_id", d.ClientID).Post(url) if err != nil { d.Status = err.Error() op.MustSaveDriverStorage(d) return err } if e.ErrorCode != 0 { if e.ErrorCode == 4126 { // 1. 未填写 username 或 password if d.Addition.Username == "" || d.Addition.Password == "" { return errors.New("refresh_token invalid, please re-provide refresh_token") } else { // refresh_token invalid, re-login return d.login() } } d.Status = e.Error() op.MustSaveDriverStorage(d) return errors.New(e.Error()) } data := res.Body() d.Status = "work" d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() d.AccessToken = jsoniter.Get(data, "access_token").ToString() d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) d.Addition.RefreshToken = d.RefreshToken op.MustSaveDriverStorage(d) return nil } func (d *PikPak) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeaders(map[string]string{ //"Authorization": "Bearer " + d.AccessToken, "User-Agent": d.GetUserAgent(), "X-Device-ID": d.GetDeviceID(), "X-Captcha-Token": d.GetCaptchaToken(), }) if d.AccessToken != "" { req.SetHeader("Authorization", "Bearer "+d.AccessToken) } if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e ErrResp req.SetError(&e) res, err := req.Execute(method, url) if err != nil { return nil, err } switch e.ErrorCode { case 0: return res.Body(), nil case 4122, 4121, 16: // access_token 过期 if err1 := d.refreshToken(d.RefreshToken); err1 != nil { return nil, err1 } return d.request(url, method, callback, resp) case 9: // 验证码token过期 if err = d.RefreshCaptchaTokenAtLogin(GetAction(method, url), d.GetUserID()); err != nil { return nil, err } return d.request(url, method, callback, resp) case 10: // 操作频繁 return nil, errors.New(e.ErrorDescription) default: return nil, errors.New(e.Error()) } } func (d *PikPak) getFiles(id string) ([]File, error) { res := make([]File, 0) pageToken := "first" for pageToken != "" { if pageToken == "first" { pageToken = "" } query := map[string]string{ "parent_id": id, "thumbnail_size": "SIZE_LARGE", "with_audit": "true", "limit": "100", "filters": `{"phase":{"eq":"PHASE_TYPE_COMPLETE"},"trashed":{"eq":false}}`, "page_token": pageToken, } var resp Files _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return nil, err } pageToken = resp.NextPageToken res = append(res, resp.Files...) } return res, nil } func GetAction(method string, url string) string { urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] return method + ":" + urlpath } type Common struct { client *resty.Client CaptchaToken string UserID string // 必要值,签名相关 ClientID string ClientSecret string ClientVersion string PackageName string Algorithms []string DeviceID string UserAgent string // 验证码token刷新成功回调 RefreshCTokenCk func(token string) } func generateDeviceSign(deviceID, packageName string) string { signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, "1", "appkey") sha1Hash := sha1.New() sha1Hash.Write([]byte(signatureBase)) sha1Result := sha1Hash.Sum(nil) sha1String := hex.EncodeToString(sha1Result) md5Hash := md5.New() md5Hash.Write([]byte(sha1String)) md5Result := md5Hash.Sum(nil) md5String := hex.EncodeToString(md5Result) deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) return deviceSign } func BuildCustomUserAgent(deviceID, clientID, appName, sdkVersion, clientVersion, packageName, userID string) string { deviceSign := generateDeviceSign(deviceID, packageName) var sb strings.Builder sb.WriteString(fmt.Sprintf("ANDROID-%s/%s ", appName, clientVersion)) sb.WriteString("protocolVersion/200 ") sb.WriteString("accesstype/ ") sb.WriteString(fmt.Sprintf("clientid/%s ", clientID)) sb.WriteString(fmt.Sprintf("clientversion/%s ", clientVersion)) sb.WriteString("action_type/ ") sb.WriteString("networktype/WIFI ") sb.WriteString("sessionid/ ") sb.WriteString(fmt.Sprintf("deviceid/%s ", deviceID)) sb.WriteString("providername/NONE ") sb.WriteString(fmt.Sprintf("devicesign/%s ", deviceSign)) sb.WriteString("refresh_token/ ") sb.WriteString(fmt.Sprintf("sdkversion/%s ", sdkVersion)) sb.WriteString(fmt.Sprintf("datetime/%d ", time.Now().UnixMilli())) sb.WriteString(fmt.Sprintf("usrno/%s ", userID)) sb.WriteString(fmt.Sprintf("appname/android-%s ", appName)) sb.WriteString(fmt.Sprintf("session_origin/ ")) sb.WriteString(fmt.Sprintf("grant_type/ ")) sb.WriteString(fmt.Sprintf("appid/ ")) sb.WriteString(fmt.Sprintf("clientip/ ")) sb.WriteString(fmt.Sprintf("devicename/Xiaomi_M2004j7ac ")) sb.WriteString(fmt.Sprintf("osversion/13 ")) sb.WriteString(fmt.Sprintf("platformversion/10 ")) sb.WriteString(fmt.Sprintf("accessmode/ ")) sb.WriteString(fmt.Sprintf("devicemodel/M2004J7AC ")) return sb.String() } func (c *Common) SetDeviceID(deviceID string) { c.DeviceID = deviceID } func (c *Common) SetUserID(userID string) { c.UserID = userID } func (c *Common) SetUserAgent(userAgent string) { c.UserAgent = userAgent } func (c *Common) SetCaptchaToken(captchaToken string) { c.CaptchaToken = captchaToken } func (c *Common) GetCaptchaToken() string { return c.CaptchaToken } func (c *Common) GetUserAgent() string { return c.UserAgent } func (c *Common) GetDeviceID() string { return c.DeviceID } func (c *Common) GetUserID() string { return c.UserID } // RefreshCaptchaTokenAtLogin 刷新验证码token(登录后) func (d *PikPak) RefreshCaptchaTokenAtLogin(action, userID string) error { metas := map[string]string{ "client_version": d.ClientVersion, "package_name": d.PackageName, "user_id": userID, } metas["timestamp"], metas["captcha_sign"] = d.Common.GetCaptchaSign() return d.refreshCaptchaToken(action, metas) } // RefreshCaptchaTokenInLogin 刷新验证码token(登录时) func (d *PikPak) RefreshCaptchaTokenInLogin(action, username string) error { metas := make(map[string]string) if ok, _ := regexp.MatchString(`\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*`, username); ok { metas["email"] = username } else if len(username) >= 11 && len(username) <= 18 { metas["phone_number"] = username } else { metas["username"] = username } return d.refreshCaptchaToken(action, metas) } // GetCaptchaSign 获取验证码签名 func (c *Common) GetCaptchaSign() (timestamp, sign string) { timestamp = fmt.Sprint(time.Now().UnixMilli()) str := fmt.Sprint(c.ClientID, c.ClientVersion, c.PackageName, c.DeviceID, timestamp) for _, algorithm := range c.Algorithms { str = utils.GetMD5EncodeStr(str + algorithm) } sign = "1." + str return } // refreshCaptchaToken 刷新CaptchaToken func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) error { param := CaptchaTokenRequest{ Action: action, CaptchaToken: d.GetCaptchaToken(), ClientID: d.ClientID, DeviceID: d.GetDeviceID(), Meta: metas, RedirectUri: "xlaccsdk01://xbase.cloud/callback?state=harbor", } var e ErrResp var resp CaptchaTokenResponse _, err := d.request("https://user.mypikpak.net/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { req.SetError(&e).SetBody(param).SetQueryParam("client_id", d.ClientID) }, &resp) if err != nil { return err } if e.IsError() { return errors.New(e.Error()) } if resp.Url != "" { return fmt.Errorf(`need verify: Click Here`, resp.Url) } if d.Common.RefreshCTokenCk != nil { d.Common.RefreshCTokenCk(resp.CaptchaToken) } d.Common.SetCaptchaToken(resp.CaptchaToken) return nil } func (d *PikPak) UploadByOSS(ctx context.Context, params *S3Params, s model.FileStreamer, up driver.UpdateProgress) error { ossClient, err := netutil.NewOSSClient(params.Endpoint, params.AccessKeyID, params.AccessKeySecret) if err != nil { return err } bucket, err := ossClient.Bucket(params.Bucket) if err != nil { return err } err = bucket.PutObject(params.Key, driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, }), OssOption(params)...) if err != nil { return err } return nil } func (d *PikPak) UploadByMultipart(ctx context.Context, params *S3Params, fileSize int64, s model.FileStreamer, up driver.UpdateProgress) error { tmpF, err := s.CacheFullAndWriter(&up, nil) if err != nil { return err } var ( chunks []oss.FileChunk parts []oss.UploadPart imur oss.InitiateMultipartUploadResult ossClient *oss.Client bucket *oss.Bucket ) if ossClient, err = netutil.NewOSSClient(params.Endpoint, params.AccessKeyID, params.AccessKeySecret); err != nil { return err } if bucket, err = ossClient.Bucket(params.Bucket); err != nil { return err } ticker := time.NewTicker(time.Hour * 12) defer ticker.Stop() // 设置超时 timeout := time.NewTimer(time.Hour * 24) if chunks, err = SplitFile(fileSize); err != nil { return err } if imur, err = bucket.InitiateMultipartUpload(params.Key, oss.SetHeader(OssSecurityTokenHeaderName, params.SecurityToken), oss.UserAgentHeader(OSSUserAgent), ); err != nil { return err } wg := sync.WaitGroup{} wg.Add(len(chunks)) chunksCh := make(chan oss.FileChunk) errCh := make(chan error) UploadedPartsCh := make(chan oss.UploadPart) quit := make(chan struct{}) // producer go chunksProducer(chunksCh, chunks) go func() { wg.Wait() quit <- struct{}{} }() completedNum := atomic.Int32{} // consumers for i := 0; i < ThreadsNum; i++ { go func(threadId int) { defer func() { if r := recover(); r != nil { errCh <- fmt.Errorf("recovered in %v", r) } }() for chunk := range chunksCh { var part oss.UploadPart // 出现错误就继续尝试，共尝试3次 for retry := 0; retry < 3; retry++ { select { case <-ctx.Done(): break case <-ticker.C: errCh <- errors.Wrap(err, "ossToken 过期") default: } buf := make([]byte, chunk.Size) if _, err = tmpF.ReadAt(buf, chunk.Offset); err != nil && !errors.Is(err, io.EOF) { continue } b := driver.NewLimitedUploadStream(ctx, bytes.NewReader(buf)) if part, err = bucket.UploadPart(imur, b, chunk.Size, chunk.Number, OssOption(params)...); err == nil { break } } if err != nil { errCh <- errors.Wrap(err, fmt.Sprintf("上传 %s 的第%d个分片时出现错误：%v", s.GetName(), chunk.Number, err)) } else { num := completedNum.Add(1) up(float64(num) * 100.0 / float64(len(chunks))) } UploadedPartsCh <- part } }(i) } go func() { for part := range UploadedPartsCh { parts = append(parts, part) wg.Done() } }() LOOP: for { select { case <-ticker.C: // ossToken 过期 return err case <-quit: break LOOP case <-errCh: return err case <-timeout.C: return fmt.Errorf("time out") } } // EOF错误是xml的Unmarshal导致的，响应其实是json格式，所以实际上上传是成功的 if _, err = bucket.CompleteMultipartUpload(imur, parts, OssOption(params)...); err != nil && !errors.Is(err, io.EOF) { // 当文件名含有 &< 这两个字符之一时响应的xml解析会出现错误，实际上上传是成功的 if filename := filepath.Base(s.GetName()); !strings.ContainsAny(filename, "&<") { return err } } return nil } func chunksProducer(ch chan oss.FileChunk, chunks []oss.FileChunk) { for _, chunk := range chunks { ch <- chunk } } func SplitFile(fileSize int64) (chunks []oss.FileChunk, err error) { for i := int64(1); i < 10; i++ { if fileSize < i*utils.GB { // 文件大小小于iGB时分为i*100片 if chunks, err = SplitFileByPartNum(fileSize, int(i*100)); err != nil { return } break } } if fileSize > 9*utils.GB { // 文件大小大于9GB时分为1000片 if chunks, err = SplitFileByPartNum(fileSize, 1000); err != nil { return } } // 单个分片大小不能小于1MB if chunks[0].Size < 1*utils.MB { if chunks, err = SplitFileByPartSize(fileSize, 1*utils.MB); err != nil { return } } return } // SplitFileByPartNum splits big file into parts by the num of parts. // Split the file with specified parts count, returns the split result when error is nil. func SplitFileByPartNum(fileSize int64, chunkNum int) ([]oss.FileChunk, error) { if chunkNum <= 0 || chunkNum > 10000 { return nil, errors.New("chunkNum invalid") } if int64(chunkNum) > fileSize { return nil, errors.New("oss: chunkNum invalid") } var chunks []oss.FileChunk chunk := oss.FileChunk{} chunkN := (int64)(chunkNum) for i := int64(0); i < chunkN; i++ { chunk.Number = int(i + 1) chunk.Offset = i * (fileSize / chunkN) if i == chunkN-1 { chunk.Size = fileSize/chunkN + fileSize%chunkN } else { chunk.Size = fileSize / chunkN } chunks = append(chunks, chunk) } return chunks, nil } // SplitFileByPartSize splits big file into parts by the size of parts. // Splits the file by the part size. Returns the FileChunk when error is nil. func SplitFileByPartSize(fileSize int64, chunkSize int64) ([]oss.FileChunk, error) { if chunkSize <= 0 { return nil, errors.New("chunkSize invalid") } chunkN := fileSize / chunkSize if chunkN >= 10000 { return nil, errors.New("Too many parts, please increase part size") } var chunks []oss.FileChunk chunk := oss.FileChunk{} for i := int64(0); i < chunkN; i++ { chunk.Number = int(i + 1) chunk.Offset = i * chunkSize chunk.Size = chunkSize chunks = append(chunks, chunk) } if fileSize%chunkSize > 0 { chunk.Number = len(chunks) + 1 chunk.Offset = int64(len(chunks)) * chunkSize chunk.Size = fileSize % chunkSize chunks = append(chunks, chunk) } return chunks, nil } // OssOption get options func OssOption(params *S3Params) []oss.Option { options := []oss.Option{ oss.SetHeader(OssSecurityTokenHeaderName, params.SecurityToken), oss.UserAgentHeader(OSSUserAgent), } return options } ================================================ FILE: drivers/pikpak_share/driver.go ================================================ package pikpak_share import ( "context" "net/http" "time" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) type PikPakShare struct { model.Storage Addition *Common PassCodeToken string } func (d *PikPakShare) Config() driver.Config { return config } func (d *PikPakShare) GetAddition() driver.Additional { return &d.Addition } func (d *PikPakShare) Init(ctx context.Context) error { if d.Common == nil { d.Common = &Common{ DeviceID: utils.GetMD5EncodeStr(d.Addition.ShareId + d.Addition.SharePwd + time.Now().String()), UserAgent: "", RefreshCTokenCk: func(token string) { d.Common.CaptchaToken = token op.MustSaveDriverStorage(d) }, } } if d.Addition.DeviceID != "" { d.SetDeviceID(d.Addition.DeviceID) } else { d.Addition.DeviceID = d.Common.DeviceID op.MustSaveDriverStorage(d) } if d.Platform == "android" { d.ClientID = AndroidClientID d.ClientSecret = AndroidClientSecret d.ClientVersion = AndroidClientVersion d.PackageName = AndroidPackageName d.Algorithms = AndroidAlgorithms d.UserAgent = BuildCustomUserAgent(d.GetDeviceID(), AndroidClientID, AndroidPackageName, AndroidSdkVersion, AndroidClientVersion, AndroidPackageName, "") } else if d.Platform == "web" { d.ClientID = WebClientID d.ClientSecret = WebClientSecret d.ClientVersion = WebClientVersion d.PackageName = WebPackageName d.Algorithms = WebAlgorithms d.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36" } else if d.Platform == "pc" { d.ClientID = PCClientID d.ClientSecret = PCClientSecret d.ClientVersion = PCClientVersion d.PackageName = PCPackageName d.Algorithms = PCAlgorithms d.UserAgent = "MainWindow Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) PikPak/2.6.11.4955 Chrome/100.0.4896.160 Electron/18.3.15 Safari/537.36" } // 获取CaptchaToken err := d.RefreshCaptchaToken(GetAction(http.MethodGet, "https://api-drive.mypikpak.net/drive/v1/share:batch_file_info"), "") if err != nil { return err } if d.SharePwd != "" { return d.getSharePassToken() } return nil } func (d *PikPakShare) Drop(ctx context.Context) error { return nil } func (d *PikPakShare) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(dir.GetID()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return fileToObj(src), nil }) } func (d *PikPakShare) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp ShareResp query := map[string]string{ "share_id": d.ShareId, "file_id": file.GetID(), "pass_code_token": d.PassCodeToken, } _, err := d.request("https://api-drive.mypikpak.net/drive/v1/share/file_info", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return nil, err } downloadUrl := resp.FileInfo.WebContentLink if downloadUrl == "" && len(resp.FileInfo.Medias) > 0 { // 使用转码后的链接 if d.Addition.UseTransCodingAddress && len(resp.FileInfo.Medias) > 1 { downloadUrl = resp.FileInfo.Medias[1].Link.Url } else { downloadUrl = resp.FileInfo.Medias[0].Link.Url } } return &model.Link{ URL: downloadUrl, }, nil } var _ driver.Driver = (*PikPakShare)(nil) ================================================ FILE: drivers/pikpak_share/meta.go ================================================ package pikpak_share import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootID ShareId string `json:"share_id" required:"true"` SharePwd string `json:"share_pwd"` Platform string `json:"platform" default:"web" required:"true" type:"select" options:"android,web,pc"` DeviceID string `json:"device_id" required:"false" default:""` UseTransCodingAddress bool `json:"use_transcoding_address" required:"true" default:"false"` } var config = driver.Config{ Name: "PikPakShare", LocalSort: true, NoUpload: true, } func init() { op.RegisterDriver(func() driver.Driver { return &PikPakShare{} }) } ================================================ FILE: drivers/pikpak_share/types.go ================================================ package pikpak_share import ( "fmt" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type ShareResp struct { ShareStatus string `json:"share_status"` ShareStatusText string `json:"share_status_text"` FileInfo File `json:"file_info"` Files []File `json:"files"` NextPageToken string `json:"next_page_token"` PassCodeToken string `json:"pass_code_token"` } type File struct { Id string `json:"id"` ShareId string `json:"share_id"` Kind string `json:"kind"` Name string `json:"name"` ModifiedTime time.Time `json:"modified_time"` Size string `json:"size"` ThumbnailLink string `json:"thumbnail_link"` WebContentLink string `json:"web_content_link"` Medias []Media `json:"medias"` } func fileToObj(f File) *model.ObjThumb { size, _ := strconv.ParseInt(f.Size, 10, 64) return &model.ObjThumb{ Object: model.Object{ ID: f.Id, Name: f.Name, Size: size, Modified: f.ModifiedTime, IsFolder: f.Kind == "drive#folder", }, Thumbnail: model.Thumbnail{ Thumbnail: f.ThumbnailLink, }, } } type Media struct { MediaId string `json:"media_id"` MediaName string `json:"media_name"` Video struct { Height int `json:"height"` Width int `json:"width"` Duration int `json:"duration"` BitRate int `json:"bit_rate"` FrameRate int `json:"frame_rate"` VideoCodec string `json:"video_codec"` AudioCodec string `json:"audio_codec"` VideoType string `json:"video_type"` } `json:"video"` Link struct { Url string `json:"url"` Token string `json:"token"` Expire time.Time `json:"expire"` } `json:"link"` NeedMoreQuota bool `json:"need_more_quota"` VipTypes []interface{} `json:"vip_types"` RedirectLink string `json:"redirect_link"` IconLink string `json:"icon_link"` IsDefault bool `json:"is_default"` Priority int `json:"priority"` IsOrigin bool `json:"is_origin"` ResolutionName string `json:"resolution_name"` IsVisible bool `json:"is_visible"` Category string `json:"category"` } type CaptchaTokenRequest struct { Action string `json:"action"` CaptchaToken string `json:"captcha_token"` ClientID string `json:"client_id"` DeviceID string `json:"device_id"` Meta map[string]string `json:"meta"` RedirectUri string `json:"redirect_uri"` } type CaptchaTokenResponse struct { CaptchaToken string `json:"captcha_token"` ExpiresIn int64 `json:"expires_in"` Url string `json:"url"` } type ErrResp struct { ErrorCode int64 `json:"error_code"` ErrorMsg string `json:"error"` ErrorDescription string `json:"error_description"` } func (e *ErrResp) IsError() bool { return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" } func (e *ErrResp) Error() string { return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ErrorDescription: %s ", e.ErrorCode, e.ErrorMsg, e.ErrorDescription) } ================================================ FILE: drivers/pikpak_share/util.go ================================================ package pikpak_share import ( "crypto/md5" "crypto/sha1" "encoding/hex" "errors" "fmt" "net/http" "regexp" "strings" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/go-resty/resty/v2" ) var AndroidAlgorithms = []string{ "SOP04dGzk0TNO7t7t9ekDbAmx+eq0OI1ovEx", "nVBjhYiND4hZ2NCGyV5beamIr7k6ifAsAbl", "Ddjpt5B/Cit6EDq2a6cXgxY9lkEIOw4yC1GDF28KrA", "VVCogcmSNIVvgV6U+AochorydiSymi68YVNGiz", "u5ujk5sM62gpJOsB/1Gu/zsfgfZO", "dXYIiBOAHZgzSruaQ2Nhrqc2im", "z5jUTBSIpBN9g4qSJGlidNAutX6", "KJE2oveZ34du/g1tiimm", } var WebAlgorithms = []string{ "C9qPpZLN8ucRTaTiUMWYS9cQvWOE", "+r6CQVxjzJV6LCV", "F", "pFJRC", "9WXYIDGrwTCz2OiVlgZa90qpECPD6olt", "/750aCr4lm/Sly/c", "RB+DT/gZCrbV", "", "CyLsf7hdkIRxRm215hl", "7xHvLi2tOYP0Y92b", "ZGTXXxu8E/MIWaEDB+Sm/", "1UI3", "E7fP5Pfijd+7K+t6Tg/NhuLq0eEUVChpJSkrKxpO", "ihtqpG6FMt65+Xk+tWUH2", "NhXXU9rg4XXdzo7u5o", } var PCAlgorithms = []string{ "KHBJ07an7ROXDoK7Db", "G6n399rSWkl7WcQmw5rpQInurc1DkLmLJqE", "JZD1A3M4x+jBFN62hkr7VDhkkZxb9g3rWqRZqFAAb", "fQnw/AmSlbbI91Ik15gpddGgyU7U", "/Dv9JdPYSj3sHiWjouR95NTQff", "yGx2zuTjbWENZqecNI+edrQgqmZKP", "ljrbSzdHLwbqcRn", "lSHAsqCkGDGxQqqwrVu", "TsWXI81fD1", "vk7hBjawK/rOSrSWajtbMk95nfgf3", } const ( AndroidClientID = "YNxT9w7GMdWvEOKa" AndroidClientSecret = "dbw2OtmVEeuUvIptb1Coyg" AndroidClientVersion = "1.53.2" AndroidPackageName = "com.pikcloud.pikpak" AndroidSdkVersion = "2.0.6.206003" WebClientID = "YUMx5nI8ZU8Ap8pm" WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" WebClientVersion = "2.0.0" WebPackageName = "mypikpak.com" WebSdkVersion = "8.0.3" PCClientID = "YvtoWO6GNHiuCl7x" PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" PCClientVersion = "undefined" // 2.6.11.4955 PCPackageName = "mypikpak.com" PCSdkVersion = "8.0.3" ) func (d *PikPakShare) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeaders(map[string]string{ "User-Agent": d.GetUserAgent(), "X-Client-ID": d.GetClientID(), "X-Device-ID": d.GetDeviceID(), "X-Captcha-Token": d.GetCaptchaToken(), }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e ErrResp req.SetError(&e) res, err := req.Execute(method, url) if err != nil { return nil, err } switch e.ErrorCode { case 0: return res.Body(), nil case 9: // 验证码token过期 if err = d.RefreshCaptchaToken(GetAction(method, url), ""); err != nil { return nil, err } return d.request(url, method, callback, resp) case 10: // 操作频繁 return nil, errors.New(e.ErrorDescription) default: return nil, errors.New(e.Error()) } } func (d *PikPakShare) getSharePassToken() error { query := map[string]string{ "share_id": d.ShareId, "pass_code": d.SharePwd, "thumbnail_size": "SIZE_LARGE", "limit": "100", } var resp ShareResp _, err := d.request("https://api-drive.mypikpak.net/drive/v1/share", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return err } d.PassCodeToken = resp.PassCodeToken return nil } func (d *PikPakShare) getFiles(id string) ([]File, error) { res := make([]File, 0) pageToken := "first" for pageToken != "" { if pageToken == "first" { pageToken = "" } query := map[string]string{ "parent_id": id, "share_id": d.ShareId, "thumbnail_size": "SIZE_LARGE", "with_audit": "true", "limit": "100", "filters": `{"phase":{"eq":"PHASE_TYPE_COMPLETE"},"trashed":{"eq":false}}`, "page_token": pageToken, "pass_code_token": d.PassCodeToken, } var resp ShareResp _, err := d.request("https://api-drive.mypikpak.net/drive/v1/share/detail", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return nil, err } if resp.ShareStatus != "OK" { if resp.ShareStatus == "PASS_CODE_EMPTY" || resp.ShareStatus == "PASS_CODE_ERROR" { err = d.getSharePassToken() if err != nil { return nil, err } return d.getFiles(id) } return nil, errors.New(resp.ShareStatusText) } pageToken = resp.NextPageToken res = append(res, resp.Files...) } return res, nil } func GetAction(method string, url string) string { urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] return method + ":" + urlpath } type Common struct { client *resty.Client CaptchaToken string // 必要值,签名相关 ClientID string ClientSecret string ClientVersion string PackageName string Algorithms []string DeviceID string UserAgent string // 验证码token刷新成功回调 RefreshCTokenCk func(token string) } func (c *Common) SetUserAgent(userAgent string) { c.UserAgent = userAgent } func (c *Common) SetCaptchaToken(captchaToken string) { c.CaptchaToken = captchaToken } func (c *Common) SetDeviceID(deviceID string) { c.DeviceID = deviceID } func (c *Common) GetCaptchaToken() string { return c.CaptchaToken } func (c *Common) GetClientID() string { return c.ClientID } func (c *Common) GetUserAgent() string { return c.UserAgent } func (c *Common) GetDeviceID() string { return c.DeviceID } func generateDeviceSign(deviceID, packageName string) string { signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, "1", "appkey") sha1Hash := sha1.New() sha1Hash.Write([]byte(signatureBase)) sha1Result := sha1Hash.Sum(nil) sha1String := hex.EncodeToString(sha1Result) md5Hash := md5.New() md5Hash.Write([]byte(sha1String)) md5Result := md5Hash.Sum(nil) md5String := hex.EncodeToString(md5Result) deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) return deviceSign } func BuildCustomUserAgent(deviceID, clientID, appName, sdkVersion, clientVersion, packageName, userID string) string { deviceSign := generateDeviceSign(deviceID, packageName) var sb strings.Builder sb.WriteString(fmt.Sprintf("ANDROID-%s/%s ", appName, clientVersion)) sb.WriteString("protocolVersion/200 ") sb.WriteString("accesstype/ ") sb.WriteString(fmt.Sprintf("clientid/%s ", clientID)) sb.WriteString(fmt.Sprintf("clientversion/%s ", clientVersion)) sb.WriteString("action_type/ ") sb.WriteString("networktype/WIFI ") sb.WriteString("sessionid/ ") sb.WriteString(fmt.Sprintf("deviceid/%s ", deviceID)) sb.WriteString("providername/NONE ") sb.WriteString(fmt.Sprintf("devicesign/%s ", deviceSign)) sb.WriteString("refresh_token/ ") sb.WriteString(fmt.Sprintf("sdkversion/%s ", sdkVersion)) sb.WriteString(fmt.Sprintf("datetime/%d ", time.Now().UnixMilli())) sb.WriteString(fmt.Sprintf("usrno/%s ", userID)) sb.WriteString(fmt.Sprintf("appname/android-%s ", appName)) sb.WriteString(fmt.Sprintf("session_origin/ ")) sb.WriteString(fmt.Sprintf("grant_type/ ")) sb.WriteString(fmt.Sprintf("appid/ ")) sb.WriteString(fmt.Sprintf("clientip/ ")) sb.WriteString(fmt.Sprintf("devicename/Xiaomi_M2004j7ac ")) sb.WriteString(fmt.Sprintf("osversion/13 ")) sb.WriteString(fmt.Sprintf("platformversion/10 ")) sb.WriteString(fmt.Sprintf("accessmode/ ")) sb.WriteString(fmt.Sprintf("devicemodel/M2004J7AC ")) return sb.String() } // RefreshCaptchaToken 刷新验证码token func (d *PikPakShare) RefreshCaptchaToken(action, userID string) error { metas := map[string]string{ "client_version": d.ClientVersion, "package_name": d.PackageName, "user_id": userID, } metas["timestamp"], metas["captcha_sign"] = d.Common.GetCaptchaSign() return d.refreshCaptchaToken(action, metas) } // GetCaptchaSign 获取验证码签名 func (c *Common) GetCaptchaSign() (timestamp, sign string) { timestamp = fmt.Sprint(time.Now().UnixMilli()) str := fmt.Sprint(c.ClientID, c.ClientVersion, c.PackageName, c.DeviceID, timestamp) for _, algorithm := range c.Algorithms { str = utils.GetMD5EncodeStr(str + algorithm) } sign = "1." + str return } // refreshCaptchaToken 刷新CaptchaToken func (d *PikPakShare) refreshCaptchaToken(action string, metas map[string]string) error { param := CaptchaTokenRequest{ Action: action, CaptchaToken: d.GetCaptchaToken(), ClientID: d.ClientID, DeviceID: d.GetDeviceID(), Meta: metas, } var e ErrResp var resp CaptchaTokenResponse _, err := d.request("https://user.mypikpak.net/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { req.SetError(&e).SetBody(param) }, &resp) if err != nil { return err } if e.IsError() { return errors.New(e.Error()) } //if resp.Url != "" { // return fmt.Errorf(`need verify: Click Here`, resp.Url) //} if d.Common.RefreshCTokenCk != nil { d.Common.RefreshCTokenCk(resp.CaptchaToken) } d.Common.SetCaptchaToken(resp.CaptchaToken) return nil } ================================================ FILE: drivers/proton_drive/driver.go ================================================ package protondrive /* Package protondrive Author: Da3zKi7 Date: 2025-09-18 Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge The power of open-source, the force of teamwork and the magic of reverse engineering! D@' 3z K!7 - The King Of Cracking Да здравствует Родина)) */ import ( "context" "fmt" "io" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/ProtonMail/gopenpgp/v2/crypto" proton_api_bridge "github.com/henrybear327/Proton-API-Bridge" "github.com/henrybear327/Proton-API-Bridge/common" "github.com/henrybear327/go-proton-api" ) type ProtonDrive struct { model.Storage Addition protonDrive *proton_api_bridge.ProtonDrive apiBase string appVersion string protonJson string userAgent string sdkVersion string webDriveAV string c *proton.Client // userKR *crypto.KeyRing addrKRs map[string]*crypto.KeyRing addrData map[string]proton.Address MainShare *proton.Share DefaultAddrKR *crypto.KeyRing MainShareKR *crypto.KeyRing } func (d *ProtonDrive) Config() driver.Config { return config } func (d *ProtonDrive) GetAddition() driver.Additional { return &d.Addition } func (d *ProtonDrive) Init(ctx context.Context) (err error) { defer func() { if r := recover(); err == nil && r != nil { err = fmt.Errorf("ProtonDrive initialization panic: %v", r) } }() if d.Email == "" { return fmt.Errorf("email is required") } if d.Password == "" { return fmt.Errorf("password is required") } config := &common.Config{ AppVersion: d.appVersion, UserAgent: d.userAgent, FirstLoginCredential: &common.FirstLoginCredentialData{ Username: d.Email, Password: d.Password, TwoFA: d.TwoFACode, }, EnableCaching: true, ConcurrentBlockUploadCount: setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers), //ConcurrentFileCryptoCount: 2, UseReusableLogin: d.UseReusableLogin && d.ReusableCredential != (common.ReusableCredentialData{}), ReplaceExistingDraft: true, ReusableCredential: &d.ReusableCredential, } protonDrive, _, err := proton_api_bridge.NewProtonDrive( ctx, config, d.authHandler, func() {}, ) if err != nil && config.UseReusableLogin { config.UseReusableLogin = false protonDrive, _, err = proton_api_bridge.NewProtonDrive(ctx, config, d.authHandler, func() {}, ) if err == nil { op.MustSaveDriverStorage(d) } } if err != nil { return fmt.Errorf("failed to initialize ProtonDrive: %w", err) } if err := d.initClient(ctx); err != nil { return err } d.protonDrive = protonDrive d.MainShare = protonDrive.MainShare if d.RootFolderID == "root" || d.RootFolderID == "" { d.RootFolderID = protonDrive.RootLink.LinkID } d.MainShareKR = protonDrive.MainShareKR d.DefaultAddrKR = protonDrive.DefaultAddrKR return nil } func (d *ProtonDrive) Drop(ctx context.Context) error { return nil } func (d *ProtonDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { entries, err := d.protonDrive.ListDirectory(ctx, dir.GetID()) if err != nil { return nil, fmt.Errorf("failed to list directory: %w", err) } objects := make([]model.Obj, 0, len(entries)) for _, entry := range entries { obj := &model.Object{ ID: entry.Link.LinkID, Name: entry.Name, Size: entry.Link.Size, Modified: time.Unix(entry.Link.ModifyTime, 0), IsFolder: entry.IsFolder, } objects = append(objects, obj) } return objects, nil } func (d *ProtonDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { link, err := d.getLink(ctx, file.GetID()) if err != nil { return nil, fmt.Errorf("failed get file link: %+v", err) } fileSystemAttrs, err := d.protonDrive.GetActiveRevisionAttrs(ctx, link) if err != nil { return nil, fmt.Errorf("failed get file revision: %+v", err) } // 解密后的文件大小 size := fileSystemAttrs.Size rangeReaderFunc := func(rangeCtx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { length := httpRange.Length if length < 0 || httpRange.Start+length > size { length = size - httpRange.Start } reader, _, _, err := d.protonDrive.DownloadFile(rangeCtx, link, httpRange.Start) if err != nil { return nil, fmt.Errorf("failed start download: %+v", err) } return utils.ReadCloser{ Reader: io.LimitReader(reader, length), Closer: reader, }, nil } expiration := time.Minute return &model.Link{ RangeReader: stream.RateLimitRangeReaderFunc(rangeReaderFunc), ContentLength: size, Expiration: &expiration, }, nil } func (d *ProtonDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { id, err := d.protonDrive.CreateNewFolderByID(ctx, parentDir.GetID(), dirName) if err != nil { return nil, fmt.Errorf("failed to create directory: %w", err) } newDir := &model.Object{ ID: id, Name: dirName, IsFolder: true, Modified: time.Now(), } return newDir, nil } func (d *ProtonDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return d.DirectMove(ctx, srcObj, dstDir) } func (d *ProtonDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { if d.protonDrive == nil { return nil, fmt.Errorf("protonDrive bridge is nil") } return d.DirectRename(ctx, srcObj, newName) } func (d *ProtonDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if srcObj.IsDir() { return nil, fmt.Errorf("directory copy not supported") } srcLink, err := d.getLink(ctx, srcObj.GetID()) if err != nil { return nil, err } reader, linkSize, fileSystemAttrs, err := d.protonDrive.DownloadFile(ctx, srcLink, 0) if err != nil { return nil, fmt.Errorf("failed to download source file: %w", err) } defer reader.Close() actualSize := linkSize if fileSystemAttrs != nil && fileSystemAttrs.Size > 0 { actualSize = fileSystemAttrs.Size } file := &stream.FileStream{ Ctx: ctx, Obj: &model.Object{ Name: srcObj.GetName(), // Use the accurate and real size Size: actualSize, Modified: srcObj.ModTime(), }, Reader: reader, } defer file.Close() return d.Put(ctx, dstDir, file, func(percentage float64) {}) } func (d *ProtonDrive) Remove(ctx context.Context, obj model.Obj) error { if obj.IsDir() { return d.protonDrive.MoveFolderToTrashByID(ctx, obj.GetID(), false) } else { return d.protonDrive.MoveFileToTrashByID(ctx, obj.GetID()) } } func (d *ProtonDrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { return d.uploadFile(ctx, dstDir.GetID(), file, up) } func (d *ProtonDrive) GetDetails(ctx context.Context) (*model.StorageDetails, error) { about, err := d.protonDrive.About(ctx) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: about.MaxSpace, UsedSpace: about.UsedSpace, }, }, nil } var _ driver.Driver = (*ProtonDrive)(nil) ================================================ FILE: drivers/proton_drive/meta.go ================================================ package protondrive /* Package protondrive Author: Da3zKi7 Date: 2025-09-18 Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge The power of open-source, the force of teamwork and the magic of reverse engineering! D@' 3z K!7 - The King Of Cracking Да здравствует Родина)) */ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/henrybear327/Proton-API-Bridge/common" ) type Addition struct { driver.RootID Email string `json:"email" required:"true" type:"string"` Password string `json:"password" required:"true" type:"string"` TwoFACode string `json:"two_fa_code" type:"string"` ChunkSize int64 `json:"chunk_size" type:"number" default:"100"` UseReusableLogin bool `json:"use_reusable_login" type:"bool" default:"true" help:"Use reusable login credentials instead of username/password"` ReusableCredential common.ReusableCredentialData } var config = driver.Config{ Name: "ProtonDrive", LocalSort: true, OnlyProxy: true, DefaultRoot: "root", NoLinkURL: true, } func init() { op.RegisterDriver(func() driver.Driver { return &ProtonDrive{ Addition: Addition{ UseReusableLogin: true, }, apiBase: "https://drive.proton.me/api", appVersion: "windows-drive@1.11.3+rclone+proton", protonJson: "application/vnd.protonmail.v1+json", sdkVersion: "js@0.3.0", userAgent: "ProtonDrive/v1.70.0 (Windows NT 10.0.22000; Win64; x64)", webDriveAV: "web-drive@5.2.0+0f69f7a8", } }) } ================================================ FILE: drivers/proton_drive/types.go ================================================ package protondrive /* Package protondrive Author: Da3zKi7 Date: 2025-09-18 Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge The power of open-source, the force of teamwork and the magic of reverse engineering! D@' 3z K!7 - The King Of Cracking Да здравствует Родина)) */ type MoveRequest struct { ParentLinkID string `json:"ParentLinkID"` NodePassphrase string `json:"NodePassphrase"` NodePassphraseSignature *string `json:"NodePassphraseSignature"` Name string `json:"Name"` NameSignatureEmail string `json:"NameSignatureEmail"` Hash string `json:"Hash"` OriginalHash string `json:"OriginalHash"` ContentHash *string `json:"ContentHash"` // Maybe null } type RenameRequest struct { Name string `json:"Name"` // PGP encrypted name NameSignatureEmail string `json:"NameSignatureEmail"` // User's signature email Hash string `json:"Hash"` // New name hash OriginalHash string `json:"OriginalHash"` // Current name hash } type RenameResponse struct { Code int `json:"Code"` } ================================================ FILE: drivers/proton_drive/util.go ================================================ package protondrive /* Package protondrive Author: Da3zKi7 Date: 2025-09-18 Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge The power of open-source, the force of teamwork and the magic of reverse engineering! D@' 3z K!7 - The King Of Cracking Да здравствует Родина)) */ import ( "bufio" "bytes" "context" "encoding/base64" "encoding/json" "errors" "fmt" "io" "net/http" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/ProtonMail/gopenpgp/v2/crypto" "github.com/henrybear327/go-proton-api" ) func (d *ProtonDrive) uploadFile(ctx context.Context, parentLinkID string, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { _, err := d.getLink(ctx, parentLinkID) if err != nil { return nil, fmt.Errorf("failed to get parent link: %w", err) } var reader io.Reader // Use buffered reader with larger buffer for better performance var bufferSize int // File > 100MB (default) if file.GetSize() > d.ChunkSize*1024*1024 { // 256KB for large files bufferSize = 256 * 1024 // File > 10MB } else if file.GetSize() > 10*1024*1024 { // 128KB for medium files bufferSize = 128 * 1024 } else { // 64KB for small files bufferSize = 64 * 1024 } // reader = bufio.NewReader(file) reader = bufio.NewReaderSize(file, bufferSize) reader = &driver.ReaderUpdatingProgress{ Reader: &stream.SimpleReaderWithSize{ Reader: reader, Size: file.GetSize(), }, UpdateProgress: up, } reader = driver.NewLimitedUploadStream(ctx, reader) id, _, err := d.protonDrive.UploadFileByReader(ctx, parentLinkID, file.GetName(), file.ModTime(), reader, 0) if err != nil { return nil, fmt.Errorf("failed to upload file: %w", err) } return &model.Object{ ID: id, Name: file.GetName(), Size: file.GetSize(), Modified: file.ModTime(), IsFolder: false, }, nil } func (d *ProtonDrive) encryptFileName(ctx context.Context, name string, parentLinkID string) (string, error) { parentLink, err := d.getLink(ctx, parentLinkID) if err != nil { return "", fmt.Errorf("failed to get parent link: %w", err) } // Get parent node keyring parentNodeKR, err := d.getLinkKR(ctx, parentLink) if err != nil { return "", fmt.Errorf("failed to get parent keyring: %w", err) } // Temporary file (request) tempReq := proton.CreateFileReq{ SignatureAddress: d.MainShare.Creator, } // Encrypt the filename err = tempReq.SetName(name, d.DefaultAddrKR, parentNodeKR) if err != nil { return "", fmt.Errorf("failed to encrypt filename: %w", err) } return tempReq.Name, nil } func (d *ProtonDrive) generateFileNameHash(ctx context.Context, name string, parentLinkID string) (string, error) { parentLink, err := d.getLink(ctx, parentLinkID) if err != nil { return "", fmt.Errorf("failed to get parent link: %w", err) } // Get parent node keyring parentNodeKR, err := d.getLinkKR(ctx, parentLink) if err != nil { return "", fmt.Errorf("failed to get parent keyring: %w", err) } signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR) if err != nil { return "", fmt.Errorf("failed to get signature verification keyring: %w", err) } parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR) if err != nil { return "", fmt.Errorf("failed to get parent hash key: %w", err) } nameHash, err := proton.GetNameHash(name, parentHashKey) if err != nil { return "", fmt.Errorf("failed to generate name hash: %w", err) } return nameHash, nil } func (d *ProtonDrive) getOriginalNameHash(link *proton.Link) (string, error) { if link == nil { return "", fmt.Errorf("link cannot be nil") } if link.Hash == "" { return "", fmt.Errorf("link hash is empty") } return link.Hash, nil } func (d *ProtonDrive) getLink(ctx context.Context, linkID string) (*proton.Link, error) { if linkID == "" { return nil, fmt.Errorf("linkID cannot be empty") } link, err := d.c.GetLink(ctx, d.MainShare.ShareID, linkID) if err != nil { return nil, err } return &link, nil } func (d *ProtonDrive) getLinkKR(ctx context.Context, link *proton.Link) (*crypto.KeyRing, error) { if link == nil { return nil, fmt.Errorf("link cannot be nil") } // Root Link or Root Dir if link.ParentLinkID == "" { signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail}) if err != nil { return nil, err } return link.GetKeyRing(d.MainShareKR, signatureVerificationKR) } // Get parent keyring recursively parentLink, err := d.getLink(ctx, link.ParentLinkID) if err != nil { return nil, err } parentNodeKR, err := d.getLinkKR(ctx, parentLink) if err != nil { return nil, err } signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail}) if err != nil { return nil, err } return link.GetKeyRing(parentNodeKR, signatureVerificationKR) } var ( ErrKeyPassOrSaltedKeyPassMustBeNotNil = errors.New("either keyPass or saltedKeyPass must be not nil") ErrFailedToUnlockUserKeys = errors.New("failed to unlock user keys") ) func getAccountKRs(ctx context.Context, c *proton.Client, keyPass, saltedKeyPass []byte) (*crypto.KeyRing, map[string]*crypto.KeyRing, map[string]proton.Address, []byte, error) { user, err := c.GetUser(ctx) if err != nil { return nil, nil, nil, nil, err } // fmt.Printf("user %#v", user) addrsArr, err := c.GetAddresses(ctx) if err != nil { return nil, nil, nil, nil, err } // fmt.Printf("addr %#v", addr) if saltedKeyPass == nil { if keyPass == nil { return nil, nil, nil, nil, ErrKeyPassOrSaltedKeyPassMustBeNotNil } // Due to limitations, salts are stored using cacheCredentialToFile salts, err := c.GetSalts(ctx) if err != nil { return nil, nil, nil, nil, err } // fmt.Printf("salts %#v", salts) saltedKeyPass, err = salts.SaltForKey(keyPass, user.Keys.Primary().ID) if err != nil { return nil, nil, nil, nil, err } // fmt.Printf("saltedKeyPass ok") } userKR, addrKRs, err := proton.Unlock(user, addrsArr, saltedKeyPass, nil) if err != nil { return nil, nil, nil, nil, err } else if userKR.CountDecryptionEntities() == 0 { return nil, nil, nil, nil, ErrFailedToUnlockUserKeys } addrs := make(map[string]proton.Address) for _, addr := range addrsArr { addrs[addr.Email] = addr } return userKR, addrKRs, addrs, saltedKeyPass, nil } func (d *ProtonDrive) getSignatureVerificationKeyring(emailAddresses []string, verificationAddrKRs ...*crypto.KeyRing) (*crypto.KeyRing, error) { ret, err := crypto.NewKeyRing(nil) if err != nil { return nil, err } for _, emailAddress := range emailAddresses { if addr, ok := d.addrData[emailAddress]; ok { if addrKR, exists := d.addrKRs[addr.ID]; exists { err = d.addKeysFromKR(ret, addrKR) if err != nil { return nil, err } } } } for _, kr := range verificationAddrKRs { err = d.addKeysFromKR(ret, kr) if err != nil { return nil, err } } if ret.CountEntities() == 0 { return nil, fmt.Errorf("no keyring for signature verification") } return ret, nil } func (d *ProtonDrive) addKeysFromKR(kr *crypto.KeyRing, newKRs ...*crypto.KeyRing) error { for i := range newKRs { for _, key := range newKRs[i].GetKeys() { err := kr.AddKey(key) if err != nil { return err } } } return nil } func (d *ProtonDrive) DirectRename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { // fmt.Printf("DEBUG DirectRename: path=%s, newName=%s", srcObj.GetPath(), newName) if d.MainShare == nil || d.DefaultAddrKR == nil { return nil, fmt.Errorf("missing required fields: MainShare=%v, DefaultAddrKR=%v", d.MainShare != nil, d.DefaultAddrKR != nil) } if d.protonDrive == nil { return nil, fmt.Errorf("protonDrive bridge is nil") } srcLink, err := d.getLink(ctx, srcObj.GetID()) if err != nil { return nil, fmt.Errorf("failed to find source: %w", err) } parentLinkID := srcLink.ParentLinkID if parentLinkID == "" { return nil, fmt.Errorf("cannot rename root folder") } encryptedName, err := d.encryptFileName(ctx, newName, parentLinkID) if err != nil { return nil, fmt.Errorf("failed to encrypt filename: %w", err) } newHash, err := d.generateFileNameHash(ctx, newName, parentLinkID) if err != nil { return nil, fmt.Errorf("failed to generate new hash: %w", err) } originalHash, err := d.getOriginalNameHash(srcLink) if err != nil { return nil, fmt.Errorf("failed to get original hash: %w", err) } renameReq := RenameRequest{ Name: encryptedName, NameSignatureEmail: d.MainShare.Creator, Hash: newHash, OriginalHash: originalHash, } err = d.executeRenameAPI(ctx, srcLink.LinkID, renameReq) if err != nil { return nil, fmt.Errorf("rename API call failed: %w", err) } return &model.Object{ ID: srcLink.LinkID, Name: newName, Size: srcObj.GetSize(), Modified: srcObj.ModTime(), IsFolder: srcObj.IsDir(), }, nil } func (d *ProtonDrive) executeRenameAPI(ctx context.Context, linkID string, req RenameRequest) error { renameURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/rename", d.MainShare.VolumeID, linkID) reqBody, err := json.Marshal(req) if err != nil { return fmt.Errorf("failed to marshal rename request: %w", err) } httpReq, err := http.NewRequestWithContext(ctx, "PUT", renameURL, bytes.NewReader(reqBody)) if err != nil { return fmt.Errorf("failed to create HTTP request: %w", err) } httpReq.Header.Set("Content-Type", "application/json") httpReq.Header.Set("Accept", d.protonJson) httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV) httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion) httpReq.Header.Set("X-Pm-Uid", d.ReusableCredential.UID) httpReq.Header.Set("Authorization", "Bearer "+d.ReusableCredential.AccessToken) client := &http.Client{} resp, err := client.Do(httpReq) if err != nil { return fmt.Errorf("failed to execute rename request: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("rename failed with status %d", resp.StatusCode) } var renameResp RenameResponse if err := json.NewDecoder(resp.Body).Decode(&renameResp); err != nil { return fmt.Errorf("failed to decode rename response: %w", err) } if renameResp.Code != 1000 { return fmt.Errorf("rename failed with code %d", renameResp.Code) } return nil } func (d *ProtonDrive) executeMoveAPI(ctx context.Context, linkID string, req MoveRequest) error { // fmt.Printf("DEBUG Move Request - Name: %s\n", req.Name) // fmt.Printf("DEBUG Move Request - Hash: %s\n", req.Hash) // fmt.Printf("DEBUG Move Request - OriginalHash: %s\n", req.OriginalHash) // fmt.Printf("DEBUG Move Request - ParentLinkID: %s\n", req.ParentLinkID) // fmt.Printf("DEBUG Move Request - Name length: %d\n", len(req.Name)) // fmt.Printf("DEBUG Move Request - NameSignatureEmail: %s\n", req.NameSignatureEmail) // fmt.Printf("DEBUG Move Request - ContentHash: %v\n", req.ContentHash) // fmt.Printf("DEBUG Move Request - NodePassphrase length: %d\n", len(req.NodePassphrase)) // fmt.Printf("DEBUG Move Request - NodePassphraseSignature length: %d\n", len(req.NodePassphraseSignature)) // fmt.Printf("DEBUG Move Request - SrcLinkID: %s\n", linkID) // fmt.Printf("DEBUG Move Request - DstParentLinkID: %s\n", req.ParentLinkID) // fmt.Printf("DEBUG Move Request - ShareID: %s\n", d.MainShare.ShareID) srcLink, _ := d.getLink(ctx, linkID) if srcLink != nil && srcLink.ParentLinkID == req.ParentLinkID { return fmt.Errorf("cannot move to same parent directory") } moveURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/move", d.MainShare.VolumeID, linkID) reqBody, err := json.Marshal(req) if err != nil { return fmt.Errorf("failed to marshal move request: %w", err) } httpReq, err := http.NewRequestWithContext(ctx, "PUT", moveURL, bytes.NewReader(reqBody)) if err != nil { return fmt.Errorf("failed to create HTTP request: %w", err) } httpReq.Header.Set("Authorization", "Bearer "+d.ReusableCredential.AccessToken) httpReq.Header.Set("Accept", d.protonJson) httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV) httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion) httpReq.Header.Set("X-Pm-Uid", d.ReusableCredential.UID) httpReq.Header.Set("Content-Type", "application/json") client := &http.Client{} resp, err := client.Do(httpReq) if err != nil { return fmt.Errorf("failed to execute move request: %w", err) } defer resp.Body.Close() var moveResp RenameResponse if err := json.NewDecoder(resp.Body).Decode(&moveResp); err != nil { return fmt.Errorf("failed to decode move response: %w", err) } if moveResp.Code != 1000 { return fmt.Errorf("move operation failed with code: %d", moveResp.Code) } return nil } func (d *ProtonDrive) DirectMove(ctx context.Context, srcObj model.Obj, dstDir model.Obj) (model.Obj, error) { // fmt.Printf("DEBUG DirectMove: srcPath=%s, dstPath=%s", srcObj.GetPath(), dstDir.GetPath()) srcLink, err := d.getLink(ctx, srcObj.GetID()) if err != nil { return nil, fmt.Errorf("failed to find source: %w", err) } dstParentLinkID := dstDir.GetID() if srcObj.IsDir() { // Check if destination is a descendant of source if err := d.checkCircularMove(ctx, srcLink.LinkID, dstParentLinkID); err != nil { return nil, err } } // Encrypt the filename for the new location encryptedName, err := d.encryptFileName(ctx, srcObj.GetName(), dstParentLinkID) if err != nil { return nil, fmt.Errorf("failed to encrypt filename: %w", err) } newHash, err := d.generateNameHash(ctx, srcObj.GetName(), dstParentLinkID) if err != nil { return nil, fmt.Errorf("failed to generate new hash: %w", err) } originalHash, err := d.getOriginalNameHash(srcLink) if err != nil { return nil, fmt.Errorf("failed to get original hash: %w", err) } // Re-encrypt node passphrase for new parent context reencryptedPassphrase, err := d.reencryptNodePassphrase(ctx, srcLink, dstParentLinkID) if err != nil { return nil, fmt.Errorf("failed to re-encrypt node passphrase: %w", err) } moveReq := MoveRequest{ ParentLinkID: dstParentLinkID, NodePassphrase: reencryptedPassphrase, Name: encryptedName, NameSignatureEmail: d.MainShare.Creator, Hash: newHash, OriginalHash: originalHash, ContentHash: nil, // *** Causes rejection *** /* NodePassphraseSignature: srcLink.NodePassphraseSignature, */ } //fmt.Printf("DEBUG MoveRequest validation:\n") //fmt.Printf(" Name length: %d\n", len(moveReq.Name)) //fmt.Printf(" Hash: %s\n", moveReq.Hash) //fmt.Printf(" OriginalHash: %s\n", moveReq.OriginalHash) //fmt.Printf(" NodePassphrase length: %d\n", len(moveReq.NodePassphrase)) /* fmt.Printf(" NodePassphraseSignature length: %d\n", len(moveReq.NodePassphraseSignature)) */ //fmt.Printf(" NameSignatureEmail: %s\n", moveReq.NameSignatureEmail) err = d.executeMoveAPI(ctx, srcLink.LinkID, moveReq) if err != nil { return nil, fmt.Errorf("move API call failed: %w", err) } return &model.Object{ ID: srcLink.LinkID, Name: srcObj.GetName(), Size: srcObj.GetSize(), Modified: srcObj.ModTime(), IsFolder: srcObj.IsDir(), }, nil } func (d *ProtonDrive) reencryptNodePassphrase(ctx context.Context, srcLink *proton.Link, dstParentLinkID string) (string, error) { // Get source parent link with metadata srcParentLink, err := d.getLink(ctx, srcLink.ParentLinkID) if err != nil { return "", fmt.Errorf("failed to get source parent link: %w", err) } // Get source parent keyring using link object srcParentKR, err := d.getLinkKR(ctx, srcParentLink) if err != nil { return "", fmt.Errorf("failed to get source parent keyring: %w", err) } // Get destination parent link with metadata dstParentLink, err := d.getLink(ctx, dstParentLinkID) if err != nil { return "", fmt.Errorf("failed to get destination parent link: %w", err) } // Get destination parent keyring using link object dstParentKR, err := d.getLinkKR(ctx, dstParentLink) if err != nil { return "", fmt.Errorf("failed to get destination parent keyring: %w", err) } // Re-encrypt the node passphrase from source parent context to destination parent context reencryptedPassphrase, err := reencryptKeyPacket(srcParentKR, dstParentKR, d.DefaultAddrKR, srcLink.NodePassphrase) if err != nil { return "", fmt.Errorf("failed to re-encrypt key packet: %w", err) } return reencryptedPassphrase, nil } func (d *ProtonDrive) generateNameHash(ctx context.Context, name string, parentLinkID string) (string, error) { parentLink, err := d.getLink(ctx, parentLinkID) if err != nil { return "", fmt.Errorf("failed to get parent link: %w", err) } // Get parent node keyring parentNodeKR, err := d.getLinkKR(ctx, parentLink) if err != nil { return "", fmt.Errorf("failed to get parent keyring: %w", err) } // Get signature verification keyring signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR) if err != nil { return "", fmt.Errorf("failed to get signature verification keyring: %w", err) } parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR) if err != nil { return "", fmt.Errorf("failed to get parent hash key: %w", err) } nameHash, err := proton.GetNameHash(name, parentHashKey) if err != nil { return "", fmt.Errorf("failed to generate name hash: %w", err) } return nameHash, nil } func reencryptKeyPacket(srcKR, dstKR, _ *crypto.KeyRing, passphrase string) (string, error) { // addrKR (3) oldSplitMessage, err := crypto.NewPGPSplitMessageFromArmored(passphrase) if err != nil { return "", err } sessionKey, err := srcKR.DecryptSessionKey(oldSplitMessage.KeyPacket) if err != nil { return "", err } newKeyPacket, err := dstKR.EncryptSessionKey(sessionKey) if err != nil { return "", err } newSplitMessage := crypto.NewPGPSplitMessage(newKeyPacket, oldSplitMessage.DataPacket) return newSplitMessage.GetArmored() } func (d *ProtonDrive) checkCircularMove(ctx context.Context, srcLinkID, dstParentLinkID string) error { currentLinkID := dstParentLinkID for currentLinkID != "" && currentLinkID != d.RootFolderID { if currentLinkID == srcLinkID { return fmt.Errorf("cannot move folder into itself or its subfolder") } currentLink, err := d.getLink(ctx, currentLinkID) if err != nil { return err } currentLinkID = currentLink.ParentLinkID } return nil } func (d *ProtonDrive) authHandler(auth proton.Auth) { if auth.AccessToken != d.ReusableCredential.AccessToken || auth.RefreshToken != d.ReusableCredential.RefreshToken { d.ReusableCredential.UID = auth.UID d.ReusableCredential.AccessToken = auth.AccessToken d.ReusableCredential.RefreshToken = auth.RefreshToken if err := d.initClient(context.Background()); err != nil { fmt.Printf("ProtonDrive: failed to reinitialize client after auth refresh: %v\n", err) } op.MustSaveDriverStorage(d) } } func (d *ProtonDrive) initClient(ctx context.Context) error { clientOptions := []proton.Option{ proton.WithAppVersion(d.appVersion), proton.WithUserAgent(d.userAgent), } manager := proton.New(clientOptions...) d.c = manager.NewClient(d.ReusableCredential.UID, d.ReusableCredential.AccessToken, d.ReusableCredential.RefreshToken) saltedKeyPassBytes, err := base64.StdEncoding.DecodeString(d.ReusableCredential.SaltedKeyPass) if err != nil { return fmt.Errorf("failed to decode salted key pass: %w", err) } _, addrKRs, addrs, _, err := getAccountKRs(ctx, d.c, nil, saltedKeyPassBytes) if err != nil { return fmt.Errorf("failed to get account keyrings: %w", err) } d.addrKRs = addrKRs d.addrData = addrs return nil } ================================================ FILE: drivers/quark_open/driver.go ================================================ package quark_open import ( "context" "encoding/hex" "errors" "fmt" "hash" "io" "net/http" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" ) type QuarkOpen struct { model.Storage Addition config driver.Config conf Conf } func (d *QuarkOpen) Config() driver.Config { return d.config } func (d *QuarkOpen) GetAddition() driver.Additional { return &d.Addition } func (d *QuarkOpen) Init(ctx context.Context) error { var resp UserInfoResp _, err := d.request(ctx, "/open/v1/user/info", http.MethodGet, nil, &resp) if err != nil { return err } if resp.Data.UserID != "" { d.conf.userId = resp.Data.UserID } else { return errors.New("failed to get user ID") } return err } func (d *QuarkOpen) Drop(ctx context.Context) error { return nil } func (d *QuarkOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.GetFiles(ctx, dir.GetID()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return fileToObj(src), nil }) } func (d *QuarkOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { data := base.Json{ "fid": file.GetID(), } var resp FileLikeResp _, err := d.request(ctx, "/open/v1/file/get_download_url", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) if err != nil { return nil, err } return &model.Link{ URL: resp.Data.DownloadURL, Header: http.Header{ "Cookie": []string{d.generateAuthCookie()}, }, Concurrency: 3, PartSize: 10 * utils.MB, }, nil } func (d *QuarkOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { data := base.Json{ "dir_path": dirName, "pdir_fid": parentDir.GetID(), } _, err := d.request(ctx, "/open/v1/dir", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *QuarkOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) error { data := base.Json{ "action_type": 1, "fid_list": []string{srcObj.GetID()}, "to_pdir_fid": dstDir.GetID(), } _, err := d.request(ctx, "/open/v1/file/move", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *QuarkOpen) Rename(ctx context.Context, srcObj model.Obj, newName string) error { data := base.Json{ "fid": srcObj.GetID(), "file_name": newName, "conflict_mode": "REUSE", } _, err := d.request(ctx, "/open/v1/file/rename", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *QuarkOpen) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return errs.NotSupport } func (d *QuarkOpen) Remove(ctx context.Context, obj model.Obj) error { data := base.Json{ "action_type": 1, "fid_list": []string{obj.GetID()}, } _, err := d.request(ctx, "/open/v1/file/delete", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { md5Str, sha1Str := stream.GetHash().GetHash(utils.MD5), stream.GetHash().GetHash(utils.SHA1) var ( md5 hash.Hash sha1 hash.Hash ) writers := []io.Writer{} if len(md5Str) != utils.MD5.Width { md5 = utils.MD5.NewFunc() writers = append(writers, md5) } if len(sha1Str) != utils.SHA1.Width { sha1 = utils.SHA1.NewFunc() writers = append(writers, sha1) } if len(writers) > 0 { _, err := stream.CacheFullAndWriter(&up, io.MultiWriter(writers...)) if err != nil { return err } if md5 != nil { md5Str = hex.EncodeToString(md5.Sum(nil)) } if sha1 != nil { sha1Str = hex.EncodeToString(sha1.Sum(nil)) } } // pre pre, err := d.upPre(ctx, stream, dstDir.GetID(), md5Str, sha1Str) if err != nil { return err } // 如果预上传已经完成，直接返回--秒传 if pre.Data.Finish { up(100) return nil } // get part info partInfo := d._getPartInfo(stream, pre.Data.PartSize) // get upload url info upUrlInfo, err := d.upUrl(ctx, pre, partInfo) if err != nil { return err } // part up ss, err := streamPkg.NewStreamSectionReader(stream, int(pre.Data.PartSize), &up) if err != nil { return err } total := stream.GetSize() // 用于存储每个分片的ETag，后续commit时需要 etags := make([]string, 0, len(partInfo)) // 遍历上传每个分片 for i := range len(upUrlInfo.UploadUrls) { if utils.IsCanceled(ctx) { return ctx.Err() } offset := int64(i) * pre.Data.PartSize size := min(pre.Data.PartSize, total-offset) rd, err := ss.GetSectionReader(offset, size) if err != nil { return err } err = retry.Do(func() error { rd.Seek(0, io.SeekStart) etag, err := d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) if err != nil { return err } etags = append(etags, etag) return nil }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) ss.FreeSectionReader(rd) if err != nil { return fmt.Errorf("failed to upload part %d: %w", i, err) } up(95 * float64(offset+size) / float64(total)) } defer up(100) return d.upFinish(ctx, pre, partInfo, etags) } var _ driver.Driver = (*QuarkOpen)(nil) ================================================ FILE: drivers/quark_open/meta.go ================================================ package quark_open import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootID OrderBy string `json:"order_by" type:"select" options:"none,file_type,file_name,updated_at,created_at" default:"none"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` UseOnlineAPI bool `json:"use_online_api" default:"true"` APIAddress string `json:"api_url_address" default:"https://api.oplist.org/quarkyun/renewapi"` AccessToken string `json:"access_token" required:"false" default:""` RefreshToken string `json:"refresh_token" required:"true"` AppID string `json:"app_id" required:"true" help:"Keep it empty if you don't have one"` SignKey string `json:"sign_key" required:"true" help:"Keep it empty if you don't have one"` } type Conf struct { ua string api string userId string } func init() { op.RegisterDriver(func() driver.Driver { return &QuarkOpen{ config: driver.Config{ Name: "QuarkOpen", OnlyProxy: true, DefaultRoot: "0", NoOverwriteUpload: true, }, conf: Conf{ ua: "go-resty/3.0.0-beta.1 (https://resty.dev)", api: "https://open-api-drive.quark.cn", }, } }) } ================================================ FILE: drivers/quark_open/types.go ================================================ package quark_open import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "time" ) type Resp struct { CommonRsp Errno int `json:"errno"` ErrorInfo string `json:"error_info"` } type CommonRsp struct { Status int `json:"status"` ReqID string `json:"req_id"` } type UserInfo struct { UserID string `json:"user_id"` Nickname string `json:"nickname"` AvatarURL string `json:"avatar_url"` } type UserInfoResp struct { CommonRsp Data UserInfo `json:"data"` } type RefreshTokenOnlineAPIResp struct { RefreshToken string `json:"refresh_token"` AccessToken string `json:"access_token"` AppID string `json:"app_id"` SignKey string `json:"sign_key"` ErrorMessage string `json:"text"` } type File struct { Fid string `json:"fid"` ParentFid string `json:"parent_fid"` Category int64 `json:"category"` FileName string `json:"filename"` Size int64 `json:"size"` FileType string `json:"file_type"` ThumbnailURL string `json:"thumbnail_url"` ContentHash string `json:"content_hash"` CreatedAt int64 `json:"created_at"` UpdatedAt int64 `json:"updated_at"` } func fileToObj(f File) *model.ObjThumb { return &model.ObjThumb{ Object: model.Object{ ID: f.Fid, Name: f.FileName, Size: f.Size, Modified: time.UnixMilli(f.UpdatedAt), IsFolder: f.FileType == "0", Ctime: time.UnixMilli(f.CreatedAt), }, Thumbnail: model.Thumbnail{Thumbnail: f.ThumbnailURL}, } } type QueryCursor struct { Version string `json:"version"` Token string `json:"token"` } type FileListResp struct { CommonRsp Data struct { FileList []File `json:"file_list"` LastPage bool `json:"last_page"` NextQueryCursor QueryCursor `json:"next_query_cursor"` } `json:"data"` } type FileLikeResp struct { CommonRsp Data struct { Fid string `json:"fid"` Size int `json:"size"` FileName string `json:"file_name"` DownloadURL string `json:"download_url"` } `json:"data"` } type UpPreResp struct { CommonRsp Data struct { Finish bool `json:"finish"` TaskID string `json:"task_id"` Fid string `json:"fid"` CommonHeaders struct { XOssContentSha256 string `json:"X-Oss-Content-Sha256"` XOssDate string `json:"X-Oss-Date"` } `json:"common_headers"` UploadUrls []struct { PartNumber int `json:"part_number"` SignatureInfo struct { AuthType string `json:"auth_type"` Signature string `json:"signature"` } `json:"signature_info"` UploadURL string `json:"upload_url"` Expired int64 `json:"expired"` } `json:"upload_urls"` PartSize int64 `json:"part_size"` } `json:"data"` } type UpUrlInfo struct { UploadUrls []struct { PartNumber int `json:"part_number"` PartSize int `json:"part_size"` SignatureInfo struct { AuthType string `json:"auth_type"` Signature string `json:"signature"` } `json:"signature_info"` UploadURL string `json:"upload_url"` } `json:"upload_urls"` CommonHeaders struct { XOssContentSha256 string `json:"X-Oss-Content-Sha256"` XOssDate string `json:"X-Oss-Date"` } `json:"common_headers"` UploadID string `json:"upload_id"` } type UpUrlResp struct { CommonRsp Data UpUrlInfo `json:"data"` } type UpFinishResp struct { CommonRsp Data struct { TaskID string `json:"task_id"` Fid string `json:"fid"` Finish bool `json:"finish"` PdirFid string `json:"pdir_fid"` Thumbnail string `json:"thumbnail"` FormatType string `json:"format_type"` Size int `json:"size"` } `json:"data"` } ================================================ FILE: drivers/quark_open/util.go ================================================ package quark_open import ( "context" "crypto/md5" "crypto/sha256" "encoding/base64" "encoding/hex" "errors" "fmt" "io" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/google/uuid" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) func (d *QuarkOpen) request(ctx context.Context, pathname string, method string, callback base.ReqCallback, resp interface{}, manualSign ...*ManualSign) ([]byte, error) { u := d.conf.api + pathname var tm, token, reqID string // 检查是否手动传入签名参数 if len(manualSign) > 0 && manualSign[0] != nil { tm = manualSign[0].Tm token = manualSign[0].Token reqID = manualSign[0].ReqID } else { // 自动生成签名参数 tm, token, reqID = d.generateReqSign(method, pathname, d.Addition.SignKey) } req := base.RestyClient.R() req.SetContext(ctx) req.SetHeaders(map[string]string{ "Accept": "application/json, text/plain, */*", "User-Agent": d.conf.ua, "x-pan-tm": tm, "x-pan-token": token, "x-pan-client-id": d.Addition.AppID, }) req.SetQueryParams(map[string]string{ "req_id": reqID, "access_token": d.Addition.AccessToken, }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e Resp req.SetError(&e) res, err := req.Execute(method, u) if err != nil { return nil, err } // 判断 是否需要 刷新 access_token if e.Status == -1 && (e.Errno == 11001 || (e.Errno == 14001 && strings.Contains(e.ErrorInfo, "access_token"))) { // token 过期 err = d.refreshToken() if err != nil { return nil, err } ctx1, cancelFunc := context.WithTimeout(ctx, 10*time.Second) defer cancelFunc() return d.request(ctx1, pathname, method, callback, resp) } if e.Status >= 400 || e.Errno != 0 { return nil, errors.New(e.ErrorInfo) } return res.Body(), nil } func (d *QuarkOpen) GetFiles(ctx context.Context, parent string) ([]File, error) { files := make([]File, 0) var queryCursor QueryCursor for { reqBody := map[string]interface{}{ "parent_fid": parent, "size": 100, // 默认每页100个文件 "sort": "file_name:asc", // 基本排序方式 } // 如果有排序设置 if d.OrderBy != "none" { reqBody["sort"] = d.OrderBy + ":" + d.OrderDirection } // 设置查询游标（用于分页） if queryCursor.Token != "" { reqBody["query_cursor"] = queryCursor } var resp FileListResp _, err := d.request(ctx, "/open/v1/file/list", http.MethodPost, func(req *resty.Request) { req.SetBody(reqBody) }, &resp) if err != nil { return nil, err } files = append(files, resp.Data.FileList...) if resp.Data.LastPage { break } queryCursor = resp.Data.NextQueryCursor } return files, nil } func (d *QuarkOpen) upPre(ctx context.Context, file model.FileStreamer, parentId, md5, sha1 string) (UpPreResp, error) { // 获取当前时间 now := time.Now() // 获取文件大小 fileSize := file.GetSize() // 手动生成 x-pan-token httpMethod := "POST" apiPath := "/open/v1/file/upload_pre" tm, xPanToken, reqID := d.generateReqSign(httpMethod, apiPath, d.Addition.SignKey) // 生成proof相关字段，传入 x-pan-token proofVersion, proofSeed1, proofSeed2, proofCode1, proofCode2, err := d.generateProof(file, xPanToken) if err != nil { return UpPreResp{}, fmt.Errorf("failed to generate proof: %w", err) } data := base.Json{ "file_name": file.GetName(), "size": fileSize, "format_type": file.GetMimetype(), "md5": md5, "sha1": sha1, "l_created_at": now.UnixMilli(), "l_updated_at": now.UnixMilli(), "pdir_fid": parentId, "same_path_reuse": true, "proof_version": proofVersion, "proof_seed1": proofSeed1, "proof_seed2": proofSeed2, "proof_code1": proofCode1, "proof_code2": proofCode2, } var resp UpPreResp // 使用手动生成的签名参数 manualSign := &ManualSign{ Tm: tm, Token: xPanToken, ReqID: reqID, } _, err = d.request(ctx, "/open/v1/file/upload_pre", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp, manualSign) return resp, err } // generateProof 生成夸克云盘文件上传的proof验证信息 func (d *QuarkOpen) generateProof(file model.FileStreamer, xPanToken string) (proofVersion, proofSeed1, proofSeed2, proofCode1, proofCode2 string, err error) { // 获取文件大小 fileSize := file.GetSize() // 设置proof_version (固定为"v1") proofVersion = "v1" // 生成proof_seed1 - 算法: md5(userid+x-pan-token) proofSeed1 = d.generateProofSeed1(xPanToken) // 生成proof_seed2 - 算法: md5(fileSize) proofSeed2 = d.generateProofSeed2(fileSize) // 生成proof_code1和proof_code2 proofCode1, err = d.generateProofCode(file, proofSeed1, fileSize) if err != nil { return "", "", "", "", "", fmt.Errorf("failed to generate proof_code1: %w", err) } proofCode2, err = d.generateProofCode(file, proofSeed2, fileSize) if err != nil { return "", "", "", "", "", fmt.Errorf("failed to generate proof_code2: %w", err) } return proofVersion, proofSeed1, proofSeed2, proofCode1, proofCode2, nil } // generateProofSeed1 生成proof_seed1，基于 userId、x-pan-token func (d *QuarkOpen) generateProofSeed1(xPanToken string) string { concatString := d.conf.userId + xPanToken md5Hash := md5.Sum([]byte(concatString)) return hex.EncodeToString(md5Hash[:]) } // generateProofSeed2 生成proof_seed2，基于 fileSize func (d *QuarkOpen) generateProofSeed2(fileSize int64) string { md5Hash := md5.Sum([]byte(strconv.FormatInt(fileSize, 10))) return hex.EncodeToString(md5Hash[:]) } type ProofRange struct { Start int64 End int64 } // generateProofCode 根据proof_seed和文件大小生成proof_code func (d *QuarkOpen) generateProofCode(file model.FileStreamer, proofSeed string, fileSize int64) (string, error) { // 获取读取范围 proofRange, err := d.getProofRange(proofSeed, fileSize) if err != nil { return "", fmt.Errorf("failed to get proof range: %w", err) } // 计算需要读取的长度 length := proofRange.End - proofRange.Start if length == 0 { return "", nil } // 使用FileStreamer的RangeRead方法读取特定范围的数据 reader, err := file.RangeRead(http_range.Range{ Start: proofRange.Start, Length: length, }) if err != nil { return "", fmt.Errorf("failed to range read: %w", err) } defer func() { if closer, ok := reader.(io.Closer); ok { closer.Close() } }() // 读取数据 buf := make([]byte, length) n, err := io.ReadFull(reader, buf) if n != int(length) { return "", fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) } // Base64编码 return base64.StdEncoding.EncodeToString(buf), nil } // getProofRange 根据proof_seed和文件大小计算需要读取的文件范围 func (d *QuarkOpen) getProofRange(proofSeed string, fileSize int64) (*ProofRange, error) { if fileSize == 0 { return &ProofRange{}, nil } // 对 proofSeed 进行 MD5 处理，取前16个字符 md5Hash := md5.Sum([]byte(proofSeed)) tmpStr := hex.EncodeToString(md5Hash[:])[:16] // 转为 uint64 tmpInt, err := strconv.ParseUint(tmpStr, 16, 64) if err != nil { return nil, fmt.Errorf("failed to parse hex string: %w", err) } // 计算索引位置 index := tmpInt % uint64(fileSize) pr := &ProofRange{ Start: int64(index), End: int64(index) + 8, } // 确保 End 不超过文件大小 if pr.End > fileSize { pr.End = fileSize } return pr, nil } func (d *QuarkOpen) _getPartInfo(stream model.FileStreamer, partSize int64) []base.Json { // 计算分片信息 partInfo := make([]base.Json, 0) total := stream.GetSize() left := total partNumber := 1 // 计算每个分片的大小和编号 for left > 0 { size := partSize if left < partSize { size = left } partInfo = append(partInfo, base.Json{ "part_number": partNumber, "part_size": size, }) left -= size partNumber++ } return partInfo } func (d *QuarkOpen) upUrl(ctx context.Context, pre UpPreResp, partInfo []base.Json) (upUrlInfo UpUrlInfo, err error) { // 构建请求体 data := base.Json{ "task_id": pre.Data.TaskID, "part_info_list": partInfo, } var resp UpUrlResp _, err = d.request(ctx, "/open/v1/file/get_upload_urls", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) if err != nil { return upUrlInfo, err } return resp.Data, nil } func (d *QuarkOpen) upPart(ctx context.Context, upUrlInfo UpUrlInfo, partNumber int, bytes io.Reader) (string, error) { // 创建请求 req, err := http.NewRequestWithContext(ctx, http.MethodPut, upUrlInfo.UploadUrls[partNumber].UploadURL, bytes) if err != nil { return "", err } req.Header.Set("Authorization", upUrlInfo.UploadUrls[partNumber].SignatureInfo.Signature) req.Header.Set("X-Oss-Date", upUrlInfo.CommonHeaders.XOssDate) req.Header.Set("X-Oss-Content-Sha256", upUrlInfo.CommonHeaders.XOssContentSha256) req.Header.Set("Accept-Encoding", "gzip") req.Header.Set("User-Agent", "Go-http-client/1.1") // 发送请求 resp, err := base.HttpClient.Do(req) if err != nil { return "", err } defer resp.Body.Close() if resp.StatusCode != 200 { body, _ := io.ReadAll(resp.Body) return "", fmt.Errorf("up status: %d, error: %s", resp.StatusCode, string(body)) } // 返回 Etag 作为分片上传的标识 return resp.Header.Get("Etag"), nil } func (d *QuarkOpen) upFinish(ctx context.Context, pre UpPreResp, partInfo []base.Json, etags []string) error { // 创建 part_info_list partInfoList := make([]base.Json, len(partInfo)) // 确保 partInfo 和 etags 长度一致 if len(partInfo) != len(etags) { return fmt.Errorf("part info count (%d) does not match etags count (%d)", len(partInfo), len(etags)) } // 组合 part_info_list for i, part := range partInfo { partInfoList[i] = base.Json{ "part_number": part["part_number"], "part_size": part["part_size"], "etag": etags[i], } } // 构建请求体 data := base.Json{ "task_id": pre.Data.TaskID, "part_info_list": partInfoList, } // 发送请求 var resp UpFinishResp _, err := d.request(ctx, "/open/v1/file/upload_finish", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) if err != nil { return err } if resp.Data.Finish != true { return fmt.Errorf("upload finish failed, task_id: %s", resp.Data.TaskID) } return nil } // ManualSign 用于手动签名URL的结构体 type ManualSign struct { Tm string Token string ReqID string } func (d *QuarkOpen) generateReqSign(method string, pathname string, signKey string) (string, string, string) { // 生成时间戳 (13位毫秒级) timestamp := strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10) // 生成 x-pan-token token的组成是: method + "&" + pathname + "&" + timestamp + "&" + signKey tokenData := method + "&" + pathname + "&" + timestamp + "&" + signKey tokenHash := sha256.Sum256([]byte(tokenData)) xPanToken := hex.EncodeToString(tokenHash[:]) // 生成 req_id reqUuid, _ := uuid.NewRandom() reqID := reqUuid.String() return timestamp, xPanToken, reqID } func (d *QuarkOpen) refreshToken() error { refresh, access, err := d._refreshToken() for i := 0; i < 3; i++ { if err == nil { break } else { log.Errorf("[quark_open] failed to refresh token: %s", err) } refresh, access, err = d._refreshToken() } if err != nil { return err } log.Infof("[quark_open] token exchange: %s -> %s", d.RefreshToken, refresh) d.RefreshToken, d.AccessToken = refresh, access op.MustSaveDriverStorage(d) return nil } func (d *QuarkOpen) _refreshToken() (string, string, error) { if d.UseOnlineAPI && d.APIAddress != "" { u := d.APIAddress var resp RefreshTokenOnlineAPIResp _, err := base.RestyClient.R(). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, "server_use": "true", "driver_txt": "quarkyun_oa", }). Get(u) if err != nil { return "", "", err } if resp.RefreshToken == "" || resp.AccessToken == "" { if resp.ErrorMessage != "" { return "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) } return "", "", fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") } return resp.RefreshToken, resp.AccessToken, nil } // TODO 本地刷新逻辑 return "", "", fmt.Errorf("local refresh token logic is not implemented yet, please use online API or contact the developer") } // 生成认证 Cookie func (d *QuarkOpen) generateAuthCookie() string { return fmt.Sprintf("x_pan_client_id=%s; x_pan_access_token=%s", d.Addition.AppID, d.Addition.AccessToken) } ================================================ FILE: drivers/quark_uc/driver.go ================================================ package quark import ( "context" "encoding/hex" "hash" "io" "net/http" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" ) type QuarkOrUC struct { model.Storage Addition config driver.Config conf Conf } func (d *QuarkOrUC) Config() driver.Config { return d.config } func (d *QuarkOrUC) GetAddition() driver.Additional { return &d.Addition } func (d *QuarkOrUC) Init(ctx context.Context) error { _, err := d.request("/config", http.MethodGet, nil, nil) if err == nil { if d.AdditionVersion != 2 { d.AdditionVersion = 2 if !d.UseTransCodingAddress && len(d.DownProxyURL) == 0 { d.WebProxy = true d.WebdavPolicy = "native_proxy" } } } return err } func (d *QuarkOrUC) Drop(ctx context.Context) error { return nil } func (d *QuarkOrUC) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.GetFiles(dir.GetID()) if err != nil { return nil, err } return files, nil } func (d *QuarkOrUC) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { f := file.(*File) if d.UseTransCodingAddress && d.config.Name == "Quark" && f.Category == 1 && f.Size > 0 { return d.getTranscodingLink(file) } return d.getDownloadLink(file) } func (d *QuarkOrUC) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { data := base.Json{ "dir_init_lock": false, "dir_path": "", "file_name": dirName, "pdir_fid": parentDir.GetID(), } _, err := d.request("/file", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) if err == nil { time.Sleep(time.Second) } return err } func (d *QuarkOrUC) Move(ctx context.Context, srcObj, dstDir model.Obj) error { data := base.Json{ "action_type": 1, "exclude_fids": []string{}, "filelist": []string{srcObj.GetID()}, "to_pdir_fid": dstDir.GetID(), } _, err := d.request("/file/move", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *QuarkOrUC) Rename(ctx context.Context, srcObj model.Obj, newName string) error { data := base.Json{ "fid": srcObj.GetID(), "file_name": newName, } _, err := d.request("/file/rename", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *QuarkOrUC) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return errs.NotSupport } func (d *QuarkOrUC) Remove(ctx context.Context, obj model.Obj) error { data := base.Json{ "action_type": 1, "exclude_fids": []string{}, "filelist": []string{obj.GetID()}, } _, err := d.request("/file/delete", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { md5Str, sha1Str := stream.GetHash().GetHash(utils.MD5), stream.GetHash().GetHash(utils.SHA1) var ( md5 hash.Hash sha1 hash.Hash ) writers := []io.Writer{} if len(md5Str) != utils.MD5.Width { md5 = utils.MD5.NewFunc() writers = append(writers, md5) } if len(sha1Str) != utils.SHA1.Width { sha1 = utils.SHA1.NewFunc() writers = append(writers, sha1) } if len(writers) > 0 { _, err := stream.CacheFullAndWriter(&up, io.MultiWriter(writers...)) if err != nil { return err } if md5 != nil { md5Str = hex.EncodeToString(md5.Sum(nil)) } if sha1 != nil { sha1Str = hex.EncodeToString(sha1.Sum(nil)) } } // pre pre, err := d.upPre(stream, dstDir.GetID()) if err != nil { return err } // hash finish, err := d.upHash(md5Str, sha1Str, pre.Data.TaskId) if err != nil { return err } if finish { up(100) return nil } // part up ss, err := streamPkg.NewStreamSectionReader(stream, pre.Metadata.PartSize, &up) if err != nil { return err } total := stream.GetSize() partSize := int64(pre.Metadata.PartSize) uploadNums := int((total + partSize - 1) / partSize) md5s := make([]string, 0, uploadNums) for partIndex := range uploadNums { if utils.IsCanceled(ctx) { return ctx.Err() } offset := int64(partIndex) * partSize size := min(partSize, total-offset) rd, err := ss.GetSectionReader(offset, size) if err != nil { return err } err = retry.Do(func() error { rd.Seek(0, io.SeekStart) m, err := d.upPart(ctx, pre, stream.GetMimetype(), partIndex+1, driver.NewLimitedUploadStream(ctx, rd)) if err != nil { return err } if m == "finish" { up(100) return nil } md5s = append(md5s, m) return nil }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) ss.FreeSectionReader(rd) if err != nil { return err } up(95 * float64(offset+size) / float64(total)) } up(97) err = d.upCommit(pre, md5s) if err != nil { return err } defer up(100) return d.upFinish(pre) } func (d *QuarkOrUC) GetDetails(ctx context.Context) (*model.StorageDetails, error) { memberInfo, err := d.memberInfo(ctx) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: memberInfo.Data.TotalCapacity, UsedSpace: memberInfo.Data.UseCapacity, }, }, nil } var _ driver.Driver = (*QuarkOrUC)(nil) ================================================ FILE: drivers/quark_uc/meta.go ================================================ package quark import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { Cookie string `json:"cookie" required:"true"` driver.RootID OrderBy string `json:"order_by" type:"select" options:"none,file_type,file_name,updated_at" default:"none"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` UseTransCodingAddress bool `json:"use_transcoding_address" help:"You can watch the transcoded video and support 302 redirection" required:"true" default:"false"` OnlyListVideoFile bool `json:"only_list_video_file" default:"false"` AdditionVersion int } type Conf struct { ua string referer string api string pr string } func init() { op.RegisterDriver(func() driver.Driver { return &QuarkOrUC{ config: driver.Config{ Name: "Quark", DefaultRoot: "0", NoOverwriteUpload: true, }, conf: Conf{ ua: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch", referer: "https://pan.quark.cn", api: "https://drive.quark.cn/1/clouddrive", pr: "ucpro", }, } }) op.RegisterDriver(func() driver.Driver { return &QuarkOrUC{ config: driver.Config{ Name: "UC", OnlyProxy: true, DefaultRoot: "0", NoOverwriteUpload: true, }, conf: Conf{ ua: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) uc-cloud-drive/2.5.20 Chrome/100.0.4896.160 Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch", referer: "https://drive.uc.cn", api: "https://pc-api.uc.cn/1/clouddrive", pr: "UCBrowser", }, } }) } ================================================ FILE: drivers/quark_uc/types.go ================================================ package quark import ( "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type Resp struct { Status int `json:"status"` Code int `json:"code"` Message string `json:"message"` // ReqId string `json:"req_id"` // Timestamp int `json:"timestamp"` } var _ model.Obj = (*File)(nil) type File struct { Fid string `json:"fid"` FileName string `json:"file_name"` // PdirFid string `json:"pdir_fid"` Category int `json:"category"` // FileType int `json:"file_type"` Size int64 `json:"size"` // FormatType string `json:"format_type"` // Status int `json:"status"` // Tags string `json:"tags,omitempty"` LCreatedAt int64 `json:"l_created_at"` LUpdatedAt int64 `json:"l_updated_at"` // NameSpace int `json:"name_space"` // IncludeItems int `json:"include_items,omitempty"` // RiskType int `json:"risk_type"` // BackupSign int `json:"backup_sign"` // Duration int `json:"duration"` // FileSource string `json:"file_source"` File bool `json:"file"` CreatedAt int64 `json:"created_at"` UpdatedAt int64 `json:"updated_at"` // PrivateExtra struct {} `json:"_private_extra"` // ObjCategory string `json:"obj_category,omitempty"` // Thumbnail string `json:"thumbnail,omitempty"` } func fileToObj(f File) *model.Object { return &model.Object{ ID: f.Fid, Name: f.FileName, Size: f.Size, Modified: time.UnixMilli(f.UpdatedAt), Ctime: time.UnixMilli(f.CreatedAt), IsFolder: !f.File, } } func (f *File) GetSize() int64 { return f.Size } func (f *File) GetName() string { return f.FileName } func (f *File) ModTime() time.Time { return time.UnixMilli(f.UpdatedAt) } func (f *File) CreateTime() time.Time { return time.UnixMilli(f.CreatedAt) } func (f *File) IsDir() bool { return !f.File } func (f *File) GetHash() utils.HashInfo { return utils.HashInfo{} } func (f *File) GetID() string { return f.Fid } func (f *File) GetPath() string { return "" } type SortResp struct { Resp Data struct { List []File `json:"list"` } `json:"data"` Metadata struct { Size int `json:"_size"` Page int `json:"_page"` Count int `json:"_count"` Total int `json:"_total"` Way string `json:"way"` } `json:"metadata"` } type DownResp struct { Resp Data []struct { // Fid string `json:"fid"` // FileName string `json:"file_name"` // PdirFid string `json:"pdir_fid"` // Category int `json:"category"` // FileType int `json:"file_type"` // Size int `json:"size"` // FormatType string `json:"format_type"` // Status int `json:"status"` // Tags string `json:"tags"` // LCreatedAt int64 `json:"l_created_at"` // LUpdatedAt int64 `json:"l_updated_at"` // NameSpace int `json:"name_space"` // Thumbnail string `json:"thumbnail"` DownloadUrl string `json:"download_url"` //Md5 string `json:"md5"` //RiskType int `json:"risk_type"` //RangeSize int `json:"range_size"` //BackupSign int `json:"backup_sign"` //ObjCategory string `json:"obj_category"` //Duration int `json:"duration"` //FileSource string `json:"file_source"` //File bool `json:"file"` //CreatedAt int64 `json:"created_at"` //UpdatedAt int64 `json:"updated_at"` //PrivateExtra struct { //} `json:"_private_extra"` } `json:"data"` //Metadata struct { // Acc2 string `json:"acc2"` // Acc1 string `json:"acc1"` //} `json:"metadata"` } type TranscodingResp struct { Resp Data struct { DefaultResolution string `json:"default_resolution"` OriginDefaultResolution string `json:"origin_default_resolution"` VideoList []struct { Resolution string `json:"resolution"` VideoInfo struct { Duration int `json:"duration"` Size int64 `json:"size"` Format string `json:"format"` Width int `json:"width"` Height int `json:"height"` Bitrate float64 `json:"bitrate"` Codec string `json:"codec"` Fps float64 `json:"fps"` Rotate int `json:"rotate"` Audio struct { Duration int `json:"duration"` Bitrate float64 `json:"bitrate"` Codec string `json:"codec"` Channels int `json:"channels"` } `json:"audio"` UpdateTime int64 `json:"update_time"` URL string `json:"url"` Resolution string `json:"resolution"` HlsType string `json:"hls_type"` Finish bool `json:"finish"` Resoultion string `json:"resoultion"` Success bool `json:"success"` } `json:"video_info,omitempty"` // Right string `json:"right"` // MemberRight string `json:"member_right"` // TransStatus string `json:"trans_status"` // Accessable bool `json:"accessable"` // SupportsFormat string `json:"supports_format"` // VideoFuncType string `json:"video_func_type,omitempty"` } `json:"video_list"` // AudioList []interface{} `json:"audio_list"` FileName string `json:"file_name"` NameSpace int `json:"name_space"` Size int64 `json:"size"` Thumbnail string `json:"thumbnail"` //LastPlayInfo struct { // Time int `json:"time"` //} `json:"last_play_info"` //SeekPreviewData struct { // TotalFrameCount int `json:"total_frame_count"` // TotalSpriteCount int `json:"total_sprite_count"` // FrameWidth int `json:"frame_width"` // FrameHeight int `json:"frame_height"` // SpriteRow int `json:"sprite_row"` // SpriteColumn int `json:"sprite_column"` // PreviewSpriteInfos []struct { // URL string `json:"url"` // FrameCount int `json:"frame_count"` // Times []int `json:"times"` // } `json:"preview_sprite_infos"` //} `json:"seek_preview_data"` //ObjKey string `json:"obj_key"` //Meta struct { // Duration int `json:"duration"` // Size int64 `json:"size"` // Format string `json:"format"` // Width int `json:"width"` // Height int `json:"height"` // Bitrate float64 `json:"bitrate"` // Codec string `json:"codec"` // Fps float64 `json:"fps"` // Rotate int `json:"rotate"` //} `json:"meta"` //PreloadLevel int `json:"preload_level"` //HasSeekPreviewData bool `json:"has_seek_preview_data"` } `json:"data"` } type UpPreResp struct { Resp Data struct { TaskId string `json:"task_id"` Finish bool `json:"finish"` UploadId string `json:"upload_id"` ObjKey string `json:"obj_key"` UploadUrl string `json:"upload_url"` Fid string `json:"fid"` Bucket string `json:"bucket"` Callback struct { CallbackUrl string `json:"callbackUrl"` CallbackBody string `json:"callbackBody"` } `json:"callback"` FormatType string `json:"format_type"` Size int `json:"size"` AuthInfo string `json:"auth_info"` } `json:"data"` Metadata struct { PartThread int `json:"part_thread"` Acc2 string `json:"acc2"` Acc1 string `json:"acc1"` PartSize int `json:"part_size"` // 分片大小 } `json:"metadata"` } type HashResp struct { Resp Data struct { Finish bool `json:"finish"` Fid string `json:"fid"` Thumbnail string `json:"thumbnail"` FormatType string `json:"format_type"` } `json:"data"` Metadata struct{} `json:"metadata"` } type UpAuthResp struct { Resp Data struct { AuthKey string `json:"auth_key"` Speed int `json:"speed"` Headers []interface{} `json:"headers"` } `json:"data"` Metadata struct{} `json:"metadata"` } type MemberResp struct { Resp Data struct { MemberType string `json:"member_type"` CreatedAt uint64 `json:"created_at"` SecretUseCapacity int64 `json:"secret_use_capacity"` UseCapacity int64 `json:"use_capacity"` IsNewUser bool `json:"is_new_user"` MemberStatus struct { Vip string `json:"VIP"` ZVip string `json:"Z_VIP"` MiniVip string `json:"MINI_VIP"` SuperVip string `json:"SUPER_VIP"` } `json:"member_status"` SecretTotalCapacity int64 `json:"secret_total_capacity"` TotalCapacity int64 `json:"total_capacity"` } `json:"data"` Metadata struct { RangeSize int `json:"range_size"` ServerCurTime uint64 `json:"server_cur_time"` } `json:"metadata"` } ================================================ FILE: drivers/quark_uc/util.go ================================================ package quark import ( "context" "crypto/md5" "encoding/base64" "errors" "fmt" "html" "io" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/cookie" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) // do others that not defined in Driver interface func (d *QuarkOrUC) request(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { u := d.conf.api + pathname req := base.RestyClient.R() req.SetHeaders(map[string]string{ "Cookie": d.Cookie, "Accept": "application/json, text/plain, */*", "Referer": d.conf.referer, }) req.SetQueryParam("pr", d.conf.pr) req.SetQueryParam("fr", "pc") if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e Resp req.SetError(&e) res, err := req.Execute(method, u) if err != nil { return nil, err } __puus := cookie.GetCookie(res.Cookies(), "__puus") if __puus != nil { d.Cookie = cookie.SetStr(d.Cookie, "__puus", __puus.Value) op.MustSaveDriverStorage(d) } if d.UseTransCodingAddress && d.config.Name == "Quark" { __pus := cookie.GetCookie(res.Cookies(), "__pus") if __pus != nil { d.Cookie = cookie.SetStr(d.Cookie, "__pus", __pus.Value) op.MustSaveDriverStorage(d) } } if e.Status >= 400 || e.Code != 0 { return nil, errors.New(e.Message) } return res.Body(), nil } func (d *QuarkOrUC) GetFiles(parent string) ([]model.Obj, error) { files := make([]model.Obj, 0) page := 1 size := 100 query := map[string]string{ "pdir_fid": parent, "_size": strconv.Itoa(size), "_fetch_total": "1", "fetch_all_file": "1", "fetch_risk_file_name": "1", } if d.OrderBy != "none" { query["_sort"] = "file_type:asc," + d.OrderBy + ":" + d.OrderDirection } for { query["_page"] = strconv.Itoa(page) var resp SortResp _, err := d.request("/file/sort", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return nil, err } for _, file := range resp.Data.List { file.FileName = html.UnescapeString(file.FileName) if d.OnlyListVideoFile { // 开启后 只列出视频文件和文件夹 if file.IsDir() || file.Category == 1 { files = append(files, &file) } } else { files = append(files, &file) } } if page*size >= resp.Metadata.Total { break } page++ } return files, nil } func (d *QuarkOrUC) getDownloadLink(file model.Obj) (*model.Link, error) { data := base.Json{ "fids": []string{file.GetID()}, } var resp DownResp ua := d.conf.ua _, err := d.request("/file/download", http.MethodPost, func(req *resty.Request) { req.SetHeader("User-Agent", ua). SetBody(data) }, &resp) if err != nil { return nil, err } return &model.Link{ URL: resp.Data[0].DownloadUrl, Header: http.Header{ "Cookie": []string{d.Cookie}, "Referer": []string{d.conf.referer}, "User-Agent": []string{ua}, }, Concurrency: 3, PartSize: 10 * utils.MB, }, nil } func (d *QuarkOrUC) getTranscodingLink(file model.Obj) (*model.Link, error) { data := base.Json{ "fid": file.GetID(), "resolutions": "low,normal,high,super,2k,4k", "supports": "fmp4_av,m3u8,dolby_vision", } var resp TranscodingResp ua := d.conf.ua _, err := d.request("/file/v2/play/project", http.MethodPost, func(req *resty.Request) { req.SetHeader("User-Agent", ua). SetBody(data) }, &resp) if err != nil { return nil, err } for _, info := range resp.Data.VideoList { if info.VideoInfo.URL != "" { return &model.Link{ URL: info.VideoInfo.URL, ContentLength: info.VideoInfo.Size, Concurrency: 3, PartSize: 10 * utils.MB, }, nil } } return nil, errors.New("no link found") } func (d *QuarkOrUC) upPre(file model.FileStreamer, parentId string) (UpPreResp, error) { now := time.Now() data := base.Json{ "ccp_hash_update": true, "dir_name": "", "file_name": file.GetName(), "format_type": file.GetMimetype(), "l_created_at": now.UnixMilli(), "l_updated_at": now.UnixMilli(), "pdir_fid": parentId, "size": file.GetSize(), //"same_path_reuse": true, } var resp UpPreResp _, err := d.request("/file/upload/pre", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) return resp, err } func (d *QuarkOrUC) upHash(md5, sha1, taskId string) (bool, error) { data := base.Json{ "md5": md5, "sha1": sha1, "task_id": taskId, } log.Debugf("hash: %+v", data) var resp HashResp _, err := d.request("/file/update/hash", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) return resp.Data.Finish, err } func (d *QuarkOrUC) upPart(ctx context.Context, pre UpPreResp, mineType string, partNumber int, bytes io.Reader) (string, error) { // func (driver QuarkOrUC) UpPart(pre UpPreResp, mineType string, partNumber int, bytes []byte, account *model.Account, md5Str, sha1Str string) (string, error) { timeStr := time.Now().UTC().Format(http.TimeFormat) data := base.Json{ "auth_info": pre.Data.AuthInfo, "auth_meta": fmt.Sprintf(`PUT %s %s x-oss-date:%s x-oss-user-agent:aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit /%s/%s?partNumber=%d&uploadId=%s`, mineType, timeStr, timeStr, pre.Data.Bucket, pre.Data.ObjKey, partNumber, pre.Data.UploadId), "task_id": pre.Data.TaskId, } var resp UpAuthResp _, err := d.request("/file/upload/auth", http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetContext(ctx) }, &resp) if err != nil { return "", err } //if partNumber == 1 { // finish, err := driver.UpHash(md5Str, sha1Str, pre.Data.TaskId, account) // if err != nil { // return "", err // } // if finish { // return "finish", nil // } //} u := fmt.Sprintf("https://%s.%s/%s", pre.Data.Bucket, pre.Data.UploadUrl[7:], pre.Data.ObjKey) req, err := http.NewRequestWithContext(ctx, http.MethodPut, u, bytes) if err != nil { return "", err } req.Header.Set("Authorization", resp.Data.AuthKey) req.Header.Set("Content-Type", mineType) req.Header.Set("Referer", "https://pan.quark.cn/") req.Header.Set("x-oss-date", timeStr) req.Header.Set("x-oss-user-agent", "aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit") q := req.URL.Query() q.Add("partNumber", strconv.Itoa(partNumber)) q.Add("uploadId", pre.Data.UploadId) req.URL.RawQuery = q.Encode() res, err := base.HttpClient.Do(req) if err != nil { return "", err } defer res.Body.Close() if res.StatusCode != 200 { respBody, _ := io.ReadAll(res.Body) return "", fmt.Errorf("up status: %d, error: %s", res.StatusCode, string(respBody)) } return res.Header.Get("Etag"), nil } func (d *QuarkOrUC) upCommit(pre UpPreResp, md5s []string) error { timeStr := time.Now().UTC().Format(http.TimeFormat) log.Debugf("md5s: %+v", md5s) bodyBuilder := strings.Builder{} bodyBuilder.WriteString(` `) for i, m := range md5s { bodyBuilder.WriteString(fmt.Sprintf(` %d %s `, i+1, m)) } bodyBuilder.WriteString("") body := bodyBuilder.String() m := md5.New() m.Write([]byte(body)) contentMd5 := base64.StdEncoding.EncodeToString(m.Sum(nil)) callbackBytes, err := utils.Json.Marshal(pre.Data.Callback) if err != nil { return err } callbackBase64 := base64.StdEncoding.EncodeToString(callbackBytes) data := base.Json{ "auth_info": pre.Data.AuthInfo, "auth_meta": fmt.Sprintf(`POST %s application/xml %s x-oss-callback:%s x-oss-date:%s x-oss-user-agent:aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit /%s/%s?uploadId=%s`, contentMd5, timeStr, callbackBase64, timeStr, pre.Data.Bucket, pre.Data.ObjKey, pre.Data.UploadId), "task_id": pre.Data.TaskId, } log.Debugf("xml: %s", body) log.Debugf("auth data: %+v", data) var resp UpAuthResp _, err = d.request("/file/upload/auth", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) if err != nil { return err } u := fmt.Sprintf("https://%s.%s/%s", pre.Data.Bucket, pre.Data.UploadUrl[7:], pre.Data.ObjKey) res, err := base.RestyClient.R(). SetHeaders(map[string]string{ "Authorization": resp.Data.AuthKey, "Content-MD5": contentMd5, "Content-Type": "application/xml", "Referer": "https://pan.quark.cn/", "x-oss-callback": callbackBase64, "x-oss-date": timeStr, "x-oss-user-agent": "aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit", }). SetQueryParams(map[string]string{ "uploadId": pre.Data.UploadId, }).SetBody(body).Post(u) if err != nil { return err } if res.StatusCode() != 200 { return fmt.Errorf("up status: %d, error: %s", res.StatusCode(), res.String()) } return nil } func (d *QuarkOrUC) upFinish(pre UpPreResp) error { data := base.Json{ "obj_key": pre.Data.ObjKey, "task_id": pre.Data.TaskId, } _, err := d.request("/file/upload/finish", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) if err != nil { return err } time.Sleep(time.Second) return nil } func (d *QuarkOrUC) memberInfo(ctx context.Context) (*MemberResp, error) { var resp MemberResp query := map[string]string{ "fetch_subscribe": "false", "_ch": "home", "fetch_identity": "false", } _, err := d.request("/member", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) req.SetContext(ctx) }, &resp) if err != nil { return nil, err } return &resp, nil } ================================================ FILE: drivers/quark_uc_tv/driver.go ================================================ package quark_uc_tv import ( "context" "fmt" "net/http" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type QuarkUCTV struct { *QuarkUCTVCommon model.Storage Addition config driver.Config conf Conf } func (d *QuarkUCTV) Config() driver.Config { return d.config } func (d *QuarkUCTV) GetAddition() driver.Additional { return &d.Addition } func (d *QuarkUCTV) Init(ctx context.Context) error { if d.Addition.DeviceID == "" { d.Addition.DeviceID = utils.GetMD5EncodeStr(time.Now().String()) } op.MustSaveDriverStorage(d) if d.QuarkUCTVCommon == nil { d.QuarkUCTVCommon = &QuarkUCTVCommon{ AccessToken: "", } } ctx1, cancelFunc := context.WithTimeout(ctx, 5*time.Second) defer cancelFunc() if d.Addition.RefreshToken == "" { if d.Addition.QueryToken == "" { qrData, err := d.getLoginCode(ctx1) if err != nil { return err } // 展示二维码 qrTemplate := ` ` qrPage := fmt.Sprintf(qrTemplate, qrData) return fmt.Errorf("need verify: \n%s", qrPage) } else { // 通过query token获取code -> refresh token code, err := d.getCode(ctx1) if err != nil { return err } // 通过code获取refresh token err = d.getRefreshTokenByTV(ctx1, code, false) if err != nil { return err } } } // 通过refresh token获取access token if d.QuarkUCTVCommon.AccessToken == "" { err := d.getRefreshTokenByTV(ctx1, d.Addition.RefreshToken, true) if err != nil { return err } } // 验证 access token 是否有效 _, err := d.isLogin(ctx1) if err != nil { return err } return nil } func (d *QuarkUCTV) Drop(ctx context.Context) error { return nil } func (d *QuarkUCTV) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files := make([]model.Obj, 0) pageIndex := int64(0) pageSize := int64(100) desc := "1" orderBy := "3" if d.OrderDirection == "asc" { desc = "0" } if d.OrderBy == "file_name" { orderBy = "1" } for { var filesData FilesData _, err := d.request(ctx, "/file", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "method": "list", "parent_fid": dir.GetID(), "order_by": orderBy, "desc": desc, "category": "", "source": "", "ex_source": "", "list_all": "0", "page_size": strconv.FormatInt(pageSize, 10), "page_index": strconv.FormatInt(pageIndex, 10), }) }, &filesData) if err != nil { return nil, err } for i := range filesData.Data.Files { files = append(files, &filesData.Data.Files[i]) } if pageIndex*pageSize >= filesData.Data.TotalCount { break } else { pageIndex++ } } return files, nil } func (d *QuarkUCTV) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { f := file.(*Files) if d.Addition.VideoLinkMethod == "streaming" && f.Category == 1 && f.Size > 0 { return d.getTranscodingLink(ctx, file) } return d.getDownloadLink(ctx, file) } func (d *QuarkUCTV) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { return nil, errs.NotImplement } func (d *QuarkUCTV) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return nil, errs.NotImplement } func (d *QuarkUCTV) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { return nil, errs.NotImplement } func (d *QuarkUCTV) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return nil, errs.NotImplement } func (d *QuarkUCTV) Remove(ctx context.Context, obj model.Obj) error { return errs.NotImplement } func (d *QuarkUCTV) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { return nil, errs.NotImplement } type QuarkUCTVCommon struct { AccessToken string } var _ driver.Driver = (*QuarkUCTV)(nil) ================================================ FILE: drivers/quark_uc_tv/meta.go ================================================ package quark_uc_tv import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { // Usually one of two driver.RootID OrderBy string `json:"order_by" type:"select" options:"file_name,updated_at" default:"updated_at"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"desc"` // define other RefreshToken string `json:"refresh_token" required:"false" default:""` // 必要且影响登录,由签名决定 DeviceID string `json:"device_id" required:"false" default:""` // 登陆所用的数据 无需手动填写 QueryToken string `json:"query_token" required:"false" default:"" help:"don't edit'"` // 视频文件链接获取方式 download(可获取源视频) or streaming(获取转码后的视频) VideoLinkMethod string `json:"link_method" required:"true" type:"select" options:"download,streaming" default:"download"` } type Conf struct { api string clientID string signKey string appVer string channel string codeApi string } func init() { op.RegisterDriver(func() driver.Driver { return &QuarkUCTV{ config: driver.Config{ Name: "QuarkTV", DefaultRoot: "0", NoOverwriteUpload: true, NoUpload: true, }, conf: Conf{ api: "https://open-api-drive.quark.cn", clientID: "d3194e61504e493eb6222857bccfed94", signKey: "kw2dvtd7p4t3pjl2d9ed9yc8yej8kw2d", appVer: "1.8.2.2", channel: "GENERAL", codeApi: "http://api.extscreen.com/quarkdrive", }, } }) op.RegisterDriver(func() driver.Driver { return &QuarkUCTV{ config: driver.Config{ Name: "UCTV", DefaultRoot: "0", NoOverwriteUpload: true, NoUpload: true, }, conf: Conf{ api: "https://open-api-drive.uc.cn", clientID: "5acf882d27b74502b7040b0c65519aa7", signKey: "l3srvtd7p42l0d0x1u8d7yc8ye9kki4d", appVer: "1.7.2.2", channel: "UCTVOFFICIALWEB", codeApi: "http://api.extscreen.com/ucdrive", }, } }) } ================================================ FILE: drivers/quark_uc_tv/types.go ================================================ package quark_uc_tv import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type Resp struct { CommonRsp Errno int `json:"errno"` ErrorInfo string `json:"error_info"` } type CommonRsp struct { Status int `json:"status"` ReqID string `json:"req_id"` } type RefreshTokenAuthResp struct { Code int `json:"code"` Message string `json:"message"` Data struct { Status int `json:"status"` Errno int `json:"errno"` ErrorInfo string `json:"error_info"` ReqID string `json:"req_id"` AccessToken string `json:"access_token"` RefreshToken string `json:"refresh_token"` ExpiresIn int `json:"expires_in"` Scope string `json:"scope"` } `json:"data"` } type Files struct { Fid string `json:"fid"` ParentFid string `json:"parent_fid"` Category int `json:"category"` Filename string `json:"filename"` Size int64 `json:"size"` FileType string `json:"file_type"` SubItems int `json:"sub_items,omitempty"` Isdir int `json:"isdir"` Duration int `json:"duration"` CreatedAt int64 `json:"created_at"` UpdatedAt int64 `json:"updated_at"` IsBackup int `json:"is_backup"` ThumbnailURL string `json:"thumbnail_url,omitempty"` } func (f *Files) GetSize() int64 { return f.Size } func (f *Files) GetName() string { return f.Filename } func (f *Files) ModTime() time.Time { //return time.Unix(f.UpdatedAt, 0) return time.Unix(0, f.UpdatedAt*int64(time.Millisecond)) } func (f *Files) CreateTime() time.Time { //return time.Unix(f.CreatedAt, 0) return time.Unix(0, f.CreatedAt*int64(time.Millisecond)) } func (f *Files) IsDir() bool { return f.Isdir == 1 } func (f *Files) GetHash() utils.HashInfo { return utils.HashInfo{} } func (f *Files) GetID() string { return f.Fid } func (f *Files) GetPath() string { return "" } var _ model.Obj = (*Files)(nil) type FilesData struct { CommonRsp Data struct { TotalCount int64 `json:"total_count"` Files []Files `json:"files"` } `json:"data"` } type StreamingFileLink struct { CommonRsp Data struct { DefaultResolution string `json:"default_resolution"` LastPlayTime int `json:"last_play_time"` VideoInfo []struct { Resolution string `json:"resolution"` Accessable int `json:"accessable"` TransStatus string `json:"trans_status"` Duration int `json:"duration,omitempty"` Size int64 `json:"size,omitempty"` Format string `json:"format,omitempty"` Width int `json:"width,omitempty"` Height int `json:"height,omitempty"` URL string `json:"url,omitempty"` Bitrate float64 `json:"bitrate,omitempty"` DolbyVision struct { Profile int `json:"profile"` Level int `json:"level"` } `json:"dolby_vision,omitempty"` } `json:"video_info"` AudioInfo []interface{} `json:"audio_info"` } `json:"data"` } type DownloadFileLink struct { CommonRsp Data struct { Fid string `json:"fid"` FileName string `json:"file_name"` Size int64 `json:"size"` DownloadURL string `json:"download_url"` } `json:"data"` } ================================================ FILE: drivers/quark_uc_tv/util.go ================================================ package quark_uc_tv import ( "context" "crypto/md5" "crypto/sha256" "encoding/hex" "errors" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) const ( UserAgent = "Mozilla/5.0 (Linux; U; Android 13; zh-cn; M2004J7AC Build/UKQ1.231108.001) AppleWebKit/533.1 (KHTML, like Gecko) Mobile Safari/533.1" DeviceBrand = "Xiaomi" Platform = "tv" DeviceName = "M2004J7AC" DeviceModel = "M2004J7AC" BuildDevice = "M2004J7AC" BuildProduct = "M2004J7AC" DeviceGpu = "Adreno (TM) 550" ActivityRect = "{}" ) func (d *QuarkUCTV) request(ctx context.Context, pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { u := d.conf.api + pathname tm, token, reqID := d.generateReqSign(method, pathname, d.conf.signKey) req := base.RestyClient.R() req.SetContext(ctx) req.SetHeaders(map[string]string{ "Accept": "application/json, text/plain, */*", "User-Agent": UserAgent, "x-pan-tm": tm, "x-pan-token": token, "x-pan-client-id": d.conf.clientID, }) req.SetQueryParams(map[string]string{ "req_id": reqID, "access_token": d.QuarkUCTVCommon.AccessToken, "app_ver": d.conf.appVer, "device_id": d.Addition.DeviceID, "device_brand": DeviceBrand, "platform": Platform, "device_name": DeviceName, "device_model": DeviceModel, "build_device": BuildDevice, "build_product": BuildProduct, "device_gpu": DeviceGpu, "activity_rect": ActivityRect, "channel": d.conf.channel, }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e Resp req.SetError(&e) res, err := req.Execute(method, u) if err != nil { return nil, err } // 判断 是否需要 刷新 access_token errInfoLower := strings.ToLower(strings.TrimSpace(e.ErrorInfo)) maybeTokenInvalid := (e.Status == -1 && (e.Errno == 10001 || e.Errno == 11001)) || (errInfoLower != "" && (strings.Contains(errInfoLower, "access token") || strings.Contains(errInfoLower, "access_token") || strings.Contains(errInfoLower, "token无效") || strings.Contains(errInfoLower, "token 无效"))) if maybeTokenInvalid { // token 过期 / 无效 err = d.getRefreshTokenByTV(ctx, d.Addition.RefreshToken, true) if err != nil { return nil, err } ctx1, cancelFunc := context.WithTimeout(ctx, 10*time.Second) defer cancelFunc() return d.request(ctx1, pathname, method, callback, resp) } if e.Status >= 400 || e.Errno != 0 { return nil, errors.New(e.ErrorInfo) } return res.Body(), nil } func (d *QuarkUCTV) getLoginCode(ctx context.Context) (string, error) { // 获取登录二维码 pathname := "/oauth/authorize" var resp struct { CommonRsp QrData string `json:"qr_data"` QueryToken string `json:"query_token"` } _, err := d.request(ctx, pathname, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "auth_type": "code", "client_id": d.conf.clientID, "scope": "netdisk", "qrcode": "1", "qr_width": "460", "qr_height": "460", }) }, &resp) if err != nil { return "", err } // 保存query_token 用于后续登录 if resp.QueryToken != "" { d.Addition.QueryToken = resp.QueryToken op.MustSaveDriverStorage(d) } return resp.QrData, nil } func (d *QuarkUCTV) getCode(ctx context.Context) (string, error) { // 通过query token获取code pathname := "/oauth/code" var resp struct { CommonRsp Code string `json:"code"` } _, err := d.request(ctx, pathname, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "client_id": d.conf.clientID, "scope": "netdisk", "query_token": d.Addition.QueryToken, }) }, &resp) if err != nil { return "", err } return resp.Code, nil } func (d *QuarkUCTV) getRefreshTokenByTV(ctx context.Context, code string, isRefresh bool) error { pathname := "/token" _, _, reqID := d.generateReqSign(http.MethodPost, pathname, d.conf.signKey) u := d.conf.codeApi + pathname var resp RefreshTokenAuthResp body := map[string]string{ "req_id": reqID, "app_ver": d.conf.appVer, "device_id": d.Addition.DeviceID, "device_brand": DeviceBrand, "platform": Platform, "device_name": DeviceName, "device_model": DeviceModel, "build_device": BuildDevice, "build_product": BuildProduct, "device_gpu": DeviceGpu, "activity_rect": ActivityRect, "channel": d.conf.channel, } if isRefresh { body["refresh_token"] = code } else { body["code"] = code } _, err := base.RestyClient.R(). SetHeader("Content-Type", "application/json"). SetBody(body). SetResult(&resp). SetContext(ctx). Post(u) if err != nil { return err } if resp.Code != 200 { return errors.New(resp.Message) } if resp.Data.RefreshToken != "" { d.Addition.RefreshToken = resp.Data.RefreshToken op.MustSaveDriverStorage(d) d.QuarkUCTVCommon.AccessToken = resp.Data.AccessToken } else { return errors.New("refresh token is empty") } return nil } func (d *QuarkUCTV) isLogin(ctx context.Context) (bool, error) { _, err := d.request(ctx, "/user", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "method": "user_info", }) }, nil) return err == nil, err } func (d *QuarkUCTV) generateReqSign(method string, pathname string, key string) (string, string, string) { //timestamp 13位时间戳 timestamp := strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10) deviceID := d.Addition.DeviceID if deviceID == "" { deviceID = utils.GetMD5EncodeStr(timestamp) d.Addition.DeviceID = deviceID op.MustSaveDriverStorage(d) } // 生成req_id reqID := md5.Sum([]byte(deviceID + timestamp)) reqIDHex := hex.EncodeToString(reqID[:]) // 生成x_pan_token tokenData := method + "&" + pathname + "&" + timestamp + "&" + key xPanToken := sha256.Sum256([]byte(tokenData)) xPanTokenHex := hex.EncodeToString(xPanToken[:]) return timestamp, xPanTokenHex, reqIDHex } func (d *QuarkUCTV) getTranscodingLink(ctx context.Context, file model.Obj) (*model.Link, error) { var fileLink StreamingFileLink _, err := d.request(ctx, "/file", "GET", func(req *resty.Request) { req.SetQueryParams(map[string]string{ "method": "streaming", "group_by": "source", "fid": file.GetID(), "resolution": "low,normal,high,super,2k,4k", "support": "dolby_vision", }) }, &fileLink) if err != nil { return nil, err } for _, info := range fileLink.Data.VideoInfo { if info.URL != "" { return &model.Link{ URL: info.URL, ContentLength: info.Size, Concurrency: 3, PartSize: 10 * utils.MB, }, nil } } return nil, errors.New("no link found") } func (d *QuarkUCTV) getDownloadLink(ctx context.Context, file model.Obj) (*model.Link, error) { var fileLink DownloadFileLink _, err := d.request(ctx, "/file", "GET", func(req *resty.Request) { req.SetQueryParams(map[string]string{ "method": "download", "group_by": "source", "fid": file.GetID(), "resolution": "low,normal,high,super,2k,4k", "support": "dolby_vision", }) }, &fileLink) if err != nil { return nil, err } return &model.Link{ URL: fileLink.Data.DownloadURL, Concurrency: 3, PartSize: 10 * utils.MB, }, nil } ================================================ FILE: drivers/s3/doge.go ================================================ package s3 import ( "crypto/hmac" "crypto/sha1" "encoding/hex" "encoding/json" "io" "net/http" "strings" ) type TmpTokenResponse struct { Code int `json:"code"` Msg string `json:"msg"` Data TmpTokenResponseData `json:"data,omitempty"` } type TmpTokenResponseData struct { Credentials Credentials `json:"Credentials"` ExpiredAt int `json:"ExpiredAt"` } type Credentials struct { AccessKeyId string `json:"accessKeyId,omitempty"` SecretAccessKey string `json:"secretAccessKey,omitempty"` SessionToken string `json:"sessionToken,omitempty"` } func getCredentials(AccessKey, SecretKey string) (rst Credentials, err error) { apiPath := "/auth/tmp_token.json" reqBody, err := json.Marshal(map[string]interface{}{"channel": "OSS_FULL", "scopes": []string{"*"}}) if err != nil { return rst, err } signStr := apiPath + "\n" + string(reqBody) hmacObj := hmac.New(sha1.New, []byte(SecretKey)) hmacObj.Write([]byte(signStr)) sign := hex.EncodeToString(hmacObj.Sum(nil)) Authorization := "TOKEN " + AccessKey + ":" + sign req, err := http.NewRequest(http.MethodPost, "https://api.dogecloud.com"+apiPath, strings.NewReader(string(reqBody))) if err != nil { return rst, err } req.Header.Add("Content-Type", "application/json") req.Header.Add("Authorization", Authorization) client := http.Client{} resp, err := client.Do(req) if err != nil { return rst, err } defer resp.Body.Close() ret, err := io.ReadAll(resp.Body) if err != nil { return rst, err } var tmpTokenResp TmpTokenResponse err = json.Unmarshal(ret, &tmpTokenResp) if err != nil { return rst, err } return tmpTokenResp.Data.Credentials, nil } ================================================ FILE: drivers/s3/driver.go ================================================ package s3 import ( "bytes" "context" "fmt" "net/url" stdpath "path" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" log "github.com/sirupsen/logrus" ) type S3 struct { model.Storage Addition Session *session.Session client *s3.S3 linkClient *s3.S3 directUploadClient *s3.S3 config driver.Config cron *cron.Cron } func (d *S3) Config() driver.Config { return d.config } func (d *S3) GetAddition() driver.Additional { return &d.Addition } func (d *S3) Init(ctx context.Context) error { if d.Region == "" { d.Region = "openlist" } if d.config.Name == "Doge" { // 多吉云每次临时生成的秘钥有效期为 2h，所以这里设置为 118 分钟重新生成一次 d.cron = cron.NewCron(time.Minute * 118) d.cron.Do(func() { err := d.initSession() if err != nil { log.Errorln("Doge init session error:", err) } d.client = d.getClient(ClientTypeNormal) d.linkClient = d.getClient(ClientTypeLink) d.directUploadClient = d.getClient(ClientTypeDirectUpload) }) } err := d.initSession() if err != nil { return err } d.client = d.getClient(ClientTypeNormal) d.linkClient = d.getClient(ClientTypeLink) d.directUploadClient = d.getClient(ClientTypeDirectUpload) return nil } func (d *S3) Drop(ctx context.Context) error { if d.cron != nil { d.cron.Stop() } return nil } func (d *S3) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { if d.ListObjectVersion == "v2" { return d.listV2(dir.GetPath(), args) } return d.listV1(dir.GetPath(), args) } func (d *S3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { path := getKey(file.GetPath(), false) fileName := stdpath.Base(path) input := &s3.GetObjectInput{ Bucket: &d.Bucket, Key: &path, //ResponseContentDisposition: &disposition, } if d.CustomHost == "" { disposition := fmt.Sprintf(`attachment; filename*=UTF-8''%s`, url.PathEscape(fileName)) if d.AddFilenameToDisposition { disposition = utils.GenerateContentDisposition(fileName) } input.ResponseContentDisposition = &disposition } req, _ := d.linkClient.GetObjectRequest(input) if req == nil { return nil, fmt.Errorf("failed to create GetObject request") } var link model.Link var err error if d.CustomHost != "" { if d.EnableCustomHostPresign { link.URL, err = req.Presign(time.Hour * time.Duration(d.SignURLExpire)) } else { err = req.Build() link.URL = req.HTTPRequest.URL.String() } if err != nil { return nil, fmt.Errorf("failed to generate link URL: %w", err) } if d.RemoveBucket { parsedURL, parseErr := url.Parse(link.URL) if parseErr != nil { log.Errorf("Failed to parse URL for bucket removal: %v, URL: %s", parseErr, link.URL) return nil, fmt.Errorf("failed to parse URL for bucket removal: %w", parseErr) } path := parsedURL.Path bucketPrefix := "/" + d.Bucket if strings.HasPrefix(path, bucketPrefix) { path = strings.TrimPrefix(path, bucketPrefix) if path == "" { path = "/" } parsedURL.Path = path link.URL = parsedURL.String() log.Debugf("Removed bucket '%s' from URL path: %s -> %s", d.Bucket, bucketPrefix, path) } else { log.Warnf("URL path does not contain expected bucket prefix '%s': %s", bucketPrefix, path) } } } else { if common.ShouldProxy(d, fileName) { err = req.Sign() link.URL = req.HTTPRequest.URL.String() link.Header = req.HTTPRequest.Header } else { link.URL, err = req.Presign(time.Hour * time.Duration(d.SignURLExpire)) } } if err != nil { return nil, err } return &link, nil } func (d *S3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { return d.Put(ctx, &model.Object{ Path: stdpath.Join(parentDir.GetPath(), dirName), }, &stream.FileStream{ Obj: &model.Object{ Name: getPlaceholderName(d.Placeholder), Modified: time.Now(), }, Reader: bytes.NewReader([]byte{}), Mimetype: "application/octet-stream", }, func(float64) {}) } func (d *S3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { err := d.Copy(ctx, srcObj, dstDir) if err != nil { return err } return d.Remove(ctx, srcObj) } func (d *S3) Rename(ctx context.Context, srcObj model.Obj, newName string) error { err := d.copy(ctx, srcObj.GetPath(), stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), srcObj.IsDir()) if err != nil { return err } return d.Remove(ctx, srcObj) } func (d *S3) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return d.copy(ctx, srcObj.GetPath(), stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.IsDir()) } func (d *S3) Remove(ctx context.Context, obj model.Obj) error { if obj.IsDir() { return d.removeDir(ctx, obj.GetPath()) } return d.removeFile(obj.GetPath()) } func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { uploader := s3manager.NewUploader(d.Session) if s.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { uploader.PartSize = s.GetSize() / (s3manager.MaxUploadParts - 1) } key := getKey(stdpath.Join(dstDir.GetPath(), s.GetName()), false) contentType := s.GetMimetype() log.Debugln("key:", key) input := &s3manager.UploadInput{ Bucket: &d.Bucket, Key: &key, Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, }), ContentType: &contentType, } _, err := uploader.UploadWithContext(ctx, input) return err } func (d *S3) GetDirectUploadTools() []string { if !d.EnableDirectUpload { return nil } return []string{"HttpDirect"} } func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) { if !d.EnableDirectUpload { return nil, errs.NotImplement } path := getKey(stdpath.Join(dstDir.GetPath(), fileName), false) req, _ := d.directUploadClient.PutObjectRequest(&s3.PutObjectInput{ Bucket: &d.Bucket, Key: &path, }) if req == nil { return nil, fmt.Errorf("failed to create PutObject request") } link, err := req.Presign(time.Hour * time.Duration(d.SignURLExpire)) if err != nil { return nil, err } return &model.HttpDirectUploadInfo{ UploadURL: link, Method: "PUT", }, nil } var _ driver.Driver = (*S3)(nil) ================================================ FILE: drivers/s3/meta.go ================================================ package s3 import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Bucket string `json:"bucket" required:"true"` Endpoint string `json:"endpoint" required:"true"` Region string `json:"region"` AccessKeyID string `json:"access_key_id" required:"true"` SecretAccessKey string `json:"secret_access_key" required:"true"` SessionToken string `json:"session_token"` CustomHost string `json:"custom_host"` EnableCustomHostPresign bool `json:"enable_custom_host_presign"` SignURLExpire int `json:"sign_url_expire" type:"number" default:"4"` Placeholder string `json:"placeholder"` ForcePathStyle bool `json:"force_path_style"` ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"` RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."` AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` EnableDirectUpload bool `json:"enable_direct_upload" default:"false"` DirectUploadHost string `json:"direct_upload_host" required:"false"` } func init() { op.RegisterDriver(func() driver.Driver { return &S3{ config: driver.Config{ Name: "S3", DefaultRoot: "/", LocalSort: true, CheckStatus: true, }, } }) op.RegisterDriver(func() driver.Driver { return &S3{ config: driver.Config{ Name: "Doge", DefaultRoot: "/", LocalSort: true, CheckStatus: true, }, } }) } ================================================ FILE: drivers/s3/types.go ================================================ package s3 ================================================ FILE: drivers/s3/util.go ================================================ package s3 import ( "context" "errors" "net/http" "net/url" "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" log "github.com/sirupsen/logrus" ) // do others that not defined in Driver interface func (d *S3) initSession() error { var err error accessKeyID, secretAccessKey, sessionToken := d.AccessKeyID, d.SecretAccessKey, d.SessionToken if d.config.Name == "Doge" { credentialsTmp, err := getCredentials(d.AccessKeyID, d.SecretAccessKey) if err != nil { return err } accessKeyID, secretAccessKey, sessionToken = credentialsTmp.AccessKeyId, credentialsTmp.SecretAccessKey, credentialsTmp.SessionToken } cfg := &aws.Config{ Credentials: credentials.NewStaticCredentials(accessKeyID, secretAccessKey, sessionToken), Region: &d.Region, Endpoint: &d.Endpoint, S3ForcePathStyle: aws.Bool(d.ForcePathStyle), } d.Session, err = session.NewSession(cfg) return err } const ( ClientTypeNormal = iota ClientTypeLink ClientTypeDirectUpload ) func (d *S3) getClient(clientType int) *s3.S3 { client := s3.New(d.Session) if clientType == ClientTypeLink && d.CustomHost != "" { client.Handlers.Build.PushBack(func(r *request.Request) { if r.HTTPRequest.Method != http.MethodGet { return } //判断CustomHost是否以http://或https://开头 split := strings.SplitN(d.CustomHost, "://", 2) if utils.SliceContains([]string{"http", "https"}, split[0]) { r.HTTPRequest.URL.Scheme = split[0] r.HTTPRequest.URL.Host = split[1] } else { r.HTTPRequest.URL.Host = d.CustomHost } }) } if clientType == ClientTypeDirectUpload && d.DirectUploadHost != "" { client.Handlers.Build.PushBack(func(r *request.Request) { if r.HTTPRequest.Method != http.MethodPut { return } split := strings.SplitN(d.DirectUploadHost, "://", 2) if utils.SliceContains([]string{"http", "https"}, split[0]) { r.HTTPRequest.URL.Scheme = split[0] r.HTTPRequest.URL.Host = split[1] } else { r.HTTPRequest.URL.Host = d.DirectUploadHost } }) } return client } func getKey(path string, dir bool) string { path = strings.TrimPrefix(path, "/") if path != "" && dir { path += "/" } return path } var defaultPlaceholderName = ".openlist" func getPlaceholderName(placeholder string) string { if placeholder == "" { return defaultPlaceholderName } return placeholder } func (d *S3) listV1(dirPath string, args model.ListArgs) ([]model.Obj, error) { prefix := getKey(dirPath, true) log.Debugf("list: %s", prefix) files := make([]model.Obj, 0) marker := "" for { input := &s3.ListObjectsInput{ Bucket: &d.Bucket, Marker: &marker, Prefix: &prefix, Delimiter: aws.String("/"), } listObjectsResult, err := d.client.ListObjects(input) if err != nil { return nil, err } for _, object := range listObjectsResult.CommonPrefixes { name := path.Base(strings.Trim(*object.Prefix, "/")) file := model.Object{ Path: path.Join(dirPath, name), Name: name, Modified: d.Modified, IsFolder: true, } files = append(files, &file) } for _, object := range listObjectsResult.Contents { name := path.Base(*object.Key) if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { continue } file := model.Object{ Path: path.Join(dirPath, name), Name: name, Size: *object.Size, Modified: *object.LastModified, } files = append(files, &file) } if listObjectsResult.IsTruncated == nil { return nil, errors.New("IsTruncated nil") } if *listObjectsResult.IsTruncated { marker = *listObjectsResult.NextMarker } else { break } } return files, nil } func (d *S3) listV2(dirPath string, args model.ListArgs) ([]model.Obj, error) { prefix := getKey(dirPath, true) files := make([]model.Obj, 0) var continuationToken, startAfter *string for { input := &s3.ListObjectsV2Input{ Bucket: &d.Bucket, ContinuationToken: continuationToken, Prefix: &prefix, Delimiter: aws.String("/"), StartAfter: startAfter, } listObjectsResult, err := d.client.ListObjectsV2(input) if err != nil { return nil, err } log.Debugf("resp: %+v", listObjectsResult) for _, object := range listObjectsResult.CommonPrefixes { name := path.Base(strings.Trim(*object.Prefix, "/")) file := model.Object{ Path: path.Join(dirPath, name), Name: name, Modified: d.Modified, IsFolder: true, } files = append(files, &file) } for _, object := range listObjectsResult.Contents { if strings.HasSuffix(*object.Key, "/") { continue } name := path.Base(*object.Key) if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { continue } file := model.Object{ Path: path.Join(dirPath, name), Name: name, Size: *object.Size, Modified: *object.LastModified, } files = append(files, &file) } if !aws.BoolValue(listObjectsResult.IsTruncated) { break } if listObjectsResult.NextContinuationToken != nil { continuationToken = listObjectsResult.NextContinuationToken continue } if len(listObjectsResult.Contents) == 0 { break } startAfter = listObjectsResult.Contents[len(listObjectsResult.Contents)-1].Key } return files, nil } func (d *S3) copy(ctx context.Context, src string, dst string, isDir bool) error { if isDir { return d.copyDir(ctx, src, dst) } return d.copyFile(ctx, src, dst) } func (d *S3) copyFile(ctx context.Context, src string, dst string) error { srcKey := getKey(src, false) dstKey := getKey(dst, false) encodedKey := strings.ReplaceAll(url.PathEscape(d.Bucket+"/"+srcKey), "+", "%2B") input := &s3.CopyObjectInput{ Bucket: &d.Bucket, CopySource: aws.String(encodedKey), Key: &dstKey, } _, err := d.client.CopyObject(input) return err } func (d *S3) copyDir(ctx context.Context, src string, dst string) error { objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true}) if err != nil { return err } for _, obj := range objs { cSrc := path.Join(src, obj.GetName()) cDst := path.Join(dst, obj.GetName()) if obj.IsDir() { err = d.copyDir(ctx, cSrc, cDst) } else { err = d.copyFile(ctx, cSrc, cDst) } if err != nil { return err } } return nil } func (d *S3) removeDir(ctx context.Context, src string) error { objs, err := op.List(ctx, d, src, model.ListArgs{}) if err != nil { return err } for _, obj := range objs { cSrc := path.Join(src, obj.GetName()) if obj.IsDir() { err = d.removeDir(ctx, cSrc) } else { err = d.removeFile(cSrc) } if err != nil { return err } } _ = d.removeFile(path.Join(src, getPlaceholderName(d.Placeholder))) _ = d.removeFile(path.Join(src, d.Placeholder)) return nil } func (d *S3) removeFile(src string) error { key := getKey(src, false) input := &s3.DeleteObjectInput{ Bucket: &d.Bucket, Key: &key, } _, err := d.client.DeleteObject(input) return err } ================================================ FILE: drivers/seafile/driver.go ================================================ package seafile import ( "context" "fmt" "net/http" stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) type Seafile struct { model.Storage Addition authorization string root model.Obj } func (d *Seafile) Config() driver.Config { return config } func (d *Seafile) GetAddition() driver.Additional { return &d.Addition } func (d *Seafile) Init(ctx context.Context) error { d.Address = strings.TrimSuffix(d.Address, "/") err := d.getToken() if err != nil { return err } d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) if d.RepoId != "" { library, err := d.getLibraryInfo(d.RepoId) if err != nil { return err } library.path = d.RootFolderPath library.ObjMask = model.Locked d.root = &LibraryInfo{ LibraryItemResp: library, } return nil } if len(d.RootFolderPath) <= 1 { d.root = &model.Object{ Name: "root", Path: d.RootFolderPath, IsFolder: true, Modified: d.Modified, Mask: model.Locked, } return nil } var resp []LibraryItemResp _, err = d.request(http.MethodGet, "/api2/repos/", func(req *resty.Request) { req.SetResult(&resp) }) if err != nil { return err } for _, library := range resp { p, found := strings.CutPrefix(d.RootFolderPath[1:], library.Name) if !found { continue } if p == "" { p = "/" } else if p[0] != '/' { continue } // d.RepoId = library.Id // d.RootFolderPath = p library.path = p library.ObjMask = model.Locked d.root = &LibraryInfo{ LibraryItemResp: library, } return nil } return fmt.Errorf("Library for root folder path %q not found", d.RootFolderPath) } func (d *Seafile) Drop(ctx context.Context) error { d.root = nil return nil } func (d *Seafile) GetRoot(ctx context.Context) (model.Obj, error) { if d.root == nil { return nil, errs.StorageNotInit } return d.root, nil } func (d *Seafile) List(ctx context.Context, dir model.Obj, args model.ListArgs) (result []model.Obj, err error) { path := dir.GetPath() switch o := dir.(type) { default: var resp []LibraryItemResp _, err = d.request(http.MethodGet, "/api2/repos/", func(req *resty.Request) { req.SetResult(&resp) }) return utils.SliceConvert(resp, func(f LibraryItemResp) (model.Obj, error) { f.path = path return &LibraryInfo{ LibraryItemResp: f, }, nil }) case *LibraryInfo: if o.Encrypted { err = d.decryptLibrary(o) if err != nil { return nil, err } } case *RepoItemResp: // do nothing } var resp []RepoItemResp _, err = d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/dir/", dir.GetID()), func(req *resty.Request) { req.SetResult(&resp).SetQueryParams(map[string]string{ "p": path, }) }) if err != nil { return nil, err } return utils.SliceConvert(resp, func(f RepoItemResp) (model.Obj, error) { f.path = stdpath.Join(path, f.Name) f.repoID = dir.GetID() return &f, nil }) } func (d *Seafile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/file/", file.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ "p": file.GetPath(), "reuse": "1", }) }) if err != nil { return nil, err } u := string(res) u = u[1 : len(u)-1] // remove quotes return &model.Link{URL: u}, nil } func (d *Seafile) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/dir/", parentDir.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ "p": stdpath.Join(parentDir.GetPath(), dirName), }).SetFormData(map[string]string{ "operation": "mkdir", }) }) return err } func (d *Seafile) Move(ctx context.Context, srcObj, dstDir model.Obj) error { _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", srcObj.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ "p": srcObj.GetPath(), }).SetFormData(map[string]string{ "operation": "move", "dst_repo": dstDir.GetID(), "dst_dir": dstDir.GetPath(), }) }, true) return err } func (d *Seafile) Rename(ctx context.Context, srcObj model.Obj, newName string) error { _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", srcObj.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ "p": srcObj.GetPath(), }).SetFormData(map[string]string{ "operation": "rename", "newname": newName, }) }, true) return err } func (d *Seafile) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", srcObj.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ "p": srcObj.GetPath(), }).SetFormData(map[string]string{ "operation": "copy", "dst_repo": dstDir.GetID(), "dst_dir": dstDir.GetPath(), }) }) return err } func (d *Seafile) Remove(ctx context.Context, obj model.Obj) error { _, err := d.request(http.MethodDelete, fmt.Sprintf("/api2/repos/%s/file/", obj.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ "p": obj.GetPath(), }) }) return err } func (d *Seafile) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/upload-link/", dstDir.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ "p": dstDir.GetPath(), }) }) if err != nil { return err } u := string(res) u = u[1 : len(u)-1] // remove quotes _, err = d.request(http.MethodPost, u, func(req *resty.Request) { r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, }) req.SetFileReader("file", s.GetName(), r). SetFormData(map[string]string{ "parent_dir": dstDir.GetPath(), "replace": "1", }). SetContext(ctx) }) return err } var _ driver.Driver = (*Seafile)(nil) ================================================ FILE: drivers/seafile/meta.go ================================================ package seafile import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Address string `json:"address" required:"true"` UserName string `json:"username" required:"false"` Password string `json:"password" required:"false"` Token string `json:"token" required:"false"` RepoId string `json:"repoId" required:"false"` RepoPwd string `json:"repoPwd" required:"false"` } var config = driver.Config{ Name: "Seafile", DefaultRoot: "/", } func init() { op.RegisterDriver(func() driver.Driver { return &Seafile{} }) } ================================================ FILE: drivers/seafile/types.go ================================================ package seafile import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type AuthTokenResp struct { Token string `json:"token"` } type RepoItemResp struct { Id string `json:"id"` Type string `json:"type"` // repo, dir, file Name string `json:"name"` Size int64 `json:"size"` Modified int64 `json:"mtime"` Permission string `json:"permission"` path string model.ObjMask repoID string } func (l *RepoItemResp) IsDir() bool { return l.Type == "dir" } func (l *RepoItemResp) GetPath() string { return l.path } func (l *RepoItemResp) GetName() string { return l.Name } func (l *RepoItemResp) ModTime() time.Time { return time.Unix(l.Modified, 0) } func (l *RepoItemResp) CreateTime() time.Time { return l.ModTime() } func (l *RepoItemResp) GetSize() int64 { return l.Size } func (l *RepoItemResp) GetID() string { if l.repoID != "" { return l.repoID } return l.Id } func (l *RepoItemResp) GetHash() utils.HashInfo { return utils.HashInfo{} } var _ model.Obj = (*RepoItemResp)(nil) type LibraryItemResp struct { RepoItemResp OwnerContactEmail string `json:"owner_contact_email"` OwnerName string `json:"owner_name"` Owner string `json:"owner"` ModifierEmail string `json:"modifier_email"` ModifierContactEmail string `json:"modifier_contact_email"` ModifierName string `json:"modifier_name"` Virtual bool `json:"virtual"` MtimeRelative string `json:"mtime_relative"` Encrypted bool `json:"encrypted"` Version int `json:"version"` HeadCommitId string `json:"head_commit_id"` Root string `json:"root"` Salt string `json:"salt"` SizeFormatted string `json:"size_formatted"` } type LibraryInfo struct { LibraryItemResp decryptedTime time.Time decryptedSuccess bool } func (l *LibraryInfo) IsDir() bool { return true } ================================================ FILE: drivers/seafile/util.go ================================================ package seafile import ( "errors" "fmt" "net/http" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/go-resty/resty/v2" ) func (d *Seafile) getToken() error { if d.Token != "" { d.authorization = fmt.Sprintf("Token %s", d.Token) return nil } var authResp AuthTokenResp res, err := base.RestyClient.R(). SetResult(&authResp). SetFormData(map[string]string{ "username": d.UserName, "password": d.Password, }). Post(d.Address + "/api2/auth-token/") if err != nil { return err } if res.StatusCode() >= 400 { return fmt.Errorf("get token failed: %s", res.String()) } d.authorization = fmt.Sprintf("Token %s", authResp.Token) return nil } func (d *Seafile) request(method string, pathname string, callback base.ReqCallback, noRedirect ...bool) ([]byte, error) { full := pathname if !strings.HasPrefix(pathname, "http") { full = d.Address + pathname } req := base.RestyClient.R() if len(noRedirect) > 0 && noRedirect[0] { req = base.NoRedirectClient.R() } req.SetHeader("Authorization", d.authorization) callback(req) var ( res *resty.Response err error ) for i := 0; i < 2; i++ { res, err = req.Execute(method, full) if err != nil { return nil, err } if res.StatusCode() != 401 { // Unauthorized break } err = d.getToken() if err != nil { return nil, err } } if res.StatusCode() >= 400 { return nil, fmt.Errorf("request failed: %s", res.String()) } return res.Body(), nil } func (d *Seafile) getLibraryInfo(repoId string) (LibraryItemResp, error) { var oneResp LibraryItemResp _, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/", repoId), func(req *resty.Request) { req.SetResult(&oneResp) }) return oneResp, err } var repoPwdNotConfigured = errors.New("library password not configured") var repoPwdIncorrect = errors.New("library password is incorrect") func (d *Seafile) decryptLibrary(repo *LibraryInfo) (err error) { if !repo.Encrypted { return nil } if d.RepoPwd == "" { return repoPwdNotConfigured } now := time.Now() decryptedTime := repo.decryptedTime if repo.decryptedSuccess { if now.Sub(decryptedTime).Minutes() <= 30 { return nil } } else { if now.Sub(decryptedTime).Seconds() <= 10 { return repoPwdIncorrect } } var resp string _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/", repo.Id), func(req *resty.Request) { req.SetResult(&resp).SetFormData(map[string]string{ "password": d.RepoPwd, }) }) repo.decryptedTime = time.Now() if err != nil || !strings.Contains(resp, "success") { repo.decryptedSuccess = false return err } repo.decryptedSuccess = true return nil } ================================================ FILE: drivers/sftp/driver.go ================================================ package sftp import ( "context" "os" "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/sftp" log "github.com/sirupsen/logrus" ) type SFTP struct { model.Storage Addition client *sftp.Client clientConnectionError error } func (d *SFTP) Config() driver.Config { return config } func (d *SFTP) GetAddition() driver.Additional { return &d.Addition } func (d *SFTP) Init(ctx context.Context) error { return d._initClient() } func (d *SFTP) Drop(ctx context.Context) error { if d.client != nil { _ = d.client.Close() } return nil } func (d *SFTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { if err := d.clientReconnectOnConnectionError(); err != nil { return nil, err } log.Debugf("[sftp] list dir: %s", dir.GetPath()) files, err := d.client.ReadDir(dir.GetPath()) if err != nil { return nil, err } objs, err := utils.SliceConvert(files, func(src os.FileInfo) (model.Obj, error) { return d.fileToObj(src, dir.GetPath()) }) return objs, err } func (d *SFTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if err := d.clientReconnectOnConnectionError(); err != nil { return nil, err } remoteFile, err := d.client.Open(file.GetPath()) if err != nil { return nil, err } mFile := &stream.RateLimitFile{ File: remoteFile, Limiter: stream.ServerDownloadLimit, Ctx: ctx, } return &model.Link{ RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), mFile), SyncClosers: utils.NewSyncClosers(remoteFile), RequireReference: true, }, nil } func (d *SFTP) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { if err := d.clientReconnectOnConnectionError(); err != nil { return err } return d.client.MkdirAll(path.Join(parentDir.GetPath(), dirName)) } func (d *SFTP) Move(ctx context.Context, srcObj, dstDir model.Obj) error { if err := d.clientReconnectOnConnectionError(); err != nil { return err } return d.client.Rename(srcObj.GetPath(), path.Join(dstDir.GetPath(), srcObj.GetName())) } func (d *SFTP) Rename(ctx context.Context, srcObj model.Obj, newName string) error { if err := d.clientReconnectOnConnectionError(); err != nil { return err } return d.client.Rename(srcObj.GetPath(), path.Join(path.Dir(srcObj.GetPath()), newName)) } func (d *SFTP) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return errs.NotSupport } func (d *SFTP) Remove(ctx context.Context, obj model.Obj) error { if err := d.clientReconnectOnConnectionError(); err != nil { return err } return d.remove(obj.GetPath()) } func (d *SFTP) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { if err := d.clientReconnectOnConnectionError(); err != nil { return err } dstFile, err := d.client.Create(path.Join(dstDir.GetPath(), stream.GetName())) if err != nil { return err } defer func() { _ = dstFile.Close() }() err = utils.CopyWithCtx(ctx, dstFile, driver.NewLimitedUploadStream(ctx, stream), stream.GetSize(), up) return err } func (d *SFTP) GetDetails(ctx context.Context) (*model.StorageDetails, error) { stat, err := d.client.StatVFS(d.RootFolderPath) if err != nil { if strings.Contains(err.Error(), "unimplemented") { return nil, errs.NotImplement } return nil, err } total := int64(stat.Blocks * stat.Bsize) free := int64(stat.Bfree * stat.Bsize) return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: total - free, }, }, nil } var _ driver.Driver = (*SFTP)(nil) ================================================ FILE: drivers/sftp/meta.go ================================================ package sftp import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { Address string `json:"address" required:"true"` Username string `json:"username" required:"true"` PrivateKey string `json:"private_key" type:"text"` Password string `json:"password"` Passphrase string `json:"passphrase"` driver.RootPath IgnoreSymlinkError bool `json:"ignore_symlink_error" default:"false" info:"Ignore symlink error"` } var config = driver.Config{ Name: "SFTP", LocalSort: true, OnlyProxy: true, DefaultRoot: "/", CheckStatus: true, NoLinkURL: true, } func init() { op.RegisterDriver(func() driver.Driver { return &SFTP{} }) } ================================================ FILE: drivers/sftp/types.go ================================================ package sftp import ( "os" stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/model" log "github.com/sirupsen/logrus" ) func (d *SFTP) fileToObj(f os.FileInfo, dir string) (model.Obj, error) { symlink := f.Mode()&os.ModeSymlink != 0 path := stdpath.Join(dir, f.Name()) if !symlink { return &model.Object{ Path: path, Name: f.Name(), Size: f.Size(), Modified: f.ModTime(), IsFolder: f.IsDir(), }, nil } // set target path target, err := d.client.ReadLink(path) if err != nil { return nil, err } if !strings.HasPrefix(target, "/") { target = stdpath.Join(dir, target) } _f, err := d.client.Stat(target) if err != nil { if d.IgnoreSymlinkError { return &model.Object{ Path: path, Name: f.Name(), Size: f.Size(), Modified: f.ModTime(), IsFolder: f.IsDir(), }, nil } return nil, err } // set basic info obj := &model.Object{ Name: f.Name(), Size: _f.Size(), Modified: _f.ModTime(), IsFolder: _f.IsDir(), Path: target, } log.Debugf("[sftp] obj: %+v, is symlink: %v", obj, symlink) return obj, nil } ================================================ FILE: drivers/sftp/util.go ================================================ package sftp import ( "fmt" "path" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/pkg/sftp" log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" ) // do others that not defined in Driver interface func (d *SFTP) initClient() error { _, err, _ := singleflight.AnyGroup.Do(fmt.Sprintf("SFTP.initClient:%p", d), func() (any, error) { return nil, d._initClient() }) return err } func (d *SFTP) _initClient() error { var auth ssh.AuthMethod if len(d.PrivateKey) > 0 { var err error var signer ssh.Signer if len(d.Passphrase) > 0 { signer, err = ssh.ParsePrivateKeyWithPassphrase([]byte(d.PrivateKey), []byte(d.Passphrase)) } else { signer, err = ssh.ParsePrivateKey([]byte(d.PrivateKey)) } if err != nil { return err } auth = ssh.PublicKeys(signer) } else { auth = ssh.Password(d.Password) } config := &ssh.ClientConfig{ User: d.Username, Auth: []ssh.AuthMethod{auth}, HostKeyCallback: ssh.InsecureIgnoreHostKey(), } conn, err := ssh.Dial("tcp", d.Address, config) if err != nil { return err } d.client, err = sftp.NewClient(conn) if err == nil { d.clientConnectionError = nil go func(d *SFTP) { d.clientConnectionError = d.client.Wait() }(d) } return err } func (d *SFTP) clientReconnectOnConnectionError() error { err := d.clientConnectionError if err == nil { return nil } log.Debugf("[sftp] discarding closed sftp connection: %v", err) if d.client != nil { _ = d.client.Close() } err = d.initClient() return err } func (d *SFTP) remove(remotePath string) error { f, err := d.client.Stat(remotePath) if err != nil { return nil } if f.IsDir() { return d.removeDirectory(remotePath) } else { return d.removeFile(remotePath) } } func (d *SFTP) removeDirectory(remotePath string) error { remoteFiles, err := d.client.ReadDir(remotePath) if err != nil { return err } for _, backupDir := range remoteFiles { remoteFilePath := path.Join(remotePath, backupDir.Name()) if backupDir.IsDir() { err := d.removeDirectory(remoteFilePath) if err != nil { return err } } else { err := d.removeFile(remoteFilePath) if err != nil { return err } } } return d.client.RemoveDirectory(remotePath) } func (d *SFTP) removeFile(remotePath string) error { return d.client.Remove(path.Join(remotePath)) } ================================================ FILE: drivers/smb/driver.go ================================================ package smb import ( "context" "errors" "path" "path/filepath" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/cloudsoda/go-smb2" ) type SMB struct { lastConnTime int64 model.Storage Addition fs *smb2.Share } func (d *SMB) Config() driver.Config { return config } func (d *SMB) GetAddition() driver.Additional { return &d.Addition } func (d *SMB) Init(ctx context.Context) error { if !strings.Contains(d.Addition.Address, ":") { d.Addition.Address = d.Addition.Address + ":445" } return d._initFS(ctx) } func (d *SMB) Drop(ctx context.Context) error { if d.fs != nil { _ = d.fs.Umount() } return nil } func (d *SMB) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { if err := d.checkConn(ctx); err != nil { return nil, err } fullPath := dir.GetPath() rawFiles, err := d.fs.ReadDir(fullPath) if err != nil { d.cleanLastConnTime() return nil, err } d.updateLastConnTime() files := make([]model.Obj, 0, len(rawFiles)) for _, f := range rawFiles { file := model.Object{ Path: path.Join(fullPath, f.Name()), Name: f.Name(), Modified: f.ModTime(), Size: f.Size(), IsFolder: f.IsDir(), Ctime: f.(*smb2.FileStat).CreationTime, } files = append(files, &file) } return files, nil } func (d *SMB) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if err := d.checkConn(ctx); err != nil { return nil, err } fullPath := file.GetPath() remoteFile, err := d.fs.Open(fullPath) if err != nil { d.cleanLastConnTime() return nil, err } d.updateLastConnTime() mFile := &stream.RateLimitFile{ File: remoteFile, Limiter: stream.ServerDownloadLimit, Ctx: ctx, } return &model.Link{ RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), mFile), SyncClosers: utils.NewSyncClosers(remoteFile), RequireReference: true, }, nil } func (d *SMB) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { if err := d.checkConn(ctx); err != nil { return err } fullPath := filepath.Join(parentDir.GetPath(), dirName) err := d.fs.MkdirAll(fullPath, 0700) if err != nil { d.cleanLastConnTime() return err } d.updateLastConnTime() return nil } func (d *SMB) Move(ctx context.Context, srcObj, dstDir model.Obj) error { if err := d.checkConn(ctx); err != nil { return err } srcPath := srcObj.GetPath() dstPath := filepath.Join(dstDir.GetPath(), srcObj.GetName()) err := d.fs.Rename(srcPath, dstPath) if err != nil { d.cleanLastConnTime() return err } d.updateLastConnTime() return nil } func (d *SMB) Rename(ctx context.Context, srcObj model.Obj, newName string) error { if err := d.checkConn(ctx); err != nil { return err } srcPath := srcObj.GetPath() dstPath := filepath.Join(filepath.Dir(srcPath), newName) err := d.fs.Rename(srcPath, dstPath) if err != nil { d.cleanLastConnTime() return err } d.updateLastConnTime() return nil } func (d *SMB) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { if err := d.checkConn(ctx); err != nil { return err } srcPath := srcObj.GetPath() dstPath := filepath.Join(dstDir.GetPath(), srcObj.GetName()) var err error if srcObj.IsDir() { err = d.CopyDir(srcPath, dstPath) } else { err = d.CopyFile(srcPath, dstPath) } if err != nil { d.cleanLastConnTime() return err } d.updateLastConnTime() return nil } func (d *SMB) Remove(ctx context.Context, obj model.Obj) error { if err := d.checkConn(ctx); err != nil { return err } var err error fullPath := obj.GetPath() if obj.IsDir() { err = d.fs.RemoveAll(fullPath) } else { err = d.fs.Remove(fullPath) } if err != nil { d.cleanLastConnTime() return err } d.updateLastConnTime() return nil } func (d *SMB) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { if err := d.checkConn(ctx); err != nil { return err } fullPath := filepath.Join(dstDir.GetPath(), stream.GetName()) out, err := d.fs.Create(fullPath) if err != nil { d.cleanLastConnTime() return err } d.updateLastConnTime() defer func() { _ = out.Close() if errors.Is(err, context.Canceled) { _ = d.fs.Remove(fullPath) } }() err = utils.CopyWithCtx(ctx, out, driver.NewLimitedUploadStream(ctx, stream), stream.GetSize(), up) if err != nil { return err } return nil } func (d *SMB) GetDetails(ctx context.Context) (*model.StorageDetails, error) { if err := d.checkConn(ctx); err != nil { return nil, err } stat, err := d.fs.Statfs(d.RootFolderPath) if err != nil { return nil, err } total := int64(stat.BlockSize() * stat.TotalBlockCount()) free := int64(stat.BlockSize() * stat.AvailableBlockCount()) return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: total - free, }, }, nil } //func (d *SMB) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*SMB)(nil) ================================================ FILE: drivers/smb/meta.go ================================================ package smb import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Address string `json:"address" required:"true"` Username string `json:"username" required:"true"` Password string `json:"password"` ShareName string `json:"share_name" required:"true"` } var config = driver.Config{ Name: "SMB", LocalSort: true, OnlyProxy: true, DefaultRoot: ".", NoCache: true, NoLinkURL: true, } func init() { op.RegisterDriver(func() driver.Driver { return &SMB{} }) } ================================================ FILE: drivers/smb/types.go ================================================ package smb ================================================ FILE: drivers/smb/util.go ================================================ package smb import ( "context" "fmt" "io/fs" "os" "path/filepath" "sync/atomic" "time" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/cloudsoda/go-smb2" ) func (d *SMB) updateLastConnTime() { atomic.StoreInt64(&d.lastConnTime, time.Now().Unix()) } func (d *SMB) cleanLastConnTime() { atomic.StoreInt64(&d.lastConnTime, 0) } func (d *SMB) getLastConnTime() time.Time { return time.Unix(atomic.LoadInt64(&d.lastConnTime), 0) } func (d *SMB) initFS(ctx context.Context) error { _, err, _ := singleflight.AnyGroup.Do(fmt.Sprintf("SMB.initFS:%p", d), func() (any, error) { return nil, d._initFS(ctx) }) return err } func (d *SMB) _initFS(ctx context.Context) error { dialer := &smb2.Dialer{ Initiator: &smb2.NTLMInitiator{ User: d.Username, Password: d.Password, }, } s, err := dialer.Dial(ctx, d.Address) if err != nil { return err } d.fs, err = s.Mount(d.ShareName) if err != nil { return err } d.updateLastConnTime() return err } func (d *SMB) checkConn(ctx context.Context) error { if time.Since(d.getLastConnTime()) < 5*time.Minute { return nil } if d.fs != nil { _ = d.fs.Umount() } return d.initFS(ctx) } // CopyFile File copies a single file from src to dst func (d *SMB) CopyFile(src, dst string) error { var err error var srcfd *smb2.File var dstfd *smb2.File var srcinfo fs.FileInfo if srcfd, err = d.fs.Open(src); err != nil { return err } defer srcfd.Close() if dstfd, err = d.CreateNestedFile(dst); err != nil { return err } defer dstfd.Close() if _, err = utils.CopyWithBuffer(dstfd, srcfd); err != nil { return err } if srcinfo, err = d.fs.Stat(src); err != nil { return err } return d.fs.Chmod(dst, srcinfo.Mode()) } // CopyDir Dir copies a whole directory recursively func (d *SMB) CopyDir(src string, dst string) error { var err error var fds []fs.FileInfo var srcinfo fs.FileInfo if srcinfo, err = d.fs.Stat(src); err != nil { return err } if err = d.fs.MkdirAll(dst, srcinfo.Mode()); err != nil { return err } if fds, err = d.fs.ReadDir(src); err != nil { return err } for _, fd := range fds { srcfp := filepath.Join(src, fd.Name()) dstfp := filepath.Join(dst, fd.Name()) if fd.IsDir() { if err = d.CopyDir(srcfp, dstfp); err != nil { return err } } else { if err = d.CopyFile(srcfp, dstfp); err != nil { return err } } } return nil } // Exists determine whether the file exists func (d *SMB) Exists(name string) bool { if _, err := d.fs.Stat(name); err != nil { if os.IsNotExist(err) { return false } } return true } // CreateNestedFile create nested file func (d *SMB) CreateNestedFile(path string) (*smb2.File, error) { basePath := filepath.Dir(path) if !d.Exists(basePath) { err := d.fs.MkdirAll(basePath, 0700) if err != nil { return nil, err } } return d.fs.Create(path) } ================================================ FILE: drivers/strm/driver.go ================================================ package strm import ( "context" "errors" "fmt" stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" log "github.com/sirupsen/logrus" ) type Strm struct { model.Storage Addition pathMap map[string][]string autoFlatten bool oneKey string supportSuffix map[string]struct{} downloadSuffix map[string]struct{} } func (d *Strm) Config() driver.Config { return config } func (d *Strm) GetAddition() driver.Additional { return &d.Addition } func (d *Strm) Init(ctx context.Context) error { if d.Paths == "" { return errors.New("paths is required") } if d.SaveStrmToLocal && len(d.SaveStrmLocalPath) <= 0 { return errors.New("SaveStrmLocalPath is required") } d.pathMap = make(map[string][]string) for _, path := range strings.Split(d.Paths, "\n") { path = strings.TrimSpace(path) if path == "" { continue } k, v := getPair(path) d.pathMap[k] = append(d.pathMap[k], v) if d.SaveStrmToLocal { err := InsertStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d) if err != nil { log.Errorf("insert strmTrie error: %v", err) continue } } } if len(d.pathMap) == 1 { for k := range d.pathMap { d.oneKey = k } d.autoFlatten = true } else { d.oneKey = "" d.autoFlatten = false } var supportTypes []string if d.FilterFileTypes == "" { d.FilterFileTypes = "mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" } supportTypes = strings.Split(d.FilterFileTypes, ",") d.supportSuffix = map[string]struct{}{} for _, ext := range supportTypes { ext = strings.ToLower(strings.TrimSpace(ext)) if ext != "" { d.supportSuffix[ext] = struct{}{} } } var downloadTypes []string if d.DownloadFileTypes == "" { d.DownloadFileTypes = "ass,srt,vtt,sub,strm" } downloadTypes = strings.Split(d.DownloadFileTypes, ",") d.downloadSuffix = map[string]struct{}{} for _, ext := range downloadTypes { ext = strings.ToLower(strings.TrimSpace(ext)) if ext != "" { d.downloadSuffix[ext] = struct{}{} } } if d.Version != 5 { types := strings.Split("mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac", ",") for _, ext := range types { if _, ok := d.supportSuffix[ext]; !ok { d.supportSuffix[ext] = struct{}{} supportTypes = append(supportTypes, ext) } } d.FilterFileTypes = strings.Join(supportTypes, ",") types = strings.Split("ass,srt,vtt,sub,strm", ",") for _, ext := range types { if _, ok := d.downloadSuffix[ext]; !ok { d.downloadSuffix[ext] = struct{}{} downloadTypes = append(downloadTypes, ext) } } d.DownloadFileTypes = strings.Join(downloadTypes, ",") d.PathPrefix = "/d" d.Version = 5 } if len(d.SaveLocalMode) == 0 { d.SaveLocalMode = SaveLocalInsertMode } return nil } func (d *Strm) Drop(ctx context.Context) error { d.pathMap = nil d.downloadSuffix = nil d.supportSuffix = nil for _, path := range strings.Split(d.Paths, "\n") { RemoveStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d) } return nil } func (Addition) GetRootPath() string { return "/" } func (d *Strm) Get(ctx context.Context, path string) (model.Obj, error) { root, sub := d.getRootAndPath(path) dsts, ok := d.pathMap[root] if !ok { return nil, errs.ObjectNotFound } for _, dst := range dsts { reqPath := stdpath.Join(dst, sub) obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) if err != nil { continue } // fs.Get 没报错，说明不是strm驱动映射的路径，需要直接返回 size := int64(0) if !obj.IsDir() { size = obj.GetSize() path = reqPath //把路径设置为真实的，供Link直接读取 } return &model.Object{ Path: path, Name: obj.GetName(), Size: size, Modified: obj.ModTime(), IsFolder: obj.IsDir(), HashInfo: obj.GetHash(), }, nil } if strings.HasSuffix(path, ".strm") { // 上面fs.Get都没找到且后缀为.strm // 返回errs.NotSupport使得op.Get尝试从op.List中查找 return nil, errs.NotSupport } return nil, errs.ObjectNotFound } func (d *Strm) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { path := dir.GetPath() if utils.PathEqual(path, "/") && !d.autoFlatten { return d.listRoot(), nil } root, sub := d.getRootAndPath(path) dsts, ok := d.pathMap[root] if !ok { return nil, errs.ObjectNotFound } var objs []model.Obj fsArgs := &fs.ListArgs{NoLog: true, Refresh: args.Refresh} for _, dst := range dsts { tmp, err := d.list(ctx, dst, sub, fsArgs) if err == nil { objs = append(objs, tmp...) } } return objs, nil } func (d *Strm) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if file.GetID() == "strm" { link := d.getLink(ctx, file.GetPath()) return &model.Link{ RangeReader: stream.GetRangeReaderFromMFile(int64(len(link)), strings.NewReader(link)), }, nil } // ftp,s3 if common.GetApiUrl(ctx) == "" { args.Redirect = false } reqPath := file.GetPath() link, _, err := d.link(ctx, reqPath, args) if err != nil { return nil, err } if link == nil { return &model.Link{ URL: fmt.Sprintf("%s/p%s?sign=%s", common.GetApiUrl(ctx), utils.EncodePath(reqPath, true), sign.Sign(reqPath)), }, nil } resultLink := *link resultLink.SyncClosers = utils.NewSyncClosers(link) return &resultLink, nil } var _ driver.Driver = (*Strm)(nil) ================================================ FILE: drivers/strm/hook.go ================================================ package strm import ( "bytes" "context" "crypto/sha256" "errors" "io" "os" stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" "github.com/tchap/go-patricia/v2/patricia" ) var strmTrie = patricia.NewTrie() func UpdateLocalStrm(ctx context.Context, path string, objs []model.Obj) { path = utils.FixAndCleanPath(path) updateLocal := func(driver *Strm, basePath string, objs []model.Obj) { relParent := strings.TrimPrefix(basePath, utils.GetActualMountPath(driver.MountPath)) localParentPath := stdpath.Join(driver.SaveStrmLocalPath, relParent) for _, obj := range objs { localPath := stdpath.Join(localParentPath, obj.GetName()) generateStrm(ctx, driver, obj, localPath) } deleteExtraFiles(driver, localParentPath, objs) } _ = strmTrie.VisitPrefixes(patricia.Prefix(path), func(needPathPrefix patricia.Prefix, item patricia.Item) error { strmDrivers := item.([]*Strm) needPath := string(needPathPrefix) restPath := strings.TrimPrefix(path, needPath) if len(restPath) > 0 && restPath[0] != '/' { return nil } for _, strmDriver := range strmDrivers { strmObjs := strmDriver.convert2strmObjs(ctx, path, objs) updateLocal(strmDriver, stdpath.Join(stdpath.Base(needPath), restPath), strmObjs) } return nil }) } func InsertStrm(dstPath string, d *Strm) error { prefix := patricia.Prefix(strings.TrimRight(dstPath, "/")) existing := strmTrie.Get(prefix) if existing == nil { if !strmTrie.Insert(prefix, []*Strm{d}) { return errors.New("failed to insert strm") } return nil } if lst, ok := existing.([]*Strm); ok { strmTrie.Set(prefix, append(lst, d)) } else { return errors.New("invalid trie item type") } return nil } func RemoveStrm(dstPath string, d *Strm) { prefix := patricia.Prefix(strings.TrimRight(dstPath, "/")) existing := strmTrie.Get(prefix) if existing == nil { return } lst, ok := existing.([]*Strm) if !ok { return } if len(lst) == 1 && lst[0] == d { strmTrie.Delete(prefix) return } for i, di := range lst { if di == d { newList := append(lst[:i], lst[i+1:]...) strmTrie.Set(prefix, newList) return } } } func generateStrm(ctx context.Context, driver *Strm, obj model.Obj, localPath string) { if !obj.IsDir() { if utils.Exists(localPath) && driver.SaveLocalMode == SaveLocalInsertMode { return } link, err := driver.Link(ctx, obj, model.LinkArgs{}) if err != nil { log.Warnf("failed to generate strm of obj %s: failed to link: %v", localPath, err) return } defer link.Close() size := link.ContentLength if size <= 0 { size = obj.GetSize() } rrf, err := stream.GetRangeReaderFromLink(size, link) if err != nil { log.Warnf("failed to generate strm of obj %s: failed to get range reader: %v", localPath, err) return } rc, err := rrf.RangeRead(ctx, http_range.Range{Length: -1}) if err != nil { log.Warnf("failed to generate strm of obj %s: failed to read range: %v", localPath, err) return } defer rc.Close() same, err := isSameContent(localPath, size, rc) if err != nil { log.Warnf("failed to compare content of obj %s: %v", localPath, err) return } if same { return } rc, err = rrf.RangeRead(ctx, http_range.Range{Length: -1}) if err != nil { log.Warnf("failed to generate strm of obj %s: failed to reread range: %v", localPath, err) return } defer rc.Close() file, err := utils.CreateNestedFile(localPath) if err != nil { log.Warnf("failed to generate strm of obj %s: failed to create local file: %v", localPath, err) return } defer file.Close() if _, err := utils.CopyWithBuffer(file, rc); err != nil { log.Warnf("failed to generate strm of obj %s: copy failed: %v", localPath, err) } } } func isSameContent(localPath string, size int64, rc io.Reader) (bool, error) { info, err := os.Stat(localPath) if err != nil { if os.IsNotExist(err) { return false, nil } return false, err } if info.Size() != size { return false, nil } localFile, err := os.Open(localPath) if err != nil { return false, err } defer localFile.Close() h1 := sha256.New() h2 := sha256.New() if _, err := io.Copy(h1, localFile); err != nil { return false, err } if _, err := io.Copy(h2, rc); err != nil { return false, err } return bytes.Equal(h1.Sum(nil), h2.Sum(nil)), nil } func deleteExtraFiles(driver *Strm, localPath string, objs []model.Obj) { if driver.SaveLocalMode != SaveLocalSyncMode { return } localFiles, localDirs, err := getLocalDirsAndFiles(localPath) if err != nil { log.Errorf("Failed to read local files from %s: %v", localPath, err) return } fileSet := make(map[string]struct{}) dirSet := make(map[string]struct{}) for _, obj := range objs { objPath := stdpath.Join(localPath, obj.GetName()) if obj.IsDir() { dirSet[objPath] = struct{}{} } else { fileSet[objPath] = struct{}{} } } for _, localFile := range localFiles { if _, exists := fileSet[localFile]; !exists { err := os.Remove(localFile) if err != nil { log.Errorf("Failed to delete file: %s, error: %v\n", localFile, err) } else { log.Infof("Deleted file %s", localFile) } } } for _, localDir := range localDirs { if _, exists := dirSet[localDir]; !exists { err := os.RemoveAll(localDir) if err != nil { log.Errorf("Failed to delete directory: %s, error: %v\n", localDir, err) } else { log.Infof("Deleted directory %s", localDir) } } } } func getLocalDirsAndFiles(localPath string) ([]string, []string, error) { var files, dirs []string entries, err := os.ReadDir(localPath) if err != nil { return nil, nil, err } for _, entry := range entries { fullPath := stdpath.Join(localPath, entry.Name()) if entry.IsDir() { dirs = append(dirs, fullPath) } else { files = append(files, fullPath) } } return files, dirs, nil } func init() { op.RegisterObjsUpdateHook(UpdateLocalStrm) } ================================================ FILE: drivers/strm/meta.go ================================================ package strm import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) const ( SaveLocalInsertMode = "insert" SaveLocalUpdateMode = "update" SaveLocalSyncMode = "sync" ) type Addition struct { Paths string `json:"paths" required:"true" type:"text"` SiteUrl string `json:"siteUrl" type:"text" required:"false" help:"The prefix URL of the strm file"` PathPrefix string `json:"PathPrefix" type:"text" required:"false" default:"/d" help:"Path prefix"` DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass,srt,vtt,sub,strm" required:"false" help:"Files need to download with strm (usally subtitles)"` FilterFileTypes string `json:"filterFileTypes" type:"text" default:"mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" required:"false" help:"Supports suffix name of strm file"` EncodePath bool `json:"encodePath" default:"true" required:"true" help:"encode the path in the strm file"` WithoutUrl bool `json:"withoutUrl" default:"false" help:"strm file content without URL prefix"` WithSign bool `json:"withSign" default:"false"` SaveStrmToLocal bool `json:"SaveStrmToLocal" default:"false" help:"save strm file locally"` SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"save strm file local path"` SaveLocalMode string `json:"SaveLocalMode" type:"select" help:"save strm file locally mode" options:"insert,update,sync" default:"insert"` Version int } var config = driver.Config{ Name: "Strm", LocalSort: true, OnlyProxy: true, NoCache: true, NoUpload: true, DefaultRoot: "/", NoLinkURL: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Strm{ Addition: Addition{ EncodePath: true, }, } }) } ================================================ FILE: drivers/strm/util.go ================================================ package strm import ( "context" "fmt" stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" ) func (d *Strm) listRoot() []model.Obj { var objs []model.Obj for k := range d.pathMap { obj := model.Object{ Path: "/" + k, Name: k, IsFolder: true, Modified: d.Modified, } objs = append(objs, &obj) } return objs } // do others that not defined in Driver interface func getPair(path string) (string, string) { //path = strings.TrimSpace(path) if strings.Contains(path, ":") { pair := strings.SplitN(path, ":", 2) if !strings.Contains(pair[0], "/") { return pair[0], pair[1] } } return stdpath.Base(path), path } func (d *Strm) getRootAndPath(path string) (string, string) { if d.autoFlatten { return d.oneKey, path } path = strings.TrimPrefix(path, "/") parts := strings.SplitN(path, "/", 2) if len(parts) == 1 { return parts[0], "" } return parts[0], parts[1] } func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([]model.Obj, error) { reqPath := stdpath.Join(dst, sub) objs, err := fs.List(ctx, reqPath, args) if err != nil { return nil, err } return d.convert2strmObjs(ctx, reqPath, objs), nil } func (d *Strm) convert2strmObjs(ctx context.Context, reqPath string, objs []model.Obj) []model.Obj { var validObjs []model.Obj for _, obj := range objs { id, name, path := "", obj.GetName(), "" size := int64(0) if !obj.IsDir() { path = stdpath.Join(reqPath, obj.GetName()) sourceExt := utils.SourceExt(name) ext := strings.ToLower(sourceExt) if _, ok := d.downloadSuffix[ext]; ok { size = obj.GetSize() } else if _, ok := d.supportSuffix[ext]; ok { id = "strm" name = strings.TrimSuffix(name, sourceExt) + "strm" size = int64(len(d.getLink(ctx, path))) } else { continue } } objRes := model.Object{ ID: id, Path: path, Name: name, Size: size, Modified: obj.ModTime(), IsFolder: obj.IsDir(), } thumb, ok := model.GetThumb(obj) if !ok { validObjs = append(validObjs, &objRes) continue } validObjs = append(validObjs, &model.ObjThumb{ Object: objRes, Thumbnail: model.Thumbnail{ Thumbnail: thumb, }, }) } return validObjs } func (d *Strm) getLink(ctx context.Context, path string) string { finalPath := path if d.EncodePath { finalPath = utils.EncodePath(path, true) } if d.WithSign { signPath := sign.Sign(path) finalPath = fmt.Sprintf("%s?sign=%s", finalPath, signPath) } pathPrefix := d.PathPrefix if len(pathPrefix) > 0 { finalPath = stdpath.Join(pathPrefix, finalPath) } if !strings.HasPrefix(finalPath, "/") { finalPath = "/" + finalPath } if d.WithoutUrl { return finalPath } apiUrl := d.SiteUrl if len(apiUrl) > 0 { apiUrl = strings.TrimSuffix(apiUrl, "/") } else { apiUrl = common.GetApiUrl(ctx) } return fmt.Sprintf("%s%s", apiUrl, finalPath) } func (d *Strm) link(ctx context.Context, reqPath string, args model.LinkArgs) (*model.Link, model.Obj, error) { storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { return nil, nil, err } if !args.Redirect { return op.Link(ctx, storage, reqActualPath, args) } obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) if err != nil { return nil, nil, err } if common.ShouldProxy(storage, stdpath.Base(reqPath)) { return nil, obj, nil } return op.Link(ctx, storage, reqActualPath, args) } ================================================ FILE: drivers/teambition/driver.go ================================================ package teambition import ( "context" "errors" "net/http" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/go-resty/resty/v2" ) type Teambition struct { model.Storage Addition } func (d *Teambition) Config() driver.Config { return config } func (d *Teambition) GetAddition() driver.Additional { return &d.Addition } func (d *Teambition) Init(ctx context.Context) error { _, err := d.request("/api/v2/roles", http.MethodGet, nil, nil) return err } func (d *Teambition) Drop(ctx context.Context) error { return nil } func (d *Teambition) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { return d.getFiles(dir.GetID()) } func (d *Teambition) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if u, ok := file.(model.URL); ok { url := u.URL() res, _ := base.NoRedirectClient.R().Get(url) if res.StatusCode() == 302 { url = res.Header().Get("location") } return &model.Link{URL: url}, nil } return nil, errors.New("can't convert obj to URL") } func (d *Teambition) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { data := base.Json{ "objectType": "collection", "_projectId": d.ProjectID, "_creatorId": "", "created": "", "updated": "", "title": dirName, "color": "blue", "description": "", "workCount": 0, "collectionType": "", "recentWorks": []interface{}{}, "_parentId": parentDir.GetID(), "subCount": nil, } _, err := d.request("/api/collections", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *Teambition) Move(ctx context.Context, srcObj, dstDir model.Obj) error { pre := "/api/works/" if srcObj.IsDir() { pre = "/api/collections/" } _, err := d.request(pre+srcObj.GetID()+"/move", http.MethodPut, func(req *resty.Request) { req.SetBody(base.Json{ "_parentId": dstDir.GetID(), }) }, nil) return err } func (d *Teambition) Rename(ctx context.Context, srcObj model.Obj, newName string) error { pre := "/api/works/" data := base.Json{ "fileName": newName, } if srcObj.IsDir() { pre = "/api/collections/" data = base.Json{ "title": newName, } } _, err := d.request(pre+srcObj.GetID(), http.MethodPut, func(req *resty.Request) { req.SetBody(data) }, nil) return err } func (d *Teambition) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { pre := "/api/works/" if srcObj.IsDir() { pre = "/api/collections/" } _, err := d.request(pre+srcObj.GetID()+"/fork", http.MethodPut, func(req *resty.Request) { req.SetBody(base.Json{ "_parentId": dstDir.GetID(), }) }, nil) return err } func (d *Teambition) Remove(ctx context.Context, obj model.Obj) error { pre := "/api/works/" if obj.IsDir() { pre = "/api/collections/" } _, err := d.request(pre+obj.GetID()+"/archive", http.MethodPost, nil, nil) return err } func (d *Teambition) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { if d.UseS3UploadMethod { return d.newUpload(ctx, dstDir, stream, up) } var ( token string err error ) if d.isInternational() { res, err := d.request("/projects", http.MethodGet, nil, nil) if err != nil { return err } token = getBetweenStr(string(res), "strikerAuth":"", "","phoneForLogin") } else { res, err := d.request("/api/v2/users/me", http.MethodGet, nil, nil) if err != nil { return err } token = utils.Json.Get(res, "strikerAuth").ToString() } var newFile *FileUpload if stream.GetSize() <= 20971520 { // post upload newFile, err = d.upload(ctx, stream, token, up) } else { // chunk upload //err = base.ErrNotImplement newFile, err = d.chunkUpload(ctx, stream, token, up) } if err != nil { return err } return d.finishUpload(newFile, dstDir.GetID()) } var _ driver.Driver = (*Teambition)(nil) ================================================ FILE: drivers/teambition/help.go ================================================ package teambition import "strings" func getBetweenStr(str, start, end string) string { n := strings.Index(str, start) if n == -1 { return "" } n = n + len(start) str = string([]byte(str)[n:]) m := strings.Index(str, end) if m == -1 { return "" } str = string([]byte(str)[:m]) return str } ================================================ FILE: drivers/teambition/meta.go ================================================ package teambition import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { Region string `json:"region" type:"select" options:"china,international" required:"true"` Cookie string `json:"cookie" required:"true"` ProjectID string `json:"project_id" required:"true"` driver.RootID OrderBy string `json:"order_by" type:"select" options:"fileName,fileSize,updated,created" default:"fileName"` OrderDirection string `json:"order_direction" type:"select" options:"Asc,Desc" default:"Asc"` UseS3UploadMethod bool `json:"use_s3_upload_method" default:"true"` } var config = driver.Config{ Name: "Teambition", } func init() { op.RegisterDriver(func() driver.Driver { return &Teambition{} }) } ================================================ FILE: drivers/teambition/types.go ================================================ package teambition import "time" type ErrResp struct { Name string `json:"name"` Message string `json:"message"` } type Collection struct { ID string `json:"_id"` Title string `json:"title"` Updated time.Time `json:"updated"` } type Work struct { ID string `json:"_id"` FileName string `json:"fileName"` FileSize int64 `json:"fileSize"` FileKey string `json:"fileKey"` FileCategory string `json:"fileCategory"` DownloadURL string `json:"downloadUrl"` ThumbnailURL string `json:"thumbnailUrl"` Thumbnail string `json:"thumbnail"` Updated time.Time `json:"updated"` PreviewURL string `json:"previewUrl"` } type FileUpload struct { FileKey string `json:"fileKey"` FileName string `json:"fileName"` FileType string `json:"fileType"` FileSize int `json:"fileSize"` FileCategory string `json:"fileCategory"` ImageWidth int `json:"imageWidth"` ImageHeight int `json:"imageHeight"` InvolveMembers []interface{} `json:"involveMembers"` Source string `json:"source"` Visible string `json:"visible"` ParentId string `json:"_parentId"` } type ChunkUpload struct { FileUpload Storage string `json:"storage"` MimeType string `json:"mimeType"` Chunks int `json:"chunks"` ChunkSize int `json:"chunkSize"` Created time.Time `json:"created"` FileMD5 string `json:"fileMD5"` LastUpdated time.Time `json:"lastUpdated"` UploadedChunks []interface{} `json:"uploadedChunks"` Token struct { AppID string `json:"AppID"` OrganizationID string `json:"OrganizationID"` UserID string `json:"UserID"` Exp time.Time `json:"Exp"` Storage string `json:"Storage"` Resource string `json:"Resource"` Speed int `json:"Speed"` } `json:"token"` DownloadUrl string `json:"downloadUrl"` ThumbnailUrl string `json:"thumbnailUrl"` PreviewUrl string `json:"previewUrl"` ImmPreviewUrl string `json:"immPreviewUrl"` PreviewExt string `json:"previewExt"` LastUploadTime interface{} `json:"lastUploadTime"` } type UploadToken struct { Sdk struct { Endpoint string `json:"endpoint"` Region string `json:"region"` S3ForcePathStyle bool `json:"s3ForcePathStyle"` Credentials struct { AccessKeyId string `json:"accessKeyId"` SecretAccessKey string `json:"secretAccessKey"` SessionToken string `json:"sessionToken"` } `json:"credentials"` } `json:"sdk"` Upload struct { Bucket string `json:"Bucket"` Key string `json:"Key"` ContentDisposition string `json:"ContentDisposition"` ContentType string `json:"ContentType"` } `json:"upload"` Token string `json:"token"` DownloadUrl string `json:"downloadUrl"` } ================================================ FILE: drivers/teambition/util.go ================================================ package teambition import ( "bytes" "context" "errors" "fmt" "io" "net/http" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) // do others that not defined in Driver interface func (d *Teambition) isInternational() bool { return d.Region == "international" } func (d *Teambition) request(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { url := "https://www.teambition.com" + pathname if d.isInternational() { url = "https://us.teambition.com" + pathname } req := base.RestyClient.R() req.SetHeader("Cookie", d.Cookie) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e ErrResp req.SetError(&e) res, err := req.Execute(method, url) if err != nil { return nil, err } if e.Name != "" { return nil, errors.New(e.Message) } return res.Body(), nil } func (d *Teambition) getFiles(parentId string) ([]model.Obj, error) { files := make([]model.Obj, 0) page := 1 for { var collections []Collection _, err := d.request("/api/collections", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "_parentId": parentId, "_projectId": d.ProjectID, "order": d.OrderBy + d.OrderDirection, "count": "50", "page": strconv.Itoa(page), }) }, &collections) if err != nil { return nil, err } if len(collections) == 0 { break } page++ for _, collection := range collections { if collection.Title == "" { continue } files = append(files, &model.Object{ ID: collection.ID, Name: collection.Title, IsFolder: true, Modified: collection.Updated, }) } } page = 1 for { var works []Work _, err := d.request("/api/works", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "_parentId": parentId, "_projectId": d.ProjectID, "order": d.OrderBy + d.OrderDirection, "count": "50", "page": strconv.Itoa(page), }) }, &works) if err != nil { return nil, err } if len(works) == 0 { break } page++ for _, work := range works { files = append(files, &model.ObjThumbURL{ Object: model.Object{ ID: work.ID, Name: work.FileName, Size: work.FileSize, Modified: work.Updated, }, Thumbnail: model.Thumbnail{Thumbnail: work.Thumbnail}, Url: model.Url{Url: work.DownloadURL}, }) } } return files, nil } func (d *Teambition) upload(ctx context.Context, file model.FileStreamer, token string, up driver.UpdateProgress) (*FileUpload, error) { prefix := "tcs" if d.isInternational() { prefix = "us-tcs" } reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: file, UpdateProgress: up, }) var newFile FileUpload res, err := base.RestyClient.R(). SetContext(ctx). SetResult(&newFile).SetHeader("Authorization", token). SetMultipartFormData(map[string]string{ "name": file.GetName(), "type": file.GetMimetype(), "size": strconv.FormatInt(file.GetSize(), 10), "lastModifiedDate": time.Now().Format("Mon Jan 02 2006 15:04:05 GMT+0800 (中国标准时间)"), }). SetMultipartField("file", file.GetName(), file.GetMimetype(), reader). Post(fmt.Sprintf("https://%s.teambition.net/upload", prefix)) if err != nil { return nil, err } log.Debugf("[teambition] upload response: %s", res.String()) return &newFile, nil } func (d *Teambition) chunkUpload(ctx context.Context, file model.FileStreamer, token string, up driver.UpdateProgress) (*FileUpload, error) { prefix := "tcs" referer := "https://www.teambition.com/" if d.isInternational() { prefix = "us-tcs" referer = "https://us.teambition.com/" } var newChunk ChunkUpload _, err := base.RestyClient.R().SetResult(&newChunk).SetHeader("Authorization", token). SetBody(base.Json{ "fileName": file.GetName(), "fileSize": file.GetSize(), "lastUpdated": time.Now(), }).Post(fmt.Sprintf("https://%s.teambition.net/upload/chunk", prefix)) if err != nil { return nil, err } for i := 0; i < newChunk.Chunks; i++ { if utils.IsCanceled(ctx) { return nil, ctx.Err() } chunkSize := newChunk.ChunkSize if i == newChunk.Chunks-1 { chunkSize = int(file.GetSize()) - i*chunkSize } log.Debugf("%d : %d", i, chunkSize) chunkData := make([]byte, chunkSize) _, err = io.ReadFull(file, chunkData) if err != nil { return nil, err } u := fmt.Sprintf("https://%s.teambition.net/upload/chunk/%s?chunk=%d&chunks=%d", prefix, newChunk.FileKey, i+1, newChunk.Chunks) log.Debugf("url: %s", u) _, err := base.RestyClient.R(). SetContext(ctx). SetHeaders(map[string]string{ "Authorization": token, "Content-Type": "application/octet-stream", "Referer": referer, }). SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(chunkData))). Post(u) if err != nil { return nil, err } up(float64(i) * 100 / float64(newChunk.Chunks)) } _, err = base.RestyClient.R().SetHeader("Authorization", token).Post( fmt.Sprintf("https://%s.teambition.net/upload/chunk/%s", prefix, newChunk.FileKey)) if err != nil { return nil, err } return &newChunk.FileUpload, nil } func (d *Teambition) finishUpload(file *FileUpload, parentId string) error { file.InvolveMembers = []interface{}{} file.Visible = "members" file.ParentId = parentId _, err := d.request("/api/works", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "works": []FileUpload{*file}, "_parentId": parentId, }) }, nil) return err } func (d *Teambition) newUpload(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { var uploadToken UploadToken _, err := d.request("/api/awos/upload-token", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "category": "work", "fileName": stream.GetName(), "fileSize": stream.GetSize(), "fileType": stream.GetMimetype(), "payload": base.Json{ "involveMembers": []struct{}{}, "visible": "members", }, "scope": "project:" + d.ProjectID, }) }, &uploadToken) if err != nil { return err } cfg := &aws.Config{ Credentials: credentials.NewStaticCredentials( uploadToken.Sdk.Credentials.AccessKeyId, uploadToken.Sdk.Credentials.SecretAccessKey, uploadToken.Sdk.Credentials.SessionToken), Region: &uploadToken.Sdk.Region, Endpoint: &uploadToken.Sdk.Endpoint, S3ForcePathStyle: &uploadToken.Sdk.S3ForcePathStyle, } ss, err := session.NewSession(cfg) if err != nil { return err } uploader := s3manager.NewUploader(ss) if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) } input := &s3manager.UploadInput{ Bucket: &uploadToken.Upload.Bucket, Key: &uploadToken.Upload.Key, ContentDisposition: &uploadToken.Upload.ContentDisposition, ContentType: &uploadToken.Upload.ContentType, Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: stream, UpdateProgress: up, }), } _, err = uploader.UploadWithContext(ctx, input) if err != nil { return err } // finish upload _, err = d.request("/api/works", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "fileTokens": []string{uploadToken.Token}, "involveMembers": []struct{}{}, "visible": "members", "works": []struct{}{}, "_parentId": dstDir.GetID(), }) }, nil) return err } ================================================ FILE: drivers/teldrive/copy.go ================================================ package teldrive import ( "fmt" "net/http" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" "golang.org/x/net/context" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" ) func NewCopyManager(ctx context.Context, concurrent int, d *Teldrive) *CopyManager { g, ctx := errgroup.WithContext(ctx) return &CopyManager{ TaskChan: make(chan CopyTask, concurrent*2), Sem: semaphore.NewWeighted(int64(concurrent)), G: g, Ctx: ctx, d: d, } } func (cm *CopyManager) startWorkers() { workerCount := cap(cm.TaskChan) / 2 for i := 0; i < workerCount; i++ { cm.G.Go(func() error { return cm.worker() }) } } func (cm *CopyManager) worker() error { for { select { case task, ok := <-cm.TaskChan: if !ok { return nil } if err := cm.Sem.Acquire(cm.Ctx, 1); err != nil { return err } var err error err = cm.processFile(task) cm.Sem.Release(1) if err != nil { return fmt.Errorf("task processing failed: %w", err) } case <-cm.Ctx.Done(): return cm.Ctx.Err() } } } func (cm *CopyManager) generateTasks(ctx context.Context, srcObj, dstDir model.Obj) error { if srcObj.IsDir() { return cm.generateFolderTasks(ctx, srcObj, dstDir) } else { // add single file task directly select { case cm.TaskChan <- CopyTask{SrcObj: srcObj, DstDir: dstDir}: return nil case <-ctx.Done(): return ctx.Err() } } } func (cm *CopyManager) generateFolderTasks(ctx context.Context, srcDir, dstDir model.Obj) error { objs, err := cm.d.List(ctx, srcDir, model.ListArgs{}) if err != nil { return fmt.Errorf("failed to list directory %s: %w", srcDir.GetPath(), err) } err = cm.d.MakeDir(cm.Ctx, dstDir, srcDir.GetName()) if err != nil || len(objs) == 0 { return err } newDstDir := &model.Object{ ID: dstDir.GetID(), Path: dstDir.GetPath() + "/" + srcDir.GetName(), Name: srcDir.GetName(), IsFolder: true, } for _, file := range objs { if utils.IsCanceled(ctx) { return ctx.Err() } srcFile := &model.Object{ ID: file.GetID(), Path: srcDir.GetPath() + "/" + file.GetName(), Name: file.GetName(), IsFolder: file.IsDir(), } // 递归生成任务 if err := cm.generateTasks(ctx, srcFile, newDstDir); err != nil { return err } } return nil } func (cm *CopyManager) processFile(task CopyTask) error { return cm.copySingleFile(cm.Ctx, task.SrcObj, task.DstDir) } func (cm *CopyManager) copySingleFile(ctx context.Context, srcObj, dstDir model.Obj) error { // `override copy mode` should delete the existing file if obj, err := cm.d.getFile(dstDir.GetPath(), srcObj.GetName(), srcObj.IsDir()); err == nil { if err := cm.d.Remove(ctx, obj); err != nil { return fmt.Errorf("failed to remove existing file: %w", err) } } // Do copy return cm.d.request(http.MethodPost, "/api/files/{id}/copy", func(req *resty.Request) { req.SetPathParam("id", srcObj.GetID()) req.SetBody(base.Json{ "newName": srcObj.GetName(), "destination": dstDir.GetPath(), }) }, nil) } ================================================ FILE: drivers/teldrive/driver.go ================================================ package teldrive import ( "context" "fmt" "math" "net/http" "net/url" "path" "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" "github.com/google/uuid" "golang.org/x/sync/errgroup" ) type Teldrive struct { model.Storage Addition } func (d *Teldrive) Config() driver.Config { return config } func (d *Teldrive) GetAddition() driver.Additional { return &d.Addition } func (d *Teldrive) Init(ctx context.Context) error { d.Address = strings.TrimSuffix(d.Address, "/") if d.Cookie == "" || !strings.HasPrefix(d.Cookie, "access_token=") { return fmt.Errorf("cookie must start with 'access_token='") } if d.UploadConcurrency == 0 { d.UploadConcurrency = 4 } if d.ChunkSize == 0 { d.ChunkSize = 10 } op.MustSaveDriverStorage(d) return nil } func (d *Teldrive) Drop(ctx context.Context) error { return nil } func (d *Teldrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var firstResp ListResp err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { req.SetQueryParams(map[string]string{ "path": dir.GetPath(), "limit": "500", "page": "1", }) }, &firstResp) if err != nil { return nil, err } pagesData := make([][]Object, firstResp.Meta.TotalPages) pagesData[0] = firstResp.Items if firstResp.Meta.TotalPages > 1 { g, _ := errgroup.WithContext(ctx) g.SetLimit(8) for i := 2; i <= firstResp.Meta.TotalPages; i++ { page := i g.Go(func() error { var resp ListResp err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { req.SetQueryParams(map[string]string{ "path": dir.GetPath(), "limit": "500", "page": strconv.Itoa(page), }) }, &resp) if err != nil { return err } pagesData[page-1] = resp.Items return nil }) } if err := g.Wait(); err != nil { return nil, err } } var allItems []Object for _, items := range pagesData { allItems = append(allItems, items...) } return utils.SliceConvert(allItems, func(src Object) (model.Obj, error) { return &model.Object{ Path: path.Join(dir.GetPath(), src.Name), ID: src.ID, Name: src.Name, Size: func() int64 { if src.Type == "folder" { return 0 } return src.Size }(), IsFolder: src.Type == "folder", Modified: src.UpdatedAt, }, nil }) } func (d *Teldrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if d.UseShareLink { shareObj, err := d.getShareFileById(file.GetID()) if err != nil || shareObj == nil { if err := d.createShareFile(file.GetID()); err != nil { return nil, err } shareObj, err = d.getShareFileById(file.GetID()) if err != nil { return nil, err } } return &model.Link{ URL: d.Address + "/api/shares/" + url.PathEscape(shareObj.Id) + "/files/" + url.PathEscape(file.GetID()) + "/" + url.PathEscape(file.GetName()), }, nil } return &model.Link{ URL: d.Address + "/api/files/" + url.PathEscape(file.GetID()) + "/" + url.PathEscape(file.GetName()), Header: http.Header{ "Cookie": {d.Cookie}, }, }, nil } func (d *Teldrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { return d.request(http.MethodPost, "/api/files/mkdir", func(req *resty.Request) { req.SetBody(map[string]interface{}{ "path": parentDir.GetPath() + "/" + dirName, }) }, nil) } func (d *Teldrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { body := base.Json{ "ids": []string{srcObj.GetID()}, "destinationParent": dstDir.GetID(), } return d.request(http.MethodPost, "/api/files/move", func(req *resty.Request) { req.SetBody(body) }, nil) } func (d *Teldrive) Rename(ctx context.Context, srcObj model.Obj, newName string) error { body := base.Json{ "name": newName, } return d.request(http.MethodPatch, "/api/files/{id}", func(req *resty.Request) { req.SetPathParam("id", srcObj.GetID()) req.SetBody(body) }, nil) } func (d *Teldrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { copyConcurrentLimit := 4 copyManager := NewCopyManager(ctx, copyConcurrentLimit, d) copyManager.startWorkers() copyManager.G.Go(func() error { defer close(copyManager.TaskChan) return copyManager.generateTasks(ctx, srcObj, dstDir) }) return copyManager.G.Wait() } func (d *Teldrive) Remove(ctx context.Context, obj model.Obj) error { body := base.Json{ "ids": []string{obj.GetID()}, } return d.request(http.MethodPost, "/api/files/delete", func(req *resty.Request) { req.SetBody(body) }, nil) } func (d *Teldrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { fileId := uuid.New().String() chunkSizeInMB := d.ChunkSize chunkSize := chunkSizeInMB * 1024 * 1024 // Convert MB to bytes totalSize := file.GetSize() totalParts := int(math.Ceil(float64(totalSize) / float64(chunkSize))) maxRetried := 3 // delete the upload task when finished or failed defer func() { _ = d.request(http.MethodDelete, "/api/uploads/{id}", func(req *resty.Request) { req.SetPathParam("id", fileId) }, nil) }() if obj, err := d.getFile(dstDir.GetPath(), file.GetName(), file.IsDir()); err == nil { if err = d.Remove(ctx, obj); err != nil { return err } } // start the upload process if err := d.request(http.MethodGet, "/api/uploads/fileId", func(req *resty.Request) { req.SetPathParam("id", fileId) }, nil); err != nil { return err } if totalSize == 0 { return d.touch(file.GetName(), dstDir.GetPath()) } if totalParts <= 1 { return d.doSingleUpload(ctx, dstDir, file, up, maxRetried, totalParts, chunkSize, fileId) } return d.doMultiUpload(ctx, dstDir, file, up, maxRetried, totalParts, chunkSize, fileId) } func (d *Teldrive) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *Teldrive) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *Teldrive) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *Teldrive) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir // return errs.NotImplement to use an internal archive tool return nil, errs.NotImplement } //func (d *Teldrive) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*Teldrive)(nil) ================================================ FILE: drivers/teldrive/meta.go ================================================ package teldrive import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Address string `json:"url" required:"true"` Cookie string `json:"cookie" type:"string" required:"true" help:"access_token=xxx"` UseShareLink bool `json:"use_share_link" type:"bool" default:"false" help:"Create share link when getting link to support 302. If disabled, you need to enable web proxy."` ChunkSize int64 `json:"chunk_size" type:"number" default:"10" help:"Chunk size in MiB"` RandomChunkName bool `json:"random_chunk_name" type:"bool" default:"true" help:"Random chunk name"` UploadConcurrency int64 `json:"upload_concurrency" type:"number" default:"4" help:"Concurrency upload requests"` } var config = driver.Config{ Name: "Teldrive", DefaultRoot: "/", } func init() { op.RegisterDriver(func() driver.Driver { return &Teldrive{} }) } ================================================ FILE: drivers/teldrive/types.go ================================================ package teldrive import ( "context" "io" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" ) type ErrResp struct { Code int `json:"code"` Message string `json:"message"` } type Object struct { ID string `json:"id"` Name string `json:"name"` Type string `json:"type"` MimeType string `json:"mimeType"` Category string `json:"category,omitempty"` ParentId string `json:"parentId"` Size int64 `json:"size"` Encrypted bool `json:"encrypted"` UpdatedAt time.Time `json:"updatedAt"` } type ListResp struct { Items []Object `json:"items"` Meta struct { Count int `json:"count"` TotalPages int `json:"totalPages"` CurrentPage int `json:"currentPage"` } `json:"meta"` } type FilePart struct { Name string `json:"name"` PartId int `json:"partId"` PartNo int `json:"partNo"` ChannelId int `json:"channelId"` Size int `json:"size"` Encrypted bool `json:"encrypted"` Salt string `json:"salt"` } type chunkTask struct { chunkIdx int fileName string chunkSize int64 reader io.ReadSeeker ss stream.StreamSectionReaderIF } type CopyManager struct { TaskChan chan CopyTask Sem *semaphore.Weighted G *errgroup.Group Ctx context.Context d *Teldrive } type CopyTask struct { SrcObj model.Obj DstDir model.Obj } type ShareObj struct { Id string `json:"id"` Protected bool `json:"protected"` UserId int `json:"userId"` Type string `json:"type"` Name string `json:"name"` ExpiresAt time.Time `json:"expiresAt"` } ================================================ FILE: drivers/teldrive/upload.go ================================================ package teldrive import ( "crypto/md5" "encoding/hex" "fmt" "io" "net/http" "sort" "strconv" "sync" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" "github.com/google/uuid" "github.com/pkg/errors" "golang.org/x/net/context" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" ) // create empty file func (d *Teldrive) touch(name, path string) error { uploadBody := base.Json{ "name": name, "type": "file", "path": path, } if err := d.request(http.MethodPost, "/api/files", func(req *resty.Request) { req.SetBody(uploadBody) }, nil); err != nil { return err } return nil } func getMD5Hash(text string) string { hash := md5.Sum([]byte(text)) return hex.EncodeToString(hash[:]) } func (d *Teldrive) createFileOnUploadSuccess(name, id, path string, uploadedFileParts []FilePart, totalSize int64) error { remoteFileParts, err := d.getFilePart(id) if err != nil { return err } // check if the uploaded file parts match the remote file parts if len(remoteFileParts) != len(uploadedFileParts) { return fmt.Errorf("[Teldrive] file parts count mismatch: expected %d, got %d", len(uploadedFileParts), len(remoteFileParts)) } formatParts := make([]base.Json, 0) for _, p := range remoteFileParts { formatParts = append(formatParts, base.Json{ "id": p.PartId, "salt": p.Salt, }) } uploadBody := base.Json{ "name": name, "type": "file", "path": path, "parts": formatParts, "size": totalSize, } // create file here if err := d.request(http.MethodPost, "/api/files", func(req *resty.Request) { req.SetBody(uploadBody) }, nil); err != nil { return err } return nil } func (d *Teldrive) checkFilePartExist(fileId string, partId int) (FilePart, error) { var uploadedParts []FilePart var filePart FilePart if err := d.request(http.MethodGet, "/api/uploads/{id}", func(req *resty.Request) { req.SetPathParam("id", fileId) }, &uploadedParts); err != nil { return filePart, err } for _, part := range uploadedParts { if part.PartId == partId { return part, nil } } return filePart, nil } func (d *Teldrive) getFilePart(fileId string) ([]FilePart, error) { var uploadedParts []FilePart if err := d.request(http.MethodGet, "/api/uploads/{id}", func(req *resty.Request) { req.SetPathParam("id", fileId) }, &uploadedParts); err != nil { return nil, err } return uploadedParts, nil } func (d *Teldrive) singleUploadRequest(ctx context.Context, fileId string, callback base.ReqCallback, resp any) error { url := d.Address + "/api/uploads/" + fileId client := resty.New().SetTimeout(0) req := client.R(). SetContext(ctx) req.SetHeader("Cookie", d.Cookie) req.SetHeader("Content-Type", "application/octet-stream") req.SetContentLength(true) req.AddRetryCondition(func(r *resty.Response, err error) bool { return false }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e ErrResp req.SetError(&e) _req, err := req.Execute(http.MethodPost, url) if err != nil { return err } if _req.IsError() { return &e } return nil } func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up model.UpdateProgress, maxRetried, totalParts int, chunkSize int64, fileId string) error { totalSize := file.GetSize() var fileParts []FilePart var uploaded int64 = 0 var partName string chunkSize = min(totalSize, chunkSize) ss, err := stream.NewStreamSectionReader(file, int(chunkSize), &up) if err != nil { return err } chunkCnt := 0 for uploaded < totalSize { if utils.IsCanceled(ctx) { return ctx.Err() } curChunkSize := min(totalSize-uploaded, chunkSize) rd, err := ss.GetSectionReader(uploaded, curChunkSize) if err != nil { return err } chunkCnt += 1 filePart := &FilePart{} if err := retry.Do(func() error { if _, err := rd.Seek(0, io.SeekStart); err != nil { return err } if d.RandomChunkName { partName = getMD5Hash(uuid.New().String()) } else { partName = file.GetName() if totalParts > 1 { partName = fmt.Sprintf("%s.part.%03d", file.GetName(), chunkCnt) } } if err := d.singleUploadRequest(ctx, fileId, func(req *resty.Request) { uploadParams := map[string]string{ "partName": partName, "partNo": strconv.Itoa(chunkCnt), "fileName": file.GetName(), } req.SetQueryParams(uploadParams) req.SetBody(driver.NewLimitedUploadStream(ctx, rd)) req.SetHeader("Content-Length", strconv.FormatInt(curChunkSize, 10)) }, filePart); err != nil { return err } return nil }, retry.Context(ctx), retry.Attempts(uint(maxRetried)), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)); err != nil { return err } if filePart.Name != "" { fileParts = append(fileParts, *filePart) uploaded += curChunkSize up(float64(uploaded) / float64(totalSize) * 100) ss.FreeSectionReader(rd) } else { // For common situation this code won't reach return fmt.Errorf("[Teldrive] upload chunk %d failed: filePart Somehow missing", chunkCnt) } } return d.createFileOnUploadSuccess(file.GetName(), fileId, dstDir.GetPath(), fileParts, totalSize) } func (d *Teldrive) doMultiUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up model.UpdateProgress, maxRetried, totalParts int, chunkSize int64, fileId string) error { concurrent := d.UploadConcurrency g, ctx := errgroup.WithContext(ctx) sem := semaphore.NewWeighted(int64(concurrent)) chunkChan := make(chan chunkTask, concurrent*2) resultChan := make(chan FilePart, concurrent) totalSize := file.GetSize() ss, err := stream.NewStreamSectionReader(file, int(totalSize), &up) if err != nil { return err } ssLock := sync.Mutex{} g.Go(func() error { defer close(chunkChan) chunkIdx := 0 for chunkIdx < totalParts { select { case <-ctx.Done(): return ctx.Err() default: } offset := int64(chunkIdx) * chunkSize curChunkSize := min(totalSize-offset, chunkSize) ssLock.Lock() reader, err := ss.GetSectionReader(offset, curChunkSize) ssLock.Unlock() if err != nil { return err } task := chunkTask{ chunkIdx: chunkIdx + 1, chunkSize: curChunkSize, fileName: file.GetName(), reader: reader, ss: ss, } // freeSectionReader will be called in d.uploadSingleChunk select { case chunkChan <- task: chunkIdx++ case <-ctx.Done(): return ctx.Err() } } return nil }) for i := 0; i < int(concurrent); i++ { g.Go(func() error { for task := range chunkChan { if err := sem.Acquire(ctx, 1); err != nil { return err } filePart, err := d.uploadSingleChunk(ctx, fileId, task, totalParts, maxRetried) sem.Release(1) if err != nil { return fmt.Errorf("upload chunk %d failed: %w", task.chunkIdx, err) } select { case resultChan <- *filePart: case <-ctx.Done(): return ctx.Err() } } return nil }) } var fileParts []FilePart var collectErr error collectDone := make(chan struct{}) go func() { defer close(collectDone) fileParts = make([]FilePart, 0, totalParts) done := make(chan error, 1) go func() { done <- g.Wait() close(resultChan) }() for { select { case filePart, ok := <-resultChan: if !ok { collectErr = <-done return } fileParts = append(fileParts, filePart) case err := <-done: collectErr = err return } } }() <-collectDone if collectErr != nil { return fmt.Errorf("multi-upload failed: %w", collectErr) } sort.Slice(fileParts, func(i, j int) bool { return fileParts[i].PartNo < fileParts[j].PartNo }) return d.createFileOnUploadSuccess(file.GetName(), fileId, dstDir.GetPath(), fileParts, totalSize) } func (d *Teldrive) uploadSingleChunk(ctx context.Context, fileId string, task chunkTask, totalParts, maxRetried int) (*FilePart, error) { filePart := &FilePart{} retryCount := 0 var partName string defer task.ss.FreeSectionReader(task.reader) for { select { case <-ctx.Done(): return nil, ctx.Err() default: } if existingPart, err := d.checkFilePartExist(fileId, task.chunkIdx); err == nil && existingPart.Name != "" { return &existingPart, nil } if _, err := task.reader.Seek(0, io.SeekStart); err != nil { return nil, err } if d.RandomChunkName { partName = getMD5Hash(uuid.New().String()) } else { partName = task.fileName if totalParts > 1 { partName = fmt.Sprintf("%s.part.%03d", task.fileName, task.chunkIdx) } } err := d.singleUploadRequest(ctx, fileId, func(req *resty.Request) { uploadParams := map[string]string{ "partName": partName, "partNo": strconv.Itoa(task.chunkIdx), "fileName": task.fileName, } req.SetQueryParams(uploadParams) req.SetBody(driver.NewLimitedUploadStream(ctx, task.reader)) req.SetHeader("Content-Length", strconv.Itoa(int(task.chunkSize))) }, filePart) if err == nil { return filePart, nil } if retryCount >= maxRetried { return nil, fmt.Errorf("upload failed after %d retries: %w", maxRetried, err) } if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { continue } retryCount++ utils.Log.Errorf("[Teldrive] upload error: %v, retrying %d times", err, retryCount) backoffDuration := time.Duration(retryCount*retryCount) * time.Second if backoffDuration > 30*time.Second { backoffDuration = 30 * time.Second } select { case <-time.After(backoffDuration): case <-ctx.Done(): return nil, ctx.Err() } } } ================================================ FILE: drivers/teldrive/util.go ================================================ package teldrive import ( "fmt" "net/http" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/go-resty/resty/v2" ) // do others that not defined in Driver interface func (d *Teldrive) request(method string, pathname string, callback base.ReqCallback, resp interface{}) error { url := d.Address + pathname req := base.RestyClient.R() req.SetHeader("Cookie", d.Cookie) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e ErrResp req.SetError(&e) _req, err := req.Execute(method, url) if err != nil { return err } if _req.IsError() { return &e } return nil } func (d *Teldrive) getFile(path, name string, isFolder bool) (model.Obj, error) { resp := &ListResp{} err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { req.SetQueryParams(map[string]string{ "path": path, "name": name, "type": func() string { if isFolder { return "folder" } return "file" }(), "operation": "find", }) }, resp) if err != nil { return nil, err } if len(resp.Items) == 0 { return nil, fmt.Errorf("file not found: %s/%s", path, name) } obj := resp.Items[0] return &model.Object{ ID: obj.ID, Name: obj.Name, Size: obj.Size, IsFolder: obj.Type == "folder", }, err } func (err *ErrResp) Error() string { if err == nil { return "" } return fmt.Sprintf("[Teldrive] message:%s Error code:%d", err.Message, err.Code) } func (d *Teldrive) createShareFile(fileId string) error { var errResp ErrResp if err := d.request(http.MethodPost, "/api/files/{id}/share", func(req *resty.Request) { req.SetPathParam("id", fileId) req.SetBody(base.Json{ "expiresAt": getDateTime(), }) }, &errResp); err != nil { return err } if errResp.Message != "" { return &errResp } return nil } func (d *Teldrive) getShareFileById(fileId string) (*ShareObj, error) { var shareObj ShareObj if err := d.request(http.MethodGet, "/api/files/{id}/share", func(req *resty.Request) { req.SetPathParam("id", fileId) }, &shareObj); err != nil { return nil, err } return &shareObj, nil } func getDateTime() string { now := time.Now().UTC() formattedWithMs := now.Add(time.Hour * 1).Format("2006-01-02T15:04:05.000Z") return formattedWithMs } ================================================ FILE: drivers/template/driver.go ================================================ package template import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type Template struct { model.Storage Addition } func (d *Template) Config() driver.Config { return config } func (d *Template) GetAddition() driver.Additional { return &d.Addition } func (d *Template) Init(ctx context.Context) error { // TODO login / refresh token //op.MustSaveDriverStorage(d) return nil } func (d *Template) Drop(ctx context.Context) error { return nil } func (d *Template) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { // TODO return the files list, required return nil, errs.NotImplement } func (d *Template) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { // TODO return link of file, required return nil, errs.NotImplement } func (d *Template) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { // TODO create folder, optional return nil, errs.NotImplement } func (d *Template) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { // TODO move obj, optional return nil, errs.NotImplement } func (d *Template) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { // TODO rename obj, optional return nil, errs.NotImplement } func (d *Template) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { // TODO copy obj, optional return nil, errs.NotImplement } func (d *Template) Remove(ctx context.Context, obj model.Obj) error { // TODO remove obj, optional return errs.NotImplement } func (d *Template) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { // TODO upload file, optional return nil, errs.NotImplement } func (d *Template) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *Template) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *Template) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional return nil, errs.NotImplement } func (d *Template) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir // return errs.NotImplement to use an internal archive tool return nil, errs.NotImplement } func (d *Template) GetDetails(ctx context.Context) (*model.StorageDetails, error) { // TODO return storage details (total space, free space, etc.) return nil, errs.NotImplement } //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*Template)(nil) ================================================ FILE: drivers/template/meta.go ================================================ package template import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { // Usually one of two driver.RootPath driver.RootID // define other Field string `json:"field" type:"select" required:"true" options:"a,b,c" default:"a"` } var config = driver.Config{ Name: "Template", LocalSort: false, OnlyProxy: false, NoCache: false, NoUpload: false, NeedMs: false, DefaultRoot: "root, / or other", CheckStatus: false, Alert: "", NoOverwriteUpload: false, NoLinkURL: false, } func init() { op.RegisterDriver(func() driver.Driver { return &Template{} }) } ================================================ FILE: drivers/template/types.go ================================================ package template ================================================ FILE: drivers/template/util.go ================================================ package template // do others that not defined in Driver interface ================================================ FILE: drivers/terabox/driver.go ================================================ package terabox import ( "bytes" "context" "crypto/md5" "encoding/hex" "fmt" "io" stdpath "path" "strconv" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" log "github.com/sirupsen/logrus" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type Terabox struct { model.Storage Addition JsToken string url_domain_prefix string base_url string } func (d *Terabox) Config() driver.Config { return config } func (d *Terabox) GetAddition() driver.Additional { return &d.Addition } func (d *Terabox) Init(ctx context.Context) error { var resp CheckLoginResp d.base_url = "https://www.terabox.com" d.url_domain_prefix = "jp" _, err := d.get("/api/check/login", nil, &resp) if err != nil { return err } if resp.Errno != 0 { if resp.Errno == 9000 { return fmt.Errorf("terabox is not yet available in this area") } return fmt.Errorf("failed to check login status according to cookie") } return err } func (d *Terabox) Drop(ctx context.Context) error { return nil } func (d *Terabox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(dir.GetPath()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { obj := fileToObj(src) obj.Path = stdpath.Join(dir.GetPath(), obj.Name) return obj, nil }) } func (d *Terabox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if d.DownloadAPI == "crack" { return d.linkCrack(file, args) } return d.linkOfficial(file, args) } func (d *Terabox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { params := map[string]string{ "a": "commit", } data := map[string]string{ "path": stdpath.Join(parentDir.GetPath(), dirName), "isdir": "1", "block_list": "[]", } res, err := d.post_form("/api/create", params, data, nil) log.Debugln(string(res)) return err } func (d *Terabox) Move(ctx context.Context, srcObj, dstDir model.Obj) error { data := []base.Json{ { "path": srcObj.GetPath(), "dest": dstDir.GetPath(), "newname": srcObj.GetName(), }, } _, err := d.manage("move", data) return err } func (d *Terabox) Rename(ctx context.Context, srcObj model.Obj, newName string) error { data := []base.Json{ { "path": srcObj.GetPath(), "newname": newName, }, } _, err := d.manage("rename", data) return err } func (d *Terabox) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { data := []base.Json{ { "path": srcObj.GetPath(), "dest": dstDir.GetPath(), "newname": srcObj.GetName(), }, } _, err := d.manage("copy", data) return err } func (d *Terabox) Remove(ctx context.Context, obj model.Obj) error { data := []string{obj.GetPath()} _, err := d.manage("delete", data) return err } func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { resp, err := base.RestyClient.R(). SetContext(ctx). Get("https://" + d.url_domain_prefix + "-data.terabox.com/rest/2.0/pcs/file?method=locateupload") if err != nil { return err } var locateupload_resp LocateUploadResp err = utils.Json.Unmarshal(resp.Body(), &locateupload_resp) if err != nil { log.Debugln(resp) return err } log.Debugln(locateupload_resp) // precreate file rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName()) path := encodeURIComponent(rawPath) var precreateBlockListStr string if stream.GetSize() > initialChunkSize { precreateBlockListStr = `["5910a591dd8fc18c32a8f3df4fdc1761","a5fc157d78e6ad1c7e114b056c92821e"]` } else { precreateBlockListStr = `["5910a591dd8fc18c32a8f3df4fdc1761"]` } data := map[string]string{ "path": rawPath, "autoinit": "1", "target_path": dstDir.GetPath(), "block_list": precreateBlockListStr, "local_mtime": strconv.FormatInt(stream.ModTime().Unix(), 10), "file_limit_switch_v34": "true", } var precreateResp PrecreateResp log.Debugln(data) res, err := d.post_form("/api/precreate", nil, data, &precreateResp) if err != nil { return err } log.Debugf("%+v", precreateResp) if precreateResp.Errno != 0 { log.Debugln(string(res)) return fmt.Errorf("[terabox] failed to precreate file, errno: %d", precreateResp.Errno) } if precreateResp.ReturnType == 2 { return nil } // upload chunks tempFile, err := stream.CacheFullAndWriter(&up, nil) if err != nil { return err } params := map[string]string{ "method": "upload", "path": path, "uploadid": precreateResp.Uploadid, } streamSize := stream.GetSize() chunkSize := calculateChunkSize(streamSize) chunkByteData := make([]byte, chunkSize) count := int((streamSize + chunkSize - 1) / chunkSize) left := streamSize uploadBlockList := make([]string, 0, count) h := md5.New() for partseq := 0; partseq < count; partseq++ { if utils.IsCanceled(ctx) { return ctx.Err() } byteSize := chunkSize var byteData []byte if left >= chunkSize { byteData = chunkByteData } else { byteSize = left byteData = make([]byte, byteSize) } left -= byteSize _, err = io.ReadFull(tempFile, byteData) if err != nil { return err } // calculate md5 h.Write(byteData) localMD5 := hex.EncodeToString(h.Sum(nil)) uploadBlockList = append(uploadBlockList, localMD5) h.Reset() u := "https://" + locateupload_resp.Host + "/rest/2.0/pcs/superfile2" params["partseq"] = strconv.Itoa(partseq) log.Debugf("%+v", params) err = retry.Do( func() error { fileReader := driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData)) res, err := d.post_multipart(u, params, "file", stream.GetName(), fileReader, nil) log.Debugln(string(res)) if err != nil { return err } rspmd5 := utils.Json.Get(res, "md5").ToString() if localMD5 != rspmd5 { log.Debugf("MD5 mismatch, our MD5: %s, server: %s", localMD5, rspmd5) return fmt.Errorf("MD5 mismatch") } return nil }, retry.Attempts(5), retry.DelayType(retry.FixedDelay), retry.Context(ctx), ) if err != nil { return err } if count > 0 { up(float64(partseq) * 100 / float64(count)) } } // create file params = map[string]string{ "isdir": "0", "rtype": "1", } uploadBlockListStr, err := utils.Json.MarshalToString(uploadBlockList) if err != nil { return err } data = map[string]string{ "path": rawPath, "size": strconv.FormatInt(stream.GetSize(), 10), "uploadid": precreateResp.Uploadid, "target_path": dstDir.GetPath(), "block_list": uploadBlockListStr, "local_mtime": strconv.FormatInt(stream.ModTime().Unix(), 10), } var createResp CreateResp res, err = d.post_form("/api/create", params, data, &createResp) log.Debugln(string(res)) if err != nil { return err } if createResp.Errno != 0 { return fmt.Errorf("[terabox] failed to create file, errno: %d", createResp.Errno) } return nil } var _ driver.Driver = (*Terabox)(nil) ================================================ FILE: drivers/terabox/meta.go ================================================ package terabox import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Cookie string `json:"cookie" required:"true"` //JsToken string `json:"js_token" type:"string" required:"true"` DownloadAPI string `json:"download_api" type:"select" options:"official,crack" default:"official"` OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` } var config = driver.Config{ Name: "Terabox", DefaultRoot: "/", } func init() { op.RegisterDriver(func() driver.Driver { return &Terabox{} }) } ================================================ FILE: drivers/terabox/types.go ================================================ package terabox import ( "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type File struct { //TkbindId int `json:"tkbind_id"` //OwnerType int `json:"owner_type"` //Category int `json:"category"` //RealCategory string `json:"real_category"` FsId int64 `json:"fs_id"` ServerMtime int64 `json:"server_mtime"` //OperId int `json:"oper_id"` //ServerCtime int `json:"server_ctime"` Thumbs struct { //Icon string `json:"icon"` Url3 string `json:"url3"` //Url2 string `json:"url2"` //Url1 string `json:"url1"` } `json:"thumbs"` //Wpfile int `json:"wpfile"` //LocalMtime int `json:"local_mtime"` Size int64 `json:"size"` //ExtentTinyint7 int `json:"extent_tinyint7"` Path string `json:"path"` //Share int `json:"share"` //ServerAtime int `json:"server_atime"` //Pl int `json:"pl"` //LocalCtime int `json:"local_ctime"` ServerFilename string `json:"server_filename"` //Md5 string `json:"md5"` //OwnerId int `json:"owner_id"` //Unlist int `json:"unlist"` Isdir int `json:"isdir"` } type ListResp struct { Errno int `json:"errno"` GuidInfo string `json:"guid_info"` List []File `json:"list"` //RequestId int64 `json:"request_id"` 接口返回有时是int有时是string Guid int `json:"guid"` } func fileToObj(f File) *model.ObjThumb { return &model.ObjThumb{ Object: model.Object{ ID: strconv.FormatInt(f.FsId, 10), Name: f.ServerFilename, Size: f.Size, Modified: time.Unix(f.ServerMtime, 0), IsFolder: f.Isdir == 1, }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, } } type DownloadResp struct { Errno int `json:"errno"` Dlink []struct { Dlink string `json:"dlink"` } `json:"dlink"` } type DownloadResp2 struct { Errno int `json:"errno"` Info []struct { Dlink string `json:"dlink"` } `json:"info"` //RequestID int64 `json:"request_id"` } type HomeInfoResp struct { Errno int `json:"errno"` Data struct { Sign1 string `json:"sign1"` Sign3 string `json:"sign3"` Timestamp int `json:"timestamp"` } `json:"data"` } type PrecreateResp struct { Path string `json:"path"` Uploadid string `json:"uploadid"` ReturnType int `json:"return_type"` BlockList []int `json:"block_list"` Errno int `json:"errno"` //RequestId int64 `json:"request_id"` } type CheckLoginResp struct { Errno int `json:"errno"` } type LocateUploadResp struct { Host string `json:"host"` } type CreateResp struct { Errno int `json:"errno"` } ================================================ FILE: drivers/terabox/util.go ================================================ package terabox import ( "encoding/base64" "fmt" "io" "net/http" "net/url" "regexp" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) const ( initialChunkSize int64 = 4 << 20 // 4MB initialSizeThreshold int64 = 4 << 30 // 4GB ) func getStrBetween(raw, start, end string) string { regexPattern := fmt.Sprintf(`%s(.*?)%s`, regexp.QuoteMeta(start), regexp.QuoteMeta(end)) regex := regexp.MustCompile(regexPattern) matches := regex.FindStringSubmatch(raw) if len(matches) < 2 { return "" } mid := matches[1] return mid } func (d *Terabox) resetJsToken() error { u := d.base_url res, err := base.RestyClient.R().SetHeaders(map[string]string{ "Cookie": d.Cookie, "Accept": "application/json, text/plain, */*", "Referer": d.base_url, "User-Agent": "terabox;1.37.0.7;PC;PC-Windows;10.0.22631;WindowsTeraBox", "X-Requested-With": "XMLHttpRequest", }).Get(u) if err != nil { return err } html := res.String() jsToken := getStrBetween(html, "`function%20fn%28a%29%7Bwindow.jsToken%20%3D%20a%7D%3Bfn%28%22", "%22%29`") if jsToken == "" { return fmt.Errorf("jsToken not found, html: %s", html) } d.JsToken = jsToken return nil } func (d *Terabox) request(rurl string, method string, callback base.ReqCallback, resp interface{}, noRetry ...bool) ([]byte, error) { req := base.RestyClient.R() req.SetHeaders(map[string]string{ "Cookie": d.Cookie, "Accept": "application/json, text/plain, */*", "Referer": d.base_url, "User-Agent": "terabox;1.37.0.7;PC;PC-Windows;10.0.22631;WindowsTeraBox", "X-Requested-With": "XMLHttpRequest", }) req.SetQueryParams(map[string]string{ "app_id": "250528", "web": "1", "channel": "dubox", "clienttype": "0", "jsToken": d.JsToken, }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } full_url := d.base_url + rurl if strings.HasPrefix(rurl, "https://") { full_url = rurl } res, err := req.Execute(method, full_url) if err != nil { return nil, err } errno := utils.Json.Get(res.Body(), "errno").ToInt() if errno == 4000023 || errno == 450016 { // reget jsToken err = d.resetJsToken() if err != nil { return nil, err } if !utils.IsBool(noRetry...) { return d.request(rurl, method, callback, resp, true) } } else if errno == -6 { header := res.Header() log.Debugln(header) urlDomainPrefix := header.Get("Url-Domain-Prefix") if len(urlDomainPrefix) > 0 { d.url_domain_prefix = urlDomainPrefix d.base_url = "https://" + d.url_domain_prefix + ".terabox.com" log.Debugln("Redirect base_url to", d.base_url) return d.request(rurl, method, callback, resp, noRetry...) } } return res.Body(), nil } func (d *Terabox) get(pathname string, params map[string]string, resp interface{}) ([]byte, error) { return d.request(pathname, http.MethodGet, func(req *resty.Request) { if params != nil { req.SetQueryParams(params) } }, resp) } func (d *Terabox) post(pathname string, params map[string]string, data interface{}, resp interface{}) ([]byte, error) { return d.request(pathname, http.MethodPost, func(req *resty.Request) { if params != nil { req.SetQueryParams(params) } req.SetBody(data) }, resp) } func (d *Terabox) post_form(pathname string, params map[string]string, data map[string]string, resp interface{}) ([]byte, error) { return d.request(pathname, http.MethodPost, func(req *resty.Request) { if params != nil { req.SetQueryParams(params) } req.SetFormData(data) }, resp) } func (d *Terabox) post_multipart( pathname string, params map[string]string, fileFieldName string, fileName string, fileReader io.Reader, resp interface{}, ) ([]byte, error) { return d.request(pathname, http.MethodPost, func(req *resty.Request) { if params != nil { req.SetQueryParams(params) } req.SetFileReader(fileFieldName, fileName, fileReader) }, resp) } func (d *Terabox) getFiles(dir string) ([]File, error) { page := 1 num := 100 params := map[string]string{ "dir": dir, } if d.OrderBy != "" { params["order"] = d.OrderBy if d.OrderDirection == "desc" { params["desc"] = "1" } } res := make([]File, 0) for { params["page"] = strconv.Itoa(page) params["num"] = strconv.Itoa(num) var resp ListResp _, err := d.get("/api/list", params, &resp) if err != nil { return nil, err } if resp.Errno == 9000 { return nil, fmt.Errorf("terabox is not yet available in this area") } if len(resp.List) == 0 { break } res = append(res, resp.List...) page++ } return res, nil } func sign(s1, s2 string) string { var a = make([]int, 256) var p = make([]int, 256) var o []byte var v = len(s1) for q := 0; q < 256; q++ { a[q] = int(s1[(q % v) : (q%v)+1][0]) p[q] = q } for u, q := 0, 0; q < 256; q++ { u = (u + p[q] + a[q]) % 256 p[q], p[u] = p[u], p[q] } for i, u, q := 0, 0, 0; q < len(s2); q++ { i = (i + 1) % 256 u = (u + p[i]) % 256 p[i], p[u] = p[u], p[i] k := p[((p[i] + p[u]) % 256)] o = append(o, byte(int(s2[q])^k)) } return base64.StdEncoding.EncodeToString(o) } func (d *Terabox) genSign() (string, error) { var resp HomeInfoResp _, err := d.get("/api/home/info", map[string]string{}, &resp) if err != nil { return "", err } return sign(resp.Data.Sign3, resp.Data.Sign1), nil } func (d *Terabox) linkOfficial(file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp DownloadResp signString, err := d.genSign() if err != nil { return nil, err } params := map[string]string{ "type": "dlink", "fidlist": fmt.Sprintf("[%s]", file.GetID()), "sign": signString, "vip": "2", "timestamp": strconv.FormatInt(time.Now().Unix(), 10), } _, err = d.get("/api/download", params, &resp) if err != nil { return nil, err } if len(resp.Dlink) == 0 { return nil, fmt.Errorf("fid %s no dlink found, errno: %d", file.GetID(), resp.Errno) } res, err := base.NoRedirectClient.R().SetHeader("Cookie", d.Cookie).SetHeader("User-Agent", "terabox;1.37.0.7;PC;PC-Windows;10.0.22631;WindowsTeraBox").Get(resp.Dlink[0].Dlink) if err != nil { return nil, err } u := res.Header().Get("location") return &model.Link{ URL: u, Header: http.Header{ "User-Agent": []string{"terabox;1.37.0.7;PC;PC-Windows;10.0.22631;WindowsTeraBox"}, }, }, nil } func (d *Terabox) linkCrack(file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp DownloadResp2 param := map[string]string{ "target": fmt.Sprintf("[\"%s\"]", file.GetPath()), "dlink": "1", "origin": "dlna", } _, err := d.get("/api/filemetas", param, &resp) if err != nil { return nil, err } return &model.Link{ URL: resp.Info[0].Dlink, Header: http.Header{ "User-Agent": []string{"terabox;1.37.0.7;PC;PC-Windows;10.0.22631;WindowsTeraBox"}, }, }, nil } func (d *Terabox) manage(opera string, filelist interface{}) ([]byte, error) { params := map[string]string{ "onnest": "fail", "opera": opera, } marshal, err := utils.Json.Marshal(filelist) if err != nil { return nil, err } data := fmt.Sprintf("async=0&filelist=%s&ondup=newcopy", encodeURIComponent(string(marshal))) return d.post("/api/filemanager", params, data, nil) } func encodeURIComponent(str string) string { r := url.QueryEscape(str) r = strings.ReplaceAll(r, "+", "%20") return r } func calculateChunkSize(streamSize int64) int64 { chunkSize := initialChunkSize sizeThreshold := initialSizeThreshold if streamSize < chunkSize { return streamSize } for streamSize > sizeThreshold { chunkSize <<= 1 sizeThreshold <<= 1 } return chunkSize } ================================================ FILE: drivers/thunder/driver.go ================================================ package thunder import ( "context" "fmt" "net/http" "net/url" "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" ) type Thunder struct { *XunLeiCommon model.Storage Addition identity string } func (x *Thunder) Config() driver.Config { return config } func (x *Thunder) GetAddition() driver.Additional { return &x.Addition } func (x *Thunder) Init(ctx context.Context) (err error) { // 初始化所需参数 if x.XunLeiCommon == nil { x.XunLeiCommon = &XunLeiCommon{ Common: &Common{ client: base.NewRestyClient(), Algorithms: []string{ "9uJNVj/wLmdwKrJaVj/omlQ", "Oz64Lp0GigmChHMf/6TNfxx7O9PyopcczMsnf", "Eb+L7Ce+Ej48u", "jKY0", "ASr0zCl6v8W4aidjPK5KHd1Lq3t+vBFf41dqv5+fnOd", "wQlozdg6r1qxh0eRmt3QgNXOvSZO6q/GXK", "gmirk+ciAvIgA/cxUUCema47jr/YToixTT+Q6O", "5IiCoM9B1/788ntB", "P07JH0h6qoM6TSUAK2aL9T5s2QBVeY9JWvalf", "+oK0AN", }, DeviceID: func() string { if len(x.DeviceID) != 32 { return utils.GetMD5EncodeStr(x.Username + x.Password) } return x.DeviceID }(), ClientID: "Xp6vsxz_7IYVw2BB", ClientSecret: "Xp6vsy4tN9toTVdMSpomVdXpRmES", ClientVersion: "8.31.0.9726", PackageName: "com.xunlei.downloadprovider", UserAgent: "ANDROID-com.xunlei.downloadprovider/8.31.0.9726 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/512000 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)", DownloadUserAgent: "Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)", Space: x.Space, refreshCTokenCk: func(token string) { x.CaptchaToken = token op.MustSaveDriverStorage(x) }, }, refreshTokenFunc: func() error { // 通过RefreshToken刷新 token, err := x.RefreshToken(x.TokenResp.RefreshToken) if err != nil { // 重新登录 token, err = x.Login(x.Username, x.Password) if err != nil { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) op.MustSaveDriverStorage(x) } // 清空 信任密钥 x.Addition.CreditKey = "" } x.SetTokenResp(token) return err }, } } // 自定义验证码token ctoekn := strings.TrimSpace(x.CaptchaToken) if ctoekn != "" { x.SetCaptchaToken(ctoekn) } if x.Addition.CreditKey != "" { x.SetCreditKey(x.Addition.CreditKey) } if x.Addition.DeviceID != "" { x.Common.DeviceID = x.Addition.DeviceID } else { x.Addition.DeviceID = x.Common.DeviceID op.MustSaveDriverStorage(x) } // 防止重复登录 identity := x.GetIdentity() if x.identity != identity || !x.IsLogin() { x.identity = identity // 登录 token, err := x.Login(x.Username, x.Password) if err != nil { return err } // 清空 信任密钥 x.Addition.CreditKey = "" x.SetTokenResp(token) } return nil } func (x *Thunder) Drop(ctx context.Context) error { return nil } type ThunderExpert struct { *XunLeiCommon model.Storage ExpertAddition identity string } func (x *ThunderExpert) Config() driver.Config { return configExpert } func (x *ThunderExpert) GetAddition() driver.Additional { return &x.ExpertAddition } func (x *ThunderExpert) Init(ctx context.Context) (err error) { // 防止重复登录 identity := x.GetIdentity() if identity != x.identity || !x.IsLogin() { x.identity = identity x.XunLeiCommon = &XunLeiCommon{ Common: &Common{ client: base.NewRestyClient(), DeviceID: func() string { if len(x.DeviceID) != 32 { return utils.GetMD5EncodeStr(x.DeviceID) } return x.DeviceID }(), ClientID: x.ClientID, ClientSecret: x.ClientSecret, ClientVersion: x.ClientVersion, PackageName: x.PackageName, UserAgent: x.UserAgent, DownloadUserAgent: x.DownloadUserAgent, UseVideoUrl: x.UseVideoUrl, Space: x.Space, refreshCTokenCk: func(token string) { x.CaptchaToken = token op.MustSaveDriverStorage(x) }, }, } if x.CaptchaToken != "" { x.SetCaptchaToken(x.CaptchaToken) } if x.ExpertAddition.CreditKey != "" { x.SetCreditKey(x.ExpertAddition.CreditKey) } if x.ExpertAddition.DeviceID != "" { x.Common.DeviceID = x.ExpertAddition.DeviceID } else { x.ExpertAddition.DeviceID = x.Common.DeviceID op.MustSaveDriverStorage(x) } // 签名方法 if x.SignType == "captcha_sign" { x.Common.Timestamp = x.Timestamp x.Common.CaptchaSign = x.CaptchaSign } else { x.Common.Algorithms = strings.Split(x.Algorithms, ",") } // 登录方式 if x.LoginType == "refresh_token" { // 通过RefreshToken登录 token, err := x.XunLeiCommon.RefreshToken(x.ExpertAddition.RefreshToken) if err != nil { return err } x.SetTokenResp(token) // 刷新token方法 x.SetRefreshTokenFunc(func() error { token, err := x.XunLeiCommon.RefreshToken(x.TokenResp.RefreshToken) if err != nil { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) } x.SetTokenResp(token) op.MustSaveDriverStorage(x) return err }) } else { // 通过用户密码登录 token, err := x.Login(x.Username, x.Password) if err != nil { return err } // 清空 信任密钥 x.ExpertAddition.CreditKey = "" x.SetTokenResp(token) x.SetRefreshTokenFunc(func() error { token, err := x.XunLeiCommon.RefreshToken(x.TokenResp.RefreshToken) if err != nil { token, err = x.Login(x.Username, x.Password) if err != nil { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) } // 清空 信任密钥 x.ExpertAddition.CreditKey = "" } x.SetTokenResp(token) op.MustSaveDriverStorage(x) return err }) } } else { // 仅修改验证码token if x.CaptchaToken != "" { x.SetCaptchaToken(x.CaptchaToken) } x.XunLeiCommon.UserAgent = x.UserAgent x.XunLeiCommon.DownloadUserAgent = x.DownloadUserAgent x.XunLeiCommon.UseVideoUrl = x.UseVideoUrl } return nil } func (x *ThunderExpert) Drop(ctx context.Context) error { return nil } func (x *ThunderExpert) SetTokenResp(token *TokenResp) { x.XunLeiCommon.SetTokenResp(token) if token != nil { x.ExpertAddition.RefreshToken = token.RefreshToken } } type XunLeiCommon struct { *Common *TokenResp // 登录信息 *CoreLoginResp // core登录信息 refreshTokenFunc func() error } func (xc *XunLeiCommon) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { return xc.getFiles(ctx, dir.GetID()) } func (xc *XunLeiCommon) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var lFile Files _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", file.GetID()) r.SetQueryParam("space", xc.Space) }, &lFile) if err != nil { return nil, err } link := &model.Link{ URL: lFile.WebContentLink, Header: http.Header{ "User-Agent": {xc.DownloadUserAgent}, }, } if xc.UseVideoUrl { for _, media := range lFile.Medias { if media.Link.URL != "" { link.URL = media.Link.URL break } } } /* strs := regexp.MustCompile(`e=([0-9]*)`).FindStringSubmatch(lFile.WebContentLink) if len(strs) == 2 { timestamp, err := strconv.ParseInt(strs[1], 10, 64) if err == nil { expired := time.Duration(timestamp-time.Now().Unix()) * time.Second link.Expiration = &expired } } */ return link, nil } func (xc *XunLeiCommon) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ "kind": FOLDER, "name": dirName, "parent_id": parentDir.GetID(), "space": xc.Space, }) }, nil) return err } func (xc *XunLeiCommon) Move(ctx context.Context, srcObj, dstDir model.Obj) error { _, err := xc.Request(FILE_API_URL+":batchMove", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ "to": base.Json{"parent_id": dstDir.GetID()}, "ids": []string{srcObj.GetID()}, "space": xc.Space, }) }, nil) return err } func (xc *XunLeiCommon) Rename(ctx context.Context, srcObj model.Obj, newName string) error { _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodPatch, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", srcObj.GetID()) r.SetBody(&base.Json{ "name": newName, "space": xc.Space, }) }, nil) return err } func (xc *XunLeiCommon) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { _, err := xc.Request(FILE_API_URL+":batchCopy", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ "to": base.Json{"parent_id": dstDir.GetID()}, "ids": []string{srcObj.GetID()}, "space": xc.Space, }) }, nil) return err } func (xc *XunLeiCommon) Remove(ctx context.Context, obj model.Obj) error { _, err := xc.Request(FILE_API_URL+"/{fileID}/trash", http.MethodPatch, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", obj.GetID()) r.SetQueryParam("space", xc.Space) r.SetBody("{}") }, nil) return err } func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { gcid := file.GetHash().GetHash(hash_extend.GCID) var err error if len(gcid) < hash_extend.GCID.Width { _, gcid, err = stream.CacheFullAndHash(file, &up, hash_extend.GCID, file.GetSize()) if err != nil { return err } } var resp UploadTaskResponse _, err = xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ "kind": FILE, "parent_id": dstDir.GetID(), "name": file.GetName(), "size": file.GetSize(), "hash": gcid, "upload_type": UPLOAD_TYPE_RESUMABLE, "space": xc.Space, }) }, &resp) if err != nil { return err } param := resp.Resumable.Params if resp.UploadType == UPLOAD_TYPE_RESUMABLE { param.Endpoint = strings.TrimLeft(param.Endpoint, param.Bucket+".") s, err := session.NewSession(&aws.Config{ Credentials: credentials.NewStaticCredentials(param.AccessKeyID, param.AccessKeySecret, param.SecurityToken), Region: aws.String("xunlei"), Endpoint: aws.String(param.Endpoint), }) if err != nil { return err } uploader := s3manager.NewUploader(s) if file.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { uploader.PartSize = file.GetSize() / (s3manager.MaxUploadParts - 1) } _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ Bucket: aws.String(param.Bucket), Key: aws.String(param.Key), Expires: aws.Time(param.Expiration), Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: file, UpdateProgress: up, }), }) return err } return nil } func (xc *XunLeiCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { var about AboutResponse _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) }, &about) if err != nil { return nil, err } total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) if err != nil { return nil, err } used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: used, }, }, nil } func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string for { var fileList FileList _, err := xc.Request(FILE_API_URL, http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(map[string]string{ "space": xc.Space, "__type": "drive", "refresh": "true", "__sync": "true", "parent_id": folderId, "page_token": pageToken, "with_audit": "true", "limit": "100", "filters": `{"phase":{"eq":"PHASE_TYPE_COMPLETE"},"trashed":{"eq":false}}`, }) // 获取硬盘挂载目录等 if xc.Space != "" { r.SetQueryParamsFromValues(url.Values{ "with": []string{ "withCategoryDiskMountPath", "withCategoryDriveCachePath", "withCategoryHistoryDownloadPath", "withReadOnlyFS", }, }) } }, &fileList) if err != nil { return nil, err } for i := 0; i < len(fileList.Files); i++ { files = append(files, &fileList.Files[i]) } if fileList.NextPageToken == "" { break } pageToken = fileList.NextPageToken } return files, nil } // 设置刷新Token的方法 func (xc *XunLeiCommon) SetRefreshTokenFunc(fn func() error) { xc.refreshTokenFunc = fn } // 设置Token func (xc *XunLeiCommon) SetTokenResp(tr *TokenResp) { xc.TokenResp = tr } func (xc *XunLeiCommon) SetCoreTokenResp(tr *CoreLoginResp) { xc.CoreLoginResp = tr } // 携带Authorization和CaptchaToken的请求 func (xc *XunLeiCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { data, err := xc.Common.Request(url, method, func(req *resty.Request) { req.SetHeaders(map[string]string{ "Authorization": xc.Token(), "X-Captcha-Token": xc.GetCaptchaToken(), }) if callback != nil { callback(req) } }, resp) errResp, ok := err.(*ErrResp) if !ok { return nil, err } switch errResp.ErrorCode { case 0: return data, nil case 4122, 4121, 10, 16: if xc.refreshTokenFunc != nil { if err = xc.refreshTokenFunc(); err == nil { break } } return nil, err case 9: // 验证码token过期 if err = xc.RefreshCaptchaTokenAtLogin(GetAction(method, url), xc.TokenResp.UserID); err != nil { return nil, err } default: return nil, err } return xc.Request(url, method, callback, resp) } // 刷新Token func (xc *XunLeiCommon) RefreshToken(refreshToken string) (*TokenResp, error) { var resp TokenResp _, err := xc.Common.Request(XLUSER_API_URL+"/auth/token", http.MethodPost, func(req *resty.Request) { req.SetBody(&base.Json{ "grant_type": "refresh_token", "refresh_token": refreshToken, "client_id": xc.ClientID, "client_secret": xc.ClientSecret, }) }, &resp) if err != nil { return nil, err } if resp.RefreshToken == "" { return nil, errs.EmptyToken } return &resp, nil } // 登录 func (xc *XunLeiCommon) Login(username, password string) (*TokenResp, error) { //v3 login拿到 sessionID sessionID, err := xc.CoreLogin(username, password) if err != nil { return nil, err } //v1 login拿到令牌 url := XLUSER_API_URL + "/auth/signin/token" if err = xc.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), username); err != nil { return nil, err } var resp TokenResp _, err = xc.Common.Request(url, http.MethodPost, func(req *resty.Request) { req.SetPathParam("client_id", xc.ClientID) req.SetBody(&SignInRequest{ ClientID: xc.ClientID, ClientSecret: xc.ClientSecret, Provider: SignProvider, SigninToken: sessionID, }) }, &resp) if err != nil { return nil, err } return &resp, nil } func (xc *XunLeiCommon) IsLogin() bool { if xc.TokenResp == nil { return false } _, err := xc.Request(XLUSER_API_URL+"/user/me", http.MethodGet, nil, nil) return err == nil } // 离线下载文件 func (xc *XunLeiCommon) OfflineDownload(ctx context.Context, fileUrl string, parentDir model.Obj, fileName string) (*OfflineTask, error) { var resp OfflineDownloadResp _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ "kind": FILE, "name": fileName, "parent_id": parentDir.GetID(), "upload_type": UPLOAD_TYPE_URL, "space": xc.Space, "url": base.Json{ "url": fileUrl, }, }) }, &resp) if err != nil { return nil, err } return &resp.Task, err } /* 获取离线下载任务列表 */ func (xc *XunLeiCommon) OfflineList(ctx context.Context, nextPageToken string) ([]OfflineTask, error) { res := make([]OfflineTask, 0) var resp OfflineListResp _, err := xc.Request(TASK_API_URL, http.MethodGet, func(req *resty.Request) { req.SetContext(ctx). SetQueryParams(map[string]string{ "type": "offline", "limit": "10000", "page_token": nextPageToken, "space": xc.Space, }) }, &resp) if err != nil { return nil, fmt.Errorf("failed to get offline list: %w", err) } res = append(res, resp.Tasks...) return res, nil } func (xc *XunLeiCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error { _, err := xc.Request(TASK_API_URL, http.MethodDelete, func(req *resty.Request) { req.SetContext(ctx). SetQueryParams(map[string]string{ "task_ids": strings.Join(taskIDs, ","), "delete_files": strconv.FormatBool(deleteFiles), "space": xc.Space, }) }, nil) if err != nil { return fmt.Errorf("failed to delete tasks %v: %w", taskIDs, err) } return nil } func (xc *XunLeiCommon) CoreLogin(username string, password string) (sessionID string, err error) { url := XLUSER_API_BASE_URL + "/xluser.core.login/v3/login" var resp CoreLoginResp res, err := xc.Common.Request(url, http.MethodPost, func(req *resty.Request) { req.SetHeader("User-Agent", "android-ok-http-client/xl-acc-sdk/version-5.0.12.512000") req.SetBody(&CoreLoginRequest{ ProtocolVersion: "301", SequenceNo: "1000012", PlatformVersion: "10", IsCompressed: "0", Appid: APPID, ClientVersion: "8.31.0.9726", PeerID: "00000000000000000000000000000000", AppName: "ANDROID-com.xunlei.downloadprovider", SdkVersion: "512000", Devicesign: generateDeviceSign(xc.DeviceID, xc.PackageName), NetWorkType: "WIFI", ProviderName: "NONE", DeviceModel: "M2004J7AC", DeviceName: "Xiaomi_M2004j7ac", OSVersion: "12", Creditkey: xc.GetCreditKey(), Hl: "zh-CN", UserName: username, PassWord: password, VerifyKey: "", VerifyCode: "", IsMd5Pwd: "0", }) }, nil) if err != nil { return "", err } if err = utils.Json.Unmarshal(res, &resp); err != nil { return "", err } xc.SetCoreTokenResp(&resp) sessionID = resp.SessionID return sessionID, nil } ================================================ FILE: drivers/thunder/meta.go ================================================ package thunder import ( "crypto/md5" "encoding/hex" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) // 高级设置 type ExpertAddition struct { driver.RootID LoginType string `json:"login_type" type:"select" options:"user,refresh_token" default:"user"` SignType string `json:"sign_type" type:"select" options:"algorithms,captcha_sign" default:"algorithms"` // 登录方式1 Username string `json:"username" required:"true" help:"login type is user,this is required"` Password string `json:"password" required:"true" help:"login type is user,this is required"` // 登录方式2 RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` // 签名方法1 Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"9uJNVj/wLmdwKrJaVj/omlQ,Oz64Lp0GigmChHMf/6TNfxx7O9PyopcczMsnf,Eb+L7Ce+Ej48u,jKY0,ASr0zCl6v8W4aidjPK5KHd1Lq3t+vBFf41dqv5+fnOd,wQlozdg6r1qxh0eRmt3QgNXOvSZO6q/GXK,gmirk+ciAvIgA/cxUUCema47jr/YToixTT+Q6O,5IiCoM9B1/788ntB,P07JH0h6qoM6TSUAK2aL9T5s2QBVeY9JWvalf,+oK0AN"` // 签名方法2 CaptchaSign string `json:"captcha_sign" required:"true" help:"sign type is captcha_sign,this is required"` Timestamp string `json:"timestamp" required:"true" help:"sign type is captcha_sign,this is required"` // 验证码 CaptchaToken string `json:"captcha_token"` // 信任密钥 CreditKey string `json:"credit_key" help:"credit key,used for login"` // 必要且影响登录,由签名决定 DeviceID string `json:"device_id" default:""` ClientID string `json:"client_id" required:"true" default:"Xp6vsxz_7IYVw2BB"` ClientSecret string `json:"client_secret" required:"true" default:"Xp6vsy4tN9toTVdMSpomVdXpRmES"` ClientVersion string `json:"client_version" required:"true" default:"8.31.0.9726"` PackageName string `json:"package_name" required:"true" default:"com.xunlei.downloadprovider"` //不影响登录,影响下载速度 UserAgent string `json:"user_agent" required:"true" default:"ANDROID-com.xunlei.downloadprovider/8.31.0.9726 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/512000 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)"` DownloadUserAgent string `json:"download_user_agent" required:"true" default:"Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)"` //优先使用视频链接代替下载链接 UseVideoUrl bool `json:"use_video_url"` Space string `json:"space" default:"" help:"device id for remote device"` } // 登录特征,用于判断是否重新登录 func (i *ExpertAddition) GetIdentity() string { hash := md5.New() if i.LoginType == "refresh_token" { hash.Write([]byte(i.RefreshToken)) } else { hash.Write([]byte(i.Username + i.Password)) } if i.SignType == "captcha_sign" { hash.Write([]byte(i.CaptchaSign + i.Timestamp)) } else { hash.Write([]byte(i.Algorithms)) } hash.Write([]byte(i.DeviceID)) hash.Write([]byte(i.ClientID)) hash.Write([]byte(i.ClientSecret)) hash.Write([]byte(i.ClientVersion)) hash.Write([]byte(i.PackageName)) return hex.EncodeToString(hash.Sum(nil)) } type Addition struct { driver.RootID Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` CaptchaToken string `json:"captcha_token"` // 信任密钥 CreditKey string `json:"credit_key" help:"credit key,used for login"` // 登录设备ID DeviceID string `json:"device_id" default:""` Space string `json:"space" default:"" help:"device id for remote device"` } // 登录特征,用于判断是否重新登录 func (i *Addition) GetIdentity() string { return utils.GetMD5EncodeStr(i.Username + i.Password) } var config = driver.Config{ Name: "Thunder", LocalSort: true, } var configExpert = driver.Config{ Name: "ThunderExpert", LocalSort: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Thunder{} }) op.RegisterDriver(func() driver.Driver { return &ThunderExpert{} }) } ================================================ FILE: drivers/thunder/types.go ================================================ package thunder import ( "fmt" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" ) type ErrResp struct { ErrorCode int64 `json:"error_code"` ErrorMsg string `json:"error"` ErrorDescription string `json:"error_description"` // ErrorDetails interface{} `json:"error_details"` } func (e *ErrResp) IsError() bool { if e.ErrorMsg == "success" { return false } return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" } func (e *ErrResp) Error() string { return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ErrorDescription: %s ", e.ErrorCode, e.ErrorMsg, e.ErrorDescription) } /* * 验证码Token **/ type CaptchaTokenRequest struct { Action string `json:"action"` CaptchaToken string `json:"captcha_token"` ClientID string `json:"client_id"` DeviceID string `json:"device_id"` Meta map[string]string `json:"meta"` RedirectUri string `json:"redirect_uri"` } type CaptchaTokenResponse struct { CaptchaToken string `json:"captcha_token"` ExpiresIn int64 `json:"expires_in"` Url string `json:"url"` } /* * 登录 **/ type TokenResp struct { TokenType string `json:"token_type"` AccessToken string `json:"access_token"` RefreshToken string `json:"refresh_token"` ExpiresIn int64 `json:"expires_in"` Sub string `json:"sub"` UserID string `json:"user_id"` } func (t *TokenResp) Token() string { return fmt.Sprint(t.TokenType, " ", t.AccessToken) } type SignInRequest struct { ClientID string `json:"client_id"` ClientSecret string `json:"client_secret"` Provider string `json:"provider"` SigninToken string `json:"signin_token"` } type CoreLoginRequest struct { ProtocolVersion string `json:"protocolVersion"` SequenceNo string `json:"sequenceNo"` PlatformVersion string `json:"platformVersion"` IsCompressed string `json:"isCompressed"` Appid string `json:"appid"` ClientVersion string `json:"clientVersion"` PeerID string `json:"peerID"` AppName string `json:"appName"` SdkVersion string `json:"sdkVersion"` Devicesign string `json:"devicesign"` NetWorkType string `json:"netWorkType"` ProviderName string `json:"providerName"` DeviceModel string `json:"deviceModel"` DeviceName string `json:"deviceName"` OSVersion string `json:"OSVersion"` Creditkey string `json:"creditkey"` Hl string `json:"hl"` UserName string `json:"userName"` PassWord string `json:"passWord"` VerifyKey string `json:"verifyKey"` VerifyCode string `json:"verifyCode"` IsMd5Pwd string `json:"isMd5Pwd"` } type CoreLoginResp struct { Account string `json:"account"` Creditkey string `json:"creditkey"` /* Error string `json:"error"` ErrorCode string `json:"errorCode"` ErrorDescription string `json:"error_description"`*/ ExpiresIn int `json:"expires_in"` IsCompressed string `json:"isCompressed"` IsSetPassWord string `json:"isSetPassWord"` KeepAliveMinPeriod string `json:"keepAliveMinPeriod"` KeepAlivePeriod string `json:"keepAlivePeriod"` LoginKey string `json:"loginKey"` NickName string `json:"nickName"` PlatformVersion string `json:"platformVersion"` ProtocolVersion string `json:"protocolVersion"` SecureKey string `json:"secureKey"` SequenceNo string `json:"sequenceNo"` SessionID string `json:"sessionID"` Timestamp string `json:"timestamp"` UserID string `json:"userID"` UserName string `json:"userName"` UserNewNo string `json:"userNewNo"` Version string `json:"version"` /* VipList []struct { ExpireDate string `json:"expireDate"` IsAutoDeduct string `json:"isAutoDeduct"` IsVip string `json:"isVip"` IsYear string `json:"isYear"` PayID string `json:"payId"` PayName string `json:"payName"` Register string `json:"register"` Vasid string `json:"vasid"` VasType string `json:"vasType"` VipDayGrow string `json:"vipDayGrow"` VipGrow string `json:"vipGrow"` VipLevel string `json:"vipLevel"` Icon struct { General string `json:"general"` Small string `json:"small"` } `json:"icon"` } `json:"vipList"`*/ } /* * 文件 **/ type FileList struct { Kind string `json:"kind"` NextPageToken string `json:"next_page_token"` Files []Files `json:"files"` Version string `json:"version"` VersionOutdated bool `json:"version_outdated"` } type Link struct { URL string `json:"url"` Token string `json:"token"` Expire time.Time `json:"expire"` Type string `json:"type"` } var _ model.Obj = (*Files)(nil) type Files struct { Kind string `json:"kind"` ID string `json:"id"` ParentID string `json:"parent_id"` Name string `json:"name"` //UserID string `json:"user_id"` Size string `json:"size"` //Revision string `json:"revision"` //FileExtension string `json:"file_extension"` //MimeType string `json:"mime_type"` //Starred bool `json:"starred"` WebContentLink string `json:"web_content_link"` CreatedTime time.Time `json:"created_time"` ModifiedTime time.Time `json:"modified_time"` IconLink string `json:"icon_link"` ThumbnailLink string `json:"thumbnail_link"` // Md5Checksum string `json:"md5_checksum"` Hash string `json:"hash"` // Links map[string]Link `json:"links"` // Phase string `json:"phase"` // Audit struct { // Status string `json:"status"` // Message string `json:"message"` // Title string `json:"title"` // } `json:"audit"` Medias []struct { //Category string `json:"category"` //IconLink string `json:"icon_link"` //IsDefault bool `json:"is_default"` //IsOrigin bool `json:"is_origin"` //IsVisible bool `json:"is_visible"` Link Link `json:"link"` //MediaID string `json:"media_id"` //MediaName string `json:"media_name"` //NeedMoreQuota bool `json:"need_more_quota"` //Priority int `json:"priority"` //RedirectLink string `json:"redirect_link"` //ResolutionName string `json:"resolution_name"` // Video struct { // AudioCodec string `json:"audio_codec"` // BitRate int `json:"bit_rate"` // Duration int `json:"duration"` // FrameRate int `json:"frame_rate"` // Height int `json:"height"` // VideoCodec string `json:"video_codec"` // VideoType string `json:"video_type"` // Width int `json:"width"` // } `json:"video"` // VipTypes []string `json:"vip_types"` } `json:"medias"` Trashed bool `json:"trashed"` DeleteTime string `json:"delete_time"` OriginalURL string `json:"original_url"` //Params struct{} `json:"params"` //OriginalFileIndex int `json:"original_file_index"` //Space string `json:"space"` //Apps []interface{} `json:"apps"` //Writable bool `json:"writable"` //FolderType string `json:"folder_type"` //Collection interface{} `json:"collection"` } func (c *Files) GetHash() utils.HashInfo { return utils.NewHashInfo(hash_extend.GCID, c.Hash) } func (c *Files) GetSize() int64 { size, _ := strconv.ParseInt(c.Size, 10, 64); return size } func (c *Files) GetName() string { return c.Name } func (c *Files) CreateTime() time.Time { return c.CreatedTime } func (c *Files) ModTime() time.Time { return c.ModifiedTime } func (c *Files) IsDir() bool { return c.Kind == FOLDER } func (c *Files) GetID() string { return c.ID } func (c *Files) GetPath() string { return "" } func (c *Files) Thumb() string { return c.ThumbnailLink } /* * 上传 **/ type UploadTaskResponse struct { UploadType string `json:"upload_type"` /*//UPLOAD_TYPE_FORM Form struct { //Headers struct{} `json:"headers"` Kind string `json:"kind"` Method string `json:"method"` MultiParts struct { OSSAccessKeyID string `json:"OSSAccessKeyId"` Signature string `json:"Signature"` Callback string `json:"callback"` Key string `json:"key"` Policy string `json:"policy"` XUserData string `json:"x:user_data"` } `json:"multi_parts"` URL string `json:"url"` } `json:"form"`*/ //UPLOAD_TYPE_RESUMABLE Resumable struct { Kind string `json:"kind"` Params struct { AccessKeyID string `json:"access_key_id"` AccessKeySecret string `json:"access_key_secret"` Bucket string `json:"bucket"` Endpoint string `json:"endpoint"` Expiration time.Time `json:"expiration"` Key string `json:"key"` SecurityToken string `json:"security_token"` } `json:"params"` Provider string `json:"provider"` } `json:"resumable"` File Files `json:"file"` } // 添加离线下载响应 type OfflineDownloadResp struct { File *string `json:"file"` Task OfflineTask `json:"task"` UploadType string `json:"upload_type"` URL struct { Kind string `json:"kind"` } `json:"url"` } // 离线下载列表 type OfflineListResp struct { ExpiresIn int64 `json:"expires_in"` NextPageToken string `json:"next_page_token"` Tasks []OfflineTask `json:"tasks"` } // offlineTask type OfflineTask struct { Callback string `json:"callback"` CreatedTime string `json:"created_time"` FileID string `json:"file_id"` FileName string `json:"file_name"` FileSize string `json:"file_size"` IconLink string `json:"icon_link"` ID string `json:"id"` Kind string `json:"kind"` Message string `json:"message"` Name string `json:"name"` Params Params `json:"params"` Phase string `json:"phase"` // PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING Progress int64 `json:"progress"` Space string `json:"space"` StatusSize int64 `json:"status_size"` Statuses []string `json:"statuses"` ThirdTaskID string `json:"third_task_id"` Type string `json:"type"` UpdatedTime string `json:"updated_time"` UserID string `json:"user_id"` } type Params struct { FolderType string `json:"folder_type"` PredictSpeed string `json:"predict_speed"` PredictType string `json:"predict_type"` } // LoginReviewResp 登录验证响应 type LoginReviewResp struct { Creditkey string `json:"creditkey"` Error string `json:"error"` ErrorCode string `json:"errorCode"` ErrorDesc string `json:"errorDesc"` ErrorDescURL string `json:"errorDescUrl"` ErrorIsRetry int `json:"errorIsRetry"` ErrorDescription string `json:"error_description"` IsCompressed string `json:"isCompressed"` PlatformVersion string `json:"platformVersion"` ProtocolVersion string `json:"protocolVersion"` Reviewurl string `json:"reviewurl"` SequenceNo string `json:"sequenceNo"` UserID string `json:"userID"` VerifyType string `json:"verifyType"` } // ReviewData 验证数据 type ReviewData struct { Creditkey string `json:"creditkey"` Reviewurl string `json:"reviewurl"` Deviceid string `json:"deviceid"` Devicesign string `json:"devicesign"` } type AboutResponse struct { // Kind string `json:"kind"` Quota struct { // Kind string `json:"kind"` Limit string `json:"limit"` Usage string `json:"usage"` // UsageInTrash string `json:"usage_in_trash"` // PlayTimesLimit string `json:"play_times_limit"` // PlayTimesUsage string `json:"play_times_usage"` // IsUnlimited bool `json:"is_unlimited"` // UpgradeType string `json:"upgrade_type"` } `json:"quota"` // ExpiresAt string `json:"expires_at"` // Quotas struct { // } `json:"quotas"` // IsSearchFlushed bool `json:"is_search_flushed"` } ================================================ FILE: drivers/thunder/util.go ================================================ package thunder import ( "crypto/md5" "crypto/sha1" "encoding/hex" "encoding/json" "fmt" "io" "net/http" "regexp" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) const ( API_URL = "https://api-pan.xunlei.com/drive/v1" FILE_API_URL = API_URL + "/files" TASK_API_URL = API_URL + "/tasks" XLUSER_API_BASE_URL = "https://xluser-ssl.xunlei.com" XLUSER_API_URL = XLUSER_API_BASE_URL + "/v1" ) const ( FOLDER = "drive#folder" FILE = "drive#file" RESUMABLE = "drive#resumable" ) const ( UPLOAD_TYPE_UNKNOWN = "UPLOAD_TYPE_UNKNOWN" //UPLOAD_TYPE_FORM = "UPLOAD_TYPE_FORM" UPLOAD_TYPE_RESUMABLE = "UPLOAD_TYPE_RESUMABLE" UPLOAD_TYPE_URL = "UPLOAD_TYPE_URL" ) const ( SignProvider = "access_end_point_token" APPID = "40" APPKey = "34a062aaa22f906fca4fefe9fb3a3021" ) func GetAction(method string, url string) string { urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] return method + ":" + urlpath } type Common struct { client *resty.Client captchaToken string creditKey string // 签名相关,二选一 Algorithms []string Timestamp, CaptchaSign string // 必要值,签名相关 DeviceID string ClientID string ClientSecret string ClientVersion string PackageName string UserAgent string DownloadUserAgent string UseVideoUrl bool Space string // 验证码token刷新成功回调 refreshCTokenCk func(token string) } func (c *Common) SetCaptchaToken(captchaToken string) { c.captchaToken = captchaToken } func (c *Common) GetCaptchaToken() string { return c.captchaToken } func (c *Common) SetCreditKey(creditKey string) { c.creditKey = creditKey } func (c *Common) GetCreditKey() string { return c.creditKey } // 刷新验证码token(登录后) func (c *Common) RefreshCaptchaTokenAtLogin(action, userID string) error { metas := map[string]string{ "client_version": c.ClientVersion, "package_name": c.PackageName, "user_id": userID, } metas["timestamp"], metas["captcha_sign"] = c.GetCaptchaSign() return c.refreshCaptchaToken(action, metas) } // 刷新验证码token(登录时) func (c *Common) RefreshCaptchaTokenInLogin(action, username string) error { metas := make(map[string]string) if ok, _ := regexp.MatchString(`\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*`, username); ok { metas["email"] = username } else if len(username) >= 11 && len(username) <= 18 { metas["phone_number"] = username } else { metas["username"] = username } return c.refreshCaptchaToken(action, metas) } // 获取验证码签名 func (c *Common) GetCaptchaSign() (timestamp, sign string) { if len(c.Algorithms) == 0 { return c.Timestamp, c.CaptchaSign } timestamp = fmt.Sprint(time.Now().UnixMilli()) str := fmt.Sprint(c.ClientID, c.ClientVersion, c.PackageName, c.DeviceID, timestamp) for _, algorithm := range c.Algorithms { str = utils.GetMD5EncodeStr(str + algorithm) } sign = "1." + str return } // 刷新验证码token func (c *Common) refreshCaptchaToken(action string, metas map[string]string) error { param := CaptchaTokenRequest{ Action: action, CaptchaToken: c.captchaToken, ClientID: c.ClientID, DeviceID: c.DeviceID, Meta: metas, RedirectUri: "xlaccsdk01://xunlei.com/callback?state=harbor", } var e ErrResp var resp CaptchaTokenResponse _, err := c.Request(XLUSER_API_URL+"/shield/captcha/init", http.MethodPost, func(req *resty.Request) { req.SetError(&e).SetBody(param) }, &resp) if err != nil { return err } if e.IsError() { return &e } if resp.Url != "" { return fmt.Errorf(`need verify: Click Here`, resp.Url) } if resp.CaptchaToken == "" { return fmt.Errorf("empty captchaToken") } if c.refreshCTokenCk != nil { c.refreshCTokenCk(resp.CaptchaToken) } c.SetCaptchaToken(resp.CaptchaToken) return nil } // 只有基础信息的请求 func (c *Common) Request(url, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := c.client.R().SetHeaders(map[string]string{ "user-agent": c.UserAgent, "accept": "application/json;charset=UTF-8", "x-device-id": c.DeviceID, "x-client-id": c.ClientID, "x-client-version": c.ClientVersion, }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } res, err := req.Execute(method, url) if err != nil { return nil, err } var erron ErrResp utils.Json.Unmarshal(res.Body(), &erron) if erron.IsError() { // review_panel 表示需要短信验证码进行验证 if erron.ErrorMsg == "review_panel" { return nil, c.getReviewData(res) } return nil, &erron } return res.Body(), nil } // 获取验证所需内容 func (c *Common) getReviewData(res *resty.Response) error { var reviewResp LoginReviewResp var reviewData ReviewData if err := utils.Json.Unmarshal(res.Body(), &reviewResp); err != nil { return err } deviceSign := generateDeviceSign(c.DeviceID, c.PackageName) reviewData = ReviewData{ Creditkey: reviewResp.Creditkey, Reviewurl: reviewResp.Reviewurl + "&deviceid=" + deviceSign, Deviceid: deviceSign, Devicesign: deviceSign, } // 将reviewData转为JSON字符串 reviewDataJSON, _ := json.MarshalIndent(reviewData, "", " ") //reviewDataJSON, _ := json.Marshal(reviewData) return fmt.Errorf(`
🔒 本次登录需要验证
This login requires verification

下面是验证所需要的数据，具体使用方法请参照对应的驱动文档
Below are the relevant verification data. For specific usage methods, please refer to the corresponding driver documentation.

%s
`, string(reviewDataJSON)) } // 计算文件Gcid func getGcid(r io.Reader, size int64) (string, error) { calcBlockSize := func(j int64) int64 { var psize int64 = 0x40000 for float64(j)/float64(psize) > 0x200 && psize < 0x200000 { psize = psize << 1 } return psize } hash1 := sha1.New() hash2 := sha1.New() readSize := calcBlockSize(size) for { hash2.Reset() if n, err := utils.CopyWithBufferN(hash2, r, readSize); err != nil && n == 0 { if err != io.EOF { return "", err } break } hash1.Write(hash2.Sum(nil)) } return hex.EncodeToString(hash1.Sum(nil)), nil } func generateDeviceSign(deviceID, packageName string) string { signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, APPID, APPKey) sha1Hash := sha1.New() sha1Hash.Write([]byte(signatureBase)) sha1Result := sha1Hash.Sum(nil) sha1String := hex.EncodeToString(sha1Result) md5Hash := md5.New() md5Hash.Write([]byte(sha1String)) md5Result := md5Hash.Sum(nil) md5String := hex.EncodeToString(md5Result) deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) return deviceSign } ================================================ FILE: drivers/thunder_browser/driver.go ================================================ package thunder_browser import ( "context" "errors" "fmt" "io" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" ) type ThunderBrowser struct { *XunLeiBrowserCommon model.Storage Addition identity string } func (x *ThunderBrowser) Config() driver.Config { return config } func (x *ThunderBrowser) GetAddition() driver.Additional { return &x.Addition } func (x *ThunderBrowser) Init(ctx context.Context) (err error) { spaceTokenFunc := func() error { // 如果用户未设置 "超级保险柜" 密码 则直接返回 if x.SafePassword == "" { return nil } // 通过 GetSafeAccessToken 获取 token, err := x.GetSafeAccessToken(x.SafePassword) x.SetSpaceTokenResp(token) return err } // 初始化所需参数 if x.XunLeiBrowserCommon == nil { x.XunLeiBrowserCommon = &XunLeiBrowserCommon{ Common: &Common{ client: base.NewRestyClient(), Algorithms: Algorithms, DeviceID: utils.GetMD5EncodeStr(x.Username + x.Password), ClientID: ClientID, ClientSecret: ClientSecret, ClientVersion: ClientVersion, PackageName: PackageName, UserAgent: BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), PackageName, SdkVersion, ClientVersion, PackageName), DownloadUserAgent: DownloadUserAgent, UseVideoUrl: x.UseVideoUrl, UseFluentPlay: x.UseFluentPlay, RemoveWay: x.Addition.RemoveWay, refreshCTokenCk: func(token string) { x.CaptchaToken = token op.MustSaveDriverStorage(x) }, }, refreshTokenFunc: func() error { // 通过RefreshToken刷新 token, err := x.RefreshToken(x.TokenResp.RefreshToken) if err != nil { // 重新登录 token, err = x.Login(x.Username, x.Password) if err != nil { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) op.MustSaveDriverStorage(x) } // 清空 信任密钥 x.Addition.CreditKey = "" } x.SetTokenResp(token) return err }, } } // 自定义验证码token ctoekn := strings.TrimSpace(x.CaptchaToken) if ctoekn != "" { x.SetCaptchaToken(ctoekn) } if x.Addition.CreditKey != "" { x.SetCreditKey(x.Addition.CreditKey) } if x.Addition.DeviceID != "" { x.Common.DeviceID = x.Addition.DeviceID } else { x.Addition.DeviceID = x.Common.DeviceID op.MustSaveDriverStorage(x) } x.XunLeiBrowserCommon.UseVideoUrl = x.UseVideoUrl x.XunLeiBrowserCommon.UseFluentPlay = x.UseFluentPlay x.Addition.RootFolderID = x.RootFolderID // 防止重复登录 identity := x.GetIdentity() if x.identity != identity || !x.IsLogin() { x.identity = identity // 登录 token, err := x.Login(x.Username, x.Password) if err != nil { return err } // 清空 信任密钥 x.Addition.CreditKey = "" x.SetTokenResp(token) } // 获取 spaceToken err = spaceTokenFunc() if err != nil { return err } return nil } func (x *ThunderBrowser) Drop(ctx context.Context) error { return nil } type ThunderBrowserExpert struct { *XunLeiBrowserCommon model.Storage ExpertAddition identity string } func (x *ThunderBrowserExpert) Config() driver.Config { return configExpert } func (x *ThunderBrowserExpert) GetAddition() driver.Additional { return &x.ExpertAddition } func (x *ThunderBrowserExpert) Init(ctx context.Context) (err error) { spaceTokenFunc := func() error { // 如果用户未设置 "超级保险柜" 密码 则直接返回 if x.SafePassword == "" { return nil } // 通过 GetSafeAccessToken 获取 token, err := x.GetSafeAccessToken(x.SafePassword) x.SetSpaceTokenResp(token) return err } // 防止重复登录 identity := x.GetIdentity() if identity != x.identity || !x.IsLogin() { x.identity = identity x.XunLeiBrowserCommon = &XunLeiBrowserCommon{ Common: &Common{ client: base.NewRestyClient(), DeviceID: func() string { if len(x.DeviceID) != 32 { if x.LoginType == "user" { return utils.GetMD5EncodeStr(x.Username + x.Password) } return utils.GetMD5EncodeStr(x.ExpertAddition.RefreshToken) } return x.DeviceID }(), ClientID: x.ClientID, ClientSecret: x.ClientSecret, ClientVersion: x.ClientVersion, PackageName: x.PackageName, UserAgent: func() string { if x.ExpertAddition.UserAgent != "" { return x.ExpertAddition.UserAgent } if x.LoginType == "user" { return BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), x.PackageName, SdkVersion, x.ClientVersion, x.PackageName) } return BuildCustomUserAgent(utils.GetMD5EncodeStr(x.ExpertAddition.RefreshToken), x.PackageName, SdkVersion, x.ClientVersion, x.PackageName) }(), DownloadUserAgent: func() string { if x.ExpertAddition.DownloadUserAgent != "" { return x.ExpertAddition.DownloadUserAgent } return DownloadUserAgent }(), UseVideoUrl: x.UseVideoUrl, UseFluentPlay: x.UseFluentPlay, RemoveWay: x.ExpertAddition.RemoveWay, refreshCTokenCk: func(token string) { x.CaptchaToken = token op.MustSaveDriverStorage(x) }, }, } if x.ExpertAddition.CaptchaToken != "" { x.SetCaptchaToken(x.ExpertAddition.CaptchaToken) op.MustSaveDriverStorage(x) } if x.ExpertAddition.CreditKey != "" { x.SetCreditKey(x.ExpertAddition.CreditKey) } if x.ExpertAddition.DeviceID != "" { x.Common.DeviceID = x.ExpertAddition.DeviceID } else { x.ExpertAddition.DeviceID = x.Common.DeviceID op.MustSaveDriverStorage(x) } if x.Common.UserAgent != "" { x.ExpertAddition.UserAgent = x.Common.UserAgent op.MustSaveDriverStorage(x) } if x.Common.DownloadUserAgent != "" { x.ExpertAddition.DownloadUserAgent = x.Common.DownloadUserAgent op.MustSaveDriverStorage(x) } x.XunLeiBrowserCommon.UseVideoUrl = x.UseVideoUrl x.XunLeiBrowserCommon.UseFluentPlay = x.UseFluentPlay x.ExpertAddition.RootFolderID = x.RootFolderID // 签名方法 if x.SignType == "captcha_sign" { x.Common.Timestamp = x.Timestamp x.Common.CaptchaSign = x.CaptchaSign } else { x.Common.Algorithms = strings.Split(x.Algorithms, ",") } // 登录方式 if x.LoginType == "refresh_token" { // 通过RefreshToken登录 token, err := x.XunLeiBrowserCommon.RefreshToken(x.ExpertAddition.RefreshToken) if err != nil { return err } x.SetTokenResp(token) // 刷新token方法 x.SetRefreshTokenFunc(func() error { token, err := x.XunLeiBrowserCommon.RefreshToken(x.TokenResp.RefreshToken) if err != nil { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) } x.SetTokenResp(token) op.MustSaveDriverStorage(x) return err }) err = spaceTokenFunc() if err != nil { return err } } else { // 通过用户密码登录 token, err := x.Login(x.Username, x.Password) if err != nil { return err } // 清空 信任密钥 x.ExpertAddition.CreditKey = "" x.SetTokenResp(token) x.SetRefreshTokenFunc(func() error { token, err := x.XunLeiBrowserCommon.RefreshToken(x.TokenResp.RefreshToken) if err != nil { token, err = x.Login(x.Username, x.Password) if err != nil { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) } // 清空 信任密钥 x.ExpertAddition.CreditKey = "" } x.SetTokenResp(token) op.MustSaveDriverStorage(x) return err }) err = spaceTokenFunc() if err != nil { return err } } } else { // 仅修改验证码token if x.CaptchaToken != "" { x.SetCaptchaToken(x.CaptchaToken) } err = spaceTokenFunc() if err != nil { return err } x.XunLeiBrowserCommon.UserAgent = x.UserAgent x.XunLeiBrowserCommon.DownloadUserAgent = x.DownloadUserAgent x.XunLeiBrowserCommon.UseVideoUrl = x.UseVideoUrl x.XunLeiBrowserCommon.UseFluentPlay = x.UseFluentPlay x.ExpertAddition.RootFolderID = x.RootFolderID } return nil } func (x *ThunderBrowserExpert) Drop(ctx context.Context) error { return nil } func (x *ThunderBrowserExpert) SetTokenResp(token *TokenResp) { x.XunLeiBrowserCommon.SetTokenResp(token) if token != nil { x.ExpertAddition.RefreshToken = token.RefreshToken } } type XunLeiBrowserCommon struct { *Common *TokenResp // 登录信息 *CoreLoginResp // core登录信息 refreshTokenFunc func() error } func (xc *XunLeiBrowserCommon) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { return xc.getFiles(ctx, dir, args.ReqPath) } func (xc *XunLeiBrowserCommon) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var lFile Files params := map[string]string{ "_magic": "2021", "space": file.(*Files).GetSpace(), "thumbnail_size": "SIZE_LARGE", "with": "url", } _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", file.GetID()) r.SetQueryParams(params) //r.SetQueryParam("space", "") }, &lFile) if err != nil { return nil, err } link := &model.Link{ URL: lFile.WebContentLink, Header: http.Header{ "User-Agent": {xc.DownloadUserAgent}, }, } if xc.UseVideoUrl { for _, media := range lFile.Medias { if media.Link.URL != "" { link.URL = media.Link.URL break } } } return link, nil } func (xc *XunLeiBrowserCommon) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { js := base.Json{ "kind": FOLDER, "name": dirName, "parent_id": parentDir.GetID(), "space": parentDir.(*Files).GetSpace(), } _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&js) }, nil) return err } func (xc *XunLeiBrowserCommon) Move(ctx context.Context, srcObj, dstDir model.Obj) error { params := map[string]string{ "_from": srcObj.(*Files).GetSpace(), } js := base.Json{ "to": base.Json{"parent_id": dstDir.GetID(), "space": dstDir.(*Files).GetSpace()}, "space": srcObj.(*Files).GetSpace(), "ids": []string{srcObj.GetID()}, } _, err := xc.Request(FILE_API_URL+":batchMove", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&js) r.SetQueryParams(params) }, nil) return err } func (xc *XunLeiBrowserCommon) Rename(ctx context.Context, srcObj model.Obj, newName string) error { params := map[string]string{ "space": srcObj.(*Files).GetSpace(), } _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodPatch, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", srcObj.GetID()) r.SetBody(&base.Json{"name": newName}) r.SetQueryParams(params) }, nil) return err } func (xc *XunLeiBrowserCommon) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { params := map[string]string{ "_from": srcObj.(*Files).GetSpace(), } js := base.Json{ "to": base.Json{"parent_id": dstDir.GetID(), "space": dstDir.(*Files).GetSpace()}, "space": srcObj.(*Files).GetSpace(), "ids": []string{srcObj.GetID()}, } _, err := xc.Request(FILE_API_URL+":batchCopy", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&js) r.SetQueryParams(params) }, nil) return err } func (xc *XunLeiBrowserCommon) Remove(ctx context.Context, obj model.Obj) error { js := base.Json{ "ids": []string{obj.GetID()}, "space": obj.(*Files).GetSpace(), } // 先判断是否是特殊情况 if obj.(*Files).GetSpace() == ThunderDriveSpace { _, err := xc.Request(FILE_API_URL+"/{fileID}/trash", http.MethodPatch, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", obj.GetID()) r.SetBody("{}") }, nil) return err } else if obj.(*Files).GetSpace() == ThunderBrowserDriveSafeSpace || obj.(*Files).GetSpace() == ThunderDriveSafeSpace { _, err := xc.Request(FILE_API_URL+":batchDelete", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&js) }, nil) return err } // 根据用户选择的删除方式进行删除 if xc.RemoveWay == "delete" { _, err := xc.Request(FILE_API_URL+":batchDelete", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&js) }, nil) return err } else { _, err := xc.Request(FILE_API_URL+":batchTrash", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&js) }, nil) return err } } func (xc *XunLeiBrowserCommon) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { gcid := stream.GetHash().GetHash(hash_extend.GCID) var err error if len(gcid) < hash_extend.GCID.Width { _, gcid, err = streamPkg.CacheFullAndHash(stream, &up, hash_extend.GCID, stream.GetSize()) if err != nil { return err } } js := base.Json{ "kind": FILE, "parent_id": dstDir.GetID(), "name": stream.GetName(), "size": stream.GetSize(), "hash": gcid, "upload_type": UPLOAD_TYPE_RESUMABLE, "space": dstDir.(*Files).GetSpace(), } var resp UploadTaskResponse _, err = xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&js) }, &resp) if err != nil { return err } param := resp.Resumable.Params if resp.UploadType == UPLOAD_TYPE_RESUMABLE { param.Endpoint = strings.TrimLeft(param.Endpoint, param.Bucket+".") s, err := session.NewSession(&aws.Config{ Credentials: credentials.NewStaticCredentials(param.AccessKeyID, param.AccessKeySecret, param.SecurityToken), Region: aws.String("xunlei"), Endpoint: aws.String(param.Endpoint), }) if err != nil { return err } uploader := s3manager.NewUploader(s) if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) } _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ Bucket: aws.String(param.Bucket), Key: aws.String(param.Key), Expires: aws.Time(param.Expiration), Body: driver.NewLimitedUploadStream(ctx, io.TeeReader(stream, driver.NewProgress(stream.GetSize(), up))), }) return err } return nil } func (xc *XunLeiBrowserCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { var about AboutResponse _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) }, &about) if err != nil { return nil, err } total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) if err != nil { return nil, err } used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: used, }, }, nil } func (xc *XunLeiBrowserCommon) getFiles(ctx context.Context, dir model.Obj, path string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string for { var fileList FileList folderSpace := "" switch dirF := dir.(type) { case *Files: folderSpace = dirF.GetSpace() default: // 处理 根目录的情况 //folderSpace = ThunderBrowserDriveSpace folderSpace = ThunderDriveSpace // 迅雷浏览器已经合并到迅雷云盘，因此变更根目录 } params := map[string]string{ "parent_id": dir.GetID(), "page_token": pageToken, "space": folderSpace, "filters": `{"trashed":{"eq":false}}`, "with": "url", "with_audit": "true", "thumbnail_size": "SIZE_LARGE", } _, err := xc.Request(FILE_API_URL, http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(params) }, &fileList) if err != nil { return nil, err } for i := range fileList.Files { // 解决 "迅雷云盘" 重复出现问题————迅雷后端发送错误 if fileList.Files[i].FolderType == ThunderDriveFolderType && fileList.Files[i].ID == "" && fileList.Files[i].Space == "" && dir.GetID() != "" { continue } files = append(files, &fileList.Files[i]) } if fileList.NextPageToken == "" { break } pageToken = fileList.NextPageToken } return files, nil } // SetRefreshTokenFunc 设置刷新Token的方法 func (xc *XunLeiBrowserCommon) SetRefreshTokenFunc(fn func() error) { xc.refreshTokenFunc = fn } // SetTokenResp 设置Token func (xc *XunLeiBrowserCommon) SetTokenResp(tr *TokenResp) { xc.TokenResp = tr } // SetCoreTokenResp 设置CoreToken func (xc *XunLeiBrowserCommon) SetCoreTokenResp(tr *CoreLoginResp) { xc.CoreLoginResp = tr } // SetSpaceTokenResp 设置Token func (xc *XunLeiBrowserCommon) SetSpaceTokenResp(spaceToken string) { xc.TokenResp.Token = spaceToken } // Request 携带Authorization和CaptchaToken的请求 func (xc *XunLeiBrowserCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { data, err := xc.Common.Request(url, method, func(req *resty.Request) { req.SetHeaders(map[string]string{ "Authorization": xc.GetToken(), "X-Captcha-Token": xc.GetCaptchaToken(), "X-Space-Authorization": xc.GetSpaceToken(), }) if callback != nil { callback(req) } }, resp) errResp, ok := err.(*ErrResp) if !ok { return nil, err } switch errResp.ErrorCode { case 0: return data, nil case 4122, 4121, 10, 16: if xc.refreshTokenFunc != nil { if err = xc.refreshTokenFunc(); err == nil { break } } return nil, err case 9: // space_token 获取失败 if errResp.ErrorMsg == "space_token_invalid" { if token, err := xc.GetSafeAccessToken(xc.Token); err != nil { return nil, err } else { xc.SetSpaceTokenResp(token) } } if errResp.ErrorMsg == "captcha_invalid" { // 验证码token过期 if err = xc.RefreshCaptchaTokenAtLogin(GetAction(method, url), xc.TokenResp.UserID); err != nil { return nil, err } } return nil, errors.New(errResp.ErrorMsg) default: // 处理未捕获到的验证码错误 if errResp.ErrorMsg == "captcha_invalid" { // 验证码token过期 if err = xc.RefreshCaptchaTokenAtLogin(GetAction(method, url), xc.TokenResp.UserID); err != nil { return nil, err } } return nil, err } return xc.Request(url, method, callback, resp) } // RefreshToken 刷新Token func (xc *XunLeiBrowserCommon) RefreshToken(refreshToken string) (*TokenResp, error) { var resp TokenResp _, err := xc.Common.Request(XLUSER_API_URL+"/auth/token", http.MethodPost, func(req *resty.Request) { req.SetBody(&base.Json{ "grant_type": "refresh_token", "refresh_token": refreshToken, "client_id": xc.ClientID, "client_secret": xc.ClientSecret, }) }, &resp) if err != nil { return nil, err } if resp.RefreshToken == "" { return nil, errors.New("refresh token is empty") } return &resp, nil } // GetSafeAccessToken 获取 超级保险柜 AccessToken func (xc *XunLeiBrowserCommon) GetSafeAccessToken(safePassword string) (string, error) { var resp TokenResp _, err := xc.Request(XLUSER_API_URL+"/password/check", http.MethodPost, func(req *resty.Request) { req.SetBody(&base.Json{ "scene": "box", "password": EncryptPassword(safePassword), }) }, &resp) if err != nil { return "", err } if resp.Token == "" { return "", errors.New("SafePassword is incorrect ") } return resp.Token, nil } // Login 登录 func (xc *XunLeiBrowserCommon) Login(username, password string) (*TokenResp, error) { //v3 login拿到 sessionID sessionID, err := xc.CoreLogin(username, password) if err != nil { return nil, err } //v1 login拿到令牌 url := XLUSER_API_URL + "/auth/signin/token" if err = xc.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), username); err != nil { return nil, err } var resp TokenResp _, err = xc.Common.Request(url, http.MethodPost, func(req *resty.Request) { req.SetPathParam("client_id", xc.ClientID) req.SetBody(&SignInRequest{ ClientID: xc.ClientID, ClientSecret: xc.ClientSecret, Provider: SignProvider, SigninToken: sessionID, }) }, &resp) if err != nil { return nil, err } return &resp, nil } func (xc *XunLeiBrowserCommon) IsLogin() bool { if xc.TokenResp == nil { return false } _, err := xc.Request(XLUSER_API_URL+"/user/me", http.MethodGet, nil, nil) return err == nil } // OfflineDownload 离线下载文件 func (xc *XunLeiBrowserCommon) OfflineDownload(ctx context.Context, fileUrl string, parentDir model.Obj, fileName string) (*OfflineTask, error) { var resp OfflineDownloadResp body := base.Json{} from := "cloudadd/" if xc.UseFluentPlay { body = base.Json{ "kind": FILE, "name": fileName, // 流畅播接口 强制将文件放在 "SPACE_FAVORITE" 文件夹 //"parent_id": parentDir.GetID(), "upload_type": UPLOAD_TYPE_URL, "url": base.Json{ "url": fileUrl, //"files": []string{"0"}, // 0 表示只下载第一个文件 }, "params": base.Json{ "cookie": "null", "web_title": "", "lastSession": "", "flags": "9", "scene": "smart_spot_panel", "referer": "https://x.xunlei.com", "dedup_index": "0", }, "need_dedup": true, "folder_type": "FAVORITE", "space": ThunderBrowserDriveFluentPlayFolderType, } from = "FLUENT_PLAY/sniff_ball/fluent_play/SPACE_FAVORITE" } else { body = base.Json{ "kind": FILE, "name": fileName, "parent_id": parentDir.GetID(), "upload_type": UPLOAD_TYPE_URL, "url": base.Json{ "url": fileUrl, }, } if files, ok := parentDir.(*Files); ok { body["space"] = files.GetSpace() } else { // 如果不是 Files 类型，则默认使用 ThunderDriveSpace body["space"] = ThunderDriveSpace } } _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParam("_from", from) r.SetBody(&body) }, &resp) if err != nil { return nil, err } return &resp.Task, err } // OfflineList 获取离线下载任务列表 func (xc *XunLeiBrowserCommon) OfflineList(ctx context.Context, nextPageToken string) ([]OfflineTask, error) { res := make([]OfflineTask, 0) var resp OfflineListResp _, err := xc.Request(TASK_API_URL, http.MethodGet, func(req *resty.Request) { req.SetContext(ctx). SetQueryParams(map[string]string{ "type": "offline", "limit": "10000", "page_token": nextPageToken, "space": "default/*", }) }, &resp) if err != nil { return nil, fmt.Errorf("failed to get offline list: %w", err) } res = append(res, resp.Tasks...) return res, nil } func (xc *XunLeiBrowserCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string) error { queryParams := map[string]string{ "task_ids": strings.Join(taskIDs, ","), "_t": strconv.FormatInt(time.Now().UnixMilli(), 10), } if xc.UseFluentPlay { queryParams["space"] = ThunderBrowserDriveFluentPlayFolderType } _, err := xc.Request(TASK_API_URL, http.MethodDelete, func(req *resty.Request) { req.SetContext(ctx). SetQueryParams(queryParams) }, nil) if err != nil { return fmt.Errorf("failed to delete tasks %v: %w", taskIDs, err) } return nil } func (xc *XunLeiBrowserCommon) CoreLogin(username string, password string) (sessionID string, err error) { url := XLUSER_API_BASE_URL + "/xluser.core.login/v3/login" var resp CoreLoginResp res, err := xc.Common.Request(url, http.MethodPost, func(req *resty.Request) { req.SetHeader("User-Agent", "android-ok-http-client/xl-acc-sdk/version-5.0.9.509300") req.SetBody(&CoreLoginRequest{ ProtocolVersion: "301", SequenceNo: "1000010", PlatformVersion: "10", IsCompressed: "0", Appid: APPID, ClientVersion: xc.Common.ClientVersion, PeerID: "00000000000000000000000000000000", AppName: "ANDROID-com.xunlei.browser", SdkVersion: "509300", Devicesign: generateDeviceSign(xc.DeviceID, xc.PackageName), NetWorkType: "WIFI", ProviderName: "NONE", DeviceModel: "M2004J7AC", DeviceName: "Xiaomi_M2004j7ac", OSVersion: "12", Creditkey: xc.GetCreditKey(), Hl: "zh-CN", UserName: username, PassWord: password, VerifyKey: "", VerifyCode: "", IsMd5Pwd: "0", }) }, nil) if err != nil { return "", err } if err = utils.Json.Unmarshal(res, &resp); err != nil { return "", err } xc.SetCoreTokenResp(&resp) sessionID = resp.SessionID return sessionID, nil } ================================================ FILE: drivers/thunder_browser/meta.go ================================================ package thunder_browser import ( "crypto/md5" "encoding/hex" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) // ExpertAddition 高级设置 type ExpertAddition struct { driver.RootID LoginType string `json:"login_type" type:"select" options:"user,refresh_token" default:"user"` SignType string `json:"sign_type" type:"select" options:"algorithms,captcha_sign" default:"algorithms"` // 登录方式1 Username string `json:"username" required:"true" help:"login type is user,this is required"` Password string `json:"password" required:"true" help:"login type is user,this is required"` // 登录方式2 RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` SafePassword string `json:"safe_password" required:"true" help:"super safe password"` // 超级保险箱密码 // 签名方法1 Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"Cw4kArmKJ/aOiFTxnQ0ES+D4mbbrIUsFn,HIGg0Qfbpm5ThZ/RJfjoao4YwgT9/M,u/PUD,OlAm8tPkOF1qO5bXxRN2iFttuDldrg,FFIiM6sFhWhU7tIMVUKOF7CUv/KzgwwV8FE,yN,4m5mglrIHksI6wYdq,LXEfS7,T+p+C+F2yjgsUtiXWU/cMNYEtJI4pq7GofW,14BrGIEMXkbvFvZ49nDUfVCRcHYFOJ1BP1Y,kWIH3Row,RAmRTKNCjucPWC"` // 签名方法2 CaptchaSign string `json:"captcha_sign" required:"true" help:"sign type is captcha_sign,this is required"` Timestamp string `json:"timestamp" required:"true" help:"sign type is captcha_sign,this is required"` // 验证码 CaptchaToken string `json:"captcha_token"` // 信任密钥 CreditKey string `json:"credit_key" help:"credit key,used for login"` // 必要且影响登录,由签名决定 DeviceID string `json:"device_id" required:"false" default:""` ClientID string `json:"client_id" required:"true" default:"ZUBzD9J_XPXfn7f7"` ClientSecret string `json:"client_secret" required:"true" default:"yESVmHecEe6F0aou69vl-g"` ClientVersion string `json:"client_version" required:"true" default:"1.40.0.7208"` PackageName string `json:"package_name" required:"true" default:"com.xunlei.browser"` // 不影响登录,影响下载速度 UserAgent string `json:"user_agent" required:"false" default:""` DownloadUserAgent string `json:"download_user_agent" required:"false" default:""` // 优先使用视频链接代替下载链接 UseVideoUrl bool `json:"use_video_url"` // 离线下载是否使用 流畅播(Fluent Play)接口 UseFluentPlay bool `json:"use_fluent_play" default:"false" help:"use fluent play for offline download,only magnet links supported"` // 移除方式 RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` } // GetIdentity 登录特征,用于判断是否重新登录 func (i *ExpertAddition) GetIdentity() string { hash := md5.New() if i.LoginType == "refresh_token" { hash.Write([]byte(i.RefreshToken)) } else { hash.Write([]byte(i.Username + i.Password)) } if i.SignType == "captcha_sign" { hash.Write([]byte(i.CaptchaSign + i.Timestamp)) } else { hash.Write([]byte(i.Algorithms)) } hash.Write([]byte(i.DeviceID)) hash.Write([]byte(i.ClientID)) hash.Write([]byte(i.ClientSecret)) hash.Write([]byte(i.ClientVersion)) hash.Write([]byte(i.PackageName)) return hex.EncodeToString(hash.Sum(nil)) } type Addition struct { driver.RootID Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` SafePassword string `json:"safe_password" required:"true"` // 超级保险箱密码 CaptchaToken string `json:"captcha_token"` CreditKey string `json:"credit_key" help:"credit key,used for login"` // 信任密钥 DeviceID string `json:"device_id" default:""` // 登录设备ID UseVideoUrl bool `json:"use_video_url" default:"false"` // 离线下载是否使用 流畅播(Fluent Play)接口 UseFluentPlay bool `json:"use_fluent_play" default:"false" help:"use fluent play for offline download,only magnet links supported"` RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` } // GetIdentity 登录特征,用于判断是否重新登录 func (i *Addition) GetIdentity() string { return utils.GetMD5EncodeStr(i.Username + i.Password) } var config = driver.Config{ Name: "ThunderBrowser", LocalSort: true, } var configExpert = driver.Config{ Name: "ThunderBrowserExpert", LocalSort: true, } func init() { op.RegisterDriver(func() driver.Driver { return &ThunderBrowser{} }) op.RegisterDriver(func() driver.Driver { return &ThunderBrowserExpert{} }) } ================================================ FILE: drivers/thunder_browser/types.go ================================================ package thunder_browser import ( "fmt" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" ) type ErrResp struct { ErrorCode int64 `json:"error_code"` ErrorMsg string `json:"error"` ErrorDescription string `json:"error_description"` // ErrorDetails interface{} `json:"error_details"` } func (e *ErrResp) IsError() bool { if e.ErrorMsg == "success" { return false } return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" } func (e *ErrResp) Error() string { return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ErrorDescription: %s ", e.ErrorCode, e.ErrorMsg, e.ErrorDescription) } /* * 验证码Token **/ type CaptchaTokenRequest struct { Action string `json:"action"` CaptchaToken string `json:"captcha_token"` ClientID string `json:"client_id"` DeviceID string `json:"device_id"` Meta map[string]string `json:"meta"` RedirectUri string `json:"redirect_uri"` } type CaptchaTokenResponse struct { CaptchaToken string `json:"captcha_token"` ExpiresIn int64 `json:"expires_in"` Url string `json:"url"` } /* * 登录 **/ type TokenResp struct { TokenType string `json:"token_type"` AccessToken string `json:"access_token"` RefreshToken string `json:"refresh_token"` ExpiresIn int64 `json:"expires_in"` Sub string `json:"sub"` UserID string `json:"user_id"` Token string `json:"token"` // "超级保险箱" 访问Token } func (t *TokenResp) GetToken() string { return fmt.Sprint(t.TokenType, " ", t.AccessToken) } // GetSpaceToken 获取"超级保险箱" 访问Token func (t *TokenResp) GetSpaceToken() string { return t.Token } type SignInRequest struct { ClientID string `json:"client_id"` ClientSecret string `json:"client_secret"` Provider string `json:"provider"` SigninToken string `json:"signin_token"` } type CoreLoginRequest struct { ProtocolVersion string `json:"protocolVersion"` SequenceNo string `json:"sequenceNo"` PlatformVersion string `json:"platformVersion"` IsCompressed string `json:"isCompressed"` Appid string `json:"appid"` ClientVersion string `json:"clientVersion"` PeerID string `json:"peerID"` AppName string `json:"appName"` SdkVersion string `json:"sdkVersion"` Devicesign string `json:"devicesign"` NetWorkType string `json:"netWorkType"` ProviderName string `json:"providerName"` DeviceModel string `json:"deviceModel"` DeviceName string `json:"deviceName"` OSVersion string `json:"OSVersion"` Creditkey string `json:"creditkey"` Hl string `json:"hl"` UserName string `json:"userName"` PassWord string `json:"passWord"` VerifyKey string `json:"verifyKey"` VerifyCode string `json:"verifyCode"` IsMd5Pwd string `json:"isMd5Pwd"` } type CoreLoginResp struct { Account string `json:"account"` Creditkey string `json:"creditkey"` /* Error string `json:"error"` ErrorCode string `json:"errorCode"` ErrorDescription string `json:"error_description"`*/ ExpiresIn int `json:"expires_in"` IsCompressed string `json:"isCompressed"` IsSetPassWord string `json:"isSetPassWord"` KeepAliveMinPeriod string `json:"keepAliveMinPeriod"` KeepAlivePeriod string `json:"keepAlivePeriod"` LoginKey string `json:"loginKey"` NickName string `json:"nickName"` PlatformVersion string `json:"platformVersion"` ProtocolVersion string `json:"protocolVersion"` SecureKey string `json:"secureKey"` SequenceNo string `json:"sequenceNo"` SessionID string `json:"sessionID"` Timestamp string `json:"timestamp"` UserID string `json:"userID"` UserName string `json:"userName"` UserNewNo string `json:"userNewNo"` Version string `json:"version"` /* VipList []struct { ExpireDate string `json:"expireDate"` IsAutoDeduct string `json:"isAutoDeduct"` IsVip string `json:"isVip"` IsYear string `json:"isYear"` PayID string `json:"payId"` PayName string `json:"payName"` Register string `json:"register"` Vasid string `json:"vasid"` VasType string `json:"vasType"` VipDayGrow string `json:"vipDayGrow"` VipGrow string `json:"vipGrow"` VipLevel string `json:"vipLevel"` Icon struct { General string `json:"general"` Small string `json:"small"` } `json:"icon"` } `json:"vipList"`*/ } /* * 文件 **/ type FileList struct { Kind string `json:"kind"` NextPageToken string `json:"next_page_token"` Files []Files `json:"files"` Version string `json:"version"` VersionOutdated bool `json:"version_outdated"` FolderType int8 } type Link struct { URL string `json:"url"` Token string `json:"token"` Expire time.Time `json:"expire"` Type string `json:"type"` } var _ model.Obj = (*Files)(nil) type Files struct { Kind string `json:"kind"` ID string `json:"id"` ParentID string `json:"parent_id"` Name string `json:"name"` //UserID string `json:"user_id"` Size string `json:"size"` //Revision string `json:"revision"` //FileExtension string `json:"file_extension"` //MimeType string `json:"mime_type"` //Starred bool `json:"starred"` WebContentLink string `json:"web_content_link"` CreatedTime CustomTime `json:"created_time"` ModifiedTime CustomTime `json:"modified_time"` IconLink string `json:"icon_link"` ThumbnailLink string `json:"thumbnail_link"` Md5Checksum string `json:"md5_checksum"` Hash string `json:"hash"` // Links map[string]Link `json:"links"` // Phase string `json:"phase"` // Audit struct { // Status string `json:"status"` // Message string `json:"message"` // Title string `json:"title"` // } `json:"audit"` Medias []struct { //Category string `json:"category"` //IconLink string `json:"icon_link"` //IsDefault bool `json:"is_default"` //IsOrigin bool `json:"is_origin"` //IsVisible bool `json:"is_visible"` Link Link `json:"link"` //MediaID string `json:"media_id"` //MediaName string `json:"media_name"` //NeedMoreQuota bool `json:"need_more_quota"` //Priority int `json:"priority"` //RedirectLink string `json:"redirect_link"` //ResolutionName string `json:"resolution_name"` // Video struct { // AudioCodec string `json:"audio_codec"` // BitRate int `json:"bit_rate"` // Duration int `json:"duration"` // FrameRate int `json:"frame_rate"` // Height int `json:"height"` // VideoCodec string `json:"video_codec"` // VideoType string `json:"video_type"` // Width int `json:"width"` // } `json:"video"` // VipTypes []string `json:"vip_types"` } `json:"medias"` Trashed bool `json:"trashed"` DeleteTime string `json:"delete_time"` OriginalURL string `json:"original_url"` //Params struct{} `json:"params"` //OriginalFileIndex int `json:"original_file_index"` Space string `json:"space"` //Apps []interface{} `json:"apps"` //Writable bool `json:"writable"` FolderType string `json:"folder_type"` //Collection interface{} `json:"collection"` SortName string `json:"sort_name"` UserModifiedTime CustomTime `json:"user_modified_time"` //SpellName []interface{} `json:"spell_name"` //FileCategory string `json:"file_category"` //Tags []interface{} `json:"tags"` //ReferenceEvents []interface{} `json:"reference_events"` //ReferenceResource interface{} `json:"reference_resource"` //Params0 struct { // PlatformIcon string `json:"platform_icon"` // SmallThumbnail string `json:"small_thumbnail"` //} `json:"params,omitempty"` } func (c *Files) GetHash() utils.HashInfo { return utils.NewHashInfo(hash_extend.GCID, c.Hash) } func (c *Files) GetSize() int64 { size, _ := strconv.ParseInt(c.Size, 10, 64); return size } func (c *Files) GetName() string { return c.Name } func (c *Files) CreateTime() time.Time { return c.CreatedTime.Time } func (c *Files) ModTime() time.Time { return c.ModifiedTime.Time } func (c *Files) IsDir() bool { return c.Kind == FOLDER } func (c *Files) GetID() string { return c.ID } func (c *Files) GetPath() string { return "" } func (c *Files) Thumb() string { return c.ThumbnailLink } func (c *Files) GetSpace() string { if c.Space != "" { return c.Space } else { // "迅雷云盘" 文件夹内 Space 为空 return "" } } /* * 上传 **/ type UploadTaskResponse struct { UploadType string `json:"upload_type"` /*//UPLOAD_TYPE_FORM Form struct { //Headers struct{} `json:"headers"` Kind string `json:"kind"` Method string `json:"method"` MultiParts struct { OSSAccessKeyID string `json:"OSSAccessKeyId"` Signature string `json:"Signature"` Callback string `json:"callback"` Key string `json:"key"` Policy string `json:"policy"` XUserData string `json:"x:user_data"` } `json:"multi_parts"` URL string `json:"url"` } `json:"form"`*/ //UPLOAD_TYPE_RESUMABLE Resumable struct { Kind string `json:"kind"` Params struct { AccessKeyID string `json:"access_key_id"` AccessKeySecret string `json:"access_key_secret"` Bucket string `json:"bucket"` Endpoint string `json:"endpoint"` Expiration time.Time `json:"expiration"` Key string `json:"key"` SecurityToken string `json:"security_token"` } `json:"params"` Provider string `json:"provider"` } `json:"resumable"` File Files `json:"file"` } // OfflineDownloadResp 离线下载响应 type OfflineDownloadResp struct { File *string `json:"file"` Task OfflineTask `json:"task"` UploadType string `json:"upload_type"` URL struct { Kind string `json:"kind"` } `json:"url"` } // OfflineListResp 离线下载列表响应 type OfflineListResp struct { ExpiresIn int64 `json:"expires_in"` NextPageToken string `json:"next_page_token"` Tasks []OfflineTask `json:"tasks"` } // OfflineTask 离线下载任务响应 type OfflineTask struct { Callback string `json:"callback"` CreatedTime string `json:"created_time"` FileID string `json:"file_id"` FileName string `json:"file_name"` FileSize string `json:"file_size"` IconLink string `json:"icon_link"` ID string `json:"id"` Kind string `json:"kind"` Message string `json:"message"` Name string `json:"name"` Params Params `json:"params"` Phase string `json:"phase"` // PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING Progress int64 `json:"progress"` Space string `json:"space"` StatusSize int64 `json:"status_size"` Statuses []string `json:"statuses"` ThirdTaskID string `json:"third_task_id"` Type string `json:"type"` UpdatedTime string `json:"updated_time"` UserID string `json:"user_id"` } type Params struct { FolderType string `json:"folder_type"` PredictSpeed string `json:"predict_speed"` PredictType string `json:"predict_type"` } // LoginReviewResp 登录验证响应 type LoginReviewResp struct { Creditkey string `json:"creditkey"` Error string `json:"error"` ErrorCode string `json:"errorCode"` ErrorDesc string `json:"errorDesc"` ErrorDescURL string `json:"errorDescUrl"` ErrorIsRetry int `json:"errorIsRetry"` ErrorDescription string `json:"error_description"` IsCompressed string `json:"isCompressed"` PlatformVersion string `json:"platformVersion"` ProtocolVersion string `json:"protocolVersion"` Reviewurl string `json:"reviewurl"` SequenceNo string `json:"sequenceNo"` UserID string `json:"userID"` VerifyType string `json:"verifyType"` } // ReviewData 验证数据 type ReviewData struct { Creditkey string `json:"creditkey"` Reviewurl string `json:"reviewurl"` Deviceid string `json:"deviceid"` Devicesign string `json:"devicesign"` } type AboutResponse struct { Quota struct { Limit string `json:"limit"` Usage string `json:"usage"` } `json:"quota"` } ================================================ FILE: drivers/thunder_browser/util.go ================================================ package thunder_browser import ( "crypto/md5" "crypto/sha1" "encoding/hex" "encoding/json" "fmt" "io" "net/http" "regexp" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) const ( API_URL = "https://x-api-pan.xunlei.com/drive/v1" FILE_API_URL = API_URL + "/files" TASK_API_URL = API_URL + "/tasks" XLUSER_API_BASE_URL = "https://xluser-ssl.xunlei.com" XLUSER_API_URL = XLUSER_API_BASE_URL + "/v1" ) var Algorithms = []string{ "Cw4kArmKJ/aOiFTxnQ0ES+D4mbbrIUsFn", "HIGg0Qfbpm5ThZ/RJfjoao4YwgT9/M", "u/PUD", "OlAm8tPkOF1qO5bXxRN2iFttuDldrg", "FFIiM6sFhWhU7tIMVUKOF7CUv/KzgwwV8FE", "yN", "4m5mglrIHksI6wYdq", "LXEfS7", "T+p+C+F2yjgsUtiXWU/cMNYEtJI4pq7GofW", "14BrGIEMXkbvFvZ49nDUfVCRcHYFOJ1BP1Y", "kWIH3Row", "RAmRTKNCjucPWC", } const ( ClientID = "ZUBzD9J_XPXfn7f7" ClientSecret = "yESVmHecEe6F0aou69vl-g" ClientVersion = "1.40.0.7208" PackageName = "com.xunlei.browser" DownloadUserAgent = "AndroidDownloadManager/13 (Linux; U; Android 13; M2004J7AC Build/SP1A.210812.016)" SdkVersion = "509300" ) const ( FOLDER = "drive#folder" FILE = "drive#file" RESUMABLE = "drive#resumable" ) const ( UPLOAD_TYPE_UNKNOWN = "UPLOAD_TYPE_UNKNOWN" //UPLOAD_TYPE_FORM = "UPLOAD_TYPE_FORM" UPLOAD_TYPE_RESUMABLE = "UPLOAD_TYPE_RESUMABLE" UPLOAD_TYPE_URL = "UPLOAD_TYPE_URL" ) const ( ThunderDriveSpace = "" ThunderDriveSafeSpace = "SPACE_SAFE" ThunderBrowserDriveSpace = "SPACE_BROWSER" ThunderBrowserDriveSafeSpace = "SPACE_BROWSER_SAFE" ThunderDriveFolderType = "DEFAULT_ROOT" ThunderBrowserDriveSafeFolderType = "BROWSER_SAFE" ThunderBrowserDriveFluentPlayFolderType = "SPACE_FAVORITE" // 流畅播文件夹标识 ) const ( SignProvider = "access_end_point_token" APPID = "22062" APPKey = "a5d7416858147a4ab99573872ffccef8" ) func GetAction(method string, url string) string { urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] return method + ":" + urlpath } type Common struct { client *resty.Client captchaToken string creditKey string // 签名相关,二选一 Algorithms []string Timestamp, CaptchaSign string // 必要值,签名相关 DeviceID string ClientID string ClientSecret string ClientVersion string PackageName string UserAgent string DownloadUserAgent string UseVideoUrl bool UseFluentPlay bool RemoveWay string // 验证码token刷新成功回调 refreshCTokenCk func(token string) } func (c *Common) SetDeviceID(deviceID string) { c.DeviceID = deviceID } func (c *Common) SetCaptchaToken(captchaToken string) { c.captchaToken = captchaToken } func (c *Common) GetCaptchaToken() string { return c.captchaToken } func (c *Common) SetCreditKey(creditKey string) { c.creditKey = creditKey } func (c *Common) GetCreditKey() string { return c.creditKey } // RefreshCaptchaTokenAtLogin 刷新验证码token(登录后) func (c *Common) RefreshCaptchaTokenAtLogin(action, userID string) error { metas := map[string]string{ "client_version": c.ClientVersion, "package_name": c.PackageName, "user_id": userID, } metas["timestamp"], metas["captcha_sign"] = c.GetCaptchaSign() return c.refreshCaptchaToken(action, metas) } // RefreshCaptchaTokenInLogin 刷新验证码token(登录时) func (c *Common) RefreshCaptchaTokenInLogin(action, username string) error { metas := make(map[string]string) if ok, _ := regexp.MatchString(`\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*`, username); ok { metas["email"] = username } else if len(username) >= 11 && len(username) <= 18 { metas["phone_number"] = username } else { metas["username"] = username } return c.refreshCaptchaToken(action, metas) } // GetCaptchaSign 获取验证码签名 func (c *Common) GetCaptchaSign() (timestamp, sign string) { if len(c.Algorithms) == 0 { return c.Timestamp, c.CaptchaSign } timestamp = fmt.Sprint(time.Now().UnixMilli()) str := fmt.Sprint(c.ClientID, c.ClientVersion, c.PackageName, c.DeviceID, timestamp) for _, algorithm := range c.Algorithms { str = utils.GetMD5EncodeStr(str + algorithm) } sign = "1." + str return } // 刷新验证码token func (c *Common) refreshCaptchaToken(action string, metas map[string]string) error { param := CaptchaTokenRequest{ Action: action, CaptchaToken: c.captchaToken, ClientID: c.ClientID, DeviceID: c.DeviceID, Meta: metas, RedirectUri: "xlaccsdk01://xunlei.com/callback?state=harbor", } var e ErrResp var resp CaptchaTokenResponse _, err := c.Request(XLUSER_API_URL+"/shield/captcha/init", http.MethodPost, func(req *resty.Request) { req.SetError(&e).SetBody(param) }, &resp) if err != nil { return err } if e.IsError() { return &e } if resp.Url != "" { return fmt.Errorf(`need verify: Click Here`, resp.Url) } if resp.CaptchaToken == "" { return fmt.Errorf("empty captchaToken") } if c.refreshCTokenCk != nil { c.refreshCTokenCk(resp.CaptchaToken) } c.SetCaptchaToken(resp.CaptchaToken) return nil } // Request 只有基础信息的请求 func (c *Common) Request(url, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := c.client.R().SetHeaders(map[string]string{ "user-agent": c.UserAgent, "accept": "application/json;charset=UTF-8", "x-device-id": c.DeviceID, "x-client-id": c.ClientID, "x-client-version": c.ClientVersion, }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } res, err := req.Execute(method, url) if err != nil { return nil, err } var erron ErrResp utils.Json.Unmarshal(res.Body(), &erron) if erron.IsError() { // review_panel 表示需要短信验证码进行验证 if erron.ErrorMsg == "review_panel" { return nil, c.getReviewData(res) } return nil, &erron } return res.Body(), nil } // 获取验证所需内容 func (c *Common) getReviewData(res *resty.Response) error { var reviewResp LoginReviewResp var reviewData ReviewData if err := utils.Json.Unmarshal(res.Body(), &reviewResp); err != nil { return err } deviceSign := generateDeviceSign(c.DeviceID, c.PackageName) reviewData = ReviewData{ Creditkey: reviewResp.Creditkey, Reviewurl: reviewResp.Reviewurl + "&deviceid=" + deviceSign, Deviceid: deviceSign, Devicesign: deviceSign, } // 将reviewData转为JSON字符串 reviewDataJSON, _ := json.MarshalIndent(reviewData, "", " ") //reviewDataJSON, _ := json.Marshal(reviewData) return fmt.Errorf(`
🔒 本次登录需要验证
This login requires verification

下面是验证所需要的数据，具体使用方法请参照对应的驱动文档
Below are the relevant verification data. For specific usage methods, please refer to the corresponding driver documentation.

%s
`, string(reviewDataJSON)) } // 计算文件Gcid func getGcid(r io.Reader, size int64) (string, error) { calcBlockSize := func(j int64) int64 { var psize int64 = 0x40000 for float64(j)/float64(psize) > 0x200 && psize < 0x200000 { psize = psize << 1 } return psize } hash1 := sha1.New() hash2 := sha1.New() readSize := calcBlockSize(size) for { hash2.Reset() if n, err := utils.CopyWithBufferN(hash2, r, readSize); err != nil && n == 0 { if err != io.EOF { return "", err } break } hash1.Write(hash2.Sum(nil)) } return hex.EncodeToString(hash1.Sum(nil)), nil } type CustomTime struct { time.Time } const timeFormat = time.RFC3339 func (ct *CustomTime) UnmarshalJSON(b []byte) error { str := string(b) if str == `""` { *ct = CustomTime{Time: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)} return nil } t, err := time.Parse(`"`+timeFormat+`"`, str) if err != nil { return err } *ct = CustomTime{Time: t} return nil } // EncryptPassword 超级保险箱 加密 func EncryptPassword(password string) string { if password == "" { return "" } // 将字符串转换为字节数组 byteData := []byte(password) // 计算MD5哈希值 hash := md5.Sum(byteData) // 将哈希值转换为十六进制字符串 return hex.EncodeToString(hash[:]) } func generateDeviceSign(deviceID, packageName string) string { signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, APPID, APPKey) sha1Hash := sha1.New() sha1Hash.Write([]byte(signatureBase)) sha1Result := sha1Hash.Sum(nil) sha1String := hex.EncodeToString(sha1Result) md5Hash := md5.New() md5Hash.Write([]byte(sha1String)) md5Result := md5Hash.Sum(nil) md5String := hex.EncodeToString(md5Result) deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) return deviceSign } func BuildCustomUserAgent(deviceID, appName, sdkVersion, clientVersion, packageName string) string { //deviceSign := generateDeviceSign(deviceID, packageName) var sb strings.Builder sb.WriteString(fmt.Sprintf("ANDROID-%s/%s ", appName, clientVersion)) sb.WriteString("networkType/WIFI ") sb.WriteString(fmt.Sprintf("appid/%s ", APPID)) sb.WriteString(fmt.Sprintf("deviceName/Xiaomi_M2004j7ac ")) sb.WriteString(fmt.Sprintf("deviceModel/M2004J7AC ")) sb.WriteString(fmt.Sprintf("OSVersion/13 ")) sb.WriteString(fmt.Sprintf("protocolVersion/301 ")) sb.WriteString(fmt.Sprintf("platformversion/10 ")) sb.WriteString(fmt.Sprintf("sdkVersion/%s ", sdkVersion)) sb.WriteString(fmt.Sprintf("Oauth2Client/0.9 (Linux 4_9_337-perf-sn-uotan-gd9d488809c3d) (JAVA 0) ")) return sb.String() } ================================================ FILE: drivers/thunderx/driver.go ================================================ package thunderx import ( "context" "encoding/json" "errors" "fmt" "net/http" "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" ) type ThunderX struct { *XunLeiXCommon model.Storage Addition identity string } func (x *ThunderX) Config() driver.Config { return config } func (x *ThunderX) GetAddition() driver.Additional { return &x.Addition } func (x *ThunderX) Init(ctx context.Context) (err error) { // 初始化所需参数 if x.XunLeiXCommon == nil { x.XunLeiXCommon = &XunLeiXCommon{ Common: &Common{ client: base.NewRestyClient(), Algorithms: Algorithms, DeviceID: utils.GetMD5EncodeStr(x.Username + x.Password), ClientID: ClientID, ClientSecret: ClientSecret, ClientVersion: ClientVersion, PackageName: PackageName, UserAgent: BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, ""), DownloadUserAgent: DownloadUserAgent, UseVideoUrl: x.UseVideoUrl, refreshCTokenCk: func(token string) { x.CaptchaToken = token op.MustSaveDriverStorage(x) }, }, refreshTokenFunc: func() error { // 通过RefreshToken刷新 token, err := x.RefreshToken(x.TokenResp.RefreshToken) if err != nil { // 重新登录 token, err = x.Login(x.Username, x.Password) if err != nil { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) if token.UserID != "" { x.SetUserID(token.UserID) x.UserAgent = BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, token.UserID) } op.MustSaveDriverStorage(x) } } x.SetTokenResp(token) return err }, } } // 自定义验证码token ctoken := strings.TrimSpace(x.CaptchaToken) if ctoken != "" { x.SetCaptchaToken(ctoken) } if x.DeviceID == "" { x.SetDeviceID(utils.GetMD5EncodeStr(x.Username + x.Password)) } x.XunLeiXCommon.UseVideoUrl = x.UseVideoUrl x.Addition.RootFolderID = x.RootFolderID // 防止重复登录 identity := x.GetIdentity() if x.identity != identity || !x.IsLogin() { x.identity = identity // 登录 token, err := x.Login(x.Username, x.Password) if err != nil { return err } x.SetTokenResp(token) if token.UserID != "" { x.SetUserID(token.UserID) x.UserAgent = BuildCustomUserAgent(x.DeviceID, ClientID, PackageName, SdkVersion, ClientVersion, PackageName, token.UserID) } } return nil } func (x *ThunderX) Drop(ctx context.Context) error { return nil } type ThunderXExpert struct { *XunLeiXCommon model.Storage ExpertAddition identity string } func (x *ThunderXExpert) Config() driver.Config { return configExpert } func (x *ThunderXExpert) GetAddition() driver.Additional { return &x.ExpertAddition } func (x *ThunderXExpert) Init(ctx context.Context) (err error) { // 防止重复登录 identity := x.GetIdentity() if identity != x.identity || !x.IsLogin() { x.identity = identity x.XunLeiXCommon = &XunLeiXCommon{ Common: &Common{ client: base.NewRestyClient(), DeviceID: func() string { if len(x.DeviceID) != 32 { if x.LoginType == "user" { return utils.GetMD5EncodeStr(x.Username + x.Password) } return utils.GetMD5EncodeStr(x.ExpertAddition.RefreshToken) } return x.DeviceID }(), ClientID: x.ClientID, ClientSecret: x.ClientSecret, ClientVersion: x.ClientVersion, PackageName: x.PackageName, UserAgent: func() string { if x.ExpertAddition.UserAgent != "" { return x.ExpertAddition.UserAgent } if x.LoginType == "user" { return BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, "") } return BuildCustomUserAgent(utils.GetMD5EncodeStr(x.ExpertAddition.RefreshToken), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, "") }(), DownloadUserAgent: func() string { if x.ExpertAddition.DownloadUserAgent != "" { return x.ExpertAddition.DownloadUserAgent } return DownloadUserAgent }(), UseVideoUrl: x.UseVideoUrl, refreshCTokenCk: func(token string) { x.CaptchaToken = token op.MustSaveDriverStorage(x) }, }, } if x.ExpertAddition.CaptchaToken != "" { x.SetCaptchaToken(x.ExpertAddition.CaptchaToken) op.MustSaveDriverStorage(x) } if x.Common.DeviceID != "" { x.ExpertAddition.DeviceID = x.Common.DeviceID op.MustSaveDriverStorage(x) } if x.Common.DownloadUserAgent != "" { x.ExpertAddition.DownloadUserAgent = x.Common.DownloadUserAgent op.MustSaveDriverStorage(x) } x.XunLeiXCommon.UseVideoUrl = x.UseVideoUrl x.ExpertAddition.RootFolderID = x.RootFolderID // 签名方法 if x.SignType == "captcha_sign" { x.Common.Timestamp = x.Timestamp x.Common.CaptchaSign = x.CaptchaSign } else { x.Common.Algorithms = strings.Split(x.Algorithms, ",") } // 登录方式 if x.LoginType == "refresh_token" { // 通过RefreshToken登录 token, err := x.XunLeiXCommon.RefreshToken(x.ExpertAddition.RefreshToken) if err != nil { return err } x.SetTokenResp(token) // 刷新token方法 x.SetRefreshTokenFunc(func() error { token, err := x.XunLeiXCommon.RefreshToken(x.TokenResp.RefreshToken) if err != nil { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) } x.SetTokenResp(token) op.MustSaveDriverStorage(x) return err }) } else { // 通过用户密码登录 token, err := x.Login(x.Username, x.Password) if err != nil { return err } x.SetTokenResp(token) x.SetRefreshTokenFunc(func() error { token, err := x.XunLeiXCommon.RefreshToken(x.TokenResp.RefreshToken) if err != nil { token, err = x.Login(x.Username, x.Password) if err != nil { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) } } x.SetTokenResp(token) op.MustSaveDriverStorage(x) return err }) } // 更新 UserAgent if x.TokenResp.UserID != "" { x.ExpertAddition.UserAgent = BuildCustomUserAgent(x.ExpertAddition.DeviceID, ClientID, PackageName, SdkVersion, ClientVersion, PackageName, x.TokenResp.UserID) x.SetUserAgent(x.ExpertAddition.UserAgent) op.MustSaveDriverStorage(x) } } else { // 仅修改验证码token if x.CaptchaToken != "" { x.SetCaptchaToken(x.CaptchaToken) } x.XunLeiXCommon.UserAgent = x.ExpertAddition.UserAgent x.XunLeiXCommon.DownloadUserAgent = x.ExpertAddition.UserAgent x.XunLeiXCommon.UseVideoUrl = x.UseVideoUrl x.ExpertAddition.RootFolderID = x.RootFolderID } return nil } func (x *ThunderXExpert) Drop(ctx context.Context) error { return nil } func (x *ThunderXExpert) SetTokenResp(token *TokenResp) { x.XunLeiXCommon.SetTokenResp(token) if token != nil { x.ExpertAddition.RefreshToken = token.RefreshToken } } type XunLeiXCommon struct { *Common *TokenResp // 登录信息 refreshTokenFunc func() error } func (xc *XunLeiXCommon) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { return xc.getFiles(ctx, dir.GetID()) } func (xc *XunLeiXCommon) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var lFile Files _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", file.GetID()) //r.SetQueryParam("space", "") }, &lFile) if err != nil { return nil, err } link := &model.Link{ URL: lFile.WebContentLink, Header: http.Header{ "User-Agent": {xc.DownloadUserAgent}, }, } if xc.UseVideoUrl { for _, media := range lFile.Medias { if media.Link.URL != "" { link.URL = media.Link.URL break } } } /* strs := regexp.MustCompile(`e=([0-9]*)`).FindStringSubmatch(lFile.WebContentLink) if len(strs) == 2 { timestamp, err := strconv.ParseInt(strs[1], 10, 64) if err == nil { expired := time.Duration(timestamp-time.Now().Unix()) * time.Second link.Expiration = &expired } } */ return link, nil } func (xc *XunLeiXCommon) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ "kind": FOLDER, "name": dirName, "parent_id": parentDir.GetID(), }) }, nil) return err } func (xc *XunLeiXCommon) Move(ctx context.Context, srcObj, dstDir model.Obj) error { _, err := xc.Request(FILE_API_URL+":batchMove", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ "to": base.Json{"parent_id": dstDir.GetID()}, "ids": []string{srcObj.GetID()}, }) }, nil) return err } func (xc *XunLeiXCommon) Rename(ctx context.Context, srcObj model.Obj, newName string) error { _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodPatch, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", srcObj.GetID()) r.SetBody(&base.Json{"name": newName}) }, nil) return err } func (xc *XunLeiXCommon) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { _, err := xc.Request(FILE_API_URL+":batchCopy", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ "to": base.Json{"parent_id": dstDir.GetID()}, "ids": []string{srcObj.GetID()}, }) }, nil) return err } func (xc *XunLeiXCommon) Remove(ctx context.Context, obj model.Obj) error { _, err := xc.Request(FILE_API_URL+"/{fileID}/trash", http.MethodPatch, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", obj.GetID()) r.SetBody("{}") }, nil) return err } func (xc *XunLeiXCommon) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { gcid := file.GetHash().GetHash(hash_extend.GCID) var err error if len(gcid) < hash_extend.GCID.Width { _, gcid, err = stream.CacheFullAndHash(file, &up, hash_extend.GCID, file.GetSize()) if err != nil { return err } } var resp UploadTaskResponse _, err = xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ "kind": FILE, "parent_id": dstDir.GetID(), "name": file.GetName(), "size": file.GetSize(), "hash": gcid, "upload_type": UPLOAD_TYPE_RESUMABLE, }) }, &resp) if err != nil { return err } param := resp.Resumable.Params if resp.UploadType == UPLOAD_TYPE_RESUMABLE { param.Endpoint = strings.TrimLeft(param.Endpoint, param.Bucket+".") s, err := session.NewSession(&aws.Config{ Credentials: credentials.NewStaticCredentials(param.AccessKeyID, param.AccessKeySecret, param.SecurityToken), Region: aws.String("xunlei"), Endpoint: aws.String(param.Endpoint), }) if err != nil { return err } uploader := s3manager.NewUploader(s) if file.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { uploader.PartSize = file.GetSize() / (s3manager.MaxUploadParts - 1) } _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ Bucket: aws.String(param.Bucket), Key: aws.String(param.Key), Expires: aws.Time(param.Expiration), Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: file, UpdateProgress: up, }), }) return err } return nil } func (xc *XunLeiXCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { var about AboutResponse _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) }, &about) if err != nil { return nil, err } total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) if err != nil { return nil, err } used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: used, }, }, nil } func (xc *XunLeiXCommon) getFiles(ctx context.Context, folderId string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string for { var fileList FileList _, err := xc.Request(FILE_API_URL, http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(map[string]string{ "space": "", "__type": "drive", "refresh": "true", "__sync": "true", "parent_id": folderId, "page_token": pageToken, "with_audit": "true", "limit": "100", "filters": `{"phase":{"eq":"PHASE_TYPE_COMPLETE"},"trashed":{"eq":false}}`, }) }, &fileList) if err != nil { return nil, err } for i := 0; i < len(fileList.Files); i++ { files = append(files, &fileList.Files[i]) } if fileList.NextPageToken == "" { break } pageToken = fileList.NextPageToken } return files, nil } // SetRefreshTokenFunc 设置刷新Token的方法 func (xc *XunLeiXCommon) SetRefreshTokenFunc(fn func() error) { xc.refreshTokenFunc = fn } // SetTokenResp 设置Token func (xc *XunLeiXCommon) SetTokenResp(tr *TokenResp) { xc.TokenResp = tr } // Request 携带Authorization和CaptchaToken的请求 func (xc *XunLeiXCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { data, err := xc.Common.Request(url, method, func(req *resty.Request) { req.SetHeaders(map[string]string{ "Authorization": xc.Token(), "X-Captcha-Token": xc.GetCaptchaToken(), }) if callback != nil { callback(req) } }, resp) var errResp *ErrResp ok := errors.As(err, &errResp) if !ok { return nil, err } switch errResp.ErrorCode { case 0: return data, nil case 4122, 4121, 10, 16: if xc.refreshTokenFunc != nil { if err = xc.refreshTokenFunc(); err == nil { break } } return nil, err case 9: // 验证码token过期 if err = xc.RefreshCaptchaTokenAtLogin(GetAction(method, url), xc.UserID); err != nil { return nil, err } default: return nil, err } return xc.Request(url, method, callback, resp) } // RefreshToken 刷新Token func (xc *XunLeiXCommon) RefreshToken(refreshToken string) (*TokenResp, error) { var resp TokenResp _, err := xc.Common.Request(XLUSER_API_URL+"/auth/token", http.MethodPost, func(req *resty.Request) { req.SetBody(&base.Json{ "grant_type": "refresh_token", "refresh_token": refreshToken, "client_id": xc.ClientID, "client_secret": xc.ClientSecret, }) }, &resp) if err != nil { return nil, err } if resp.RefreshToken == "" { return nil, errs.EmptyToken } resp.UserID = resp.Sub return &resp, nil } // Login 登录 func (xc *XunLeiXCommon) Login(username, password string) (*TokenResp, error) { url := XLUSER_API_URL + "/auth/signin" err := xc.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), username) if err != nil { return nil, err } var resp TokenResp _, err = xc.Common.Request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(&SignInRequest{ CaptchaToken: xc.GetCaptchaToken(), ClientID: xc.ClientID, ClientSecret: xc.ClientSecret, Username: username, Password: password, }) }, &resp) if err != nil { return nil, err } resp.UserID = resp.Sub return &resp, nil } func (xc *XunLeiXCommon) IsLogin() bool { if xc.TokenResp == nil { return false } _, err := xc.Request(XLUSER_API_URL+"/user/me", http.MethodGet, nil, nil) return err == nil } // 离线下载文件，都和Pikpak接口一致 func (xc *XunLeiXCommon) OfflineDownload(ctx context.Context, fileUrl string, parentDir model.Obj, fileName string) (*OfflineTask, error) { requestBody := base.Json{ "kind": "drive#file", "name": fileName, "upload_type": "UPLOAD_TYPE_URL", "url": base.Json{ "url": fileUrl, }, "params": base.Json{}, "parent_id": parentDir.GetID(), } var resp OfflineDownloadResp // 一样的 _, err := xc.Request(FILE_API_URL, http.MethodPost, func(req *resty.Request) { req.SetContext(ctx). SetBody(requestBody) }, &resp) if err != nil { return nil, err } return &resp.Task, err } // 获取离线下载任务列表 func (xc *XunLeiXCommon) OfflineList(ctx context.Context, nextPageToken string, phase []string) ([]OfflineTask, error) { res := make([]OfflineTask, 0) if len(phase) == 0 { phase = []string{"PHASE_TYPE_RUNNING", "PHASE_TYPE_ERROR", "PHASE_TYPE_COMPLETE", "PHASE_TYPE_PENDING"} } params := map[string]string{ "type": "offline", "thumbnail_size": "SIZE_SMALL", "limit": "10000", "page_token": nextPageToken, "with": "reference_resource", } // 处理 phase 参数 if len(phase) > 0 { filters := base.Json{ "phase": map[string]string{ "in": strings.Join(phase, ","), }, } filtersJSON, err := json.Marshal(filters) if err != nil { return nil, fmt.Errorf("failed to marshal filters: %w", err) } params["filters"] = string(filtersJSON) } var resp OfflineListResp _, err := xc.Request(TASKS_API_URL, http.MethodGet, func(req *resty.Request) { req.SetContext(ctx). SetQueryParams(params) }, &resp) if err != nil { return nil, fmt.Errorf("failed to get offline list: %w", err) } res = append(res, resp.Tasks...) return res, nil } func (xc *XunLeiXCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error { params := map[string]string{ "task_ids": strings.Join(taskIDs, ","), "delete_files": strconv.FormatBool(deleteFiles), } _, err := xc.Request(TASKS_API_URL, http.MethodDelete, func(req *resty.Request) { req.SetContext(ctx). SetQueryParams(params) }, nil) if err != nil { return fmt.Errorf("failed to delete tasks %v: %w", taskIDs, err) } return nil } ================================================ FILE: drivers/thunderx/meta.go ================================================ package thunderx import ( "crypto/md5" "encoding/hex" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) // 高级设置 type ExpertAddition struct { driver.RootID LoginType string `json:"login_type" type:"select" options:"user,refresh_token" default:"user"` SignType string `json:"sign_type" type:"select" options:"algorithms,captcha_sign" default:"algorithms"` // 登录方式1 Username string `json:"username" required:"true" help:"login type is user,this is required"` Password string `json:"password" required:"true" help:"login type is user,this is required"` // 登录方式2 RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` // 签名方法1 Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"kVy0WbPhiE4v6oxXZ88DvoA3Q,lON/AUoZKj8/nBtcE85mVbkOaVdVa,rLGffQrfBKH0BgwQ33yZofvO3Or,FO6HWqw,GbgvyA2,L1NU9QvIQIH7DTRt,y7llk4Y8WfYflt6,iuDp1WPbV3HRZudZtoXChxH4HNVBX5ZALe,8C28RTXmVcco0,X5Xh,7xe25YUgfGgD0xW3ezFS,,CKCR,8EmDjBo6h3eLaK7U6vU2Qys0NsMx,t2TeZBXKqbdP09Arh9C3"` // 签名方法2 CaptchaSign string `json:"captcha_sign" required:"true" help:"sign type is captcha_sign,this is required"` Timestamp string `json:"timestamp" required:"true" help:"sign type is captcha_sign,this is required"` // 验证码 CaptchaToken string `json:"captcha_token"` // 必要且影响登录,由签名决定 DeviceID string `json:"device_id" required:"false" default:""` ClientID string `json:"client_id" required:"true" default:"ZQL_zwA4qhHcoe_2"` ClientSecret string `json:"client_secret" required:"true" default:"Og9Vr1L8Ee6bh0olFxFDRg"` ClientVersion string `json:"client_version" required:"true" default:"1.06.0.2132"` PackageName string `json:"package_name" required:"true" default:"com.thunder.downloader"` ////不影响登录,影响下载速度 UserAgent string `json:"user_agent" required:"false" default:""` DownloadUserAgent string `json:"download_user_agent" required:"false" default:""` //优先使用视频链接代替下载链接 UseVideoUrl bool `json:"use_video_url"` } // 登录特征,用于判断是否重新登录 func (i *ExpertAddition) GetIdentity() string { hash := md5.New() if i.LoginType == "refresh_token" { hash.Write([]byte(i.RefreshToken)) } else { hash.Write([]byte(i.Username + i.Password)) } if i.SignType == "captcha_sign" { hash.Write([]byte(i.CaptchaSign + i.Timestamp)) } else { hash.Write([]byte(i.Algorithms)) } hash.Write([]byte(i.DeviceID)) hash.Write([]byte(i.ClientID)) hash.Write([]byte(i.ClientSecret)) hash.Write([]byte(i.ClientVersion)) hash.Write([]byte(i.PackageName)) return hex.EncodeToString(hash.Sum(nil)) } type Addition struct { driver.RootID Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` CaptchaToken string `json:"captcha_token"` UseVideoUrl bool `json:"use_video_url" default:"true"` } // 登录特征,用于判断是否重新登录 func (i *Addition) GetIdentity() string { return utils.GetMD5EncodeStr(i.Username + i.Password) } var config = driver.Config{ Name: "ThunderX", LocalSort: true, } var configExpert = driver.Config{ Name: "ThunderXExpert", LocalSort: true, } func init() { op.RegisterDriver(func() driver.Driver { return &ThunderX{} }) op.RegisterDriver(func() driver.Driver { return &ThunderXExpert{} }) } ================================================ FILE: drivers/thunderx/types.go ================================================ package thunderx import ( "fmt" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash" ) type ErrResp struct { ErrorCode int64 `json:"error_code"` ErrorMsg string `json:"error"` ErrorDescription string `json:"error_description"` // ErrorDetails interface{} `json:"error_details"` } func (e *ErrResp) IsError() bool { return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" } func (e *ErrResp) Error() string { return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ErrorDescription: %s ", e.ErrorCode, e.ErrorMsg, e.ErrorDescription) } /* * 验证码Token **/ type CaptchaTokenRequest struct { Action string `json:"action"` CaptchaToken string `json:"captcha_token"` ClientID string `json:"client_id"` DeviceID string `json:"device_id"` Meta map[string]string `json:"meta"` RedirectUri string `json:"redirect_uri"` } type CaptchaTokenResponse struct { CaptchaToken string `json:"captcha_token"` ExpiresIn int64 `json:"expires_in"` Url string `json:"url"` } /* * 登录 **/ type TokenResp struct { TokenType string `json:"token_type"` AccessToken string `json:"access_token"` RefreshToken string `json:"refresh_token"` ExpiresIn int64 `json:"expires_in"` Sub string `json:"sub"` UserID string `json:"user_id"` } func (t *TokenResp) Token() string { return fmt.Sprint(t.TokenType, " ", t.AccessToken) } type SignInRequest struct { CaptchaToken string `json:"captcha_token"` ClientID string `json:"client_id"` ClientSecret string `json:"client_secret"` Username string `json:"username"` Password string `json:"password"` } /* * 文件 **/ type FileList struct { Kind string `json:"kind"` NextPageToken string `json:"next_page_token"` Files []Files `json:"files"` Version string `json:"version"` VersionOutdated bool `json:"version_outdated"` } type Link struct { URL string `json:"url"` Token string `json:"token"` Expire time.Time `json:"expire"` Type string `json:"type"` } var _ model.Obj = (*Files)(nil) type Files struct { Kind string `json:"kind"` ID string `json:"id"` ParentID string `json:"parent_id"` Name string `json:"name"` //UserID string `json:"user_id"` Size string `json:"size"` //Revision string `json:"revision"` //FileExtension string `json:"file_extension"` //MimeType string `json:"mime_type"` //Starred bool `json:"starred"` WebContentLink string `json:"web_content_link"` CreatedTime time.Time `json:"created_time"` ModifiedTime time.Time `json:"modified_time"` IconLink string `json:"icon_link"` ThumbnailLink string `json:"thumbnail_link"` // Md5Checksum string `json:"md5_checksum"` Hash string `json:"hash"` // Links map[string]Link `json:"links"` // Phase string `json:"phase"` // Audit struct { // Status string `json:"status"` // Message string `json:"message"` // Title string `json:"title"` // } `json:"audit"` Medias []struct { //Category string `json:"category"` //IconLink string `json:"icon_link"` //IsDefault bool `json:"is_default"` //IsOrigin bool `json:"is_origin"` //IsVisible bool `json:"is_visible"` Link Link `json:"link"` //MediaID string `json:"media_id"` //MediaName string `json:"media_name"` //NeedMoreQuota bool `json:"need_more_quota"` //Priority int `json:"priority"` //RedirectLink string `json:"redirect_link"` //ResolutionName string `json:"resolution_name"` // Video struct { // AudioCodec string `json:"audio_codec"` // BitRate int `json:"bit_rate"` // Duration int `json:"duration"` // FrameRate int `json:"frame_rate"` // Height int `json:"height"` // VideoCodec string `json:"video_codec"` // VideoType string `json:"video_type"` // Width int `json:"width"` // } `json:"video"` // VipTypes []string `json:"vip_types"` } `json:"medias"` Trashed bool `json:"trashed"` DeleteTime string `json:"delete_time"` OriginalURL string `json:"original_url"` //Params struct{} `json:"params"` //OriginalFileIndex int `json:"original_file_index"` //Space string `json:"space"` //Apps []interface{} `json:"apps"` //Writable bool `json:"writable"` //FolderType string `json:"folder_type"` //Collection interface{} `json:"collection"` } func (c *Files) GetHash() utils.HashInfo { return utils.NewHashInfo(hash_extend.GCID, c.Hash) } func (c *Files) GetSize() int64 { size, _ := strconv.ParseInt(c.Size, 10, 64); return size } func (c *Files) GetName() string { return c.Name } func (c *Files) CreateTime() time.Time { return c.CreatedTime } func (c *Files) ModTime() time.Time { return c.ModifiedTime } func (c *Files) IsDir() bool { return c.Kind == FOLDER } func (c *Files) GetID() string { return c.ID } func (c *Files) GetPath() string { return "" } func (c *Files) Thumb() string { return c.ThumbnailLink } /* * 上传 **/ type UploadTaskResponse struct { UploadType string `json:"upload_type"` /*//UPLOAD_TYPE_FORM Form struct { //Headers struct{} `json:"headers"` Kind string `json:"kind"` Method string `json:"method"` MultiParts struct { OSSAccessKeyID string `json:"OSSAccessKeyId"` Signature string `json:"Signature"` Callback string `json:"callback"` Key string `json:"key"` Policy string `json:"policy"` XUserData string `json:"x:user_data"` } `json:"multi_parts"` URL string `json:"url"` } `json:"form"`*/ //UPLOAD_TYPE_RESUMABLE Resumable struct { Kind string `json:"kind"` Params struct { AccessKeyID string `json:"access_key_id"` AccessKeySecret string `json:"access_key_secret"` Bucket string `json:"bucket"` Endpoint string `json:"endpoint"` Expiration time.Time `json:"expiration"` Key string `json:"key"` SecurityToken string `json:"security_token"` } `json:"params"` Provider string `json:"provider"` } `json:"resumable"` File Files `json:"file"` } // 添加离线下载响应 type OfflineDownloadResp struct { File *string `json:"file"` Task OfflineTask `json:"task"` UploadType string `json:"upload_type"` URL struct { Kind string `json:"kind"` } `json:"url"` } // 离线下载列表 type OfflineListResp struct { ExpiresIn int64 `json:"expires_in"` NextPageToken string `json:"next_page_token"` Tasks []OfflineTask `json:"tasks"` } // offlineTask type OfflineTask struct { Callback string `json:"callback"` CreatedTime string `json:"created_time"` FileID string `json:"file_id"` FileName string `json:"file_name"` FileSize string `json:"file_size"` IconLink string `json:"icon_link"` ID string `json:"id"` Kind string `json:"kind"` Message string `json:"message"` Name string `json:"name"` Params Params `json:"params"` Phase string `json:"phase"` // PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING Progress int64 `json:"progress"` ReferenceResource ReferenceResource `json:"reference_resource"` Space string `json:"space"` StatusSize int64 `json:"status_size"` Statuses []string `json:"statuses"` ThirdTaskID string `json:"third_task_id"` Type string `json:"type"` UpdatedTime string `json:"updated_time"` UserID string `json:"user_id"` } type Params struct { Age string `json:"age"` MIMEType *string `json:"mime_type,omitempty"` PredictType string `json:"predict_type"` URL string `json:"url"` } type ReferenceResource struct { Type string `json:"@type"` Audit interface{} `json:"audit"` Hash string `json:"hash"` IconLink string `json:"icon_link"` ID string `json:"id"` Kind string `json:"kind"` Medias []Media `json:"medias"` MIMEType string `json:"mime_type"` Name string `json:"name"` Params map[string]interface{} `json:"params"` ParentID string `json:"parent_id"` Phase string `json:"phase"` Size string `json:"size"` Space string `json:"space"` Starred bool `json:"starred"` Tags []string `json:"tags"` ThumbnailLink string `json:"thumbnail_link"` } type Media struct { MediaId string `json:"media_id"` MediaName string `json:"media_name"` Video struct { Height int `json:"height"` Width int `json:"width"` Duration int `json:"duration"` BitRate int `json:"bit_rate"` FrameRate int `json:"frame_rate"` VideoCodec string `json:"video_codec"` AudioCodec string `json:"audio_codec"` VideoType string `json:"video_type"` } `json:"video"` Link struct { Url string `json:"url"` Token string `json:"token"` Expire time.Time `json:"expire"` } `json:"link"` NeedMoreQuota bool `json:"need_more_quota"` VipTypes []interface{} `json:"vip_types"` RedirectLink string `json:"redirect_link"` IconLink string `json:"icon_link"` IsDefault bool `json:"is_default"` Priority int `json:"priority"` IsOrigin bool `json:"is_origin"` ResolutionName string `json:"resolution_name"` IsVisible bool `json:"is_visible"` Category string `json:"category"` } type AboutResponse struct { Quota struct { Limit string `json:"limit"` Usage string `json:"usage"` } `json:"quota"` } ================================================ FILE: drivers/thunderx/util.go ================================================ package thunderx import ( "crypto/md5" "crypto/sha1" "encoding/hex" "fmt" "io" "net/http" "regexp" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) const ( API_URL = "https://api-pan.xunleix.com/drive/v1" FILE_API_URL = API_URL + "/files" TASKS_API_URL = API_URL + "/tasks" XLUSER_API_URL = "https://xluser-ssl.xunleix.com/v1" ) var Algorithms = []string{ "kVy0WbPhiE4v6oxXZ88DvoA3Q", "lON/AUoZKj8/nBtcE85mVbkOaVdVa", "rLGffQrfBKH0BgwQ33yZofvO3Or", "FO6HWqw", "GbgvyA2", "L1NU9QvIQIH7DTRt", "y7llk4Y8WfYflt6", "iuDp1WPbV3HRZudZtoXChxH4HNVBX5ZALe", "8C28RTXmVcco0", "X5Xh", "7xe25YUgfGgD0xW3ezFS", "", "CKCR", "8EmDjBo6h3eLaK7U6vU2Qys0NsMx", "t2TeZBXKqbdP09Arh9C3", } const ( ClientID = "ZQL_zwA4qhHcoe_2" ClientSecret = "Og9Vr1L8Ee6bh0olFxFDRg" ClientVersion = "1.06.0.2132" PackageName = "com.thunder.downloader" DownloadUserAgent = "Dalvik/2.1.0 (Linux; U; Android 13; M2004J7AC Build/SP1A.210812.016)" SdkVersion = "2.0.3.203100 " ) const ( FOLDER = "drive#folder" FILE = "drive#file" RESUMABLE = "drive#resumable" ) const ( UPLOAD_TYPE_UNKNOWN = "UPLOAD_TYPE_UNKNOWN" //UPLOAD_TYPE_FORM = "UPLOAD_TYPE_FORM" UPLOAD_TYPE_RESUMABLE = "UPLOAD_TYPE_RESUMABLE" UPLOAD_TYPE_URL = "UPLOAD_TYPE_URL" ) func GetAction(method string, url string) string { urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] return method + ":" + urlpath } type Common struct { client *resty.Client captchaToken string userID string // 签名相关,二选一 Algorithms []string Timestamp, CaptchaSign string // 必要值,签名相关 DeviceID string ClientID string ClientSecret string ClientVersion string PackageName string UserAgent string DownloadUserAgent string UseVideoUrl bool // 验证码token刷新成功回调 refreshCTokenCk func(token string) } func (c *Common) SetDeviceID(deviceID string) { c.DeviceID = deviceID } func (c *Common) SetUserID(userID string) { c.userID = userID } func (c *Common) SetUserAgent(userAgent string) { c.UserAgent = userAgent } func (c *Common) SetCaptchaToken(captchaToken string) { c.captchaToken = captchaToken } func (c *Common) GetCaptchaToken() string { return c.captchaToken } // 刷新验证码token(登录后) func (c *Common) RefreshCaptchaTokenAtLogin(action, userID string) error { metas := map[string]string{ "client_version": c.ClientVersion, "package_name": c.PackageName, "user_id": userID, } metas["timestamp"], metas["captcha_sign"] = c.GetCaptchaSign() return c.refreshCaptchaToken(action, metas) } // 刷新验证码token(登录时) func (c *Common) RefreshCaptchaTokenInLogin(action, username string) error { metas := make(map[string]string) if ok, _ := regexp.MatchString(`\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*`, username); ok { metas["email"] = username } else if len(username) >= 11 && len(username) <= 18 { metas["phone_number"] = username } else { metas["username"] = username } return c.refreshCaptchaToken(action, metas) } // 获取验证码签名 func (c *Common) GetCaptchaSign() (timestamp, sign string) { if len(c.Algorithms) == 0 { return c.Timestamp, c.CaptchaSign } timestamp = fmt.Sprint(time.Now().UnixMilli()) str := fmt.Sprint(c.ClientID, c.ClientVersion, c.PackageName, c.DeviceID, timestamp) for _, algorithm := range c.Algorithms { str = utils.GetMD5EncodeStr(str + algorithm) } sign = "1." + str return } // 刷新验证码token func (c *Common) refreshCaptchaToken(action string, metas map[string]string) error { param := CaptchaTokenRequest{ Action: action, CaptchaToken: c.captchaToken, ClientID: c.ClientID, DeviceID: c.DeviceID, Meta: metas, RedirectUri: "xlaccsdk01://xbase.cloud/callback?state=harbor", } var e ErrResp var resp CaptchaTokenResponse _, err := c.Request(XLUSER_API_URL+"/shield/captcha/init", http.MethodPost, func(req *resty.Request) { req.SetError(&e).SetBody(param) }, &resp) if err != nil { return err } if e.IsError() { return &e } if resp.Url != "" { return fmt.Errorf(`need verify: Click Here`, resp.Url) } if resp.CaptchaToken == "" { return fmt.Errorf("empty captchaToken") } if c.refreshCTokenCk != nil { c.refreshCTokenCk(resp.CaptchaToken) } c.SetCaptchaToken(resp.CaptchaToken) return nil } // Request 只有基础信息的请求 func (c *Common) Request(url, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := c.client.R().SetHeaders(map[string]string{ "user-agent": c.UserAgent, "accept": "application/json;charset=UTF-8", "x-device-id": c.DeviceID, "x-client-id": c.ClientID, "x-client-version": c.ClientVersion, }) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } res, err := req.Execute(method, url) if err != nil { return nil, err } var erron ErrResp utils.Json.Unmarshal(res.Body(), &erron) if erron.IsError() { return nil, &erron } return res.Body(), nil } // 计算文件Gcid func getGcid(r io.Reader, size int64) (string, error) { calcBlockSize := func(j int64) int64 { var psize int64 = 0x40000 for float64(j)/float64(psize) > 0x200 && psize < 0x200000 { psize = psize << 1 } return psize } hash1 := sha1.New() hash2 := sha1.New() readSize := calcBlockSize(size) for { hash2.Reset() if n, err := utils.CopyWithBufferN(hash2, r, readSize); err != nil && n == 0 { if err != io.EOF { return "", err } break } hash1.Write(hash2.Sum(nil)) } return hex.EncodeToString(hash1.Sum(nil)), nil } func generateDeviceSign(deviceID, packageName string) string { signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, "1", "appkey") sha1Hash := sha1.New() sha1Hash.Write([]byte(signatureBase)) sha1Result := sha1Hash.Sum(nil) sha1String := hex.EncodeToString(sha1Result) md5Hash := md5.New() md5Hash.Write([]byte(sha1String)) md5Result := md5Hash.Sum(nil) md5String := hex.EncodeToString(md5Result) deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) return deviceSign } func BuildCustomUserAgent(deviceID, clientID, appName, sdkVersion, clientVersion, packageName, userID string) string { deviceSign := generateDeviceSign(deviceID, packageName) var sb strings.Builder sb.WriteString(fmt.Sprintf("ANDROID-%s/%s ", appName, clientVersion)) sb.WriteString("protocolVersion/200 ") sb.WriteString("accesstype/ ") sb.WriteString(fmt.Sprintf("clientid/%s ", clientID)) sb.WriteString(fmt.Sprintf("clientversion/%s ", clientVersion)) sb.WriteString("action_type/ ") sb.WriteString("networktype/WIFI ") sb.WriteString("sessionid/ ") sb.WriteString(fmt.Sprintf("deviceid/%s ", deviceID)) sb.WriteString("providername/NONE ") sb.WriteString(fmt.Sprintf("devicesign/%s ", deviceSign)) sb.WriteString("refresh_token/ ") sb.WriteString(fmt.Sprintf("sdkversion/%s ", sdkVersion)) sb.WriteString(fmt.Sprintf("datetime/%d ", time.Now().UnixMilli())) sb.WriteString(fmt.Sprintf("usrno/%s ", userID)) sb.WriteString(fmt.Sprintf("appname/%s ", appName)) sb.WriteString(fmt.Sprintf("session_origin/ ")) sb.WriteString(fmt.Sprintf("grant_type/ ")) sb.WriteString(fmt.Sprintf("appid/ ")) sb.WriteString(fmt.Sprintf("clientip/ ")) sb.WriteString(fmt.Sprintf("devicename/Xiaomi_M2004j7ac ")) sb.WriteString(fmt.Sprintf("osversion/13 ")) sb.WriteString(fmt.Sprintf("platformversion/10 ")) sb.WriteString(fmt.Sprintf("accessmode/ ")) sb.WriteString(fmt.Sprintf("devicemodel/M2004J7AC ")) return sb.String() } ================================================ FILE: drivers/url_tree/driver.go ================================================ package url_tree import ( "context" "errors" stdpath "path" "strings" "sync" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" ) type Urls struct { model.Storage Addition root *Node mutex sync.RWMutex } func (d *Urls) Config() driver.Config { return config } func (d *Urls) GetAddition() driver.Additional { return &d.Addition } func (d *Urls) Init(ctx context.Context) error { node, err := BuildTree(d.UrlStructure, d.HeadSize) if err != nil { return err } node.calSize() d.root = node return nil } func (d *Urls) Drop(ctx context.Context) error { return nil } func (Addition) GetRootPath() string { return "/" } func (d *Urls) Get(ctx context.Context, path string) (model.Obj, error) { d.mutex.RLock() defer d.mutex.RUnlock() node := GetNodeFromRootByPath(d.root, path) return nodeToObj(node, path) } func (d *Urls) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { d.mutex.RLock() defer d.mutex.RUnlock() node := GetNodeFromRootByPath(d.root, dir.GetPath()) log.Debugf("path: %s, node: %+v", dir.GetPath(), node) if node == nil { return nil, errs.ObjectNotFound } if node.isFile() { return nil, errs.NotFolder } return utils.SliceConvert(node.Children, func(node *Node) (model.Obj, error) { return nodeToObj(node, stdpath.Join(dir.GetPath(), node.Name)) }) } func (d *Urls) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { d.mutex.RLock() defer d.mutex.RUnlock() node := GetNodeFromRootByPath(d.root, file.GetPath()) log.Debugf("path: %s, node: %+v", file.GetPath(), node) if node == nil { return nil, errs.ObjectNotFound } if node.isFile() { return &model.Link{ URL: node.Url, }, nil } return nil, errs.NotFile } func (d *Urls) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if !d.Writable { return nil, errs.PermissionDenied } d.mutex.Lock() defer d.mutex.Unlock() node := GetNodeFromRootByPath(d.root, parentDir.GetPath()) if node == nil { return nil, errs.ObjectNotFound } if node.isFile() { return nil, errs.NotFolder } dir := &Node{ Name: dirName, Level: node.Level + 1, } node.Children = append(node.Children, dir) d.updateStorage() return nodeToObj(dir, stdpath.Join(parentDir.GetPath(), dirName)) } func (d *Urls) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if !d.Writable { return nil, errs.PermissionDenied } if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { return nil, errors.New("cannot move parent dir to child") } d.mutex.Lock() defer d.mutex.Unlock() dstNode := GetNodeFromRootByPath(d.root, dstDir.GetPath()) if dstNode == nil || dstNode.isFile() { return nil, errs.NotFolder } srcDir, srcName := stdpath.Split(srcObj.GetPath()) srcParentNode := GetNodeFromRootByPath(d.root, srcDir) if srcParentNode == nil { return nil, errs.ObjectNotFound } newChildren := make([]*Node, 0, len(srcParentNode.Children)) var srcNode *Node for _, child := range srcParentNode.Children { if child.Name == srcName { srcNode = child } else { newChildren = append(newChildren, child) } } if srcNode == nil { return nil, errs.ObjectNotFound } srcParentNode.Children = newChildren srcNode.setLevel(dstNode.Level + 1) dstNode.Children = append(dstNode.Children, srcNode) d.root.calSize() d.updateStorage() return nodeToObj(srcNode, stdpath.Join(dstDir.GetPath(), srcName)) } func (d *Urls) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { if !d.Writable { return nil, errs.PermissionDenied } d.mutex.Lock() defer d.mutex.Unlock() srcNode := GetNodeFromRootByPath(d.root, srcObj.GetPath()) if srcNode == nil { return nil, errs.ObjectNotFound } srcNode.Name = newName d.updateStorage() return nodeToObj(srcNode, stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName)) } func (d *Urls) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if !d.Writable { return nil, errs.PermissionDenied } if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { return nil, errors.New("cannot copy parent dir to child") } d.mutex.Lock() defer d.mutex.Unlock() dstNode := GetNodeFromRootByPath(d.root, dstDir.GetPath()) if dstNode == nil || dstNode.isFile() { return nil, errs.NotFolder } srcNode := GetNodeFromRootByPath(d.root, srcObj.GetPath()) if srcNode == nil { return nil, errs.ObjectNotFound } newNode := srcNode.deepCopy(dstNode.Level + 1) dstNode.Children = append(dstNode.Children, newNode) d.root.calSize() d.updateStorage() return nodeToObj(newNode, stdpath.Join(dstDir.GetPath(), stdpath.Base(srcObj.GetPath()))) } func (d *Urls) Remove(ctx context.Context, obj model.Obj) error { if !d.Writable { return errs.PermissionDenied } d.mutex.Lock() defer d.mutex.Unlock() objDir, objName := stdpath.Split(obj.GetPath()) nodeParent := GetNodeFromRootByPath(d.root, objDir) if nodeParent == nil { return errs.ObjectNotFound } newChildren := make([]*Node, 0, len(nodeParent.Children)) var deletedObj *Node for _, child := range nodeParent.Children { if child.Name != objName { newChildren = append(newChildren, child) } else { deletedObj = child } } if deletedObj == nil { return errs.ObjectNotFound } nodeParent.Children = newChildren if deletedObj.Size > 0 { d.root.calSize() } d.updateStorage() return nil } func (d *Urls) PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) { if !d.Writable { return nil, errs.PermissionDenied } d.mutex.Lock() defer d.mutex.Unlock() dirNode := GetNodeFromRootByPath(d.root, dstDir.GetPath()) if dirNode == nil || dirNode.isFile() { return nil, errs.NotFolder } newNode := &Node{ Name: name, Level: dirNode.Level + 1, Url: url, } dirNode.Children = append(dirNode.Children, newNode) if d.HeadSize { size, err := getSizeFromUrl(url) if err != nil { log.Errorf("get size from url error: %s", err) } else { newNode.Size = size d.root.calSize() } } d.updateStorage() return nodeToObj(newNode, stdpath.Join(dstDir.GetPath(), name)) } func (d *Urls) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { if !d.Writable { return errs.PermissionDenied } d.mutex.Lock() defer d.mutex.Unlock() node := GetNodeFromRootByPath(d.root, dstDir.GetPath()) // parent if node == nil { return errs.ObjectNotFound } if node.isFile() { return errs.NotFolder } file, err := parseFileLine(stream.GetName(), d.HeadSize) if err != nil { return err } node.Children = append(node.Children, file) d.updateStorage() return nil } func (d *Urls) updateStorage() { d.UrlStructure = StringifyTree(d.root) op.MustSaveDriverStorage(d) } //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*Urls)(nil) ================================================ FILE: drivers/url_tree/meta.go ================================================ package url_tree import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { UrlStructure string `json:"url_structure" type:"text" required:"true" default:"https://raw.githubusercontent.com/OpenListTeam/OpenList/main/README.md\nhttps://raw.githubusercontent.com/OpenListTeam/OpenList/main/README_cn.md\nfolder:\n CONTRIBUTING.md:1635:https://raw.githubusercontent.com/OpenListTeam/OpenList/main/CONTRIBUTING.md\n CODE_OF_CONDUCT.md:2093:https://raw.githubusercontent.com/OpenListTeam/OpenList/main/CODE_OF_CONDUCT.md" help:"structure:FolderName:\n [FileName:][FileSize:][Modified:]Url"` HeadSize bool `json:"head_size" type:"bool" default:"false" help:"Use head method to get file size, but it may be failed."` Writable bool `json:"writable" type:"bool" default:"false"` } var config = driver.Config{ Name: "UrlTree", LocalSort: true, NoCache: true, CheckStatus: true, OnlyIndices: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Urls{} }) } ================================================ FILE: drivers/url_tree/types.go ================================================ package url_tree import "github.com/OpenListTeam/OpenList/v4/pkg/utils" // Node is a node in the folder tree type Node struct { Url string Name string Level int Modified int64 Size int64 Children []*Node } func (node *Node) getByPath(paths []string) *Node { if len(paths) == 0 || node == nil { return nil } if node.Name != paths[0] { return nil } if len(paths) == 1 { return node } for _, child := range node.Children { tmp := child.getByPath(paths[1:]) if tmp != nil { return tmp } } return nil } func (node *Node) isFile() bool { return node.Url != "" } func (node *Node) calSize() int64 { if node.isFile() { return node.Size } var size int64 = 0 for _, child := range node.Children { size += child.calSize() } node.Size = size return size } func (node *Node) setLevel(level int) { node.Level = level for _, child := range node.Children { child.setLevel(level + 1) } } func (node *Node) deepCopy(level int) *Node { ret := *node ret.Level = level ret.Children, _ = utils.SliceConvert(ret.Children, func(child *Node) (*Node, error) { return child.deepCopy(level + 1), nil }) return &ret } ================================================ FILE: drivers/url_tree/urls_test.go ================================================ package url_tree_test import ( "testing" "github.com/OpenListTeam/OpenList/v4/drivers/url_tree" ) func testTree() (*url_tree.Node, error) { text := `folder1: name1:https://url1 http://url2 folder2: http://url3 http://url4 http://url5 folder3: http://url6 http://url7 http://url8` return url_tree.BuildTree(text, false) } func TestBuildTree(t *testing.T) { node, err := testTree() if err != nil { t.Errorf("failed to build tree: %+v", err) } else { t.Logf("tree: %+v", node) } } func TestGetNode(t *testing.T) { root, err := testTree() if err != nil { t.Errorf("failed to build tree: %+v", err) return } node := url_tree.GetNodeFromRootByPath(root, "/") if node != root { t.Errorf("got wrong node: %+v", node) } url3 := url_tree.GetNodeFromRootByPath(root, "/folder1/folder2/url3") if url3 != root.Children[0].Children[2].Children[0] { t.Errorf("got wrong node: %+v", url3) } } ================================================ FILE: drivers/url_tree/util.go ================================================ package url_tree import ( "fmt" stdpath "path" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" log "github.com/sirupsen/logrus" ) // build tree from text, text structure definition: /** * FolderName: * [FileName:][FileSize:][Modified:]Url */ /** * For example: * folder1: * name1:url1 * url2 * folder2: * url3 * url4 * url5 * folder3: * url6 * url7 * url8 */ // if there are no name, use the last segment of url as name func BuildTree(text string, headSize bool) (*Node, error) { lines := strings.Split(text, "\n") var root = &Node{Level: -1, Name: "root"} stack := []*Node{root} for _, line := range lines { // calculate indent indent := 0 for i := 0; i < len(line); i++ { if line[i] != ' ' { break } indent++ } // if indent is not a multiple of 2, it is an error if indent%2 != 0 { return nil, fmt.Errorf("the line '%s' is not a multiple of 2", line) } // calculate level level := indent / 2 line = strings.TrimSpace(line[indent:]) // if the line is empty, skip if line == "" { continue } // if level isn't greater than the level of the top of the stack // it is not the child of the top of the stack for level <= stack[len(stack)-1].Level { // pop the top of the stack stack = stack[:len(stack)-1] } // if the line is a folder if isFolder(line) { // create a new node node := &Node{ Level: level, Name: strings.TrimSuffix(line, ":"), } // add the node to the top of the stack stack[len(stack)-1].Children = append(stack[len(stack)-1].Children, node) // push the node to the stack stack = append(stack, node) } else { // if the line is a file // create a new node node, err := parseFileLine(line, headSize) if err != nil { return nil, err } node.Level = level // add the node to the top of the stack stack[len(stack)-1].Children = append(stack[len(stack)-1].Children, node) } } return root, nil } func isFolder(line string) bool { return strings.HasSuffix(line, ":") } // line definition: // [FileName:][FileSize:][Modified:]Url func parseFileLine(line string, headSize bool) (*Node, error) { // if there is no url, it is an error if !strings.Contains(line, "http://") && !strings.Contains(line, "https://") { return nil, fmt.Errorf("invalid line: %s, because url is required for file", line) } index := strings.Index(line, "http://") if index == -1 { index = strings.Index(line, "https://") } url := line[index:] info := line[:index] node := &Node{ Url: url, } haveSize := false if index > 0 { if !strings.HasSuffix(info, ":") { return nil, fmt.Errorf("invalid line: %s, because file info must end with ':'", line) } info = info[:len(info)-1] if info == "" { return nil, fmt.Errorf("invalid line: %s, because file name can't be empty", line) } infoParts := strings.Split(info, ":") node.Name = infoParts[0] if len(infoParts) > 1 { size, err := strconv.ParseInt(infoParts[1], 10, 64) if err != nil { return nil, fmt.Errorf("invalid line: %s, because file size must be an integer", line) } node.Size = size haveSize = true if len(infoParts) > 2 { modified, err := strconv.ParseInt(infoParts[2], 10, 64) if err != nil { return nil, fmt.Errorf("invalid line: %s, because file modified must be an unix timestamp", line) } node.Modified = modified } } } else { node.Name = stdpath.Base(url) } if !haveSize && headSize { size, err := getSizeFromUrl(url) if err != nil { log.Errorf("get size from url error: %s", err) } else { node.Size = size } } return node, nil } func splitPath(path string) []string { if path == "/" { return []string{"root"} } if strings.HasSuffix(path, "/") { path = path[:len(path)-1] } parts := strings.Split(path, "/") parts[0] = "root" return parts } func GetNodeFromRootByPath(root *Node, path string) *Node { return root.getByPath(splitPath(path)) } func nodeToObj(node *Node, path string) (model.Obj, error) { if node == nil { return nil, errs.ObjectNotFound } return &model.Object{ Name: node.Name, Size: node.Size, Modified: time.Unix(node.Modified, 0), IsFolder: !node.isFile(), Path: path, }, nil } func getSizeFromUrl(url string) (int64, error) { res, err := base.RestyClient.R().SetDoNotParseResponse(true).Head(url) if err != nil { return 0, err } defer res.RawResponse.Body.Close() if res.StatusCode() >= 300 { return 0, fmt.Errorf("get size from url %s failed, status code: %d", url, res.StatusCode()) } size, err := strconv.ParseInt(res.Header().Get("Content-Length"), 10, 64) if err != nil { return 0, err } return size, nil } func StringifyTree(node *Node) string { sb := strings.Builder{} if node.Level == -1 { for i, child := range node.Children { sb.WriteString(StringifyTree(child)) if i < len(node.Children)-1 { sb.WriteString("\n") } } return sb.String() } for i := 0; i < node.Level; i++ { sb.WriteString(" ") } if node.Url == "" { sb.WriteString(node.Name) sb.WriteString(":") for _, child := range node.Children { sb.WriteString("\n") sb.WriteString(StringifyTree(child)) } } else if node.Size == 0 && node.Modified == 0 { if stdpath.Base(node.Url) == node.Name { sb.WriteString(node.Url) } else { sb.WriteString(fmt.Sprintf("%s:%s", node.Name, node.Url)) } } else { sb.WriteString(node.Name) sb.WriteString(":") if node.Size != 0 || node.Modified != 0 { sb.WriteString(strconv.FormatInt(node.Size, 10)) sb.WriteString(":") } if node.Modified != 0 { sb.WriteString(strconv.FormatInt(node.Modified, 10)) sb.WriteString(":") } sb.WriteString(node.Url) } return sb.String() } ================================================ FILE: drivers/uss/driver.go ================================================ package uss import ( "context" "fmt" "net/url" "path" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/upyun/go-sdk/v3/upyun" ) type USS struct { model.Storage Addition client *upyun.UpYun } func (d *USS) Config() driver.Config { return config } func (d *USS) GetAddition() driver.Additional { return &d.Addition } func (d *USS) Init(ctx context.Context) error { d.client = upyun.NewUpYun(&upyun.UpYunConfig{ Bucket: d.Bucket, Operator: d.OperatorName, Password: d.OperatorPassword, }) return nil } func (d *USS) Drop(ctx context.Context) error { return nil } func (d *USS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { prefix := getKey(dir.GetPath(), true) objsChan := make(chan *upyun.FileInfo, 10) var err error go func() { err = d.client.List(&upyun.GetObjectsConfig{ Path: prefix, ObjectsChan: objsChan, MaxListObjects: 0, MaxListLevel: 1, }) }() if err != nil { return nil, err } res := make([]model.Obj, 0) for obj := range objsChan { t := obj.Time f := model.Object{ Path: path.Join(dir.GetPath(), obj.Name), Name: obj.Name, Size: obj.Size, Modified: t, IsFolder: obj.IsDir, } res = append(res, &f) } return res, err } func (d *USS) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { key := getKey(file.GetPath(), false) host := d.Endpoint if !strings.Contains(host, "://") { //判断是否包含协议头，否则https host = "https://" + host } u := fmt.Sprintf("%s/%s", host, key) downExp := time.Hour * time.Duration(d.SignURLExpire) expireAt := time.Now().Add(downExp).Unix() upd := url.QueryEscape(path.Base(file.GetPath())) tokenOrPassword := d.AntiTheftChainToken if tokenOrPassword == "" { tokenOrPassword = d.OperatorPassword } signStr := strings.Join([]string{tokenOrPassword, fmt.Sprint(expireAt), fmt.Sprintf("/%s", key)}, "&") upt := utils.GetMD5EncodeStr(signStr)[12:20] + fmt.Sprint(expireAt) link := fmt.Sprintf("%s?_upd=%s&_upt=%s", u, upd, upt) return &model.Link{URL: link}, nil } func (d *USS) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { return d.client.Mkdir(getKey(path.Join(parentDir.GetPath(), dirName), true)) } func (d *USS) Move(ctx context.Context, srcObj, dstDir model.Obj) error { return d.client.Move(&upyun.MoveObjectConfig{ SrcPath: getKey(srcObj.GetPath(), srcObj.IsDir()), DestPath: getKey(path.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.IsDir()), }) } func (d *USS) Rename(ctx context.Context, srcObj model.Obj, newName string) error { return d.client.Move(&upyun.MoveObjectConfig{ SrcPath: getKey(srcObj.GetPath(), srcObj.IsDir()), DestPath: getKey(path.Join(path.Dir(srcObj.GetPath()), newName), srcObj.IsDir()), }) } func (d *USS) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return d.client.Copy(&upyun.CopyObjectConfig{ SrcPath: getKey(srcObj.GetPath(), srcObj.IsDir()), DestPath: getKey(path.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.IsDir()), }) } func (d *USS) Remove(ctx context.Context, obj model.Obj) error { return d.client.Delete(&upyun.DeleteObjectConfig{ Path: getKey(obj.GetPath(), obj.IsDir()), Async: false, }) } func (d *USS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { return d.client.Put(&upyun.PutObjectConfig{ Path: getKey(path.Join(dstDir.GetPath(), s.GetName()), false), Reader: driver.NewLimitedUploadStream(ctx, &stream.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, }), }) } var _ driver.Driver = (*USS)(nil) ================================================ FILE: drivers/uss/meta.go ================================================ package uss import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Bucket string `json:"bucket" required:"true"` Endpoint string `json:"endpoint" required:"true"` OperatorName string `json:"operator_name" required:"true"` OperatorPassword string `json:"operator_password" required:"true"` AntiTheftChainToken string `json:"anti_theft_chain_token" required:"false" default:""` //CustomHost string `json:"custom_host"` //Endpoint与CustomHost作用相同，去除 SignURLExpire int `json:"sign_url_expire" type:"number" default:"4"` } var config = driver.Config{ Name: "USS", LocalSort: true, DefaultRoot: "/", } func init() { op.RegisterDriver(func() driver.Driver { return &USS{} }) } ================================================ FILE: drivers/uss/types.go ================================================ package uss ================================================ FILE: drivers/uss/util.go ================================================ package uss import "strings" // do others that not defined in Driver interface func getKey(path string, dir bool) string { path = strings.TrimPrefix(path, "/") if dir { path += "/" } return path } ================================================ FILE: drivers/virtual/driver.go ================================================ package virtual import ( "context" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" ) type Virtual struct { model.Storage Addition } func (d *Virtual) Config() driver.Config { return config } func (d *Virtual) Init(ctx context.Context) error { return nil } func (d *Virtual) Drop(ctx context.Context) error { return nil } func (d *Virtual) GetAddition() driver.Additional { return &d.Addition } func (d *Virtual) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var res []model.Obj for i := 0; i < d.NumFile; i++ { res = append(res, d.genObj(false)) } for i := 0; i < d.NumFolder; i++ { res = append(res, d.genObj(true)) } return res, nil } type DummyMFile struct{} func (f DummyMFile) Read(p []byte) (n int, err error) { return random.Rand.Read(p) } func (f DummyMFile) ReadAt(p []byte, off int64) (n int, err error) { return random.Rand.Read(p) } func (DummyMFile) Seek(offset int64, whence int) (int64, error) { return offset, nil } func (d *Virtual) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { return &model.Link{ RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), DummyMFile{}), }, nil } func (d *Virtual) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { dir := &model.Object{ Name: dirName, Size: 0, IsFolder: true, Modified: time.Now(), } return dir, nil } func (d *Virtual) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return srcObj, nil } func (d *Virtual) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { obj := &model.Object{ Name: newName, Size: srcObj.GetSize(), IsFolder: srcObj.IsDir(), Modified: time.Now(), } return obj, nil } func (d *Virtual) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { return srcObj, nil } func (d *Virtual) Remove(ctx context.Context, obj model.Obj) error { return nil } func (d *Virtual) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { file := &model.Object{ Name: stream.GetName(), Size: stream.GetSize(), Modified: time.Now(), } return file, nil } var _ driver.Driver = (*Virtual)(nil) ================================================ FILE: drivers/virtual/meta.go ================================================ package virtual import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath NumFile int `json:"num_file" type:"number" default:"30" required:"true"` NumFolder int `json:"num_folder" type:"number" default:"30" required:"true"` MaxFileSize int64 `json:"max_file_size" type:"number" default:"1073741824" required:"true"` MinFileSize int64 `json:"min_file_size" type:"number" default:"1048576" required:"true"` } var config = driver.Config{ Name: "Virtual", LocalSort: true, OnlyProxy: true, NeedMs: true, NoLinkURL: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Virtual{} }) } ================================================ FILE: drivers/virtual/util.go ================================================ package virtual import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" ) func (d *Virtual) genObj(dir bool) model.Obj { obj := &model.Object{ Name: random.String(10), Size: 0, IsFolder: true, Modified: time.Now(), } if !dir { obj.Size = random.RangeInt64(d.MinFileSize, d.MaxFileSize) obj.IsFolder = false } return obj } ================================================ FILE: drivers/webdav/driver.go ================================================ package webdav import ( "context" "fmt" "net/http" "os" "path" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/gowebdav" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type WebDav struct { model.Storage Addition client *gowebdav.Client cron *cron.Cron } func (d *WebDav) Config() driver.Config { return config } func (d *WebDav) GetAddition() driver.Additional { return &d.Addition } func (d *WebDav) Init(ctx context.Context) error { err := d.setClient() if err == nil { d.cron = cron.NewCron(time.Hour * 12) d.cron.Do(func() { _ = d.setClient() }) } return err } func (d *WebDav) Drop(ctx context.Context) error { if d.cron != nil { d.cron.Stop() } return nil } func (d *WebDav) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.client.ReadDir(dir.GetPath()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src os.FileInfo) (model.Obj, error) { return &model.Object{ Path: path.Join(dir.GetPath(), src.Name()), Name: src.Name(), Size: src.Size(), Modified: src.ModTime(), IsFolder: src.IsDir(), }, nil }) } func (d *WebDav) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { url, header, err := d.client.Link(file.GetPath()) if err != nil { return nil, err } if args.Redirect { // get the url after redirect req := base.NoRedirectClient.R() req.Header = header req.SetDoNotParseResponse(true) res, err := req.Get(url) if err != nil { return nil, err } _ = res.RawResponse.Body.Close() if (res.StatusCode() == 302 || res.StatusCode() == 307 || res.StatusCode() == 308) && res.Header().Get("location") != "" { url = res.Header().Get("location") } else { return nil, fmt.Errorf("redirect failed, status: %d", res.StatusCode()) } } return &model.Link{ URL: url, Header: header, }, nil } func (d *WebDav) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { return d.client.MkdirAll(path.Join(parentDir.GetPath(), dirName), 0644) } func (d *WebDav) Move(ctx context.Context, srcObj, dstDir model.Obj) error { return d.client.Rename(getPath(srcObj), path.Join(dstDir.GetPath(), srcObj.GetName()), true) } func (d *WebDav) Rename(ctx context.Context, srcObj model.Obj, newName string) error { return d.client.Rename(getPath(srcObj), path.Join(path.Dir(srcObj.GetPath()), newName), true) } func (d *WebDav) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return d.client.Copy(getPath(srcObj), path.Join(dstDir.GetPath(), srcObj.GetName()), true) } func (d *WebDav) Remove(ctx context.Context, obj model.Obj) error { return d.client.RemoveAll(getPath(obj)) } func (d *WebDav) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { callback := func(r *http.Request) { r.Header.Set("Content-Type", s.GetMimetype()) r.ContentLength = s.GetSize() } reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, }) err := d.client.WriteStream(path.Join(dstDir.GetPath(), s.GetName()), reader, 0644, callback) return err } var _ driver.Driver = (*WebDav)(nil) ================================================ FILE: drivers/webdav/meta.go ================================================ package webdav import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { Vendor string `json:"vendor" type:"select" options:"sharepoint,other" default:"other"` Address string `json:"address" required:"true"` Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` driver.RootPath TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" default:"false"` } var config = driver.Config{ Name: "WebDav", LocalSort: true, DefaultRoot: "/", PreferProxy: true, } func init() { op.RegisterDriver(func() driver.Driver { return &WebDav{} }) } ================================================ FILE: drivers/webdav/odrvcookie/cookie.go ================================================ package odrvcookie import ( "net/http" "github.com/OpenListTeam/OpenList/v4/pkg/cookie" ) //type SpCookie struct { // Cookie string // expire time.Time //} // //func (sp SpCookie) IsExpire() bool { // return time.Now().After(sp.expire) //} // //var cookiesMap = struct { // sync.Mutex // m map[string]*SpCookie //}{m: make(map[string]*SpCookie)} func GetCookie(username, password, siteUrl string) (string, error) { //cookiesMap.Lock() //defer cookiesMap.Unlock() //spCookie, ok := cookiesMap.m[username] //if ok { // if !spCookie.IsExpire() { // log.Debugln("sp use old cookie.") // return spCookie.Cookie, nil // } //} //log.Debugln("fetch new cookie") ca := New(username, password, siteUrl) tokenConf, err := ca.Cookies() if err != nil { return "", err } return cookie.ToString([]*http.Cookie{&tokenConf.RtFa, &tokenConf.FedAuth}), nil //spCookie = &SpCookie{ // Cookie: cookie.ToString([]*http.Cookie{&tokenConf.RtFa, &tokenConf.FedAuth}), // expire: time.Now().Add(time.Hour * 12), //} //cookiesMap.m[username] = spCookie //return spCookie.Cookie, nil } ================================================ FILE: drivers/webdav/odrvcookie/fetch.go ================================================ // Package odrvcookie can fetch authentication cookies for a sharepoint webdav endpoint package odrvcookie import ( "bytes" "encoding/xml" "fmt" "html/template" "net/http" "net/http/cookiejar" "net/url" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "golang.org/x/net/publicsuffix" ) // CookieAuth hold the authentication information // These are username and password as well as the authentication endpoint type CookieAuth struct { user string pass string endpoint string } // CookieResponse contains the requested cookies type CookieResponse struct { RtFa http.Cookie FedAuth http.Cookie } // SuccessResponse hold a response from the sharepoint webdav type SuccessResponse struct { XMLName xml.Name `xml:"Envelope"` Succ SuccessResponseBody `xml:"Body"` } // SuccessResponseBody is the body of a success response, it holds the token type SuccessResponseBody struct { XMLName xml.Name Type string `xml:"RequestSecurityTokenResponse>TokenType"` Created time.Time `xml:"RequestSecurityTokenResponse>Lifetime>Created"` Expires time.Time `xml:"RequestSecurityTokenResponse>Lifetime>Expires"` Token string `xml:"RequestSecurityTokenResponse>RequestedSecurityToken>BinarySecurityToken"` } // reqString is a template that gets populated with the user data in order to retrieve a "BinarySecurityToken" const reqString = ` http://schemas.xmlsoap.org/ws/2005/02/trust/RST/Issue http://www.w3.org/2005/08/addressing/anonymous {{ .LoginUrl }} {{ .Username }} {{ .Password }} {{ .Address }} http://schemas.xmlsoap.org/ws/2005/05/identity/NoProofKey http://schemas.xmlsoap.org/ws/2005/02/trust/Issue urn:oasis:names:tc:SAML:1.0:assertion ` // New creates a new CookieAuth struct func New(pUser, pPass, pEndpoint string) CookieAuth { retStruct := CookieAuth{ user: pUser, pass: pPass, endpoint: pEndpoint, } return retStruct } // Cookies creates a CookieResponse. It fetches the auth token and then // retrieves the Cookies func (ca *CookieAuth) Cookies() (CookieResponse, error) { spToken, err := ca.getSPToken() if err != nil { return CookieResponse{}, err } return ca.getSPCookie(spToken) } func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (CookieResponse, error) { spRoot, err := url.Parse(ca.endpoint) if err != nil { return CookieResponse{}, err } u, err := url.Parse("https://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0") if err != nil { return CookieResponse{}, err } // To authenticate with davfs or anything else we need two cookies (rtFa and FedAuth) // In order to get them we use the token we got earlier and a cookieJar jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) if err != nil { return CookieResponse{}, err } client := &http.Client{ Jar: jar, } // Send the previously acquired Token as a Post parameter if _, err = client.Post(u.String(), "text/xml", strings.NewReader(conf.Succ.Token)); err != nil { return CookieResponse{}, err } cookieResponse := CookieResponse{} for _, cookie := range jar.Cookies(u) { if (cookie.Name == "rtFa") || (cookie.Name == "FedAuth") { switch cookie.Name { case "rtFa": cookieResponse.RtFa = *cookie case "FedAuth": cookieResponse.FedAuth = *cookie } } } return cookieResponse, err } var loginUrlsMap = map[string]string{ "com": "https://login.microsoftonline.com", "cn": "https://login.chinacloudapi.cn", "us": "https://login.microsoftonline.us", "de": "https://login.microsoftonline.de", } func getLoginUrl(endpoint string) (string, error) { spRoot, err := url.Parse(endpoint) if err != nil { return "", err } domains := strings.Split(spRoot.Host, ".") tld := domains[len(domains)-1] loginUrl, ok := loginUrlsMap[tld] if !ok { return "", fmt.Errorf("tld %s is not supported", tld) } return loginUrl + "/extSTS.srf", nil } func (ca *CookieAuth) getSPToken() (*SuccessResponse, error) { loginUrl, err := getLoginUrl(ca.endpoint) if err != nil { return nil, err } reqData := map[string]string{ "Username": ca.user, "Password": ca.pass, "Address": ca.endpoint, "LoginUrl": loginUrl, } t := template.Must(template.New("authXML").Parse(reqString)) buf := &bytes.Buffer{} if err := t.Execute(buf, reqData); err != nil { return nil, err } // Execute the first request which gives us an auth token for the sharepoint service // With this token we can authenticate on the login page and save the returned cookies req, err := http.NewRequest(http.MethodPost, loginUrl, buf) if err != nil { return nil, err } client := base.HttpClient resp, err := client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() respBuf := bytes.Buffer{} respBuf.ReadFrom(resp.Body) s := respBuf.Bytes() var conf SuccessResponse err = xml.Unmarshal(s, &conf) if err != nil { return nil, err } return &conf, err } ================================================ FILE: drivers/webdav/types.go ================================================ package webdav ================================================ FILE: drivers/webdav/util.go ================================================ package webdav import ( "crypto/tls" "net/http" "net/http/cookiejar" "github.com/OpenListTeam/OpenList/v4/drivers/webdav/odrvcookie" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/gowebdav" ) // do others that not defined in Driver interface func (d *WebDav) isSharepoint() bool { return d.Vendor == "sharepoint" } func (d *WebDav) setClient() error { c := gowebdav.NewClient(d.Address, d.Username, d.Password) c.SetTransport(&http.Transport{ Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{InsecureSkipVerify: d.TlsInsecureSkipVerify}, }) if d.isSharepoint() { cookie, err := odrvcookie.GetCookie(d.Username, d.Password, d.Address) if err == nil { c.SetInterceptor(func(method string, rq *http.Request) { rq.Header.Del("Authorization") rq.Header.Set("Cookie", cookie) }) } else { return err } } else { cookieJar, err := cookiejar.New(nil) if err == nil { c.SetJar(cookieJar) } else { return err } } d.client = c return nil } func getPath(obj model.Obj) string { if obj.IsDir() { return obj.GetPath() + "/" } return obj.GetPath() } ================================================ FILE: drivers/weiyun/driver.go ================================================ package weiyun import ( "context" "fmt" "io" "math" "net/http" "strconv" "sync/atomic" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" weiyunsdkgo "github.com/foxxorcat/weiyun-sdk-go" "github.com/go-resty/resty/v2" ) type WeiYun struct { model.Storage Addition client *weiyunsdkgo.WeiYunClient cron *cron.Cron rootFolder *Folder uploadThread int } func (d *WeiYun) Config() driver.Config { return config } func (d *WeiYun) GetAddition() driver.Additional { return &d.Addition } func (d *WeiYun) Init(ctx context.Context) error { // 限制上传线程数 d.uploadThread, _ = strconv.Atoi(d.UploadThread) if d.uploadThread < 4 || d.uploadThread > 32 { d.uploadThread, d.UploadThread = 4, "4" } d.client = weiyunsdkgo.NewWeiYunClientWithRestyClient(base.NewRestyClient()) err := d.client.SetCookiesStr(d.Cookies).RefreshCtoken() if err != nil { return err } // Cookie过期回调 d.client.SetOnCookieExpired(func(err error) { d.Status = err.Error() op.MustSaveDriverStorage(d) }) // cookie更新回调 d.client.SetOnCookieUpload(func(c []*http.Cookie) { d.Cookies = weiyunsdkgo.CookieToString(weiyunsdkgo.ClearCookie(c)) op.MustSaveDriverStorage(d) }) // qqCookie保活 if d.client.LoginType() == weiyunsdkgo.AccountTypeQQ || d.client.LoginType() == weiyunsdkgo.AccountTypeQQOpenID { d.cron = cron.NewCron(time.Minute * 5) d.cron.Do(func() { _ = d.client.KeepAlive() }) } // 获取默认根目录dirKey if d.RootFolderID == "" { userInfo, err := d.client.DiskUserInfoGet() if err != nil { return err } d.RootFolderID = userInfo.MainDirKey } // 处理目录ID，找到PdirKey folders, err := d.client.LibDirPathGet(d.RootFolderID) if err != nil { return err } if len(folders) == 0 { return fmt.Errorf("invalid directory ID") } folder := folders[len(folders)-1] d.rootFolder = &Folder{ PFolder: &Folder{ Folder: weiyunsdkgo.Folder{ DirKey: folder.PdirKey, }, }, Folder: folder.Folder, } return nil } func (d *WeiYun) Drop(ctx context.Context) error { d.client = nil if d.cron != nil { d.cron.Stop() d.cron = nil } return nil } func (d *WeiYun) GetRoot(ctx context.Context) (model.Obj, error) { return d.rootFolder, nil } func (d *WeiYun) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { if folder, ok := dir.(*Folder); ok { var files []model.Obj for { data, err := d.client.DiskDirFileList(folder.GetID(), weiyunsdkgo.WarpParamOption( weiyunsdkgo.QueryFileOptionOffest(int64(len(files))), weiyunsdkgo.QueryFileOptionGetType(weiyunsdkgo.FileAndDir), weiyunsdkgo.QueryFileOptionSort(func() weiyunsdkgo.OrderBy { switch d.OrderBy { case "name": return weiyunsdkgo.FileName case "size": return weiyunsdkgo.FileSize case "updated_at": return weiyunsdkgo.FileMtime default: return weiyunsdkgo.FileName } }(), d.OrderDirection == "desc"), )) if err != nil { return nil, err } if files == nil { files = make([]model.Obj, 0, data.TotalDirCount+data.TotalFileCount) } for _, dir := range data.DirList { files = append(files, &Folder{ PFolder: folder, Folder: dir, }) } for _, file := range data.FileList { files = append(files, &File{ PFolder: folder, File: file, }) } if data.FinishFlag || len(data.DirList)+len(data.FileList) == 0 { return files, nil } } } return nil, errs.NotSupport } func (d *WeiYun) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if file, ok := file.(*File); ok { data, err := d.client.DiskFileDownload(weiyunsdkgo.FileParam{PdirKey: file.GetPKey(), FileID: file.GetID()}) if err != nil { return nil, err } return &model.Link{ URL: data.DownloadUrl, Header: http.Header{ "Cookie": []string{data.CookieName + "=" + data.CookieValue}, }, }, nil } return nil, errs.NotSupport } func (d *WeiYun) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if folder, ok := parentDir.(*Folder); ok { newFolder, err := d.client.DiskDirCreate(weiyunsdkgo.FolderParam{ PPdirKey: folder.GetPKey(), PdirKey: folder.DirKey, DirName: dirName, }) if err != nil { return nil, err } return &Folder{ PFolder: folder, Folder: *newFolder, }, nil } return nil, errs.NotSupport } func (d *WeiYun) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { // TODO: 默认策略为重命名，使用缓存可能出现冲突。微云app也有这个冲突，不知道腾讯怎么搞的 if dstDir, ok := dstDir.(*Folder); ok { dstParam := weiyunsdkgo.FolderParam{ PdirKey: dstDir.GetPKey(), DirKey: dstDir.GetID(), DirName: dstDir.GetName(), } switch srcObj := srcObj.(type) { case *File: err := d.client.DiskFileMove(weiyunsdkgo.FileParam{ PPdirKey: srcObj.PFolder.GetPKey(), PdirKey: srcObj.GetPKey(), FileID: srcObj.GetID(), FileName: srcObj.GetName(), }, dstParam) if err != nil { return nil, err } return &File{ PFolder: dstDir, File: srcObj.File, }, nil case *Folder: err := d.client.DiskDirMove(weiyunsdkgo.FolderParam{ PPdirKey: srcObj.PFolder.GetPKey(), PdirKey: srcObj.GetPKey(), DirKey: srcObj.GetID(), DirName: srcObj.GetName(), }, dstParam) if err != nil { return nil, err } return &Folder{ PFolder: dstDir, Folder: srcObj.Folder, }, nil } } return nil, errs.NotSupport } func (d *WeiYun) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { switch srcObj := srcObj.(type) { case *File: err := d.client.DiskFileRename(weiyunsdkgo.FileParam{ PPdirKey: srcObj.PFolder.GetPKey(), PdirKey: srcObj.GetPKey(), FileID: srcObj.GetID(), FileName: srcObj.GetName(), }, newName) if err != nil { return nil, err } newFile := srcObj.File newFile.FileName = newName newFile.FileCtime = weiyunsdkgo.TimeStamp(time.Now()) return &File{ PFolder: srcObj.PFolder, File: newFile, }, nil case *Folder: err := d.client.DiskDirAttrModify(weiyunsdkgo.FolderParam{ PPdirKey: srcObj.PFolder.GetPKey(), PdirKey: srcObj.GetPKey(), DirKey: srcObj.GetID(), DirName: srcObj.GetName(), }, newName) if err != nil { return nil, err } newFolder := srcObj.Folder newFolder.DirName = newName newFolder.DirCtime = weiyunsdkgo.TimeStamp(time.Now()) return &Folder{ PFolder: srcObj.PFolder, Folder: newFolder, }, nil } return nil, errs.NotSupport } func (d *WeiYun) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return errs.NotImplement } func (d *WeiYun) Remove(ctx context.Context, obj model.Obj) error { switch obj := obj.(type) { case *File: return d.client.DiskFileDelete(weiyunsdkgo.FileParam{ PPdirKey: obj.PFolder.GetPKey(), PdirKey: obj.GetPKey(), FileID: obj.GetID(), FileName: obj.GetName(), }) case *Folder: return d.client.DiskDirDelete(weiyunsdkgo.FolderParam{ PPdirKey: obj.PFolder.GetPKey(), PdirKey: obj.GetPKey(), DirKey: obj.GetID(), DirName: obj.GetName(), }) } return errs.NotSupport } func (d *WeiYun) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { // NOTE: // 秒传需要sha1最后一个状态,但sha1无法逆运算需要读完整个文件(或许可以??) // 服务器支持上传进度恢复,不需要额外实现 var folder *Folder var ok bool if folder, ok = dstDir.(*Folder); !ok { return nil, errs.NotSupport } file, err := stream.CacheFullAndWriter(&up, nil) if err != nil { return nil, err } // step 1. preData, err := d.client.PreUpload(ctx, weiyunsdkgo.UpdloadFileParam{ PdirKey: folder.GetPKey(), DirKey: folder.DirKey, FileName: stream.GetName(), FileSize: stream.GetSize(), File: file, ChannelCount: 4, FileExistOption: 1, }) if err != nil { return nil, err } // not fast upload if !preData.FileExist { // step.2 增加上传通道 if len(preData.ChannelList) < d.uploadThread { newCh, err := d.client.AddUploadChannel(len(preData.ChannelList), d.uploadThread, preData.UploadAuthData) if err != nil { return nil, err } preData.ChannelList = append(preData.ChannelList, newCh.AddChannels...) } // step.3 上传 threadG, upCtx := errgroup.NewGroupWithContext(ctx, len(preData.ChannelList), retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) total := atomic.Int64{} for _, channel := range preData.ChannelList { if utils.IsCanceled(upCtx) { break } var channel = channel threadG.Go(func(ctx context.Context) error { for { channel.Len = int(math.Min(float64(stream.GetSize()-channel.Offset), float64(channel.Len))) len64 := int64(channel.Len) upData, err := d.client.UploadFile(upCtx, channel, preData.UploadAuthData, driver.NewLimitedUploadStream(ctx, io.NewSectionReader(file, channel.Offset, len64))) if err != nil { return err } cur := total.Add(len64) up(float64(cur) * 100.0 / float64(stream.GetSize())) // 上传完成 if upData.UploadState != 1 { return nil } channel = upData.Channel } }) } if err = threadG.Wait(); err != nil { return nil, err } } return &File{ PFolder: folder, File: preData.File, }, nil } func (d *WeiYun) GetDetails(ctx context.Context) (*model.StorageDetails, error) { info, err := d.client.DiskUserInfoGet(func(request *resty.Request) { request.SetContext(ctx) }) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: info.TotalSpace, UsedSpace: info.UsedSpace, }, }, nil } // func (d *WeiYun) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport // } var _ driver.Driver = (*WeiYun)(nil) var _ driver.GetRooter = (*WeiYun)(nil) var _ driver.MkdirResult = (*WeiYun)(nil) // var _ driver.CopyResult = (*WeiYun)(nil) var _ driver.MoveResult = (*WeiYun)(nil) var _ driver.Remove = (*WeiYun)(nil) var _ driver.PutResult = (*WeiYun)(nil) var _ driver.RenameResult = (*WeiYun)(nil) var _ driver.WithDetails = (*WeiYun)(nil) ================================================ FILE: drivers/weiyun/meta.go ================================================ package weiyun import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { RootFolderID string `json:"root_folder_id"` Cookies string `json:"cookies" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at" default:"name"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` UploadThread string `json:"upload_thread" default:"4" help:"4<=thread<=32"` } var config = driver.Config{ Name: "WeiYun", OnlyProxy: true, CheckStatus: true, } func init() { op.RegisterDriver(func() driver.Driver { return &WeiYun{} }) } ================================================ FILE: drivers/weiyun/types.go ================================================ package weiyun import ( "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" weiyunsdkgo "github.com/foxxorcat/weiyun-sdk-go" ) type File struct { PFolder *Folder weiyunsdkgo.File } func (f *File) GetID() string { return f.FileID } func (f *File) GetSize() int64 { return f.FileSize } func (f *File) GetName() string { return f.FileName } func (f *File) ModTime() time.Time { return time.Time(f.FileMtime) } func (f *File) IsDir() bool { return false } func (f *File) GetPath() string { return "" } func (f *File) GetPKey() string { return f.PFolder.DirKey } func (f *File) CreateTime() time.Time { return time.Time(f.FileCtime) } func (f *File) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.SHA1, f.FileSha) } type Folder struct { PFolder *Folder weiyunsdkgo.Folder } func (f *Folder) CreateTime() time.Time { return time.Time(f.DirCtime) } func (f *Folder) GetHash() utils.HashInfo { return utils.HashInfo{} } func (f *Folder) GetID() string { return f.DirKey } func (f *Folder) GetSize() int64 { return 0 } func (f *Folder) GetName() string { return f.DirName } func (f *Folder) ModTime() time.Time { return time.Time(f.DirMtime) } func (f *Folder) IsDir() bool { return true } func (f *Folder) GetPath() string { return "" } func (f *Folder) GetPKey() string { return f.PFolder.DirKey } ================================================ FILE: drivers/wopan/driver.go ================================================ package template import ( "context" "fmt" "strconv" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/wopan-sdk-go" "github.com/go-resty/resty/v2" ) type Wopan struct { model.Storage Addition client *wopan.WoClient defaultFamilyID string } func (d *Wopan) Config() driver.Config { return config } func (d *Wopan) GetAddition() driver.Additional { return &d.Addition } func (d *Wopan) Init(ctx context.Context) error { d.client = wopan.DefaultWithRefreshToken(d.RefreshToken) d.client.SetAccessToken(d.AccessToken) d.client.OnRefreshToken(func(accessToken, refreshToken string) { d.AccessToken = accessToken d.RefreshToken = refreshToken op.MustSaveDriverStorage(d) }) fml, err := d.client.FamilyUserCurrentEncode() if err != nil { return err } d.defaultFamilyID = strconv.Itoa(fml.DefaultHomeId) return d.client.InitData() } func (d *Wopan) Drop(ctx context.Context) error { return nil } func (d *Wopan) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var res []model.Obj pageNum := 0 pageSize := 100 for { data, err := d.client.QueryAllFiles(d.getSpaceType(), dir.GetID(), pageNum, pageSize, 0, d.FamilyID, func(req *resty.Request) { req.SetContext(ctx) }) if err != nil { return nil, err } objs, err := utils.SliceConvert(data.Files, fileToObj) if err != nil { return nil, err } res = append(res, objs...) if len(data.Files) < pageSize { break } pageNum++ } return res, nil } func (d *Wopan) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if f, ok := file.(*Object); ok { res, err := d.client.GetDownloadUrlV2([]string{f.FID}, func(req *resty.Request) { req.SetContext(ctx) }) if err != nil { return nil, err } return &model.Link{ URL: res.List[0].DownloadUrl, }, nil } return nil, fmt.Errorf("unable to convert file to Object") } func (d *Wopan) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { familyID := d.FamilyID if familyID == "" { familyID = d.defaultFamilyID } _, err := d.client.CreateDirectory(d.getSpaceType(), parentDir.GetID(), dirName, familyID, func(req *resty.Request) { req.SetContext(ctx) }) return err } func (d *Wopan) Move(ctx context.Context, srcObj, dstDir model.Obj) error { dirList := make([]string, 0) fileList := make([]string, 0) if srcObj.IsDir() { dirList = append(dirList, srcObj.GetID()) } else { fileList = append(fileList, srcObj.GetID()) } return d.client.MoveFile(dirList, fileList, dstDir.GetID(), d.getSpaceType(), d.getSpaceType(), d.FamilyID, d.FamilyID, func(req *resty.Request) { req.SetContext(ctx) }) } func (d *Wopan) Rename(ctx context.Context, srcObj model.Obj, newName string) error { _type := 1 if srcObj.IsDir() { _type = 0 } return d.client.RenameFileOrDirectory(d.getSpaceType(), _type, srcObj.GetID(), newName, d.FamilyID, func(req *resty.Request) { req.SetContext(ctx) }) } func (d *Wopan) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { dirList := make([]string, 0) fileList := make([]string, 0) if srcObj.IsDir() { dirList = append(dirList, srcObj.GetID()) } else { fileList = append(fileList, srcObj.GetID()) } return d.client.CopyFile(dirList, fileList, dstDir.GetID(), d.getSpaceType(), d.getSpaceType(), d.FamilyID, d.FamilyID, func(req *resty.Request) { req.SetContext(ctx) }) } func (d *Wopan) Remove(ctx context.Context, obj model.Obj) error { dirList := make([]string, 0) fileList := make([]string, 0) if obj.IsDir() { dirList = append(dirList, obj.GetID()) } else { fileList = append(fileList, obj.GetID()) } return d.client.DeleteFile(d.getSpaceType(), dirList, fileList, func(req *resty.Request) { req.SetContext(ctx) }) } func (d *Wopan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { _, err := d.client.Upload2C(d.getSpaceType(), wopan.Upload2CFile{ Name: stream.GetName(), Size: stream.GetSize(), Content: driver.NewLimitedUploadStream(ctx, stream), ContentType: stream.GetMimetype(), }, dstDir.GetID(), d.FamilyID, wopan.Upload2COption{ OnProgress: func(current, total int64) { up(100 * float64(current) / float64(total)) }, Ctx: ctx, }) return err } func (d *Wopan) GetDetails(ctx context.Context) (*model.StorageDetails, error) { quota, err := d.client.QueryCloudUsageInfo() if err != nil { return nil, err } total, err := strconv.ParseInt(quota.UsageInfo.ByteTotalSize, 10, 64) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, UsedSpace: quota.UsageInfo.ByteUsedSize, }, }, nil } //func (d *Wopan) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} var _ driver.Driver = (*Wopan)(nil) ================================================ FILE: drivers/wopan/meta.go ================================================ package template import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { // Usually one of two driver.RootID // define other RefreshToken string `json:"refresh_token" required:"true"` FamilyID string `json:"family_id" help:"Keep it empty if you want to use your personal drive"` SortRule string `json:"sort_rule" type:"select" options:"name_asc,name_desc,time_asc,time_desc,size_asc,size_desc" default:"name_asc"` AccessToken string `json:"access_token"` } var config = driver.Config{ Name: "WoPan", DefaultRoot: "0", NoOverwriteUpload: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Wopan{} }) } ================================================ FILE: drivers/wopan/types.go ================================================ package template import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/wopan-sdk-go" ) type Object struct { model.ObjThumb FID string } func fileToObj(file wopan.File) (model.Obj, error) { t, err := getTime(file.CreateTime) if err != nil { return nil, err } return &Object{ ObjThumb: model.ObjThumb{ Object: model.Object{ ID: file.Id, //Path: "", Name: file.Name, Size: file.Size, Modified: t, IsFolder: file.Type == 0, }, Thumbnail: model.Thumbnail{ Thumbnail: file.ThumbUrl, }, }, FID: file.Fid, }, nil } ================================================ FILE: drivers/wopan/util.go ================================================ package template import ( "time" "github.com/OpenListTeam/wopan-sdk-go" ) // do others that not defined in Driver interface func (d *Wopan) getSortRule() int { switch d.SortRule { case "name_asc": return wopan.SortNameAsc case "name_desc": return wopan.SortNameDesc case "time_asc": return wopan.SortTimeAsc case "time_desc": return wopan.SortTimeDesc case "size_asc": return wopan.SortSizeAsc case "size_desc": return wopan.SortSizeDesc default: return wopan.SortNameAsc } } func (d *Wopan) getSpaceType() string { if d.FamilyID == "" { return wopan.SpaceTypePersonal } return wopan.SpaceTypeFamily } // 20230607214351 func getTime(str string) (time.Time, error) { loc := time.FixedZone("UTC+8", 8*60*60) return time.ParseInLocation("20060102150405", str, loc) } ================================================ FILE: drivers/wps/driver.go ================================================ package wps import ( "context" "fmt" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type Wps struct { model.Storage Addition companyID string } func (d *Wps) Config() driver.Config { return config } func (d *Wps) GetAddition() driver.Additional { return &d.Addition } func (d *Wps) Init(ctx context.Context) error { if d.Cookie == "" { return fmt.Errorf("cookie is empty") } return d.ensureCompanyID(ctx) } func (d *Wps) Drop(ctx context.Context) error { return nil } func (d *Wps) List(ctx context.Context, dir model.Obj, _ model.ListArgs) ([]model.Obj, error) { basePath := "/" if dir != nil { if p := dir.GetPath(); p != "" { basePath = p } } return d.list(ctx, basePath) } func (d *Wps) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*model.Link, error) { if file == nil { return nil, errs.NotSupport } return d.link(ctx, file.GetPath()) } func (d *Wps) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { return d.makeDir(ctx, parentDir, dirName) } func (d *Wps) Move(ctx context.Context, srcObj, dstDir model.Obj) error { return d.move(ctx, srcObj, dstDir) } func (d *Wps) Rename(ctx context.Context, srcObj model.Obj, newName string) error { return d.rename(ctx, srcObj, newName) } func (d *Wps) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return d.copy(ctx, srcObj, dstDir) } func (d *Wps) Remove(ctx context.Context, obj model.Obj) error { return d.remove(ctx, obj) } func (d *Wps) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { return d.put(ctx, dstDir, file, up) } func (d *Wps) GetDetails(ctx context.Context) (*model.StorageDetails, error) { quota, err := d.spaces(ctx) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: quota.Total, UsedSpace: quota.Used, }, }, nil } var _ driver.Driver = (*Wps)(nil) ================================================ FILE: drivers/wps/meta.go ================================================ package wps import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { driver.RootPath Cookie string `json:"cookie" required:"true" type:"text"` Mode string `json:"mode" type:"select" options:"Personal,Business" default:"Business"` } var config = driver.Config{ Name: "WPS", LocalSort: true, DefaultRoot: "/", Alert: "", NoOverwriteUpload: true, } func init() { op.RegisterDriver(func() driver.Driver { return &Wps{} }) } ================================================ FILE: drivers/wps/types.go ================================================ package wps import ( "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type workspaceResp struct { Companies []struct { ID int64 `json:"id"` } `json:"companies"` } type Group struct { CompanyID int64 `json:"company_id"` GroupID int64 `json:"group_id"` Name string `json:"name"` Type string `json:"type"` } type groupsResp struct { Groups []Group `json:"groups"` } type filePerms struct { Download int `json:"download"` } type FileInfo struct { GroupID int64 `json:"groupid"` ParentID int64 `json:"parentid"` Name string `json:"fname"` Size int64 `json:"fsize"` Type string `json:"ftype"` Ctime int64 `json:"ctime"` Mtime int64 `json:"mtime"` ID int64 `json:"id"` Deleted bool `json:"deleted"` FilePerms filePerms `json:"file_perms_acl"` } type filesResp struct { Files []FileInfo `json:"files"` NextOffset int `json:"next_offset"` } type downloadResp struct { URL string `json:"url"` Result string `json:"result"` } type spacesResp struct { Id int64 `json:"id"` Name string `json:"name"` Result string `json:"result"` Total int64 `json:"total"` Used int64 `json:"used"` UsedParts []struct { Type string `json:"type"` Used int64 `json:"used"` } `json:"used_parts"` } type Obj struct { id string name string size int64 ctime time.Time mtime time.Time isDir bool hash utils.HashInfo path string canDownload bool } func (o *Obj) GetSize() int64 { return o.size } func (o *Obj) GetName() string { return o.name } func (o *Obj) ModTime() time.Time { return o.mtime } func (o *Obj) CreateTime() time.Time { return o.ctime } func (o *Obj) IsDir() bool { return o.isDir } func (o *Obj) GetHash() utils.HashInfo { return o.hash } func (o *Obj) GetID() string { return o.id } func (o *Obj) GetPath() string { return o.path } ================================================ FILE: drivers/wps/util.go ================================================ package wps import ( "bytes" "context" "crypto/sha1" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "io" "mime/multipart" "net/http" "strconv" "strings" "sync" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/go-resty/resty/v2" ) const endpoint = "https://365.kdocs.cn" const personalEndpoint = "https://drive.wps.cn" type resolvedNode struct { kind string group Group file *FileInfo } type resolveCacheEntry struct { node *resolvedNode expire time.Time } type resolveCacheStore struct { mu sync.RWMutex m map[string]resolveCacheEntry } var resolveCaches sync.Map type apiResult struct { Result string `json:"result"` Msg string `json:"msg"` } type uploadCreateUpdateResp struct { apiResult Method string `json:"method"` URL string `json:"url"` Store string `json:"store"` Request struct { Headers map[string]string `json:"headers"` FormData map[string]string `json:"formData"` } `json:"request"` Response struct { ExpectCode []int `json:"expect_code"` ArgsETag string `json:"args_etag"` ArgsKey string `json:"args_key"` } `json:"response"` } type uploadPutResp struct { NewFilename string `json:"newfilename"` Sha1 string `json:"sha1"` MD5 string `json:"md5"` } type personalGroupsResp struct { apiResult Groups []struct { ID int64 `json:"id"` Name string `json:"name"` } `json:"groups"` } type countingWriter struct { n *int64 } func (w countingWriter) Write(p []byte) (int, error) { *w.n += int64(len(p)) return len(p), nil } func (d *Wps) isPersonal() bool { return strings.TrimSpace(d.Mode) == "Personal" } func (d *Wps) driveHost() string { if d.isPersonal() { return personalEndpoint } return endpoint } func (d *Wps) drivePrefix() string { if d.isPersonal() { return "" } return "/3rd/drive" } func (d *Wps) driveURL(path string) string { return d.driveHost() + d.drivePrefix() + path } func (d *Wps) origin() string { return d.driveHost() } func (d *Wps) canDownload(f *FileInfo) bool { if f == nil || f.Type == "folder" { return false } if f.FilePerms.Download != 0 { return true } return d.isPersonal() } func (d *Wps) request(ctx context.Context) *resty.Request { return base.RestyClient.R(). SetHeader("Cookie", d.Cookie). SetHeader("Accept", "application/json"). SetContext(ctx) } func (d *Wps) jsonRequest(ctx context.Context) *resty.Request { return d.request(ctx). SetHeader("Content-Type", "application/json"). SetHeader("Origin", d.origin()) } func statusOK(code int, expect []int) bool { if len(expect) == 0 { return code >= 200 && code < 300 } for _, v := range expect { if v == code { return true } } return false } func respArg(arg string, resp *http.Response, body []byte) string { arg = strings.TrimSpace(arg) if arg == "" { return "" } l := strings.ToLower(arg) if strings.HasPrefix(l, "header.") { h := strings.TrimSpace(arg[len("header."):]) if h == "" { return "" } return strings.TrimSpace(resp.Header.Get(h)) } if strings.HasPrefix(l, "body.") { k := strings.TrimSpace(arg[len("body."):]) if k == "" { return "" } var m map[string]interface{} if err := json.Unmarshal(body, &m); err != nil { return "" } if v, ok := m[k]; ok { if s, ok := v.(string); ok { return strings.TrimSpace(s) } } } return "" } func extractXMLTag(v, tag string) string { s := strings.TrimSpace(v) if s == "" { return "" } lt := strings.ToLower(tag) open := "<" + lt + ">" clos := "" ls := strings.ToLower(s) i := strings.Index(ls, open) if i < 0 { return "" } i += len(open) j := strings.Index(ls[i:], clos) if j < 0 { return "" } r := strings.TrimSpace(s[i : i+j]) r = strings.ReplaceAll(r, """, "") return strings.Trim(r, `"'`) } func checkAPI(resp *resty.Response, result apiResult) error { if result.Result != "" && result.Result != "ok" { if result.Msg == "" { result.Msg = "unknown error" } return fmt.Errorf("%s: %s", result.Result, result.Msg) } if resp != nil && resp.IsError() { if result.Msg != "" { return fmt.Errorf("%s", result.Msg) } return fmt.Errorf("http error: %d", resp.StatusCode()) } return nil } func (d *Wps) ensureCompanyID(ctx context.Context) error { if d.isPersonal() { return nil } if d.companyID != "" { return nil } var resp workspaceResp r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(endpoint + "/3rd/plussvr/compose/v1/users/self/workspaces?fields=name&comp_status=active") if err != nil { return err } if r != nil && r.IsError() { return fmt.Errorf("http error: %d", r.StatusCode()) } if len(resp.Companies) == 0 { return fmt.Errorf("no company id") } d.companyID = strconv.FormatInt(resp.Companies[0].ID, 10) return nil } func (d *Wps) getGroups(ctx context.Context) ([]Group, error) { if d.isPersonal() { var resp personalGroupsResp r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(d.driveURL("/api/v3/groups")) if err != nil { return nil, err } if err := checkAPI(r, resp.apiResult); err != nil { return nil, err } res := make([]Group, 0, len(resp.Groups)) for _, g := range resp.Groups { res = append(res, Group{GroupID: g.ID, Name: g.Name}) } return res, nil } if err := d.ensureCompanyID(ctx); err != nil { return nil, err } var resp groupsResp url := fmt.Sprintf("%s/3rd/plus/groups/v1/companies/%s/users/self/groups/private", endpoint, d.companyID) r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) if err != nil { return nil, err } if r != nil && r.IsError() { return nil, fmt.Errorf("http error: %d", r.StatusCode()) } return resp.Groups, nil } func (d *Wps) getFiles(ctx context.Context, groupID, parentID int64) ([]FileInfo, error) { var resp filesResp var files []FileInfo next_offset := 0 for range 50 { url := fmt.Sprintf("%s/api/v5/groups/%d/files", d.driveHost()+d.drivePrefix(), groupID) r, err := d.request(ctx). SetQueryParam("parentid", strconv.FormatInt(parentID, 10)). SetQueryParam("offset", fmt.Sprint(next_offset)). SetResult(&resp). SetError(&resp). Get(url) if err != nil { return nil, err } if r != nil && r.IsError() { return nil, fmt.Errorf("http error: %d", r.StatusCode()) } files = append(files, resp.Files...) if resp.NextOffset == -1 { break } next_offset = resp.NextOffset } return files, nil } func parseTime(v int64) time.Time { if v <= 0 { return time.Time{} } return time.Unix(v, 0) } func joinPath(basePath, name string) string { if basePath == "" || basePath == "/" { return "/" + name } return strings.TrimRight(basePath, "/") + "/" + name } func normalizePath(path string) string { clean := strings.TrimSpace(path) if clean == "" || clean == "/" { return "/" } return "/" + strings.Trim(clean, "/") } func (d *Wps) resolveCacheStore() *resolveCacheStore { if d == nil { return nil } if v, ok := resolveCaches.Load(d); ok { if s, ok := v.(*resolveCacheStore); ok { return s } } s := &resolveCacheStore{m: make(map[string]resolveCacheEntry)} if v, loaded := resolveCaches.LoadOrStore(d, s); loaded { if s2, ok := v.(*resolveCacheStore); ok { return s2 } } return s } func (d *Wps) getResolveCache(path string) (*resolvedNode, bool) { s := d.resolveCacheStore() if s == nil { return nil, false } s.mu.RLock() e, ok := s.m[path] s.mu.RUnlock() if !ok || e.node == nil { return nil, false } if !e.expire.IsZero() && time.Now().After(e.expire) { s.mu.Lock() delete(s.m, path) s.mu.Unlock() return nil, false } return e.node, true } func (d *Wps) setResolveCache(path string, node *resolvedNode) { s := d.resolveCacheStore() if s == nil || node == nil { return } s.mu.Lock() s.m[path] = resolveCacheEntry{node: node, expire: time.Now().Add(10 * time.Minute)} s.mu.Unlock() } func (d *Wps) clearResolveCache() { s := d.resolveCacheStore() if s == nil { return } s.mu.Lock() if len(s.m) != 0 { s.m = make(map[string]resolveCacheEntry) } s.mu.Unlock() } func (d *Wps) resolvePath(ctx context.Context, path string) (*resolvedNode, error) { cacheKey := normalizePath(path) if n, ok := d.getResolveCache(cacheKey); ok { return n, nil } clean := strings.TrimSpace(path) if clean == "" { clean = "/" } clean = strings.Trim(clean, "/") if clean == "" { n := &resolvedNode{kind: "root"} d.setResolveCache("/", n) return n, nil } seg := strings.Split(clean, "/") groups, err := d.getGroups(ctx) if err != nil { return nil, err } var grp *Group for i := range groups { if groups[i].Name == seg[0] { grp = &groups[i] break } } if grp == nil { return nil, fmt.Errorf("group not found") } cur := "/" + seg[0] gn := &resolvedNode{kind: "group", group: *grp} d.setResolveCache(cur, gn) if len(seg) == 1 { return gn, nil } parentID := int64(0) var lastNode *resolvedNode for i := 1; i < len(seg); i++ { files, err := d.getFiles(ctx, grp.GroupID, parentID) if err != nil { return nil, err } var found *FileInfo for j := range files { if files[j].Name == seg[i] { found = &files[j] break } } if found == nil { return nil, fmt.Errorf("path not found") } if i < len(seg)-1 && found.Type != "folder" { return nil, fmt.Errorf("path not found") } fi := *found parentID = fi.ID cur = cur + "/" + seg[i] kind := "file" if fi.Type == "folder" { kind = "folder" } n := &resolvedNode{kind: kind, group: *grp, file: &fi} d.setResolveCache(cur, n) lastNode = n } if lastNode == nil { return nil, fmt.Errorf("path not found") } return lastNode, nil } func (d *Wps) fileToObj(basePath string, f FileInfo) *Obj { name := f.Name path := joinPath(basePath, name) obj := &Obj{ id: path, name: name, size: f.Size, ctime: parseTime(f.Ctime), mtime: parseTime(f.Mtime), isDir: f.Type == "folder", path: path, } if !obj.isDir { obj.canDownload = d.canDownload(&f) } return obj } func (d *Wps) doJSON(ctx context.Context, method, url string, body interface{}) error { var result apiResult req := d.jsonRequest(ctx).SetBody(body).SetResult(&result).SetError(&result) var ( resp *resty.Response err error ) switch method { case http.MethodPost: resp, err = req.Post(url) case http.MethodPut: resp, err = req.Put(url) default: return errs.NotSupport } if err != nil { return err } return checkAPI(resp, result) } func (d *Wps) list(ctx context.Context, basePath string) ([]model.Obj, error) { if strings.TrimSpace(basePath) == "" { basePath = "/" } node, err := d.resolvePath(ctx, basePath) if err != nil { return nil, err } if node.kind == "root" { groups, err := d.getGroups(ctx) if err != nil { return nil, err } res := make([]model.Obj, 0, len(groups)) for _, g := range groups { path := joinPath(basePath, g.Name) obj := &Obj{ id: path, name: g.Name, ctime: parseTime(0), mtime: parseTime(0), isDir: true, path: path, } res = append(res, obj) d.setResolveCache(normalizePath(path), &resolvedNode{kind: "group", group: g}) } d.setResolveCache("/", &resolvedNode{kind: "root"}) return res, nil } if node.kind != "group" && node.kind != "folder" { return nil, nil } parentID := int64(0) if node.file != nil && node.kind == "folder" { parentID = node.file.ID } files, err := d.getFiles(ctx, node.group.GroupID, parentID) if err != nil { return nil, err } res := make([]model.Obj, 0, len(files)) for _, f := range files { res = append(res, d.fileToObj(basePath, f)) path := normalizePath(joinPath(basePath, f.Name)) fi := f kind := "file" if fi.Type == "folder" { kind = "folder" } d.setResolveCache(path, &resolvedNode{kind: kind, group: node.group, file: &fi}) } return res, nil } func (d *Wps) link(ctx context.Context, path string) (*model.Link, error) { node, err := d.resolvePath(ctx, path) if err != nil { return nil, err } if node.kind != "file" || node.file == nil { return nil, errs.NotSupport } if !d.canDownload(node.file) { return nil, fmt.Errorf("no download permission") } url := fmt.Sprintf("%s/api/v5/groups/%d/files/%d/download?support_checksums=sha1", d.driveHost()+d.drivePrefix(), node.group.GroupID, node.file.ID) var resp downloadResp r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) if err != nil { return nil, err } if r != nil && r.IsError() { return nil, fmt.Errorf("http error: %d", r.StatusCode()) } if resp.URL == "" { return nil, fmt.Errorf("empty download url") } return &model.Link{URL: resp.URL, Header: http.Header{}}, nil } func (d *Wps) makeDir(ctx context.Context, parentDir model.Obj, dirName string) error { if parentDir == nil { return errs.NotSupport } node, err := d.resolvePath(ctx, parentDir.GetPath()) if err != nil { return err } if node.kind != "group" && node.kind != "folder" { return errs.NotSupport } parentID := int64(0) if node.file != nil && node.kind == "folder" { parentID = node.file.ID } body := map[string]interface{}{ "groupid": node.group.GroupID, "name": dirName, "parentid": parentID, } if err := d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/folder"), body); err != nil { return err } d.clearResolveCache() return nil } func (d *Wps) move(ctx context.Context, srcObj, dstDir model.Obj) error { if srcObj == nil || dstDir == nil { return errs.NotSupport } nodeSrc, err := d.resolvePath(ctx, srcObj.GetPath()) if err != nil { return err } nodeDst, err := d.resolvePath(ctx, dstDir.GetPath()) if err != nil { return err } if nodeSrc.kind != "file" && nodeSrc.kind != "folder" { return errs.NotSupport } if nodeDst.kind != "group" && nodeDst.kind != "folder" { return errs.NotSupport } targetParentID := int64(0) if nodeDst.file != nil && nodeDst.kind == "folder" { targetParentID = nodeDst.file.ID } body := map[string]interface{}{ "fileids": []int64{nodeSrc.file.ID}, "target_groupid": nodeDst.group.GroupID, "target_parentid": targetParentID, } url := fmt.Sprintf("/api/v3/groups/%d/files/batch/move", nodeSrc.group.GroupID) for { var res apiResult resp, err := d.jsonRequest(ctx). SetBody(body). SetResult(&res). SetError(&res). Post(d.driveURL(url)) if err != nil { return err } if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { time.Sleep(500 * time.Millisecond) continue } if err := checkAPI(resp, res); err != nil { return err } break } d.clearResolveCache() return nil } func (d *Wps) rename(ctx context.Context, srcObj model.Obj, newName string) error { if srcObj == nil { return errs.NotSupport } node, err := d.resolvePath(ctx, srcObj.GetPath()) if err != nil { return err } if node.kind != "file" && node.kind != "folder" { return errs.NotSupport } url := fmt.Sprintf("/api/v3/groups/%d/files/%d", node.group.GroupID, node.file.ID) body := map[string]string{"fname": newName} if err := d.doJSON(ctx, http.MethodPut, d.driveURL(url), body); err != nil { return err } d.clearResolveCache() return nil } func (d *Wps) copy(ctx context.Context, srcObj, dstDir model.Obj) error { if srcObj == nil || dstDir == nil { return errs.NotSupport } nodeSrc, err := d.resolvePath(ctx, srcObj.GetPath()) if err != nil { return err } nodeDst, err := d.resolvePath(ctx, dstDir.GetPath()) if err != nil { return err } if nodeSrc.kind != "file" && nodeSrc.kind != "folder" { return errs.NotSupport } if nodeDst.kind != "group" && nodeDst.kind != "folder" { return errs.NotSupport } targetParentID := int64(0) if nodeDst.file != nil && nodeDst.kind == "folder" { targetParentID = nodeDst.file.ID } body := map[string]interface{}{ "fileids": []int64{nodeSrc.file.ID}, "groupid": nodeSrc.group.GroupID, "target_groupid": nodeDst.group.GroupID, "target_parentid": targetParentID, "duplicated_name_model": 1, } url := fmt.Sprintf("/api/v3/groups/%d/files/batch/copy", nodeSrc.group.GroupID) for { var res apiResult resp, err := d.jsonRequest(ctx). SetBody(body). SetResult(&res). SetError(&res). Post(d.driveURL(url)) if err != nil { return err } if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { time.Sleep(500 * time.Millisecond) continue } if err := checkAPI(resp, res); err != nil { return err } break } d.clearResolveCache() return nil } func (d *Wps) remove(ctx context.Context, obj model.Obj) error { if obj == nil { return errs.NotSupport } node, err := d.resolvePath(ctx, obj.GetPath()) if err != nil { return err } if node.kind != "file" && node.kind != "folder" { return errs.NotSupport } body := map[string]interface{}{ "fileids": []int64{node.file.ID}, } url := fmt.Sprintf("/api/v3/groups/%d/files/batch/delete", node.group.GroupID) for { var res apiResult resp, err := d.jsonRequest(ctx). SetBody(body). SetResult(&res). SetError(&res). Post(d.driveURL(url)) if err != nil { return err } // 无法连续创建文件夹删除。如果一定要删除，每0.5s 尝试一次创建下一个删除请求，应当避免递归删除文件夹 if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { time.Sleep(500 * time.Millisecond) continue } if err := checkAPI(resp, res); err != nil { return err } break } d.clearResolveCache() return nil } func cacheAndHash(file model.FileStreamer, up driver.UpdateProgress) (model.File, int64, string, string, error) { h1 := sha1.New() h256 := sha256.New() size := file.GetSize() var counted int64 ws := []io.Writer{h1, h256} if size <= 0 { ws = append(ws, countingWriter{n: &counted}) } p := up f, err := file.CacheFullAndWriter(&p, io.MultiWriter(ws...)) if err != nil { return nil, 0, "", "", err } if size <= 0 { size = counted } return f, size, hex.EncodeToString(h1.Sum(nil)), hex.EncodeToString(h256.Sum(nil)), nil } func (d *Wps) createUpload(ctx context.Context, groupID, parentID int64, name string, size int64, sha1Hex, sha256Hex string) (*uploadCreateUpdateResp, error) { body := map[string]string{ "group_id": strconv.FormatInt(groupID, 10), "name": name, "parent_id": strconv.FormatInt(parentID, 10), "sha1": sha1Hex, "sha256": sha256Hex, "size": strconv.FormatInt(size, 10), } var resp uploadCreateUpdateResp r, err := d.jsonRequest(ctx). SetBody(body). SetResult(&resp). SetError(&resp). Put(d.driveURL("/api/v5/files/upload/create_update")) if err != nil { return nil, err } if err := checkAPI(r, resp.apiResult); err != nil { return nil, err } if resp.URL == "" { return nil, fmt.Errorf("empty upload url") } return &resp, nil } func normalizeETag(v string) string { v = strings.TrimSpace(v) if strings.HasPrefix(v, "W/") { v = strings.TrimSpace(strings.TrimPrefix(v, "W/")) } return strings.Trim(v, `"`) } func (d *Wps) commitUpload(ctx context.Context, etag, key string, groupID, parentID int64, name, sha1Hex string, size int64, store string) error { store = strings.TrimSpace(store) if store == "" { store = "ks3" } storeKey := "" if key != "" { storeKey = key } body := map[string]interface{}{ "etag": etag, "groupid": groupID, "key": key, "name": name, "parentid": parentID, "sha1": sha1Hex, "size": size, "store": store, "storekey": storeKey, } return d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/file"), body) } func (d *Wps) put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { if dstDir == nil || file == nil { return errs.NotSupport } if up == nil { up = func(float64) {} } node, err := d.resolvePath(ctx, dstDir.GetPath()) if err != nil { return err } if node.kind != "group" && node.kind != "folder" { return errs.NotSupport } parentID := int64(0) if node.file != nil && node.kind == "folder" { parentID = node.file.ID } f, size, sha1Hex, sha256Hex, err := cacheAndHash(file, func(float64) {}) if err != nil { return err } if c, ok := f.(io.Closer); ok { defer c.Close() } // 在隐藏文件名前加_上传，这是WPS的限制，无法上传隐藏文件，也无法将任何文件重命名为隐藏文件，所有隐藏文件会被自动加上_ 上传 // 甚至可以上传前缀是..的文件，但是单个点就是不行 realName := file.GetName() uploadName := realName if strings.HasPrefix(realName, ".") { uploadName = "_" + realName } info, err := d.createUpload(ctx, node.group.GroupID, parentID, uploadName, size, sha1Hex, sha256Hex) if err != nil { return err } if _, err := f.Seek(0, io.SeekStart); err != nil { return err } rf := driver.NewLimitedUploadFile(ctx, f) prog := driver.NewProgress(size, model.UpdateProgressWithRange(up, 0, 1)) method := strings.ToUpper(strings.TrimSpace(info.Method)) if method == "" { method = http.MethodPut } var req *http.Request if method == http.MethodPost && len(info.Request.FormData) > 0 { if size == 0 { var buf bytes.Buffer mw := multipart.NewWriter(&buf) for k, v := range info.Request.FormData { if err := mw.WriteField(k, v); err != nil { return err } } part, err := mw.CreateFormFile("file", uploadName) if err != nil { return err } if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { return err } if err := mw.Close(); err != nil { return err } req, err = http.NewRequestWithContext(ctx, method, info.URL, bytes.NewReader(buf.Bytes())) if err != nil { return err } for k, v := range info.Request.Headers { req.Header.Set(k, v) } req.Header.Set("Content-Type", mw.FormDataContentType()) req.ContentLength = int64(buf.Len()) req.Header.Set("Content-Length", strconv.FormatInt(req.ContentLength, 10)) } else { pr, pw := io.Pipe() mw := multipart.NewWriter(pw) req, err = http.NewRequestWithContext(ctx, method, info.URL, pr) if err != nil { return err } for k, v := range info.Request.Headers { req.Header.Set(k, v) } req.Header.Set("Content-Type", mw.FormDataContentType()) go func() { for k, v := range info.Request.FormData { if err := mw.WriteField(k, v); err != nil { pw.CloseWithError(err) return } } part, err := mw.CreateFormFile("file", uploadName) if err != nil { pw.CloseWithError(err) return } if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { pw.CloseWithError(err) return } if err := mw.Close(); err != nil { pw.CloseWithError(err) return } pw.Close() }() } } else { var body = io.TeeReader(rf, prog) if size == 0 { body = bytes.NewReader(nil) } req, err = http.NewRequestWithContext(ctx, method, info.URL, body) if err != nil { return err } for k, v := range info.Request.Headers { req.Header.Set(k, v) } req.ContentLength = size req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) } c := *base.RestyClient.GetClient() c.Timeout = 0 resp, err := (&c).Do(req) if err != nil { return err } defer resp.Body.Close() if !statusOK(resp.StatusCode, info.Response.ExpectCode) { io.Copy(io.Discard, resp.Body) return fmt.Errorf("http error: %d", resp.StatusCode) } body, err := io.ReadAll(resp.Body) if err != nil { return err } etag := normalizeETag(respArg(info.Response.ArgsETag, resp, body)) if etag == "" { etag = normalizeETag(resp.Header.Get("ETag")) } key := strings.TrimSpace(respArg(info.Response.ArgsKey, resp, body)) if key == "" { key = strings.TrimSpace(resp.Header.Get("x-obs-save-key")) } var pr uploadPutResp sha1FromServer := "" if err := json.Unmarshal(body, &pr); err == nil { sha1FromServer = strings.TrimSpace(pr.NewFilename) if sha1FromServer == "" { sha1FromServer = strings.TrimSpace(pr.Sha1) } if etag == "" && pr.MD5 != "" { etag = strings.TrimSpace(pr.MD5) } } if sha1FromServer == "" { if v := extractXMLTag(string(body), "ETag"); v != "" { sha1FromServer = v if etag == "" { etag = v } } } if sha1FromServer == "" && key != "" && len(key) == 40 { sha1FromServer = key } if sha1FromServer == "" { sha1FromServer = sha1Hex } if etag == "" { return fmt.Errorf("empty etag") } if sha1FromServer == "" { return fmt.Errorf("empty sha1") } store := strings.TrimSpace(info.Store) commitKey := "" if strings.TrimSpace(info.Response.ArgsKey) != "" { commitKey = key if commitKey == "" { commitKey = sha1FromServer } } if err := d.commitUpload(ctx, etag, commitKey, node.group.GroupID, parentID, uploadName, sha1FromServer, size, store); err != nil { return err } up(1) return nil } func (d *Wps) spaces(ctx context.Context) (*spacesResp, error) { url := fmt.Sprintf("%s/api/v3/spaces", d.driveHost()+d.drivePrefix()) var resp spacesResp r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) if err != nil { return nil, err } if r != nil && r.IsError() { return nil, fmt.Errorf("http error: %d", r.StatusCode()) } return &resp, nil } ================================================ FILE: drivers/yandex_disk/driver.go ================================================ package yandex_disk import ( "context" "net/http" "path" "strconv" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) type YandexDisk struct { model.Storage Addition AccessToken string } func (d *YandexDisk) Config() driver.Config { return config } func (d *YandexDisk) GetAddition() driver.Additional { return &d.Addition } func (d *YandexDisk) Init(ctx context.Context) error { return d.refreshToken() } func (d *YandexDisk) Drop(ctx context.Context) error { return nil } func (d *YandexDisk) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(dir.GetPath()) if err != nil { return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { obj := fileToObj(src) obj.Path = path.Join(dir.GetPath(), obj.Name) return obj, nil }) } func (d *YandexDisk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp DownResp _, err := d.request("/download", http.MethodGet, func(req *resty.Request) { req.SetQueryParam("path", file.GetPath()) }, &resp) if err != nil { return nil, err } link := model.Link{ URL: resp.Href, } return &link, nil } func (d *YandexDisk) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { _, err := d.request("", http.MethodPut, func(req *resty.Request) { req.SetQueryParam("path", path.Join(parentDir.GetPath(), dirName)) }, nil) return err } func (d *YandexDisk) Move(ctx context.Context, srcObj, dstDir model.Obj) error { _, err := d.request("/move", http.MethodPost, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "from": srcObj.GetPath(), "path": path.Join(dstDir.GetPath(), srcObj.GetName()), "overwrite": "true", }) }, nil) return err } func (d *YandexDisk) Rename(ctx context.Context, srcObj model.Obj, newName string) error { _, err := d.request("/move", http.MethodPost, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "from": srcObj.GetPath(), "path": path.Join(path.Dir(srcObj.GetPath()), newName), "overwrite": "true", }) }, nil) return err } func (d *YandexDisk) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { _, err := d.request("/copy", http.MethodPost, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "from": srcObj.GetPath(), "path": path.Join(dstDir.GetPath(), srcObj.GetName()), "overwrite": "true", }) }, nil) return err } func (d *YandexDisk) Remove(ctx context.Context, obj model.Obj) error { _, err := d.request("", http.MethodDelete, func(req *resty.Request) { req.SetQueryParam("path", obj.GetPath()) }, nil) return err } func (d *YandexDisk) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { var resp UploadResp _, err := d.request("/upload", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(map[string]string{ "path": path.Join(dstDir.GetPath(), s.GetName()), "overwrite": "true", }) }, &resp) if err != nil { return err } reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, }) req, err := http.NewRequestWithContext(ctx, resp.Method, resp.Href, reader) if err != nil { return err } req.Header.Set("Content-Length", strconv.FormatInt(s.GetSize(), 10)) req.Header.Set("Content-Type", "application/octet-stream") res, err := base.HttpClient.Do(req) if err != nil { return err } _ = res.Body.Close() return err } var _ driver.Driver = (*YandexDisk)(nil) ================================================ FILE: drivers/yandex_disk/meta.go ================================================ package yandex_disk import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Addition struct { RefreshToken string `json:"refresh_token" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,path,created,modified,size" default:"name"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` driver.RootPath UseOnlineAPI bool `json:"use_online_api" default:"true"` APIAddress string `json:"api_url_address" default:"https://api.oplist.org/yandexui/renewapi"` ClientID string `json:"client_id"` ClientSecret string `json:"client_secret"` } var config = driver.Config{ Name: "YandexDisk", DefaultRoot: "/", } func init() { op.RegisterDriver(func() driver.Driver { return &YandexDisk{} }) } ================================================ FILE: drivers/yandex_disk/types.go ================================================ package yandex_disk import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type TokenErrResp struct { ErrorDescription string `json:"error_description"` Error string `json:"error"` } type ErrResp struct { Message string `json:"message"` Description string `json:"description"` Error string `json:"error"` } type File struct { //AntivirusStatus string `json:"antivirus_status"` Size int64 `json:"size"` //CommentIds struct { // PrivateResource string `json:"private_resource"` // PublicResource string `json:"public_resource"` //} `json:"comment_ids"` Name string `json:"name"` //Exif struct { // DateTime time.Time `json:"date_time"` //} `json:"exif"` //Created time.Time `json:"created"` //ResourceId string `json:"resource_id"` Modified time.Time `json:"modified"` //MimeType string `json:"mime_type"` File string `json:"file"` //MediaType string `json:"media_type"` Preview string `json:"preview"` Path string `json:"path"` //Sha256 string `json:"sha256"` Type string `json:"type"` //Md5 string `json:"md5"` //Revision int64 `json:"revision"` } func fileToObj(f File) *model.Object { return &model.Object{ Name: f.Name, Size: f.Size, Modified: f.Modified, IsFolder: f.Type == "dir", } } type FilesResp struct { Embedded struct { Sort string `json:"sort"` Items []File `json:"items"` Limit int `json:"limit"` Offset int `json:"offset"` Path string `json:"path"` Total int `json:"total"` } `json:"_embedded"` Name string `json:"name"` Exif struct { } `json:"exif"` ResourceId string `json:"resource_id"` Created time.Time `json:"created"` Modified time.Time `json:"modified"` Path string `json:"path"` CommentIds struct { } `json:"comment_ids"` Type string `json:"type"` Revision int64 `json:"revision"` } type DownResp struct { Href string `json:"href"` Method string `json:"method"` Templated bool `json:"templated"` } type UploadResp struct { OperationId string `json:"operation_id"` Href string `json:"href"` Method string `json:"method"` Templated bool `json:"templated"` } ================================================ FILE: drivers/yandex_disk/util.go ================================================ package yandex_disk import ( "errors" "fmt" "net/http" "strconv" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/go-resty/resty/v2" ) // do others that not defined in Driver interface func (d *YandexDisk) refreshToken() error { // 使用在线API刷新Token，无需ClientID和ClientSecret if d.UseOnlineAPI && len(d.APIAddress) > 0 { u := d.APIAddress var resp struct { RefreshToken string `json:"refresh_token"` AccessToken string `json:"access_token"` ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, "server_use": "true", "driver_txt": "yandexui_go", }). Get(u) if err != nil { return err } if resp.RefreshToken == "" || resp.AccessToken == "" { if resp.ErrorMessage != "" { return fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) } return fmt.Errorf("empty token returned from official API , a wrong refresh token may have been used") } d.AccessToken = resp.AccessToken d.RefreshToken = resp.RefreshToken op.MustSaveDriverStorage(d) return nil } // 使用本地客户端的情况下检查是否为空 if d.ClientID == "" || d.ClientSecret == "" { return fmt.Errorf("empty ClientID or ClientSecret") } // 走原有的刷新逻辑 u := "https://oauth.yandex.com/token" var resp base.TokenResp var e TokenErrResp _, err := base.RestyClient.R().SetResult(&resp).SetError(&e).SetFormData(map[string]string{ "grant_type": "refresh_token", "refresh_token": d.RefreshToken, "client_id": d.ClientID, "client_secret": d.ClientSecret, }).Post(u) if err != nil { return err } if e.Error != "" { return fmt.Errorf("%s : %s", e.Error, e.ErrorDescription) } d.AccessToken, d.RefreshToken = resp.AccessToken, resp.RefreshToken op.MustSaveDriverStorage(d) return nil } func (d *YandexDisk) request(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { u := "https://cloud-api.yandex.net/v1/disk/resources" + pathname req := base.RestyClient.R() req.SetHeader("Authorization", "OAuth "+d.AccessToken) if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } var e ErrResp req.SetError(&e) res, err := req.Execute(method, u) if err != nil { return nil, err } //log.Debug(res.String()) if e.Error != "" { if e.Error == "UnauthorizedError" { err = d.refreshToken() if err != nil { return nil, err } return d.request(pathname, method, callback, resp) } return nil, errors.New(e.Description) } return res.Body(), nil } func (d *YandexDisk) getFiles(path string) ([]File, error) { limit := 100 page := 1 res := make([]File, 0) for { offset := (page - 1) * limit query := map[string]string{ "path": path, "limit": strconv.Itoa(limit), "offset": strconv.Itoa(offset), } if d.OrderBy != "" { if d.OrderDirection == "desc" { query["sort"] = "-" + d.OrderBy } else { query["sort"] = d.OrderBy } } var resp FilesResp _, err := d.request("", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return nil, err } res = append(res, resp.Embedded.Items...) if resp.Embedded.Total <= offset+limit { break } } return res, nil } ================================================ FILE: entrypoint.sh ================================================ #!/bin/sh umask ${UMASK} if [ "$1" = "version" ]; then ./openlist version else # Check file of /opt/openlist/data permissions for current user # 检查当前用户是否有当前目录的写和执行权限 if [ -d ./data ]; then if ! [ -w ./data ] || ! [ -x ./data ]; then cat </dev/null fi runsvdir /opt/service/start & else # If aria2 should NOT run and target directory exists, remove it if [ -d "$ARIA2_DIR" ]; then rm -rf "$ARIA2_DIR" fi fi exec ./openlist server --no-prefix fi ================================================ FILE: go.mod ================================================ module github.com/OpenListTeam/OpenList/v4 go 1.24.0 toolchain go1.24.13 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 github.com/KarpelesLab/reflink v1.0.2 github.com/KirCute/zip v1.0.1 github.com/OpenListTeam/go-cache v0.1.0 github.com/OpenListTeam/sftpd-openlist v1.0.1 github.com/OpenListTeam/tache v0.2.2 github.com/OpenListTeam/times v0.1.0 github.com/OpenListTeam/wopan-sdk-go v0.1.5 github.com/ProtonMail/go-crypto v1.3.0 github.com/ProtonMail/gopenpgp/v2 v2.9.0 github.com/SheltonZhu/115driver v1.2.3 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/antchfx/htmlquery v1.3.5 github.com/antchfx/xpath v1.3.5 github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.55.7 github.com/blevesearch/bleve/v2 v2.5.2 github.com/bmatcuk/doublestar/v4 v4.9.1 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.21.0 github.com/charmbracelet/bubbletea v1.3.6 github.com/charmbracelet/lipgloss v1.1.0 github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc github.com/coreos/go-oidc v2.3.0+incompatible github.com/deckarep/golang-set/v2 v2.8.0 github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 github.com/disintegration/imaging v1.6.2 github.com/dlclark/regexp2 v1.11.5 github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6 github.com/foxxorcat/mopan-sdk-go v0.1.6 github.com/foxxorcat/weiyun-sdk-go v0.1.4 github.com/gin-contrib/cors v1.7.6 github.com/gin-gonic/gin v1.10.1 github.com/go-resty/resty/v2 v2.16.5 github.com/go-webauthn/webauthn v0.13.4 github.com/golang-jwt/jwt/v4 v4.5.2 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38 github.com/hekmon/transmissionrpc/v3 v3.0.0 github.com/henrybear327/go-proton-api v1.0.0 github.com/ipfs/go-ipfs-api v0.7.0 github.com/itsHenry35/gofakes3 v0.0.8 github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 github.com/json-iterator/go v1.1.12 github.com/kdomanski/iso9660 v0.4.0 github.com/maruel/natural v1.1.1 github.com/meilisearch/meilisearch-go v0.32.0 github.com/mholt/archives v0.1.3 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/ncw/swift/v2 v2.0.4 github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.9 github.com/pquerna/otp v1.5.0 github.com/quic-go/quic-go v0.54.1 github.com/rclone/rclone v1.70.3 github.com/shirou/gopsutil/v4 v4.25.5 github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.14.0 github.com/spf13/cobra v1.9.1 github.com/stretchr/testify v1.10.0 github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 github.com/tchap/go-patricia/v2 v2.3.3 github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.6.0 github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 golang.org/x/crypto v0.46.0 golang.org/x/image v0.29.0 golang.org/x/net v0.48.0 golang.org/x/oauth2 v0.34.0 golang.org/x/time v0.14.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.5.7 gorm.io/driver/postgres v1.5.9 gorm.io/driver/sqlite v1.5.6 gorm.io/gorm v1.25.11 ) require ( cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect github.com/ProtonMail/go-srp v0.0.7 // indirect github.com/PuerkitoBio/goquery v1.10.3 // indirect github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect github.com/bradenaw/juniper v0.15.3 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cronokirby/saferith v0.33.0 // indirect github.com/ebitengine/purego v0.8.4 // indirect github.com/emersion/go-message v0.18.2 // indirect github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect github.com/geoffgarside/ber v1.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/goidentity/v6 v6.0.1 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/lanrat/extsort v1.0.2 // indirect github.com/mikelolasagasti/xz v1.0.1 // indirect github.com/minio/minlz v1.0.0 // indirect github.com/minio/xxml v0.0.3 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/relvacode/iso8601 v1.6.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.uber.org/mock v0.5.0 // indirect golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect golang.org/x/mod v0.30.0 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect ) require ( github.com/OpenListTeam/115-sdk-go v0.2.3 github.com/STARRY-S/zip v0.2.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/blevesearch/go-faiss v1.0.25 // indirect github.com/blevesearch/zapx/v16 v16.2.4 // indirect github.com/bodgit/plumbing v1.3.0 // indirect github.com/bodgit/sevenzip v1.6.1 github.com/bodgit/windows v1.0.1 // indirect github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/charmbracelet/x/ansi v0.9.3 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect github.com/cloudflare/circl v1.6.1 // indirect github.com/cloudwego/base64x v0.1.5 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/fclairamb/go-log v0.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hekmon/cunits/v2 v2.1.0 // indirect github.com/ipfs/boxo v0.12.0 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/microcosm-cc/bluemonday v1.0.27 github.com/nwaples/rardecode/v2 v2.1.1 github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/ulikunitz/xz v0.5.12 // indirect github.com/yuin/goldmark v1.7.13 go4.org v0.0.0-20260112195520-a5071408f32f resty.dev/v3 v3.0.0-beta.2 // indirect ) require ( github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect github.com/OpenListTeam/gsync v0.1.0 // indirect github.com/abbot/go-http-auth v0.4.0 // indirect github.com/aead/ecdh v0.2.0 // indirect github.com/andreburgaud/crypt2go v1.8.0 // indirect github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 // indirect github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.22.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/blevesearch/bleve_index_api v1.2.8 // indirect github.com/blevesearch/geo v0.2.3 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect github.com/blevesearch/mmap-go v1.0.4 // indirect github.com/blevesearch/scorch_segment_api/v2 v2.3.10 // indirect github.com/blevesearch/segment v0.9.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect github.com/blevesearch/vellum v1.1.0 // indirect github.com/blevesearch/zapx/v11 v11.4.2 // indirect github.com/blevesearch/zapx/v12 v12.4.2 // indirect github.com/blevesearch/zapx/v13 v13.4.2 // indirect github.com/blevesearch/zapx/v14 v14.4.2 // indirect github.com/blevesearch/zapx/v15 v15.4.2 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/bytedance/sonic v1.13.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-chi/chi/v5 v5.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.26.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/go-webauthn/x v0.1.23 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/golang-jwt/jwt/v5 v5.2.3 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-tpm v0.9.5 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/henrybear327/Proton-API-Bridge v1.0.0 github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.5.0 github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgx/v5 v5.5.5 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p v0.27.8 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr v0.9.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.64.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/rfjakob/eme v1.1.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/tklauser/go-sysconf v0.3.15 // indirect github.com/tklauser/numcpus v0.10.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/u2takey/go-utils v0.3.1 // indirect github.com/ugorji/go/codec v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.4.0 // indirect golang.org/x/arch v0.18.0 // indirect golang.org/x/sync v0.19.0 golang.org/x/sys v0.40.0 golang.org/x/term v0.38.0 // indirect golang.org/x/text v0.32.0 golang.org/x/tools v0.39.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect google.golang.org/grpc v1.78.0 google.golang.org/protobuf v1.36.11 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect ) replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton-api v1.0.0 replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed // replace github.com/OpenListTeam/115-sdk-go => ../../OpenListTeam/115-sdk-go ================================================ FILE: go.sum ================================================ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.53.0 h1:MZQCQQaRwOrAcuKjiHWHrgKykt4fZyuwF2dtiG3fGW8= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go/auth v0.16.2 h1:QvBAGFPLrDeoiNjyfVunhQ10HKNYuOwZ5noee0M5df4= cloud.google.com/go/auth v0.16.2/go.mod h1:sRBas2Y1fB1vZTdurouM0AzuYQBMZinrUYL8EufhtEA= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= github.com/KarpelesLab/reflink v1.0.2 h1:hQ1aM3TmjU2kTNUx5p/HaobDoADYk+a6AuEinG4Cv88= github.com/KarpelesLab/reflink v1.0.2/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/OpenListTeam/115-sdk-go v0.2.3 h1:nDNz0GxgliW+nT2Ds486k/rp/GgJj7Ngznc98ZBUwZo= github.com/OpenListTeam/115-sdk-go v0.2.3/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI= github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs= github.com/OpenListTeam/gsync v0.1.0/go.mod h1:h/Rvv9aX/6CdW/7B8di3xK3xNV8dUg45Fehrd/ksZ9s= github.com/OpenListTeam/sftpd-openlist v1.0.1 h1:j4S3iPFOpnXCUKRPS7uCT4mF2VCl34GyqvH6lqwnkUU= github.com/OpenListTeam/sftpd-openlist v1.0.1/go.mod h1:uO/wKnbvbdq3rBLmClMTZXuCnw7XW4wlAq4dZe91a40= github.com/OpenListTeam/tache v0.2.2 h1:CWFn6sr1AIYaEjC8ONdKs+LrxHyuErheenAjEqRhh4k= github.com/OpenListTeam/tache v0.2.2/go.mod h1:qmnZ/VpY2DUlmjg3UoDeNFy/LRqrw0biN3hYEEGc/+A= github.com/OpenListTeam/times v0.1.0 h1:qknxw+qj5CYKgXAwydA102UEpPcpU8TYNGRmwRyPYpg= github.com/OpenListTeam/times v0.1.0/go.mod h1:Jx7qen5NCYzKk2w14YuvU48YYMcPa1P9a+EJePC15Pc= github.com/OpenListTeam/wopan-sdk-go v0.1.5 h1:iKKcVzIqBgtGDbn0QbdWrCazSGxXFmYFyrnFBG+U8dI= github.com/OpenListTeam/wopan-sdk-go v0.1.5/go.mod h1:otynv0CgSNUClPpUgZ44qCZGcMRe0dc83Pkk65xAunI= github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I= github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug= github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo= github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk= github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo= github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI= github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk= github.com/ProtonMail/gopenpgp/v2 v2.9.0 h1:ruLzBmwe4dR1hdnrsEJ/S7psSBmV15gFttFUPP/+/kE= github.com/ProtonMail/gopenpgp/v2 v2.9.0/go.mod h1:IldDyh9Hv1ZCCYatTuuEt1XZJ0OPjxLpTarDfglih7s= github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2JW2gggRdg= github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= github.com/SheltonZhu/115driver v1.2.3 h1:94XMP/ey7VXIlpoBLIJHEoXu7N8YsELZlXVbxWcDDvk= github.com/SheltonZhu/115driver v1.2.3/go.mod h1:Zk7Qz7SYO1QU0SJIne6DnUD2k36S3wx/KbsQpxcfY/Y= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreburgaud/crypt2go v1.8.0 h1:J73vGTb1P6XL69SSuumbKs0DWn3ulbl9L92ZXBjw6pc= github.com/andreburgaud/crypt2go v1.8.0/go.mod h1:L5nfShQ91W78hOWhUH2tlGRPO+POAPJAF5fKOLB9SXg= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 h1:8PmGpDEZl9yDpcdEr6Odf23feCxK3LNUNMxjXg41pZQ= github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/antchfx/htmlquery v1.3.5 h1:aYthDDClnG2a2xePf6tys/UyyM/kRcsFRm+ifhFKoU0= github.com/antchfx/htmlquery v1.3.5/go.mod h1:5oyIPIa3ovYGtLqMPNjBF2Uf25NPCKsMjCnQ8lvjaoA= github.com/antchfx/xpath v1.3.5 h1:PqbXLC3TkfeZyakF5eeh3NTWEbYl4VHNVeufANzDbKQ= github.com/antchfx/xpath v1.3.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.49 h1:7gss+6H2mrrFtBrkokJRR2TzQD9qkpGA4N6BvIP/pCM= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.49/go.mod h1:30PBx0ENoUCJm2AxzgCue8j7KEjb9ci4enxy6CCOjbE= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2 h1:BCG7DCXEXpNCcpwCxg1oi9pkJWH2+eZzTn9MY56MbVw= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= github.com/aws/aws-sdk-go-v2/service/s3 v1.72.3 h1:WZOmJfCDV+4tYacLxpiojoAdT5sxTfB3nTqQNtZu+J4= github.com/aws/aws-sdk-go-v2/service/s3 v1.72.3/go.mod h1:xMekrnhmJ5aqmyxtmALs7mlvXw5xRh+eYjOjvrIIFJ4= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4= github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/blevesearch/bleve/v2 v2.5.2 h1:Ab0r0MODV2C5A6BEL87GqLBySqp/s9xFgceCju6BQk8= github.com/blevesearch/bleve/v2 v2.5.2/go.mod h1:5Dj6dUQxZM6aqYT3eutTD/GpWKGFSsV8f7LDidFbwXo= github.com/blevesearch/bleve_index_api v1.2.8 h1:Y98Pu5/MdlkRyLM0qDHostYo7i+Vv1cDNhqTeR4Sy6Y= github.com/blevesearch/bleve_index_api v1.2.8/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0= github.com/blevesearch/geo v0.2.3 h1:K9/vbGI9ehlXdxjxDRJtoAMt7zGAsMIzc6n8zWcwnhg= github.com/blevesearch/geo v0.2.3/go.mod h1:K56Q33AzXt2YExVHGObtmRSFYZKYGv0JEN5mdacJJR8= github.com/blevesearch/go-faiss v1.0.25 h1:lel1rkOUGbT1CJ0YgzKwC7k+XH0XVBHnCVWahdCXk4U= github.com/blevesearch/go-faiss v1.0.25/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= github.com/blevesearch/scorch_segment_api/v2 v2.3.10 h1:Yqk0XD1mE0fDZAJXTjawJ8If/85JxnLd8v5vG/jWE/s= github.com/blevesearch/scorch_segment_api/v2 v2.3.10/go.mod h1:Z3e6ChN3qyN35yaQpl00MfI5s8AxUJbpTR/DL8QOQ+8= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A= github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= github.com/blevesearch/vellum v1.1.0 h1:CinkGyIsgVlYf8Y2LUQHvdelgXr6PYuvoDIajq6yR9w= github.com/blevesearch/vellum v1.1.0/go.mod h1:QgwWryE8ThtNPxtgWJof5ndPfx0/YMBh+W2weHKPw8Y= github.com/blevesearch/zapx/v11 v11.4.2 h1:l46SV+b0gFN+Rw3wUI1YdMWdSAVhskYuvxlcgpQFljs= github.com/blevesearch/zapx/v11 v11.4.2/go.mod h1:4gdeyy9oGa/lLa6D34R9daXNUvfMPZqUYjPwiLmekwc= github.com/blevesearch/zapx/v12 v12.4.2 h1:fzRbhllQmEMUuAQ7zBuMvKRlcPA5ESTgWlDEoB9uQNE= github.com/blevesearch/zapx/v12 v12.4.2/go.mod h1:TdFmr7afSz1hFh/SIBCCZvcLfzYvievIH6aEISCte58= github.com/blevesearch/zapx/v13 v13.4.2 h1:46PIZCO/ZuKZYgxI8Y7lOJqX3Irkc3N8W82QTK3MVks= github.com/blevesearch/zapx/v13 v13.4.2/go.mod h1:knK8z2NdQHlb5ot/uj8wuvOq5PhDGjNYQQy0QDnopZk= github.com/blevesearch/zapx/v14 v14.4.2 h1:2SGHakVKd+TrtEqpfeq8X+So5PShQ5nW6GNxT7fWYz0= github.com/blevesearch/zapx/v14 v14.4.2/go.mod h1:rz0XNb/OZSMjNorufDGSpFpjoFKhXmppH9Hi7a877D8= github.com/blevesearch/zapx/v15 v15.4.2 h1:sWxpDE0QQOTjyxYbAVjt3+0ieu8NCE0fDRaFxEsp31k= github.com/blevesearch/zapx/v15 v15.4.2/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw= github.com/blevesearch/zapx/v16 v16.2.4 h1:tGgfvleXTAkwsD5mEzgM3zCS/7pgocTCnO1oyAUjlww= github.com/blevesearch/zapx/v16 v16.2.4/go.mod h1:Rti/REtuuMmzwsI8/C/qIzRaEoSK/wiFYw5e5ctUKKs= github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4= github.com/bodgit/sevenzip v1.6.1/go.mod h1:GVoYQbEVbOGT8n2pfqCIMRUaRjQ8F9oSqoBEqZh5fQ8= github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo= github.com/bradenaw/juniper v0.15.3/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs= github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg= github.com/charmbracelet/bubbletea v1.3.6 h1:VkHIxPJQeDt0aFJIsVxw8BQdh/F/L2KKZGsK6et5taU= github.com/charmbracelet/bubbletea v1.3.6/go.mod h1:oQD9VCRQFF8KplacJLo28/jofOI2ToOfGYeFgBBxHOc= github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= github.com/charmbracelet/x/ansi v0.9.3 h1:BXt5DHS/MKF+LjuK4huWrC6NCvHtexww7dMayh6GXd0= github.com/charmbracelet/x/ansi v0.9.3/go.mod h1:3RQDQ6lDnROptfpWuUVIUG64bD2g2BgntdxH0Ya5TeE= github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8= github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ= github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc h1:t8YjNUCt1DimB4HCIXBztwWMhgxr5yG5/YaRl9Afdfg= github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc/go.mod h1:CgWpFCFWzzEA5hVkhAc6DZZzGd3czx+BblvOzjmg6KA= github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc h1:0xCWmFKBmarCqqqLeM7jFBSw/Or81UEElFqO8MY+GDs= github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc/go.mod h1:uvR42Hb/t52HQd7x5/ZLzZEK8oihrFpgnodIJ1vte2E= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/coreos/go-oidc v2.3.0+incompatible h1:+5vEsrgprdLjjQ9FzIKAzQz1wwPD+83hQRfUIPh7rO0= github.com/coreos/go-oidc v2.3.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.8.0 h1:swm0rlPCmdWn9mESxKOjWk8hXSqoxOp+ZlfuyaAdFlQ= github.com/deckarep/golang-set/v2 v2.8.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 h1:OtSeLS5y0Uy01jaKK4mA/WVIYtpzVm63vLVAPzJXigg= github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8/go.mod h1:apkPC/CR3s48O2D7Y++n1XWEpgPNNCjXYga3PPbJe2E= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg= github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHSmFD5zgFkWCyA4L1kCDkImPAtK7D6tg= github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6 h1:q1b+gv6AG2TDPN+f0QAkbRrAvJ3ZosnwRLTKNxSXlaA= github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6/go.mod h1:MAsn6OKL24MLbGdCjt1t44XMGgX3sFqukYTKmTUOci8= github.com/fclairamb/go-log v0.6.0 h1:1V7BJ75P2PvanLHRyGBBFjncB6d4AgEmu+BPWKbMkaU= github.com/fclairamb/go-log v0.6.0/go.mod h1:cyXxOw4aJwO6lrZb8GRELSw+sxO6wwkLJdsjY5xYCWA= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/srX8vQd1Y= github.com/foxxorcat/mopan-sdk-go v0.1.6/go.mod h1:UaY6D88yBXWGrcu/PcyLWyL4lzrk5pSxSABPHftOvxs= github.com/foxxorcat/weiyun-sdk-go v0.1.4 h1:X2tFvdqikkJ7awCBbMH7XXk7+uQoJlQksJz9CUU6ZgA= github.com/foxxorcat/weiyun-sdk-go v0.1.4/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64= github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/gin-contrib/cors v1.7.6 h1:3gQ8GMzs1Ylpf70y8bMw4fVpycXIeX1ZemuSQIsnQQY= github.com/gin-contrib/cors v1.7.6/go.mod h1:Ulcl+xN4jel9t1Ry8vqph23a60FwH9xVLd+3ykmTjOk= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM= github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-webauthn/webauthn v0.13.4 h1:q68qusWPcqHbg9STSxBLBHnsKaLxNO0RnVKaAqMuAuQ= github.com/go-webauthn/webauthn v0.13.4/go.mod h1:MglN6OH9ECxvhDqoq1wMoF6P6JRYDiQpC9nc5OomQmI= github.com/go-webauthn/x v0.1.23 h1:9lEO0s+g8iTyz5Vszlg/rXTGrx3CjcD0RZQ1GPZCaxI= github.com/go-webauthn/x v0.1.23/go.mod h1:AJd3hI7NfEp/4fI6T4CHD753u91l510lglU7/NMN6+E= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0= github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU= github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0= github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38 h1:lsK2GVgI2Ox0NkRpQnN09GBOH7jtsjFK5tcIgxXlLr0= github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38/go.mod h1:8x1h4rm3s8xMcTyJrq848sQ6BJnKzl57mDY4CNshdPM= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hekmon/cunits/v2 v2.1.0 h1:k6wIjc4PlacNOHwKEMBgWV2/c8jyD4eRMs5mR1BBhI0= github.com/hekmon/cunits/v2 v2.1.0/go.mod h1:9r1TycXYXaTmEWlAIfFV8JT+Xo59U96yUJAYHxzii2M= github.com/hekmon/transmissionrpc/v3 v3.0.0 h1:0Fb11qE0IBh4V4GlOwHNYpqpjcYDp5GouolwrpmcUDQ= github.com/hekmon/transmissionrpc/v3 v3.0.0/go.mod h1:38SlNhFzinVUuY87wGj3acOmRxeYZAZfrj6Re7UgCDg= github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0LTKWQciUm8PMZb0= github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts= github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw= github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ= github.com/ipfs/boxo v0.12.0/go.mod h1:xAnfiU6PtxWCnRqu7dcXQ10bB5/kvI1kXRotuGqGBhg= github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk= github.com/ipfs/go-ipfs-api v0.7.0 h1:CMBNCUl0b45coC+lQCXEVpMhwoqjiaCwUIrM+coYW2Q= github.com/ipfs/go-ipfs-api v0.7.0/go.mod h1:AIxsTNB0+ZhkqIfTZpdZ0VR/cpX5zrXjATa3prSay3g= github.com/itsHenry35/gofakes3 v0.0.8 h1:1AgOl04IgoUV5r/WSK7ycnvwfpgharYLfVTmnzk5miw= github.com/itsHenry35/gofakes3 v0.0.8/go.mod h1:gQwOJ7LoH5QSpCVmjzC6oKp+MS71utLS7GHtonsvD0c= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 h1:ZxO6Qr2GOXPdcW80Mcn3nemvilMPvpWqxrNfK2ZnNNs= github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3/go.mod h1:dvLUr/8Fs9a2OBrEnCC5duphbkz/k/mSy5OkXg3PAgI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg= github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lanrat/extsort v1.0.2 h1:p3MLVpQEPwEGPzeLBb+1eSErzRl6Bgjgr+qnIs2RxrU= github.com/lanrat/extsort v1.0.2/go.mod h1:ivzsdLm8Tv+88qbdpMElV6Z15StlzPUtZSKsGb51hnQ= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= github.com/libp2p/go-libp2p v0.27.8 h1:IX5x/4yKwyPQeVS2AXHZ3J4YATM9oHBGH1gBc23jBAI= github.com/libp2p/go-libp2p v0.27.8/go.mod h1:eCFFtd0s5i/EVKR7+5Ki8bM7qwkNW3TPTTSSW9sz8NE= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/meilisearch/meilisearch-go v0.32.0 h1:cWcycpONSH3VLTZ5npUl1O5aXPkNM0vUx6bywnYqGbE= github.com/meilisearch/meilisearch-go v0.32.0/go.mod h1:aNtyuwurDg/ggxQIcKqWH6G9g2ptc8GyY7PLY4zMn/g= github.com/mholt/archives v0.1.3 h1:aEAaOtNra78G+TvV5ohmXrJOAzf++dIlYeDW3N9q458= github.com/mholt/archives v0.1.3/go.mod h1:LUCGp++/IbV/I0Xq4SzcIR6uwgeh2yjnQWamjRQfLTU= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0= github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc= github.com/minio/minlz v1.0.0 h1:Kj7aJZ1//LlTP1DM8Jm7lNKvvJS2m74gyyXXn3+uJWQ= github.com/minio/minlz v1.0.0/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/xxml v0.0.3 h1:ZIpPQpfyG5uZQnqqC0LZuWtPk/WT8G/qkxvO6jb7zMU= github.com/minio/xxml v0.0.3/go.mod h1:wcXErosl6IezQIMEWSK/LYC2VS7LJ1dAkgvuyIN3aH4= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sggzwC/NcqbDQ= github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= github.com/ncw/swift/v2 v2.0.4 h1:hHWVFxn5/YaTWAASmn4qyq2p6OyP/Hm3vMLzkjEqR7w= github.com/ncw/swift/v2 v2.0.4/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/nwaples/rardecode/v2 v2.1.1 h1:OJaYalXdliBUXPmC8CZGQ7oZDxzX1/5mQmgn0/GASew= github.com/nwaples/rardecode/v2 v2.1.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw= github.com/pkg/sftp v1.13.9/go.mod h1:OBN7bVXdstkFFN/gdnHPUb5TE8eb8G1Rp9wCItqjkkA= github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA= github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8cTqKc= github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg= github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= github.com/rclone/rclone v1.70.3 h1:rg/WNh4DmSVZyKP2tHZ4lAaWEyMi7h/F0r7smOMA3IE= github.com/rclone/rclone v1.70.3/go.mod h1:nLyN+hpxAsQn9Rgt5kM774lcRDad82x/KqQeBZ83cMo= github.com/relvacode/iso8601 v1.6.0 h1:eFXUhMJN3Gz8Rcq82f9DTMW0svjtAVuIEULglM7QHTU= github.com/relvacode/iso8601 v1.6.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= github.com/shirou/gopsutil/v4 v4.25.5 h1:rtd9piuSMGeU8g1RMXjZs9y9luK5BwtnG7dZaQUJAsc= github.com/shirou/gopsutil/v4 v4.25.5/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 h1:Sa+sR8aaAMFwxhXWENEnE6ZpqhZ9d7u1RT2722Rw6hc= github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5/go.mod h1:UdZiFUFu6e2WjjtjxivwXWcwc1N/8zgbkBR9QNucUOY= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 h1:6Y51mutOvRGRx6KqyMNo//xk8B8o6zW9/RVmy1VamOs= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543/go.mod h1:jpwqYA8KUVEvSUJHkCXsnBRJCSKP1BMa81QZ6kvRpow= github.com/tchap/go-patricia/v2 v2.3.3 h1:xfNEsODumaEcCcY3gI0hYPZ/PcpVv5ju6RMAhgwZDDc= github.com/tchap/go-patricia/v2 v2.3.3/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k= github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uklU= github.com/u2takey/ffmpeg-go v0.5.0/go.mod h1:ruZWkvC1FEiUNjmROowOAps3ZcWxEiOpFoHCvk97kGc= github.com/u2takey/go-utils v0.3.1 h1:TaQTgmEZZeDHQFYfd+AdUT1cT4QJgJn/XVPELhHw4ys= github.com/u2takey/go-utils v0.3.1/go.mod h1:6e+v5vEZ/6gu12w/DC2ixZdZtCrNokVxD0JUklcqdCs= github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/unknwon/goconfig v1.0.0 h1:rS7O+CmUdli1T+oDm7fYj1MwqNWtEJfNj+FqcUHML8U= github.com/unknwon/goconfig v1.0.0/go.mod h1:qu2ZQ/wcC/if2u32263HTVC39PeOQRSmidQk3DuDFQ8= github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q= github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= github.com/winfsp/cgofuse v1.6.0 h1:re3W+HTd0hj4fISPBqfsrwyvPFpzqhDu8doJ9nOPDB0= github.com/winfsp/cgofuse v1.6.0/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 h1:PSRwrE5QBufPnOjdgIkRs5KBV1Avq3SY8oksj2Z+k3o= github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3/go.mod h1:CKriYB8bkNgSbYUQF1khSpejKb5IsV6cR7MdaAR7Fc0= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw= go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4= golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas= golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.236.0 h1:CAiEiDVtO4D/Qja2IA9VzlFrgPnK3XVMmRoJZlSWbc0= google.golang.org/api v0.236.0/go.mod h1:X1WF9CU2oTc+Jml1tiIxGmWFK/UZezdqEu09gcxZAj4= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/ldap.v3 v3.1.0 h1:DIDWEjI7vQWREh0S8X5/NFPCZ3MCVd55LmXKPW4XLGE= gopkg.in/ldap.v3 v3.1.0/go.mod h1:dQjCc0R0kfyFjIlWNMH1DORwUASZyDxo2Ry1B51dXaQ= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= resty.dev/v3 v3.0.0-beta.2 h1:xu4mGAdbCLuc3kbk7eddWfWm4JfhwDtdapwss5nCjnQ= resty.dev/v3 v3.0.0-beta.2/go.mod h1:OgkqiPvTDtOuV4MGZuUDhwOpkY8enjOsjjMzeOHefy4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= ================================================ FILE: internal/archive/all.go ================================================ package archive import ( _ "github.com/OpenListTeam/OpenList/v4/internal/archive/archives" _ "github.com/OpenListTeam/OpenList/v4/internal/archive/iso9660" _ "github.com/OpenListTeam/OpenList/v4/internal/archive/rardecode" _ "github.com/OpenListTeam/OpenList/v4/internal/archive/sevenzip" _ "github.com/OpenListTeam/OpenList/v4/internal/archive/zip" ) ================================================ FILE: internal/archive/archives/archives.go ================================================ package archives import ( "fmt" "io" "io/fs" "os" "path/filepath" "strings" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type Archives struct { } func (Archives) AcceptedExtensions() []string { return []string{ ".br", ".bz2", ".gz", ".lz4", ".lz", ".mz", ".sz", ".s2", ".xz", ".zz", ".zst", ".tar", ".tgz", ".tlz4", ".tlz", ".tbz2", ".txz", ".tzst", } } func (Archives) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { return map[string]tool.MultipartExtension{} } func (Archives) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { fsys, err := getFs(ss[0], args) if err != nil { return nil, err } files, err := fsys.ReadDir(".") if err != nil { return nil, filterPassword(err) } tree := make([]model.ObjTree, 0, len(files)) for _, file := range files { info, err := file.Info() if err != nil { continue } tree = append(tree, &model.ObjectTree{Object: *toModelObj(info)}) } return &model.ArchiveMetaInfo{ Comment: "", Encrypted: false, Tree: tree, }, nil } func (Archives) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { fsys, err := getFs(ss[0], args.ArchiveArgs) if err != nil { return nil, err } innerPath := strings.TrimPrefix(args.InnerPath, "/") if innerPath == "" { innerPath = "." } obj, err := fsys.ReadDir(innerPath) if err != nil { return nil, filterPassword(err) } return utils.SliceConvert(obj, func(src os.DirEntry) (model.Obj, error) { info, err := src.Info() if err != nil { return nil, err } return toModelObj(info), nil }) } func (Archives) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { fsys, err := getFs(ss[0], args.ArchiveArgs) if err != nil { return nil, 0, err } file, err := fsys.Open(strings.TrimPrefix(args.InnerPath, "/")) if err != nil { return nil, 0, filterPassword(err) } stat, err := file.Stat() if err != nil { return nil, 0, filterPassword(err) } return file, stat.Size(), nil } func (Archives) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { fsys, err := getFs(ss[0], args.ArchiveArgs) if err != nil { return err } isDir := false path := strings.TrimPrefix(args.InnerPath, "/") if path == "" { isDir = true path = "." } else { stat, err := fsys.Stat(path) if err != nil { return filterPassword(err) } if stat.IsDir() { isDir = true outputPath = filepath.Join(outputPath, stat.Name()) err = os.Mkdir(outputPath, 0700) if err != nil { return filterPassword(err) } } } if isDir { err = fs.WalkDir(fsys, path, func(p string, d fs.DirEntry, err error) error { if err != nil { return err } relPath := strings.TrimPrefix(p, path+"/") dstPath := filepath.Join(outputPath, relPath) if !strings.HasPrefix(dstPath, outputPath+string(os.PathSeparator)) { return fmt.Errorf("illegal file path: %s", relPath) } if d.IsDir() { err = os.MkdirAll(dstPath, 0700) } else { dir := filepath.Dir(dstPath) err = decompress(fsys, p, dir, func(_ float64) {}) } return err }) } else { err = decompress(fsys, path, outputPath, up) } return filterPassword(err) } var _ tool.Tool = (*Archives)(nil) func init() { tool.RegisterTool(Archives{}) } ================================================ FILE: internal/archive/archives/utils.go ================================================ package archives import ( "fmt" "io" fs2 "io/fs" "os" "path/filepath" "strings" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/mholt/archives" ) func getFs(ss *stream.SeekableStream, args model.ArchiveArgs) (*archives.ArchiveFS, error) { reader, err := stream.NewReadAtSeeker(ss, 0) if err != nil { return nil, err } if r, ok := reader.(*stream.RangeReadReadAtSeeker); ok { r.InitHeadCache() } format, _, err := archives.Identify(ss.Ctx, ss.GetName(), reader) if err != nil { return nil, errs.UnknownArchiveFormat } extractor, ok := format.(archives.Extractor) if !ok { return nil, errs.UnknownArchiveFormat } switch f := format.(type) { case archives.SevenZip: f.Password = args.Password case archives.Rar: f.Password = args.Password } return &archives.ArchiveFS{ Stream: io.NewSectionReader(reader, 0, ss.GetSize()), Format: extractor, Context: ss.Ctx, }, nil } func toModelObj(file os.FileInfo) *model.Object { return &model.Object{ Name: file.Name(), Size: file.Size(), Modified: file.ModTime(), IsFolder: file.IsDir(), } } func filterPassword(err error) error { if err != nil && strings.Contains(err.Error(), "password") { return errs.WrongArchivePassword } return err } func decompress(fsys fs2.FS, filePath, targetPath string, up model.UpdateProgress) error { rc, err := fsys.Open(filePath) if err != nil { return err } defer rc.Close() stat, err := rc.Stat() if err != nil { return err } destPath := filepath.Join(targetPath, stat.Name()) if !strings.HasPrefix(destPath, targetPath+string(os.PathSeparator)) { return fmt.Errorf("illegal file path: %s", stat.Name()) } f, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } defer f.Close() _, err = utils.CopyWithBuffer(f, &stream.ReaderUpdatingProgress{ Reader: &stream.SimpleReaderWithSize{ Reader: rc, Size: stat.Size(), }, UpdateProgress: up, }) return err } ================================================ FILE: internal/archive/iso9660/iso9660.go ================================================ package iso9660 import ( "fmt" "io" "os" "path/filepath" "strings" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/kdomanski/iso9660" ) type ISO9660 struct { } func (ISO9660) AcceptedExtensions() []string { return []string{".iso"} } func (ISO9660) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { return map[string]tool.MultipartExtension{} } func (ISO9660) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { return &model.ArchiveMetaInfo{ Comment: "", Encrypted: false, }, nil } func (ISO9660) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { img, err := getImage(ss[0]) if err != nil { return nil, err } dir, err := getObj(img, args.InnerPath) if err != nil { return nil, err } if !dir.IsDir() { return nil, errs.NotFolder } children, err := dir.GetChildren() if err != nil { return nil, err } ret := make([]model.Obj, 0, len(children)) for _, child := range children { ret = append(ret, toModelObj(child)) } return ret, nil } func (ISO9660) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { img, err := getImage(ss[0]) if err != nil { return nil, 0, err } obj, err := getObj(img, args.InnerPath) if err != nil { return nil, 0, err } if obj.IsDir() { return nil, 0, errs.NotFile } return io.NopCloser(obj.Reader()), obj.Size(), nil } func (ISO9660) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { img, err := getImage(ss[0]) if err != nil { return err } obj, err := getObj(img, args.InnerPath) if err != nil { return err } if obj.IsDir() { if args.InnerPath != "/" { rootpath := outputPath outputPath = filepath.Join(outputPath, obj.Name()) if !strings.HasPrefix(outputPath, rootpath+string(os.PathSeparator)) { return fmt.Errorf("illegal file path: %s", obj.Name()) } if err = os.MkdirAll(outputPath, 0700); err != nil { return err } } var children []*iso9660.File if children, err = obj.GetChildren(); err == nil { err = decompressAll(children, outputPath) } } else { err = decompress(obj, outputPath, up) } return err } var _ tool.Tool = (*ISO9660)(nil) func init() { tool.RegisterTool(ISO9660{}) } ================================================ FILE: internal/archive/iso9660/utils.go ================================================ package iso9660 import ( "fmt" "os" "path/filepath" "strings" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/kdomanski/iso9660" ) func getImage(ss *stream.SeekableStream) (*iso9660.Image, error) { reader, err := stream.NewReadAtSeeker(ss, 0) if err != nil { return nil, err } return iso9660.OpenImage(reader) } func getObj(img *iso9660.Image, path string) (*iso9660.File, error) { obj, err := img.RootDir() if err != nil { return nil, err } if path == "/" { return obj, nil } paths := strings.Split(strings.TrimPrefix(path, "/"), "/") for _, p := range paths { if !obj.IsDir() { return nil, errs.ObjectNotFound } children, err := obj.GetChildren() if err != nil { return nil, err } exist := false for _, child := range children { if child.Name() == p { obj = child exist = true break } } if !exist { return nil, errs.ObjectNotFound } } return obj, nil } func toModelObj(file *iso9660.File) model.Obj { return &model.Object{ Name: file.Name(), Size: file.Size(), Modified: file.ModTime(), IsFolder: file.IsDir(), } } func decompress(f *iso9660.File, path string, up model.UpdateProgress) error { destPath := filepath.Join(path, f.Name()) if !strings.HasPrefix(destPath, path+string(os.PathSeparator)) { return fmt.Errorf("illegal file path: %s", f.Name()) } file, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } defer file.Close() _, err = utils.CopyWithBuffer(file, &stream.ReaderUpdatingProgress{ Reader: &stream.SimpleReaderWithSize{ Reader: f.Reader(), Size: f.Size(), }, UpdateProgress: up, }) return err } func decompressAll(children []*iso9660.File, path string) error { for _, child := range children { if child.IsDir() { nextChildren, err := child.GetChildren() if err != nil { return err } nextPath := filepath.Join(path, child.Name()) if !strings.HasPrefix(nextPath, path+string(os.PathSeparator)) { return fmt.Errorf("illegal file path: %s", child.Name()) } if err = os.MkdirAll(nextPath, 0700); err != nil { return err } if err = decompressAll(nextChildren, nextPath); err != nil { return err } } else { if err := decompress(child, path, func(_ float64) {}); err != nil { return err } } } return nil } ================================================ FILE: internal/archive/rardecode/rardecode.go ================================================ package rardecode import ( "io" "os" "path/filepath" "regexp" "strings" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/nwaples/rardecode/v2" ) type RarDecoder struct{} func (RarDecoder) AcceptedExtensions() []string { return []string{".rar"} } func (RarDecoder) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { return map[string]tool.MultipartExtension{ ".part1.rar": {PartFileFormat: regexp.MustCompile(`^.*\.part(\d+)\.rar$`), SecondPartIndex: 2}, } } func (RarDecoder) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { l, err := list(ss, args.Password) if err != nil { return nil, err } _, tree := tool.GenerateMetaTreeFromFolderTraversal(l) return &model.ArchiveMetaInfo{ Comment: "", Encrypted: false, Tree: tree, }, nil } func (RarDecoder) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { return nil, errs.NotSupport } func (RarDecoder) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { reader, err := getReader(ss, args.Password) if err != nil { return nil, 0, err } innerPath := strings.TrimPrefix(args.InnerPath, "/") for { var header *rardecode.FileHeader header, err = reader.Next() if err == io.EOF { break } if err != nil { return nil, 0, err } if header.Name == innerPath { if header.IsDir { break } return io.NopCloser(reader), header.UnPackedSize, nil } } return nil, 0, errs.ObjectNotFound } func (RarDecoder) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { reader, err := getReader(ss, args.Password) if err != nil { return err } if args.InnerPath == "/" { for { var header *rardecode.FileHeader header, err = reader.Next() if err == io.EOF { break } if err != nil { return err } name := header.Name if header.IsDir { name = name + "/" } err = decompress(reader, header, name, outputPath) if err != nil { return err } } } else { innerPath := strings.TrimPrefix(args.InnerPath, "/") innerBase := filepath.Base(innerPath) createdBaseDir := false for { var header *rardecode.FileHeader header, err = reader.Next() if err == io.EOF { break } if err != nil { return err } name := header.Name if header.IsDir { name = name + "/" } if name == innerPath { err = _decompress(reader, header, outputPath, up) if err != nil { return err } break } else if strings.HasPrefix(name, innerPath+"/") { targetPath := filepath.Join(outputPath, innerBase) if !createdBaseDir { err = os.Mkdir(targetPath, 0700) if err != nil { return err } createdBaseDir = true } restPath := strings.TrimPrefix(name, innerPath+"/") err = decompress(reader, header, restPath, targetPath) if err != nil { return err } } } } return nil } var _ tool.Tool = (*RarDecoder)(nil) func init() { tool.RegisterTool(RarDecoder{}) } ================================================ FILE: internal/archive/rardecode/utils.go ================================================ package rardecode import ( "fmt" "io" "io/fs" "os" "path/filepath" "sort" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/nwaples/rardecode/v2" ) type VolumeFile struct { model.File name string ss model.FileStreamer } func (v *VolumeFile) Name() string { return v.name } func (v *VolumeFile) Size() int64 { return v.ss.GetSize() } func (v *VolumeFile) Mode() fs.FileMode { return 0644 } func (v *VolumeFile) ModTime() time.Time { return v.ss.ModTime() } func (v *VolumeFile) IsDir() bool { return false } func (v *VolumeFile) Sys() any { return nil } func (v *VolumeFile) Stat() (fs.FileInfo, error) { return v, nil } func (v *VolumeFile) Close() error { return nil } type VolumeFs struct { parts map[string]*VolumeFile } func (v *VolumeFs) Open(name string) (fs.File, error) { file, ok := v.parts[name] if !ok { return nil, fs.ErrNotExist } return file, nil } func makeOpts(ss []*stream.SeekableStream) (string, rardecode.Option, error) { if len(ss) == 1 { reader, err := stream.NewReadAtSeeker(ss[0], 0) if err != nil { return "", nil, err } fileName := "file.rar" fsys := &VolumeFs{parts: map[string]*VolumeFile{ fileName: {File: reader, name: fileName}, }} return fileName, rardecode.FileSystem(fsys), nil } else { parts := make(map[string]*VolumeFile, len(ss)) for i, s := range ss { reader, err := stream.NewReadAtSeeker(s, 0) if err != nil { return "", nil, err } fileName := fmt.Sprintf("file.part%d.rar", i+1) parts[fileName] = &VolumeFile{File: reader, name: fileName, ss: s} } return "file.part1.rar", rardecode.FileSystem(&VolumeFs{parts: parts}), nil } } type WrapReader struct { files []*rardecode.File } func (r *WrapReader) Files() []tool.SubFile { ret := make([]tool.SubFile, 0, len(r.files)) for _, f := range r.files { ret = append(ret, &WrapFile{File: f}) } return ret } type WrapFile struct { *rardecode.File } func (f *WrapFile) Name() string { if f.File.IsDir { return f.File.Name + "/" } return f.File.Name } func (f *WrapFile) FileInfo() fs.FileInfo { return &WrapFileInfo{File: f.File} } type WrapFileInfo struct { *rardecode.File } func (f *WrapFileInfo) Name() string { return filepath.Base(f.File.Name) } func (f *WrapFileInfo) Size() int64 { return f.File.UnPackedSize } func (f *WrapFileInfo) ModTime() time.Time { return f.File.ModificationTime } func (f *WrapFileInfo) IsDir() bool { return f.File.IsDir } func (f *WrapFileInfo) Sys() any { return nil } func list(ss []*stream.SeekableStream, password string) (*WrapReader, error) { fileName, fsOpt, err := makeOpts(ss) if err != nil { return nil, err } opts := []rardecode.Option{fsOpt} if password != "" { opts = append(opts, rardecode.Password(password)) } files, err := rardecode.List(fileName, opts...) // rardecode输出文件列表的顺序不一定是父目录在前，子目录在后 // 父路径的长度一定比子路径短，排序后的files可保证父路径在前 sort.Slice(files, func(i, j int) bool { return len(files[i].Name) < len(files[j].Name) }) if err != nil { return nil, filterPassword(err) } return &WrapReader{files: files}, nil } func getReader(ss []*stream.SeekableStream, password string) (*rardecode.Reader, error) { fileName, fsOpt, err := makeOpts(ss) if err != nil { return nil, err } opts := []rardecode.Option{fsOpt} if password != "" { opts = append(opts, rardecode.Password(password)) } rc, err := rardecode.OpenReader(fileName, opts...) if err != nil { return nil, filterPassword(err) } ss[0].Closers.Add(rc) return &rc.Reader, nil } func decompress(reader *rardecode.Reader, header *rardecode.FileHeader, filePath, outputPath string) error { targetPath := outputPath dir, base := filepath.Split(filePath) if dir != "" { targetPath = filepath.Join(targetPath, dir) if strings.HasPrefix(targetPath, outputPath+string(os.PathSeparator)) { err := os.MkdirAll(targetPath, 0700) if err != nil { return err } } else { targetPath = outputPath } } if base != "" { err := _decompress(reader, header, targetPath, func(_ float64) {}) if err != nil { return err } } return nil } func _decompress(reader *rardecode.Reader, header *rardecode.FileHeader, targetPath string, up model.UpdateProgress) error { destPath := filepath.Join(targetPath, filepath.Base(header.Name)) if !strings.HasPrefix(destPath, targetPath+string(os.PathSeparator)) { return fmt.Errorf("illegal file path: %s", filepath.Base(header.Name)) } f, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } defer func() { _ = f.Close() }() _, err = io.Copy(f, &stream.ReaderUpdatingProgress{ Reader: &stream.SimpleReaderWithSize{ Reader: reader, Size: header.UnPackedSize, }, UpdateProgress: up, }) if err != nil { return err } return nil } func filterPassword(err error) error { if err != nil && strings.Contains(err.Error(), "password") { return errs.WrongArchivePassword } return err } ================================================ FILE: internal/archive/sevenzip/sevenzip.go ================================================ package sevenzip import ( "io" "regexp" "strings" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" ) type SevenZip struct{} func (SevenZip) AcceptedExtensions() []string { return []string{".7z"} } func (SevenZip) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { return map[string]tool.MultipartExtension{ ".7z.001": {PartFileFormat: regexp.MustCompile(`^.*\.7z\.(\d+)$`), SecondPartIndex: 2}, } } func (SevenZip) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { reader, err := getReader(ss, args.Password) if err != nil { return nil, err } _, tree := tool.GenerateMetaTreeFromFolderTraversal(&WrapReader{Reader: reader}) return &model.ArchiveMetaInfo{ Comment: "", Encrypted: args.Password != "", Tree: tree, }, nil } func (SevenZip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { return nil, errs.NotSupport } func (SevenZip) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { reader, err := getReader(ss, args.Password) if err != nil { return nil, 0, err } innerPath := strings.TrimPrefix(args.InnerPath, "/") for _, file := range reader.File { if file.Name == innerPath { r, e := file.Open() if e != nil { return nil, 0, e } return r, file.FileInfo().Size(), nil } } return nil, 0, errs.ObjectNotFound } func (SevenZip) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { reader, err := getReader(ss, args.Password) if err != nil { return err } return tool.DecompressFromFolderTraversal(&WrapReader{Reader: reader}, outputPath, args, up) } var _ tool.Tool = (*SevenZip)(nil) func init() { tool.RegisterTool(SevenZip{}) } ================================================ FILE: internal/archive/sevenzip/utils.go ================================================ package sevenzip import ( "errors" "io" "io/fs" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/bodgit/sevenzip" ) type WrapReader struct { Reader *sevenzip.Reader } func (r *WrapReader) Files() []tool.SubFile { ret := make([]tool.SubFile, 0, len(r.Reader.File)) for _, f := range r.Reader.File { ret = append(ret, &WrapFile{f: f}) } return ret } type WrapFile struct { f *sevenzip.File } func (f *WrapFile) Name() string { return f.f.Name } func (f *WrapFile) FileInfo() fs.FileInfo { return f.f.FileInfo() } func (f *WrapFile) Open() (io.ReadCloser, error) { return f.f.Open() } func getReader(ss []*stream.SeekableStream, password string) (*sevenzip.Reader, error) { readerAt, err := stream.NewMultiReaderAt(ss) if err != nil { return nil, err } sr, err := sevenzip.NewReaderWithPassword(readerAt, readerAt.Size(), password) if err != nil { return nil, filterPassword(err) } return sr, nil } func filterPassword(err error) error { if err != nil { var e *sevenzip.ReadError if errors.As(err, &e) && e.Encrypted { return errs.WrongArchivePassword } } return err } ================================================ FILE: internal/archive/tool/base.go ================================================ package tool import ( "io" "regexp" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" ) type MultipartExtension struct { PartFileFormat *regexp.Regexp SecondPartIndex int } type Tool interface { AcceptedExtensions() []string AcceptedMultipartExtensions() map[string]MultipartExtension GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error } ================================================ FILE: internal/archive/tool/helper.go ================================================ package tool import ( "fmt" "io" "io/fs" "os" "path/filepath" "strings" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" ) type SubFile interface { Name() string FileInfo() fs.FileInfo Open() (io.ReadCloser, error) } type CanEncryptSubFile interface { IsEncrypted() bool SetPassword(password string) } type ArchiveReader interface { Files() []SubFile } func GenerateMetaTreeFromFolderTraversal(r ArchiveReader) (bool, []model.ObjTree) { encrypted := false dirMap := make(map[string]*model.ObjectTree) for _, file := range r.Files() { if encrypt, ok := file.(CanEncryptSubFile); ok && encrypt.IsEncrypted() { encrypted = true } name := strings.TrimPrefix(file.Name(), "/") var dir string var dirObj *model.ObjectTree isNewFolder := false if !file.FileInfo().IsDir() { // 先将 文件 添加到 所在的文件夹 dir = filepath.Dir(name) dirObj = dirMap[dir] if dirObj == nil { isNewFolder = dir != "." dirObj = &model.ObjectTree{} dirObj.IsFolder = true dirObj.Name = filepath.Base(dir) dirObj.Modified = file.FileInfo().ModTime() dirMap[dir] = dirObj } dirObj.Children = append( dirObj.Children, &model.ObjectTree{ Object: *MakeModelObj(file.FileInfo()), }, ) } else { dir = strings.TrimSuffix(name, "/") dirObj = dirMap[dir] if dirObj == nil { isNewFolder = dir != "." dirObj = &model.ObjectTree{} dirMap[dir] = dirObj } dirObj.IsFolder = true dirObj.Name = filepath.Base(dir) dirObj.Modified = file.FileInfo().ModTime() } if isNewFolder { // 将 文件夹 添加到 父文件夹 // 考虑压缩包仅记录文件的路径，不记录文件夹 // 循环创建所有父文件夹 parentDir := filepath.Dir(dir) for { parentDirObj := dirMap[parentDir] if parentDirObj == nil { parentDirObj = &model.ObjectTree{} if parentDir != "." { parentDirObj.IsFolder = true parentDirObj.Name = filepath.Base(parentDir) parentDirObj.Modified = file.FileInfo().ModTime() } dirMap[parentDir] = parentDirObj } parentDirObj.Children = append(parentDirObj.Children, dirObj) parentDir = filepath.Dir(parentDir) if dirMap[parentDir] != nil { break } dirObj = parentDirObj } } } if len(dirMap) > 0 { return encrypted, dirMap["."].GetChildren() } else { return encrypted, nil } } func MakeModelObj(file os.FileInfo) *model.Object { return &model.Object{ Name: file.Name(), Size: file.Size(), Modified: file.ModTime(), IsFolder: file.IsDir(), } } type WrapFileInfo struct { model.Obj } func DecompressFromFolderTraversal(r ArchiveReader, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { var err error files := r.Files() if args.InnerPath == "/" { for i, file := range files { name := file.Name() err = decompress(file, name, outputPath, args.Password) if err != nil { return err } up(float64(i+1) * 100.0 / float64(len(files))) } } else { innerPath := strings.TrimPrefix(args.InnerPath, "/") innerBase := filepath.Base(innerPath) createdBaseDir := false for _, file := range files { name := file.Name() if name == innerPath { err = _decompress(file, outputPath, args.Password, up) if err != nil { return err } break } else if strings.HasPrefix(name, innerPath+"/") { targetPath := filepath.Join(outputPath, innerBase) if !createdBaseDir { err = os.Mkdir(targetPath, 0700) if err != nil { return err } createdBaseDir = true } restPath := strings.TrimPrefix(name, innerPath+"/") err = decompress(file, restPath, targetPath, args.Password) if err != nil { return err } } } } return nil } func decompress(file SubFile, filePath, outputPath, password string) error { targetPath := outputPath dir, base := filepath.Split(filePath) if dir != "" { targetPath = filepath.Join(targetPath, dir) if strings.HasPrefix(targetPath, outputPath+string(os.PathSeparator)) { err := os.MkdirAll(targetPath, 0700) if err != nil { return err } } else { targetPath = outputPath } } if base != "" { err := _decompress(file, targetPath, password, func(_ float64) {}) if err != nil { return err } } return nil } func _decompress(file SubFile, targetPath, password string, up model.UpdateProgress) error { if encrypt, ok := file.(CanEncryptSubFile); ok && encrypt.IsEncrypted() { encrypt.SetPassword(password) } rc, err := file.Open() if err != nil { return err } defer func() { _ = rc.Close() }() destPath := filepath.Join(targetPath, file.FileInfo().Name()) if !strings.HasPrefix(destPath, targetPath+string(os.PathSeparator)) { return fmt.Errorf("illegal file path: %s", file.FileInfo().Name()) } f, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } defer func() { _ = f.Close() }() _, err = io.Copy(f, &stream.ReaderUpdatingProgress{ Reader: &stream.SimpleReaderWithSize{ Reader: rc, Size: file.FileInfo().Size(), }, UpdateProgress: up, }) if err != nil { return err } return nil } ================================================ FILE: internal/archive/tool/utils.go ================================================ package tool import ( "github.com/OpenListTeam/OpenList/v4/internal/errs" ) var ( Tools = make(map[string]Tool) MultipartExtensions = make(map[string]MultipartExtension) ) func RegisterTool(tool Tool) { for _, ext := range tool.AcceptedExtensions() { Tools[ext] = tool } for mainFile, ext := range tool.AcceptedMultipartExtensions() { MultipartExtensions[mainFile] = ext Tools[mainFile] = tool } } func GetArchiveTool(ext string) (*MultipartExtension, Tool, error) { t, ok := Tools[ext] if !ok { return nil, nil, errs.UnknownArchiveFormat } partExt, ok := MultipartExtensions[ext] if !ok { return nil, t, nil } return &partExt, t, nil } ================================================ FILE: internal/archive/zip/utils.go ================================================ package zip import ( "bytes" "io" "io/fs" "strings" "github.com/KirCute/zip" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "golang.org/x/text/encoding/ianaindex" "golang.org/x/text/transform" ) type WrapReader struct { Reader *zip.Reader } func (r *WrapReader) Files() []tool.SubFile { ret := make([]tool.SubFile, 0, len(r.Reader.File)) for _, f := range r.Reader.File { ret = append(ret, &WrapFile{f: f}) } return ret } type WrapFileInfo struct { fs.FileInfo efs bool } func (f *WrapFileInfo) Name() string { return decodeName(f.FileInfo.Name(), f.efs) } type WrapFile struct { f *zip.File } func (f *WrapFile) Name() string { return decodeName(f.f.Name, isEFS(f.f.Flags)) } func (f *WrapFile) FileInfo() fs.FileInfo { return &WrapFileInfo{FileInfo: f.f.FileInfo(), efs: isEFS(f.f.Flags)} } func (f *WrapFile) Open() (io.ReadCloser, error) { return f.f.Open() } func (f *WrapFile) IsEncrypted() bool { return f.f.IsEncrypted() } func (f *WrapFile) SetPassword(password string) { f.f.SetPassword(password) } func makePart(ss *stream.SeekableStream) (zip.SizeReaderAt, error) { ra, err := stream.NewReadAtSeeker(ss, 0) if err != nil { return nil, err } return &inlineSizeReaderAt{ReaderAt: ra, size: ss.GetSize()}, nil } func (z *Zip) getReader(ss []*stream.SeekableStream) (*zip.Reader, error) { if len(ss) > 1 && z.traditionalSecondPartRegExp.MatchString(ss[1].GetName()) { ss = append(ss[1:], ss[0]) ras := make([]zip.SizeReaderAt, 0, len(ss)) for _, s := range ss { ra, err := makePart(s) if err != nil { return nil, err } ras = append(ras, ra) } return zip.NewMultipartReader(ras) } else { reader, err := stream.NewMultiReaderAt(ss) if err != nil { return nil, err } return zip.NewReader(reader, reader.Size()) } } func filterPassword(err error) error { if err != nil && strings.Contains(err.Error(), "password") { return errs.WrongArchivePassword } return err } func decodeName(name string, efs bool) string { if efs { return name } enc, err := ianaindex.IANA.Encoding(setting.GetStr(conf.NonEFSZipEncoding)) if err != nil { return name } i := bytes.NewReader([]byte(name)) decoder := transform.NewReader(i, enc.NewDecoder()) content, _ := io.ReadAll(decoder) return string(content) } func isEFS(flags uint16) bool { return (flags & 0x800) > 0 } type inlineSizeReaderAt struct { io.ReaderAt size int64 } func (i *inlineSizeReaderAt) Size() int64 { return i.size } ================================================ FILE: internal/archive/zip/zip.go ================================================ package zip import ( "io" stdpath "path" "regexp" "strings" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" ) type Zip struct { traditionalSecondPartRegExp *regexp.Regexp } func (z *Zip) AcceptedExtensions() []string { return []string{} } func (z *Zip) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { return map[string]tool.MultipartExtension{ ".zip": {PartFileFormat: regexp.MustCompile(`^.*\.z(\d+)$`), SecondPartIndex: 1}, ".zip.001": {PartFileFormat: regexp.MustCompile(`^.*\.zip\.(\d+)$`), SecondPartIndex: 2}, } } func (z *Zip) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { zipReader, err := z.getReader(ss) if err != nil { return nil, err } efs := true if len(zipReader.File) > 0 { efs = isEFS(zipReader.File[0].Flags) } encrypted, tree := tool.GenerateMetaTreeFromFolderTraversal(&WrapReader{Reader: zipReader}) return &model.ArchiveMetaInfo{ Comment: decodeName(zipReader.Comment, efs), Encrypted: encrypted, Tree: tree, }, nil } func (z *Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { zipReader, err := z.getReader(ss) if err != nil { return nil, err } if args.InnerPath == "/" { ret := make([]model.Obj, 0) passVerified := false var dir *model.Object for _, file := range zipReader.File { if !passVerified && file.IsEncrypted() { file.SetPassword(args.Password) rc, e := file.Open() if e != nil { return nil, filterPassword(e) } _ = rc.Close() passVerified = true } name := strings.TrimSuffix(decodeName(file.Name, isEFS(file.Flags)), "/") if strings.Contains(name, "/") { // 有些压缩包不压缩第一个文件夹 strs := strings.Split(name, "/") if dir == nil && len(strs) == 2 { dir = &model.Object{ Name: strs[0], Modified: ss[0].ModTime(), IsFolder: true, } } continue } ret = append(ret, tool.MakeModelObj(&WrapFileInfo{FileInfo: file.FileInfo(), efs: isEFS(file.Flags)})) } if len(ret) == 0 && dir != nil { ret = append(ret, dir) } return ret, nil } else { innerPath := strings.TrimPrefix(args.InnerPath, "/") + "/" ret := make([]model.Obj, 0) exist := false for _, file := range zipReader.File { name := decodeName(file.Name, isEFS(file.Flags)) dir := stdpath.Dir(strings.TrimSuffix(name, "/")) + "/" if dir != innerPath { continue } exist = true ret = append(ret, tool.MakeModelObj(&WrapFileInfo{file.FileInfo(), isEFS(file.Flags)})) } if !exist { return nil, errs.ObjectNotFound } return ret, nil } } func (z *Zip) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { zipReader, err := z.getReader(ss) if err != nil { return nil, 0, err } innerPath := strings.TrimPrefix(args.InnerPath, "/") for _, file := range zipReader.File { if decodeName(file.Name, isEFS(file.Flags)) == innerPath { if file.IsEncrypted() { file.SetPassword(args.Password) } r, e := file.Open() if e != nil { return nil, 0, e } return r, file.FileInfo().Size(), nil } } return nil, 0, errs.ObjectNotFound } func (z *Zip) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { zipReader, err := z.getReader(ss) if err != nil { return err } return tool.DecompressFromFolderTraversal(&WrapReader{Reader: zipReader}, outputPath, args, up) } var _ tool.Tool = (*Zip)(nil) func init() { tool.RegisterTool(&Zip{ traditionalSecondPartRegExp: regexp.MustCompile(`^.*\.z0*1$`), }) } ================================================ FILE: internal/authn/authn.go ================================================ package authn import ( "fmt" "net/url" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/go-webauthn/webauthn/webauthn" ) func NewAuthnInstance(c *gin.Context) (*webauthn.WebAuthn, error) { siteUrl, err := url.Parse(common.GetApiUrl(c.Request.Context())) if err != nil { return nil, err } return webauthn.New(&webauthn.Config{ RPDisplayName: setting.GetStr(conf.SiteTitle), RPID: siteUrl.Hostname(), //RPOrigin: siteUrl.String(), RPOrigins: []string{fmt.Sprintf("%s://%s", siteUrl.Scheme, siteUrl.Host)}, // RPOrigin: "http://localhost:5173" }) } ================================================ FILE: internal/bootstrap/config.go ================================================ package bootstrap import ( "net/url" "os" "path/filepath" "strings" "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/caarlos0/env/v9" "github.com/shirou/gopsutil/v4/mem" log "github.com/sirupsen/logrus" ) // Program working directory func PWD() string { if flags.ForceBinDir { ex, err := os.Executable() if err != nil { log.Fatal(err) } pwd := filepath.Dir(ex) return pwd } d, err := os.Getwd() if err != nil { d = "." } return d } func InitConfig() { pwd := PWD() dataDir := flags.DataDir if !filepath.IsAbs(dataDir) { flags.DataDir = filepath.Join(pwd, flags.DataDir) } // Determine config file path: use flags.ConfigPath if provided, otherwise default to /config.json configPath := flags.ConfigPath if configPath == "" { configPath = filepath.Join(flags.DataDir, "config.json") } else { // if relative, resolve relative to working directory if !filepath.IsAbs(configPath) { if absPath, err := filepath.Abs(configPath); err == nil { configPath = absPath } else { configPath = filepath.Join(pwd, configPath) } } } configPath = filepath.Clean(configPath) conf.ConfigPath = configPath log.Infof("reading config file: %s", configPath) if !utils.Exists(configPath) { log.Infof("config file not exists, creating default config file") _, err := utils.CreateNestedFile(configPath) if err != nil { log.Fatalf("failed to create config file: %+v", err) } conf.Conf = conf.DefaultConfig(dataDir) LastLaunchedVersion = conf.Version conf.Conf.LastLaunchedVersion = conf.Version if !utils.WriteJsonToFile(configPath, conf.Conf) { log.Fatalf("failed to create default config file") } } else { configBytes, err := os.ReadFile(configPath) if err != nil { log.Fatalf("reading config file error: %+v", err) } conf.Conf = conf.DefaultConfig(dataDir) err = utils.Json.Unmarshal(configBytes, conf.Conf) if err != nil { log.Fatalf("load config error: %+v", err) } LastLaunchedVersion = conf.Conf.LastLaunchedVersion if strings.HasPrefix(conf.Version, "v") || LastLaunchedVersion == "" { conf.Conf.LastLaunchedVersion = conf.Version } // update config.json struct confBody, err := utils.Json.MarshalIndent(conf.Conf, "", " ") if err != nil { log.Fatalf("marshal config error: %+v", err) } err = os.WriteFile(configPath, confBody, 0o777) if err != nil { log.Fatalf("update config struct error: %+v", err) } } if !conf.Conf.Force { confFromEnv() } if conf.Conf.MaxConcurrency > 0 { net.DefaultConcurrencyLimit = &net.ConcurrencyLimit{Limit: conf.Conf.MaxConcurrency} } if conf.Conf.MaxBufferLimit < 0 { m, _ := mem.VirtualMemory() if m != nil { conf.MaxBufferLimit = max(int(float64(m.Total)*0.05), 4*utils.MB) conf.MaxBufferLimit -= conf.MaxBufferLimit % utils.MB } else { conf.MaxBufferLimit = 16 * utils.MB } } else { conf.MaxBufferLimit = conf.Conf.MaxBufferLimit * utils.MB } log.Infof("max buffer limit: %dMB", conf.MaxBufferLimit/utils.MB) if conf.Conf.MmapThreshold > 0 { conf.MmapThreshold = conf.Conf.MmapThreshold * utils.MB } else { conf.MmapThreshold = 0 } log.Infof("mmap threshold: %dMB", conf.Conf.MmapThreshold) if len(conf.Conf.Log.Filter.Filters) == 0 { conf.Conf.Log.Filter.Enable = false } // convert abs path convertAbsPath := func(path *string) { if *path != "" && !filepath.IsAbs(*path) { *path = filepath.Join(pwd, *path) } } convertAbsPath(&conf.Conf.Database.DBFile) convertAbsPath(&conf.Conf.Scheme.CertFile) convertAbsPath(&conf.Conf.Scheme.KeyFile) convertAbsPath(&conf.Conf.Scheme.UnixFile) convertAbsPath(&conf.Conf.Log.Name) convertAbsPath(&conf.Conf.TempDir) convertAbsPath(&conf.Conf.BleveDir) convertAbsPath(&conf.Conf.DistDir) err := os.MkdirAll(conf.Conf.TempDir, 0o777) if err != nil { log.Fatalf("create temp dir error: %+v", err) } log.Debugf("config: %+v", conf.Conf) // Validate and display proxy configuration status validateProxyConfig() base.InitClient() initURL() } func confFromEnv() { prefix := "OPENLIST_" if flags.NoPrefix { prefix = "" } log.Infof("load config from env with prefix: %s", prefix) if err := env.ParseWithOptions(conf.Conf, env.Options{ Prefix: prefix, }); err != nil { log.Fatalf("load config from env error: %+v", err) } } func initURL() { if !strings.Contains(conf.Conf.SiteURL, "://") { conf.Conf.SiteURL = utils.FixAndCleanPath(conf.Conf.SiteURL) } u, err := url.Parse(conf.Conf.SiteURL) if err != nil { utils.Log.Fatalf("can't parse site_url: %+v", err) } conf.URL = u } func CleanTempDir() { files, err := os.ReadDir(conf.Conf.TempDir) if err != nil { log.Errorln("failed list temp file: ", err) } for _, file := range files { if err := os.RemoveAll(filepath.Join(conf.Conf.TempDir, file.Name())); err != nil { log.Errorln("failed delete temp file: ", err) } } } // validateProxyConfig validates proxy configuration and displays status at startup func validateProxyConfig() { if conf.Conf.ProxyAddress != "" { if _, err := url.Parse(conf.Conf.ProxyAddress); err == nil { log.Infof("Proxy enabled: %s", conf.Conf.ProxyAddress) } else { log.Errorf("Invalid proxy address format: %s, error: %v", conf.Conf.ProxyAddress, err) } } } ================================================ FILE: internal/bootstrap/data/data.go ================================================ package data import "github.com/OpenListTeam/OpenList/v4/cmd/flags" func InitData() { initUser() initSettings() initTasks() if flags.Dev { initDevData() initDevDo() } } ================================================ FILE: internal/bootstrap/data/dev.go ================================================ package data import ( "context" "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/message" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" log "github.com/sirupsen/logrus" ) func initDevData() { _, err := op.CreateStorage(context.Background(), model.Storage{ MountPath: "/", Order: 0, Driver: "Local", Status: "", Addition: `{"root_folder_path":"."}`, }) if err != nil { log.Fatalf("failed to create storage: %+v", err) } err = db.CreateUser(&model.User{ Username: "Noah", Password: "hsu", BasePath: "/data", Role: 0, Permission: 512, }) if err != nil { log.Fatalf("failed to create user: %+v", err) } } func initDevDo() { if flags.Dev { go func() { err := message.GetMessenger().WaitSend(message.Message{ Type: "string", Content: "dev mode", }, 10) if err != nil { log.Debugf("%+v", err) } m, err := message.GetMessenger().WaitReceive(10) if err != nil { log.Debugf("%+v", err) } else { log.Debugf("received: %+v", m) } }() } } ================================================ FILE: internal/bootstrap/data/setting.go ================================================ package data import ( "fmt" "sort" "strconv" "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/pkg/errors" "gorm.io/gorm" ) func initSettings() { initialSettingItems := InitialSettings() isActive := func(key string) bool { for _, item := range initialSettingItems { if item.Key == key { return true } } return false } // check deprecated settings, err := op.GetSettingItems() if err != nil { utils.Log.Fatalf("failed get settings: %+v", err) } settingMap := map[string]*model.SettingItem{} for _, v := range settings { if !isActive(v.Key) && v.Flag != model.DEPRECATED { v.Flag = model.DEPRECATED err = op.SaveSettingItem(&v) if err != nil { utils.Log.Fatalf("failed save setting: %+v", err) } } settingMap[v.Key] = &v } op.MigrationSettingItems = map[string]op.MigrationValueItem{} // create or save setting var saveItems []model.SettingItem for i := range initialSettingItems { item := &initialSettingItems[i] item.Index = uint(i) migrationValue := item.MigrationValue if len(migrationValue) > 0 { op.MigrationSettingItems[item.Key] = op.MigrationValueItem{MigrationValue: item.MigrationValue, Value: item.Value} item.MigrationValue = "" } // err stored, ok := settingMap[item.Key] if !ok { stored, err = op.GetSettingItemByKey(item.Key) if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { utils.Log.Fatalf("failed get setting: %+v", err) continue } } if item.Key != conf.VERSION && stored != nil && (len(migrationValue) == 0 || stored.Value != migrationValue) { item.Value = stored.Value } _, err = op.HandleSettingItemHook(item) if err != nil { utils.Log.Errorf("failed to execute hook on %s: %+v", item.Key, err) continue } if stored == nil || *item != *stored { saveItems = append(saveItems, *item) } } if len(saveItems) > 0 { err = db.SaveSettingItems(saveItems) if err != nil { utils.Log.Fatalf("failed save setting: %+v", err) } else { op.SettingCacheUpdate() } } } func InitialSettings() []model.SettingItem { var token string if flags.Dev { token = "dev_token" } else { token = random.Token() } siteVersion := fmt.Sprintf("%s (Commit: %s) - Frontend: %s - Build at: %s", conf.Version, conf.GitCommit, conf.WebVersion, conf.BuiltAt) initialSettingItems := []model.SettingItem{ // site settings {Key: conf.VERSION, Value: siteVersion, Type: conf.TypeString, Group: model.SITE, Flag: model.READONLY}, //{Key: conf.ApiUrl, Value: "", Type: conf.TypeString, Group: model.SITE}, //{Key: conf.BasePath, Value: "", Type: conf.TypeString, Group: model.SITE}, {Key: conf.SiteTitle, Value: "OpenList", Type: conf.TypeString, Group: model.SITE}, {Key: conf.Announcement, Value: "Welcome to the OpenList project!\nFor the latest updates, to contribute code, or to submit suggestions and issues, please visit our [project repository](https://github.com/OpenListTeam/OpenList).", Type: conf.TypeText, Group: model.SITE}, {Key: "pagination_type", Value: "all", Type: conf.TypeSelect, Options: "all,pagination,load_more,auto_load_more", Group: model.SITE}, {Key: "default_page_size", Value: "30", Type: conf.TypeNumber, Group: model.SITE}, {Key: conf.AllowIndexed, Value: "false", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.AllowMounted, Value: "true", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.RobotsTxt, Value: "User-agent: *\nAllow: /", Type: conf.TypeText, Group: model.SITE}, // style settings {Key: conf.Logo, Value: "https://res.oplist.org/logo/logo.svg", MigrationValue: "https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.svg", Type: conf.TypeText, Group: model.STYLE}, {Key: conf.Favicon, Value: "https://res.oplist.org/logo/logo.svg", MigrationValue: "https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.svg", Type: conf.TypeString, Group: model.STYLE}, {Key: conf.MainColor, Value: "#1890ff", Type: conf.TypeString, Group: model.STYLE}, {Key: "home_icon", Value: "🏠", Type: conf.TypeString, Group: model.STYLE}, {Key: "share_icon", Value: "🎁", Type: conf.TypeString, Group: model.STYLE}, {Key: "home_container", Value: "max_980px", Type: conf.TypeSelect, Options: "max_980px,hope_container", Group: model.STYLE}, {Key: "settings_layout", Value: "list", Type: conf.TypeSelect, Options: "list,responsive", Group: model.STYLE}, {Key: conf.HideStorageDetails, Value: "true", Type: conf.TypeBool, Group: model.STYLE, Flag: model.PRIVATE}, {Key: conf.HideStorageDetailsInManagePage, Value: "true", Type: conf.TypeBool, Group: model.STYLE, Flag: model.PRIVATE}, {Key: "show_disk_usage_in_plain_text", Value: "false", Type: conf.TypeBool, Group: model.STYLE, Flag: model.PUBLIC}, // preview settings {Key: conf.TextTypes, Value: "txt,htm,html,xml,java,properties,sql,js,md,json,conf,ini,vue,php,py,bat,gitignore,yml,go,sh,c,cpp,h,hpp,tsx,vtt,srt,ass,rs,lrc,strm", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: conf.AudioTypes, Value: "mp3,flac,ogg,m4a,wav,opus,wma", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: conf.VideoTypes, Value: "mp4,mkv,avi,mov,rmvb,webm,flv,m3u8", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: conf.ImageTypes, Value: "jpg,tiff,jpeg,png,gif,bmp,svg,ico,swf,webp,avif", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, //{Key: conf.OfficeTypes, Value: "doc,docx,xls,xlsx,ppt,pptx", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: conf.ProxyTypes, Value: "m3u8,url", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: conf.ProxyIgnoreHeaders, Value: "authorization,referer", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: "external_previews", Value: `{}`, Type: conf.TypeText, Group: model.PREVIEW}, {Key: "iframe_previews", Value: `{ "doc,docx,xls,xlsx,ppt,pptx": { "Microsoft":"https://view.officeapps.live.com/op/view.aspx?src=$e_url", "Google":"https://docs.google.com/gview?url=$e_url&embedded=true" }, "pdf": { "PDF.js":"https://res.oplist.org/pdf.js/web/viewer.html?file=$e_url" }, "epub": { "EPUB.js":"https://res.oplist.org/epub.js/viewer.html?url=$e_url" } }`, Type: conf.TypeText, Group: model.PREVIEW}, // {Key: conf.OfficeViewers, Value: `{ // "Microsoft":"https://view.officeapps.live.com/op/view.aspx?src=$url", // "Google":"https://docs.google.com/gview?url=$url&embedded=true", //}`, Type: conf.TypeText, Group: model.PREVIEW}, // {Key: conf.PdfViewers, Value: `{ // "pdf.js":"https://openlistteam.github.io/pdf.js/web/viewer.html?file=$url" //}`, Type: conf.TypeText, Group: model.PREVIEW}, {Key: "audio_cover", Value: "https://res.oplist.org/logo/logo.svg", MigrationValue: "https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.svg", Type: conf.TypeString, Group: model.PREVIEW}, {Key: conf.AudioAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.VideoAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.PreviewDownloadByDefault, Value: "false", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.PreviewArchivesByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.SharePreviewDownloadByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.SharePreviewArchivesByDefault, Value: "false", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.ReadMeAutoRender, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.NonEFSZipEncoding, Value: "IBM437", Type: conf.TypeString, Group: model.PREVIEW}, // global settings {Key: conf.HideFiles, Value: "/\\/README.md/i", Type: conf.TypeText, Group: model.GLOBAL}, {Key: "package_download", Value: "true", Type: conf.TypeBool, Group: model.GLOBAL}, {Key: conf.CustomizeHead, MigrationValue: ``, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.CustomizeBody, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.LinkExpiration, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.SignAll, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, { Key: conf.PrivacyRegs, Value: `(?:(?:\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5]) ([[:xdigit:]]{1,4}(?::[[:xdigit:]]{1,4}){7}|::|:(?::[[:xdigit:]]{1,4}){1,6}|[[:xdigit:]]{1,4}:(?::[[:xdigit:]]{1,4}){1,5}|(?:[[:xdigit:]]{1,4}:){2}(?::[[:xdigit:]]{1,4}){1,4}|(?:[[:xdigit:]]{1,4}:){3}(?::[[:xdigit:]]{1,4}){1,3}|(?:[[:xdigit:]]{1,4}:){4}(?::[[:xdigit:]]{1,4}){1,2}|(?:[[:xdigit:]]{1,4}:){5}:[[:xdigit:]]{1,4}|(?:[[:xdigit:]]{1,4}:){1,6}:) (?U)access_token=(.*)&`, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE, }, {Key: conf.OcrApi, Value: "https://openlistteam-ocr-api-server.hf.space/ocr/file/json", MigrationValue: "https://api.example.com/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL}, // TODO: This can be replace by a community-hosted endpoint, see https://github.com/OpenListTeam/ocr_api_server {Key: conf.FilenameCharMapping, Value: `{"/": "|"}`, Type: conf.TypeText, Group: model.GLOBAL}, {Key: conf.ForwardDirectLinkParams, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL}, {Key: conf.IgnoreDirectLinkParams, Value: "sign,openlist_ts,raw", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.WebauthnLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, {Key: conf.SharePreview, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, {Key: conf.ShareArchivePreview, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, {Key: conf.ShareForceProxy, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.ShareSummaryContent, Value: "@{{creator}} shared {{#each files}}{{#if @first}}\"{{filename this}}\"{{/if}}{{#if @last}}{{#unless (eq @index 0)}} and {{@index}} more files{{/unless}}{{/if}}{{/each}} from {{site_title}}: {{base_url}}/@s/{{id}}{{#if pwd}} , the share code is {{pwd}}{{/if}}{{#if expires}}, please access before {{dateLocaleString expires}}.{{/if}}", Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PUBLIC}, {Key: conf.HandleHookAfterWriting, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.HandleHookRateLimit, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.IgnoreSystemFiles, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `When enabled, ignores common system files during upload (.DS_Store, desktop.ini, Thumbs.db, and files starting with ._)`}, // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,meilisearch,none", Group: model.INDEX}, {Key: conf.AutoUpdateIndex, Value: "false", Type: conf.TypeBool, Group: model.INDEX}, {Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`}, {Key: conf.MaxIndexDepth, Value: "20", Type: conf.TypeNumber, Group: model.INDEX, Flag: model.PRIVATE, Help: `max depth of index`}, {Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE}, // SSO settings {Key: conf.SSOLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.SSO, Flag: model.PUBLIC}, {Key: conf.SSOLoginPlatform, Type: conf.TypeSelect, Options: "Casdoor,Github,Microsoft,Google,Dingtalk,OIDC", Group: model.SSO, Flag: model.PUBLIC}, {Key: conf.SSOClientId, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOClientSecret, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOOIDCUsernameKey, Value: "name", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOOrganizationName, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOApplicationName, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOEndpointName, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOJwtPublicKey, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOExtraScopes, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOAutoRegister, Value: "false", Type: conf.TypeBool, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSODefaultDir, Value: "/", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSODefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOCompatibilityMode, Value: "false", Type: conf.TypeBool, Group: model.SSO, Flag: model.PUBLIC}, // ldap settings {Key: conf.LdapLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.LDAP, Flag: model.PUBLIC}, {Key: conf.LdapServer, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapSkipTlsVerify, Value: "false", Type: conf.TypeBool, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapManagerDN, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapManagerPassword, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapUserSearchBase, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapUserSearchFilter, Value: "(uid=%s)", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapDefaultDir, Value: "/", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapDefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapLoginTips, Value: "login with ldap", Type: conf.TypeString, Group: model.LDAP, Flag: model.PUBLIC}, // s3 settings {Key: conf.S3AccessKeyId, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, {Key: conf.S3SecretAccessKey, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, {Key: conf.S3Buckets, Value: "[]", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, // ftp settings {Key: conf.FTPPublicHost, Value: "127.0.0.1", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, {Key: conf.FTPPasvPortMap, Value: "", Type: conf.TypeText, Group: model.FTP, Flag: model.PRIVATE}, {Key: conf.FTPMandatoryTLS, Value: "false", Type: conf.TypeBool, Group: model.FTP, Flag: model.PRIVATE}, {Key: conf.FTPImplicitTLS, Value: "false", Type: conf.TypeBool, Group: model.FTP, Flag: model.PRIVATE}, {Key: conf.FTPTLSPrivateKeyPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, {Key: conf.FTPTLSPublicCertPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, {Key: conf.SFTPDisablePasswordLogin, Value: "false", Type: conf.TypeBool, Group: model.FTP, Flag: model.PRIVATE}, // traffic settings {Key: conf.TaskOfflineDownloadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Download.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskOfflineDownloadTransferThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Transfer.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskUploadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Upload.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskCopyThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Copy.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskDecompressDownloadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Decompress.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskDecompressUploadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.DecompressUpload.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.StreamMaxClientDownloadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.StreamMaxClientUploadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.StreamMaxServerDownloadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.StreamMaxServerUploadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, } additionalSettingItems := tool.Tools.Items() // 固定顺序 sort.Slice(additionalSettingItems, func(i, j int) bool { return additionalSettingItems[i].Key < additionalSettingItems[j].Key }) initialSettingItems = append(initialSettingItems, additionalSettingItems...) if flags.Dev { initialSettingItems = append(initialSettingItems, []model.SettingItem{ {Key: "test_deprecated", Value: "test_value", Type: conf.TypeString, Flag: model.DEPRECATED}, {Key: "test_options", Value: "a", Type: conf.TypeSelect, Options: "a,b,c"}, {Key: "test_help", Type: conf.TypeString, Help: "this is a help message"}, }...) } return initialSettingItems } ================================================ FILE: internal/bootstrap/data/task.go ================================================ package data import ( "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" ) var initialTaskItems []model.TaskItem func initTasks() { InitialTasks() for i := range initialTaskItems { item := &initialTaskItems[i] taskitem, _ := db.GetTaskDataByType(item.Key) if taskitem == nil { db.CreateTaskData(item) } } } func InitialTasks() []model.TaskItem { initialTaskItems = []model.TaskItem{ {Key: "copy", PersistData: "[]"}, {Key: "move", PersistData: "[]"}, {Key: "download", PersistData: "[]"}, {Key: "transfer", PersistData: "[]"}, } return initialTaskItems } ================================================ FILE: internal/bootstrap/data/user.go ================================================ package data import ( "fmt" "os" "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/pkg/errors" "gorm.io/gorm" ) func initUser() { admin, err := op.GetAdmin() adminPassword := random.String(8) envpass := os.Getenv("OPENLIST_ADMIN_PASSWORD") if flags.Dev { adminPassword = "admin" } else if len(envpass) > 0 { adminPassword = envpass } if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { salt := random.String(16) admin = &model.User{ Username: "admin", Salt: salt, PwdHash: model.TwoHashPwd(adminPassword, salt), Role: model.ADMIN, BasePath: "/", Authn: "[]", // 0(can see hidden) - 8(webdav read) & 12(can read archives) - 14(can share) Permission: 0x71FF, } if err := op.CreateUser(admin); err != nil { panic(err) } else { // DO NOT output the password to log file. Only output to console. // utils.Log.Infof("Successfully created the admin user and the initial password is: %s", adminPassword) fmt.Printf("Successfully created the admin user and the initial password is: %s\n", adminPassword) } } else { utils.Log.Fatalf("[init user] Failed to get admin user: %v", err) } } _, err = op.GetGuest() if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { salt := random.String(16) guest := &model.User{ Username: "guest", PwdHash: model.TwoHashPwd("guest", salt), Salt: salt, Role: model.GUEST, BasePath: "/", Permission: 0, Disabled: true, Authn: "[]", } if err := db.CreateUser(guest); err != nil { utils.Log.Fatalf("[init user] Failed to create guest user: %v", err) } } else { utils.Log.Fatalf("[init user] Failed to get guest user: %v", err) } } } ================================================ FILE: internal/bootstrap/db.go ================================================ package bootstrap import ( "fmt" stdlog "log" "strings" "time" "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" log "github.com/sirupsen/logrus" "gorm.io/driver/mysql" "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" "gorm.io/gorm/schema" ) func InitDB() { logLevel := logger.Silent if flags.Debug || flags.Dev { logLevel = logger.Info } newLogger := logger.New( stdlog.New(log.StandardLogger().Out, "\r\n", stdlog.LstdFlags), logger.Config{ SlowThreshold: time.Second, LogLevel: logLevel, IgnoreRecordNotFoundError: true, Colorful: true, }, ) gormConfig := &gorm.Config{ NamingStrategy: schema.NamingStrategy{ TablePrefix: conf.Conf.Database.TablePrefix, }, Logger: newLogger, } var dB *gorm.DB var err error if flags.Dev { dB, err = gorm.Open(sqlite.Open("file::memory:?cache=shared"), gormConfig) conf.Conf.Database.Type = "sqlite3" } else { database := conf.Conf.Database switch database.Type { case "sqlite3": { if !(strings.HasSuffix(database.DBFile, ".db") && len(database.DBFile) > 3) { log.Fatalf("db name error.") } dB, err = gorm.Open(sqlite.Open(fmt.Sprintf("%s?_journal=WAL&_vacuum=incremental", database.DBFile)), gormConfig) } case "mysql": { dsn := database.DSN if dsn == "" { //[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] dsn = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local&tls=%s", database.User, database.Password, database.Host, database.Port, database.Name, database.SSLMode) } dB, err = gorm.Open(mysql.Open(dsn), gormConfig) } case "postgres": { dsn := database.DSN if dsn == "" { if database.Password != "" { dsn = fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%d sslmode=%s TimeZone=Asia/Shanghai", database.Host, database.User, database.Password, database.Name, database.Port, database.SSLMode) } else { dsn = fmt.Sprintf("host=%s user=%s dbname=%s port=%d sslmode=%s TimeZone=Asia/Shanghai", database.Host, database.User, database.Name, database.Port, database.SSLMode) } } dB, err = gorm.Open(postgres.Open(dsn), gormConfig) } default: log.Fatalf("not supported database type: %s", database.Type) } } if err != nil { log.Fatalf("failed to connect database:%s", err.Error()) } db.Init(dB) } ================================================ FILE: internal/bootstrap/index.go ================================================ package bootstrap import ( "github.com/OpenListTeam/OpenList/v4/internal/search" log "github.com/sirupsen/logrus" ) func InitIndex() { progress, err := search.Progress() if err != nil { log.Errorf("init index error: %+v", err) return } if !progress.IsDone { progress.IsDone = true search.WriteProgress(progress) } } ================================================ FILE: internal/bootstrap/log.go ================================================ package bootstrap import ( "io" "log" "os" "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/natefinch/lumberjack" "github.com/sirupsen/logrus" ) func init() { formatter := logrus.TextFormatter{ ForceColors: true, EnvironmentOverrideColors: true, TimestampFormat: "2006-01-02 15:04:05", FullTimestamp: true, } logrus.SetFormatter(&formatter) utils.Log.SetFormatter(&formatter) // logrus.SetLevel(logrus.DebugLevel) } func setLog(l *logrus.Logger) { if flags.Debug || flags.Dev { l.SetLevel(logrus.DebugLevel) l.SetReportCaller(true) } else { l.SetLevel(logrus.InfoLevel) l.SetReportCaller(false) } } func Log() { setLog(logrus.StandardLogger()) setLog(utils.Log) logConfig := conf.Conf.Log if logConfig.Enable { var w io.Writer = &lumberjack.Logger{ Filename: logConfig.Name, MaxSize: logConfig.MaxSize, // megabytes MaxBackups: logConfig.MaxBackups, MaxAge: logConfig.MaxAge, //days Compress: logConfig.Compress, // disabled by default } if flags.Debug || flags.Dev || flags.LogStd { w = io.MultiWriter(os.Stdout, w) } logrus.SetOutput(w) } log.SetOutput(logrus.StandardLogger().Out) utils.Log.Infof("init logrus...") utils.Log = logrus.StandardLogger() } ================================================ FILE: internal/bootstrap/offline_download.go ================================================ package bootstrap import ( "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) func InitOfflineDownloadTools() { for k, v := range tool.Tools { res, err := v.Init() if err != nil { utils.Log.Warnf("init offline download tool %s failed: %s", k, err) } else { utils.Log.Infof("init offline download tool %s success: %s", k, res) } } } ================================================ FILE: internal/bootstrap/patch/all.go ================================================ package patch import ( "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v3_24_0" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v3_32_0" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v3_41_0" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v4_1_8" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v4_1_9" ) type VersionPatches struct { // Version means if the system is upgraded from Version or an earlier one // to the current version, all patches in Patches will be executed. Version string Patches []func() } var UpgradePatches = []VersionPatches{ { Version: "v3.24.0", Patches: []func(){ v3_24_0.HashPwdForOldVersion, }, }, { Version: "v3.32.0", Patches: []func(){ v3_32_0.UpdateAuthnForOldVersion, }, }, { Version: "v3.41.0", Patches: []func(){ v3_41_0.GrantAdminPermissions, }, }, { Version: "v4.1.8", Patches: []func(){ v4_1_8.FixAliasConfig, }, }, { Version: "v4.1.9", Patches: []func(){ v4_1_9.EnableWebDavProxy, v4_1_9.ResetSkipTlsVerify, }, }, } ================================================ FILE: internal/bootstrap/patch/v3_24_0/hash_password.go ================================================ package v3_24_0 import ( "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) // HashPwdForOldVersion encode passwords using SHA256 // First published: 75acbcc perf: sha256 for user's password (close #3552) by Andy Hsu func HashPwdForOldVersion() { users, _, err := op.GetUsers(1, -1) if err != nil { utils.Log.Errorf("[hash pwd for old version] failed get users: %v", err) return } for i := range users { user := users[i] if user.PwdHash == "" { user.SetPassword(user.Password) user.Password = "" if err := db.UpdateUser(&user); err != nil { utils.Log.Errorf("[hash pwd for old version] failed update user: %v", err) } } } } ================================================ FILE: internal/bootstrap/patch/v3_32_0/update_authn.go ================================================ package v3_32_0 import ( "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) // UpdateAuthnForOldVersion updates users' authn // First published: bdfc159 fix: webauthn logspam (#6181) by itsHenry func UpdateAuthnForOldVersion() { users, _, err := op.GetUsers(1, -1) if err != nil { utils.Log.Errorf("[update authn for old version] failed get users: %v", err) return } for i := range users { user := users[i] if user.Authn == "" { user.Authn = "[]" if err := db.UpdateUser(&user); err != nil { utils.Log.Errorf("[update authn for old version] failed update user: %v", err) } } } } ================================================ FILE: internal/bootstrap/patch/v3_41_0/grant_permission.go ================================================ package v3_41_0 import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) // GrantAdminPermissions gives admin Permission 0(can see hidden) - 9(webdav manage) and // 12(can read archives) - 13(can decompress archives) // This patch is written to help users upgrading from older version better adapt func GrantAdminPermissions() { admin, err := op.GetAdmin() if err == nil && (admin.Permission&0x33FF) == 0 { admin.Permission |= 0x33FF err = op.UpdateUser(admin) } if err != nil { utils.Log.Errorf("Cannot grant permissions to admin: %v", err) } } ================================================ FILE: internal/bootstrap/patch/v4_1_8/alias.go ================================================ package v4_1_8 import ( "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) // FixAliasConfig upgrade the old version of the Addition of the Alias driver func FixAliasConfig() { storages, _, err := db.GetStorages(1, -1) if err != nil { utils.Log.Errorf("[FixAliasConfig] failed to get storages: %s", err.Error()) return } for _, s := range storages { if s.Driver != "Alias" { continue } addition := make(map[string]any) err = utils.Json.UnmarshalFromString(s.Addition, &addition) if err != nil { utils.Log.Errorf("[FixAliasConfig] failed to unmarshal addition of [%d]%s: %s", s.ID, s.MountPath, err.Error()) continue } if _, ok := addition["read_conflict_policy"]; ok { utils.Log.Infof("[FixAliasConfig] skip fixing [%d]%s because the addition already has \"read_conflict_policy\" key", s.ID, s.MountPath) continue } var protectSameName, parallelWrite, writable bool protectSameNameAny, ok := addition["protect_same_name"] if ok { delete(addition, "protect_same_name") protectSameName, ok = protectSameNameAny.(bool) } if !ok { protectSameName = false } parallelWriteAny, ok := addition["parallel_write"] if ok { delete(addition, "parallel_write") parallelWrite, ok = parallelWriteAny.(bool) } if !ok { parallelWrite = false } writableAny, ok := addition["writable"] if ok { delete(addition, "writable") writable, ok = writableAny.(bool) } if !ok { writable = false } if !writable { addition["write_conflict_policy"] = "disabled" addition["put_conflict_policy"] = "disabled" } else if !protectSameName && !parallelWrite { addition["write_conflict_policy"] = "first" addition["put_conflict_policy"] = "first" } else if protectSameName && !parallelWrite { addition["write_conflict_policy"] = "deterministic" addition["put_conflict_policy"] = "deterministic" } else if !protectSameName && parallelWrite { addition["write_conflict_policy"] = "all" addition["put_conflict_policy"] = "all" } else { addition["write_conflict_policy"] = "deterministic_or_all" addition["put_conflict_policy"] = "deterministic_or_all" } addition["read_conflict_policy"] = "first" s.Addition, err = utils.Json.MarshalToString(addition) if err != nil { utils.Log.Errorf("[FixAliasConfig] failed to marshal addition of [%d]%s: %s", s.ID, s.MountPath, err.Error()) continue } err = db.UpdateStorage(&s) if err != nil { utils.Log.Errorf("[FixAliasConfig] failed to update storage [%d]%s: %s", s.ID, s.MountPath, err.Error()) } } } ================================================ FILE: internal/bootstrap/patch/v4_1_9/skip_tls.go ================================================ package v4_1_9 import ( "os" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) func ResetSkipTlsVerify() { if !conf.Conf.TlsInsecureSkipVerify { return } if !strings.HasPrefix(conf.Version, "v") { return } conf.Conf.TlsInsecureSkipVerify = false confBody, err := utils.Json.MarshalIndent(conf.Conf, "", " ") if err != nil { utils.Log.Errorf("[ResetSkipTlsVerify] failed to rewrite config: marshal config error: %+v", err) return } err = os.WriteFile(conf.ConfigPath, confBody, 0o777) if err != nil { utils.Log.Errorf("[ResetSkipTlsVerify] failed to rewrite config: update config struct error: %+v", err) return } utils.Log.Infof("[ResetSkipTlsVerify] succeeded to set tls_insecure_skip_verify to false") } ================================================ FILE: internal/bootstrap/patch/v4_1_9/webdav.go ================================================ package v4_1_9 import ( "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) // EnableWebDavProxy updates Webdav driver storages to enable proxy func EnableWebDavProxy() { storages, _, err := db.GetStorages(1, -1) if err != nil { utils.Log.Errorf("[EnableWebDavProxy] failed to get storages: %s", err.Error()) return } for _, s := range storages { if s.Driver != "WebDav" { continue } if !s.WebProxy { s.WebProxy = true } if s.WebdavPolicy == "302_redirect" { s.WebdavPolicy = "native_proxy" } err = db.UpdateStorage(&s) if err != nil { utils.Log.Errorf("[EnableWebDavProxy] failed to update storage [%d]%s: %s", s.ID, s.MountPath, err.Error()) } } } ================================================ FILE: internal/bootstrap/patch.go ================================================ package bootstrap import ( "fmt" "strings" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) var LastLaunchedVersion = "" func safeCall(v string, i int, f func()) { defer func() { if r := recover(); r != nil { utils.Log.Errorf("Recovered from patch (version: %s, index: %d) panic: %v", v, i, r) } }() f() } func getVersion(v string) (major, minor, patchNum int, err error) { _, err = fmt.Sscanf(v, "v%d.%d.%d", &major, &minor, &patchNum) return major, minor, patchNum, err } func compareVersion(majorA, minorA, patchNumA, majorB, minorB, patchNumB int) bool { if majorA != majorB { return majorA > majorB } if minorA != minorB { return minorA > minorB } if patchNumA != patchNumB { return patchNumA > patchNumB } return true } func InitUpgradePatch() { if !strings.HasPrefix(conf.Version, "v") { for _, vp := range patch.UpgradePatches { for i, p := range vp.Patches { safeCall(vp.Version, i, p) } } return } if LastLaunchedVersion == conf.Version { return } if LastLaunchedVersion == "" { LastLaunchedVersion = "v0.0.0" } major, minor, patchNum, err := getVersion(LastLaunchedVersion) if err != nil { utils.Log.Warnf("Failed to parse last launched version %s: %v, skipping all patches and rewrite last launched version", LastLaunchedVersion, err) return } for _, vp := range patch.UpgradePatches { ma, mi, pn, err := getVersion(vp.Version) if err != nil { utils.Log.Errorf("Skip invalid version %s patches: %v", vp.Version, err) continue } if compareVersion(ma, mi, pn, major, minor, patchNum) { for i, p := range vp.Patches { safeCall(vp.Version, i, p) } } } } ================================================ FILE: internal/bootstrap/run.go ================================================ package bootstrap import ( "context" "fmt" "net" "net/http" "os" "strconv" "sync" "time" "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/data" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server" "github.com/OpenListTeam/OpenList/v4/server/middlewares" "github.com/OpenListTeam/sftpd-openlist" ftpserver "github.com/fclairamb/ftpserverlib" "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/pkg/errors" "github.com/quic-go/quic-go/http3" log "github.com/sirupsen/logrus" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" ) func Init() { InitConfig() Log() InitDB() data.InitData() InitStreamLimit() InitIndex() InitUpgradePatch() } func Release() { db.Close() } var ( running bool httpSrv *http.Server httpRunning bool httpsSrv *http.Server httpsRunning bool unixSrv *http.Server unixRunning bool quicSrv *http3.Server quicRunning bool s3Srv *http.Server s3Running bool ftpDriver *server.FtpMainDriver ftpServer *ftpserver.FtpServer ftpRunning bool sftpDriver *server.SftpDriver sftpServer *sftpd.SftpServer sftpRunning bool ) // Called by OpenList-Mobile func IsRunning(t string) bool { switch t { case "http": return httpRunning case "https": return httpsRunning case "unix": return unixRunning case "quic": return quicRunning case "s3": return s3Running case "sftp": return sftpRunning case "ftp": return ftpRunning } return running } func Start() { if conf.Conf.DelayedStart != 0 { utils.Log.Infof("delayed start for %d seconds", conf.Conf.DelayedStart) time.Sleep(time.Duration(conf.Conf.DelayedStart) * time.Second) } InitOfflineDownloadTools() LoadStorages() InitTaskManager() if !flags.Debug && !flags.Dev { gin.SetMode(gin.ReleaseMode) } r := gin.New() // gin log if conf.Conf.Log.Filter.Enable { r.Use(middlewares.FilteredLogger()) } else { r.Use(gin.LoggerWithWriter(log.StandardLogger().Out)) } r.Use(gin.RecoveryWithWriter(log.StandardLogger().Out)) server.Init(r) var httpHandler http.Handler = r if conf.Conf.Scheme.EnableH2c { httpHandler = h2c.NewHandler(r, &http2.Server{}) } if conf.Conf.Scheme.HttpPort != -1 { httpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpPort) fmt.Printf("start HTTP server @ %s\n", httpBase) utils.Log.Infof("start HTTP server @ %s", httpBase) httpSrv = &http.Server{Addr: httpBase, Handler: httpHandler} go func() { httpRunning = true err := httpSrv.ListenAndServe() httpRunning = false if err != nil && !errors.Is(err, http.ErrServerClosed) { handleEndpointStartFailedHooks("http", err) utils.Log.Errorf("failed to start http: %s", err.Error()) } else { handleEndpointShutdownHooks("http") } }() } if conf.Conf.Scheme.HttpsPort != -1 { httpsBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpsPort) fmt.Printf("start HTTPS server @ %s\n", httpsBase) utils.Log.Infof("start HTTPS server @ %s", httpsBase) httpsSrv = &http.Server{Addr: httpsBase, Handler: r} go func() { httpsRunning = true err := httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) httpsRunning = false if err != nil && !errors.Is(err, http.ErrServerClosed) { handleEndpointStartFailedHooks("https", err) utils.Log.Errorf("failed to start https: %s", err.Error()) } else { handleEndpointShutdownHooks("https") } }() if conf.Conf.Scheme.EnableH3 { fmt.Printf("start HTTP3 (quic) server @ %s\n", httpsBase) utils.Log.Infof("start HTTP3 (quic) server @ %s", httpsBase) r.Use(func(c *gin.Context) { if c.Request.TLS != nil { port := conf.Conf.Scheme.HttpsPort c.Header("Alt-Svc", fmt.Sprintf("h3=\":%d\"; ma=86400", port)) } c.Next() }) quicSrv = &http3.Server{Addr: httpsBase, Handler: r} go func() { quicRunning = true err := quicSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) quicRunning = false if err != nil && !errors.Is(err, http.ErrServerClosed) { handleEndpointStartFailedHooks("quic", err) utils.Log.Errorf("failed to start http3 (quic): %s", err.Error()) } else { handleEndpointShutdownHooks("quic") } }() } } if conf.Conf.Scheme.UnixFile != "" { fmt.Printf("start unix server @ %s\n", conf.Conf.Scheme.UnixFile) utils.Log.Infof("start unix server @ %s", conf.Conf.Scheme.UnixFile) unixSrv = &http.Server{Handler: httpHandler} go func() { listener, err := net.Listen("unix", conf.Conf.Scheme.UnixFile) if err != nil { utils.Log.Errorf("failed to listen unix: %+v", err) return } unixRunning = true // set socket file permission mode, err := strconv.ParseUint(conf.Conf.Scheme.UnixFilePerm, 8, 32) if err != nil { utils.Log.Errorf("failed to parse socket file permission: %+v", err) } else { err = os.Chmod(conf.Conf.Scheme.UnixFile, os.FileMode(mode)) if err != nil { utils.Log.Errorf("failed to chmod socket file: %+v", err) } } err = unixSrv.Serve(listener) unixRunning = false if err != nil && !errors.Is(err, http.ErrServerClosed) { handleEndpointStartFailedHooks("unix", err) utils.Log.Errorf("failed to start unix: %s", err.Error()) } else { handleEndpointShutdownHooks("unix") } }() } if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable { s3r := gin.New() s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out)) server.InitS3(s3r) s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port) fmt.Printf("start S3 server @ %s\n", s3Base) utils.Log.Infof("start S3 server @ %s", s3Base) go func() { s3Running = true var err error if conf.Conf.S3.SSL { s3Srv = &http.Server{Addr: s3Base, Handler: s3r} err = s3Srv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) } else { s3Srv = &http.Server{Addr: s3Base, Handler: s3r} err = s3Srv.ListenAndServe() } s3Running = false if err != nil && !errors.Is(err, http.ErrServerClosed) { handleEndpointStartFailedHooks("s3", err) utils.Log.Errorf("failed to start s3 server: %s", err.Error()) } else { handleEndpointShutdownHooks("s3") } }() } if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable { var err error ftpDriver, err = server.NewMainDriver() if err != nil { utils.Log.Errorf("failed to start ftp driver: %s", err.Error()) } else { fmt.Printf("start ftp server on %s\n", conf.Conf.FTP.Listen) utils.Log.Infof("start ftp server on %s", conf.Conf.FTP.Listen) go func() { ftpServer = ftpserver.NewFtpServer(ftpDriver) ftpRunning = true err = ftpServer.ListenAndServe() ftpRunning = false if err != nil { handleEndpointStartFailedHooks("ftp", err) utils.Log.Errorf("problem ftp server listening: %s", err.Error()) } else { handleEndpointShutdownHooks("ftp") } }() } } if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable { var err error sftpDriver, err = server.NewSftpDriver() if err != nil { utils.Log.Errorf("failed to start sftp driver: %s", err.Error()) } else { fmt.Printf("start sftp server on %s", conf.Conf.SFTP.Listen) utils.Log.Infof("start sftp server on %s", conf.Conf.SFTP.Listen) go func() { sftpServer = sftpd.NewSftpServer(sftpDriver) sftpRunning = true err = sftpServer.RunServer() sftpRunning = false if err != nil { handleEndpointStartFailedHooks("sftp", err) utils.Log.Errorf("problem sftp server listening: %s", err.Error()) } else { handleEndpointShutdownHooks("sftp") } }() } } running = true } func Shutdown(timeout time.Duration) { utils.Log.Println("Shutdown server...") fs.ArchiveContentUploadTaskManager.RemoveAll() ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() var wg sync.WaitGroup if httpSrv != nil && conf.Conf.Scheme.HttpPort != -1 { wg.Add(1) go func() { defer wg.Done() if err := httpSrv.Shutdown(ctx); err != nil { utils.Log.Error("HTTP server shutdown err: ", err) } httpSrv = nil }() } if httpsSrv != nil && conf.Conf.Scheme.HttpsPort != -1 { wg.Add(1) go func() { defer wg.Done() if err := httpsSrv.Shutdown(ctx); err != nil { utils.Log.Error("HTTPS server shutdown err: ", err) } httpsSrv = nil }() if quicSrv != nil && conf.Conf.Scheme.EnableH3 { wg.Add(1) go func() { defer wg.Done() if err := quicSrv.Shutdown(ctx); err != nil { utils.Log.Error("HTTP3 (quic) server shutdown err: ", err) } quicSrv = nil }() } } if unixSrv != nil && conf.Conf.Scheme.UnixFile != "" { wg.Add(1) go func() { defer wg.Done() if err := unixSrv.Shutdown(ctx); err != nil { utils.Log.Error("Unix server shutdown err: ", err) } unixSrv = nil }() } if s3Srv != nil && conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable { wg.Add(1) go func() { defer wg.Done() if err := s3Srv.Shutdown(ctx); err != nil { utils.Log.Error("S3 server shutdown err: ", err) } s3Srv = nil }() } if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable && ftpServer != nil { wg.Add(1) go func() { defer wg.Done() if ftpDriver != nil { ftpDriver.Stop() ftpDriver = nil } if err := ftpServer.Stop(); err != nil { utils.Log.Error("FTP server shutdown err: ", err) } ftpServer = nil }() } if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable && sftpServer != nil { wg.Add(1) go func() { defer wg.Done() if err := sftpServer.Close(); err != nil { utils.Log.Error("SFTP server shutdown err: ", err) } sftpServer = nil sftpDriver = nil }() } wg.Wait() utils.Log.Println("Server exit") running = false } type EndpointStartFailedHook func(string, string) type EndpointShutdownHook func(string) var ( endpointStartFailedHooks map[string]EndpointStartFailedHook endpointShutdownHooks map[string]EndpointShutdownHook ) func RegisterEndpointStartFailedHook(hook EndpointStartFailedHook) string { id := uuid.NewString() endpointStartFailedHooks[id] = hook return id } func RemoveEndpointStartFailedHook(id string) { delete(endpointStartFailedHooks, id) } func RegisterEndpointShutdownHook(hook EndpointShutdownHook) string { id := uuid.NewString() endpointShutdownHooks[id] = hook return id } func RemoveEndpointShutdownHook(id string) { delete(endpointShutdownHooks, id) } func handleEndpointStartFailedHooks(t string, err error) { for _, hook := range endpointStartFailedHooks { hook(t, err.Error()) } } func handleEndpointShutdownHooks(t string) { for _, hook := range endpointShutdownHooks { hook(t) } } func init() { endpointShutdownHooks = make(map[string]EndpointShutdownHook) endpointStartFailedHooks = make(map[string]EndpointStartFailedHook) } ================================================ FILE: internal/bootstrap/storage.go ================================================ package bootstrap import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) func LoadStorages() { storages, err := db.GetEnabledStorages() if err != nil { utils.Log.Fatalf("failed get enabled storages: %+v", err) } go func(storages []model.Storage) { for i := range storages { err := op.LoadStorage(context.Background(), storages[i]) if err != nil { utils.Log.Errorf("failed get enabled storages: %+v", err) } else { utils.Log.Infof("success load storage: [%s], driver: [%s], order: [%d]", storages[i].MountPath, storages[i].Driver, storages[i].Order) } } conf.SendStoragesLoadedSignal() }(storages) } ================================================ FILE: internal/bootstrap/stream_limit.go ================================================ package bootstrap import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "golang.org/x/time/rate" ) type blockBurstLimiter struct { *rate.Limiter } func (l blockBurstLimiter) WaitN(ctx context.Context, total int) error { for total > 0 { n := l.Burst() if l.Limiter.Limit() == rate.Inf || n > total { n = total } err := l.Limiter.WaitN(ctx, n) if err != nil { return err } total -= n } return nil } func streamFilterNegative(limit int) (rate.Limit, int) { if limit < 0 { return rate.Inf, 0 } return rate.Limit(limit) * 1024.0, limit * 1024 } func initLimiter(limiter *stream.Limiter, s string) { clientDownLimit, burst := streamFilterNegative(setting.GetInt(s, -1)) *limiter = blockBurstLimiter{Limiter: rate.NewLimiter(clientDownLimit, burst)} op.RegisterSettingChangingCallback(func() { newLimit, newBurst := streamFilterNegative(setting.GetInt(s, -1)) (*limiter).SetLimit(newLimit) (*limiter).SetBurst(newBurst) }) } func InitStreamLimit() { initLimiter(&stream.ClientDownloadLimit, conf.StreamMaxClientDownloadSpeed) initLimiter(&stream.ClientUploadLimit, conf.StreamMaxClientUploadSpeed) initLimiter(&stream.ServerDownloadLimit, conf.StreamMaxServerDownloadSpeed) initLimiter(&stream.ServerUploadLimit, conf.StreamMaxServerUploadSpeed) } ================================================ FILE: internal/bootstrap/task.go ================================================ package bootstrap import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/tache" ) func taskFilterNegative(num int) int64 { if num < 0 { num = 0 } return int64(num) } func InitTaskManager() { fs.UploadTaskManager = tache.NewManager[*fs.UploadTask](tache.WithWorks(setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers)), tache.WithMaxRetry(conf.Conf.Tasks.Upload.MaxRetry)) //upload will not support persist op.RegisterSettingChangingCallback(func() { fs.UploadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers))) }) fs.CopyTaskManager = tache.NewManager[*fs.FileTransferTask](tache.WithWorks(setting.GetInt(conf.TaskCopyThreadsNum, conf.Conf.Tasks.Copy.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("copy", conf.Conf.Tasks.Copy.TaskPersistant), db.UpdateTaskDataFunc("copy", conf.Conf.Tasks.Copy.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Copy.MaxRetry)) op.RegisterSettingChangingCallback(func() { fs.CopyTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskCopyThreadsNum, conf.Conf.Tasks.Copy.Workers))) }) fs.MoveTaskManager = tache.NewManager[*fs.FileTransferTask](tache.WithWorks(setting.GetInt(conf.TaskMoveThreadsNum, conf.Conf.Tasks.Move.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("move", conf.Conf.Tasks.Move.TaskPersistant), db.UpdateTaskDataFunc("move", conf.Conf.Tasks.Move.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Move.MaxRetry)) op.RegisterSettingChangingCallback(func() { fs.MoveTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskMoveThreadsNum, conf.Conf.Tasks.Move.Workers))) }) tool.DownloadTaskManager = tache.NewManager[*tool.DownloadTask](tache.WithWorks(setting.GetInt(conf.TaskOfflineDownloadThreadsNum, conf.Conf.Tasks.Download.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("download", conf.Conf.Tasks.Download.TaskPersistant), db.UpdateTaskDataFunc("download", conf.Conf.Tasks.Download.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Download.MaxRetry)) op.RegisterSettingChangingCallback(func() { tool.DownloadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskOfflineDownloadThreadsNum, conf.Conf.Tasks.Download.Workers))) }) tool.TransferTaskManager = tache.NewManager[*tool.TransferTask](tache.WithWorks(setting.GetInt(conf.TaskOfflineDownloadTransferThreadsNum, conf.Conf.Tasks.Transfer.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("transfer", conf.Conf.Tasks.Transfer.TaskPersistant), db.UpdateTaskDataFunc("transfer", conf.Conf.Tasks.Transfer.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Transfer.MaxRetry)) op.RegisterSettingChangingCallback(func() { tool.TransferTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskOfflineDownloadTransferThreadsNum, conf.Conf.Tasks.Transfer.Workers))) }) if len(tool.TransferTaskManager.GetAll()) == 0 { //prevent offline downloaded files from being deleted CleanTempDir() } fs.ArchiveDownloadTaskManager = tache.NewManager[*fs.ArchiveDownloadTask](tache.WithWorks(setting.GetInt(conf.TaskDecompressDownloadThreadsNum, conf.Conf.Tasks.Decompress.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant), db.UpdateTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Decompress.MaxRetry)) op.RegisterSettingChangingCallback(func() { fs.ArchiveDownloadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskDecompressDownloadThreadsNum, conf.Conf.Tasks.Decompress.Workers))) }) fs.ArchiveContentUploadTaskManager.Manager = tache.NewManager[*fs.ArchiveContentUploadTask](tache.WithWorks(setting.GetInt(conf.TaskDecompressUploadThreadsNum, conf.Conf.Tasks.DecompressUpload.Workers)), tache.WithMaxRetry(conf.Conf.Tasks.DecompressUpload.MaxRetry)) //decompress upload will not support persist op.RegisterSettingChangingCallback(func() { fs.ArchiveContentUploadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskDecompressUploadThreadsNum, conf.Conf.Tasks.DecompressUpload.Workers))) }) } ================================================ FILE: internal/cache/keyed_cache.go ================================================ package cache import ( "sync" "time" ) type KeyedCache[T any] struct { entries map[string]*CacheEntry[T] mu sync.RWMutex ttl time.Duration } func NewKeyedCache[T any](ttl time.Duration) *KeyedCache[T] { c := &KeyedCache[T]{ entries: make(map[string]*CacheEntry[T]), ttl: ttl, } gcFuncs = append(gcFuncs, c.GC) return c } func (c *KeyedCache[T]) Set(key string, value T) { c.SetWithExpirable(key, value, ExpirationTime(time.Now().Add(c.ttl))) } func (c *KeyedCache[T]) SetWithTTL(key string, value T, ttl time.Duration) { c.SetWithExpirable(key, value, ExpirationTime(time.Now().Add(ttl))) } func (c *KeyedCache[T]) SetWithExpirable(key string, value T, exp Expirable) { c.mu.Lock() defer c.mu.Unlock() c.entries[key] = &CacheEntry[T]{ data: value, Expirable: exp, } } func (c *KeyedCache[T]) Get(key string) (T, bool) { c.mu.RLock() entry, exists := c.entries[key] if !exists { c.mu.RUnlock() return *new(T), false } expired := entry.Expired() c.mu.RUnlock() if !expired { return entry.data, true } c.mu.Lock() if c.entries[key] == entry { delete(c.entries, key) c.mu.Unlock() return *new(T), false } c.mu.Unlock() return *new(T), false } func (c *KeyedCache[T]) Delete(key string) { c.mu.Lock() defer c.mu.Unlock() delete(c.entries, key) } func (c *KeyedCache[T]) Pop(key string) (T, bool) { c.mu.Lock() defer c.mu.Unlock() if entry, exists := c.entries[key]; exists { delete(c.entries, key) return entry.data, true } return *new(T), false } func (c *KeyedCache[T]) Clear() { c.mu.Lock() defer c.mu.Unlock() c.entries = make(map[string]*CacheEntry[T]) } func (c *KeyedCache[T]) GC() { c.mu.Lock() defer c.mu.Unlock() expiredKeys := make([]string, 0, len(c.entries)) for key, entry := range c.entries { if entry.Expired() { expiredKeys = append(expiredKeys, key) } } for _, key := range expiredKeys { delete(c.entries, key) } } ================================================ FILE: internal/cache/type.go ================================================ package cache import "time" type Expirable interface { Expired() bool } type ExpirationTime time.Time func (e ExpirationTime) Expired() bool { return time.Now().After(time.Time(e)) } type CacheEntry[T any] struct { Expirable data T } ================================================ FILE: internal/cache/typed_cache.go ================================================ package cache import ( "sync" "time" ) type TypedCache[T any] struct { entries map[string]map[string]*CacheEntry[T] mu sync.RWMutex ttl time.Duration } func NewTypedCache[T any](ttl time.Duration) *TypedCache[T] { c := &TypedCache[T]{ entries: make(map[string]map[string]*CacheEntry[T]), ttl: ttl, } gcFuncs = append(gcFuncs, c.GC) return c } func (c *TypedCache[T]) SetType(key, typeKey string, value T) { c.SetTypeWithExpirable(key, typeKey, value, ExpirationTime(time.Now().Add(c.ttl))) } func (c *TypedCache[T]) SetTypeWithTTL(key, typeKey string, value T, ttl time.Duration) { c.SetTypeWithExpirable(key, typeKey, value, ExpirationTime(time.Now().Add(ttl))) } func (c *TypedCache[T]) SetTypeWithExpirable(key, typeKey string, value T, exp Expirable) { c.mu.Lock() defer c.mu.Unlock() cache, exists := c.entries[key] if !exists { cache = make(map[string]*CacheEntry[T]) c.entries[key] = cache } cache[typeKey] = &CacheEntry[T]{ data: value, Expirable: exp, } } func (c *TypedCache[T]) GetType(key, typeKey string) (T, bool) { c.mu.RLock() cache, exists := c.entries[key] if !exists { c.mu.RUnlock() return *new(T), false } entry, exists := cache[typeKey] if !exists { c.mu.RUnlock() return *new(T), false } expired := entry.Expired() c.mu.RUnlock() if !expired { return entry.data, true } c.mu.Lock() if cache[typeKey] == entry { delete(cache, typeKey) if len(cache) == 0 { delete(c.entries, key) } c.mu.Unlock() return *new(T), false } c.mu.Unlock() return *new(T), false } func (c *TypedCache[T]) DeleteKey(key string) { c.mu.Lock() defer c.mu.Unlock() delete(c.entries, key) } func (c *TypedCache[T]) Clear() { c.mu.Lock() defer c.mu.Unlock() c.entries = make(map[string]map[string]*CacheEntry[T]) } func (c *TypedCache[T]) GC() { c.mu.Lock() defer c.mu.Unlock() expiredKeys := make(map[string][]string) for tk, entries := range c.entries { for key, entry := range entries { if !entry.Expired() { continue } if _, ok := expiredKeys[tk]; !ok { expiredKeys[tk] = make([]string, 0, len(entries)) } expiredKeys[tk] = append(expiredKeys[tk], key) } } for tk, keys := range expiredKeys { for _, key := range keys { delete(c.entries[tk], key) } if len(c.entries[tk]) == 0 { delete(c.entries, tk) } } } ================================================ FILE: internal/cache/utils.go ================================================ package cache import ( "time" "github.com/OpenListTeam/OpenList/v4/pkg/cron" log "github.com/sirupsen/logrus" ) var ( cacheGcCron *cron.Cron gcFuncs []func() ) func init() { // TODO Move to bootstrap cacheGcCron = cron.NewCron(time.Hour) cacheGcCron.Do(func() { log.Infof("Start cache GC") for _, f := range gcFuncs { f() } }) } ================================================ FILE: internal/conf/config.go ================================================ package conf import ( "path/filepath" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" ) type Database struct { Type string `json:"type" env:"TYPE"` Host string `json:"host" env:"HOST"` Port int `json:"port" env:"PORT"` User string `json:"user" env:"USER"` Password string `json:"password" env:"PASS"` Name string `json:"name" env:"NAME"` DBFile string `json:"db_file" env:"FILE"` TablePrefix string `json:"table_prefix" env:"TABLE_PREFIX"` SSLMode string `json:"ssl_mode" env:"SSL_MODE"` DSN string `json:"dsn" env:"DSN"` } type Meilisearch struct { Host string `json:"host" env:"HOST"` APIKey string `json:"api_key" env:"API_KEY"` Index string `json:"index" env:"INDEX"` } type Scheme struct { Address string `json:"address" env:"ADDR"` HttpPort int `json:"http_port" env:"HTTP_PORT"` HttpsPort int `json:"https_port" env:"HTTPS_PORT"` ForceHttps bool `json:"force_https" env:"FORCE_HTTPS"` CertFile string `json:"cert_file" env:"CERT_FILE"` KeyFile string `json:"key_file" env:"KEY_FILE"` UnixFile string `json:"unix_file" env:"UNIX_FILE"` UnixFilePerm string `json:"unix_file_perm" env:"UNIX_FILE_PERM"` EnableH2c bool `json:"enable_h2c" env:"ENABLE_H2C"` EnableH3 bool `json:"enable_h3" env:"ENABLE_H3"` } type LogConfig struct { Enable bool `json:"enable" env:"ENABLE"` Name string `json:"name" env:"NAME"` MaxSize int `json:"max_size" env:"MAX_SIZE"` MaxBackups int `json:"max_backups" env:"MAX_BACKUPS"` MaxAge int `json:"max_age" env:"MAX_AGE"` Compress bool `json:"compress" env:"COMPRESS"` Filter LogFilterConfig `json:"filter" envPrefix:"FILTER_"` } type LogFilterConfig struct { Enable bool `json:"enable" env:"ENABLE"` Filters []Filter `json:"filters"` } type Filter struct { CIDR string `json:"cidr"` Path string `json:"path"` Method string `json:"method"` } type TaskConfig struct { Workers int `json:"workers" env:"WORKERS"` MaxRetry int `json:"max_retry" env:"MAX_RETRY"` TaskPersistant bool `json:"task_persistant" env:"TASK_PERSISTANT"` } type TasksConfig struct { Download TaskConfig `json:"download" envPrefix:"DOWNLOAD_"` Transfer TaskConfig `json:"transfer" envPrefix:"TRANSFER_"` Upload TaskConfig `json:"upload" envPrefix:"UPLOAD_"` Copy TaskConfig `json:"copy" envPrefix:"COPY_"` Move TaskConfig `json:"move" envPrefix:"MOVE_"` Decompress TaskConfig `json:"decompress" envPrefix:"DECOMPRESS_"` DecompressUpload TaskConfig `json:"decompress_upload" envPrefix:"DECOMPRESS_UPLOAD_"` AllowRetryCanceled bool `json:"allow_retry_canceled" env:"ALLOW_RETRY_CANCELED"` } type Cors struct { AllowOrigins []string `json:"allow_origins" env:"ALLOW_ORIGINS"` AllowMethods []string `json:"allow_methods" env:"ALLOW_METHODS"` AllowHeaders []string `json:"allow_headers" env:"ALLOW_HEADERS"` } type S3 struct { Enable bool `json:"enable" env:"ENABLE"` Port int `json:"port" env:"PORT"` SSL bool `json:"ssl" env:"SSL"` } type FTP struct { Enable bool `json:"enable" env:"ENABLE"` Listen string `json:"listen" env:"LISTEN"` FindPasvPortAttempts int `json:"find_pasv_port_attempts" env:"FIND_PASV_PORT_ATTEMPTS"` ActiveTransferPortNon20 bool `json:"active_transfer_port_non_20" env:"ACTIVE_TRANSFER_PORT_NON_20"` IdleTimeout int `json:"idle_timeout" env:"IDLE_TIMEOUT"` ConnectionTimeout int `json:"connection_timeout" env:"CONNECTION_TIMEOUT"` DisableActiveMode bool `json:"disable_active_mode" env:"DISABLE_ACTIVE_MODE"` DefaultTransferBinary bool `json:"default_transfer_binary" env:"DEFAULT_TRANSFER_BINARY"` EnableActiveConnIPCheck bool `json:"enable_active_conn_ip_check" env:"ENABLE_ACTIVE_CONN_IP_CHECK"` EnablePasvConnIPCheck bool `json:"enable_pasv_conn_ip_check" env:"ENABLE_PASV_CONN_IP_CHECK"` } type SFTP struct { Enable bool `json:"enable" env:"ENABLE"` Listen string `json:"listen" env:"LISTEN"` } type Config struct { Force bool `json:"force" env:"FORCE"` SiteURL string `json:"site_url" env:"SITE_URL"` Cdn string `json:"cdn" env:"CDN"` JwtSecret string `json:"jwt_secret" env:"JWT_SECRET"` TokenExpiresIn int `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"` Database Database `json:"database" envPrefix:"DB_"` Meilisearch Meilisearch `json:"meilisearch" envPrefix:"MEILISEARCH_"` Scheme Scheme `json:"scheme"` TempDir string `json:"temp_dir" env:"TEMP_DIR"` BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` DistDir string `json:"dist_dir"` Log LogConfig `json:"log" envPrefix:"LOG_"` DelayedStart int `json:"delayed_start" env:"DELAYED_START"` MaxBufferLimit int `json:"max_buffer_limitMB" env:"MAX_BUFFER_LIMIT_MB"` MmapThreshold int `json:"mmap_thresholdMB" env:"MMAP_THRESHOLD_MB"` MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"` MaxConcurrency int `json:"max_concurrency" env:"MAX_CONCURRENCY"` TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"` Tasks TasksConfig `json:"tasks" envPrefix:"TASKS_"` Cors Cors `json:"cors" envPrefix:"CORS_"` S3 S3 `json:"s3" envPrefix:"S3_"` FTP FTP `json:"ftp" envPrefix:"FTP_"` SFTP SFTP `json:"sftp" envPrefix:"SFTP_"` LastLaunchedVersion string `json:"last_launched_version"` ProxyAddress string `json:"proxy_address" env:"PROXY_ADDRESS"` } func DefaultConfig(dataDir string) *Config { tempDir := filepath.Join(dataDir, "temp") indexDir := filepath.Join(dataDir, "bleve") logPath := filepath.Join(dataDir, "log/log.log") dbPath := filepath.Join(dataDir, "data.db") return &Config{ Scheme: Scheme{ Address: "0.0.0.0", UnixFile: "", HttpPort: 5244, HttpsPort: -1, ForceHttps: false, CertFile: "", KeyFile: "", }, JwtSecret: random.String(16), TokenExpiresIn: 48, TempDir: tempDir, Database: Database{ Type: "sqlite3", Port: 0, TablePrefix: "x_", DBFile: dbPath, }, Meilisearch: Meilisearch{ Host: "http://localhost:7700", Index: "openlist", }, BleveDir: indexDir, Log: LogConfig{ Enable: true, Name: logPath, MaxSize: 50, MaxBackups: 30, MaxAge: 28, Filter: LogFilterConfig{ Enable: false, Filters: []Filter{ {Path: "/ping"}, {Method: "HEAD"}, {Path: "/dav/", Method: "PROPFIND"}, }, }, }, MaxBufferLimit: -1, MmapThreshold: 4, MaxConnections: 0, MaxConcurrency: 64, TlsInsecureSkipVerify: false, Tasks: TasksConfig{ Download: TaskConfig{ Workers: 5, MaxRetry: 1, // TaskPersistant: true, }, Transfer: TaskConfig{ Workers: 5, MaxRetry: 2, // TaskPersistant: true, }, Upload: TaskConfig{ Workers: 5, }, Copy: TaskConfig{ Workers: 5, MaxRetry: 2, // TaskPersistant: true, }, Move: TaskConfig{ Workers: 5, MaxRetry: 2, // TaskPersistant: true, }, Decompress: TaskConfig{ Workers: 5, MaxRetry: 2, // TaskPersistant: true, }, DecompressUpload: TaskConfig{ Workers: 5, MaxRetry: 2, }, AllowRetryCanceled: false, }, Cors: Cors{ AllowOrigins: []string{"*"}, AllowMethods: []string{"*"}, AllowHeaders: []string{"*"}, }, S3: S3{ Enable: false, Port: 5246, SSL: false, }, FTP: FTP{ Enable: false, Listen: ":5221", FindPasvPortAttempts: 50, ActiveTransferPortNon20: false, IdleTimeout: 900, ConnectionTimeout: 30, DisableActiveMode: false, DefaultTransferBinary: false, EnableActiveConnIPCheck: true, EnablePasvConnIPCheck: true, }, SFTP: SFTP{ Enable: false, Listen: ":5222", }, LastLaunchedVersion: "", ProxyAddress: "", } } ================================================ FILE: internal/conf/const.go ================================================ package conf const ( TypeString = "string" TypeSelect = "select" TypeBool = "bool" TypeText = "text" TypeNumber = "number" ) const ( // site VERSION = "version" SiteTitle = "site_title" Announcement = "announcement" AllowIndexed = "allow_indexed" AllowMounted = "allow_mounted" RobotsTxt = "robots_txt" Logo = "logo" // multi-lines text, L1: light, EOL: dark Favicon = "favicon" MainColor = "main_color" HideStorageDetails = "hide_storage_details" HideStorageDetailsInManagePage = "hide_storage_details_in_manage_page" // preview TextTypes = "text_types" AudioTypes = "audio_types" VideoTypes = "video_types" ImageTypes = "image_types" ProxyTypes = "proxy_types" ProxyIgnoreHeaders = "proxy_ignore_headers" AudioAutoplay = "audio_autoplay" VideoAutoplay = "video_autoplay" PreviewDownloadByDefault = "preview_download_by_default" PreviewArchivesByDefault = "preview_archives_by_default" SharePreviewDownloadByDefault = "share_preview_download_by_default" SharePreviewArchivesByDefault = "share_preview_archives_by_default" ReadMeAutoRender = "readme_autorender" FilterReadMeScripts = "filter_readme_scripts" NonEFSZipEncoding = "non_efs_zip_encoding" // global HideFiles = "hide_files" CustomizeHead = "customize_head" CustomizeBody = "customize_body" LinkExpiration = "link_expiration" SignAll = "sign_all" PrivacyRegs = "privacy_regs" OcrApi = "ocr_api" FilenameCharMapping = "filename_char_mapping" ForwardDirectLinkParams = "forward_direct_link_params" IgnoreDirectLinkParams = "ignore_direct_link_params" WebauthnLoginEnabled = "webauthn_login_enabled" SharePreview = "share_preview" ShareArchivePreview = "share_archive_preview" ShareForceProxy = "share_force_proxy" ShareSummaryContent = "share_summary_content" HandleHookAfterWriting = "handle_hook_after_writing" HandleHookRateLimit = "handle_hook_rate_limit" IgnoreSystemFiles = "ignore_system_files" // index SearchIndex = "search_index" AutoUpdateIndex = "auto_update_index" IgnorePaths = "ignore_paths" MaxIndexDepth = "max_index_depth" // aria2 Aria2Uri = "aria2_uri" Aria2Secret = "aria2_secret" // transmission TransmissionUri = "transmission_uri" TransmissionSeedtime = "transmission_seedtime" // 115 Pan115TempDir = "115_temp_dir" // 123 Pan123TempDir = "123_temp_dir" // 115_open Pan115OpenTempDir = "115_open_temp_dir" // pikpak PikPakTempDir = "pikpak_temp_dir" // thunder ThunderTempDir = "thunder_temp_dir" // thunderx ThunderXTempDir = "thunderx_temp_dir" // thunder_browser ThunderBrowserTempDir = "thunder_browser_temp_dir" // single Token = "token" IndexProgress = "index_progress" // SSO SSOClientId = "sso_client_id" SSOClientSecret = "sso_client_secret" SSOLoginEnabled = "sso_login_enabled" SSOLoginPlatform = "sso_login_platform" SSOOIDCUsernameKey = "sso_oidc_username_key" SSOOrganizationName = "sso_organization_name" SSOApplicationName = "sso_application_name" SSOEndpointName = "sso_endpoint_name" SSOJwtPublicKey = "sso_jwt_public_key" SSOExtraScopes = "sso_extra_scopes" SSOAutoRegister = "sso_auto_register" SSODefaultDir = "sso_default_dir" SSODefaultPermission = "sso_default_permission" SSOCompatibilityMode = "sso_compatibility_mode" // ldap LdapLoginEnabled = "ldap_login_enabled" LdapServer = "ldap_server" LdapSkipTlsVerify = "ldap_skip_tls_verify" LdapManagerDN = "ldap_manager_dn" LdapManagerPassword = "ldap_manager_password" LdapUserSearchBase = "ldap_user_search_base" LdapUserSearchFilter = "ldap_user_search_filter" LdapDefaultPermission = "ldap_default_permission" LdapDefaultDir = "ldap_default_dir" LdapLoginTips = "ldap_login_tips" // s3 S3Buckets = "s3_buckets" S3AccessKeyId = "s3_access_key_id" S3SecretAccessKey = "s3_secret_access_key" // qbittorrent QbittorrentUrl = "qbittorrent_url" QbittorrentSeedtime = "qbittorrent_seedtime" // 123 open offline download Pan123OpenOfflineDownloadCallbackUrl = "123_open_callback_url" Pan123OpenTempDir = "123_open_temp_dir" // ftp FTPPublicHost = "ftp_public_host" FTPPasvPortMap = "ftp_pasv_port_map" FTPMandatoryTLS = "ftp_mandatory_tls" FTPImplicitTLS = "ftp_implicit_tls" FTPTLSPrivateKeyPath = "ftp_tls_private_key_path" FTPTLSPublicCertPath = "ftp_tls_public_cert_path" SFTPDisablePasswordLogin = "sftp_disable_password_login" // traffic TaskOfflineDownloadThreadsNum = "offline_download_task_threads_num" TaskOfflineDownloadTransferThreadsNum = "offline_download_transfer_task_threads_num" TaskUploadThreadsNum = "upload_task_threads_num" TaskCopyThreadsNum = "copy_task_threads_num" TaskMoveThreadsNum = "move_task_threads_num" TaskDecompressDownloadThreadsNum = "decompress_download_task_threads_num" TaskDecompressUploadThreadsNum = "decompress_upload_task_threads_num" StreamMaxClientDownloadSpeed = "max_client_download_speed" StreamMaxClientUploadSpeed = "max_client_upload_speed" StreamMaxServerDownloadSpeed = "max_server_download_speed" StreamMaxServerUploadSpeed = "max_server_upload_speed" ) const ( UNKNOWN = iota FOLDER // OFFICE VIDEO AUDIO TEXT IMAGE ) // ContextKey is the type of context keys. type ContextKey int8 const ( _ ContextKey = iota NoTaskKey ApiUrlKey UserKey MetaKey MetaPassKey ClientIPKey ProxyHeaderKey RequestHeaderKey UserAgentKey PathKey SharingIDKey SkipHookKey ) ================================================ FILE: internal/conf/var.go ================================================ package conf import ( "net/url" "regexp" "sync" ) var ( BuiltAt string = "unknown" GitAuthor string = "unknown" GitCommit string = "unknown" Version string = "dev" WebVersion string = "rolling" ) var ( Conf *Config URL *url.URL ConfigPath string ) var SlicesMap = make(map[string][]string) var FilenameCharMap = make(map[string]string) var PrivacyReg []*regexp.Regexp var ( // 单个Buffer最大限制 MaxBufferLimit = 16 * 1024 * 1024 // 超过该阈值的Buffer将使用 mmap 分配，可主动释放内存 MmapThreshold = 4 * 1024 * 1024 ) var ( RawIndexHtml string ManageHtml string IndexHtml string ) var ( // StoragesLoaded loaded success if empty StoragesLoaded = false storagesLoadMu sync.RWMutex storagesLoadSignal chan struct{} = make(chan struct{}) ) func StoragesLoadSignal() <-chan struct{} { storagesLoadMu.RLock() ch := storagesLoadSignal storagesLoadMu.RUnlock() return ch } func SendStoragesLoadedSignal() { storagesLoadMu.Lock() select { case <-storagesLoadSignal: // already closed default: StoragesLoaded = true close(storagesLoadSignal) } storagesLoadMu.Unlock() } func ResetStoragesLoadSignal() { storagesLoadMu.Lock() select { case <-storagesLoadSignal: StoragesLoaded = false storagesLoadSignal = make(chan struct{}) default: // not closed -> nothing to do } storagesLoadMu.Unlock() } ================================================ FILE: internal/db/db.go ================================================ package db import ( log "github.com/sirupsen/logrus" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "gorm.io/gorm" ) var db *gorm.DB func Init(d *gorm.DB) { db = d err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.SharingDB)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } } func AutoMigrate(dst ...interface{}) error { var err error if conf.Conf.Database.Type == "mysql" { err = db.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(dst...) } else { err = db.AutoMigrate(dst...) } return err } func GetDb() *gorm.DB { return db } func Close() { log.Info("closing db") sqlDB, err := db.DB() if err != nil { log.Errorf("failed to get db: %s", err.Error()) return } err = sqlDB.Close() if err != nil { log.Errorf("failed to close db: %s", err.Error()) return } } ================================================ FILE: internal/db/meta.go ================================================ package db import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/pkg/errors" ) func GetMetaByPath(path string) (*model.Meta, error) { meta := model.Meta{Path: path} if err := db.Where(meta).First(&meta).Error; err != nil { return nil, errors.Wrapf(err, "failed select meta") } return &meta, nil } func GetMetaById(id uint) (*model.Meta, error) { var u model.Meta if err := db.First(&u, id).Error; err != nil { return nil, errors.Wrapf(err, "failed get old meta") } return &u, nil } func CreateMeta(u *model.Meta) error { return errors.WithStack(db.Create(u).Error) } func UpdateMeta(u *model.Meta) error { return errors.WithStack(db.Save(u).Error) } func GetMetas(pageIndex, pageSize int) (metas []model.Meta, count int64, err error) { metaDB := db.Model(&model.Meta{}) if err = metaDB.Count(&count).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get metas count") } if err = metaDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&metas).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get find metas") } return metas, count, nil } func DeleteMetaById(id uint) error { return errors.WithStack(db.Delete(&model.Meta{}, id).Error) } ================================================ FILE: internal/db/searchnode.go ================================================ package db import ( "fmt" stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" "gorm.io/gorm" ) func whereInParent(parent string) *gorm.DB { if parent == "/" { return db.Where("1 = 1") } return db.Where(fmt.Sprintf("%s LIKE ?", columnName("parent")), fmt.Sprintf("%s/%%", parent)). Or(fmt.Sprintf("%s = ?", columnName("parent")), parent) } func CreateSearchNode(node *model.SearchNode) error { return db.Create(node).Error } func BatchCreateSearchNodes(nodes *[]model.SearchNode) error { return db.CreateInBatches(nodes, 1000).Error } func DeleteSearchNodesByParent(path string) error { path = utils.FixAndCleanPath(path) err := db.Where(whereInParent(path)).Delete(&model.SearchNode{}).Error if err != nil { return err } dir, name := stdpath.Split(path) return db.Where(fmt.Sprintf("%s = ? AND %s = ?", columnName("parent"), columnName("name")), dir, name).Delete(&model.SearchNode{}).Error } func ClearSearchNodes() error { return db.Where("1 = 1").Delete(&model.SearchNode{}).Error } func GetSearchNodesByParent(parent string) ([]model.SearchNode, error) { var nodes []model.SearchNode if err := db.Where(fmt.Sprintf("%s = ?", columnName("parent")), parent).Find(&nodes).Error; err != nil { return nil, err } return nodes, nil } func SearchNode(req model.SearchReq, useFullText bool) ([]model.SearchNode, int64, error) { var searchDB *gorm.DB if !useFullText || conf.Conf.Database.Type == "sqlite3" { keywordsClause := db.Where("1 = 1") for _, keyword := range strings.Fields(req.Keywords) { keywordsClause = keywordsClause.Where("name LIKE ?", fmt.Sprintf("%%%s%%", keyword)) } searchDB = db.Model(&model.SearchNode{}).Where(whereInParent(req.Parent)).Where(keywordsClause) } else { switch conf.Conf.Database.Type { case "mysql": searchDB = db.Model(&model.SearchNode{}).Where(whereInParent(req.Parent)). Where("MATCH (name) AGAINST (? IN BOOLEAN MODE)", "'*"+req.Keywords+"*'") case "postgres": searchDB = db.Model(&model.SearchNode{}).Where(whereInParent(req.Parent)). Where("to_tsvector(name) @@ to_tsquery(?)", strings.Join(strings.Fields(req.Keywords), " & ")) } } if req.Scope != 0 { isDir := req.Scope == 1 searchDB.Where(db.Where("is_dir = ?", isDir)) } var count int64 if err := searchDB.Count(&count).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get search items count") } var files []model.SearchNode if err := searchDB.Order("name asc").Offset((req.Page - 1) * req.PerPage).Limit(req.PerPage). Find(&files).Error; err != nil { return nil, 0, err } return files, count, nil } ================================================ FILE: internal/db/settingitem.go ================================================ package db import ( "fmt" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/pkg/errors" ) func GetSettingItems() ([]model.SettingItem, error) { var settingItems []model.SettingItem if err := db.Find(&settingItems).Error; err != nil { return nil, errors.WithStack(err) } return settingItems, nil } func GetSettingItemByKey(key string) (*model.SettingItem, error) { var settingItem model.SettingItem if err := db.Where(fmt.Sprintf("%s = ?", columnName("key")), key).First(&settingItem).Error; err != nil { return nil, errors.WithStack(err) } return &settingItem, nil } // func GetSettingItemInKeys(keys []string) ([]model.SettingItem, error) { // var settingItem []model.SettingItem // if err := db.Where(fmt.Sprintf("%s in ?", columnName("key")), keys).Find(&settingItem).Error; err != nil { // return nil, errors.WithStack(err) // } // return settingItem, nil // } func GetPublicSettingItems() ([]model.SettingItem, error) { var settingItems []model.SettingItem if err := db.Where(fmt.Sprintf("%s in ?", columnName("flag")), []int{model.PUBLIC, model.READONLY}).Find(&settingItems).Error; err != nil { return nil, errors.WithStack(err) } return settingItems, nil } func GetSettingItemsByGroup(group int) ([]model.SettingItem, error) { var settingItems []model.SettingItem if err := db.Where(fmt.Sprintf("%s = ?", columnName("group")), group).Find(&settingItems).Error; err != nil { return nil, errors.WithStack(err) } return settingItems, nil } func GetSettingItemsInGroups(groups []int) ([]model.SettingItem, error) { var settingItems []model.SettingItem err := db.Order(columnName("index")).Where(fmt.Sprintf("%s in ?", columnName("group")), groups).Find(&settingItems).Error if err != nil { return nil, errors.WithStack(err) } return settingItems, nil } func SaveSettingItems(items []model.SettingItem) (err error) { return errors.WithStack(db.Save(items).Error) } func SaveSettingItem(item *model.SettingItem) error { return errors.WithStack(db.Save(item).Error) } func DeleteSettingItemByKey(key string) error { return errors.WithStack(db.Delete(&model.SettingItem{Key: key}).Error) } ================================================ FILE: internal/db/sharing.go ================================================ package db import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/pkg/errors" ) func GetSharingById(id string) (*model.SharingDB, error) { s := model.SharingDB{ID: id} if err := db.Where(s).First(&s).Error; err != nil { return nil, errors.Wrapf(err, "failed get sharing") } return &s, nil } func GetSharings(pageIndex, pageSize int) (sharings []model.SharingDB, count int64, err error) { sharingDB := db.Model(&model.SharingDB{}) if err := sharingDB.Count(&count).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get sharings count") } if err := sharingDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&sharings).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get find sharings") } return sharings, count, nil } func GetSharingsByCreatorId(creator uint, pageIndex, pageSize int) (sharings []model.SharingDB, count int64, err error) { sharingDB := db.Model(&model.SharingDB{}) cond := model.SharingDB{CreatorId: creator} if err := sharingDB.Where(cond).Count(&count).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get sharings count") } if err := sharingDB.Where(cond).Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&sharings).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get find sharings") } return sharings, count, nil } func CreateSharing(s *model.SharingDB) (string, error) { if s.ID == "" { id := random.String(8) for len(id) < 12 { old := model.SharingDB{ ID: id, } if err := db.Where(old).First(&old).Error; err != nil { s.ID = id return id, errors.WithStack(db.Create(s).Error) } id += random.String(1) } return "", errors.New("failed find valid id") } else { query := model.SharingDB{ID: s.ID} if err := db.Where(query).First(&query).Error; err == nil { return "", errors.New("sharing already exist") } return s.ID, errors.WithStack(db.Create(s).Error) } } func UpdateSharing(s *model.SharingDB) error { return errors.WithStack(db.Save(s).Error) } func DeleteSharingById(id string) error { s := model.SharingDB{ID: id} return errors.WithStack(db.Where(s).Delete(&s).Error) } func DeleteSharingsByCreatorId(creatorId uint) error { return errors.WithStack(db.Where("creator_id = ?", creatorId).Delete(&model.SharingDB{}).Error) } ================================================ FILE: internal/db/sshkey.go ================================================ package db import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/pkg/errors" ) func GetSSHPublicKeyByUserId(userId uint, pageIndex, pageSize int) (keys []model.SSHPublicKey, count int64, err error) { keyDB := db.Model(&model.SSHPublicKey{}) query := model.SSHPublicKey{UserId: userId} if err := keyDB.Where(query).Count(&count).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get user's keys count") } if err := keyDB.Where(query).Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&keys).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get find user's keys") } return keys, count, nil } func GetSSHPublicKeyById(id uint) (*model.SSHPublicKey, error) { var k model.SSHPublicKey if err := db.First(&k, id).Error; err != nil { return nil, errors.Wrapf(err, "failed get old key") } return &k, nil } func GetSSHPublicKeyByUserTitle(userId uint, title string) (*model.SSHPublicKey, error) { key := model.SSHPublicKey{UserId: userId, Title: title} if err := db.Where(key).First(&key).Error; err != nil { return nil, errors.Wrapf(err, "failed find key with title of user") } return &key, nil } func CreateSSHPublicKey(k *model.SSHPublicKey) error { return errors.WithStack(db.Create(k).Error) } func UpdateSSHPublicKey(k *model.SSHPublicKey) error { return errors.WithStack(db.Save(k).Error) } func GetSSHPublicKeys(pageIndex, pageSize int) (keys []model.SSHPublicKey, count int64, err error) { keyDB := db.Model(&model.SSHPublicKey{}) if err := keyDB.Count(&count).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get keys count") } if err := keyDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&keys).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get find keys") } return keys, count, nil } func DeleteSSHPublicKeyById(id uint) error { return errors.WithStack(db.Delete(&model.SSHPublicKey{}, id).Error) } ================================================ FILE: internal/db/storage.go ================================================ package db import ( "fmt" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/pkg/errors" ) // why don't need `cache` for storage? // because all storage store in `op.storagesMap` // the most of the read operation is from `op.storagesMap` // just for persistence in database // CreateStorage just insert storage to database func CreateStorage(storage *model.Storage) error { return errors.WithStack(db.Create(storage).Error) } // UpdateStorage just update storage in database func UpdateStorage(storage *model.Storage) error { return errors.WithStack(db.Save(storage).Error) } // DeleteStorageById just delete storage from database by id func DeleteStorageById(id uint) error { return errors.WithStack(db.Delete(&model.Storage{}, id).Error) } // GetStorages Get all storages from database order by index func GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) { storageDB := db.Model(&model.Storage{}) var count int64 if err := storageDB.Count(&count).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get storages count") } var storages []model.Storage if err := addStorageOrder(storageDB).Order(columnName("order")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil { return nil, 0, errors.WithStack(err) } return storages, count, nil } // GetStorageById Get Storage by id, used to update storage usually func GetStorageById(id uint) (*model.Storage, error) { var storage model.Storage storage.ID = id if err := db.First(&storage).Error; err != nil { return nil, errors.WithStack(err) } return &storage, nil } // GetStorageByMountPath Get Storage by mountPath, used to update storage usually func GetStorageByMountPath(mountPath string) (*model.Storage, error) { var storage model.Storage if err := db.Where("mount_path = ?", mountPath).First(&storage).Error; err != nil { return nil, errors.WithStack(err) } return &storage, nil } func GetEnabledStorages() ([]model.Storage, error) { var storages []model.Storage err := addStorageOrder(db).Where(fmt.Sprintf("%s = ?", columnName("disabled")), false).Find(&storages).Error if err != nil { return nil, errors.WithStack(err) } return storages, nil } ================================================ FILE: internal/db/tasks.go ================================================ package db import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/pkg/errors" ) func GetTaskDataByType(type_s string) (*model.TaskItem, error) { task := model.TaskItem{Key: type_s} if err := db.Where(task).First(&task).Error; err != nil { return nil, errors.Wrapf(err, "failed find task") } return &task, nil } func UpdateTaskData(t *model.TaskItem) error { return errors.WithStack(db.Model(&model.TaskItem{}).Where("key = ?", t.Key).Update("persist_data", t.PersistData).Error) } func CreateTaskData(t *model.TaskItem) error { return errors.WithStack(db.Create(t).Error) } func GetTaskDataFunc(type_s string, enabled bool) func() ([]byte, error) { if !enabled { return nil } task, err := GetTaskDataByType(type_s) if err != nil { return nil } return func() ([]byte, error) { <-conf.StoragesLoadSignal() return []byte(task.PersistData), nil } } func UpdateTaskDataFunc(type_s string, enabled bool) func([]byte) error { if !enabled { return nil } return func(data []byte) error { s := string(data) if s == "null" || s == "" { s = "[]" } return UpdateTaskData(&model.TaskItem{Key: type_s, PersistData: s}) } } ================================================ FILE: internal/db/user.go ================================================ package db import ( "encoding/base64" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-webauthn/webauthn/webauthn" "github.com/pkg/errors" ) func GetUserByRole(role int) (*model.User, error) { user := model.User{Role: role} if err := db.Where(user).Take(&user).Error; err != nil { return nil, err } return &user, nil } func GetUserByName(username string) (*model.User, error) { user := model.User{Username: username} if err := db.Where(user).First(&user).Error; err != nil { return nil, errors.Wrapf(err, "failed find user") } return &user, nil } func GetUserBySSOID(ssoID string) (*model.User, error) { user := model.User{SsoID: ssoID} if err := db.Where(user).First(&user).Error; err != nil { return nil, errors.Wrapf(err, "The single sign on platform is not bound to any users") } return &user, nil } func GetUserById(id uint) (*model.User, error) { var u model.User if err := db.First(&u, id).Error; err != nil { return nil, errors.Wrapf(err, "failed get old user") } return &u, nil } func CreateUser(u *model.User) error { return errors.WithStack(db.Create(u).Error) } func UpdateUser(u *model.User) error { return errors.WithStack(db.Save(u).Error) } func GetUsers(pageIndex, pageSize int) (users []model.User, count int64, err error) { userDB := db.Model(&model.User{}) if err := userDB.Count(&count).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get users count") } if err := userDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&users).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get find users") } return users, count, nil } func DeleteUserById(id uint) error { return errors.WithStack(db.Delete(&model.User{}, id).Error) } func UpdateAuthn(userID uint, authn string) error { return db.Model(&model.User{ID: userID}).Update("authn", authn).Error } func RegisterAuthn(u *model.User, credential *webauthn.Credential) error { if u == nil { return errors.New("user is nil") } exists := u.WebAuthnCredentials() if credential != nil { exists = append(exists, *credential) } res, err := utils.Json.Marshal(exists) if err != nil { return err } return UpdateAuthn(u.ID, string(res)) } func RemoveAuthn(u *model.User, id string) error { exists := u.WebAuthnCredentials() for i := 0; i < len(exists); i++ { idEncoded := base64.StdEncoding.EncodeToString(exists[i].ID) if idEncoded == id { exists[len(exists)-1], exists[i] = exists[i], exists[len(exists)-1] exists = exists[:len(exists)-1] break } } res, err := utils.Json.Marshal(exists) if err != nil { return err } return UpdateAuthn(u.ID, string(res)) } ================================================ FILE: internal/db/util.go ================================================ package db import ( "fmt" "github.com/OpenListTeam/OpenList/v4/internal/conf" "gorm.io/gorm" ) func columnName(name string) string { if conf.Conf.Database.Type == "postgres" { return fmt.Sprintf(`"%s"`, name) } return fmt.Sprintf("`%s`", name) } func addStorageOrder(db *gorm.DB) *gorm.DB { return db.Order(fmt.Sprintf("%s, %s", columnName("order"), columnName("id"))) } ================================================ FILE: internal/driver/config.go ================================================ package driver type Config struct { Name string `json:"name"` LocalSort bool `json:"local_sort"` OnlyProxy bool `json:"only_proxy"` NoCache bool `json:"no_cache"` NoUpload bool `json:"no_upload"` // if need get message from user, such as validate code NeedMs bool `json:"need_ms"` DefaultRoot string `json:"default_root"` CheckStatus bool `json:"-"` //info,success,warning,danger Alert string `json:"alert"` // whether to support overwrite upload NoOverwriteUpload bool `json:"-"` ProxyRangeOption bool `json:"-"` // if the driver returns Link without URL, this should be set to true NoLinkURL bool `json:"-"` // Link cache behaviour: // - LinkCacheAuto: let driver decide per-path (implement driver.LinkCacheModeResolver) // - LinkCacheNone: no extra info added to cache key (default) // - flags (OR-able) can add more attributes to cache key (IP, UA, ...) LinkCacheMode `json:"-"` // if the driver only store indices of files (e.g. UrlTree) OnlyIndices bool `json:"only_indices"` // prefer proxy download even if direct link is available PreferProxy bool `json:"prefer_proxy"` } type LinkCacheMode int8 const ( LinkCacheAuto LinkCacheMode = -1 // Let the driver decide per-path (use driver.LinkCacheModeResolver) LinkCacheNone LinkCacheMode = 0 // No extra info added to cache key (default) ) const ( LinkCacheIP LinkCacheMode = 1 << iota // include client IP in cache key LinkCacheUA // include User-Agent in cache key ) func (c Config) MustProxy() bool { return c.OnlyProxy || c.NoLinkURL } func (c Config) DefaultProxy() bool { return c.PreferProxy } ================================================ FILE: internal/driver/driver.go ================================================ package driver import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type Driver interface { Meta Reader // Writer // Other } type Meta interface { Config() Config // GetStorage just get raw storage, no need to implement, because model.Storage have implemented GetStorage() *model.Storage SetStorage(model.Storage) // GetAddition Additional is used for unmarshal of JSON, so need return pointer GetAddition() Additional // Init If already initialized, drop first Init(ctx context.Context) error Drop(ctx context.Context) error } type Other interface { Other(ctx context.Context, args model.OtherArgs) (interface{}, error) } type Reader interface { // List files in the path // if identify files by path, need to set ID with path,like path.Join(dir.GetID(), obj.GetName()) // if identify files by id, need to set ID with corresponding id List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) // Link get url/filepath/reader of file Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) } type GetRooter interface { GetRoot(ctx context.Context) (model.Obj, error) } type Getter interface { // Get file by path, the path haven't been joined with root path Get(ctx context.Context, path string) (model.Obj, error) } //type Writer interface { // Mkdir // Move // Rename // Copy // Remove // Put //} type Mkdir interface { MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error } type Move interface { Move(ctx context.Context, srcObj, dstDir model.Obj) error } type Rename interface { Rename(ctx context.Context, srcObj model.Obj, newName string) error } type Copy interface { Copy(ctx context.Context, srcObj, dstDir model.Obj) error } type Remove interface { Remove(ctx context.Context, obj model.Obj) error } type Put interface { // Put a file (provided as a FileStreamer) into the driver // Besides the most basic upload functionality, the following features also need to be implemented: // 1. Canceling (when `<-ctx.Done()` returns), which can be supported by the following methods: // (1) Use request methods that carry context, such as the following: // a. http.NewRequestWithContext // b. resty.Request.SetContext // c. s3manager.Uploader.UploadWithContext // d. utils.CopyWithCtx // (2) Use a `driver.ReaderWithCtx` or `driver.NewLimitedUploadStream` // (3) Use `utils.IsCanceled` to check if the upload has been canceled during the upload process, // this is typically applicable to chunked uploads. // 2. Submit upload progress (via `up`) in real-time. There are three recommended ways as follows: // (1) Use `utils.CopyWithCtx` // (2) Use `driver.ReaderUpdatingProgress` // (3) Use `driver.Progress` with `io.TeeReader` // 3. Slow down upload speed (via `stream.ServerUploadLimit`). It requires you to wrap the read stream // in a `driver.RateLimitReader` or a `driver.RateLimitFile` after calculating the file's hash and // before uploading the file or file chunks. Or you can directly call `driver.ServerUploadLimitWaitN` // if your file chunks are sufficiently small (less than about 50KB). // NOTE that the network speed may be significantly slower than the stream's read speed. Therefore, if // you use a `errgroup.Group` to upload each chunk in parallel, you should use `Group.SetLimit` to // limit the maximum number of upload threads, preventing excessive memory usage caused by buffering // too many file chunks awaiting upload. Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up UpdateProgress) error } type PutURL interface { // PutURL directly put a URL into the storage // Applicable to index-based drivers like URL-Tree or drivers that support uploading files as URLs // Called when using SimpleHttp for offline downloading, skipping creating a download task PutURL(ctx context.Context, dstDir model.Obj, name, url string) error } type MkdirResult interface { MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) } type MoveResult interface { Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) } type RenameResult interface { Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) } type CopyResult interface { Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) } type PutResult interface { // Put a file (provided as a FileStreamer) into the driver and return the put obj // Besides the most basic upload functionality, the following features also need to be implemented: // 1. Canceling (when `<-ctx.Done()` returns), which can be supported by the following methods: // (1) Use request methods that carry context, such as the following: // a. http.NewRequestWithContext // b. resty.Request.SetContext // c. s3manager.Uploader.UploadWithContext // d. utils.CopyWithCtx // (2) Use a `driver.ReaderWithCtx` or `driver.NewLimitedUploadStream` // (3) Use `utils.IsCanceled` to check if the upload has been canceled during the upload process, // this is typically applicable to chunked uploads. // 2. Submit upload progress (via `up`) in real-time. There are three recommended ways as follows: // (1) Use `utils.CopyWithCtx` // (2) Use `driver.ReaderUpdatingProgress` // (3) Use `driver.Progress` with `io.TeeReader` // 3. Slow down upload speed (via `stream.ServerUploadLimit`). It requires you to wrap the read stream // in a `driver.RateLimitReader` or a `driver.RateLimitFile` after calculating the file's hash and // before uploading the file or file chunks. Or you can directly call `driver.ServerUploadLimitWaitN` // if your file chunks are sufficiently small (less than about 50KB). // NOTE that the network speed may be significantly slower than the stream's read speed. Therefore, if // you use a `errgroup.Group` to upload each chunk in parallel, you should use `Group.SetLimit` to // limit the maximum number of upload threads, preventing excessive memory usage caused by buffering // too many file chunks awaiting upload. Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up UpdateProgress) (model.Obj, error) } type PutURLResult interface { // PutURL directly put a URL into the storage // Applicable to index-based drivers like URL-Tree or drivers that support uploading files as URLs // Called when using SimpleHttp for offline downloading, skipping creating a download task PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) } type ArchiveReader interface { // GetArchiveMeta get the meta-info of an archive // return errs.WrongArchivePassword if the meta-info is also encrypted but provided password is wrong or empty // return errs.NotImplement to use internal archive tools to get the meta-info, such as the following cases: // 1. the driver do not support the format of the archive but there may be an internal tool do // 2. handling archives is a VIP feature, but the driver does not have VIP access GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) // ListArchive list the children of model.ArchiveArgs.InnerPath in the archive // return errs.NotImplement to use internal archive tools to list the children // return errs.NotSupport if the folder structure should be acquired from model.ArchiveMeta.GetTree ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) // Extract get url/filepath/reader of a file in the archive // return errs.NotImplement to use internal archive tools to extract Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) } type ArchiveGetter interface { // ArchiveGet get file by inner path // return errs.NotImplement to use internal archive tools to get the children // return errs.NotSupport if the folder structure should be acquired from model.ArchiveMeta.GetTree ArchiveGet(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (model.Obj, error) } type ArchiveDecompress interface { ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error } type ArchiveDecompressResult interface { // ArchiveDecompress decompress an archive // when args.PutIntoNewDir, the new sub-folder should be named the same to the archive but without the extension // return each decompressed obj from the root path of the archive when args.PutIntoNewDir is false // return only the newly created folder when args.PutIntoNewDir is true // return errs.NotImplement to use internal archive tools to decompress ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) } type WithDetails interface { // GetDetails get storage details (total space, free space, etc.) GetDetails(ctx context.Context) (*model.StorageDetails, error) } type Reference interface { InitReference(storage Driver) error } type LinkCacheModeResolver interface { // ResolveLinkCacheMode returns the LinkCacheMode for the given path. ResolveLinkCacheMode(path string) LinkCacheMode } type DirectUploader interface { // GetDirectUploadTools returns available frontend-direct upload tools GetDirectUploadTools() []string // GetDirectUploadInfo returns the information needed for direct upload from client to storage // actualPath is the path relative to the storage root (after removing mount path prefix) // return errs.NotImplement if the driver does not support the given direct upload tool GetDirectUploadInfo(ctx context.Context, tool string, dstDir model.Obj, fileName string, fileSize int64) (any, error) } ================================================ FILE: internal/driver/item.go ================================================ package driver type Additional interface{} type Select string type Item struct { Name string `json:"name"` Type string `json:"type"` Default string `json:"default"` Options string `json:"options"` Required bool `json:"required"` Help string `json:"help"` } type Info struct { Common []Item `json:"common"` Additional []Item `json:"additional"` Config Config `json:"config"` } type IRootPath interface { GetRootPath() string } type IRootId interface { GetRootId() string } type RootPath struct { RootFolderPath string `json:"root_folder_path"` } type RootID struct { RootFolderID string `json:"root_folder_id"` } func (r RootPath) GetRootPath() string { return r.RootFolderPath } func (r *RootPath) SetRootPath(path string) { r.RootFolderPath = path } func (r RootID) GetRootId() string { return r.RootFolderID } ================================================ FILE: internal/driver/utils.go ================================================ package driver import ( "context" "io" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" ) type UpdateProgress = model.UpdateProgress type Progress struct { Total int64 Done int64 up UpdateProgress } func (p *Progress) Write(b []byte) (n int, err error) { n = len(b) p.Done += int64(n) p.up(float64(p.Done) / float64(p.Total) * 100) return n, err } func NewProgress(total int64, up UpdateProgress) *Progress { return &Progress{ Total: total, up: up, } } type RateLimitReader = stream.RateLimitReader type RateLimitWriter = stream.RateLimitWriter type RateLimitFile = stream.RateLimitFile func NewLimitedUploadStream(ctx context.Context, r io.Reader) *RateLimitReader { return &RateLimitReader{ Reader: r, Limiter: stream.ServerUploadLimit, Ctx: ctx, } } func NewLimitedUploadFile(ctx context.Context, f model.File) *RateLimitFile { return &RateLimitFile{ File: f, Limiter: stream.ServerUploadLimit, Ctx: ctx, } } func ServerUploadLimitWaitN(ctx context.Context, n int) error { return stream.ServerUploadLimit.WaitN(ctx, n) } type ReaderWithCtx = stream.ReaderWithCtx type ReaderUpdatingProgress = stream.ReaderUpdatingProgress type SimpleReaderWithSize = stream.SimpleReaderWithSize ================================================ FILE: internal/errs/driver.go ================================================ package errs import "errors" var ( EmptyToken = errors.New("empty token") ) ================================================ FILE: internal/errs/errors.go ================================================ package errs import ( "errors" "fmt" pkgerr "github.com/pkg/errors" ) var ( NotImplement = errors.New("not implement") NotSupport = errors.New("not support") RelativePath = errors.New("using relative path is not allowed") UploadNotSupported = errors.New("upload not supported") MetaNotFound = errors.New("meta not found") StorageNotFound = errors.New("storage not found") StorageNotInit = errors.New("storage not init") StreamIncomplete = errors.New("upload/download stream incomplete, possible network issue") StreamPeekFail = errors.New("StreamPeekFail") UnknownArchiveFormat = errors.New("unknown archive format") WrongArchivePassword = errors.New("wrong archive password") DriverExtractNotSupported = errors.New("driver extraction not supported") WrongShareCode = errors.New("wrong share code") InvalidSharing = errors.New("invalid sharing") SharingNotFound = errors.New("sharing not found") ) // NewErr wrap constant error with an extra message // use errors.Is(err1, StorageNotFound) to check if err belongs to any internal error func NewErr(err error, format string, a ...any) error { return fmt.Errorf("%w; %s", err, fmt.Sprintf(format, a...)) } func IsNotFoundError(err error) bool { return errors.Is(pkgerr.Cause(err), ObjectNotFound) || errors.Is(pkgerr.Cause(err), StorageNotFound) } func IsNotSupportError(err error) bool { return errors.Is(pkgerr.Cause(err), NotSupport) } func IsNotImplementError(err error) bool { return errors.Is(pkgerr.Cause(err), NotImplement) } ================================================ FILE: internal/errs/errors_test.go ================================================ package errs import ( "errors" pkgerr "github.com/pkg/errors" "testing" ) func TestErrs(t *testing.T) { err1 := NewErr(StorageNotFound, "please add a storage first") t.Logf("err1: %s", err1) if !errors.Is(err1, StorageNotFound) { t.Errorf("failed, expect %s is %s", err1, StorageNotFound) } if !errors.Is(pkgerr.Cause(err1), StorageNotFound) { t.Errorf("failed, expect %s is %s", err1, StorageNotFound) } err2 := pkgerr.WithMessage(err1, "failed get storage") t.Logf("err2: %s", err2) if !errors.Is(err2, StorageNotFound) { t.Errorf("failed, expect %s is %s", err2, StorageNotFound) } if !errors.Is(pkgerr.Cause(err2), StorageNotFound) { t.Errorf("failed, expect %s is %s", err2, StorageNotFound) } } ================================================ FILE: internal/errs/object.go ================================================ package errs import ( "errors" pkgerr "github.com/pkg/errors" ) var ( ObjectNotFound = errors.New("object not found") ObjectAlreadyExists = errors.New("object already exists") NotFolder = errors.New("not a folder") NotFile = errors.New("not a file") IgnoredSystemFile = errors.New("system file upload ignored") ) func IsObjectNotFound(err error) bool { return errors.Is(pkgerr.Cause(err), ObjectNotFound) } ================================================ FILE: internal/errs/operate.go ================================================ package errs import "errors" var ( PermissionDenied = errors.New("permission denied") ) ================================================ FILE: internal/errs/search.go ================================================ package errs import "fmt" var ( SearchNotAvailable = fmt.Errorf("search not available") BuildIndexIsRunning = fmt.Errorf("build index is running, please try later") ) ================================================ FILE: internal/errs/unwrap.go ================================================ package errs func UnwrapOrSelf(err error) error { u, ok := err.(interface { Unwrap() error }) if !ok { return err } return u.Unwrap() } ================================================ FILE: internal/errs/user.go ================================================ package errs import "errors" var ( EmptyUsername = errors.New("username is empty") EmptyPassword = errors.New("password is empty") WrongPassword = errors.New("password is incorrect") DeleteAdminOrGuest = errors.New("cannot delete admin or guest") ) ================================================ FILE: internal/fs/archive.go ================================================ package fs import ( "context" stderrors "errors" "fmt" "io" "math/rand" "os" stdpath "path" "path/filepath" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/internal/task_group" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) type ArchiveDownloadTask struct { TaskData model.ArchiveDecompressArgs } func (t *ArchiveDownloadTask) GetName() string { return fmt.Sprintf("decompress [%s](%s)[%s] to [%s](%s) with password <%s>", t.SrcStorageMp, t.SrcActualPath, t.InnerPath, t.DstStorageMp, t.DstActualPath, t.Password) } func (t *ArchiveDownloadTask) Run() error { if t.SrcStorage == nil { if srcStorage, _, err := op.GetStorageAndActualPath(t.SrcStorageMp); err == nil { t.SrcStorage = srcStorage } else { return err } if dstStorage, _, err := op.GetStorageAndActualPath(t.DstStorageMp); err == nil { t.DstStorage = dstStorage } else { return err } } t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() uploadTask, err := t.RunWithoutPushUploadTask() if err != nil { return err } uploadTask.groupID = stdpath.Join(uploadTask.DstStorageMp, uploadTask.DstActualPath) task_group.TransferCoordinator.AddTask(uploadTask.groupID, nil) ArchiveContentUploadTaskManager.Add(uploadTask) return nil } func (t *ArchiveDownloadTask) RunWithoutPushUploadTask() (*ArchiveContentUploadTask, error) { srcObj, tool, ss, err := op.GetArchiveToolAndStream(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.LinkArgs{}) if err != nil { return nil, err } defer func() { var e error for _, s := range ss { e = stderrors.Join(e, s.Close()) } if e != nil { log.Errorf("failed to close file streamer, %v", e) } }() var decompressUp model.UpdateProgress if t.CacheFull { total := int64(0) for _, s := range ss { total += s.GetSize() } t.SetTotalBytes(total) t.Status = "getting src object" part := 100 / float64(len(ss)+1) for i, s := range ss { if s.GetFile() != nil { continue } _, err = s.CacheFullAndWriter(nil, nil) if err != nil { return nil, err } else { t.SetProgress(float64(i+1) * part) } } decompressUp = model.UpdateProgressWithRange(t.SetProgress, 100-part, 100) } else { decompressUp = t.SetProgress } t.Status = "walking and decompressing" dir, err := os.MkdirTemp(conf.Conf.TempDir, "dir-*") if err != nil { return nil, err } err = tool.Decompress(ss, dir, t.ArchiveInnerArgs, decompressUp) if err != nil { return nil, err } baseName := strings.TrimSuffix(srcObj.GetName(), stdpath.Ext(srcObj.GetName())) uploadTask := &ArchiveContentUploadTask{ TaskExtension: task.TaskExtension{ Creator: t.Creator, ApiUrl: t.ApiUrl, }, ObjName: baseName, InPlace: !t.PutIntoNewDir, FilePath: dir, DstActualPath: t.DstActualPath, dstStorage: t.DstStorage, DstStorageMp: t.DstStorageMp, overwrite: t.Overwrite, } return uploadTask, nil } var ArchiveDownloadTaskManager *tache.Manager[*ArchiveDownloadTask] type ArchiveContentUploadTask struct { task.TaskExtension status string ObjName string InPlace bool FilePath string DstActualPath string dstStorage driver.Driver DstStorageMp string finalized bool groupID string overwrite bool } func (t *ArchiveContentUploadTask) GetName() string { return fmt.Sprintf("upload %s to [%s](%s)", t.ObjName, t.DstStorageMp, t.DstActualPath) } func (t *ArchiveContentUploadTask) GetStatus() string { return t.status } func (t *ArchiveContentUploadTask) Run() error { t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() return t.RunWithNextTaskCallback(func(nextTsk *ArchiveContentUploadTask) error { task_group.TransferCoordinator.AddTask(t.groupID, nil) ArchiveContentUploadTaskManager.Add(nextTsk) return nil }) } func (t *ArchiveContentUploadTask) OnSucceeded() { task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, true) } func (t *ArchiveContentUploadTask) OnFailed() { task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, false) } func (t *ArchiveContentUploadTask) SetRetry(retry int, maxRetry int) { t.TaskExtension.SetRetry(retry, maxRetry) if retry == 0 && (len(t.groupID) == 0 || // 重启恢复 (t.GetErr() == nil && t.GetState() != tache.StatePending)) { // 手动重试 t.groupID = stdpath.Join(t.DstStorageMp, t.DstActualPath) task_group.TransferCoordinator.AddTask(t.groupID, nil) } } func (t *ArchiveContentUploadTask) RunWithNextTaskCallback(f func(nextTask *ArchiveContentUploadTask) error) error { info, err := os.Stat(t.FilePath) if err != nil { return err } if info.IsDir() { t.status = "src object is dir, listing objs" nextDstActualPath := t.DstActualPath if !t.InPlace { nextDstActualPath = stdpath.Join(nextDstActualPath, t.ObjName) err = op.MakeDir(t.Ctx(), t.dstStorage, nextDstActualPath) if err != nil { return err } } entries, err := os.ReadDir(t.FilePath) if err != nil { return err } if !t.InPlace { task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(nextDstActualPath)) } var es error for _, entry := range entries { var nextFilePath string if entry.IsDir() { nextFilePath, err = moveToTempPath(stdpath.Join(t.FilePath, entry.Name()), "dir-") } else { nextFilePath, err = moveToTempPath(stdpath.Join(t.FilePath, entry.Name()), "file-") } if err != nil { es = stderrors.Join(es, err) continue } err = f(&ArchiveContentUploadTask{ TaskExtension: task.TaskExtension{ Creator: t.Creator, ApiUrl: t.ApiUrl, }, ObjName: entry.Name(), InPlace: false, FilePath: nextFilePath, DstActualPath: nextDstActualPath, dstStorage: t.dstStorage, DstStorageMp: t.DstStorageMp, groupID: t.groupID, overwrite: t.overwrite, }) if err != nil { es = stderrors.Join(es, err) } } if es != nil { return es } } else { if !t.overwrite { dstPath := stdpath.Join(t.DstActualPath, t.ObjName) if res, _ := op.Get(t.Ctx(), t.dstStorage, dstPath); res != nil { return errs.ObjectAlreadyExists } } file, err := os.Open(t.FilePath) if err != nil { return err } t.SetTotalBytes(info.Size()) fs := &stream.FileStream{ Obj: &model.Object{ Name: t.ObjName, Size: info.Size(), Modified: time.Now(), }, Mimetype: utils.GetMimeType(stdpath.Ext(t.ObjName)), WebPutAsTask: true, Reader: file, } fs.Closers.Add(file) t.status = "uploading" err = op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.dstStorage, t.DstActualPath, fs, t.SetProgress) if err != nil { return err } } t.deleteSrcFile() return nil } func (t *ArchiveContentUploadTask) Cancel() { t.TaskExtension.Cancel() if !conf.Conf.Tasks.AllowRetryCanceled { t.deleteSrcFile() } } func (t *ArchiveContentUploadTask) deleteSrcFile() { if !t.finalized { _ = os.RemoveAll(t.FilePath) t.finalized = true } } func moveToTempPath(path, prefix string) (string, error) { newPath, err := genTempFileName(prefix) if err != nil { return "", err } err = os.Rename(path, newPath) if err != nil { return "", err } return newPath, nil } func genTempFileName(prefix string) (string, error) { retry := 0 t := time.Now().UnixMilli() for retry < 10000 { newPath := filepath.Join(conf.Conf.TempDir, prefix+fmt.Sprintf("%x-%x", t, rand.Uint32())) if _, err := os.Stat(newPath); err != nil { if os.IsNotExist(err) { return newPath, nil } else { return "", err } } retry++ } return "", errors.New("failed to generate temp-file name: too many retries") } type archiveContentUploadTaskManagerType struct { *tache.Manager[*ArchiveContentUploadTask] } func (m *archiveContentUploadTaskManagerType) Remove(id string) { if t, ok := m.GetByID(id); ok { t.deleteSrcFile() m.Manager.Remove(id) } } func (m *archiveContentUploadTaskManagerType) RemoveAll() { tasks := m.GetAll() for _, t := range tasks { m.Remove(t.GetID()) } } func (m *archiveContentUploadTaskManagerType) RemoveByState(state ...tache.State) { tasks := m.GetByState(state...) for _, t := range tasks { m.Remove(t.GetID()) } } func (m *archiveContentUploadTaskManagerType) RemoveByCondition(condition func(task *ArchiveContentUploadTask) bool) { tasks := m.GetByCondition(condition) for _, t := range tasks { m.Remove(t.GetID()) } } var ArchiveContentUploadTaskManager = &archiveContentUploadTaskManagerType{ Manager: nil, } func archiveMeta(ctx context.Context, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) { storage, actualPath, err := op.GetStorageAndActualPath(path) if err != nil { return nil, errors.WithMessage(err, "failed get storage") } return op.GetArchiveMeta(ctx, storage, actualPath, args) } func archiveList(ctx context.Context, path string, args model.ArchiveListArgs) ([]model.Obj, error) { storage, actualPath, err := op.GetStorageAndActualPath(path) if err != nil { return nil, errors.WithMessage(err, "failed get storage") } return op.ListArchive(ctx, storage, actualPath, args) } func archiveDecompress(ctx context.Context, srcObjPath, dstDirPath string, args model.ArchiveDecompressArgs, lazyCache ...bool) (task.TaskExtensionInfo, error) { srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(srcObjPath) if err != nil { return nil, errors.WithMessage(err, "failed get src storage") } dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { return nil, errors.WithMessage(err, "failed get dst storage") } if srcStorage.GetStorage() == dstStorage.GetStorage() { err = op.ArchiveDecompress(ctx, srcStorage, srcObjActualPath, dstDirActualPath, args, lazyCache...) if !errors.Is(err, errs.NotImplement) { return nil, err } } tsk := &ArchiveDownloadTask{ TaskData: TaskData{ SrcStorage: srcStorage, DstStorage: dstStorage, SrcActualPath: srcObjActualPath, DstActualPath: dstDirActualPath, SrcStorageMp: srcStorage.GetStorage().MountPath, DstStorageMp: dstStorage.GetStorage().MountPath, }, ArchiveDecompressArgs: args, } if ctx.Value(conf.NoTaskKey) != nil { tsk.Base.SetCtx(ctx) uploadTask, err := tsk.RunWithoutPushUploadTask() if err != nil { return nil, errors.WithMessagef(err, "failed download [%s]", srcObjPath) } defer uploadTask.deleteSrcFile() var callback func(t *ArchiveContentUploadTask) error var hasSuccess bool callback = func(t *ArchiveContentUploadTask) error { t.Base.SetCtx(ctx) e := t.RunWithNextTaskCallback(callback) if e == nil { hasSuccess = true } t.deleteSrcFile() return e } uploadTask.Base.SetCtx(ctx) uploadTask.groupID = stdpath.Join(uploadTask.DstStorageMp, uploadTask.DstActualPath) task_group.TransferCoordinator.AddTask(uploadTask.groupID, nil) err = uploadTask.RunWithNextTaskCallback(callback) task_group.TransferCoordinator.Done(context.WithoutCancel(ctx), uploadTask.groupID, hasSuccess) return nil, err } else { tsk.Creator, _ = ctx.Value(conf.UserKey).(*model.User) tsk.ApiUrl = common.GetApiUrl(ctx) ArchiveDownloadTaskManager.Add(tsk) return tsk, nil } } func archiveDriverExtract(ctx context.Context, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) { storage, actualPath, err := op.GetStorageAndActualPath(path) if err != nil { return nil, nil, errors.WithMessage(err, "failed get storage") } return op.DriverExtract(ctx, storage, actualPath, args) } func archiveInternalExtract(ctx context.Context, path string, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { storage, actualPath, err := op.GetStorageAndActualPath(path) if err != nil { return nil, 0, errors.WithMessage(err, "failed get storage") } return op.InternalExtract(ctx, storage, actualPath, args) } ================================================ FILE: internal/fs/copy_move.go ================================================ package fs import ( "context" "fmt" stdpath "path" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/internal/task_group" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" "github.com/pkg/errors" ) type taskType uint8 func (t taskType) String() string { switch t { case copy: return "copy" case move: return "move" case merge: return "merge" default: return "unknown" } } const ( copy taskType = iota move merge ) type FileTransferTask struct { TaskData TaskType taskType groupID string } func (t *FileTransferTask) GetName() string { return fmt.Sprintf("%s [%s](%s) to [%s](%s)", t.TaskType, t.SrcStorageMp, t.SrcActualPath, t.DstStorageMp, t.DstActualPath) } func (t *FileTransferTask) Run() error { if t.SrcStorage == nil { if srcStorage, _, err := op.GetStorageAndActualPath(t.SrcStorageMp); err == nil { t.SrcStorage = srcStorage } else { return err } if dstStorage, _, err := op.GetStorageAndActualPath(t.DstStorageMp); err == nil { t.DstStorage = dstStorage } else { return err } } t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() return t.RunWithNextTaskCallback(func(nextTask *FileTransferTask) error { task_group.TransferCoordinator.AddTask(t.groupID, nil) if t.TaskType == copy || t.TaskType == merge { CopyTaskManager.Add(nextTask) } else { MoveTaskManager.Add(nextTask) } return nil }) } func (t *FileTransferTask) OnSucceeded() { task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, true) } func (t *FileTransferTask) OnFailed() { task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, false) } func (t *FileTransferTask) SetRetry(retry int, maxRetry int) { t.TaskData.SetRetry(retry, maxRetry) if retry == 0 && (len(t.groupID) == 0 || // 重启恢复 (t.GetErr() == nil && t.GetState() != tache.StatePending)) { // 手动重试 t.groupID = stdpath.Join(t.DstStorageMp, t.DstActualPath) var payload any if t.TaskType == move { payload = task_group.SrcPathToRemove(stdpath.Join(t.SrcStorageMp, t.SrcActualPath)) } task_group.TransferCoordinator.AddTask(t.groupID, payload) } } func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(srcObjPath) if err != nil { return nil, errors.WithMessage(err, "failed get src storage") } dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { return nil, errors.WithMessage(err, "failed get dst storage") } if srcStorage.GetStorage() == dstStorage.GetStorage() { if utils.IsBool(skipHook...) { ctx = context.WithValue(ctx, conf.SkipHookKey, struct{}{}) } if taskType == copy || taskType == merge { err = op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath) if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) { return nil, err } } else { err = op.Move(ctx, srcStorage, srcObjActualPath, dstDirActualPath) if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) { return nil, err } } } // not in the same storage t := &FileTransferTask{ TaskData: TaskData{ SrcStorage: srcStorage, DstStorage: dstStorage, SrcActualPath: srcObjActualPath, DstActualPath: dstDirActualPath, SrcStorageMp: srcStorage.GetStorage().MountPath, DstStorageMp: dstStorage.GetStorage().MountPath, }, TaskType: taskType, } t.groupID = stdpath.Join(t.DstStorageMp, t.DstActualPath) task_group.TransferCoordinator.AddTask(t.groupID, nil) if ctx.Value(conf.NoTaskKey) != nil { var callback func(nextTask *FileTransferTask) error hasSuccess := false callback = func(nextTask *FileTransferTask) error { nextTask.Base.SetCtx(ctx) err := nextTask.RunWithNextTaskCallback(callback) if err == nil { hasSuccess = true } return err } t.Base.SetCtx(ctx) err = t.RunWithNextTaskCallback(callback) if err == nil { hasSuccess = true } if taskType == move { task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.SrcPathToRemove(srcObjPath)) } task_group.TransferCoordinator.Done(context.WithoutCancel(ctx), t.groupID, hasSuccess) return nil, err } t.Creator, _ = ctx.Value(conf.UserKey).(*model.User) t.ApiUrl = common.GetApiUrl(ctx) if taskType == copy || taskType == merge { CopyTaskManager.Add(t) } else { task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.SrcPathToRemove(srcObjPath)) MoveTaskManager.Add(t) } return t, nil } func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransferTask) error) error { t.Status = "getting src object" srcObj, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcActualPath) if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", t.SrcActualPath) } if srcObj.IsDir() { t.Status = "src object is dir, listing objs" objs, err := op.List(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.ListArgs{}) if err != nil { return errors.WithMessagef(err, "failed list src [%s] objs", t.SrcActualPath) } dstActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName()) task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstActualPath)) existedObjs := make(map[string]bool) if t.TaskType == merge { dstObjs, err := op.List(t.Ctx(), t.DstStorage, dstActualPath, model.ListArgs{}) if err != nil && !errors.Is(err, errs.ObjectNotFound) { // 目标文件夹不存在的情况不是错误，会在之后新建文件夹 // 这种情况显然不需要统计existedObjs，dstObjs保持为nil，下面这个for将不会执行 return errors.WithMessagef(err, "failed list dst [%s] objs", dstActualPath) } for _, obj := range dstObjs { if err := t.Ctx().Err(); err != nil { return err } if !obj.IsDir() { existedObjs[obj.GetName()] = true } } } for _, obj := range objs { if err := t.Ctx().Err(); err != nil { return err } if t.TaskType == merge && !obj.IsDir() && existedObjs[obj.GetName()] { // skip existed file continue } err = f(&FileTransferTask{ TaskType: t.TaskType, TaskData: TaskData{ TaskExtension: task.TaskExtension{ Creator: t.Creator, ApiUrl: t.ApiUrl, }, SrcStorage: t.SrcStorage, DstStorage: t.DstStorage, SrcActualPath: stdpath.Join(t.SrcActualPath, obj.GetName()), DstActualPath: dstActualPath, SrcStorageMp: t.SrcStorageMp, DstStorageMp: t.DstStorageMp, }, groupID: t.groupID, }) if err != nil { return err } } t.Status = fmt.Sprintf("src object is dir, added all %s tasks of objs", t.TaskType) return nil } t.Status = "getting src object link" link, srcObj, err := op.Link(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.LinkArgs{}) if err != nil { return errors.WithMessagef(err, "failed get [%s] link", t.SrcActualPath) } // any link provided is seekable ss, err := stream.NewSeekableStream(&stream.FileStream{ Obj: srcObj, Ctx: t.Ctx(), }, link) if err != nil { _ = link.Close() return errors.WithMessagef(err, "failed get [%s] stream", t.SrcActualPath) } t.SetTotalBytes(ss.GetSize()) t.Status = "uploading" return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, ss, t.SetProgress) } var ( CopyTaskManager *tache.Manager[*FileTransferTask] MoveTaskManager *tache.Manager[*FileTransferTask] ) ================================================ FILE: internal/fs/fs.go ================================================ package fs import ( "context" "io" log "github.com/sirupsen/logrus" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/pkg/errors" ) // the param named path of functions in this package is a mount path // So, the purpose of this package is to convert mount path to actual path // then pass the actual path to the op package type ListArgs struct { Refresh bool NoLog bool WithStorageDetails bool } func List(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) { res, err := list(ctx, path, args) if err != nil { if !args.NoLog { log.Errorf("failed list %s: %+v", path, err) } return nil, err } return res, nil } type GetArgs struct { NoLog bool WithStorageDetails bool } func Get(ctx context.Context, path string, args *GetArgs) (model.Obj, error) { res, err := get(ctx, path, args) if err != nil { if !args.NoLog { log.Warnf("failed get %s: %s", path, err) } return nil, err } return res, nil } func Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { res, file, err := link(ctx, path, args) if err != nil { log.Errorf("failed link %s: %+v", path, err) return nil, nil, err } return res, file, nil } func MakeDir(ctx context.Context, path string) error { err := makeDir(ctx, path) if err != nil { log.Errorf("failed make dir %s: %+v", path, err) } return err } func Move(ctx context.Context, srcPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { req, err := transfer(ctx, move, srcPath, dstDirPath, skipHook...) if err != nil { log.Errorf("failed move %s to %s: %+v", srcPath, dstDirPath, err) } return req, err } func Copy(ctx context.Context, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { res, err := transfer(ctx, copy, srcObjPath, dstDirPath, skipHook...) if err != nil { log.Errorf("failed copy %s to %s: %+v", srcObjPath, dstDirPath, err) } return res, err } func Merge(ctx context.Context, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { res, err := transfer(ctx, merge, srcObjPath, dstDirPath, skipHook...) if err != nil { log.Errorf("failed merge %s to %s: %+v", srcObjPath, dstDirPath, err) } return res, err } func Rename(ctx context.Context, srcPath, dstName string, skipHook ...bool) error { err := rename(ctx, srcPath, dstName, skipHook...) if err != nil { log.Errorf("failed rename %s to %s: %+v", srcPath, dstName, err) } return err } func Remove(ctx context.Context, path string) error { err := remove(ctx, path) if err != nil { log.Errorf("failed remove %s: %+v", path, err) } return err } func PutDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer, skipHook ...bool) error { err := putDirectly(ctx, dstDirPath, file, skipHook...) if err != nil { log.Errorf("failed put %s: %+v", dstDirPath, err) } return err } func PutAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) (task.TaskExtensionInfo, error) { t, err := putAsTask(ctx, dstDirPath, file) if err != nil { log.Errorf("failed put %s: %+v", dstDirPath, err) } return t, err } func ArchiveMeta(ctx context.Context, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) { meta, err := archiveMeta(ctx, path, args) if err != nil { log.Errorf("failed get archive meta %s: %+v", path, err) } return meta, err } func ArchiveList(ctx context.Context, path string, args model.ArchiveListArgs) ([]model.Obj, error) { objs, err := archiveList(ctx, path, args) if err != nil { log.Errorf("failed list archive [%s]%s: %+v", path, args.InnerPath, err) } return objs, err } func ArchiveDecompress(ctx context.Context, srcObjPath, dstDirPath string, args model.ArchiveDecompressArgs, lazyCache ...bool) (task.TaskExtensionInfo, error) { t, err := archiveDecompress(ctx, srcObjPath, dstDirPath, args, lazyCache...) if err != nil { log.Errorf("failed decompress [%s]%s: %+v", srcObjPath, args.InnerPath, err) } return t, err } func ArchiveDriverExtract(ctx context.Context, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) { l, obj, err := archiveDriverExtract(ctx, path, args) if err != nil { log.Errorf("failed extract [%s]%s: %+v", path, args.InnerPath, err) } return l, obj, err } func ArchiveInternalExtract(ctx context.Context, path string, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { l, obj, err := archiveInternalExtract(ctx, path, args) if err != nil { log.Errorf("failed extract [%s]%s: %+v", path, args.InnerPath, err) } return l, obj, err } type GetStoragesArgs struct { } func GetStorage(path string, args *GetStoragesArgs) (driver.Driver, error) { storageDriver, _, err := op.GetStorageAndActualPath(path) if err != nil { return nil, err } return storageDriver, nil } func Other(ctx context.Context, args model.FsOtherArgs) (interface{}, error) { res, err := other(ctx, args) if err != nil { log.Errorf("failed get other %s: %+v", args.Path, err) } return res, err } func PutURL(ctx context.Context, path, dstName, urlStr string) error { storage, dstDirActualPath, err := op.GetStorageAndActualPath(path) if err != nil { return errors.WithMessage(err, "failed get storage") } if storage.Config().NoUpload { return errors.WithStack(errs.UploadNotSupported) } _, ok := storage.(driver.PutURL) _, okResult := storage.(driver.PutURLResult) if !ok && !okResult { return errs.NotImplement } return op.PutURL(ctx, storage, dstDirActualPath, dstName, urlStr) } func GetDirectUploadInfo(ctx context.Context, tool, path, dstName string, fileSize int64) (any, error) { info, err := getDirectUploadInfo(ctx, tool, path, dstName, fileSize) if err != nil { log.Errorf("failed get %s direct upload info for %s(%d bytes): %+v", path, dstName, fileSize, err) } return info, err } ================================================ FILE: internal/fs/get.go ================================================ package fs import ( "context" stdpath "path" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" ) func get(ctx context.Context, path string, args *GetArgs) (model.Obj, error) { path = utils.FixAndCleanPath(path) // maybe a virtual file if path != "/" { dir, name := stdpath.Split(path) virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, dir, !args.WithStorageDetails, false, name) for _, f := range virtualFiles { if f.GetName() == name { return f, nil } } } storage, actualPath, err := op.GetStorageAndActualPath(path) if err != nil { // if there are no storage prefix with path, maybe root folder if path == "/" { return &model.Object{ Name: "root", IsFolder: true, Mask: model.ReadOnly | model.Virtual, }, nil } return nil, errors.WithMessage(err, "failed get storage") } return op.Get(ctx, storage, actualPath) } ================================================ FILE: internal/fs/link.go ================================================ package fs import ( "context" "strings" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" ) func link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { storage, actualPath, err := op.GetStorageAndActualPath(path) if err != nil { return nil, nil, errors.WithMessage(err, "failed get storage") } l, obj, err := op.Link(ctx, storage, actualPath, args) if err != nil { return nil, nil, errors.WithMessage(err, "failed link") } if l.URL != "" && !strings.HasPrefix(l.URL, "http://") && !strings.HasPrefix(l.URL, "https://") { l.URL = common.GetApiUrl(ctx) + l.URL } return l, obj, nil } ================================================ FILE: internal/fs/list.go ================================================ package fs import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) // List files func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) { meta, _ := ctx.Value(conf.MetaKey).(*model.Meta) user, _ := ctx.Value(conf.UserKey).(*model.User) virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, path, !args.WithStorageDetails, args.Refresh, "") storage, actualPath, err := op.GetStorageAndActualPath(path) if err != nil && len(virtualFiles) == 0 { return nil, errors.WithMessage(err, "failed get storage") } var _objs []model.Obj if storage != nil { _objs, err = op.List(ctx, storage, actualPath, model.ListArgs{ ReqPath: path, Refresh: args.Refresh, WithStorageDetails: args.WithStorageDetails, }) if err != nil { if !args.NoLog { log.Errorf("fs/list: %+v", err) } if len(virtualFiles) == 0 { return nil, errors.WithMessage(err, "failed get objs") } } } om := model.NewObjMerge() if whetherHide(user, meta, path) { om.InitHideReg(meta.Hide) } objs := om.Merge(_objs, virtualFiles...) return objs, nil } func whetherHide(user *model.User, meta *model.Meta, path string) bool { // if is admin, don't hide if user == nil || user.CanSeeHides() { return false } // if meta is nil, don't hide if meta == nil { return false } // if meta.Hide is empty, don't hide if meta.Hide == "" { return false } // if meta doesn't apply to sub_folder, don't hide if !utils.PathEqual(meta.Path, path) && !meta.HSub { return false } // if is guest, hide return true } ================================================ FILE: internal/fs/other.go ================================================ package fs import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" ) func makeDir(ctx context.Context, path string) error { storage, actualPath, err := op.GetStorageAndActualPath(path) if err != nil { return errors.WithMessage(err, "failed get storage") } return op.MakeDir(ctx, storage, actualPath) } func rename(ctx context.Context, srcPath, dstName string, skipHook ...bool) error { storage, srcActualPath, err := op.GetStorageAndActualPath(srcPath) if err != nil { return errors.WithMessage(err, "failed get storage") } if utils.IsBool(skipHook...) { ctx = context.WithValue(ctx, conf.SkipHookKey, struct{}{}) } return op.Rename(ctx, storage, srcActualPath, dstName) } func remove(ctx context.Context, path string) error { storage, actualPath, err := op.GetStorageAndActualPath(path) if err != nil { return errors.WithMessage(err, "failed get storage") } return op.Remove(ctx, storage, actualPath) } func other(ctx context.Context, args model.FsOtherArgs) (interface{}, error) { storage, actualPath, err := op.GetStorageAndActualPath(args.Path) if err != nil { return nil, errors.WithMessage(err, "failed get storage") } args.Path = actualPath return op.Other(ctx, storage, args) } type TaskData struct { task.TaskExtension Status string `json:"-"` //don't save status to save space SrcActualPath string `json:"src_path"` DstActualPath string `json:"dst_path"` SrcStorage driver.Driver `json:"-"` DstStorage driver.Driver `json:"-"` SrcStorageMp string `json:"src_storage_mp"` DstStorageMp string `json:"dst_storage_mp"` } func (t *TaskData) GetStatus() string { return t.Status } ================================================ FILE: internal/fs/put.go ================================================ package fs import ( "context" "fmt" stdpath "path" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/internal/task_group" "github.com/OpenListTeam/tache" "github.com/pkg/errors" ) type UploadTask struct { task.TaskExtension storage driver.Driver dstDirActualPath string file model.FileStreamer } func (t *UploadTask) GetName() string { return fmt.Sprintf("upload %s to [%s](%s)", t.file.GetName(), t.storage.GetStorage().MountPath, t.dstDirActualPath) } func (t *UploadTask) GetStatus() string { return "uploading" } func (t *UploadTask) Run() error { t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.storage, t.dstDirActualPath, t.file, t.SetProgress) } func (t *UploadTask) OnSucceeded() { task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), stdpath.Join(t.storage.GetStorage().MountPath, t.dstDirActualPath), true) } func (t *UploadTask) OnFailed() { task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), stdpath.Join(t.storage.GetStorage().MountPath, t.dstDirActualPath), false) } func (t *UploadTask) SetRetry(retry int, maxRetry int) { t.TaskExtension.SetRetry(retry, maxRetry) if retry == 0 && (t.GetErr() == nil && t.GetState() != tache.StatePending) { // 手动重试 task_group.TransferCoordinator.AddTask(stdpath.Join(t.storage.GetStorage().MountPath, t.dstDirActualPath), nil) } } var UploadTaskManager *tache.Manager[*UploadTask] // putAsTask add as a put task and return immediately func putAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) (task.TaskExtensionInfo, error) { storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { return nil, errors.WithMessage(err, "failed get storage") } if storage.Config().NoUpload { return nil, errors.WithStack(errs.UploadNotSupported) } if file.NeedStore() { _, err := file.CacheFullAndWriter(nil, nil) if err != nil { return nil, errors.Wrapf(err, "failed to create temp file") } //file.SetReader(tempFile) //file.SetTmpFile(tempFile) } taskCreator, _ := ctx.Value(conf.UserKey).(*model.User) // taskCreator is nil when convert failed t := &UploadTask{ TaskExtension: task.TaskExtension{ Creator: taskCreator, ApiUrl: common.GetApiUrl(ctx), }, storage: storage, dstDirActualPath: dstDirActualPath, file: file, } t.SetTotalBytes(file.GetSize()) task_group.TransferCoordinator.AddTask(stdpath.Join(storage.GetStorage().MountPath, dstDirActualPath), nil) UploadTaskManager.Add(t) return t, nil } // putDirect put the file and return after finish func putDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer, skipHook ...bool) error { storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { _ = file.Close() return errors.WithMessage(err, "failed get storage") } if storage.Config().NoUpload { _ = file.Close() return errors.WithStack(errs.UploadNotSupported) } if utils.IsBool(skipHook...) { ctx = context.WithValue(ctx, conf.SkipHookKey, struct{}{}) } return op.Put(ctx, storage, dstDirActualPath, file, nil) } func getDirectUploadInfo(ctx context.Context, tool, dstDirPath, dstName string, fileSize int64) (any, error) { storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { return nil, errors.WithMessage(err, "failed get storage") } return op.GetDirectUploadInfo(ctx, tool, storage, dstDirActualPath, dstName, fileSize) } ================================================ FILE: internal/fs/walk.go ================================================ package fs import ( "context" "path" "path/filepath" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" ) // WalkFS traverses filesystem fs starting at name up to depth levels. // // WalkFS will stop when current depth > `depth`. For each visited node, // WalkFS calls walkFn. If a visited file system node is a directory and // walkFn returns path.SkipDir, walkFS will skip traversal of this node. func WalkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn func(reqPath string, info model.Obj) error) error { // This implementation is based on Walk's code in the standard path/path package. walkFnErr := walkFn(name, info) if walkFnErr != nil { if info.IsDir() && walkFnErr == filepath.SkipDir { return nil } return walkFnErr } if !info.IsDir() || depth == 0 { return nil } meta, _ := op.GetNearestMeta(name) // Read directory names. objs, err := List(context.WithValue(ctx, conf.MetaKey, meta), name, &ListArgs{}) if err != nil { return walkFnErr } for _, fileInfo := range objs { filename := path.Join(name, fileInfo.GetName()) if err := WalkFS(ctx, depth-1, filename, fileInfo, walkFn); err != nil { if err == filepath.SkipDir { break } return err } } return nil } ================================================ FILE: internal/fuse/fs.go ================================================ package fuse import "github.com/winfsp/cgofuse/fuse" type Fs struct { RootFolder string fuse.FileSystemBase } func (fs *Fs) Init() { //TODO implement me panic("implement me") } func (fs *Fs) Destroy() { //TODO implement me panic("implement me") } func (fs *Fs) Statfs(path string, stat *fuse.Statfs_t) int { //TODO implement me panic("implement me") } func (fs *Fs) Mknod(path string, mode uint32, dev uint64) int { //TODO implement me panic("implement me") } func (fs *Fs) Mkdir(path string, mode uint32) int { //TODO implement me panic("implement me") } func (fs *Fs) Unlink(path string) int { //TODO implement me panic("implement me") } func (fs *Fs) Rmdir(path string) int { //TODO implement me panic("implement me") } func (fs *Fs) Link(oldpath string, newpath string) int { //TODO implement me panic("implement me") } func (fs *Fs) Symlink(target string, newpath string) int { //TODO implement me panic("implement me") } func (fs *Fs) Readlink(path string) (int, string) { //TODO implement me panic("implement me") } func (fs *Fs) Rename(oldpath string, newpath string) int { //TODO implement me panic("implement me") } func (fs *Fs) Chmod(path string, mode uint32) int { //TODO implement me panic("implement me") } func (fs *Fs) Chown(path string, uid uint32, gid uint32) int { //TODO implement me panic("implement me") } func (fs *Fs) Utimens(path string, tmsp []fuse.Timespec) int { //TODO implement me panic("implement me") } func (fs *Fs) Access(path string, mask uint32) int { //TODO implement me panic("implement me") } func (fs *Fs) Create(path string, flags int, mode uint32) (int, uint64) { //TODO implement me panic("implement me") } func (fs *Fs) Open(path string, flags int) (int, uint64) { //TODO implement me panic("implement me") } func (fs *Fs) Getattr(path string, stat *fuse.Stat_t, fh uint64) int { //TODO implement me panic("implement me") } func (fs *Fs) Truncate(path string, size int64, fh uint64) int { //TODO implement me panic("implement me") } func (fs *Fs) Read(path string, buff []byte, ofst int64, fh uint64) int { //TODO implement me panic("implement me") } func (fs *Fs) Write(path string, buff []byte, ofst int64, fh uint64) int { //TODO implement me panic("implement me") } func (fs *Fs) Flush(path string, fh uint64) int { //TODO implement me panic("implement me") } func (fs *Fs) Release(path string, fh uint64) int { //TODO implement me panic("implement me") } func (fs *Fs) Fsync(path string, datasync bool, fh uint64) int { //TODO implement me panic("implement me") } func (fs *Fs) Opendir(path string) (int, uint64) { //TODO implement me panic("implement me") } func (fs *Fs) Readdir(path string, fill func(name string, stat *fuse.Stat_t, ofst int64) bool, ofst int64, fh uint64) int { //TODO implement me panic("implement me") } func (fs *Fs) Releasedir(path string, fh uint64) int { //TODO implement me panic("implement me") } func (fs *Fs) Fsyncdir(path string, datasync bool, fh uint64) int { //TODO implement me panic("implement me") } func (fs *Fs) Setxattr(path string, name string, value []byte, flags int) int { //TODO implement me panic("implement me") } func (fs *Fs) Getxattr(path string, name string) (int, []byte) { //TODO implement me panic("implement me") } func (fs *Fs) Removexattr(path string, name string) int { //TODO implement me panic("implement me") } func (fs *Fs) Listxattr(path string, fill func(name string) bool) int { //TODO implement me panic("implement me") } var _ fuse.FileSystemInterface = (*Fs)(nil) ================================================ FILE: internal/fuse/mount.go ================================================ package fuse import "github.com/winfsp/cgofuse/fuse" func Mount(mountSrc, mountDst string, opts []string) { fs := &Fs{RootFolder: mountSrc} host := fuse.NewFileSystemHost(fs) go host.Mount(mountDst, opts) } ================================================ FILE: internal/message/http.go ================================================ package message import ( "time" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" ) type Http struct { Received chan string // received messages from web ToSend chan Message // messages to send to web } type Req struct { Message string `json:"message" form:"message"` } func (p *Http) GetHandle(c *gin.Context) { select { case message := <-p.ToSend: common.SuccessResp(c, message) default: common.ErrorStrResp(c, "no message", 404) } } func (p *Http) SendHandle(c *gin.Context) { var req Req if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } select { case p.Received <- req.Message: common.SuccessResp(c) default: common.ErrorStrResp(c, "nowhere needed", 500) } } func (p *Http) Send(message Message) error { select { case p.ToSend <- message: return nil default: return errors.New("send failed") } } func (p *Http) Receive() (string, error) { select { case message := <-p.Received: return message, nil default: return "", errors.New("receive failed") } } func (p *Http) WaitSend(message Message, d int) error { select { case p.ToSend <- message: return nil case <-time.After(time.Duration(d) * time.Second): return errors.New("send timeout") } } func (p *Http) WaitReceive(d int) (string, error) { select { case message := <-p.Received: return message, nil case <-time.After(time.Duration(d) * time.Second): return "", errors.New("receive timeout") } } var HttpInstance = &Http{ Received: make(chan string), ToSend: make(chan Message), } ================================================ FILE: internal/message/message.go ================================================ package message type Message struct { Type string `json:"type"` Content interface{} `json:"content"` } type Messenger interface { Send(Message) error Receive() (string, error) WaitSend(Message, int) error WaitReceive(int) (string, error) } func GetMessenger() Messenger { return HttpInstance } ================================================ FILE: internal/message/ws.go ================================================ package message // TODO websocket implementation ================================================ FILE: internal/model/archive.go ================================================ package model import "time" type ObjTree interface { Obj GetChildren() []ObjTree } type ObjectTree struct { Object Children []ObjTree } func (t *ObjectTree) GetChildren() []ObjTree { return t.Children } type ArchiveMeta interface { GetComment() string // IsEncrypted means if the content of the archive requires a password to access // GetArchiveMeta should return errs.WrongArchivePassword if the meta-info is also encrypted, // and the provided password is empty. IsEncrypted() bool // GetTree directly returns the full folder structure // returns nil if the folder structure should be acquired by calling driver.ArchiveReader.ListArchive GetTree() []ObjTree } type ArchiveMetaInfo struct { Comment string Encrypted bool Tree []ObjTree } func (m *ArchiveMetaInfo) GetComment() string { return m.Comment } func (m *ArchiveMetaInfo) IsEncrypted() bool { return m.Encrypted } func (m *ArchiveMetaInfo) GetTree() []ObjTree { return m.Tree } type ArchiveMetaProvider struct { ArchiveMeta *Sort DriverProviding bool Expiration *time.Duration } ================================================ FILE: internal/model/args.go ================================================ package model import ( "context" "io" "net/http" "time" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type ListArgs struct { ReqPath string S3ShowPlaceholder bool Refresh bool WithStorageDetails bool SkipHook bool } type LinkArgs struct { IP string Header http.Header Type string Redirect bool } type Link struct { URL string `json:"url"` // most common way Header http.Header `json:"header"` // needed header (for url) RangeReader RangeReaderIF `json:"-"` // recommended way if can't use URL Expiration *time.Duration // local cache expire Duration //for accelerating request, use multi-thread downloading Concurrency int `json:"concurrency"` PartSize int `json:"part_size"` ContentLength int64 `json:"content_length"` // 转码视频、缩略图 utils.SyncClosers `json:"-"` // 如果SyncClosers中的资源被关闭后Link将不可用，则此值应为 true RequireReference bool `json:"-"` } type OtherArgs struct { Obj Obj Method string Data interface{} } type FsOtherArgs struct { Path string `json:"path" form:"path"` Method string `json:"method" form:"method"` Data interface{} `json:"data" form:"data"` } type ArchiveArgs struct { Password string LinkArgs } type ArchiveInnerArgs struct { ArchiveArgs InnerPath string } type ArchiveMetaArgs struct { ArchiveArgs Refresh bool } type ArchiveListArgs struct { ArchiveInnerArgs Refresh bool } type ArchiveDecompressArgs struct { ArchiveInnerArgs CacheFull bool PutIntoNewDir bool Overwrite bool } type SharingListArgs struct { Refresh bool Pwd string } type SharingArchiveMetaArgs struct { ArchiveMetaArgs Pwd string } type SharingArchiveListArgs struct { ArchiveListArgs Pwd string } type SharingLinkArgs struct { Pwd string LinkArgs } type RangeReaderIF interface { RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) } type RangeReadCloserIF interface { RangeReaderIF utils.ClosersIF } var _ RangeReadCloserIF = (*RangeReadCloser)(nil) type RangeReadCloser struct { RangeReader RangeReaderIF utils.Closers } func (r *RangeReadCloser) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { rc, err := r.RangeReader.RangeRead(ctx, httpRange) r.Add(rc) return rc, err } ================================================ FILE: internal/model/direct_upload.go ================================================ package model type HttpDirectUploadInfo struct { UploadURL string `json:"upload_url"` // The URL to upload the file ChunkSize int64 `json:"chunk_size"` // The chunk size for uploading, 0 means no chunking required Headers map[string]string `json:"headers,omitempty"` // Optional headers to include in the upload request Method string `json:"method,omitempty"` // HTTP method, default is PUT } ================================================ FILE: internal/model/file.go ================================================ package model import ( "errors" "io" ) // File is basic file level accessing interface type File interface { io.Reader io.ReaderAt io.Seeker } type FileCloser struct { File io.Closer } func (f *FileCloser) Close() error { var errs []error if clr, ok := f.File.(io.Closer); ok { errs = append(errs, clr.Close()) } if f.Closer != nil { errs = append(errs, f.Closer.Close()) } return errors.Join(errs...) } // FileRangeReader 是对 RangeReaderIF 的轻量包装，表明由 RangeReaderIF.RangeRead // 返回的 io.ReadCloser 同时实现了 model.File（即支持 Read/ReadAt/Seek）。 // 只有满足这些才需要使用 FileRangeReader，否则直接使用 RangeReaderIF 即可。 type FileRangeReader struct { RangeReaderIF } ================================================ FILE: internal/model/meta.go ================================================ package model type Meta struct { ID uint `json:"id" gorm:"primaryKey"` Path string `json:"path" gorm:"unique" binding:"required"` Password string `json:"password"` PSub bool `json:"p_sub"` Write bool `json:"write"` WSub bool `json:"w_sub"` Hide string `json:"hide"` HSub bool `json:"h_sub"` Readme string `json:"readme"` RSub bool `json:"r_sub"` Header string `json:"header"` HeaderSub bool `json:"header_sub"` } ================================================ FILE: internal/model/obj.go ================================================ package model import ( "io" "sort" "strings" "time" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/dlclark/regexp2" mapset "github.com/deckarep/golang-set/v2" "github.com/maruel/natural" ) type ObjUnwrap interface { Unwrap() Obj } type Obj interface { GetSize() int64 GetName() string ModTime() time.Time CreateTime() time.Time IsDir() bool GetHash() utils.HashInfo // The internal information of the driver. // If you want to use it, please understand what it means GetID() string GetPath() string } // FileStreamer ->check FileStream for more comments type FileStreamer interface { io.Reader utils.ClosersIF Obj GetMimetype() string NeedStore() bool IsForceStreamUpload() bool GetExist() Obj SetExist(Obj) // for a non-seekable Stream, RangeRead supports peeking some data, and CacheFullAndWriter still works RangeRead(http_range.Range) (io.Reader, error) // for a non-seekable Stream, if Read is called, this function won't work. // caches the full Stream and writes it to writer (if provided, even if the stream is already cached). CacheFullAndWriter(up *UpdateProgress, writer io.Writer) (File, error) // if the Stream is not a File and is not cached, returns nil. GetFile() File } type UpdateProgress func(percentage float64) func UpdateProgressWithRange(inner UpdateProgress, start, end float64) UpdateProgress { return func(p float64) { if p < 0 { p = 0 } if p > 100 { p = 100 } scaled := start + (end-start)*(p/100.0) inner(scaled) } } type URL interface { URL() string } type Thumb interface { Thumb() string } type SetPath interface { SetPath(path string) } type ObjWithProvider interface { GetProvider() string } func SortFiles(objs []Obj, orderBy, orderDirection string) { if orderBy == "" { return } sort.Slice(objs, func(i, j int) bool { switch orderBy { case "name": { c := natural.Less(objs[i].GetName(), objs[j].GetName()) if orderDirection == "desc" { return !c } return c } case "size": { if orderDirection == "desc" { return objs[i].GetSize() >= objs[j].GetSize() } return objs[i].GetSize() <= objs[j].GetSize() } case "modified": if orderDirection == "desc" { return objs[i].ModTime().After(objs[j].ModTime()) } return objs[i].ModTime().Before(objs[j].ModTime()) } return false }) } func ExtractFolder(objs []Obj, extractFolder string) { if extractFolder == "" { return } front := extractFolder == "front" sort.SliceStable(objs, func(i, j int) bool { if objs[i].IsDir() || objs[j].IsDir() { if !objs[i].IsDir() { return !front } if !objs[j].IsDir() { return front } } return false }) } func WrapObjName(objs Obj) Obj { return &ObjWrapName{Name: utils.MappingName(objs.GetName()), Obj: objs} } func WrapObjsName(objs []Obj) { for i := range objs { objs[i] = &ObjWrapName{Name: utils.MappingName(objs[i].GetName()), Obj: objs[i]} } } func UnwrapObjName(obj Obj) Obj { if n, ok := obj.(*ObjWrapName); ok { return n.Obj } return obj } func GetThumb(obj Obj) (thumb string, ok bool) { for { switch o := obj.(type) { case Thumb: return o.Thumb(), true case ObjUnwrap: obj = o.Unwrap() default: return } } } func GetUrl(obj Obj) (url string, ok bool) { for { switch o := obj.(type) { case URL: return o.URL(), true case ObjUnwrap: obj = o.Unwrap() default: return } } } func GetProvider(obj Obj) (string, bool) { for { switch o := obj.(type) { case ObjWithProvider: return o.GetProvider(), true case ObjUnwrap: obj = o.Unwrap() default: return "unknown", false } } } // Merge func NewObjMerge() *ObjMerge { return &ObjMerge{ set: mapset.NewSet[string](), } } type ObjMerge struct { regs []*regexp2.Regexp set mapset.Set[string] } func (om *ObjMerge) Merge(objs []Obj, objs_ ...Obj) []Obj { newObjs := make([]Obj, 0, len(objs)+len(objs_)) newObjs = om.insertObjs(om.insertObjs(newObjs, objs...), objs_...) return newObjs } func (om *ObjMerge) insertObjs(objs []Obj, objs_ ...Obj) []Obj { for _, obj := range objs_ { if om.clickObj(obj) { objs = append(objs, obj) } } return objs } func (om *ObjMerge) clickObj(obj Obj) bool { for _, reg := range om.regs { if isMatch, _ := reg.MatchString(obj.GetName()); isMatch { return false } } return om.set.Add(obj.GetName()) } func (om *ObjMerge) InitHideReg(hides string) { rs := strings.Split(hides, "\n") om.regs = make([]*regexp2.Regexp, 0, len(rs)) for _, r := range rs { om.regs = append(om.regs, regexp2.MustCompile(r, regexp2.None)) } } func (om *ObjMerge) Reset() { om.set.Clear() } type ObjMask uint8 func (m ObjMask) GetObjMask() ObjMask { return m } const ( Virtual ObjMask = 1 << iota NoRename NoRemove NoMove NoCopy NoWrite Temp ) const ( Locked = NoRename | NoRemove | NoMove ReadOnly = Locked | NoWrite // NoRename | NoDelete | NoMove | NoWrite ) type ObjWrapMask struct { Obj Mask ObjMask } func (m *ObjWrapMask) Unwrap() Obj { return m.Obj } func (m *ObjWrapMask) GetObjMask() ObjMask { return m.Mask } func GetObjMask(obj Obj) ObjMask { for { switch o := obj.(type) { case interface{ GetObjMask() ObjMask }: return o.GetObjMask() case ObjUnwrap: obj = o.Unwrap() default: return 0 } } } func ObjHasMask(obj Obj, mask ObjMask) bool { return GetObjMask(obj)&mask != 0 } ================================================ FILE: internal/model/object.go ================================================ package model import ( "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type ObjWrapName struct { Name string Obj } func (o *ObjWrapName) Unwrap() Obj { return o.Obj } func (o *ObjWrapName) GetName() string { return o.Name } type Object struct { ID string Path string Name string Size int64 Modified time.Time Ctime time.Time // file create time IsFolder bool HashInfo utils.HashInfo Mask ObjMask } func (o *Object) GetName() string { return o.Name } func (o *Object) GetSize() int64 { return o.Size } func (o *Object) ModTime() time.Time { return o.Modified } func (o *Object) CreateTime() time.Time { if o.Ctime.IsZero() { return o.ModTime() } return o.Ctime } func (o *Object) IsDir() bool { return o.IsFolder } func (o *Object) GetID() string { return o.ID } func (o *Object) GetPath() string { return o.Path } func (o *Object) SetPath(path string) { o.Path = path } func (o *Object) GetHash() utils.HashInfo { return o.HashInfo } func (o *Object) GetObjMask() ObjMask { return o.Mask } type Thumbnail struct { Thumbnail string } type Url struct { Url string } func (w Url) URL() string { return w.Url } func (t Thumbnail) Thumb() string { return t.Thumbnail } type ObjThumb struct { Object Thumbnail } type ObjectURL struct { Object Url } type ObjThumbURL struct { Object Thumbnail Url } type Provider struct { Provider string } func (p Provider) GetProvider() string { return p.Provider } type ObjectProvider struct { Object Provider } ================================================ FILE: internal/model/req.go ================================================ package model type PageReq struct { Page int `json:"page" form:"page"` PerPage int `json:"per_page" form:"per_page"` } const MaxUint = ^uint(0) const MinUint = 0 const MaxInt = int(MaxUint >> 1) const MinInt = -MaxInt - 1 func (p *PageReq) Validate() { if p.Page < 1 { p.Page = 1 } if p.PerPage < 1 { p.PerPage = MaxInt } } ================================================ FILE: internal/model/search.go ================================================ package model import ( "fmt" "time" ) type IndexProgress struct { ObjCount uint64 `json:"obj_count"` IsDone bool `json:"is_done"` LastDoneTime *time.Time `json:"last_done_time"` Error string `json:"error"` } type SearchReq struct { Parent string `json:"parent"` Keywords string `json:"keywords"` // 0 for all, 1 for dir, 2 for file Scope int `json:"scope"` PageReq } type SearchNode struct { Parent string `json:"parent" gorm:"index"` Name string `json:"name"` IsDir bool `json:"is_dir"` Size int64 `json:"size"` } func (p *SearchReq) Validate() error { if p.Page < 1 { return fmt.Errorf("page can't < 1") } if p.PerPage < 1 { return fmt.Errorf("per_page can't < 1") } return nil } func (s *SearchNode) Type() string { return "SearchNode" } ================================================ FILE: internal/model/setting.go ================================================ package model const ( SINGLE = iota SITE STYLE PREVIEW GLOBAL OFFLINE_DOWNLOAD INDEX SSO LDAP S3 FTP TRAFFIC ) const ( PUBLIC = iota PRIVATE READONLY DEPRECATED ) type SettingItem struct { Key string `json:"key" gorm:"primaryKey" binding:"required"` // unique key Value string `json:"value"` // value MigrationValue string `json:"-" gorm:"-:all"` // deprecated value Help string `json:"help"` // help message Type string `json:"type"` // string, number, bool, select Options string `json:"options"` // values for select Group int `json:"group"` // use to group setting in frontend Flag int `json:"flag"` // 0 = public, 1 = private, 2 = readonly, 3 = deprecated, etc. Index uint `json:"index"` } func (s SettingItem) IsDeprecated() bool { return s.Flag == DEPRECATED } ================================================ FILE: internal/model/sharing.go ================================================ package model import "time" type SharingDB struct { ID string `json:"id" gorm:"type:char(12);primaryKey"` FilesRaw string `json:"-" gorm:"type:text"` Expires *time.Time `json:"expires"` Pwd string `json:"pwd"` Accessed int `json:"accessed"` MaxAccessed int `json:"max_accessed"` CreatorId uint `json:"-"` Disabled bool `json:"disabled"` Remark string `json:"remark"` Readme string `json:"readme" gorm:"type:text"` Header string `json:"header" gorm:"type:text"` Sort } type Sharing struct { *SharingDB Files []string `json:"files"` Creator *User `json:"-"` } func (s *Sharing) Valid() bool { if s.Disabled { return false } if s.MaxAccessed > 0 && s.Accessed >= s.MaxAccessed { return false } if len(s.Files) == 0 { return false } if s.Creator == nil || !s.Creator.CanShare() { return false } if s.Expires != nil && !s.Expires.IsZero() && s.Expires.Before(time.Now()) { return false } return true } func (s *Sharing) Verify(pwd string) bool { return s.Pwd == "" || s.Pwd == pwd } ================================================ FILE: internal/model/sshkey.go ================================================ package model import ( "golang.org/x/crypto/ssh" "time" ) type SSHPublicKey struct { ID uint `json:"id" gorm:"primaryKey"` UserId uint `json:"-"` Title string `json:"title"` Fingerprint string `json:"fingerprint"` KeyStr string `gorm:"type:text" json:"-"` AddedTime time.Time `json:"added_time"` LastUsedTime time.Time `json:"last_used_time"` } func (k *SSHPublicKey) GetKey() (ssh.PublicKey, error) { pubKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(k.KeyStr)) if err != nil { return nil, err } return pubKey, nil } func (k *SSHPublicKey) UpdateLastUsedTime() { k.LastUsedTime = time.Now() } ================================================ FILE: internal/model/storage.go ================================================ package model import ( "encoding/json" "time" ) type Storage struct { ID uint `json:"id" gorm:"primaryKey"` // unique key MountPath string `json:"mount_path" gorm:"unique" binding:"required"` // must be standardized Order int `json:"order"` // use to sort Driver string `json:"driver"` // driver used CacheExpiration int `json:"cache_expiration"` // cache expire time CustomCachePolicies string `json:"custom_cache_policies" gorm:"type:text"` Status string `json:"status"` Addition string `json:"addition" gorm:"type:text"` // Additional information, defined in the corresponding driver Remark string `json:"remark"` Modified time.Time `json:"modified"` Disabled bool `json:"disabled"` // if disabled DisableIndex bool `json:"disable_index"` EnableSign bool `json:"enable_sign"` Sort Proxy } type Sort struct { OrderBy string `json:"order_by"` OrderDirection string `json:"order_direction"` ExtractFolder string `json:"extract_folder"` } type Proxy struct { WebProxy bool `json:"web_proxy"` WebdavPolicy string `json:"webdav_policy"` ProxyRange bool `json:"proxy_range"` DownProxyURL string `json:"down_proxy_url"` // Disable sign for DownProxyURL DisableProxySign bool `json:"disable_proxy_sign"` } func (s *Storage) GetStorage() *Storage { return s } func (s *Storage) SetStorage(storage Storage) { *s = storage } func (s *Storage) SetStatus(status string) { s.Status = status } func (p Proxy) Webdav302() bool { return p.WebdavPolicy == "302_redirect" } func (p Proxy) WebdavProxyURL() bool { return p.WebdavPolicy == "use_proxy_url" } type DiskUsage struct { TotalSpace int64 UsedSpace int64 } func (d DiskUsage) FreeSpace() int64 { return d.TotalSpace - d.UsedSpace } func (d DiskUsage) MarshalJSON() ([]byte, error) { return json.Marshal(map[string]interface{}{ "total_space": d.TotalSpace, "used_space": d.UsedSpace, "free_space": d.FreeSpace(), }) } type StorageDetails struct { DiskUsage } type ObjWithStorageDetails interface { GetStorageDetails() *StorageDetails } type ObjStorageDetails struct { Obj *StorageDetails } func (o *ObjStorageDetails) Unwrap() Obj { return o.Obj } func (o *ObjStorageDetails) GetStorageDetails() *StorageDetails { return o.StorageDetails } func GetStorageDetails(obj Obj) (*StorageDetails, bool) { if obj, ok := obj.(ObjWithStorageDetails); ok { return obj.GetStorageDetails(), true } if unwrap, ok := obj.(ObjUnwrap); ok { return GetStorageDetails(unwrap.Unwrap()) } return nil, false } ================================================ FILE: internal/model/task.go ================================================ package model type TaskItem struct { Key string `json:"key"` PersistData string `gorm:"type:text" json:"persist_data"` } ================================================ FILE: internal/model/user.go ================================================ package model import ( "encoding/binary" "encoding/json" "fmt" "time" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/OpenListTeam/go-cache" "github.com/go-webauthn/webauthn/webauthn" "github.com/pkg/errors" ) const ( GENERAL = iota GUEST // only one exists ADMIN ) const ( StaticHashSalt = "https://github.com/alist-org/alist" InvalidUsernameOrPassword = "Invalid username or password" Invalid2FACode = "Invalid 2FA code" TooManyAttempts = "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later." GuestCannotUpdateProfile = "Guest user can not update profile" GuestCannotGenerate2FA = "Guest user can not generate 2FA code" ) var LoginCache = cache.NewMemCache[int]() var ( DefaultLockDuration = time.Minute * 5 DefaultMaxAuthRetries = 5 ) type User struct { ID uint `json:"id" gorm:"primaryKey"` // unique key Username string `json:"username" gorm:"unique" binding:"required"` // username PwdHash string `json:"-"` // password hash PwdTS int64 `json:"-"` // password timestamp Salt string `json:"-"` // unique salt Password string `json:"password"` // password BasePath string `json:"base_path"` // base path Role int `json:"role"` // user's role Disabled bool `json:"disabled"` // Determine permissions by bit // 0: can see hidden files // 1: can access without password // 2: can add offline download tasks // 3: can mkdir and upload // 4: can rename // 5: can move // 6: can copy // 7: can remove // 8: webdav read // 9: webdav write // 10: ftp/sftp login and read // 11: ftp/sftp write // 12: can read archives // 13: can decompress archives // 14: can share Permission int32 `json:"permission"` OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform Authn string `gorm:"type:text" json:"-"` AllowLdap bool `json:"allow_ldap" gorm:"default:true"` } func (u *User) IsGuest() bool { return u.Role == GUEST } func (u *User) IsAdmin() bool { return u.Role == ADMIN } func (u *User) ValidateRawPassword(password string) error { return u.ValidatePwdStaticHash(StaticHash(password)) } func (u *User) ValidatePwdStaticHash(pwdStaticHash string) error { if pwdStaticHash == "" { return errors.WithStack(errs.EmptyPassword) } if u.PwdHash != HashPwd(pwdStaticHash, u.Salt) { return errors.WithStack(errs.WrongPassword) } return nil } func (u *User) SetPassword(pwd string) *User { u.Salt = random.String(16) u.PwdHash = TwoHashPwd(pwd, u.Salt) u.PwdTS = time.Now().Unix() return u } func CanSeeHides(permission int32) bool { return permission&1 == 1 } func (u *User) CanSeeHides() bool { return CanSeeHides(u.Permission) } func CanAccessWithoutPassword(permission int32) bool { return (permission>>1)&1 == 1 } func (u *User) CanAccessWithoutPassword() bool { return CanAccessWithoutPassword(u.Permission) } func CanAddOfflineDownloadTasks(permission int32) bool { return (permission>>2)&1 == 1 } func (u *User) CanAddOfflineDownloadTasks() bool { return CanAddOfflineDownloadTasks(u.Permission) } func CanWrite(permission int32) bool { return (permission>>3)&1 == 1 } func (u *User) CanWrite() bool { return CanWrite(u.Permission) } func CanRename(permission int32) bool { return (permission>>4)&1 == 1 } func (u *User) CanRename() bool { return CanRename(u.Permission) } func CanMove(permission int32) bool { return (permission>>5)&1 == 1 } func (u *User) CanMove() bool { return CanMove(u.Permission) } func CanCopy(permission int32) bool { return (permission>>6)&1 == 1 } func (u *User) CanCopy() bool { return CanCopy(u.Permission) } func CanRemove(permission int32) bool { return (permission>>7)&1 == 1 } func (u *User) CanRemove() bool { return CanRemove(u.Permission) } func CanWebdavRead(permission int32) bool { return (permission>>8)&1 == 1 } func (u *User) CanWebdavRead() bool { return CanWebdavRead(u.Permission) } func CanWebdavManage(permission int32) bool { return (permission>>9)&1 == 1 } func (u *User) CanWebdavManage() bool { return CanWebdavManage(u.Permission) } func CanFTPAccess(permission int32) bool { return (permission>>10)&1 == 1 } func (u *User) CanFTPAccess() bool { return CanFTPAccess(u.Permission) } func CanFTPManage(permission int32) bool { return (permission>>11)&1 == 1 } func (u *User) CanFTPManage() bool { return CanFTPManage(u.Permission) } func CanReadArchives(permission int32) bool { return (permission>>12)&1 == 1 } func (u *User) CanReadArchives() bool { return CanReadArchives(u.Permission) } func CanDecompress(permission int32) bool { return (permission>>13)&1 == 1 } func (u *User) CanDecompress() bool { return CanDecompress(u.Permission) } func CanShare(permission int32) bool { return (permission>>14)&1 == 1 } func (u *User) CanShare() bool { return CanShare(u.Permission) } func (u *User) JoinPath(reqPath string) (string, error) { return utils.JoinBasePath(u.BasePath, reqPath) } func StaticHash(password string) string { return utils.HashData(utils.SHA256, []byte(fmt.Sprintf("%s-%s", password, StaticHashSalt))) } func HashPwd(static string, salt string) string { return utils.HashData(utils.SHA256, []byte(fmt.Sprintf("%s-%s", static, salt))) } func TwoHashPwd(password string, salt string) string { return HashPwd(StaticHash(password), salt) } func (u *User) WebAuthnID() []byte { bs := make([]byte, 8) binary.LittleEndian.PutUint64(bs, uint64(u.ID)) return bs } func (u *User) WebAuthnName() string { return u.Username } func (u *User) WebAuthnDisplayName() string { return u.Username } func (u *User) WebAuthnCredentials() []webauthn.Credential { var res []webauthn.Credential err := json.Unmarshal([]byte(u.Authn), &res) if err != nil { fmt.Println(err) } return res } func (u *User) WebAuthnIcon() string { return "https://res.oplist.org/logo/logo.svg" } ================================================ FILE: internal/net/oss.go ================================================ package net import "github.com/aliyun/aliyun-oss-go-sdk/oss" func NewOSSClient(endpoint, accessKeyID, accessKeySecret string, options ...oss.ClientOption) (*oss.Client, error) { clientOptions := []oss.ClientOption{oss.HTTPClient(NewHttpClient())} clientOptions = append(clientOptions, options...) return oss.New(endpoint, accessKeyID, accessKeySecret, clientOptions...) } ================================================ FILE: internal/net/oss_test.go ================================================ package net import ( "net/http" "net/url" "testing" "github.com/OpenListTeam/OpenList/v4/internal/conf" ) func TestNewOSSClientUsesEnvironmentHTTPSProxy(t *testing.T) { oldConf := conf.Conf conf.Conf = conf.DefaultConfig("data") defer func() { conf.Conf = oldConf }() t.Setenv("HTTP_PROXY", "") t.Setenv("http_proxy", "") t.Setenv("HTTPS_PROXY", "http://127.0.0.1:7890") t.Setenv("https_proxy", "") t.Setenv("NO_PROXY", "") t.Setenv("no_proxy", "") client, err := NewOSSClient("https://oss-cn-hangzhou.aliyuncs.com", "test-access-key", "test-access-secret") if err != nil { t.Fatalf("expected no error, got %v", err) } if client.HTTPClient == nil { t.Fatal("expected OSS client to use a custom HTTP client") } transport, ok := client.HTTPClient.Transport.(*http.Transport) if !ok { t.Fatalf("expected *http.Transport, got %T", client.HTTPClient.Transport) } if transport.Proxy == nil { t.Fatal("expected proxy function to be configured") } req := &http.Request{URL: &url.URL{Scheme: "https", Host: "oss-cn-hangzhou.aliyuncs.com"}} proxyURL, err := transport.Proxy(req) if err != nil { t.Fatalf("expected no proxy lookup error, got %v", err) } if proxyURL == nil { t.Fatal("expected HTTPS proxy to be used") } if got, want := proxyURL.String(), "http://127.0.0.1:7890"; got != want { t.Fatalf("expected proxy %q, got %q", want, got) } } ================================================ FILE: internal/net/request.go ================================================ package net import ( "context" "errors" "fmt" "io" "net/http" "strconv" "strings" "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/rclone/rclone/lib/mmap" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/aws/aws-sdk-go/aws/awsutil" log "github.com/sirupsen/logrus" ) // DefaultDownloadPartSize is the default range of bytes to get at a time when // using Download(). const DefaultDownloadPartSize = utils.MB * 8 // DefaultDownloadConcurrency is the default number of goroutines to spin up // when using Download(). const DefaultDownloadConcurrency = 2 // DefaultPartBodyMaxRetries is the default number of retries to make when a part fails to download. const DefaultPartBodyMaxRetries = 3 var DefaultConcurrencyLimit *ConcurrencyLimit type Downloader struct { PartSize int // PartBodyMaxRetries is the number of retry attempts to make for failed part downloads. PartBodyMaxRetries int // The number of goroutines to spin up in parallel when sending parts. // If this is set to zero, the DefaultDownloadConcurrency value will be used. // // Concurrency of 1 will download the parts sequentially. Concurrency int //RequestParam HttpRequestParams HttpClient HttpRequestFunc *ConcurrencyLimit } type HttpRequestFunc func(ctx context.Context, params *HttpRequestParams) (*http.Response, error) func NewDownloader(options ...func(*Downloader)) *Downloader { d := &Downloader{ //允许不设置的选项 PartBodyMaxRetries: DefaultPartBodyMaxRetries, ConcurrencyLimit: DefaultConcurrencyLimit, } for _, option := range options { option(d) } return d } // Download The Downloader makes multi-thread http requests to remote URL, each chunk(except last one) has PartSize, // cache some data, then return Reader with assembled data // Supports range, do not support unknown FileSize, and will fail if FileSize is incorrect // memory usage is at about Concurrency*PartSize, use this wisely func (d Downloader) Download(ctx context.Context, p *HttpRequestParams) (readCloser io.ReadCloser, err error) { var finalP HttpRequestParams awsutil.Copy(&finalP, p) if finalP.Range.Length < 0 || finalP.Range.Start+finalP.Range.Length > finalP.Size { finalP.Range.Length = finalP.Size - finalP.Range.Start } impl := downloader{params: &finalP, cfg: d, ctx: ctx} // Ensures we don't need nil checks later on // 必需的选项 if impl.cfg.Concurrency == 0 { impl.cfg.Concurrency = DefaultDownloadConcurrency } if impl.cfg.PartSize == 0 { impl.cfg.PartSize = DefaultDownloadPartSize } if conf.MaxBufferLimit > 0 && impl.cfg.PartSize > conf.MaxBufferLimit { impl.cfg.PartSize = conf.MaxBufferLimit } if impl.cfg.HttpClient == nil { impl.cfg.HttpClient = DefaultHttpRequestFunc } return impl.download() } // downloader is the implementation structure used internally by Downloader. type downloader struct { ctx context.Context cancel context.CancelCauseFunc cfg Downloader params *HttpRequestParams //http request params chunkChannel chan chunk //chunk chanel //wg sync.WaitGroup m sync.Mutex nextChunk int //next chunk id bufs []*Buf written int64 //total bytes of file downloaded from remote err error concurrency int //剩余的并发数，递减。到0时停止并发 maxPart int //有多少个分片 pos int64 maxPos int64 m2 sync.Mutex readingID int // 正在被读取的id } type ConcurrencyLimit struct { _m sync.Mutex Limit int // 需要大于0 } var ErrExceedMaxConcurrency = HttpStatusCodeError(http.StatusTooManyRequests) func (l *ConcurrencyLimit) sub() error { l._m.Lock() defer l._m.Unlock() if l.Limit-1 < 0 { return ErrExceedMaxConcurrency } l.Limit-- // log.Debugf("ConcurrencyLimit.sub: %d", l.Limit) return nil } func (l *ConcurrencyLimit) add() { l._m.Lock() defer l._m.Unlock() l.Limit++ // log.Debugf("ConcurrencyLimit.add: %d", l.Limit) } // 检测是否超过限制 func (d *downloader) concurrencyCheck() error { if d.cfg.ConcurrencyLimit != nil { return d.cfg.ConcurrencyLimit.sub() } return nil } func (d *downloader) concurrencyFinish() { if d.cfg.ConcurrencyLimit != nil { d.cfg.ConcurrencyLimit.add() } } // download performs the implementation of the object download across ranged GETs. func (d *downloader) download() (io.ReadCloser, error) { if err := d.concurrencyCheck(); err != nil { return nil, err } maxPart := 1 if d.params.Range.Length > int64(d.cfg.PartSize) { maxPart = int((d.params.Range.Length + int64(d.cfg.PartSize) - 1) / int64(d.cfg.PartSize)) } if maxPart < d.cfg.Concurrency { d.cfg.Concurrency = maxPart } log.Debugf("cfgConcurrency:%d", d.cfg.Concurrency) if maxPart == 1 { resp, err := d.cfg.HttpClient(d.ctx, d.params) if err != nil { d.concurrencyFinish() return nil, err } closeFunc := resp.Body.Close resp.Body = utils.NewReadCloser(resp.Body, func() error { d.m.Lock() defer d.m.Unlock() var err error if closeFunc != nil { d.concurrencyFinish() err = closeFunc() closeFunc = nil } return err }) return resp.Body, nil } d.ctx, d.cancel = context.WithCancelCause(d.ctx) // workers d.chunkChannel = make(chan chunk, d.cfg.Concurrency) d.maxPart = maxPart d.pos = d.params.Range.Start d.maxPos = d.params.Range.Start + d.params.Range.Length d.concurrency = d.cfg.Concurrency _ = d.sendChunkTask(true) var rc io.ReadCloser = NewMultiReadCloser(d.bufs[0], d.interrupt, d.finishBuf) // Return error return rc, d.err } func (d *downloader) sendChunkTask(newConcurrency bool) error { d.m.Lock() defer d.m.Unlock() isNewBuf := d.concurrency > 0 if newConcurrency { if d.concurrency <= 0 { return nil } if d.nextChunk > 0 { // 第一个不检查，因为已经检查过了 if err := d.concurrencyCheck(); err != nil { return err } } d.concurrency-- go d.downloadPart() } var buf *Buf if isNewBuf { buf = NewBuf(d.ctx, d.cfg.PartSize) d.bufs = append(d.bufs, buf) } else { buf = d.getBuf(d.nextChunk) } if d.pos < d.maxPos { finalSize := int64(d.cfg.PartSize) switch d.nextChunk { case 0: // 最小分片在前面有助视频播放？ firstSize := d.params.Range.Length % finalSize if firstSize > 0 { minSize := finalSize / 2 if firstSize < minSize { // 最小分片太小就调整到一半 finalSize = minSize } else { finalSize = firstSize } } case 1: firstSize := d.params.Range.Length % finalSize minSize := finalSize / 2 if firstSize > 0 && firstSize < minSize { finalSize += firstSize - minSize } } err := buf.Reset(int(finalSize)) if err != nil { return err } ch := chunk{ start: d.pos, size: finalSize, id: d.nextChunk, buf: buf, newConcurrency: newConcurrency, } d.pos += finalSize d.nextChunk++ d.chunkChannel <- ch return nil } return nil } // when the final reader Close, we interrupt func (d *downloader) interrupt() error { d.m.Lock() defer d.m.Unlock() err := d.err if err == nil && d.written != d.params.Range.Length { log.Debugf("Downloader interrupt before finish") err := fmt.Errorf("interrupted") d.err = err } close(d.chunkChannel) if d.bufs != nil { d.cancel(err) for _, buf := range d.bufs { buf.Close() } d.bufs = nil if d.concurrency > 0 { d.concurrency = -d.concurrency } log.Debugf("maxConcurrency:%d", d.cfg.Concurrency+d.concurrency) } return err } func (d *downloader) getBuf(id int) (b *Buf) { return d.bufs[id%len(d.bufs)] } func (d *downloader) finishBuf(id int) (isLast bool, nextBuf *Buf) { id++ if id >= d.maxPart { return true, nil } _ = d.sendChunkTask(false) d.readingID = id return false, d.getBuf(id) } // downloadPart is an individual goroutine worker reading from the ch channel // and performing Http request on the data with a given byte range. func (d *downloader) downloadPart() { defer d.concurrencyFinish() for { select { case <-d.ctx.Done(): return case c, ok := <-d.chunkChannel: if !ok { return } if d.getErr() != nil { // Drain the channel if there is an error, to prevent deadlocking // of download producer. return } if err := d.downloadChunk(&c); err != nil { if err == errCancelConcurrency { return } if err == context.Canceled { if e := context.Cause(d.ctx); e != nil { err = e } } d.setErr(err) d.cancel(err) return } } } } // downloadChunk downloads the chunk func (d *downloader) downloadChunk(ch *chunk) error { log.Debugf("start chunk_%d, %+v", ch.id, ch) params := d.getParamsFromChunk(ch) var n int64 var err error for retry := 0; retry <= d.cfg.PartBodyMaxRetries; retry++ { if d.getErr() != nil { return nil } n, err = d.tryDownloadChunk(params, ch) if err == nil { d.incrWritten(n) log.Debugf("chunk_%d downloaded", ch.id) break } if d.getErr() != nil { return nil } if utils.IsCanceled(d.ctx) { return d.ctx.Err() } // Check if the returned error is an errNeedRetry. // If this occurs we unwrap the err to set the underlying error // and attempt any remaining retries. if e, ok := err.(*errNeedRetry); ok { err = e.Unwrap() if n > 0 { // 测试：下载时 断开openlist向云盘发起的下载连接 // 校验：下载完后校验文件哈希值 一致 d.incrWritten(n) ch.start += n ch.size -= n params.Range.Start = ch.start params.Range.Length = ch.size } log.Warnf("err chunk_%d, object part download error %s, retrying attempt %d. %v", ch.id, params.URL, retry, err) } else if err == errInfiniteRetry { retry-- continue } else { break } } return err } var errCancelConcurrency = errors.New("cancel concurrency") var errInfiniteRetry = errors.New("infinite retry") func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int64, error) { resp, err := d.cfg.HttpClient(d.ctx, params) if err != nil { statusCode, ok := errs.UnwrapOrSelf(err).(HttpStatusCodeError) if !ok { return 0, err } if statusCode == http.StatusRequestedRangeNotSatisfiable { return 0, err } if ch.id == 0 { //第1个任务 有限的重试，超过重试就会结束请求 switch statusCode { default: return 0, err case http.StatusTooManyRequests: case http.StatusBadGateway: case http.StatusServiceUnavailable: case http.StatusGatewayTimeout: } <-time.After(time.Millisecond * 200) return 0, &errNeedRetry{err: err} } // 来到这 说明第1个分片下载 连接成功了 // 后续分片下载出错都当超载处理 log.Debugf("err chunk_%d, try downloading:%v", ch.id, err) d.m.Lock() isCancelConcurrency := ch.newConcurrency if d.concurrency > 0 { // 取消剩余的并发任务 // 用于计算实际的并发数 d.concurrency = -d.concurrency isCancelConcurrency = true } if isCancelConcurrency { d.concurrency-- d.chunkChannel <- *ch d.m.Unlock() return 0, errCancelConcurrency } d.m.Unlock() if ch.id != d.readingID { //正在被读取的优先重试 d.m2.Lock() defer d.m2.Unlock() <-time.After(time.Millisecond * 200) } return 0, errInfiniteRetry } defer resp.Body.Close() //only check file size on the first task if ch.id == 0 { err = d.checkTotalBytes(resp) if err != nil { return 0, err } } _ = d.sendChunkTask(true) n, err := utils.CopyWithBuffer(ch.buf, resp.Body) if err != nil { return n, &errNeedRetry{err: err} } if n != ch.size { err = fmt.Errorf("chunk download size incorrect, expected=%d, got=%d", ch.size, n) return n, &errNeedRetry{err: err} } return n, nil } func (d *downloader) getParamsFromChunk(ch *chunk) *HttpRequestParams { var params HttpRequestParams awsutil.Copy(¶ms, d.params) // Get the getBuf byte range of data params.Range = http_range.Range{Start: ch.start, Length: ch.size} return ¶ms } func (d *downloader) checkTotalBytes(resp *http.Response) error { var err error totalBytes := int64(-1) contentRange := resp.Header.Get("Content-Range") if len(contentRange) == 0 { // ContentRange is nil when the full file contents is provided, and // is not chunked. Use ContentLength instead. if resp.ContentLength > 0 { totalBytes = resp.ContentLength } } else { parts := strings.Split(contentRange, "/") total := int64(-1) // Checking for whether a numbered total exists // If one does not exist, we will assume the total to be -1, undefined, // and sequentially download each chunk until hitting a 416 error totalStr := parts[len(parts)-1] if totalStr != "*" { total, err = strconv.ParseInt(totalStr, 10, 64) if err != nil { err = fmt.Errorf("failed extracting file size") } } else { err = fmt.Errorf("file size unknown") } totalBytes = total } if totalBytes != d.params.Size && err == nil { err = fmt.Errorf("expect file size=%d unmatch remote report size=%d, need refresh cache", d.params.Size, totalBytes) } if err != nil { // _ = d.interrupt() d.setErr(err) d.cancel(err) } return err } func (d *downloader) incrWritten(n int64) { d.m.Lock() defer d.m.Unlock() d.written += n } // getErr is a thread-safe getter for the error object func (d *downloader) getErr() error { d.m.Lock() defer d.m.Unlock() return d.err } // setErr is a thread-safe setter for the error object func (d *downloader) setErr(e error) { d.m.Lock() defer d.m.Unlock() d.err = e } // Chunk represents a single chunk of data to write by the worker routine. // This structure also implements an io.SectionReader style interface for // io.WriterAt, effectively making it an io.SectionWriter (which does not // exist). type chunk struct { start int64 size int64 buf *Buf id int newConcurrency bool } func DefaultHttpRequestFunc(ctx context.Context, params *HttpRequestParams) (*http.Response, error) { header := http_range.ApplyRangeToHttpHeader(params.Range, params.HeaderRef) return RequestHttp(ctx, "GET", header, params.URL) } func GetRangeReaderHttpRequestFunc(rangeReader model.RangeReaderIF) HttpRequestFunc { return func(ctx context.Context, params *HttpRequestParams) (*http.Response, error) { rc, err := rangeReader.RangeRead(ctx, params.Range) if err != nil { return nil, err } return &http.Response{ StatusCode: http.StatusPartialContent, Status: http.StatusText(http.StatusPartialContent), Body: rc, Header: http.Header{ "Content-Range": {params.Range.ContentRange(params.Size)}, }, ContentLength: params.Range.Length, }, nil } } type HttpRequestParams struct { URL string //only want data within this range Range http_range.Range HeaderRef http.Header //total file size Size int64 } type errNeedRetry struct { err error } func (e *errNeedRetry) Error() string { return e.err.Error() } func (e *errNeedRetry) Unwrap() error { return e.err } type MultiReadCloser struct { cfg *cfg closer closerFunc finish finishBufFUnc } type cfg struct { rPos int //current reader position, start from 0 curBuf *Buf } type closerFunc func() error type finishBufFUnc func(id int) (isLast bool, buf *Buf) // NewMultiReadCloser to save memory, we re-use limited Buf, and feed data to Read() func NewMultiReadCloser(buf *Buf, c closerFunc, fb finishBufFUnc) *MultiReadCloser { return &MultiReadCloser{closer: c, finish: fb, cfg: &cfg{curBuf: buf}} } func (mr MultiReadCloser) Read(p []byte) (n int, err error) { if mr.cfg.curBuf == nil { return 0, io.EOF } n, err = mr.cfg.curBuf.Read(p) //log.Debugf("read_%d read current buffer, n=%d ,err=%+v", mr.cfg.rPos, n, err) if err == io.EOF { log.Debugf("read_%d finished current buffer", mr.cfg.rPos) isLast, next := mr.finish(mr.cfg.rPos) if isLast { return n, io.EOF } mr.cfg.curBuf = next mr.cfg.rPos++ return n, nil } if err == context.Canceled { if e := context.Cause(mr.cfg.curBuf.ctx); e != nil { err = e } } return n, err } func (mr MultiReadCloser) Close() error { return mr.closer() } type Buf struct { size int //expected size ctx context.Context offR int offW int rw sync.Mutex buf []byte mmap bool readSignal chan struct{} readPending bool } // NewBuf is a buffer that can have 1 read & 1 write at the same time. // when read is faster write, immediately feed data to read after written func NewBuf(ctx context.Context, maxSize int) *Buf { br := &Buf{ ctx: ctx, size: maxSize, readSignal: make(chan struct{}, 1), } if conf.MmapThreshold > 0 && maxSize >= conf.MmapThreshold { m, err := mmap.Alloc(maxSize) if err == nil { br.buf = m br.mmap = true return br } } br.buf = make([]byte, maxSize) return br } func (br *Buf) Reset(size int) error { br.rw.Lock() defer br.rw.Unlock() if br.buf == nil { return io.ErrClosedPipe } if size > cap(br.buf) { return fmt.Errorf("reset size %d exceeds max size %d", size, cap(br.buf)) } br.size = size br.offR = 0 br.offW = 0 return nil } func (br *Buf) Read(p []byte) (int, error) { if err := br.ctx.Err(); err != nil { return 0, err } if len(p) == 0 { return 0, nil } if br.offR >= br.size { return 0, io.EOF } for { br.rw.Lock() if br.buf == nil { br.rw.Unlock() return 0, io.ErrClosedPipe } if br.offW < br.offR { br.rw.Unlock() return 0, io.ErrUnexpectedEOF } if br.offW == br.offR { br.readPending = true br.rw.Unlock() select { case <-br.ctx.Done(): return 0, br.ctx.Err() case _, ok := <-br.readSignal: if !ok { return 0, io.ErrClosedPipe } continue } } n := copy(p, br.buf[br.offR:br.offW]) br.offR += n br.rw.Unlock() if n < len(p) && br.offR >= br.size { return n, io.EOF } return n, nil } } func (br *Buf) Write(p []byte) (int, error) { if err := br.ctx.Err(); err != nil { return 0, err } if len(p) == 0 { return 0, nil } br.rw.Lock() defer br.rw.Unlock() if br.buf == nil { return 0, io.ErrClosedPipe } if br.offW >= br.size { return 0, io.ErrShortWrite } n := copy(br.buf[br.offW:], p[:min(br.size-br.offW, len(p))]) br.offW += n if br.readPending { br.readPending = false select { case br.readSignal <- struct{}{}: default: } } if n < len(p) { return n, io.ErrShortWrite } return n, nil } func (br *Buf) Close() error { br.rw.Lock() defer br.rw.Unlock() var err error if br.mmap { err = mmap.Free(br.buf) br.mmap = false } br.buf = nil close(br.readSignal) return err } ================================================ FILE: internal/net/request_test.go ================================================ package net //no http range // import ( "bytes" "context" "fmt" "io" "net/http" "sync" "testing" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/sirupsen/logrus" ) var buf22MB = make([]byte, 1024*1024*22) func containsString(slice []string, val string) bool { for _, item := range slice { if item == val { return true } } return false } func dummyHttpRequest(data []byte, p http_range.Range) io.ReadCloser { end := p.Start + p.Length - 1 if end >= int64(len(data)) { end = int64(len(data)) } bodyBytes := data[p.Start:end] return io.NopCloser(bytes.NewReader(bodyBytes)) } func TestDownloadOrder(t *testing.T) { buff := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} downloader, invocations, ranges := newDownloadRangeClient(buff) con, partSize := 3, 3 d := NewDownloader(func(d *Downloader) { d.Concurrency = con d.PartSize = partSize d.HttpClient = downloader.HttpRequest }) var start, length int64 = 2, 10 length2 := length if length2 == -1 { length2 = int64(len(buff)) - start } req := &HttpRequestParams{ Range: http_range.Range{Start: start, Length: length}, Size: int64(len(buff)), } readCloser, err := d.Download(context.Background(), req) if err != nil { t.Fatalf("expect no error, got %v", err) } resultBuf, err := io.ReadAll(readCloser) if err != nil { t.Fatalf("expect no error, got %v", err) } if exp, a := int(length), len(resultBuf); exp != a { t.Errorf("expect buffer length=%d, got %d", exp, a) } chunkSize := int(length+int64(partSize)-1) / partSize if e, a := chunkSize, *invocations; e != a { t.Errorf("expect %v API calls, got %v", e, a) } expectRngs := []string{"2-1", "6-3", "3-3", "9-3"} for _, rng := range expectRngs { if !containsString(*ranges, rng) { t.Errorf("expect range %v, but absent in return", rng) } } if e, a := expectRngs, *ranges; len(e) != len(a) { t.Errorf("expect %v ranges, got %v", e, a) } } func init() { Formatter := new(logrus.TextFormatter) Formatter.TimestampFormat = "2006-01-02T15:04:05.999999999" Formatter.FullTimestamp = true Formatter.ForceColors = true logrus.SetFormatter(Formatter) logrus.SetLevel(logrus.DebugLevel) logrus.Debugf("Download start") } func TestDownloadSingle(t *testing.T) { buff := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} downloader, invocations, ranges := newDownloadRangeClient(buff) con, partSize := 1, 4 d := NewDownloader(func(d *Downloader) { d.Concurrency = con d.PartSize = partSize d.HttpClient = downloader.HttpRequest }) var start, length int64 = 2, 10 req := &HttpRequestParams{ Range: http_range.Range{Start: start, Length: length}, Size: int64(len(buff)), } readCloser, err := d.Download(context.Background(), req) if err != nil { t.Fatalf("expect no error, got %v", err) } resultBuf, err := io.ReadAll(readCloser) if err != nil { t.Fatalf("expect no error, got %v", err) } if exp, a := int(length), len(resultBuf); exp != a { t.Errorf("expect buffer length=%d, got %d", exp, a) } if e, a := int(length+int64(partSize)-1)/partSize, *invocations; e != a { t.Errorf("expect %v API calls, got %v", e, a) } expectRngs := []string{"2-2", "4-4", "8-4"} for _, rng := range expectRngs { if !containsString(*ranges, rng) { t.Errorf("expect range %v, but absent in return", rng) } } if e, a := expectRngs, *ranges; len(e) != len(a) { t.Errorf("expect %v ranges, got %v", e, a) } } type downloadCaptureClient struct { mockedHttpRequest func(params *HttpRequestParams) (*http.Response, error) GetObjectInvocations int RetrievedRanges []string lock sync.Mutex } func (c *downloadCaptureClient) HttpRequest(ctx context.Context, params *HttpRequestParams) (*http.Response, error) { c.lock.Lock() defer c.lock.Unlock() c.GetObjectInvocations++ if params.Range.Length != 0 { c.RetrievedRanges = append(c.RetrievedRanges, fmt.Sprintf("%d-%d", params.Range.Start, params.Range.Length)) } return c.mockedHttpRequest(params) } func newDownloadRangeClient(data []byte) (*downloadCaptureClient, *int, *[]string) { capture := &downloadCaptureClient{} capture.mockedHttpRequest = func(params *HttpRequestParams) (*http.Response, error) { start, fin := params.Range.Start, params.Range.Start+params.Range.Length if params.Range.Length == -1 || fin >= int64(len(data)) { fin = int64(len(data)) } bodyBytes := data[start:fin] header := &http.Header{} header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, fin-1, len(data))) return &http.Response{ Body: io.NopCloser(bytes.NewReader(bodyBytes)), Header: *header, ContentLength: int64(len(bodyBytes)), }, nil } return capture, &capture.GetObjectInvocations, &capture.RetrievedRanges } ================================================ FILE: internal/net/serve.go ================================================ package net import ( "compress/gzip" "context" "crypto/tls" "fmt" "io" "mime/multipart" "net/http" "strconv" "strings" "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) //this file is inspired by GO_SDK net.http.ServeContent //type RangeReadCloser struct { // GetReaderForRange RangeReaderFunc //} // ServeHTTP replies to the request using the content in the // provided RangeReadCloser. The main benefit of ServeHTTP over io.Copy // is that it handles Range requests properly, sets the MIME type, and // handles If-Match, If-Unmodified-Since, If-None-Match, If-Modified-Since, // and If-Range requests. // // If the response's Content-Type header is not set, ServeHTTP // first tries to deduce the type from name's file extension and, // if that fails, falls back to reading the first block of the content // and passing it to DetectContentType. // The name is otherwise unused; in particular it can be empty and is // never sent in the response. // // If modtime is not the zero time or Unix epoch, ServeHTTP // includes it in a Last-Modified header in the response. If the // request includes an If-Modified-Since header, ServeHTTP uses // modtime to decide whether the content needs to be sent at all. // // The content's RangeReadCloser method must work: ServeHTTP gives a range, // caller will give the reader for that Range. // // If the caller has set w's ETag header formatted per RFC 7232, section 2.3, // ServeHTTP uses it to handle requests using If-Match, If-None-Match, or If-Range. func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, size int64, RangeReadCloser model.RangeReadCloserIF) error { defer RangeReadCloser.Close() setLastModified(w, modTime) done, rangeReq := checkPreconditions(w, r, modTime) if done { return nil } if size < 0 { // since too many functions need file size to work, // will not implement the support of unknown file size here http.Error(w, "negative content size not supported", http.StatusInternalServerError) return nil } code := http.StatusOK // If Content-Type isn't set, use the file's extension to find it, but // if the Content-Type is unset explicitly, do not sniff the type. contentTypes, haveType := w.Header()["Content-Type"] var contentType string if !haveType { contentType = utils.GetMimeType(name) w.Header().Set("Content-Type", contentType) } else if len(contentTypes) > 0 { contentType = contentTypes[0] } // handle Content-Range header. sendSize := size var sendContent io.ReadCloser ranges, err := http_range.ParseRange(rangeReq, size) switch { case err == nil: case errors.Is(err, http_range.ErrNoOverlap): if size == 0 { // Some clients add a Range header to all requests to // limit the size of the response. If the file is empty, // ignore the range header and respond with a 200 rather // than a 416. ranges = nil break } w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", size)) fallthrough default: http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) return nil } if sumRangesSize(ranges) > size { // The total number of bytes in all the ranges is larger than the size of the file // or unknown file size, ignore the range request. ranges = nil } // 使用请求的Context // 不然从sendContent读不到数据，即使请求断开CopyBuffer也会一直堵塞 ctx := r.Context() switch { case len(ranges) == 0: reader, err := RangeReadCloser.RangeRead(ctx, http_range.Range{Length: -1}) if err != nil { code = http.StatusRequestedRangeNotSatisfiable if statusCode, ok := errs.UnwrapOrSelf(err).(HttpStatusCodeError); ok { code = int(statusCode) } http.Error(w, err.Error(), code) return nil } sendContent = reader case len(ranges) == 1: // RFC 7233, Section 4.1: // "If a single part is being transferred, the server // generating the 206 response MUST generate a // Content-Range header field, describing what range // of the selected representation is enclosed, and a // payload consisting of the range. // ... // A server MUST NOT generate a multipart response to // a request for a single range, since a client that // does not request multiple parts might not support // multipart responses." ra := ranges[0] sendContent, err = RangeReadCloser.RangeRead(ctx, ra) if err != nil { code = http.StatusRequestedRangeNotSatisfiable if statusCode, ok := errs.UnwrapOrSelf(err).(HttpStatusCodeError); ok { code = int(statusCode) } http.Error(w, err.Error(), code) return nil } sendSize = ra.Length code = http.StatusPartialContent w.Header().Set("Content-Range", ra.ContentRange(size)) case len(ranges) > 1: sendSize, err = rangesMIMESize(ranges, contentType, size) if err != nil { http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) } code = http.StatusPartialContent pr, pw := io.Pipe() mw := multipart.NewWriter(pw) w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary()) sendContent = pr defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish. go func() { for _, ra := range ranges { part, err := mw.CreatePart(ra.MimeHeader(contentType, size)) if err != nil { pw.CloseWithError(err) return } reader, err := RangeReadCloser.RangeRead(ctx, ra) if err != nil { pw.CloseWithError(err) return } if _, err := utils.CopyWithBufferN(part, reader, ra.Length); err != nil { pw.CloseWithError(err) return } } mw.Close() pw.Close() }() } w.Header().Set("Accept-Ranges", "bytes") if w.Header().Get("Content-Encoding") == "" { w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) } w.WriteHeader(code) if r.Method != "HEAD" { written, err := utils.CopyWithBufferN(w, sendContent, sendSize) if err != nil { if errors.Is(context.Cause(ctx), context.Canceled) { return nil } log.Warnf("ServeHttp error. err: %s ", err) if written != sendSize { log.Warnf("Maybe size incorrect or reader not giving correct/full data, or connection closed before finish. written bytes: %d ,sendSize:%d, ", written, sendSize) } code = http.StatusInternalServerError if statusCode, ok := errs.UnwrapOrSelf(err).(HttpStatusCodeError); ok { code = int(statusCode) } w.WriteHeader(code) return err } } return nil } func ProcessHeader(origin, override http.Header) http.Header { result := http.Header{} // client header for h, val := range origin { if utils.SliceContains(conf.SlicesMap[conf.ProxyIgnoreHeaders], strings.ToLower(h)) { continue } result[h] = val } // needed header for h, val := range override { result[h] = val } return result } // RequestHttp deal with Header properly then send the request func RequestHttp(ctx context.Context, httpMethod string, headerOverride http.Header, URL string) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, httpMethod, URL, nil) if err != nil { return nil, err } req.Header = headerOverride res, err := HttpClient().Do(req) if err != nil { return nil, err } // TODO clean header with blocklist or passlist res.Header.Del("set-cookie") var reader io.Reader if res.StatusCode >= 400 { // 根据 Content-Encoding 判断 Body 是否压缩 switch res.Header.Get("Content-Encoding") { case "gzip": // 使用gzip.NewReader解压缩 reader, _ = gzip.NewReader(res.Body) defer reader.(*gzip.Reader).Close() default: // 没有Content-Encoding，直接读取 reader = res.Body } all, _ := io.ReadAll(reader) _ = res.Body.Close() msg := string(all) log.Debugln(msg) return nil, fmt.Errorf("http request [%s] failure,status: %w response:%s", URL, HttpStatusCodeError(res.StatusCode), msg) } return res, nil } type HttpStatusCodeError int func (e HttpStatusCodeError) Error() string { return fmt.Sprintf("%d|%s", e, http.StatusText(int(e))) } var once sync.Once var httpClient *http.Client func HttpClient() *http.Client { once.Do(func() { httpClient = NewHttpClient() httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { if len(via) >= 10 { return errors.New("stopped after 10 redirects") } req.Header.Del("Referer") return nil } }) return httpClient } func NewHttpClient() *http.Client { transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, } SetProxyIfConfigured(transport) return &http.Client{ Timeout: time.Hour * 48, Transport: transport, } } ================================================ FILE: internal/net/util.go ================================================ package net import ( "io" "mime/multipart" "net/http" "net/textproto" "net/url" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/rclone/rclone/lib/readers" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) // scanETag determines if a syntactically valid ETag is present at s. If so, // the ETag and remaining text after consuming ETag is returned. Otherwise, // it returns "", "". func scanETag(s string) (etag string, remain string) { s = textproto.TrimString(s) start := 0 if strings.HasPrefix(s, "W/") { start = 2 } if len(s[start:]) < 2 || s[start] != '"' { return "", "" } // ETag is either W/"text" or "text". // See RFC 7232 2.3. for i := start + 1; i < len(s); i++ { c := s[i] switch { // Character values allowed in ETags. case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80: case c == '"': return s[:i+1], s[i+1:] default: return "", "" } } return "", "" } // etagStrongMatch reports whether a and b match using strong ETag comparison. // Assumes a and b are valid ETags. func etagStrongMatch(a, b string) bool { return a == b && a != "" && a[0] == '"' } // etagWeakMatch reports whether a and b match using weak ETag comparison. // Assumes a and b are valid ETags. func etagWeakMatch(a, b string) bool { return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/") } // condResult is the result of an HTTP request precondition check. // See https://tools.ietf.org/html/rfc7232 section 3. type condResult int const ( condNone condResult = iota condTrue condFalse ) func checkIfMatch(w http.ResponseWriter, r *http.Request) condResult { im := r.Header.Get("If-Match") if im == "" { return condNone } r.Header.Del("If-Match") for { im = textproto.TrimString(im) if len(im) == 0 { break } if im[0] == ',' { im = im[1:] continue } if im[0] == '*' { return condTrue } etag, remain := scanETag(im) if etag == "" { break } if etagStrongMatch(etag, w.Header().Get("Etag")) { return condTrue } im = remain } return condFalse } func checkIfUnmodifiedSince(r *http.Request, modtime time.Time) condResult { ius := r.Header.Get("If-Unmodified-Since") if ius == "" { return condNone } r.Header.Del("If-Unmodified-Since") if isZeroTime(modtime) { return condNone } t, err := http.ParseTime(ius) if err != nil { return condNone } // The Last-Modified header truncates sub-second precision so // the modtime needs to be truncated too. modtime = modtime.Truncate(time.Second) if ret := modtime.Compare(t); ret <= 0 { return condTrue } return condFalse } func checkIfNoneMatch(w http.ResponseWriter, r *http.Request) condResult { inm := r.Header.Get("If-None-Match") if inm == "" { return condNone } r.Header.Del("If-None-Match") buf := inm for { buf = textproto.TrimString(buf) if len(buf) == 0 { break } if buf[0] == ',' { buf = buf[1:] continue } if buf[0] == '*' { return condFalse } etag, remain := scanETag(buf) if etag == "" { break } if etagWeakMatch(etag, w.Header().Get("Etag")) { return condFalse } buf = remain } return condTrue } func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult { if r.Method != "GET" && r.Method != "HEAD" { return condNone } ims := r.Header.Get("If-Modified-Since") if ims == "" { return condNone } r.Header.Del("If-Modified-Since") if isZeroTime(modtime) { return condNone } t, err := http.ParseTime(ims) if err != nil { return condNone } // The Last-Modified header truncates sub-second precision so // the modtime needs to be truncated too. modtime = modtime.Truncate(time.Second) if ret := modtime.Compare(t); ret <= 0 { return condFalse } return condTrue } func checkIfRange(w http.ResponseWriter, r *http.Request, modtime time.Time) condResult { if r.Method != "GET" && r.Method != "HEAD" { return condNone } ir := r.Header.Get("If-Range") if ir == "" { return condNone } r.Header.Del("If-Range") etag, _ := scanETag(ir) if etag != "" { if etagStrongMatch(etag, w.Header().Get("Etag")) { return condTrue } return condFalse } // The If-Range value is typically the ETag value, but it may also be // the modtime date. See golang.org/issue/8367. if modtime.IsZero() { return condFalse } t, err := http.ParseTime(ir) if err != nil { return condFalse } if t.Unix() == modtime.Unix() { return condTrue } return condFalse } var unixEpochTime = time.Unix(0, 0) // isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0). func isZeroTime(t time.Time) bool { return t.IsZero() || t.Equal(unixEpochTime) } func setLastModified(w http.ResponseWriter, modtime time.Time) { if !isZeroTime(modtime) { w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) } } func writeNotModified(w http.ResponseWriter) { // RFC 7232 section 4.1: // a sender SHOULD NOT generate representation metadata other than the // above listed fields unless said metadata exists for the purpose of // guiding cache updates (e.g., Last-Modified might be useful if the // response does not have an ETag field). h := w.Header() delete(h, "Content-Type") delete(h, "Content-Length") delete(h, "Content-Encoding") if h.Get("Etag") != "" { delete(h, "Last-Modified") } w.WriteHeader(http.StatusNotModified) } // checkPreconditions evaluates request preconditions and reports whether a precondition // resulted in sending StatusNotModified or StatusPreconditionFailed. func checkPreconditions(w http.ResponseWriter, r *http.Request, modtime time.Time) (done bool, rangeHeader string) { // This function carefully follows RFC 7232 section 6. ch := checkIfMatch(w, r) if ch == condNone { ch = checkIfUnmodifiedSince(r, modtime) } if ch == condFalse { w.WriteHeader(http.StatusPreconditionFailed) return true, "" } switch checkIfNoneMatch(w, r) { case condFalse: if r.Method == "GET" || r.Method == "HEAD" { writeNotModified(w) return true, "" } w.WriteHeader(http.StatusPreconditionFailed) return true, "" case condNone: if checkIfModifiedSince(r, modtime) == condFalse { writeNotModified(w) return true, "" } } rangeHeader = r.Header.Get("Range") if rangeHeader != "" && checkIfRange(w, r, modtime) == condFalse { rangeHeader = "" } return false, rangeHeader } func sumRangesSize(ranges []http_range.Range) (size int64) { for _, ra := range ranges { size += ra.Length } return } // countingWriter counts how many bytes have been written to it. type countingWriter int64 func (w *countingWriter) Write(p []byte) (n int, err error) { *w += countingWriter(len(p)) return len(p), nil } // rangesMIMESize returns the number of bytes it takes to encode the // provided ranges as a multipart response. func rangesMIMESize(ranges []http_range.Range, contentType string, contentSize int64) (encSize int64, err error) { var w countingWriter mw := multipart.NewWriter(&w) for _, ra := range ranges { _, err := mw.CreatePart(ra.MimeHeader(contentType, contentSize)) if err != nil { return 0, err } encSize += ra.Length } err = mw.Close() if err != nil { return 0, err } encSize += int64(w) return encSize, nil } // GetRangedHttpReader some http server doesn't support "Range" header, // so this function read readCloser with whole data, skip offset, then return ReaderCloser. func GetRangedHttpReader(readCloser io.ReadCloser, offset, length int64) (io.ReadCloser, error) { if offset > 100*1024*1024 { log.Warnf("offset is more than 100MB, if loading data from internet, high-latency and wasting of bandwidth is expected") } if _, err := utils.CopyWithBuffer(io.Discard, io.LimitReader(readCloser, offset)); err != nil { return nil, err } // return an io.ReadCloser that is limited to `length` bytes. return readers.NewLimitedReadCloser(readCloser, length), nil } // SetProxyIfConfigured sets proxy for HTTP Transport if configured func SetProxyIfConfigured(transport *http.Transport) { // If proxy address is configured, override environment variable settings if conf.Conf.ProxyAddress != "" { if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } } } // SetRestyProxyIfConfigured sets proxy for Resty client if configured func SetRestyProxyIfConfigured(client *resty.Client) { // If proxy address is configured, override environment variable settings if conf.Conf.ProxyAddress != "" { if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil { client.SetProxy(proxyURL.String()) } } } ================================================ FILE: internal/offline_download/115/client.go ================================================ package _115 import ( "context" "fmt" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Cloud115 struct { refreshTaskCache bool } func (p *Cloud115) Name() string { return "115 Cloud" } func (p *Cloud115) Items() []model.SettingItem { return nil } func (p *Cloud115) Run(task *tool.DownloadTask) error { return errs.NotSupport } func (p *Cloud115) Init() (string, error) { p.refreshTaskCache = false return "ok", nil } func (p *Cloud115) IsReady() bool { tempDir := setting.GetStr(conf.Pan115TempDir) if tempDir == "" { return false } storage, _, err := op.GetStorageAndActualPath(tempDir) if err != nil { return false } if _, ok := storage.(*_115.Pan115); !ok { return false } return true } func (p *Cloud115) AddURL(args *tool.AddUrlArgs) (string, error) { // 添加新任务刷新缓存 p.refreshTaskCache = true storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) if err != nil { return "", err } driver115, ok := storage.(*_115.Pan115) if !ok { return "", fmt.Errorf("unsupported storage driver for offline download, only 115 Cloud is supported") } ctx := context.Background() if err := op.MakeDir(ctx, storage, actualPath); err != nil { return "", err } parentDir, err := op.GetUnwrap(ctx, storage, actualPath) if err != nil { return "", err } hashs, err := driver115.OfflineDownload(ctx, []string{args.Url}, parentDir) if err != nil || len(hashs) < 1 { return "", fmt.Errorf("failed to add offline download task: %w", err) } return hashs[0], nil } func (p *Cloud115) Remove(task *tool.DownloadTask) error { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return err } driver115, ok := storage.(*_115.Pan115) if !ok { return fmt.Errorf("unsupported storage driver for offline download, only 115 Cloud is supported") } ctx := context.Background() if err := driver115.DeleteOfflineTasks(ctx, []string{task.GID}, false); err != nil { return err } return nil } func (p *Cloud115) Status(task *tool.DownloadTask) (*tool.Status, error) { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return nil, err } driver115, ok := storage.(*_115.Pan115) if !ok { return nil, fmt.Errorf("unsupported storage driver for offline download, only 115 Cloud is supported") } tasks, err := driver115.OfflineList(context.Background()) if err != nil { return nil, err } s := &tool.Status{ Progress: 0, NewGID: "", Completed: false, Status: "the task has been deleted", Err: nil, } for _, t := range tasks { if t.InfoHash == task.GID { s.Progress = t.Percent s.Status = t.GetStatus() s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { s.Err = fmt.Errorf(t.GetStatus()) } return s, nil } } s.Err = fmt.Errorf("the task has been deleted") return nil, nil } var _ tool.Tool = (*Cloud115)(nil) func init() { tool.Tools.Add(&Cloud115{}) } ================================================ FILE: internal/offline_download/115_open/client.go ================================================ package _115_open import ( "context" "fmt" _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Open115 struct { } func (o *Open115) Name() string { return "115 Open" } func (o *Open115) Items() []model.SettingItem { return nil } func (o *Open115) Run(task *tool.DownloadTask) error { return errs.NotSupport } func (o *Open115) Init() (string, error) { return "ok", nil } func (o *Open115) IsReady() bool { tempDir := setting.GetStr(conf.Pan115OpenTempDir) if tempDir == "" { return false } storage, _, err := op.GetStorageAndActualPath(tempDir) if err != nil { return false } if _, ok := storage.(*_115_open.Open115); !ok { return false } return true } func (o *Open115) AddURL(args *tool.AddUrlArgs) (string, error) { storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) if err != nil { return "", err } driver115Open, ok := storage.(*_115_open.Open115) if !ok { return "", fmt.Errorf("unsupported storage driver for offline download, only 115 Cloud is supported") } ctx := context.Background() if err := op.MakeDir(ctx, storage, actualPath); err != nil { return "", err } parentDir, err := op.GetUnwrap(ctx, storage, actualPath) if err != nil { return "", err } hashs, err := driver115Open.OfflineDownload(ctx, []string{args.Url}, parentDir) if err != nil || len(hashs) < 1 { return "", fmt.Errorf("failed to add offline download task: %w", err) } return hashs[0], nil } func (o *Open115) Remove(task *tool.DownloadTask) error { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return err } driver115Open, ok := storage.(*_115_open.Open115) if !ok { return fmt.Errorf("unsupported storage driver for offline download, only 115 Open is supported") } ctx := context.Background() if err := driver115Open.DeleteOfflineTask(ctx, task.GID, false); err != nil { return err } return nil } func (o *Open115) Status(task *tool.DownloadTask) (*tool.Status, error) { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return nil, err } driver115Open, ok := storage.(*_115_open.Open115) if !ok { return nil, fmt.Errorf("unsupported storage driver for offline download, only 115 Open is supported") } tasks, err := driver115Open.OfflineList(context.Background()) if err != nil { return nil, err } s := &tool.Status{ Progress: 0, NewGID: "", Completed: false, Status: "the task has been deleted", Err: nil, } for _, t := range tasks.Tasks { if t.InfoHash == task.GID { s.Progress = float64(t.PercentDone) s.Status = t.GetStatus() s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { s.Err = fmt.Errorf(t.GetStatus()) } return s, nil } } s.Err = fmt.Errorf("the task has been deleted") return nil, nil } var _ tool.Tool = (*Open115)(nil) func init() { tool.Tools.Add(&Open115{}) } ================================================ FILE: internal/offline_download/123/client.go ================================================ package _123_pan import ( "context" "fmt" "strconv" _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" ) type Pan123 struct{} func (*Pan123) Name() string { return "123Pan" } func (*Pan123) Items() []model.SettingItem { return []model.SettingItem{ {Key: conf.Pan123TempDir, Value: "", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } } func (*Pan123) Run(_ *tool.DownloadTask) error { return errs.NotSupport } func (*Pan123) Init() (string, error) { return "ok", nil } func (*Pan123) IsReady() bool { tempDir := setting.GetStr(conf.Pan123TempDir) if tempDir == "" { return false } storage, _, err := op.GetStorageAndActualPath(tempDir) if err != nil { return false } if _, ok := storage.(*_123.Pan123); !ok { return false } return true } func (*Pan123) AddURL(args *tool.AddUrlArgs) (string, error) { storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) if err != nil { return "", err } driver123, ok := storage.(*_123.Pan123) if !ok { return "", fmt.Errorf("unsupported storage driver for offline download, only 123Pan is supported") } ctx := context.Background() if err := op.MakeDir(ctx, storage, actualPath); err != nil { return "", err } parentDir, err := op.GetUnwrap(ctx, storage, actualPath) if err != nil { return "", err } taskID, err := driver123.OfflineDownload(ctx, args.Url, parentDir) if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } return strconv.FormatInt(taskID, 10), nil } func (*Pan123) Remove(task *tool.DownloadTask) error { taskID, err := strconv.ParseInt(task.GID, 10, 64) if err != nil { return fmt.Errorf("failed to parse task ID: %s", task.GID) } storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return err } driver123, ok := storage.(*_123.Pan123) if !ok { return fmt.Errorf("unsupported storage driver for offline download, only 123Pan is supported") } return driver123.DeleteOfflineTasks(context.Background(), []int64{taskID}) } func (*Pan123) Status(task *tool.DownloadTask) (*tool.Status, error) { taskID, err := strconv.ParseInt(task.GID, 10, 64) if err != nil { return nil, fmt.Errorf("failed to parse task ID: %s", task.GID) } storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return nil, err } driver123, ok := storage.(*_123.Pan123) if !ok { return nil, fmt.Errorf("unsupported storage driver for offline download, only 123Pan is supported") } t, err := driver123.GetOfflineTask(context.Background(), taskID) if err != nil { return nil, err } var statusStr string completed := false var taskErr error switch t.Status { case 0: statusStr = "downloading" case 2: statusStr = "succeed" completed = true case 1: statusStr = "failed" taskErr = fmt.Errorf("offline download failed") case 3: statusStr = "retrying" default: statusStr = fmt.Sprintf("status_%d", t.Status) } return &tool.Status{ TotalBytes: t.Size, Progress: t.Progress, Completed: completed, Status: statusStr, Err: taskErr, }, nil } var _ tool.Tool = (*Pan123)(nil) func init() { tool.Tools.Add(&Pan123{}) } ================================================ FILE: internal/offline_download/123_open/client.go ================================================ package _123_open import ( "context" "fmt" "strconv" _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" ) type Open123 struct{} func (*Open123) Name() string { return "123 Open" } func (*Open123) Items() []model.SettingItem { return nil } func (*Open123) Run(_ *tool.DownloadTask) error { return errs.NotSupport } func (*Open123) Init() (string, error) { return "ok", nil } func (*Open123) IsReady() bool { tempDir := setting.GetStr(conf.Pan123OpenTempDir) if tempDir == "" { return false } storage, _, err := op.GetStorageAndActualPath(tempDir) if err != nil { return false } if _, ok := storage.(*_123_open.Open123); !ok { return false } return true } func (*Open123) AddURL(args *tool.AddUrlArgs) (string, error) { storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) if err != nil { return "", err } driver123Open, ok := storage.(*_123_open.Open123) if !ok { return "", fmt.Errorf("unsupported storage driver for offline download, only 123 Open is supported") } ctx := context.Background() if err := op.MakeDir(ctx, storage, actualPath); err != nil { return "", err } parentDir, err := op.GetUnwrap(ctx, storage, actualPath) if err != nil { return "", err } cb := setting.GetStr(conf.Pan123OpenOfflineDownloadCallbackUrl) taskID, err := driver123Open.OfflineDownload(ctx, args.Url, parentDir, cb) if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } return strconv.Itoa(taskID), nil } func (*Open123) Remove(_ *tool.DownloadTask) error { return errs.NotSupport } func (*Open123) Status(task *tool.DownloadTask) (*tool.Status, error) { taskID, err := strconv.Atoi(task.GID) if err != nil { return nil, fmt.Errorf("failed to parse task ID: %s", task.GID) } storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return nil, err } driver123Open, ok := storage.(*_123_open.Open123) if !ok { return nil, fmt.Errorf("unsupported storage driver for offline download, only 123 Open is supported") } process, status, err := driver123Open.OfflineDownloadProcess(context.Background(), taskID) if err != nil { return nil, err } var statusStr string switch status { case 0: statusStr = "downloading" case 1: err = fmt.Errorf("offline download failed") case 2: statusStr = "succeed" case 3: statusStr = "retrying" } return &tool.Status{ Progress: process, Completed: status == 2, Status: statusStr, Err: err, }, nil } var _ tool.Tool = (*Open123)(nil) func init() { tool.Tools.Add(&Open123{}) } ================================================ FILE: internal/offline_download/all.go ================================================ package offline_download import ( _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/115" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/115_open" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/123" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/123_open" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/aria2" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/http" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/pikpak" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/qbit" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/thunder" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/thunder_browser" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/thunderx" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/transmission" ) ================================================ FILE: internal/offline_download/aria2/aria2.go ================================================ package aria2 import ( "context" "fmt" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/aria2/rpc" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) var notify = NewNotify() type Aria2 struct { client rpc.Client } func (a *Aria2) Run(task *tool.DownloadTask) error { return errs.NotSupport } func (a *Aria2) Name() string { return "aria2" } func (a *Aria2) Items() []model.SettingItem { // aria2 settings return []model.SettingItem{ {Key: conf.Aria2Uri, Value: "http://localhost:6800/jsonrpc", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.Aria2Secret, Value: "", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } } func (a *Aria2) Init() (string, error) { a.client = nil uri := setting.GetStr(conf.Aria2Uri) secret := setting.GetStr(conf.Aria2Secret) c, err := rpc.New(context.Background(), uri, secret, 4*time.Second, notify) if err != nil { return "", errors.Wrap(err, "failed to init aria2 client") } version, err := c.GetVersion() if err != nil { return "", errors.Wrapf(err, "failed get aria2 version") } a.client = c log.Infof("using aria2 version: %s", version.Version) return fmt.Sprintf("aria2 version: %s", version.Version), nil } func (a *Aria2) IsReady() bool { return a.client != nil } func (a *Aria2) AddURL(args *tool.AddUrlArgs) (string, error) { options := map[string]interface{}{ "dir": args.TempDir, } gid, err := a.client.AddURI([]string{args.Url}, options) if err != nil { return "", err } notify.Signals.Store(gid, args.Signal) return gid, nil } func (a *Aria2) Remove(task *tool.DownloadTask) error { _, err := a.client.Remove(task.GID) return err } func (a *Aria2) Status(task *tool.DownloadTask) (*tool.Status, error) { info, err := a.client.TellStatus(task.GID) if err != nil { return nil, err } total, err := strconv.ParseInt(info.TotalLength, 10, 64) if err != nil { total = 0 } downloaded, err := strconv.ParseUint(info.CompletedLength, 10, 64) if err != nil { downloaded = 0 } s := &tool.Status{ Completed: info.Status == "complete", Err: err, TotalBytes: total, } s.Progress = float64(downloaded) / float64(total) * 100 if len(info.FollowedBy) != 0 { s.NewGID = info.FollowedBy[0] notify.Signals.Delete(task.GID) notify.Signals.Store(s.NewGID, task.Signal) } switch info.Status { case "complete": s.Completed = true case "error": s.Err = errors.Errorf("failed to download %s, error: %s", task.GID, info.ErrorMessage) case "active": s.Status = "aria2: " + info.Status if info.Seeder == "true" { s.Completed = true } case "waiting", "paused": s.Status = "aria2: " + info.Status case "removed": s.Err = errors.Errorf("failed to download %s, removed", task.GID) default: return nil, errors.Errorf("[aria2] unknown status %s", info.Status) } return s, nil } var _ tool.Tool = (*Aria2)(nil) func init() { tool.Tools.Add(&Aria2{}) } ================================================ FILE: internal/offline_download/aria2/notify.go ================================================ package aria2 import ( "github.com/OpenListTeam/OpenList/v4/pkg/aria2/rpc" "github.com/OpenListTeam/OpenList/v4/pkg/generic_sync" ) const ( Downloading = iota Paused Stopped Completed Errored ) type Notify struct { Signals generic_sync.MapOf[string, chan int] } func NewNotify() *Notify { return &Notify{Signals: generic_sync.MapOf[string, chan int]{}} } func (n *Notify) OnDownloadStart(events []rpc.Event) { for _, e := range events { if signal, ok := n.Signals.Load(e.Gid); ok { signal <- Downloading } } } func (n *Notify) OnDownloadPause(events []rpc.Event) { for _, e := range events { if signal, ok := n.Signals.Load(e.Gid); ok { signal <- Paused } } } func (n *Notify) OnDownloadStop(events []rpc.Event) { for _, e := range events { if signal, ok := n.Signals.Load(e.Gid); ok { signal <- Stopped } } } func (n *Notify) OnDownloadComplete(events []rpc.Event) { for _, e := range events { if signal, ok := n.Signals.Load(e.Gid); ok { signal <- Completed } } } func (n *Notify) OnDownloadError(events []rpc.Event) { for _, e := range events { if signal, ok := n.Signals.Load(e.Gid); ok { signal <- Errored } } } func (n *Notify) OnBtDownloadComplete(events []rpc.Event) { for _, e := range events { if signal, ok := n.Signals.Load(e.Gid); ok { signal <- Completed } } } ================================================ FILE: internal/offline_download/http/client.go ================================================ package http import ( "fmt" "math/rand/v2" "net/http" "os" "path" "path/filepath" "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type SimpleHttp struct { client http.Client } func (s SimpleHttp) Name() string { return "SimpleHttp" } func (s SimpleHttp) Items() []model.SettingItem { return nil } func (s SimpleHttp) Init() (string, error) { return "ok", nil } func (s SimpleHttp) IsReady() bool { return true } func (s SimpleHttp) AddURL(args *tool.AddUrlArgs) (string, error) { panic("should not be called") } func (s SimpleHttp) Remove(task *tool.DownloadTask) error { panic("should not be called") } func (s SimpleHttp) Status(task *tool.DownloadTask) (*tool.Status, error) { panic("should not be called") } func (s SimpleHttp) Run(task *tool.DownloadTask) error { streamPut := task.DeletePolicy == tool.UploadDownloadStream method := http.MethodGet if streamPut { method = http.MethodHead } req, err := http.NewRequestWithContext(task.Ctx(), method, task.Url, nil) if err != nil { return err } req.Header.Set("User-Agent", base.UserAgent) if streamPut { req.Header.Set("Range", "bytes=0-") } resp, err := s.client.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode >= 400 { return fmt.Errorf("http status code %d", resp.StatusCode) } filename, err := parseFilenameFromContentDisposition(resp.Header.Get("Content-Disposition")) if err != nil { filename = path.Base(resp.Request.URL.Path) } filename = strings.Trim(filename, "/") if len(filename) == 0 { filename = fmt.Sprintf("%s-%d-%x", strings.ReplaceAll(req.URL.Host, ".", "_"), time.Now().UnixMilli(), rand.Uint32()) } fileSize := resp.ContentLength if streamPut { if fileSize == 0 { start, end, _ := http_range.ParseContentRange(resp.Header.Get("Content-Range")) fileSize = start + end } task.SetTotalBytes(fileSize) task.TempDir = filename return nil } task.SetTotalBytes(fileSize) // save to temp dir _ = os.MkdirAll(task.TempDir, os.ModePerm) filePath := filepath.Join(task.TempDir, filename) file, err := os.Create(filePath) if err != nil { return err } defer file.Close() err = utils.CopyWithCtx(task.Ctx(), file, resp.Body, fileSize, task.SetProgress) return err } func init() { tool.Tools.Add(&SimpleHttp{}) } ================================================ FILE: internal/offline_download/http/util.go ================================================ package http import ( "fmt" "mime" ) func parseFilenameFromContentDisposition(contentDisposition string) (string, error) { if contentDisposition == "" { return "", fmt.Errorf("Content-Disposition is empty") } _, params, err := mime.ParseMediaType(contentDisposition) if err != nil { return "", err } filename := params["filename"] if filename == "" { return "", fmt.Errorf("filename not found in Content-Disposition: [%s]", contentDisposition) } return filename, nil } ================================================ FILE: internal/offline_download/pikpak/pikpak.go ================================================ package pikpak import ( "context" "fmt" "strconv" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type PikPak struct { refreshTaskCache bool } func (p *PikPak) Name() string { return "PikPak" } func (p *PikPak) Items() []model.SettingItem { return nil } func (p *PikPak) Run(task *tool.DownloadTask) error { return errs.NotSupport } func (p *PikPak) Init() (string, error) { p.refreshTaskCache = false return "ok", nil } func (p *PikPak) IsReady() bool { tempDir := setting.GetStr(conf.PikPakTempDir) if tempDir == "" { return false } storage, _, err := op.GetStorageAndActualPath(tempDir) if err != nil { return false } if _, ok := storage.(*pikpak.PikPak); !ok { return false } return true } func (p *PikPak) AddURL(args *tool.AddUrlArgs) (string, error) { // 添加新任务刷新缓存 p.refreshTaskCache = true storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) if err != nil { return "", err } pikpakDriver, ok := storage.(*pikpak.PikPak) if !ok { return "", fmt.Errorf("unsupported storage driver for offline download, only Pikpak is supported") } ctx := context.Background() if err := op.MakeDir(ctx, storage, actualPath); err != nil { return "", err } parentDir, err := op.GetUnwrap(ctx, storage, actualPath) if err != nil { return "", err } t, err := pikpakDriver.OfflineDownload(ctx, args.Url, parentDir, "") if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } return t.ID, nil } func (p *PikPak) Remove(task *tool.DownloadTask) error { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return err } pikpakDriver, ok := storage.(*pikpak.PikPak) if !ok { return fmt.Errorf("unsupported storage driver for offline download, only Pikpak is supported") } ctx := context.Background() err = pikpakDriver.DeleteOfflineTasks(ctx, []string{task.GID}, false) if err != nil { return err } return nil } func (p *PikPak) Status(task *tool.DownloadTask) (*tool.Status, error) { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return nil, err } pikpakDriver, ok := storage.(*pikpak.PikPak) if !ok { return nil, fmt.Errorf("unsupported storage driver for offline download, only Pikpak is supported") } tasks, err := p.GetTasks(pikpakDriver) if err != nil { return nil, err } s := &tool.Status{ Progress: 0, NewGID: "", Completed: false, Status: "the task has been deleted", Err: nil, } for _, t := range tasks { if t.ID == task.GID { s.Progress = float64(t.Progress) s.Status = t.Message s.Completed = (t.Phase == "PHASE_TYPE_COMPLETE") s.TotalBytes, err = strconv.ParseInt(t.FileSize, 10, 64) if err != nil { s.TotalBytes = 0 } if t.Phase == "PHASE_TYPE_ERROR" { s.Err = fmt.Errorf(t.Message) } return s, nil } } s.Err = fmt.Errorf("the task has been deleted") return s, nil } func init() { tool.Tools.Add(&PikPak{}) } ================================================ FILE: internal/offline_download/pikpak/util.go ================================================ package pikpak import ( "context" "time" "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/go-cache" ) var taskCache = cache.NewMemCache(cache.WithShards[[]pikpak.OfflineTask](16)) var taskG singleflight.Group[[]pikpak.OfflineTask] func (p *PikPak) GetTasks(pikpakDriver *pikpak.PikPak) ([]pikpak.OfflineTask, error) { key := op.Key(pikpakDriver, "/drive/v1/task") if !p.refreshTaskCache { if tasks, ok := taskCache.Get(key); ok { return tasks, nil } } p.refreshTaskCache = false tasks, err, _ := taskG.Do(key, func() ([]pikpak.OfflineTask, error) { ctx := context.Background() phase := []string{"PHASE_TYPE_RUNNING", "PHASE_TYPE_ERROR", "PHASE_TYPE_PENDING", "PHASE_TYPE_COMPLETE"} tasks, err := pikpakDriver.OfflineList(ctx, "", phase) if err != nil { return nil, err } // 添加缓存 10s if len(tasks) > 0 { taskCache.Set(key, tasks, cache.WithEx[[]pikpak.OfflineTask](time.Second*10)) } else { taskCache.Del(key) } return tasks, nil }) if err != nil { return nil, err } return tasks, nil } ================================================ FILE: internal/offline_download/qbit/qbit.go ================================================ package qbit import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/qbittorrent" "github.com/pkg/errors" ) type QBittorrent struct { client qbittorrent.Client } func (a *QBittorrent) Run(task *tool.DownloadTask) error { return errs.NotSupport } func (a *QBittorrent) Name() string { return "qBittorrent" } func (a *QBittorrent) Items() []model.SettingItem { // qBittorrent settings return []model.SettingItem{ {Key: conf.QbittorrentUrl, Value: "http://admin:adminadmin@localhost:8080/", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.QbittorrentSeedtime, Value: "0", Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } } func (a *QBittorrent) Init() (string, error) { a.client = nil url := setting.GetStr(conf.QbittorrentUrl) qbClient, err := qbittorrent.New(url) if err != nil { return "", err } a.client = qbClient return "ok", nil } func (a *QBittorrent) IsReady() bool { return a.client != nil } func (a *QBittorrent) AddURL(args *tool.AddUrlArgs) (string, error) { err := a.client.AddFromLink(args.Url, args.TempDir, args.UID) if err != nil { return "", err } return args.UID, nil } func (a *QBittorrent) Remove(task *tool.DownloadTask) error { err := a.client.Delete(task.GID, false) return err } func (a *QBittorrent) Status(task *tool.DownloadTask) (*tool.Status, error) { info, err := a.client.GetInfo(task.GID) if err != nil { return nil, err } s := &tool.Status{} s.TotalBytes = info.Size s.Progress = float64(info.Completed) / float64(info.Size) * 100 switch info.State { case qbittorrent.UPLOADING, qbittorrent.PAUSEDUP, qbittorrent.QUEUEDUP, qbittorrent.STALLEDUP, qbittorrent.FORCEDUP, qbittorrent.CHECKINGUP: s.Completed = true case qbittorrent.ALLOCATING, qbittorrent.DOWNLOADING, qbittorrent.METADL, qbittorrent.PAUSEDDL, qbittorrent.QUEUEDDL, qbittorrent.STALLEDDL, qbittorrent.CHECKINGDL, qbittorrent.FORCEDDL, qbittorrent.CHECKINGRESUMEDATA, qbittorrent.MOVING: s.Status = "[qBittorrent] downloading" case qbittorrent.ERROR, qbittorrent.MISSINGFILES, qbittorrent.UNKNOWN: s.Err = errors.Errorf("[qBittorrent] failed to download %s, error: %s", task.GID, info.State) default: s.Err = errors.Errorf("[qBittorrent] unknown error occurred downloading %s", task.GID) } return s, nil } var _ tool.Tool = (*QBittorrent)(nil) func init() { tool.Tools.Add(&QBittorrent{}) } ================================================ FILE: internal/offline_download/thunder/thunder.go ================================================ package thunder import ( "context" "errors" "fmt" "strconv" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/drivers/thunder" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type Thunder struct { refreshTaskCache bool } func (t *Thunder) Name() string { return "Thunder" } func (t *Thunder) Items() []model.SettingItem { return nil } func (t *Thunder) Run(task *tool.DownloadTask) error { return errs.NotSupport } func (t *Thunder) Init() (string, error) { t.refreshTaskCache = false return "ok", nil } func (t *Thunder) IsReady() bool { tempDir := setting.GetStr(conf.ThunderTempDir) if tempDir == "" { return false } storage, _, err := op.GetStorageAndActualPath(tempDir) if err != nil { return false } if _, ok := storage.(*thunder.Thunder); !ok { return false } return true } func (t *Thunder) AddURL(args *tool.AddUrlArgs) (string, error) { // 添加新任务刷新缓存 t.refreshTaskCache = true storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) if err != nil { return "", err } thunderDriver, ok := storage.(*thunder.Thunder) if !ok { return "", fmt.Errorf("unsupported storage driver for offline download, only Thunder is supported") } ctx := context.Background() if err := op.MakeDir(ctx, storage, actualPath); err != nil { return "", err } parentDir, err := op.GetUnwrap(ctx, storage, actualPath) if err != nil { return "", err } task, err := thunderDriver.OfflineDownload(ctx, args.Url, parentDir, "") if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } return task.ID, nil } func (t *Thunder) Remove(task *tool.DownloadTask) error { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return err } thunderDriver, ok := storage.(*thunder.Thunder) if !ok { return fmt.Errorf("unsupported storage driver for offline download, only Thunder is supported") } ctx := context.Background() err = thunderDriver.DeleteOfflineTasks(ctx, []string{task.GID}, false) if err != nil { return err } return nil } func (t *Thunder) Status(task *tool.DownloadTask) (*tool.Status, error) { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return nil, err } thunderDriver, ok := storage.(*thunder.Thunder) if !ok { return nil, fmt.Errorf("unsupported storage driver for offline download, only Thunder is supported") } tasks, err := t.GetTasks(thunderDriver) if err != nil { return nil, err } s := &tool.Status{ Progress: 0, NewGID: "", Completed: false, Status: "the task has been deleted", Err: nil, } for _, t := range tasks { if t.ID == task.GID { s.Progress = float64(t.Progress) s.Status = t.Message s.Completed = (t.Phase == "PHASE_TYPE_COMPLETE") s.TotalBytes, err = strconv.ParseInt(t.FileSize, 10, 64) if err != nil { s.TotalBytes = 0 } if t.Phase == "PHASE_TYPE_ERROR" { s.Err = errors.New(t.Message) } return s, nil } } s.Err = fmt.Errorf("the task has been deleted") return s, nil } func init() { tool.Tools.Add(&Thunder{}) } ================================================ FILE: internal/offline_download/thunder/util.go ================================================ package thunder import ( "context" "time" "github.com/OpenListTeam/OpenList/v4/drivers/thunder" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/go-cache" ) var taskCache = cache.NewMemCache(cache.WithShards[[]thunder.OfflineTask](16)) var taskG singleflight.Group[[]thunder.OfflineTask] func (t *Thunder) GetTasks(thunderDriver *thunder.Thunder) ([]thunder.OfflineTask, error) { key := op.Key(thunderDriver, "/drive/v1/task") if !t.refreshTaskCache { if tasks, ok := taskCache.Get(key); ok { return tasks, nil } } t.refreshTaskCache = false tasks, err, _ := taskG.Do(key, func() ([]thunder.OfflineTask, error) { ctx := context.Background() tasks, err := thunderDriver.OfflineList(ctx, "") if err != nil { return nil, err } // 添加缓存 10s if len(tasks) > 0 { taskCache.Set(key, tasks, cache.WithEx[[]thunder.OfflineTask](time.Second*10)) } else { taskCache.Del(key) } return tasks, nil }) if err != nil { return nil, err } return tasks, nil } ================================================ FILE: internal/offline_download/thunder_browser/thunder_browser.go ================================================ package thunder_browser import ( "context" "errors" "fmt" "strconv" "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" ) type ThunderBrowser struct { refreshTaskCache bool } func (t *ThunderBrowser) Name() string { return "ThunderBrowser" } func (t *ThunderBrowser) Items() []model.SettingItem { return nil } func (t *ThunderBrowser) Run(task *tool.DownloadTask) error { return errs.NotSupport } func (t *ThunderBrowser) Init() (string, error) { t.refreshTaskCache = false return "ok", nil } func (t *ThunderBrowser) IsReady() bool { tempDir := setting.GetStr(conf.ThunderBrowserTempDir) if tempDir == "" { return false } storage, _, err := op.GetStorageAndActualPath(tempDir) if err != nil { return false } switch storage.(type) { case *thunder_browser.ThunderBrowser, *thunder_browser.ThunderBrowserExpert: return true default: return false } } func (t *ThunderBrowser) AddURL(args *tool.AddUrlArgs) (string, error) { // 添加新任务刷新缓存 t.refreshTaskCache = true storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) if err != nil { return "", err } ctx := context.Background() if err := op.MakeDir(ctx, storage, actualPath); err != nil { return "", err } parentDir, err := op.GetUnwrap(ctx, storage, actualPath) if err != nil { return "", err } var task *thunder_browser.OfflineTask switch v := storage.(type) { case *thunder_browser.ThunderBrowser: task, err = v.OfflineDownload(ctx, args.Url, parentDir, "") case *thunder_browser.ThunderBrowserExpert: task, err = v.OfflineDownload(ctx, args.Url, parentDir, "") default: return "", fmt.Errorf("unsupported storage driver for offline download, only ThunderBrowser is supported") } if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } if task == nil { return "", fmt.Errorf("failed to add offline download task: task is nil") } return task.ID, nil } func (t *ThunderBrowser) Remove(task *tool.DownloadTask) error { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return err } ctx := context.Background() switch v := storage.(type) { case *thunder_browser.ThunderBrowser: err = v.DeleteOfflineTasks(ctx, []string{task.GID}) case *thunder_browser.ThunderBrowserExpert: err = v.DeleteOfflineTasks(ctx, []string{task.GID}) default: return fmt.Errorf("unsupported storage driver for offline download, only ThunderBrowser is supported") } if err != nil { return err } return nil } func (t *ThunderBrowser) Status(task *tool.DownloadTask) (*tool.Status, error) { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return nil, err } var tasks []thunder_browser.OfflineTask switch v := storage.(type) { case *thunder_browser.ThunderBrowser: tasks, err = t.GetTasks(v) case *thunder_browser.ThunderBrowserExpert: tasks, err = t.GetTasksExpert(v) default: return nil, fmt.Errorf("unsupported storage driver for offline download, only ThunderBrowser is supported") } if err != nil { return nil, err } s := &tool.Status{ Progress: 0, NewGID: "", Completed: false, Status: "the task has been deleted", Err: nil, } for _, t := range tasks { if t.ID == task.GID { s.Progress = float64(t.Progress) s.Status = t.Message s.Completed = t.Phase == "PHASE_TYPE_COMPLETE" s.TotalBytes, err = strconv.ParseInt(t.FileSize, 10, 64) if err != nil { s.TotalBytes = 0 } if t.Phase == "PHASE_TYPE_ERROR" { s.Err = errors.New(t.Message) } return s, nil } } s.Err = fmt.Errorf("the task has been deleted") return s, nil } func init() { tool.Tools.Add(&ThunderBrowser{}) } ================================================ FILE: internal/offline_download/thunder_browser/util.go ================================================ package thunder_browser import ( "context" "time" "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/go-cache" ) var taskCache = cache.NewMemCache(cache.WithShards[[]thunder_browser.OfflineTask](16)) var taskG singleflight.Group[[]thunder_browser.OfflineTask] func (t *ThunderBrowser) GetTasks(thunderDriver *thunder_browser.ThunderBrowser) ([]thunder_browser.OfflineTask, error) { key := op.Key(thunderDriver, "/drive/v1/task") if !t.refreshTaskCache { if tasks, ok := taskCache.Get(key); ok { return tasks, nil } } t.refreshTaskCache = false tasks, err, _ := taskG.Do(key, func() ([]thunder_browser.OfflineTask, error) { ctx := context.Background() tasks, err := thunderDriver.OfflineList(ctx, "") if err != nil { return nil, err } // 添加缓存 10s if len(tasks) > 0 { taskCache.Set(key, tasks, cache.WithEx[[]thunder_browser.OfflineTask](time.Second*10)) } else { taskCache.Del(key) } return tasks, nil }) if err != nil { return nil, err } return tasks, nil } func (t *ThunderBrowser) GetTasksExpert(thunderDriver *thunder_browser.ThunderBrowserExpert) ([]thunder_browser.OfflineTask, error) { key := op.Key(thunderDriver, "/drive/v1/task") if !t.refreshTaskCache { if tasks, ok := taskCache.Get(key); ok { return tasks, nil } } t.refreshTaskCache = false tasks, err, _ := taskG.Do(key, func() ([]thunder_browser.OfflineTask, error) { ctx := context.Background() tasks, err := thunderDriver.OfflineList(ctx, "") if err != nil { return nil, err } // 添加缓存 10s if len(tasks) > 0 { taskCache.Set(key, tasks, cache.WithEx[[]thunder_browser.OfflineTask](time.Second*10)) } else { taskCache.Del(key) } return tasks, nil }) if err != nil { return nil, err } return tasks, nil } ================================================ FILE: internal/offline_download/thunderx/thunderx.go ================================================ package thunderx import ( "context" "errors" "fmt" "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "strconv" ) type ThunderX struct { refreshTaskCache bool } func (t *ThunderX) Name() string { return "ThunderX" } func (t *ThunderX) Items() []model.SettingItem { return nil } func (t *ThunderX) Init() (string, error) { t.refreshTaskCache = false return "ok", nil } func (t *ThunderX) IsReady() bool { tempDir := setting.GetStr(conf.ThunderXTempDir) if tempDir == "" { return false } storage, _, err := op.GetStorageAndActualPath(tempDir) if err != nil { return false } if _, ok := storage.(*thunderx.ThunderX); !ok { return false } return true } func (t *ThunderX) AddURL(args *tool.AddUrlArgs) (string, error) { // 添加新任务刷新缓存 t.refreshTaskCache = true storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) if err != nil { return "", err } thunderXDriver, ok := storage.(*thunderx.ThunderX) if !ok { return "", fmt.Errorf("unsupported storage driver for offline download, only ThunderX is supported") } ctx := context.Background() if err := op.MakeDir(ctx, storage, actualPath); err != nil { return "", err } parentDir, err := op.GetUnwrap(ctx, storage, actualPath) if err != nil { return "", err } task, err := thunderXDriver.OfflineDownload(ctx, args.Url, parentDir, "") if err != nil { return "", fmt.Errorf("failed to add offline download task: %w", err) } return task.ID, nil } func (t *ThunderX) Remove(task *tool.DownloadTask) error { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return err } thunderXDriver, ok := storage.(*thunderx.ThunderX) if !ok { return fmt.Errorf("unsupported storage driver for offline download, only ThunderX is supported") } ctx := context.Background() err = thunderXDriver.DeleteOfflineTasks(ctx, []string{task.GID}, false) if err != nil { return err } return nil } func (t *ThunderX) Status(task *tool.DownloadTask) (*tool.Status, error) { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return nil, err } thunderXDriver, ok := storage.(*thunderx.ThunderX) if !ok { return nil, fmt.Errorf("unsupported storage driver for offline download, only ThunderX is supported") } tasks, err := t.GetTasks(thunderXDriver) if err != nil { return nil, err } s := &tool.Status{ Progress: 0, NewGID: "", Completed: false, Status: "the task has been deleted", Err: nil, } for _, t := range tasks { if t.ID == task.GID { s.Progress = float64(t.Progress) s.Status = t.Message s.Completed = t.Phase == "PHASE_TYPE_COMPLETE" s.TotalBytes, err = strconv.ParseInt(t.FileSize, 10, 64) if err != nil { s.TotalBytes = 0 } if t.Phase == "PHASE_TYPE_ERROR" { s.Err = errors.New(t.Message) } return s, nil } } s.Err = fmt.Errorf("the task has been deleted") return s, nil } func (t *ThunderX) Run(task *tool.DownloadTask) error { return errs.NotSupport } func init() { tool.Tools.Add(&ThunderX{}) } ================================================ FILE: internal/offline_download/thunderx/utils.go ================================================ package thunderx import ( "context" "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/go-cache" "time" ) var taskCache = cache.NewMemCache(cache.WithShards[[]thunderx.OfflineTask](16)) var taskG singleflight.Group[[]thunderx.OfflineTask] func (t *ThunderX) GetTasks(thunderxDriver *thunderx.ThunderX) ([]thunderx.OfflineTask, error) { key := op.Key(thunderxDriver, "/drive/v1/task") if !t.refreshTaskCache { if tasks, ok := taskCache.Get(key); ok { return tasks, nil } } t.refreshTaskCache = false tasks, err, _ := taskG.Do(key, func() ([]thunderx.OfflineTask, error) { ctx := context.Background() phase := []string{"PHASE_TYPE_RUNNING", "PHASE_TYPE_ERROR", "PHASE_TYPE_PENDING", "PHASE_TYPE_COMPLETE"} tasks, err := thunderxDriver.OfflineList(ctx, "", phase) if err != nil { return nil, err } // 添加缓存 10s if len(tasks) > 0 { taskCache.Set(key, tasks, cache.WithEx[[]thunderx.OfflineTask](time.Second*10)) } else { taskCache.Del(key) } return tasks, nil }) if err != nil { return nil, err } return tasks, nil } ================================================ FILE: internal/offline_download/tool/add.go ================================================ package tool import ( "context" "net/url" stdpath "path" "path/filepath" _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" "github.com/OpenListTeam/OpenList/v4/drivers/thunder" "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/google/uuid" "github.com/pkg/errors" ) type DeletePolicy string const ( DeleteOnUploadSucceed DeletePolicy = "delete_on_upload_succeed" DeleteOnUploadFailed DeletePolicy = "delete_on_upload_failed" DeleteNever DeletePolicy = "delete_never" DeleteAlways DeletePolicy = "delete_always" UploadDownloadStream DeletePolicy = "upload_download_stream" ) type AddURLArgs struct { URL string DstDirPath string Tool string DeletePolicy DeletePolicy } func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, error) { // check storage storage, dstDirActualPath, err := op.GetStorageAndActualPath(args.DstDirPath) if err != nil { return nil, errors.WithMessage(err, "failed get storage") } // check is it could upload if storage.Config().NoUpload { return nil, errors.WithStack(errs.UploadNotSupported) } // check path is valid obj, err := op.Get(ctx, storage, dstDirActualPath) if err != nil { if !errs.IsObjectNotFound(err) { return nil, errors.WithMessage(err, "failed get object") } } else { if !obj.IsDir() { // can't add to a file return nil, errors.WithStack(errs.NotFolder) } } // try putting url if args.Tool == "SimpleHttp" { err = tryPutUrl(ctx, args.DstDirPath, args.URL) if err == nil || !errors.Is(err, errs.NotImplement) { return nil, err } } // get tool tool, err := Tools.Get(args.Tool) if err != nil { return nil, errors.Wrapf(err, "failed get offline download tool") } // check tool is ready if !tool.IsReady() { // try to init tool if _, err := tool.Init(); err != nil { return nil, errors.Wrapf(err, "failed init offline download tool %s", args.Tool) } } uid := uuid.NewString() tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid) deletePolicy := args.DeletePolicy // 如果当前 storage 是对应网盘，则直接下载到目标路径，无需转存 switch args.Tool { case "115 Cloud": if _, ok := storage.(*_115.Pan115); ok { tempDir = args.DstDirPath } else { tempDir = filepath.Join(setting.GetStr(conf.Pan115TempDir), uid) } case "115 Open": if _, ok := storage.(*_115_open.Open115); ok { tempDir = args.DstDirPath } else { tempDir = filepath.Join(setting.GetStr(conf.Pan115OpenTempDir), uid) } case "123 Open": if _, ok := storage.(*_123_open.Open123); ok && dstDirActualPath != "/" { // directly offline downloading to the root path is not allowed via 123 open platform tempDir = args.DstDirPath } else { tempDir = filepath.Join(setting.GetStr(conf.Pan123OpenTempDir), uid) } case "123Pan": if _, ok := storage.(*_123.Pan123); ok { tempDir = args.DstDirPath } else { tempDir = filepath.Join(setting.GetStr(conf.Pan123TempDir), uid) } case "PikPak": if _, ok := storage.(*pikpak.PikPak); ok { tempDir = args.DstDirPath } else { tempDir = filepath.Join(setting.GetStr(conf.PikPakTempDir), uid) } case "Thunder": if _, ok := storage.(*thunder.Thunder); ok { tempDir = args.DstDirPath } else { tempDir = filepath.Join(setting.GetStr(conf.ThunderTempDir), uid) } case "ThunderBrowser": switch storage.(type) { case *thunder_browser.ThunderBrowser, *thunder_browser.ThunderBrowserExpert: tempDir = args.DstDirPath default: tempDir = filepath.Join(setting.GetStr(conf.ThunderBrowserTempDir), uid) } case "ThunderX": if _, ok := storage.(*thunderx.ThunderX); ok { tempDir = args.DstDirPath } else { tempDir = filepath.Join(setting.GetStr(conf.ThunderXTempDir), uid) } } taskCreator, _ := ctx.Value(conf.UserKey).(*model.User) // taskCreator is nil when convert failed t := &DownloadTask{ TaskExtension: task.TaskExtension{ Creator: taskCreator, ApiUrl: common.GetApiUrl(ctx), }, Url: args.URL, DstDirPath: args.DstDirPath, TempDir: tempDir, DeletePolicy: deletePolicy, Toolname: args.Tool, tool: tool, } DownloadTaskManager.Add(t) return t, nil } func tryPutUrl(ctx context.Context, path, urlStr string) error { var dstName string u, err := url.Parse(urlStr) if err == nil { dstName = stdpath.Base(u.Path) } else { dstName = "UnnamedURL" } return fs.PutURL(ctx, path, dstName, urlStr) } ================================================ FILE: internal/offline_download/tool/base.go ================================================ package tool import ( "github.com/OpenListTeam/OpenList/v4/internal/model" ) type AddUrlArgs struct { Url string UID string TempDir string Signal chan int } type Status struct { TotalBytes int64 Progress float64 NewGID string Completed bool Status string Err error } type Tool interface { Name() string // Items return the setting items the tool need Items() []model.SettingItem Init() (string, error) IsReady() bool // AddURL add an uri to download, return the task id AddURL(args *AddUrlArgs) (string, error) // Remove the download if task been canceled Remove(task *DownloadTask) error // Status return the status of the download task, if an error occurred, return the error in Status.Err Status(task *DownloadTask) (*Status, error) // Run for simple http download Run(task *DownloadTask) error } ================================================ FILE: internal/offline_download/tool/download.go ================================================ package tool import ( "fmt" "path" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/internal/task_group" "github.com/OpenListTeam/tache" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) type DownloadTask struct { task.TaskExtension Url string `json:"url"` DstDirPath string `json:"dst_dir_path"` TempDir string `json:"temp_dir"` DeletePolicy DeletePolicy `json:"delete_policy"` Toolname string `json:"toolname"` Status string `json:"-"` Signal chan int `json:"-"` GID string `json:"-"` tool Tool callStatusRetried int } func (t *DownloadTask) Run() error { t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() if t.tool == nil { tool, err := Tools.Get(t.Toolname) if err != nil { return errors.WithMessage(err, "failed get tool") } t.tool = tool } if err := t.tool.Run(t); !errs.IsNotSupportError(err) { if err == nil { return t.Transfer() } return err } t.Signal = make(chan int) defer func() { t.Signal = nil }() gid, err := t.tool.AddURL(&AddUrlArgs{ Url: t.Url, UID: t.ID, TempDir: t.TempDir, Signal: t.Signal, }) if err != nil { return err } t.GID = gid var ok bool outer: for { select { case <-t.CtxDone(): err := t.tool.Remove(t) return err case <-t.Signal: ok, err = t.Update() if ok { break outer } case <-time.After(time.Second * 3): ok, err = t.Update() if ok { break outer } } } if err != nil { return err } if t.tool.Name() == "Pikpak" { return nil } if t.tool.Name() == "Thunder" { return nil } if t.tool.Name() == "ThunderBrowser" { return nil } if t.tool.Name() == "ThunderX" { return nil } if t.tool.Name() == "115 Cloud" { // hack for 115 <-time.After(time.Second * 1) err := t.tool.Remove(t) if err != nil { log.Errorln(err.Error()) } return nil } if t.tool.Name() == "115 Open" { return nil } if t.tool.Name() == "123 Open" { return nil } t.Status = "offline download completed, maybe transferring" // hack for qBittorrent if t.tool.Name() == "qBittorrent" { seedTime := setting.GetInt(conf.QbittorrentSeedtime, 0) if seedTime >= 0 { t.Status = "offline download completed, waiting for seeding" <-time.After(time.Minute * time.Duration(seedTime)) err := t.tool.Remove(t) if err != nil { log.Errorln(err.Error()) } } } if t.tool.Name() == "Transmission" { // hack for transmission seedTime := setting.GetInt(conf.TransmissionSeedtime, 0) if seedTime >= 0 { t.Status = "offline download completed, waiting for seeding" <-time.After(time.Minute * time.Duration(seedTime)) err := t.tool.Remove(t) if err != nil { log.Errorln(err.Error()) } } } return nil } // Update download status, return true if download completed func (t *DownloadTask) Update() (bool, error) { info, err := t.tool.Status(t) if err != nil { t.callStatusRetried++ log.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) return false, nil } if t.callStatusRetried > 5 { return true, errors.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) } t.callStatusRetried = 0 t.SetProgress(info.Progress) t.SetTotalBytes(info.TotalBytes) t.Status = fmt.Sprintf("[%s]: %s", t.tool.Name(), info.Status) if info.NewGID != "" { log.Debugf("followen by: %+v", info.NewGID) t.GID = info.NewGID return false, nil } // if download completed if info.Completed { err := t.Transfer() return true, errors.WithMessage(err, "failed to transfer file") } // if download failed if info.Err != nil { return true, errors.Errorf("failed to download %s, error: %s", t.ID, info.Err.Error()) } return false, nil } func (t *DownloadTask) Transfer() error { toolName := t.tool.Name() if toolName == "115 Cloud" || toolName == "115 Open" || toolName == "123 Open" || toolName == "123Pan" || toolName == "PikPak" || toolName == "Thunder" || toolName == "ThunderX" || toolName == "ThunderBrowser" { // 如果不是直接下载到目标路径，则进行转存 if t.TempDir != t.DstDirPath { return transferObj(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) } return nil } if t.DeletePolicy == UploadDownloadStream { dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(t.DstDirPath) if err != nil { return errors.WithMessage(err, "failed get dst storage") } taskCreator, _ := t.Ctx().Value(conf.UserKey).(*model.User) tsk := &TransferTask{ TaskData: fs.TaskData{ TaskExtension: task.TaskExtension{ Creator: taskCreator, ApiUrl: t.ApiUrl, }, SrcActualPath: t.TempDir, DstActualPath: dstDirActualPath, DstStorage: dstStorage, DstStorageMp: dstStorage.GetStorage().MountPath, }, DeletePolicy: t.DeletePolicy, Url: t.Url, } tsk.SetTotalBytes(t.GetTotalBytes()) tsk.groupID = path.Join(tsk.DstStorageMp, tsk.DstActualPath) task_group.TransferCoordinator.AddTask(tsk.groupID, nil) TransferTaskManager.Add(tsk) return nil } return transferStd(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) } func (t *DownloadTask) GetName() string { return fmt.Sprintf("download %s to (%s)", t.Url, t.DstDirPath) } func (t *DownloadTask) GetStatus() string { return t.Status } var DownloadTaskManager *tache.Manager[*DownloadTask] ================================================ FILE: internal/offline_download/tool/tools.go ================================================ package tool import ( "fmt" "sort" "github.com/OpenListTeam/OpenList/v4/internal/model" ) var ( Tools = make(ToolsManager) ) type ToolsManager map[string]Tool func (t ToolsManager) Get(name string) (Tool, error) { if tool, ok := t[name]; ok { return tool, nil } return nil, fmt.Errorf("tool %s not found", name) } func (t ToolsManager) Add(tool Tool) { t[tool.Name()] = tool } func (t ToolsManager) Names() []string { names := make([]string, 0, len(t)) for name := range t { if tool, err := t.Get(name); err == nil && tool.IsReady() { names = append(names, name) } } sort.Strings(names) return names } func (t ToolsManager) Items() []model.SettingItem { var items []model.SettingItem for _, tool := range t { items = append(items, tool.Items()...) } return items } ================================================ FILE: internal/offline_download/tool/transfer.go ================================================ package tool import ( "context" "fmt" "os" "path" stdpath "path" "path/filepath" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/internal/task_group" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) type TransferTask struct { fs.TaskData DeletePolicy DeletePolicy `json:"delete_policy"` Url string `json:"url"` groupID string `json:"-"` } func (t *TransferTask) Run() error { if t.SrcStorage == nil && t.SrcStorageMp != "" { if srcStorage, _, err := op.GetStorageAndActualPath(t.SrcStorageMp); err == nil { t.SrcStorage = srcStorage } else { return err } if t.DstStorage == nil { if dstStorage, _, err := op.GetStorageAndActualPath(t.DstStorageMp); err == nil { t.DstStorage = dstStorage } else { return err } } } t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() if t.SrcStorage == nil { if t.DeletePolicy == UploadDownloadStream { rr, err := stream.GetRangeReaderFromLink(t.GetTotalBytes(), &model.Link{URL: t.Url}) if err != nil { return err } r, err := rr.RangeRead(t.Ctx(), http_range.Range{Length: t.GetTotalBytes()}) if err != nil { return err } name := t.SrcActualPath mimetype := utils.GetMimeType(name) s := &stream.FileStream{ Ctx: t.Ctx(), Obj: &model.Object{ Name: name, Size: t.GetTotalBytes(), Modified: time.Now(), IsFolder: false, }, Reader: r, Mimetype: mimetype, Closers: utils.NewClosers(r), } return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, s, t.SetProgress) } return transferStdPath(t) } return transferObjPath(t) } func (t *TransferTask) GetName() string { if t.DeletePolicy == UploadDownloadStream { return fmt.Sprintf("upload [%s](%s) to [%s](%s)", t.SrcActualPath, t.Url, t.DstStorageMp, t.DstActualPath) } return fmt.Sprintf("transfer [%s](%s) to [%s](%s)", t.SrcStorageMp, t.SrcActualPath, t.DstStorageMp, t.DstActualPath) } func (t *TransferTask) OnSucceeded() { if t.DeletePolicy == DeleteOnUploadSucceed || t.DeletePolicy == DeleteAlways { if t.SrcStorage == nil { removeStdTemp(t) } else { removeObjTemp(t) } } task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, true) } func (t *TransferTask) OnFailed() { if t.DeletePolicy == DeleteOnUploadFailed || t.DeletePolicy == DeleteAlways { if t.SrcStorage == nil { removeStdTemp(t) } else { removeObjTemp(t) } } task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, false) } func (t *TransferTask) SetRetry(retry int, maxRetry int) { if retry == 0 && (len(t.groupID) == 0 || // 重启恢复 (t.GetErr() == nil && t.GetState() != tache.StatePending)) { // 手动重试 t.groupID = stdpath.Join(t.DstStorageMp, t.DstActualPath) task_group.TransferCoordinator.AddTask(t.groupID, nil) } t.TaskData.SetRetry(retry, maxRetry) } var ( TransferTaskManager *tache.Manager[*TransferTask] ) func transferStd(ctx context.Context, tempDir, dstDirPath string, deletePolicy DeletePolicy) error { dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { return errors.WithMessage(err, "failed get dst storage") } entries, err := os.ReadDir(tempDir) if err != nil { return err } taskCreator, _ := ctx.Value(conf.UserKey).(*model.User) for _, entry := range entries { t := &TransferTask{ TaskData: fs.TaskData{ TaskExtension: task.TaskExtension{ Creator: taskCreator, ApiUrl: common.GetApiUrl(ctx), }, SrcActualPath: stdpath.Join(tempDir, entry.Name()), DstActualPath: dstDirActualPath, DstStorage: dstStorage, DstStorageMp: dstStorage.GetStorage().MountPath, }, DeletePolicy: deletePolicy, } t.groupID = path.Join(t.DstStorageMp, t.DstActualPath) task_group.TransferCoordinator.AddTask(t.groupID, nil) TransferTaskManager.Add(t) } return nil } func transferStdPath(t *TransferTask) error { t.Status = "getting src object" info, err := os.Stat(t.SrcActualPath) if err != nil { return err } if info.IsDir() { t.Status = "src object is dir, listing objs" entries, err := os.ReadDir(t.SrcActualPath) if err != nil { return err } dstDirActualPath := stdpath.Join(t.DstActualPath, info.Name()) task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstDirActualPath)) for _, entry := range entries { srcRawPath := stdpath.Join(t.SrcActualPath, entry.Name()) task := &TransferTask{ TaskData: fs.TaskData{ TaskExtension: task.TaskExtension{ Creator: t.Creator, ApiUrl: t.ApiUrl, }, SrcActualPath: srcRawPath, DstActualPath: dstDirActualPath, DstStorage: t.DstStorage, SrcStorageMp: t.SrcStorageMp, DstStorageMp: t.DstStorageMp, }, groupID: t.groupID, DeletePolicy: t.DeletePolicy, } task_group.TransferCoordinator.AddTask(t.groupID, nil) TransferTaskManager.Add(task) } t.Status = "src object is dir, added all transfer tasks of files" return nil } return transferStdFile(t) } func transferStdFile(t *TransferTask) error { rc, err := os.Open(t.SrcActualPath) if err != nil { return errors.Wrapf(err, "failed to open file %s", t.SrcActualPath) } info, err := rc.Stat() if err != nil { return errors.Wrapf(err, "failed to get file %s", t.SrcActualPath) } mimetype := utils.GetMimeType(t.SrcActualPath) s := &stream.FileStream{ Ctx: t.Ctx(), Obj: &model.Object{ Name: filepath.Base(t.SrcActualPath), Size: info.Size(), Modified: info.ModTime(), IsFolder: false, }, Reader: rc, Mimetype: mimetype, Closers: utils.NewClosers(rc), } t.SetTotalBytes(info.Size()) return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, s, t.SetProgress) } func removeStdTemp(t *TransferTask) { info, err := os.Stat(t.SrcActualPath) if err != nil || info.IsDir() { return } if err := os.Remove(t.SrcActualPath); err != nil { log.Errorf("failed to delete temp file %s, error: %s", t.SrcActualPath, err.Error()) } } func transferObj(ctx context.Context, tempDir, dstDirPath string, deletePolicy DeletePolicy) error { srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(tempDir) if err != nil { return errors.WithMessage(err, "failed get src storage") } dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { return errors.WithMessage(err, "failed get dst storage") } objs, err := op.List(ctx, srcStorage, srcObjActualPath, model.ListArgs{}) if err != nil { return errors.WithMessagef(err, "failed list src [%s] objs", tempDir) } taskCreator, _ := ctx.Value(conf.UserKey).(*model.User) // taskCreator is nil when convert failed for _, obj := range objs { t := &TransferTask{ TaskData: fs.TaskData{ TaskExtension: task.TaskExtension{ Creator: taskCreator, ApiUrl: common.GetApiUrl(ctx), }, SrcActualPath: stdpath.Join(srcObjActualPath, obj.GetName()), DstActualPath: dstDirActualPath, SrcStorage: srcStorage, DstStorage: dstStorage, SrcStorageMp: srcStorage.GetStorage().MountPath, DstStorageMp: dstStorage.GetStorage().MountPath, }, DeletePolicy: deletePolicy, } t.groupID = path.Join(t.DstStorageMp, t.DstActualPath) task_group.TransferCoordinator.AddTask(t.groupID, nil) TransferTaskManager.Add(t) } return nil } func transferObjPath(t *TransferTask) error { t.Status = "getting src object" srcObj, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcActualPath) if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", t.SrcActualPath) } if srcObj.IsDir() { t.Status = "src object is dir, listing objs" objs, err := op.List(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.ListArgs{}) if err != nil { return errors.WithMessagef(err, "failed list src [%s] objs", t.SrcActualPath) } dstDirActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName()) task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstDirActualPath)) for _, obj := range objs { if utils.IsCanceled(t.Ctx()) { return nil } srcObjPath := stdpath.Join(t.SrcActualPath, obj.GetName()) task_group.TransferCoordinator.AddTask(t.groupID, nil) TransferTaskManager.Add(&TransferTask{ TaskData: fs.TaskData{ TaskExtension: task.TaskExtension{ Creator: t.Creator, ApiUrl: t.ApiUrl, }, SrcActualPath: srcObjPath, DstActualPath: dstDirActualPath, SrcStorage: t.SrcStorage, DstStorage: t.DstStorage, SrcStorageMp: t.SrcStorageMp, DstStorageMp: t.DstStorageMp, }, groupID: t.groupID, DeletePolicy: t.DeletePolicy, }) } t.Status = "src object is dir, added all transfer tasks of objs" return nil } return transferObjFile(t) } func transferObjFile(t *TransferTask) error { _, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcActualPath) if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", t.SrcActualPath) } link, srcFile, err := op.Link(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.LinkArgs{}) if err != nil { return errors.WithMessagef(err, "failed get [%s] link", t.SrcActualPath) } // any link provided is seekable ss, err := stream.NewSeekableStream(&stream.FileStream{ Obj: srcFile, Ctx: t.Ctx(), }, link) if err != nil { _ = link.Close() return errors.WithMessagef(err, "failed get [%s] stream", t.SrcActualPath) } t.SetTotalBytes(ss.GetSize()) return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, ss, t.SetProgress) } func removeObjTemp(t *TransferTask) { srcObj, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcActualPath) if err != nil || srcObj.IsDir() { return } if err := op.Remove(t.Ctx(), t.SrcStorage, t.SrcActualPath); err != nil { log.Errorf("failed to delete temp obj %s, error: %s", t.SrcActualPath, err.Error()) } } ================================================ FILE: internal/offline_download/transmission/client.go ================================================ package transmission import ( "bytes" "context" "encoding/base64" "fmt" "net/http" "net/url" "strconv" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/hekmon/transmissionrpc/v3" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) type Transmission struct { client *transmissionrpc.Client } func (t *Transmission) Run(task *tool.DownloadTask) error { return errs.NotSupport } func (t *Transmission) Name() string { return "Transmission" } func (t *Transmission) Items() []model.SettingItem { // transmission settings return []model.SettingItem{ {Key: conf.TransmissionUri, Value: "http://localhost:9091/transmission/rpc", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.TransmissionSeedtime, Value: "0", Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } } func (t *Transmission) Init() (string, error) { t.client = nil uri := setting.GetStr(conf.TransmissionUri) endpoint, err := url.Parse(uri) if err != nil { return "", errors.Wrap(err, "failed to init transmission client") } c, err := transmissionrpc.New(endpoint, nil) if err != nil { return "", errors.Wrap(err, "failed to init transmission client") } ok, serverVersion, serverMinimumVersion, err := c.RPCVersion(context.Background()) if err != nil { return "", errors.Wrapf(err, "failed get transmission version") } if !ok { return "", fmt.Errorf("remote transmission RPC version (v%d) is incompatible with the transmission library (v%d): remote needs at least v%d", serverVersion, transmissionrpc.RPCVersion, serverMinimumVersion) } t.client = c log.Infof("remote transmission RPC version (v%d) is compatible with our transmissionrpc library (v%d)\n", serverVersion, transmissionrpc.RPCVersion) log.Infof("using transmission version: %d", serverVersion) return fmt.Sprintf("transmission version: %d", serverVersion), nil } func (t *Transmission) IsReady() bool { return t.client != nil } func (t *Transmission) AddURL(args *tool.AddUrlArgs) (string, error) { endpoint, err := url.Parse(args.Url) if err != nil { return "", errors.Wrap(err, "failed to parse transmission uri") } rpcPayload := transmissionrpc.TorrentAddPayload{ DownloadDir: &args.TempDir, } // http url for .torrent file if endpoint.Scheme == "http" || endpoint.Scheme == "https" { resp, err := http.Get(args.Url) if err != nil { return "", errors.Wrap(err, "failed to get .torrent file") } defer resp.Body.Close() buffer := new(bytes.Buffer) encoder := base64.NewEncoder(base64.StdEncoding, buffer) // Stream file to the encoder if _, err = utils.CopyWithBuffer(encoder, resp.Body); err != nil { return "", errors.Wrap(err, "can't copy file content into the base64 encoder") } // Flush last bytes if err = encoder.Close(); err != nil { return "", errors.Wrap(err, "can't flush last bytes of the base64 encoder") } // Get the string form b64 := buffer.String() rpcPayload.MetaInfo = &b64 } else { // magnet uri rpcPayload.Filename = &args.Url } torrent, err := t.client.TorrentAdd(context.TODO(), rpcPayload) if err != nil { return "", err } if torrent.ID == nil { return "", fmt.Errorf("failed get torrent ID") } gid := strconv.FormatInt(*torrent.ID, 10) return gid, nil } func (t *Transmission) Remove(task *tool.DownloadTask) error { gid, err := strconv.ParseInt(task.GID, 10, 64) if err != nil { return err } err = t.client.TorrentRemove(context.TODO(), transmissionrpc.TorrentRemovePayload{ IDs: []int64{gid}, DeleteLocalData: false, }) return err } func (t *Transmission) Status(task *tool.DownloadTask) (*tool.Status, error) { gid, err := strconv.ParseInt(task.GID, 10, 64) if err != nil { return nil, err } infos, err := t.client.TorrentGetAllFor(context.TODO(), []int64{gid}) if err != nil { return nil, err } if len(infos) < 1 { return nil, fmt.Errorf("failed get status, wrong gid: %s", task.GID) } info := infos[0] s := &tool.Status{ Completed: *info.IsFinished, Err: err, } s.Progress = *info.PercentDone * 100 s.TotalBytes = int64(*info.SizeWhenDone / 8) switch *info.Status { case transmissionrpc.TorrentStatusCheckWait, transmissionrpc.TorrentStatusDownloadWait, transmissionrpc.TorrentStatusCheck, transmissionrpc.TorrentStatusDownload, transmissionrpc.TorrentStatusIsolated: s.Status = "[transmission] " + info.Status.String() case transmissionrpc.TorrentStatusSeedWait, transmissionrpc.TorrentStatusSeed: s.Completed = true case transmissionrpc.TorrentStatusStopped: s.Err = errors.Errorf("[transmission] failed to download %s, status: %s, error: %s", task.GID, info.Status.String(), *info.ErrorString) default: s.Err = errors.Errorf("[transmission] unknown status occurred downloading %s, err: %s", task.GID, *info.ErrorString) } return s, nil } var _ tool.Tool = (*Transmission)(nil) func init() { tool.Tools.Add(&Transmission{}) } ================================================ FILE: internal/op/archive.go ================================================ package op import ( "context" stderrors "errors" "io" stdpath "path" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" "github.com/OpenListTeam/OpenList/v4/internal/cache" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" gocache "github.com/OpenListTeam/go-cache" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) var ( archiveMetaCache = gocache.NewMemCache(gocache.WithShards[*model.ArchiveMetaProvider](64)) archiveMetaG singleflight.Group[*model.ArchiveMetaProvider] ) func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) key := Key(storage, path) fn := func() (*model.ArchiveMetaProvider, error) { _, m, err := getArchiveMeta(ctx, storage, path, args) if err != nil { return nil, errors.Wrapf(err, "failed to get %s archive met: %+v", path, err) } if m.Expiration != nil { archiveMetaCache.Set(key, m, gocache.WithEx[*model.ArchiveMetaProvider](*m.Expiration)) } return m, nil } // if storage.Config().NoLinkSingleflight { // meta, err := fn() // return meta, err // } if !args.Refresh { if meta, ok := archiveMetaCache.Get(key); ok { log.Debugf("use cache when get %s archive meta", path) return meta, nil } } meta, err, _ := archiveMetaG.Do(key, fn) return meta, err } func GetArchiveToolAndStream(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (model.Obj, tool.Tool, []*stream.SeekableStream, error) { l, obj, err := Link(ctx, storage, path, args) if err != nil { return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] link", path) } // Get archive tool var partExt *tool.MultipartExtension var t tool.Tool ext := obj.GetName() for { var found bool _, ext, found = strings.Cut(ext, ".") if !found { _ = l.Close() return nil, nil, nil, errors.Errorf("failed get archive tool: the obj does not have an extension.") } partExt, t, err = tool.GetArchiveTool("." + ext) if err == nil { break } } // Get first part stream ss, err := stream.NewSeekableStream(&stream.FileStream{Ctx: ctx, Obj: obj}, l) if err != nil { _ = l.Close() return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] stream", path) } ret := []*stream.SeekableStream{ss} if partExt == nil { return obj, t, ret, nil } // Merge multi-part archive dir := stdpath.Dir(path) objs, err := List(ctx, storage, dir, model.ListArgs{}) if err != nil { return obj, t, ret, nil } for _, o := range objs { submatch := partExt.PartFileFormat.FindStringSubmatch(o.GetName()) if submatch == nil { continue } partIdx, e := strconv.Atoi(submatch[1]) if e != nil { continue } partIdx = partIdx - partExt.SecondPartIndex + 1 if partIdx < 1 { continue } p := stdpath.Join(dir, o.GetName()) l1, o1, e := Link(ctx, storage, p, args) if e != nil { err = errors.WithMessagef(e, "failed get [%s] link", p) break } ss1, e := stream.NewSeekableStream(&stream.FileStream{Ctx: ctx, Obj: o1}, l1) if e != nil { _ = l1.Close() err = errors.WithMessagef(e, "failed get [%s] stream", p) break } for partIdx >= len(ret) { ret = append(ret, nil) } ret[partIdx] = ss1 } closeAll := func(r []*stream.SeekableStream) { for _, s := range r { if s != nil { _ = s.Close() } } } if err != nil { closeAll(ret) return nil, nil, nil, err } for i, ss1 := range ret { if ss1 == nil { closeAll(ret) return nil, nil, nil, errors.Errorf("failed merge [%s] parts, missing part %d", path, i) } } return obj, t, ret, nil } func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (model.Obj, *model.ArchiveMetaProvider, error) { storageAr, ok := storage.(driver.ArchiveReader) if ok { obj, err := GetUnwrap(ctx, storage, path) if err != nil { return nil, nil, errors.WithMessage(err, "failed to get file") } if obj.IsDir() { return nil, nil, errors.WithStack(errs.NotFile) } meta, err := storageAr.GetArchiveMeta(ctx, obj, args.ArchiveArgs) if !errors.Is(err, errs.NotImplement) { archiveMetaProvider := &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: true} if meta != nil && meta.GetTree() != nil { archiveMetaProvider.Sort = &storage.GetStorage().Sort } if !storage.Config().NoCache { Expiration := time.Minute * time.Duration(storage.GetStorage().CacheExpiration) archiveMetaProvider.Expiration = &Expiration } return obj, archiveMetaProvider, err } } obj, t, ss, err := GetArchiveToolAndStream(ctx, storage, path, args.LinkArgs) if err != nil { return nil, nil, err } defer func() { var e error for _, s := range ss { e = stderrors.Join(e, s.Close()) } if e != nil { log.Errorf("failed to close file streamer, %v", e) } }() meta, err := t.GetMeta(ss, args.ArchiveArgs) if err != nil { return nil, nil, err } archiveMetaProvider := &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: false} if meta.GetTree() != nil { archiveMetaProvider.Sort = &storage.GetStorage().Sort } if !storage.Config().NoCache { Expiration := time.Minute * time.Duration(storage.GetStorage().CacheExpiration) archiveMetaProvider.Expiration = &Expiration } return obj, archiveMetaProvider, err } var ( archiveListCache = gocache.NewMemCache(gocache.WithShards[[]model.Obj](64)) archiveListG singleflight.Group[[]model.Obj] ) func ListArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) metaKey := Key(storage, path) key := stdpath.Join(metaKey, args.InnerPath) if !args.Refresh { if files, ok := archiveListCache.Get(key); ok { log.Debugf("use cache when list archive [%s]%s", path, args.InnerPath) return files, nil } // if meta, ok := archiveMetaCache.Get(metaKey); ok { // log.Debugf("use meta cache when list archive [%s]%s", path, args.InnerPath) // return getChildrenFromArchiveMeta(meta, args.InnerPath) // } } objs, err, _ := archiveListG.Do(key, func() ([]model.Obj, error) { files, err := listArchive(ctx, storage, path, args) if err != nil { return nil, errors.Wrapf(err, "failed to list archive [%s]%s: %+v", path, args.InnerPath, err) } // warp obj name model.WrapObjsName(files) // sort objs if storage.Config().LocalSort { model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) } model.ExtractFolder(files, storage.GetStorage().ExtractFolder) if !storage.Config().NoCache { if len(files) > 0 { log.Debugf("set cache: %s => %+v", key, files) archiveListCache.Set(key, files, gocache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) } else { log.Debugf("del cache: %s", key) archiveListCache.Del(key) } } return files, nil }) return objs, err } func _listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) { storageAr, ok := storage.(driver.ArchiveReader) if ok { obj, err := GetUnwrap(ctx, storage, path) if err != nil { return nil, errors.WithMessage(err, "failed to get file") } if obj.IsDir() { return nil, errors.WithStack(errs.NotFile) } files, err := storageAr.ListArchive(ctx, obj, args.ArchiveInnerArgs) if !errors.Is(err, errs.NotImplement) { return files, err } } _, t, ss, err := GetArchiveToolAndStream(ctx, storage, path, args.LinkArgs) if err != nil { return nil, err } defer func() { var e error for _, s := range ss { e = stderrors.Join(e, s.Close()) } if e != nil { log.Errorf("failed to close file streamer, %v", e) } }() files, err := t.List(ss, args.ArchiveInnerArgs) return files, err } func listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) { files, err := _listArchive(ctx, storage, path, args) if errors.Is(err, errs.NotSupport) { var meta model.ArchiveMeta meta, err = GetArchiveMeta(ctx, storage, path, model.ArchiveMetaArgs{ ArchiveArgs: args.ArchiveArgs, Refresh: args.Refresh, }) if err != nil { return nil, err } files, err = getChildrenFromArchiveMeta(meta, args.InnerPath) if err != nil { return nil, err } } if err != nil { return nil, err } return files, err } func getChildrenFromArchiveMeta(meta model.ArchiveMeta, innerPath string) ([]model.Obj, error) { obj := meta.GetTree() if obj == nil { return nil, errors.WithStack(errs.NotImplement) } dirs := splitPath(innerPath) for _, dir := range dirs { var next model.ObjTree for _, c := range obj { if c.GetName() == dir { next = c break } } if next == nil { return nil, errors.WithStack(errs.ObjectNotFound) } if !next.IsDir() || next.GetChildren() == nil { return nil, errors.WithStack(errs.NotFolder) } obj = next.GetChildren() } return utils.SliceConvert(obj, func(src model.ObjTree) (model.Obj, error) { return src, nil }) } func splitPath(path string) []string { var parts []string for { dir, file := stdpath.Split(path) if file == "" { break } parts = append([]string{file}, parts...) path = strings.TrimSuffix(dir, "/") } return parts } func ArchiveGet(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) af, err := GetUnwrap(ctx, storage, path) if err != nil { return nil, nil, errors.WithMessage(err, "failed to get file") } if af.IsDir() { return nil, nil, errors.WithStack(errs.NotFile) } if g, ok := storage.(driver.ArchiveGetter); ok { obj, err := g.ArchiveGet(ctx, af, args.ArchiveInnerArgs) if err == nil { return af, model.WrapObjName(obj), nil } } if utils.PathEqual(args.InnerPath, "/") { return af, &model.ObjWrapName{ Name: RootName, Obj: &model.Object{ Name: af.GetName(), Path: af.GetPath(), ID: af.GetID(), Size: af.GetSize(), Modified: af.ModTime(), IsFolder: true, }, }, nil } innerDir, name := stdpath.Split(args.InnerPath) args.InnerPath = strings.TrimSuffix(innerDir, "/") files, err := ListArchive(ctx, storage, path, args) if err != nil { return nil, nil, errors.WithMessage(err, "failed get parent list") } for _, f := range files { if f.GetName() == name { return af, f, nil } } return nil, nil, errors.WithStack(errs.ObjectNotFound) } type objWithLink struct { link *model.Link obj model.Obj } var ( extractCache = cache.NewKeyedCache[*objWithLink](5 * time.Minute) extractG = singleflight.Group[*objWithLink]{} ) func DriverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } key := stdpath.Join(Key(storage, path), args.InnerPath) if ol, ok := extractCache.Get(key); ok { if ol.link.Expiration != nil || ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { return ol.link, ol.obj, nil } } fn := func() (*objWithLink, error) { ol, err := driverExtract(ctx, storage, path, args) if err != nil { return nil, errors.Wrapf(err, "failed extract archive") } if ol.link.Expiration != nil { extractCache.SetWithTTL(key, ol, *ol.link.Expiration) } else { extractCache.SetWithExpirable(key, ol, &ol.link.SyncClosers) } return ol, nil } for { ol, err, _ := extractG.Do(key, fn) if err != nil { return nil, nil, err } if ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { return ol.link, ol.obj, nil } } } func driverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*objWithLink, error) { storageAr, ok := storage.(driver.ArchiveReader) if !ok { return nil, errs.DriverExtractNotSupported } archiveFile, extracted, err := ArchiveGet(ctx, storage, path, model.ArchiveListArgs{ ArchiveInnerArgs: args, Refresh: false, }) if err != nil { return nil, errors.WithMessage(err, "failed to get file") } if extracted.IsDir() { return nil, errors.WithStack(errs.NotFile) } link, err := storageAr.Extract(ctx, archiveFile, args) return &objWithLink{link: link, obj: extracted}, err } type streamWithParent struct { rc io.ReadCloser parents []*stream.SeekableStream } func (s *streamWithParent) Read(p []byte) (int, error) { return s.rc.Read(p) } func (s *streamWithParent) Close() error { err := s.rc.Close() for _, ss := range s.parents { err = stderrors.Join(err, ss.Close()) } return err } func InternalExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { _, t, ss, err := GetArchiveToolAndStream(ctx, storage, path, args.LinkArgs) if err != nil { return nil, 0, err } rc, size, err := t.Extract(ss, args) if err != nil { var e error for _, s := range ss { e = stderrors.Join(e, s.Close()) } if e != nil { log.Errorf("failed to close file streamer, %v", e) err = stderrors.Join(err, e) } return nil, 0, err } return &streamWithParent{rc: rc, parents: ss}, size, nil } func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, args model.ArchiveDecompressArgs, lazyCache ...bool) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } srcPath = utils.FixAndCleanPath(srcPath) dstDirPath = utils.FixAndCleanPath(dstDirPath) srcObj, err := GetUnwrap(ctx, storage, srcPath) if err != nil { return errors.WithMessage(err, "failed to get src object") } dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { return errors.WithMessage(err, "failed to get dst dir") } var newObjs []model.Obj switch s := storage.(type) { case driver.ArchiveDecompressResult: newObjs, err = s.ArchiveDecompress(ctx, srcObj, dstDir, args) if err == nil { if len(newObjs) > 0 { if !storage.Config().NoCache { if cache, exist := Cache.dirCache.Get(Key(storage, dstDirPath)); exist { for _, newObj := range newObjs { cache.UpdateObject(newObj.GetName(), newObj) } } } } else if !utils.IsBool(lazyCache...) { Cache.DeleteDirectory(storage, dstDirPath) } } case driver.ArchiveDecompress: err = s.ArchiveDecompress(ctx, srcObj, dstDir, args) if err == nil && !utils.IsBool(lazyCache...) { Cache.DeleteDirectory(storage, dstDirPath) } default: return errs.NotImplement } if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() { onlyList := false targetPath := dstDirPath if len(newObjs) == 1 && newObjs[0].IsDir() { targetPath = stdpath.Join(dstDirPath, newObjs[0].GetName()) } else if len(newObjs) == 1 && !newObjs[0].IsDir() { onlyList = true } else if args.PutIntoNewDir { targetPath = stdpath.Join(dstDirPath, strings.TrimSuffix(srcObj.GetName(), stdpath.Ext(srcObj.GetName()))) } else if innerBase := stdpath.Base(args.InnerPath); innerBase != "." && innerBase != "/" { targetPath = stdpath.Join(dstDirPath, innerBase) dstObj, e := Get(ctx, storage, targetPath) onlyList = e != nil || !dstObj.IsDir() } if onlyList { go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true}) } else { var limiter *rate.Limiter if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil { if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 { limiter = rate.NewLimiter(rate.Limit(f), 1) } } go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil) } } return errors.WithStack(err) } ================================================ FILE: internal/op/cache.go ================================================ package op import ( stdpath "path" "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/cache" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type CacheManager struct { dirCache *cache.KeyedCache[*directoryCache] // Cache for directory listings linkCache *cache.TypedCache[*objWithLink] // Cache for file links userCache *cache.KeyedCache[*model.User] // Cache for user data settingCache *cache.KeyedCache[any] // Cache for settings detailCache *cache.KeyedCache[*model.StorageDetails] // Cache for storage details } func NewCacheManager() *CacheManager { return &CacheManager{ dirCache: cache.NewKeyedCache[*directoryCache](time.Minute * 5), linkCache: cache.NewTypedCache[*objWithLink](time.Minute * 30), userCache: cache.NewKeyedCache[*model.User](time.Hour), settingCache: cache.NewKeyedCache[any](time.Hour), detailCache: cache.NewKeyedCache[*model.StorageDetails](time.Minute * 30), } } // global instance var Cache = NewCacheManager() func Key(storage driver.Driver, path string) string { return utils.GetFullPath(storage.GetStorage().MountPath, path) } // recursively delete directory and its children from dirCache func (cm *CacheManager) DeleteDirectoryTree(storage driver.Driver, dirPath string) { if storage.Config().NoCache { return } cm.deleteDirectoryTree(Key(storage, dirPath)) } func (cm *CacheManager) deleteDirectoryTree(key string) { if dirCache, exists := cm.dirCache.Pop(key); exists { for _, obj := range dirCache.objs { if obj.IsDir() { cm.deleteDirectoryTree(stdpath.Join(key, obj.GetName())) } else { cm.linkCache.DeleteKey(stdpath.Join(key, obj.GetName())) } } } } // remove directory from dirCache func (cm *CacheManager) DeleteDirectory(storage driver.Driver, dirPath string) { if storage.Config().NoCache { return } cm.dirCache.Delete(Key(storage, dirPath)) } // remove object from dirCache. // if it's a directory, remove all its children from dirCache too. // if it's a file, remove its link from linkCache. func (cm *CacheManager) removeDirectoryObject(storage driver.Driver, dirPath string, obj model.Obj) { key := Key(storage, dirPath) if !obj.IsDir() { cm.linkCache.DeleteKey(stdpath.Join(key, obj.GetName())) } if storage.Config().NoCache { return } if cache, exist := cm.dirCache.Get(key); exist { if obj.IsDir() { cm.deleteDirectoryTree(stdpath.Join(key, obj.GetName())) } cache.RemoveObject(obj.GetName()) } } // cache user data func (cm *CacheManager) SetUser(username string, user *model.User) { cm.userCache.Set(username, user) } // cached user data func (cm *CacheManager) GetUser(username string) (*model.User, bool) { return cm.userCache.Get(username) } // remove user data from cache func (cm *CacheManager) DeleteUser(username string) { cm.userCache.Delete(username) } // caches setting func (cm *CacheManager) SetSetting(key string, setting *model.SettingItem) { cm.settingCache.Set(key, setting) } // cached setting func (cm *CacheManager) GetSetting(key string) (*model.SettingItem, bool) { if data, exists := cm.settingCache.Get(key); exists { if setting, ok := data.(*model.SettingItem); ok { return setting, true } } return nil, false } // cache setting groups func (cm *CacheManager) SetSettingGroup(key string, settings []model.SettingItem) { cm.settingCache.Set(key, settings) } // cached setting group func (cm *CacheManager) GetSettingGroup(key string) ([]model.SettingItem, bool) { if data, exists := cm.settingCache.Get(key); exists { if settings, ok := data.([]model.SettingItem); ok { return settings, true } } return nil, false } func (cm *CacheManager) SetStorageDetails(storage driver.Driver, details *model.StorageDetails) { if storage.Config().NoCache { return } expiration := time.Minute * time.Duration(storage.GetStorage().CacheExpiration) cm.detailCache.SetWithTTL(utils.GetActualMountPath(storage.GetStorage().MountPath), details, expiration) } func (cm *CacheManager) GetStorageDetails(storage driver.Driver) (*model.StorageDetails, bool) { return cm.detailCache.Get(utils.GetActualMountPath(storage.GetStorage().MountPath)) } func (cm *CacheManager) InvalidateStorageDetails(storage driver.Driver) { cm.detailCache.Delete(utils.GetActualMountPath(storage.GetStorage().MountPath)) } // clears all caches func (cm *CacheManager) ClearAll() { cm.dirCache.Clear() cm.linkCache.Clear() cm.userCache.Clear() cm.settingCache.Clear() cm.detailCache.Clear() } type directoryCache struct { objs []model.Obj sorted []model.Obj mu sync.RWMutex dirtyFlags uint8 } const ( dirtyRemove uint8 = 1 << iota // 对象删除：刷新 sorted 副本，但不需要 full sort/extract dirtyUpdate // 对象更新：需要执行 full sort + extract ) func newDirectoryCache(objs []model.Obj) *directoryCache { sorted := make([]model.Obj, len(objs)) copy(sorted, objs) return &directoryCache{ objs: objs, sorted: sorted, } } func (dc *directoryCache) RemoveObject(name string) { dc.mu.Lock() defer dc.mu.Unlock() for i, obj := range dc.objs { if obj.GetName() == name { dc.objs = append(dc.objs[:i], dc.objs[i+1:]...) dc.dirtyFlags |= dirtyRemove break } } } func (dc *directoryCache) UpdateObject(oldName string, newObj model.Obj) { dc.mu.Lock() defer dc.mu.Unlock() if oldName != "" { for i, obj := range dc.objs { if obj.GetName() == oldName { dc.objs[i] = newObj dc.dirtyFlags |= dirtyUpdate return } } } dc.objs = append(dc.objs, newObj) dc.dirtyFlags |= dirtyUpdate } func (dc *directoryCache) GetSortedObjects(meta driver.Meta) []model.Obj { dc.mu.RLock() if dc.dirtyFlags == 0 { dc.mu.RUnlock() return dc.sorted } dc.mu.RUnlock() dc.mu.Lock() defer dc.mu.Unlock() sorted := make([]model.Obj, len(dc.objs)) copy(sorted, dc.objs) dc.sorted = sorted if dc.dirtyFlags&dirtyUpdate != 0 { storage := meta.GetStorage() if meta.Config().LocalSort { model.SortFiles(sorted, storage.OrderBy, storage.OrderDirection) } model.ExtractFolder(sorted, storage.ExtractFolder) } dc.dirtyFlags = 0 return sorted } ================================================ FILE: internal/op/const.go ================================================ package op const ( WORK = "work" DISABLED = "disabled" RootName = "root" ) ================================================ FILE: internal/op/driver.go ================================================ package op import ( "reflect" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/pkg/errors" ) type DriverConstructor func() driver.Driver var driverMap = map[string]DriverConstructor{} var driverInfoMap = map[string]driver.Info{} func RegisterDriver(driver DriverConstructor) { // log.Infof("register driver: [%s]", config.Name) tempDriver := driver() tempConfig := tempDriver.Config() registerDriverItems(tempConfig, tempDriver.GetAddition()) driverMap[tempConfig.Name] = driver } func GetDriver(name string) (DriverConstructor, error) { n, ok := driverMap[name] if !ok { return nil, errors.Errorf("no driver named: %s", name) } return n, nil } func GetDriverNames() []string { var driverNames []string for k := range driverInfoMap { driverNames = append(driverNames, k) } return driverNames } func GetDriverInfoMap() map[string]driver.Info { return driverInfoMap } func registerDriverItems(config driver.Config, addition driver.Additional) { // log.Debugf("addition of %s: %+v", config.Name, addition) tAddition := reflect.TypeOf(addition) for tAddition.Kind() == reflect.Pointer { tAddition = tAddition.Elem() } mainItems := getMainItems(config) additionalItems := getAdditionalItems(tAddition, config.DefaultRoot) driverInfoMap[config.Name] = driver.Info{ Common: mainItems, Additional: additionalItems, Config: config, } } func getMainItems(config driver.Config) []driver.Item { items := []driver.Item{{ Name: "mount_path", Type: conf.TypeString, Required: true, Help: "The path you want to mount to, it is unique and cannot be repeated", }, { Name: "order", Type: conf.TypeNumber, Help: "use to sort", }, { Name: "remark", Type: conf.TypeText, }} if !config.NoCache { items = append(items, driver.Item{ Name: "cache_expiration", Type: conf.TypeNumber, Default: "30", Required: true, Help: "The cache expiration time for this storage", }) items = append(items, driver.Item{ Name: "custom_cache_policies", Type: conf.TypeText, Default: "", Required: false, Help: "The cache expiration rules for this storage", }) } if config.MustProxy() { items = append(items, driver.Item{ Name: "webdav_policy", Type: conf.TypeSelect, Default: "native_proxy", Options: "use_proxy_url,native_proxy", Required: true, }) } else { if config.DefaultProxy() { items = append(items, []driver.Item{{ Name: "web_proxy", Type: conf.TypeBool, Default: "true", }, { Name: "webdav_policy", Type: conf.TypeSelect, Options: "302_redirect,use_proxy_url,native_proxy", Default: "native_proxy", Required: true, }, }...) } else { items = append(items, []driver.Item{{ Name: "web_proxy", Type: conf.TypeBool, }, { Name: "webdav_policy", Type: conf.TypeSelect, Options: "302_redirect,use_proxy_url,native_proxy", Default: "302_redirect", Required: true, }, }...) } if config.ProxyRangeOption { item := driver.Item{ Name: "proxy_range", Type: conf.TypeBool, Help: "Need to enable proxy", } if config.Name == "139Yun" { item.Default = "true" } items = append(items, item) } } items = append(items, driver.Item{ Name: "down_proxy_url", Type: conf.TypeText, }) items = append(items, driver.Item{ Name: "disable_proxy_sign", Type: conf.TypeBool, Default: "false", Help: "Disable sign for Download proxy URL", }) if config.LocalSort { items = append(items, []driver.Item{{ Name: "order_by", Type: conf.TypeSelect, Options: "name,size,modified", }, { Name: "order_direction", Type: conf.TypeSelect, Options: "asc,desc", }}...) } items = append(items, driver.Item{ Name: "extract_folder", Type: conf.TypeSelect, Options: "front,back", }) items = append(items, driver.Item{ Name: "disable_index", Type: conf.TypeBool, Default: "false", Required: true, }) items = append(items, driver.Item{ Name: "enable_sign", Type: conf.TypeBool, Default: "false", Required: true, }) return items } func getAdditionalItems(t reflect.Type, defaultRoot string) []driver.Item { var items []driver.Item for i := 0; i < t.NumField(); i++ { field := t.Field(i) if field.Type.Kind() == reflect.Struct { items = append(items, getAdditionalItems(field.Type, defaultRoot)...) continue } tag := field.Tag ignore, ok1 := tag.Lookup("ignore") name, ok2 := tag.Lookup("json") if (ok1 && ignore == "true") || !ok2 { continue } item := driver.Item{ Name: name, Type: strings.ToLower(field.Type.Name()), Default: tag.Get("default"), Options: tag.Get("options"), Required: tag.Get("required") == "true", Help: tag.Get("help"), } if tag.Get("type") != "" { item.Type = tag.Get("type") } if item.Name == "root_folder_id" || item.Name == "root_folder_path" { if item.Default == "" { item.Default = defaultRoot } item.Required = item.Default != "" } // set default type to string if item.Type == "" { item.Type = "string" } items = append(items, item) } return items } ================================================ FILE: internal/op/driver_test.go ================================================ package op_test import ( "testing" _ "github.com/OpenListTeam/OpenList/v4/drivers" "github.com/OpenListTeam/OpenList/v4/internal/op" ) func TestDriverItemsMap(t *testing.T) { itemsMap := op.GetDriverInfoMap() if len(itemsMap) != 0 { t.Logf("driverInfoMap: %v", itemsMap) } else { t.Errorf("expected driverInfoMap not empty, but got empty") } } ================================================ FILE: internal/op/fs.go ================================================ package op import ( "context" stdpath "path" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/bmatcuk/doublestar/v4" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) var listG singleflight.Group[[]model.Obj] // List files in storage, not contains virtual file func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs) ([]model.Obj, error) { return list(ctx, storage, path, args, nil) } func list(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, resultValidator func([]model.Obj) error) ([]model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) log.Debugf("op.List %s", path) key := Key(storage, path) if !args.Refresh { if dirCache, exists := Cache.dirCache.Get(key); exists { log.Debugf("use cache when list %s", path) objs := dirCache.GetSortedObjects(storage) if resultValidator != nil { if err := resultValidator(objs); err == nil { return objs, nil } } else { return objs, nil } } } objs, err, _ := listG.Do(key, func() ([]model.Obj, error) { dir, err := GetUnwrap(ctx, storage, path) if err != nil { return nil, errors.WithMessage(err, "failed get dir") } log.Debugf("list dir: %+v", dir) if !dir.IsDir() { return nil, errors.WithStack(errs.NotFolder) } files, err := storage.List(ctx, dir, args) if err != nil { return nil, errors.Wrapf(err, "failed to list objs") } // warp obj name wrapObjsName(storage, files) // sort objs if storage.Config().LocalSort { model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) } model.ExtractFolder(files, storage.GetStorage().ExtractFolder) if !args.SkipHook { // call hooks go func(reqPath string, files []model.Obj) { HandleObjsUpdateHook(context.WithoutCancel(ctx), reqPath, files) }(utils.GetFullPath(storage.GetStorage().MountPath, path), files) } if !storage.Config().NoCache { if len(files) > 0 { log.Debugf("set cache: %s => %+v", key, files) ttl := storage.GetStorage().CacheExpiration customCachePolicies := storage.GetStorage().CustomCachePolicies if len(customCachePolicies) > 0 { configPolicies := strings.Split(customCachePolicies, "\n") for _, configPolicy := range configPolicies { pattern, ttlstr, ok := strings.Cut(strings.TrimSpace(configPolicy), ":") if !ok { log.Warnf("Malformed custom cache policy entry: %s in storage %s for path %s. Expected format: pattern:ttl", configPolicy, storage.GetStorage().MountPath, path) continue } if match, err1 := doublestar.Match(pattern, path); err1 != nil { log.Warnf("Invalid glob pattern in custom cache policy: %s, error: %v", pattern, err1) continue } else if !match { continue } if configTtl, err1 := strconv.ParseInt(ttlstr, 10, 64); err1 == nil { ttl = int(configTtl) break } } } duration := time.Minute * time.Duration(ttl) Cache.dirCache.SetWithTTL(key, newDirectoryCache(files), duration) } else { log.Debugf("del cache: %s", key) Cache.deleteDirectoryTree(key) } } return files, nil }) if err != nil { return nil, err } if resultValidator != nil { if err := resultValidator(objs); err != nil { return nil, err } } return objs, nil } // Get object from list of files func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj ...bool) (model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) log.Debugf("op.Get %s", path) // is root folder if path == "/" { if getRooter, ok := storage.(driver.GetRooter); ok { rootObj, err := getRooter.GetRoot(ctx) if err != nil { return nil, errors.WithMessage(err, "failed get root obj") } return rootObj, nil } switch r := storage.(type) { case driver.IRootId: return &model.Object{ ID: r.GetRootId(), Name: RootName, Modified: storage.GetStorage().Modified, IsFolder: true, Mask: model.Locked, }, nil case driver.IRootPath: return &model.Object{ Path: r.GetRootPath(), Name: RootName, Modified: storage.GetStorage().Modified, Mask: model.Locked, IsFolder: true, }, nil } return nil, errors.New("please implement GetRooter or IRootPath or IRootId interface") } // try get from cache first dir, name := stdpath.Split(path) dirCache, dirCacheExists := Cache.dirCache.Get(Key(storage, dir)) refreshList := false excludeTemp := utils.IsBool(excludeTempObj...) if dirCacheExists { files := dirCache.GetSortedObjects(storage) for _, f := range files { if f.GetName() == name { if excludeTemp && model.ObjHasMask(f, model.Temp) { refreshList = true break } return f, nil } } } // get the obj directly without list so that we can reduce the io if g, ok := storage.(driver.Getter); ok { obj, err := g.Get(ctx, path) if err == nil { return obj, nil } if !errs.IsNotImplementError(err) && !errs.IsNotSupportError(err) { return nil, errors.WithMessage(err, "failed to get obj") } } if !dirCacheExists || refreshList { var obj model.Obj list(ctx, storage, dir, model.ListArgs{Refresh: refreshList}, func(objs []model.Obj) error { for _, f := range objs { if f.GetName() == name { if excludeTemp && model.ObjHasMask(f, model.Temp) { return errs.ObjectNotFound } obj = f return nil } } return nil }) if obj != nil { return obj, nil } } log.Debugf("cant find obj with name: %s", name) return nil, errors.WithStack(errs.ObjectNotFound) } func GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { obj, err := Get(ctx, storage, path, true) if err != nil { return nil, err } return model.UnwrapObjName(obj), err } var linkG = singleflight.Group[*objWithLink]{} // Link get link, if is an url. should have an expiry time func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } mode := storage.Config().LinkCacheMode if mode == -1 { mode = storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(path) } typeKey := args.Type if mode&driver.LinkCacheIP != 0 { typeKey += "/" + args.IP } if mode&driver.LinkCacheUA != 0 { typeKey += "/" + args.Header.Get("User-Agent") } key := Key(storage, path) if ol, exists := Cache.linkCache.GetType(key, typeKey); exists { if ol.link.Expiration != nil || ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { return ol.link, ol.obj, nil } } fn := func() (*objWithLink, error) { file, err := GetUnwrap(ctx, storage, path) if err != nil { return nil, errors.WithMessage(err, "failed to get file") } if file.IsDir() { return nil, errors.WithStack(errs.NotFile) } link, err := storage.Link(ctx, file, args) if err != nil { return nil, errors.Wrapf(err, "failed get link") } ol := &objWithLink{link: link, obj: file} if link.Expiration != nil { Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *link.Expiration) } else { Cache.linkCache.SetTypeWithExpirable(key, typeKey, ol, &link.SyncClosers) } return ol, nil } for { ol, err, _ := linkG.Do(key+"/"+typeKey, fn) if err != nil { return nil, nil, err } if ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { return ol.link, ol.obj, nil } } } // Other api func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (any, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } o, ok := storage.(driver.Other) if !ok { return nil, errs.NotImplement } obj, err := GetUnwrap(ctx, storage, args.Path) if err != nil { return nil, errors.WithMessagef(err, "failed to get obj") } return o.Other(ctx, model.OtherArgs{ Obj: obj, Method: args.Method, Data: args.Data, }) } var mkdirG singleflight.Group[any] func MakeDir(ctx context.Context, storage driver.Driver, path string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) key := Key(storage, path) _, err, _ := mkdirG.Do(key, func() (any, error) { // check if dir exists f, err := Get(ctx, storage, path) if err == nil { if f.IsDir() { return nil, nil } return nil, errors.New("file exists") } if !errs.IsObjectNotFound(err) { return nil, errors.WithMessage(err, "failed to check if dir exists") } parentPath, dirName := stdpath.Split(path) if err = MakeDir(ctx, storage, parentPath); err != nil { return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) } parentDir, err := GetUnwrap(ctx, storage, parentPath) // this should not happen if err != nil { return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) } if model.ObjHasMask(parentDir, model.NoWrite) { return nil, errors.WithStack(errs.PermissionDenied) } var newObj model.Obj switch s := storage.(type) { case driver.MkdirResult: newObj, err = s.MakeDir(ctx, parentDir, dirName) case driver.Mkdir: err = s.MakeDir(ctx, parentDir, dirName) default: return nil, errs.NotImplement } if err != nil { return nil, errors.WithStack(err) } if storage.Config().NoCache { return nil, nil } if dirCache, exist := Cache.dirCache.Get(Key(storage, parentPath)); exist { if newObj == nil { t := time.Now() newObj = &model.Object{ Name: dirName, IsFolder: true, Modified: t, Ctime: t, Mask: model.Temp, } } dirCache.UpdateObject("", wrapObjName(storage, newObj)) } return nil, nil }) return err } func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } srcPath = utils.FixAndCleanPath(srcPath) if utils.PathEqual(srcPath, "/") { return errors.New("move root folder is not allowed") } srcDirPath := stdpath.Dir(srcPath) dstDirPath = utils.FixAndCleanPath(dstDirPath) if dstDirPath == srcDirPath { return errors.New("move in place") } srcRawObj, err := Get(ctx, storage, srcPath, true) if err != nil { return errors.WithMessage(err, "failed to get src object") } if model.ObjHasMask(srcRawObj, model.NoMove) { return errors.WithStack(errs.PermissionDenied) } srcObj := model.UnwrapObjName(srcRawObj) dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { return errors.WithMessage(err, "failed to get dst dir") } if model.ObjHasMask(dstDir, model.NoWrite) { return errors.WithStack(errs.PermissionDenied) } var newObj model.Obj switch s := storage.(type) { case driver.MoveResult: newObj, err = s.Move(ctx, srcObj, dstDir) case driver.Move: err = s.Move(ctx, srcObj, dstDir) default: err = errs.NotImplement } if err != nil { return errors.WithStack(err) } srcKey := Key(storage, srcDirPath) dstKey := Key(storage, dstDirPath) if !srcRawObj.IsDir() { Cache.linkCache.DeleteKey(stdpath.Join(srcKey, srcRawObj.GetName())) Cache.linkCache.DeleteKey(stdpath.Join(dstKey, srcRawObj.GetName())) } if !storage.Config().NoCache { if cache, exist := Cache.dirCache.Get(srcKey); exist { if srcRawObj.IsDir() { Cache.deleteDirectoryTree(stdpath.Join(srcKey, srcRawObj.GetName())) } cache.RemoveObject(srcRawObj.GetName()) } if cache, exist := Cache.dirCache.Get(dstKey); exist { if newObj == nil { newObj = &model.ObjWrapMask{Obj: srcRawObj, Mask: model.Temp} } else { newObj = wrapObjName(storage, newObj) } cache.UpdateObject(srcRawObj.GetName(), newObj) } } if ctx.Value(conf.SkipHookKey) != nil || !needHandleObjsUpdateHook() { return nil } if !srcObj.IsDir() { go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) } else { go objsUpdateHook(context.WithoutCancel(ctx), storage, stdpath.Join(dstDirPath, srcObj.GetName()), true) } return nil } func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } srcPath = utils.FixAndCleanPath(srcPath) if utils.PathEqual(srcPath, "/") { return errors.New("rename root folder is not allowed") } srcRawObj, err := Get(ctx, storage, srcPath, true) if err != nil { return errors.WithMessage(err, "failed to get src object") } if model.ObjHasMask(srcRawObj, model.NoRename) { return errors.WithStack(errs.PermissionDenied) } srcObj := model.UnwrapObjName(srcRawObj) var newObj model.Obj switch s := storage.(type) { case driver.RenameResult: newObj, err = s.Rename(ctx, srcObj, dstName) case driver.Rename: err = s.Rename(ctx, srcObj, dstName) default: return errs.NotImplement } if err != nil { return errors.WithStack(err) } dirKey := Key(storage, stdpath.Dir(srcPath)) if !srcRawObj.IsDir() { Cache.linkCache.DeleteKey(stdpath.Join(dirKey, srcRawObj.GetName())) Cache.linkCache.DeleteKey(stdpath.Join(dirKey, dstName)) } if !storage.Config().NoCache { if cache, exist := Cache.dirCache.Get(dirKey); exist { if srcRawObj.IsDir() { Cache.deleteDirectoryTree(stdpath.Join(dirKey, srcRawObj.GetName())) } if newObj == nil { newObj = &model.ObjWrapMask{Obj: &model.ObjWrapName{Name: dstName, Obj: srcObj}, Mask: model.Temp} } newObj = wrapObjName(storage, newObj) cache.UpdateObject(srcRawObj.GetName(), newObj) } } if ctx.Value(conf.SkipHookKey) != nil || !needHandleObjsUpdateHook() { return nil } dstDirPath := stdpath.Dir(srcPath) if !srcObj.IsDir() { go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) } else { go objsUpdateHook(context.WithoutCancel(ctx), storage, stdpath.Join(dstDirPath, srcObj.GetName()), true) } return nil } // Copy Just copy file[s] in a storage func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } srcPath = utils.FixAndCleanPath(srcPath) dstDirPath = utils.FixAndCleanPath(dstDirPath) if dstDirPath == stdpath.Dir(srcPath) { return errors.New("copy in place") } srcRawObj, err := Get(ctx, storage, srcPath, true) if err != nil { return errors.WithMessage(err, "failed to get src object") } // if model.ObjHasMask(srcRawObj, model.NoCopy) { // return errors.WithStack(errs.PermissionDenied) // } srcObj := model.UnwrapObjName(srcRawObj) dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { return errors.WithMessage(err, "failed to get dst dir") } if model.ObjHasMask(dstDir, model.NoWrite) { return errors.WithStack(errs.PermissionDenied) } var newObj model.Obj switch s := storage.(type) { case driver.CopyResult: newObj, err = s.Copy(ctx, srcObj, dstDir) case driver.Copy: err = s.Copy(ctx, srcObj, dstDir) default: err = errs.NotImplement } if err != nil { return errors.WithStack(err) } dstKey := Key(storage, dstDirPath) if !srcRawObj.IsDir() { Cache.linkCache.DeleteKey(stdpath.Join(dstKey, srcRawObj.GetName())) } if !storage.Config().NoCache { if cache, exist := Cache.dirCache.Get(dstKey); exist { if newObj == nil { newObj = &model.ObjWrapMask{Obj: srcRawObj, Mask: model.Temp} } else { newObj = wrapObjName(storage, newObj) } cache.UpdateObject(srcRawObj.GetName(), newObj) } } if ctx.Value(conf.SkipHookKey) != nil || !needHandleObjsUpdateHook() { return nil } if !srcObj.IsDir() { go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) } else { go objsUpdateHook(context.WithoutCancel(ctx), storage, stdpath.Join(dstDirPath, srcObj.GetName()), true) } return nil } func Remove(ctx context.Context, storage driver.Driver, path string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) if utils.PathEqual(path, "/") { return errors.New("delete root folder is not allowed") } rawObj, err := Get(ctx, storage, path, true) if err != nil { // if object not found, it's ok if errs.IsObjectNotFound(err) { log.Debugf("%s have been removed", path) return nil } return errors.WithMessage(err, "failed to get object") } if model.ObjHasMask(rawObj, model.NoRemove) { return errors.WithStack(errs.PermissionDenied) } dirPath := stdpath.Dir(path) switch s := storage.(type) { case driver.Remove: err = s.Remove(ctx, model.UnwrapObjName(rawObj)) if err == nil { Cache.removeDirectoryObject(storage, dirPath, rawObj) } default: return errs.NotImplement } return errors.WithStack(err) } func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file model.FileStreamer, up driver.UpdateProgress) error { defer func() { if err := file.Close(); err != nil { log.Errorf("failed to close file streamer, %v", err) } }() if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } // UrlTree PUT if storage.Config().OnlyIndices { var link string dstDirPath, link = urlTreeSplitLineFormPath(stdpath.Join(dstDirPath, file.GetName())) file = &stream.FileStream{Obj: &model.Object{Name: link}, Closers: utils.Closers{file}} } // if file exist and size = 0, delete it dstDirPath = utils.FixAndCleanPath(dstDirPath) dstPath := stdpath.Join(dstDirPath, file.GetName()) tempName := file.GetName() + ".openlist_to_delete" tempPath := stdpath.Join(dstDirPath, tempName) fi, err := GetUnwrap(ctx, storage, dstPath) if err == nil { if fi.GetSize() == 0 { err = Remove(ctx, storage, dstPath) if err != nil { return errors.WithMessagef(err, "while uploading, failed remove existing file which size = 0") } } else if storage.Config().NoOverwriteUpload { // try to rename old obj err = Rename(ctx, storage, dstPath, tempName) if err != nil { return err } } else { file.SetExist(fi) } } err = MakeDir(ctx, storage, dstDirPath) if err != nil { return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) } parentDir, err := GetUnwrap(ctx, storage, dstDirPath) // this should not happen if err != nil { return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) } if model.ObjHasMask(parentDir, model.NoWrite) { return errors.WithStack(errs.PermissionDenied) } // if up is nil, set a default to prevent panic if up == nil { up = func(p float64) {} } // 如果小于0，则通过缓存获取完整大小，可能发生于流式上传 if file.GetSize() < 0 { log.Warnf("file size < 0, try to get full size from cache") file.CacheFullAndWriter(nil, nil) } var newObj model.Obj switch s := storage.(type) { case driver.PutResult: newObj, err = s.Put(ctx, parentDir, file, up) case driver.Put: err = s.Put(ctx, parentDir, file, up) default: return errs.NotImplement } if err == nil { Cache.linkCache.DeleteKey(Key(storage, dstPath)) if !storage.Config().NoCache { if cache, exist := Cache.dirCache.Get(Key(storage, dstDirPath)); exist { if newObj == nil { newObj = &model.Object{ Name: file.GetName(), Size: file.GetSize(), Modified: file.ModTime(), Ctime: file.CreateTime(), Mask: model.Temp, } } newObj = wrapObjName(storage, newObj) cache.UpdateObject(newObj.GetName(), newObj) } } if ctx.Value(conf.SkipHookKey) == nil && needHandleObjsUpdateHook() { go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) } } log.Debugf("put file [%s] done", file.GetName()) if storage.Config().NoOverwriteUpload && fi != nil && fi.GetSize() > 0 { if err != nil { // upload failed, recover old obj err := Rename(ctx, storage, tempPath, file.GetName()) if err != nil { log.Errorf("failed recover old obj: %+v", err) } } else { // upload success, remove old obj err = Remove(ctx, storage, tempPath) } } return errors.WithStack(err) } func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } dstDirPath = utils.FixAndCleanPath(dstDirPath) dstPath := stdpath.Join(dstDirPath, dstName) if _, err := Get(ctx, storage, dstPath); err == nil { return errors.WithStack(errs.ObjectAlreadyExists) } err := MakeDir(ctx, storage, dstDirPath) if err != nil { return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) } dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) } if model.ObjHasMask(dstDir, model.NoWrite) { return errors.WithStack(errs.PermissionDenied) } var newObj model.Obj switch s := storage.(type) { case driver.PutURLResult: newObj, err = s.PutURL(ctx, dstDir, dstName, url) case driver.PutURL: err = s.PutURL(ctx, dstDir, dstName, url) default: return errors.WithStack(errs.NotImplement) } if err == nil { Cache.linkCache.DeleteKey(Key(storage, dstPath)) if !storage.Config().NoCache { if cache, exist := Cache.dirCache.Get(Key(storage, dstDirPath)); exist { if newObj == nil { t := time.Now() newObj = &model.Object{ Name: dstName, Modified: t, Ctime: t, Mask: model.Temp, } } newObj = wrapObjName(storage, newObj) cache.UpdateObject(newObj.GetName(), newObj) } if ctx.Value(conf.SkipHookKey) == nil && needHandleObjsUpdateHook() { go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) } } } log.Debugf("put url [%s](%s) done", dstName, url) return errors.WithStack(err) } func GetDirectUploadTools(storage driver.Driver) []string { du, ok := storage.(driver.DirectUploader) if !ok { return nil } if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil } return du.GetDirectUploadTools() } func GetDirectUploadInfo(ctx context.Context, tool string, storage driver.Driver, dstDirPath, dstName string, fileSize int64) (any, error) { du, ok := storage.(driver.DirectUploader) if !ok { return nil, errors.WithStack(errs.NotImplement) } if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } dstDirPath = utils.FixAndCleanPath(dstDirPath) dstPath := stdpath.Join(dstDirPath, dstName) _, err := Get(ctx, storage, dstPath) if err == nil { return nil, errors.WithStack(errs.ObjectAlreadyExists) } err = MakeDir(ctx, storage, dstDirPath) if err != nil { return nil, errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) } dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { return nil, errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) } info, err := du.GetDirectUploadInfo(ctx, tool, dstDir, dstName, fileSize) if err != nil { return nil, errors.WithStack(err) } return info, nil } func objsUpdateHook(ctx context.Context, storage driver.Driver, dirPath string, recursive bool) { files, err := List(ctx, storage, dirPath, model.ListArgs{SkipHook: true}) if err != nil { return } if !recursive { HandleObjsUpdateHook(ctx, utils.GetFullPath(storage.GetStorage().MountPath, dirPath), files) return } var limiter *rate.Limiter if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil { if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 { limiter = rate.NewLimiter(rate.Limit(f), 1) } } recursivelyObjsUpdateHook(ctx, storage, dirPath, files, limiter) } func recursivelyObjsUpdateHook(ctx context.Context, storage driver.Driver, dirPath string, files []model.Obj, limiter *rate.Limiter) { HandleObjsUpdateHook(ctx, utils.GetFullPath(storage.GetStorage().MountPath, dirPath), files) for _, f := range files { if utils.IsCanceled(ctx) { return } if !f.IsDir() { continue } dstPath := stdpath.Join(dirPath, f.GetName()) if limiter != nil { if err := limiter.Wait(ctx); err != nil { return } } files, err := List(ctx, storage, dstPath, model.ListArgs{SkipHook: true}) if err == nil { recursivelyObjsUpdateHook(ctx, storage, dstPath, files, limiter) } } } func needHandleObjsUpdateHook() bool { if len(objsUpdateHooks) < 1 { return false } needHandle, _ := GetSettingItemByKey(conf.HandleHookAfterWriting) return needHandle != nil && (needHandle.Value == "true" || needHandle.Value == "1") } func wrapObjsName(storage driver.Driver, objs []model.Obj) { if _, ok := storage.(driver.Getter); !ok { model.WrapObjsName(objs) } } func wrapObjName(storage driver.Driver, obj model.Obj) model.Obj { if _, ok := storage.(driver.Getter); !ok { return model.WrapObjName(obj) } return obj } ================================================ FILE: internal/op/hook.go ================================================ package op import ( "context" "regexp" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) // Obj type ObjsUpdateHook = func(ctx context.Context, parent string, objs []model.Obj) var ( objsUpdateHooks = make([]ObjsUpdateHook, 0) ) func RegisterObjsUpdateHook(hook ObjsUpdateHook) { objsUpdateHooks = append(objsUpdateHooks, hook) } func HandleObjsUpdateHook(ctx context.Context, parent string, objs []model.Obj) { for _, hook := range objsUpdateHooks { hook(ctx, parent, objs) } } // Setting type SettingItemHook func(item *model.SettingItem) error var settingItemHooks = map[string]SettingItemHook{ conf.VideoTypes: func(item *model.SettingItem) error { conf.SlicesMap[conf.VideoTypes] = strings.Split(item.Value, ",") return nil }, conf.AudioTypes: func(item *model.SettingItem) error { conf.SlicesMap[conf.AudioTypes] = strings.Split(item.Value, ",") return nil }, conf.ImageTypes: func(item *model.SettingItem) error { conf.SlicesMap[conf.ImageTypes] = strings.Split(item.Value, ",") return nil }, conf.TextTypes: func(item *model.SettingItem) error { conf.SlicesMap[conf.TextTypes] = strings.Split(item.Value, ",") return nil }, conf.ProxyTypes: func(item *model.SettingItem) error { conf.SlicesMap[conf.ProxyTypes] = strings.Split(item.Value, ",") return nil }, conf.ProxyIgnoreHeaders: func(item *model.SettingItem) error { conf.SlicesMap[conf.ProxyIgnoreHeaders] = strings.Split(item.Value, ",") return nil }, conf.PrivacyRegs: func(item *model.SettingItem) error { regStrs := strings.Split(item.Value, "\n") regs := make([]*regexp.Regexp, 0, len(regStrs)) for _, regStr := range regStrs { reg, err := regexp.Compile(regStr) if err != nil { return errors.WithStack(err) } regs = append(regs, reg) } conf.PrivacyReg = regs return nil }, conf.FilenameCharMapping: func(item *model.SettingItem) error { err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap) if err != nil { return err } log.Debugf("filename char mapping: %+v", conf.FilenameCharMap) return nil }, conf.IgnoreDirectLinkParams: func(item *model.SettingItem) error { conf.SlicesMap[conf.IgnoreDirectLinkParams] = strings.Split(item.Value, ",") return nil }, } func RegisterSettingItemHook(key string, hook SettingItemHook) { settingItemHooks[key] = hook } func HandleSettingItemHook(item *model.SettingItem) (hasHook bool, err error) { if hook, ok := settingItemHooks[item.Key]; ok { return true, hook(item) } return false, nil } // Storage type StorageHook func(typ string, storage driver.Driver) var storageHooks = make([]StorageHook, 0) func callStorageHooks(typ string, storage driver.Driver) { for _, hook := range storageHooks { hook(typ, storage) } } func RegisterStorageHook(hook StorageHook) { storageHooks = append(storageHooks, hook) } ================================================ FILE: internal/op/meta.go ================================================ package op import ( stdpath "path" "time" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/go-cache" "github.com/pkg/errors" "gorm.io/gorm" ) var metaCache = cache.NewMemCache(cache.WithShards[*model.Meta](2)) // metaG maybe not needed var metaG singleflight.Group[*model.Meta] func GetNearestMeta(path string) (*model.Meta, error) { return getNearestMeta(utils.FixAndCleanPath(path)) } func getNearestMeta(path string) (*model.Meta, error) { meta, err := GetMetaByPath(path) if err == nil { return meta, nil } if errors.Cause(err) != errs.MetaNotFound { return nil, err } if path == "/" { return nil, errs.MetaNotFound } return getNearestMeta(stdpath.Dir(path)) } func GetMetaByPath(path string) (*model.Meta, error) { return getMetaByPath(utils.FixAndCleanPath(path)) } func getMetaByPath(path string) (*model.Meta, error) { meta, ok := metaCache.Get(path) if ok { if meta == nil { return meta, errs.MetaNotFound } return meta, nil } meta, err, _ := metaG.Do(path, func() (*model.Meta, error) { _meta, err := db.GetMetaByPath(path) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { metaCache.Set(path, nil) return nil, errs.MetaNotFound } return nil, err } metaCache.Set(path, _meta, cache.WithEx[*model.Meta](time.Hour)) return _meta, nil }) return meta, err } func DeleteMetaById(id uint) error { old, err := db.GetMetaById(id) if err != nil { return err } metaCache.Del(old.Path) return db.DeleteMetaById(id) } func UpdateMeta(u *model.Meta) error { u.Path = utils.FixAndCleanPath(u.Path) old, err := db.GetMetaById(u.ID) if err != nil { return err } metaCache.Del(old.Path) return db.UpdateMeta(u) } func CreateMeta(u *model.Meta) error { u.Path = utils.FixAndCleanPath(u.Path) metaCache.Del(u.Path) return db.CreateMeta(u) } func GetMetaById(id uint) (*model.Meta, error) { return db.GetMetaById(id) } func GetMetas(pageIndex, pageSize int) (metas []model.Meta, count int64, err error) { return db.GetMetas(pageIndex, pageSize) } ================================================ FILE: internal/op/path.go ================================================ package op import ( stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" ) // GetStorageAndActualPath Get the corresponding storage and actual path // for path: remove the mount path prefix and join the actual root folder if exists func GetStorageAndActualPath(rawPath string) (storage driver.Driver, actualPath string, err error) { rawPath = utils.FixAndCleanPath(rawPath) storage = GetBalancedStorage(rawPath) if storage == nil { if rawPath == "/" { err = errs.NewErr(errs.StorageNotFound, "please add a storage first") return } err = errs.NewErr(errs.StorageNotFound, "rawPath: %s", rawPath) return } log.Debugln("use storage: ", storage.GetStorage().MountPath) mountPath := utils.GetActualMountPath(storage.GetStorage().MountPath) actualPath = utils.FixAndCleanPath(strings.TrimPrefix(rawPath, mountPath)) return } // urlTreeSplitLineFormPath 分割path中分割真实路径和UrlTree定义字符串 func urlTreeSplitLineFormPath(path string) (pp string, file string) { // url.PathUnescape 会移除 // ，手动加回去 path = strings.Replace(path, "https:/", "https://", 1) path = strings.Replace(path, "http:/", "http://", 1) if strings.Contains(path, ":https:/") || strings.Contains(path, ":http:/") { // URL-Tree模式 /url_tree_drivr/file_name[:size[:time]]:https://example.com/file fPath := strings.SplitN(path, ":", 2)[0] pp, _ = stdpath.Split(fPath) file = path[len(pp):] } else if strings.Contains(path, "/https:/") || strings.Contains(path, "/http:/") { // URL-Tree模式 /url_tree_drivr/https://example.com/file index := strings.Index(path, "/http://") if index == -1 { index = strings.Index(path, "/https://") } pp = path[:index] file = path[index+1:] } else { pp, file = stdpath.Split(path) } if pp == "" { pp = "/" } return } ================================================ FILE: internal/op/recursive_list.go ================================================ package op import ( "context" stdpath "path" "sync" "sync/atomic" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) var ( ManualScanCancel = atomic.Pointer[context.CancelFunc]{} ScannedCount = atomic.Uint64{} ) func ManualScanRunning() bool { return ManualScanCancel.Load() != nil } func BeginManualScan(rawPath string, limit float64) error { rawPath = utils.FixAndCleanPath(rawPath) ctx, cancel := context.WithCancel(context.Background()) if !ManualScanCancel.CompareAndSwap(nil, &cancel) { cancel() return errors.New("manual scan is running, please try later") } ScannedCount.Store(0) go func() { defer func() { (*ManualScanCancel.Swap(nil))() }() err := RecursivelyList(ctx, rawPath, rate.Limit(limit), &ScannedCount) if err != nil { log.Errorf("failed recursively list: %v", err) } }() return nil } func StopManualScan() { c := ManualScanCancel.Load() if c != nil { (*c)() } } func RecursivelyList(ctx context.Context, rawPath string, limit rate.Limit, counter *atomic.Uint64) error { storage, actualPath, err := GetStorageAndActualPath(rawPath) if err != nil && !errors.Is(err, errs.StorageNotFound) { return err } else if err == nil { var limiter *rate.Limiter if limit > .0 { limiter = rate.NewLimiter(limit, 1) } RecursivelyListStorage(ctx, storage, actualPath, limiter, counter) } else { var wg sync.WaitGroup recursivelyListVirtual(ctx, rawPath, limit, counter, &wg) wg.Wait() } return nil } func recursivelyListVirtual(ctx context.Context, rawPath string, limit rate.Limit, counter *atomic.Uint64, wg *sync.WaitGroup) { objs := GetStorageVirtualFilesByPath(rawPath) if counter != nil { counter.Add(uint64(len(objs))) } for _, obj := range objs { if utils.IsCanceled(ctx) { return } nextPath := stdpath.Join(rawPath, obj.GetName()) storage, actualPath, err := GetStorageAndActualPath(nextPath) if err != nil && !errors.Is(err, errs.StorageNotFound) { log.Errorf("error recursively list: failed get storage [%s]: %v", nextPath, err) } else if err == nil { var limiter *rate.Limiter if limit > .0 { limiter = rate.NewLimiter(limit, 1) } wg.Add(1) go func() { defer wg.Done() RecursivelyListStorage(ctx, storage, actualPath, limiter, counter) }() } else { recursivelyListVirtual(ctx, nextPath, limit, counter, wg) } } } func RecursivelyListStorage(ctx context.Context, storage driver.Driver, actualPath string, limiter *rate.Limiter, counter *atomic.Uint64) { objs, err := List(ctx, storage, actualPath, model.ListArgs{Refresh: true}) if err != nil { if !errors.Is(err, context.Canceled) { log.Errorf("error recursively list: failed list (%s)[%s]: %v", storage.GetStorage().MountPath, actualPath, err) } return } if counter != nil { counter.Add(uint64(len(objs))) } for _, obj := range objs { if utils.IsCanceled(ctx) { return } if !obj.IsDir() { continue } if limiter != nil { if err = limiter.Wait(ctx); err != nil { return } } nextPath := stdpath.Join(actualPath, obj.GetName()) RecursivelyListStorage(ctx, storage, nextPath, limiter, counter) } } ================================================ FILE: internal/op/setting.go ================================================ package op import ( "fmt" "sort" "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/pkg/errors" ) var settingG singleflight.Group[*model.SettingItem] var settingCacheF = func(item *model.SettingItem) { Cache.SetSetting(item.Key, item) } var settingGroupG singleflight.Group[[]model.SettingItem] var settingGroupCacheF = func(key string, items []model.SettingItem) { Cache.SetSettingGroup(key, items) } var settingChangingCallbacks = make([]func(), 0) func RegisterSettingChangingCallback(f func()) { settingChangingCallbacks = append(settingChangingCallbacks, f) } func SettingCacheUpdate() { Cache.ClearAll() for _, cb := range settingChangingCallbacks { cb() } } func GetPublicSettingsMap() map[string]string { items, _ := GetPublicSettingItems() pSettings := make(map[string]string) for _, item := range items { pSettings[item.Key] = item.Value } return pSettings } func GetSettingsMap() map[string]string { items, _ := GetSettingItems() settings := make(map[string]string) for _, item := range items { settings[item.Key] = item.Value } return settings } func GetSettingItems() ([]model.SettingItem, error) { if items, exists := Cache.GetSettingGroup("ALL_SETTING_ITEMS"); exists { return items, nil } items, err, _ := settingGroupG.Do("ALL_SETTING_ITEMS", func() ([]model.SettingItem, error) { _items, err := db.GetSettingItems() if err != nil { return nil, err } settingGroupCacheF("ALL_SETTING_ITEMS", _items) return _items, nil }) return items, err } func GetPublicSettingItems() ([]model.SettingItem, error) { if items, exists := Cache.GetSettingGroup("ALL_PUBLIC_SETTING_ITEMS"); exists { return items, nil } items, err, _ := settingGroupG.Do("ALL_PUBLIC_SETTING_ITEMS", func() ([]model.SettingItem, error) { _items, err := db.GetPublicSettingItems() if err != nil { return nil, err } settingGroupCacheF("ALL_PUBLIC_SETTING_ITEMS", _items) return _items, nil }) return items, err } func GetSettingItemByKey(key string) (*model.SettingItem, error) { if item, exists := Cache.GetSetting(key); exists { return item, nil } item, err, _ := settingG.Do(key, func() (*model.SettingItem, error) { _item, err := db.GetSettingItemByKey(key) if err != nil { return nil, err } settingCacheF(_item) return _item, nil }) return item, err } func GetSettingItemInKeys(keys []string) ([]model.SettingItem, error) { var items []model.SettingItem for _, key := range keys { item, err := GetSettingItemByKey(key) if err != nil { return nil, err } items = append(items, *item) } return items, nil } func GetSettingItemsByGroup(group int) ([]model.SettingItem, error) { key := fmt.Sprintf("GROUP_%d", group) if items, exists := Cache.GetSettingGroup(key); exists { return items, nil } items, err, _ := settingGroupG.Do(key, func() ([]model.SettingItem, error) { _items, err := db.GetSettingItemsByGroup(group) if err != nil { return nil, err } settingGroupCacheF(key, _items) return _items, nil }) return items, err } func GetSettingItemsInGroups(groups []int) ([]model.SettingItem, error) { sort.Ints(groups) keyParts := make([]string, 0, len(groups)) for _, g := range groups { keyParts = append(keyParts, strconv.Itoa(g)) } key := "GROUPS_" + strings.Join(keyParts, "_") if items, exists := Cache.GetSettingGroup(key); exists { return items, nil } items, err, _ := settingGroupG.Do(key, func() ([]model.SettingItem, error) { _items, err := db.GetSettingItemsInGroups(groups) if err != nil { return nil, err } settingGroupCacheF(key, _items) return _items, nil }) return items, err } func SaveSettingItems(items []model.SettingItem) error { for i := range items { item := &items[i] if it, ok := MigrationSettingItems[item.Key]; ok && item.Value == it.MigrationValue { item.Value = it.Value } if ok, err := HandleSettingItemHook(item); ok && err != nil { return fmt.Errorf("failed to execute hook on %s: %+v", item.Key, err) } } err := db.SaveSettingItems(items) if err != nil { return fmt.Errorf("failed save setting: %+v", err) } SettingCacheUpdate() return nil } func SaveSettingItem(item *model.SettingItem) (err error) { if it, ok := MigrationSettingItems[item.Key]; ok && item.Value == it.MigrationValue { item.Value = it.Value } // hook if _, err := HandleSettingItemHook(item); err != nil { return fmt.Errorf("failed to execute hook on %s: %+v", item.Key, err) } // update if err = db.SaveSettingItem(item); err != nil { return fmt.Errorf("failed save setting on %s: %+v", item.Key, err) } SettingCacheUpdate() return nil } func DeleteSettingItemByKey(key string) error { old, err := GetSettingItemByKey(key) if err != nil { return errors.WithMessage(err, "failed to get settingItem") } if !old.IsDeprecated() { return errors.Errorf("setting [%s] is not deprecated", key) } SettingCacheUpdate() return db.DeleteSettingItemByKey(key) } type MigrationValueItem struct { MigrationValue, Value string } var MigrationSettingItems map[string]MigrationValueItem ================================================ FILE: internal/op/sharing.go ================================================ package op import ( "fmt" stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/go-cache" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) func makeJoined(sdb []model.SharingDB) []model.Sharing { creator := make(map[uint]*model.User) return utils.MustSliceConvert(sdb, func(s model.SharingDB) model.Sharing { var c *model.User var ok bool if c, ok = creator[s.CreatorId]; !ok { var err error if c, err = GetUserById(s.CreatorId); err != nil { c = nil } else { creator[s.CreatorId] = c } } var files []string if err := utils.Json.UnmarshalFromString(s.FilesRaw, &files); err != nil { files = make([]string, 0) } return model.Sharing{ SharingDB: &s, Files: files, Creator: c, } }) } var sharingCache = cache.NewMemCache(cache.WithShards[*model.Sharing](8)) var sharingG singleflight.Group[*model.Sharing] func GetSharingById(id string, refresh ...bool) (*model.Sharing, error) { if !utils.IsBool(refresh...) { if sharing, ok := sharingCache.Get(id); ok { log.Debugf("use cache when get sharing %s", id) return sharing, nil } } sharing, err, _ := sharingG.Do(id, func() (*model.Sharing, error) { s, err := db.GetSharingById(id) if err != nil { return nil, errors.WithMessagef(err, "failed get sharing [%s]", id) } creator, err := GetUserById(s.CreatorId) if err != nil { return nil, errors.WithMessagef(err, "failed get sharing creator [%s]", id) } var files []string if err = utils.Json.UnmarshalFromString(s.FilesRaw, &files); err != nil { files = make([]string, 0) } return &model.Sharing{ SharingDB: s, Files: files, Creator: creator, }, nil }) return sharing, err } func GetSharings(pageIndex, pageSize int) ([]model.Sharing, int64, error) { s, cnt, err := db.GetSharings(pageIndex, pageSize) if err != nil { return nil, 0, errors.WithStack(err) } return makeJoined(s), cnt, nil } func GetSharingsByCreatorId(userId uint, pageIndex, pageSize int) ([]model.Sharing, int64, error) { s, cnt, err := db.GetSharingsByCreatorId(userId, pageIndex, pageSize) if err != nil { return nil, 0, errors.WithStack(err) } return makeJoined(s), cnt, nil } func GetSharingUnwrapPath(sharing *model.Sharing, path string) (unwrapPath string, err error) { if len(sharing.Files) == 0 { return "", errors.New("cannot get actual path of an invalid sharing") } if len(sharing.Files) == 1 { return stdpath.Join(sharing.Files[0], path), nil } path = utils.FixAndCleanPath(path)[1:] if len(path) == 0 { return "", errors.New("cannot get actual path of a sharing root path") } mapPath := "" child, rest, _ := strings.Cut(path, "/") for _, c := range sharing.Files { if child == stdpath.Base(c) { mapPath = c break } } if mapPath == "" { return "", fmt.Errorf("failed find child [%s] of sharing [%s]", child, sharing.ID) } return stdpath.Join(mapPath, rest), nil } func CreateSharing(sharing *model.Sharing) (id string, err error) { sharing.CreatorId = sharing.Creator.ID sharing.FilesRaw, err = utils.Json.MarshalToString(utils.MustSliceConvert(sharing.Files, utils.FixAndCleanPath)) if err != nil { return "", errors.WithStack(err) } return db.CreateSharing(sharing.SharingDB) } func UpdateSharing(sharing *model.Sharing, skipMarshal ...bool) (err error) { if !utils.IsBool(skipMarshal...) { sharing.CreatorId = sharing.Creator.ID sharing.FilesRaw, err = utils.Json.MarshalToString(utils.MustSliceConvert(sharing.Files, utils.FixAndCleanPath)) if err != nil { return errors.WithStack(err) } } sharingCache.Del(sharing.ID) return db.UpdateSharing(sharing.SharingDB) } func DeleteSharing(sid string) error { sharingCache.Del(sid) return db.DeleteSharingById(sid) } func DeleteSharingsByCreatorId(creatorId uint) error { return db.DeleteSharingsByCreatorId(creatorId) } ================================================ FILE: internal/op/sshkey.go ================================================ package op import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/pkg/errors" "golang.org/x/crypto/ssh" ) func CreateSSHPublicKey(k *model.SSHPublicKey) (error, bool) { _, err := db.GetSSHPublicKeyByUserTitle(k.UserId, k.Title) if err == nil { return errors.New("key with the same title already exists"), true } pubKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(k.KeyStr)) if err != nil { return err, false } k.Fingerprint = ssh.FingerprintSHA256(pubKey) k.AddedTime = time.Now() k.LastUsedTime = k.AddedTime return db.CreateSSHPublicKey(k), true } func GetSSHPublicKeyByUserId(userId uint, pageIndex, pageSize int) (keys []model.SSHPublicKey, count int64, err error) { return db.GetSSHPublicKeyByUserId(userId, pageIndex, pageSize) } func GetSSHPublicKeyByIdAndUserId(id uint, userId uint) (*model.SSHPublicKey, error) { key, err := db.GetSSHPublicKeyById(id) if err != nil { return nil, err } if key.UserId != userId { return nil, errors.Wrapf(err, "failed get old key") } return key, nil } func UpdateSSHPublicKey(k *model.SSHPublicKey) error { return db.UpdateSSHPublicKey(k) } func DeleteSSHPublicKeyById(keyId uint) error { return db.DeleteSSHPublicKeyById(keyId) } ================================================ FILE: internal/op/storage.go ================================================ package op import ( "context" "fmt" "reflect" "runtime" "sort" "strings" "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/generic_sync" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) // Although the driver type is stored, // there is a storage in each driver, // so it should actually be a storage, just wrapped by the driver var storagesMap generic_sync.MapOf[string, driver.Driver] func GetAllStorages() []driver.Driver { return storagesMap.Values() } func HasStorage(mountPath string) bool { return storagesMap.Has(utils.FixAndCleanPath(mountPath)) } func GetStorageByMountPath(mountPath string) (driver.Driver, error) { mountPath = utils.FixAndCleanPath(mountPath) storageDriver, ok := storagesMap.Load(mountPath) if !ok { return nil, errors.Errorf("no mount path for an storage is: %s", mountPath) } return storageDriver, nil } // CreateStorage Save the storage to database so storage can get an id // then instantiate corresponding driver and save it in memory func CreateStorage(ctx context.Context, storage model.Storage) (uint, error) { storage.Modified = time.Now() storage.MountPath = utils.FixAndCleanPath(storage.MountPath) var err error // check driver first driverName := storage.Driver driverNew, err := GetDriver(driverName) if err != nil { return 0, errors.WithMessage(err, "failed get driver new") } storageDriver := driverNew() // insert storage to database err = db.CreateStorage(&storage) if err != nil { return storage.ID, errors.WithMessage(err, "failed create storage in database") } // already has an id err = initStorage(ctx, storage, storageDriver) go callStorageHooks("add", storageDriver) if err != nil { return storage.ID, errors.Wrap(err, "failed init storage but storage is already created") } log.Debugf("storage %+v is created", storageDriver) return storage.ID, nil } // LoadStorage load exist storage in db to memory func LoadStorage(ctx context.Context, storage model.Storage) error { storage.MountPath = utils.FixAndCleanPath(storage.MountPath) // check driver first driverName := storage.Driver driverNew, err := GetDriver(driverName) if err != nil { return errors.WithMessage(err, "failed get driver new") } storageDriver := driverNew() err = initStorage(ctx, storage, storageDriver) go callStorageHooks("add", storageDriver) log.Debugf("storage %+v is created", storageDriver) return err } func getCurrentGoroutineStack() string { buf := make([]byte, 1<<16) n := runtime.Stack(buf, false) return string(buf[:n]) } // initStorage initialize the driver and store to storagesMap func initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver) (err error) { storageDriver.SetStorage(storage) driverStorage := storageDriver.GetStorage() defer func() { if err := recover(); err != nil { errInfo := fmt.Sprintf("[panic] err: %v\nstack: %s\n", err, getCurrentGoroutineStack()) log.Errorf("panic init storage: %s", errInfo) driverStorage.SetStatus(errInfo) MustSaveDriverStorage(storageDriver) storagesMap.Store(driverStorage.MountPath, storageDriver) } }() // Unmarshal Addition err = utils.Json.UnmarshalFromString(driverStorage.Addition, storageDriver.GetAddition()) if err == nil { if ref, ok := storageDriver.(driver.Reference); ok { if strings.HasPrefix(driverStorage.Remark, "ref:/") { refMountPath := driverStorage.Remark i := strings.Index(refMountPath, "\n") if i > 0 { refMountPath = refMountPath[4:i] } else { refMountPath = refMountPath[4:] } var refStorage driver.Driver refStorage, err = GetStorageByMountPath(refMountPath) if err != nil { err = fmt.Errorf("ref: %w", err) } else { err = ref.InitReference(refStorage) if err != nil && errs.IsNotSupportError(err) { err = fmt.Errorf("ref: storage is not %s", storageDriver.Config().Name) } } } } } if err == nil { err = storageDriver.Init(ctx) } storagesMap.Store(driverStorage.MountPath, storageDriver) if err != nil { if IsUseOnlineAPI(storageDriver) { driverStorage.SetStatus(utils.SanitizeHTML(err.Error())) } else { driverStorage.SetStatus(err.Error()) } err = errors.Wrap(err, "failed init storage") } else { driverStorage.SetStatus(WORK) } MustSaveDriverStorage(storageDriver) return err } func IsUseOnlineAPI(storageDriver driver.Driver) bool { v := reflect.ValueOf(storageDriver.GetAddition()) if v.Kind() == reflect.Ptr { v = v.Elem() } if !v.IsValid() || v.Kind() != reflect.Struct { return false } field_v := v.FieldByName("UseOnlineAPI") if !field_v.IsValid() { return false } if field_v.Kind() != reflect.Bool { return false } return field_v.Bool() } func EnableStorage(ctx context.Context, id uint) error { storage, err := db.GetStorageById(id) if err != nil { return errors.WithMessage(err, "failed get storage") } if !storage.Disabled { return errors.Errorf("this storage have enabled") } storage.Disabled = false err = db.UpdateStorage(storage) if err != nil { return errors.WithMessage(err, "failed update storage in db") } err = LoadStorage(ctx, *storage) if err != nil { return errors.WithMessage(err, "failed load storage") } return nil } func DisableStorage(ctx context.Context, id uint) error { storage, err := db.GetStorageById(id) if err != nil { return errors.WithMessage(err, "failed get storage") } if storage.Disabled { return errors.Errorf("this storage have disabled") } storageDriver, err := GetStorageByMountPath(storage.MountPath) if err != nil { return errors.WithMessage(err, "failed get storage driver") } // drop the storage in the driver if err := storageDriver.Drop(ctx); err != nil { return errors.Wrap(err, "failed drop storage") } // delete the storage in the memory storage.Disabled = true storage.SetStatus(DISABLED) err = db.UpdateStorage(storage) if err != nil { return errors.WithMessage(err, "failed update storage in db") } storagesMap.Delete(storage.MountPath) go callStorageHooks("del", storageDriver) return nil } // UpdateStorage update storage // get old storage first // drop the storage then reinitialize func UpdateStorage(ctx context.Context, storage model.Storage) error { oldStorage, err := db.GetStorageById(storage.ID) if err != nil { return errors.WithMessage(err, "failed get old storage") } if oldStorage.Driver != storage.Driver { return errors.Errorf("driver cannot be changed") } storage.Modified = time.Now() storage.MountPath = utils.FixAndCleanPath(storage.MountPath) err = db.UpdateStorage(&storage) if err != nil { return errors.WithMessage(err, "failed update storage in database") } if storage.Disabled { return nil } storageDriver, err := GetStorageByMountPath(oldStorage.MountPath) if oldStorage.MountPath != storage.MountPath { // mount path renamed, need to drop the storage storagesMap.Delete(oldStorage.MountPath) Cache.DeleteDirectoryTree(storageDriver, "/") Cache.InvalidateStorageDetails(storageDriver) } if err != nil { return errors.WithMessage(err, "failed get storage driver") } err = storageDriver.Drop(ctx) if err != nil { return errors.Wrapf(err, "failed drop storage") } err = initStorage(ctx, storage, storageDriver) go callStorageHooks("update", storageDriver) log.Debugf("storage %+v is update", storageDriver) return err } func DeleteStorageById(ctx context.Context, id uint) error { storage, err := db.GetStorageById(id) if err != nil { return errors.WithMessage(err, "failed get storage") } var dropErr error = nil if !storage.Disabled { storageDriver, err := GetStorageByMountPath(storage.MountPath) if err != nil { return errors.WithMessage(err, "failed get storage driver") } // drop the storage in the driver if err := storageDriver.Drop(ctx); err != nil { dropErr = errors.Wrapf(err, "failed drop storage") } // delete the storage in the memory storagesMap.Delete(storage.MountPath) Cache.DeleteDirectoryTree(storageDriver, "/") Cache.InvalidateStorageDetails(storageDriver) go callStorageHooks("del", storageDriver) } // delete the storage in the database if err := db.DeleteStorageById(id); err != nil { return errors.WithMessage(err, "failed delete storage in database") } return dropErr } // MustSaveDriverStorage call from specific driver func MustSaveDriverStorage(driver driver.Driver) { err := saveDriverStorage(driver) if err != nil { log.Errorf("failed save driver storage: %s", err) } } func saveDriverStorage(driver driver.Driver) error { storage := driver.GetStorage() addition := driver.GetAddition() str, err := utils.Json.MarshalToString(addition) if err != nil { return errors.Wrap(err, "error while marshal addition") } storage.Addition = str err = db.UpdateStorage(storage) if err != nil { return errors.WithMessage(err, "failed update storage in database") } return nil } // getStoragesByPath get storage by longest match path, contains balance storage. // for example, there is /a/b,/a/c,/a/d/e,/a/d/e.balance // getStoragesByPath(/a/d/e/f) => /a/d/e,/a/d/e.balance func getStoragesByPath(path string) []driver.Driver { storages := make([]driver.Driver, 0) curSlashCount := 0 storagesMap.Range(func(mountPath string, value driver.Driver) bool { mountPath = utils.GetActualMountPath(mountPath) // is this path if utils.IsSubPath(mountPath, path) { slashCount := strings.Count(utils.PathAddSeparatorSuffix(mountPath), "/") // not the longest match if slashCount > curSlashCount { storages = storages[:0] curSlashCount = slashCount } if slashCount == curSlashCount { storages = append(storages, value) } } return true }) // make sure the order is the same for same input sort.Slice(storages, func(i, j int) bool { return storages[i].GetStorage().MountPath < storages[j].GetStorage().MountPath }) return storages } // GetStorageVirtualFilesByPath Obtain the virtual file generated by the storage according to the path // for example, there are: /a/b,/a/c,/a/d/e,/a/b.balance1,/av // GetStorageVirtualFilesByPath(/a) => b,c,d func GetStorageVirtualFilesByPath(prefix string) []model.Obj { return getStorageVirtualFilesByPath(prefix, nil, "") } func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, hideDetails, refresh bool, filterByName string) []model.Obj { if hideDetails { return getStorageVirtualFilesByPath(prefix, nil, filterByName) } return getStorageVirtualFilesByPath(prefix, func(d driver.Driver, obj model.Obj) model.Obj { if _, ok := obj.(*model.ObjStorageDetails); ok { return obj } ret := &model.ObjStorageDetails{ Obj: obj, StorageDetails: nil, } resultChan := make(chan *model.StorageDetails, 1) go func(dri driver.Driver) { details, err := GetStorageDetails(ctx, dri, refresh) if err != nil { if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.StorageNotInit) { log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, err) } } resultChan <- details }(d) select { case r := <-resultChan: ret.StorageDetails = r case <-time.After(time.Second): } return ret }, filterByName) } func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver, model.Obj) model.Obj, filterByName string) []model.Obj { files := make([]model.Obj, 0) storages := storagesMap.Values() sort.Slice(storages, func(i, j int) bool { if storages[i].GetStorage().Order == storages[j].GetStorage().Order { return storages[i].GetStorage().MountPath < storages[j].GetStorage().MountPath } return storages[i].GetStorage().Order < storages[j].GetStorage().Order }) if !strings.HasSuffix(prefix, "/") { prefix += "/" } set := make(map[string]int) var wg sync.WaitGroup for _, v := range storages { // Exclude prefix itself and non prefix p, found := strings.CutPrefix(utils.GetActualMountPath(v.GetStorage().MountPath), prefix) if !found || p == "" { continue } name, _, found := strings.Cut(p, "/") if filterByName != "" && name != filterByName { continue } if idx, ok := set[name]; ok { if !found { files[idx].(*model.Object).Mask = model.Locked | model.Virtual if rootCallback != nil { wg.Add(1) go func() { defer wg.Done() files[idx] = rootCallback(v, files[idx]) }() } } continue } set[name] = len(files) obj := &model.Object{ Name: name, Modified: v.GetStorage().Modified, IsFolder: true, } if !found { idx := len(files) obj.Mask = model.Locked | model.Virtual files = append(files, obj) if rootCallback != nil { wg.Add(1) go func() { defer wg.Done() files[idx] = rootCallback(v, files[idx]) }() } } else { obj.Mask = model.ReadOnly | model.Virtual files = append(files, obj) } } if rootCallback != nil { wg.Wait() } return files } var balanceMap generic_sync.MapOf[string, int] // GetBalancedStorage get storage by path func GetBalancedStorage(path string) driver.Driver { path = utils.FixAndCleanPath(path) storages := getStoragesByPath(path) storageNum := len(storages) switch storageNum { case 0: return nil case 1: return storages[0] default: virtualPath := utils.GetActualMountPath(storages[0].GetStorage().MountPath) i, _ := balanceMap.LoadOrStore(virtualPath, 0) i = (i + 1) % storageNum balanceMap.Store(virtualPath, i) return storages[i] } } var detailsG singleflight.Group[*model.StorageDetails] func GetStorageDetails(ctx context.Context, storage driver.Driver, refresh ...bool) (*model.StorageDetails, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } wd, ok := storage.(driver.WithDetails) if !ok { return nil, errs.NotImplement } if !utils.IsBool(refresh...) { if ret, ok := Cache.GetStorageDetails(storage); ok { return ret, nil } } details, err, _ := detailsG.Do(storage.GetStorage().MountPath, func() (*model.StorageDetails, error) { ret, err := wd.GetDetails(ctx) if err != nil { return nil, err } Cache.SetStorageDetails(storage, ret) return ret, nil }) return details, err } ================================================ FILE: internal/op/storage_test.go ================================================ package op_test import ( "context" "testing" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" mapset "github.com/deckarep/golang-set/v2" "gorm.io/driver/sqlite" "gorm.io/gorm" ) func init() { dB, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{}) if err != nil { panic("failed to connect database") } conf.Conf = conf.DefaultConfig("data") db.Init(dB) } func TestCreateStorage(t *testing.T) { var storages = []struct { storage model.Storage isErr bool }{ {storage: model.Storage{Driver: "Local", MountPath: "/local", Addition: `{"root_folder_path":"."}`}, isErr: false}, {storage: model.Storage{Driver: "Local", MountPath: "/local", Addition: `{"root_folder_path":"."}`}, isErr: true}, {storage: model.Storage{Driver: "None", MountPath: "/none", Addition: `{"root_folder_path":"."}`}, isErr: true}, } for _, storage := range storages { _, err := op.CreateStorage(context.Background(), storage.storage) if err != nil { if !storage.isErr { t.Errorf("failed to create storage: %+v", err) } else { t.Logf("expect failed to create storage: %+v", err) } } } } func TestGetStorageVirtualFilesByPath(t *testing.T) { setupStorages(t) virtualFiles := op.GetStorageVirtualFilesByPath("/a") var names []string for _, virtualFile := range virtualFiles { names = append(names, virtualFile.GetName()) } var expectedNames = []string{"b", "c", "d"} if utils.SliceEqual(names, expectedNames) { t.Logf("passed") } else { t.Errorf("expected: %+v, got: %+v", expectedNames, names) } } func TestGetBalancedStorage(t *testing.T) { set := mapset.NewSet[string]() for i := 0; i < 5; i++ { storage := op.GetBalancedStorage("/a/d/e1") set.Add(storage.GetStorage().MountPath) } expected := mapset.NewSet([]string{"/a/d/e1", "/a/d/e1.balance"}...) if !expected.Equal(set) { t.Errorf("expected: %+v, got: %+v", expected, set) } } func setupStorages(t *testing.T) { var storages = []model.Storage{ {Driver: "Local", MountPath: "/a/b", Order: 0, Addition: `{"root_folder_path":"."}`}, {Driver: "Local", MountPath: "/adc", Order: 0, Addition: `{"root_folder_path":"."}`}, {Driver: "Local", MountPath: "/a/c", Order: 1, Addition: `{"root_folder_path":"."}`}, {Driver: "Local", MountPath: "/a/d", Order: 2, Addition: `{"root_folder_path":"."}`}, {Driver: "Local", MountPath: "/a/d/e1", Order: 3, Addition: `{"root_folder_path":"."}`}, {Driver: "Local", MountPath: "/a/d/e", Order: 4, Addition: `{"root_folder_path":"."}`}, {Driver: "Local", MountPath: "/a/d/e1.balance", Order: 4, Addition: `{"root_folder_path":"."}`}, } for _, storage := range storages { _, err := op.CreateStorage(context.Background(), storage) if err != nil { t.Fatalf("failed to create storage: %+v", err) } } } ================================================ FILE: internal/op/user.go ================================================ package op import ( "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" ) var userG singleflight.Group[*model.User] var guestUser *model.User var adminUser *model.User func GetAdmin() (*model.User, error) { if adminUser == nil { user, err := db.GetUserByRole(model.ADMIN) if err != nil { return nil, err } adminUser = user } return adminUser, nil } func GetGuest() (*model.User, error) { if guestUser == nil { user, err := db.GetUserByRole(model.GUEST) if err != nil { return nil, err } guestUser = user } return guestUser, nil } func GetUserByRole(role int) (*model.User, error) { return db.GetUserByRole(role) } func GetUserByName(username string) (*model.User, error) { if username == "" { return nil, errs.EmptyUsername } if user, exists := Cache.GetUser(username); exists { return user, nil } user, err, _ := userG.Do(username, func() (*model.User, error) { _user, err := db.GetUserByName(username) if err != nil { return nil, err } Cache.SetUser(username, _user) return _user, nil }) return user, err } func GetUserById(id uint) (*model.User, error) { return db.GetUserById(id) } func GetUsers(pageIndex, pageSize int) (users []model.User, count int64, err error) { return db.GetUsers(pageIndex, pageSize) } func CreateUser(u *model.User) error { u.BasePath = utils.FixAndCleanPath(u.BasePath) return db.CreateUser(u) } func DeleteUserById(id uint) error { old, err := db.GetUserById(id) if err != nil { return err } if old.IsAdmin() || old.IsGuest() { return errs.DeleteAdminOrGuest } Cache.DeleteUser(old.Username) if err := DeleteSharingsByCreatorId(id); err != nil { return errors.WithMessage(err, "failed to delete user's sharings") } return db.DeleteUserById(id) } func UpdateUser(u *model.User) error { old, err := db.GetUserById(u.ID) if err != nil { return err } if u.IsAdmin() { adminUser = nil } if u.IsGuest() { guestUser = nil } Cache.DeleteUser(old.Username) u.BasePath = utils.FixAndCleanPath(u.BasePath) return db.UpdateUser(u) } func Cancel2FAByUser(u *model.User) error { u.OtpSecret = "" return UpdateUser(u) } func Cancel2FAById(id uint) error { user, err := db.GetUserById(id) if err != nil { return err } return Cancel2FAByUser(user) } func DelUserCache(username string) error { user, err := GetUserByName(username) if err != nil { return err } if user.IsAdmin() { adminUser = nil } if user.IsGuest() { guestUser = nil } Cache.DeleteUser(username) return nil } ================================================ FILE: internal/search/bleve/init.go ================================================ package bleve import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/search/searcher" "github.com/blevesearch/bleve/v2" log "github.com/sirupsen/logrus" ) var config = searcher.Config{ Name: "bleve", } func Init(indexPath *string) (bleve.Index, error) { log.Debugf("bleve path: %s", *indexPath) fileIndex, err := bleve.Open(*indexPath) if err == bleve.ErrorIndexPathDoesNotExist { log.Infof("Creating new index...") indexMapping := bleve.NewIndexMapping() searchNodeMapping := bleve.NewDocumentMapping() searchNodeMapping.AddFieldMappingsAt("is_dir", bleve.NewBooleanFieldMapping()) // TODO: appoint analyzer parentFieldMapping := bleve.NewTextFieldMapping() searchNodeMapping.AddFieldMappingsAt("parent", parentFieldMapping) // TODO: appoint analyzer nameFieldMapping := bleve.NewKeywordFieldMapping() searchNodeMapping.AddFieldMappingsAt("name", nameFieldMapping) indexMapping.AddDocumentMapping("SearchNode", searchNodeMapping) fileIndex, err = bleve.New(*indexPath, indexMapping) if err != nil { return nil, err } } else if err != nil { return nil, err } return fileIndex, nil } func init() { searcher.RegisterSearcher(config, func() (searcher.Searcher, error) { b, err := Init(&conf.Conf.BleveDir) if err != nil { return nil, err } return &Bleve{BIndex: b}, nil }) } ================================================ FILE: internal/search/bleve/search.go ================================================ package bleve import ( "context" "os" query2 "github.com/blevesearch/bleve/v2/search/query" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/search/searcher" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/blevesearch/bleve/v2" search2 "github.com/blevesearch/bleve/v2/search" "github.com/google/uuid" log "github.com/sirupsen/logrus" ) type Bleve struct { BIndex bleve.Index } func (b *Bleve) Config() searcher.Config { return config } func (b *Bleve) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { var queries []query2.Query query := bleve.NewMatchQuery(req.Keywords) query.SetField("name") queries = append(queries, query) if req.Scope != 0 { isDir := req.Scope == 1 isDirQuery := bleve.NewBoolFieldQuery(isDir) queries = append(queries, isDirQuery) } reqQuery := bleve.NewConjunctionQuery(queries...) search := bleve.NewSearchRequest(reqQuery) search.SortBy([]string{"name"}) search.From = (req.Page - 1) * req.PerPage search.Size = req.PerPage search.Fields = []string{"*"} searchResults, err := b.BIndex.Search(search) if err != nil { log.Errorf("search error: %+v", err) return nil, 0, err } res, err := utils.SliceConvert(searchResults.Hits, func(src *search2.DocumentMatch) (model.SearchNode, error) { return model.SearchNode{ Parent: src.Fields["parent"].(string), Name: src.Fields["name"].(string), IsDir: src.Fields["is_dir"].(bool), Size: int64(src.Fields["size"].(float64)), }, nil }) return res, int64(searchResults.Total), nil } func (b *Bleve) Index(ctx context.Context, node model.SearchNode) error { return b.BIndex.Index(uuid.NewString(), node) } func (b *Bleve) BatchIndex(ctx context.Context, nodes []model.SearchNode) error { batch := b.BIndex.NewBatch() for _, node := range nodes { batch.Index(uuid.NewString(), node) } return b.BIndex.Batch(batch) } func (b *Bleve) Get(ctx context.Context, parent string) ([]model.SearchNode, error) { return nil, errs.NotSupport } func (b *Bleve) Del(ctx context.Context, prefix string) error { return errs.NotSupport } func (b *Bleve) Release(ctx context.Context) error { if b.BIndex != nil { return b.BIndex.Close() } return nil } func (b *Bleve) Clear(ctx context.Context) error { err := b.Release(ctx) if err != nil { return err } log.Infof("Removing old index...") err = os.RemoveAll(conf.Conf.BleveDir) if err != nil { log.Errorf("clear bleve error: %+v", err) } bIndex, err := Init(&conf.Conf.BleveDir) if err != nil { return err } b.BIndex = bIndex return nil } var _ searcher.Searcher = (*Bleve)(nil) ================================================ FILE: internal/search/build.go ================================================ package search import ( "context" "path" "path/filepath" "strings" "sync" "sync/atomic" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/search/searcher" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/mq" "github.com/OpenListTeam/OpenList/v4/pkg/utils" mapset "github.com/deckarep/golang-set/v2" log "github.com/sirupsen/logrus" ) var ( Quit = atomic.Pointer[chan struct{}]{} ) func Running() bool { return Quit.Load() != nil } func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int, count bool) error { var ( err error objCount uint64 = 0 fi model.Obj ) log.Infof("build index for: %+v", indexPaths) log.Infof("ignore paths: %+v", ignorePaths) quit := make(chan struct{}, 1) if !Quit.CompareAndSwap(nil, &quit) { // other goroutine is running return errs.BuildIndexIsRunning } var ( indexMQ = mq.NewInMemoryMQ[ObjWithParent]() running = atomic.Bool{} // current goroutine running wg = &sync.WaitGroup{} ) running.Store(true) wg.Add(1) go func() { ticker := time.NewTicker(time.Second) defer func() { Quit.Store(nil) wg.Done() // notify walk to exit when StopIndex api called running.Store(false) ticker.Stop() }() tickCount := 0 for { select { case <-ticker.C: tickCount += 1 if indexMQ.Len() < 1000 && tickCount != 5 { continue } else if tickCount >= 5 { tickCount = 0 } log.Infof("index obj count: %d", objCount) indexMQ.ConsumeAll(func(messages []mq.Message[ObjWithParent]) { if len(messages) != 0 { log.Debugf("current index: %s", messages[len(messages)-1].Content.Parent) } if err = BatchIndex(ctx, utils.MustSliceConvert(messages, func(src mq.Message[ObjWithParent]) ObjWithParent { return src.Content })); err != nil { log.Errorf("build index in batch error: %+v", err) } else { objCount = objCount + uint64(len(messages)) } if count { WriteProgress(&model.IndexProgress{ ObjCount: objCount, IsDone: false, LastDoneTime: nil, }) } }) case <-quit: log.Debugf("build index for %+v received quit", indexPaths) eMsg := "" now := time.Now() originErr := err indexMQ.ConsumeAll(func(messages []mq.Message[ObjWithParent]) { if err = BatchIndex(ctx, utils.MustSliceConvert(messages, func(src mq.Message[ObjWithParent]) ObjWithParent { return src.Content })); err != nil { log.Errorf("build index in batch error: %+v", err) } else { objCount = objCount + uint64(len(messages)) } if originErr != nil { log.Errorf("build index error: %+v", originErr) eMsg = originErr.Error() } else { log.Infof("success build index, count: %d", objCount) } if count { WriteProgress(&model.IndexProgress{ ObjCount: objCount, IsDone: true, LastDoneTime: &now, Error: eMsg, }) } }) log.Debugf("build index for %+v quit success", indexPaths) return } } }() defer func() { if !running.Load() || Quit.Load() != &quit { log.Debugf("build index for %+v stopped by StopIndex", indexPaths) return } select { // avoid goroutine leak case quit <- struct{}{}: default: } wg.Wait() }() admin, err := op.GetAdmin() if err != nil { return err } if count { WriteProgress(&model.IndexProgress{ ObjCount: 0, IsDone: false, }) } for _, indexPath := range indexPaths { walkFn := func(indexPath string, info model.Obj) error { if !running.Load() { return filepath.SkipDir } for _, avoidPath := range ignorePaths { if strings.HasPrefix(indexPath, avoidPath) { return filepath.SkipDir } } if storage, _, err := op.GetStorageAndActualPath(indexPath); err == nil { if storage.GetStorage().DisableIndex { return filepath.SkipDir } } // ignore root if indexPath == "/" { return nil } indexMQ.Publish(mq.Message[ObjWithParent]{ Content: ObjWithParent{ Obj: info, Parent: path.Dir(indexPath), }, }) return nil } fi, err = fs.Get(ctx, indexPath, &fs.GetArgs{}) if err != nil { return err } // TODO: run walkFS concurrently err = fs.WalkFS(context.WithValue(ctx, conf.UserKey, admin), maxDepth, indexPath, fi, walkFn) if err != nil { return err } } return nil } func Del(ctx context.Context, prefix string) error { return instance.Del(ctx, prefix) } func Clear(ctx context.Context) error { return instance.Clear(ctx) } func Config(ctx context.Context) searcher.Config { return instance.Config() } func Update(ctx context.Context, parent string, objs []model.Obj) { if instance == nil || !instance.Config().AutoUpdate || !setting.GetBool(conf.AutoUpdateIndex) || Running() { return } if isIgnorePath(parent) { return } // only update when index have built progress, err := Progress() if err != nil { log.Errorf("update search index error while get progress: %+v", err) return } if !progress.IsDone { return } // Use task queue for Meilisearch to avoid race conditions with async indexing if msInstance, ok := instance.(interface { EnqueueUpdate(parent string, objs []model.Obj) }); ok { // Enqueue task for async processing (diff calculation happens at consumption time) msInstance.EnqueueUpdate(parent, objs) return } nodes, err := instance.Get(ctx, parent) if err != nil { log.Errorf("update search index error while get nodes: %+v", err) return } now := mapset.NewSet[string]() for i := range objs { now.Add(objs[i].GetName()) } old := mapset.NewSet[string]() for i := range nodes { old.Add(nodes[i].Name) } // delete data that no longer exists toDelete := old.Difference(now) toAdd := now.Difference(old) for i := range nodes { if toDelete.Contains(nodes[i].Name) && !op.HasStorage(path.Join(parent, nodes[i].Name)) { log.Debugf("delete index: %s", path.Join(parent, nodes[i].Name)) err = instance.Del(ctx, path.Join(parent, nodes[i].Name)) if err != nil { log.Errorf("update search index error while del old node: %+v", err) return } } } // collect files and folders to add in batch var toAddObjs []ObjWithParent for i := range objs { if toAdd.Contains(objs[i].GetName()) { log.Debugf("add index: %s", path.Join(parent, objs[i].GetName())) toAddObjs = append(toAddObjs, ObjWithParent{ Parent: parent, Obj: objs[i], }) } } // batch index all files and folders at once if len(toAddObjs) > 0 { err = BatchIndex(ctx, toAddObjs) if err != nil { log.Errorf("update search index error while batch index new nodes: %+v", err) return } } } func init() { op.RegisterObjsUpdateHook(Update) } ================================================ FILE: internal/search/db/init.go ================================================ package db import ( "fmt" "strings" log "github.com/sirupsen/logrus" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/search/searcher" ) var config = searcher.Config{ Name: "database", AutoUpdate: true, } func init() { searcher.RegisterSearcher(config, func() (searcher.Searcher, error) { db := db.GetDb() switch conf.Conf.Database.Type { case "mysql": tableName := fmt.Sprintf("%ssearch_nodes", conf.Conf.Database.TablePrefix) tx := db.Exec(fmt.Sprintf("CREATE FULLTEXT INDEX idx_%s_name_fulltext ON %s(name);", tableName, tableName)) if err := tx.Error; err != nil && !strings.Contains(err.Error(), "Error 1061 (42000)") { // duplicate error log.Errorf("failed to create full text index: %v", err) return nil, err } case "postgres": db.Exec("CREATE EXTENSION pg_trgm;") db.Exec("CREATE EXTENSION btree_gin;") tableName := fmt.Sprintf("%ssearch_nodes", conf.Conf.Database.TablePrefix) tx := db.Exec(fmt.Sprintf("CREATE INDEX idx_%s_name ON %s USING GIN (name);", tableName, tableName)) if err := tx.Error; err != nil && !strings.Contains(err.Error(), "SQLSTATE 42P07") { log.Errorf("failed to create index using GIN: %v", err) return nil, err } } return &DB{}, nil }) } ================================================ FILE: internal/search/db/search.go ================================================ package db import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/search/searcher" ) type DB struct{} func (D DB) Config() searcher.Config { return config } func (D DB) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { return db.SearchNode(req, true) } func (D DB) Index(ctx context.Context, node model.SearchNode) error { return db.CreateSearchNode(&node) } func (D DB) BatchIndex(ctx context.Context, nodes []model.SearchNode) error { return db.BatchCreateSearchNodes(&nodes) } func (D DB) Get(ctx context.Context, parent string) ([]model.SearchNode, error) { return db.GetSearchNodesByParent(parent) } func (D DB) Del(ctx context.Context, path string) error { return db.DeleteSearchNodesByParent(path) } func (D DB) Release(ctx context.Context) error { return nil } func (D DB) Clear(ctx context.Context) error { return db.ClearSearchNodes() } var _ searcher.Searcher = (*DB)(nil) ================================================ FILE: internal/search/db_non_full_text/init.go ================================================ package db_non_full_text import ( "github.com/OpenListTeam/OpenList/v4/internal/search/searcher" ) var config = searcher.Config{ Name: "database_non_full_text", AutoUpdate: true, } func init() { searcher.RegisterSearcher(config, func() (searcher.Searcher, error) { return &DB{}, nil }) } ================================================ FILE: internal/search/db_non_full_text/search.go ================================================ package db_non_full_text import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/search/searcher" ) type DB struct{} func (D DB) Config() searcher.Config { return config } func (D DB) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { return db.SearchNode(req, false) } func (D DB) Index(ctx context.Context, node model.SearchNode) error { return db.CreateSearchNode(&node) } func (D DB) BatchIndex(ctx context.Context, nodes []model.SearchNode) error { return db.BatchCreateSearchNodes(&nodes) } func (D DB) Get(ctx context.Context, parent string) ([]model.SearchNode, error) { return db.GetSearchNodesByParent(parent) } func (D DB) Del(ctx context.Context, path string) error { return db.DeleteSearchNodesByParent(path) } func (D DB) Release(ctx context.Context) error { return nil } func (D DB) Clear(ctx context.Context) error { return db.ClearSearchNodes() } var _ searcher.Searcher = (*DB)(nil) ================================================ FILE: internal/search/import.go ================================================ package search import ( _ "github.com/OpenListTeam/OpenList/v4/internal/search/bleve" _ "github.com/OpenListTeam/OpenList/v4/internal/search/db" _ "github.com/OpenListTeam/OpenList/v4/internal/search/db_non_full_text" _ "github.com/OpenListTeam/OpenList/v4/internal/search/meilisearch" ) ================================================ FILE: internal/search/meilisearch/init.go ================================================ package meilisearch import ( "errors" "fmt" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/search/searcher" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/meilisearch/meilisearch-go" ) var config = searcher.Config{ Name: "meilisearch", AutoUpdate: true, } func init() { searcher.RegisterSearcher(config, func() (searcher.Searcher, error) { indexUid := conf.Conf.Meilisearch.Index if len(indexUid) == 0 { return nil, errors.New("index is blank") } m := Meilisearch{ Client: meilisearch.New( conf.Conf.Meilisearch.Host, meilisearch.WithAPIKey(conf.Conf.Meilisearch.APIKey), ), IndexUid: indexUid, FilterableAttributes: []string{"parent", "is_dir", "name", "parent_hash", "parent_path_hashes"}, SearchableAttributes: []string{"name"}, } _, err := m.Client.GetIndex(m.IndexUid) if err != nil { var mErr *meilisearch.Error ok := errors.As(err, &mErr) if ok && mErr.MeilisearchApiError.Code == "index_not_found" { task, err := m.Client.CreateIndex(&meilisearch.IndexConfig{ Uid: m.IndexUid, PrimaryKey: "id", }) if err != nil { return nil, err } forTask, err := m.Client.WaitForTask(task.TaskUID, time.Second) if err != nil { return nil, err } if forTask.Status != meilisearch.TaskStatusSucceeded { return nil, fmt.Errorf("index creation failed, task status is %s", forTask.Status) } } else { return nil, err } } attributes, err := m.Client.Index(m.IndexUid).GetFilterableAttributes() if err != nil { return nil, err } if attributes == nil || !utils.SliceAllContains(*attributes, m.FilterableAttributes...) { _, err = m.Client.Index(m.IndexUid).UpdateFilterableAttributes(&m.FilterableAttributes) if err != nil { return nil, err } } attributes, err = m.Client.Index(m.IndexUid).GetSearchableAttributes() if err != nil { return nil, err } if attributes == nil || !utils.SliceAllContains(*attributes, m.SearchableAttributes...) { _, err = m.Client.Index(m.IndexUid).UpdateSearchableAttributes(&m.SearchableAttributes) if err != nil { return nil, err } } pagination, err := m.Client.Index(m.IndexUid).GetPagination() if err != nil { return nil, err } if pagination.MaxTotalHits != int64(model.MaxInt) { _, err := m.Client.Index(m.IndexUid).UpdatePagination(&meilisearch.Pagination{ MaxTotalHits: int64(model.MaxInt), }) if err != nil { return nil, err } } // Initialize and start task queue manager m.taskQueue = NewTaskQueueManager(&m) m.taskQueue.Start() return &m, nil }) } ================================================ FILE: internal/search/meilisearch/search.go ================================================ package meilisearch import ( "context" "fmt" "path" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/search/searcher" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/meilisearch/meilisearch-go" ) type searchDocument struct { // Document id, hash of the file path, // can be used for filtering a file exactly(case-sensitively). ID string `json:"id"` // Hash of parent, can be used for filtering direct children. ParentHash string `json:"parent_hash"` // One-by-one hash of parent paths (path hierarchy). // eg: A file's parent is '/home/a/b', // its parent paths are '/home/a/b', '/home/a', '/home', '/'. // Can be used for filtering all descendants exactly. // Storing path hashes instead of plaintext paths benefits disk usage and case-sensitive filter. ParentPathHashes []string `json:"parent_path_hashes"` model.SearchNode } type Meilisearch struct { Client meilisearch.ServiceManager IndexUid string FilterableAttributes []string SearchableAttributes []string taskQueue *TaskQueueManager } func (m *Meilisearch) Config() searcher.Config { return config } func (m *Meilisearch) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { mReq := &meilisearch.SearchRequest{ AttributesToSearchOn: m.SearchableAttributes, Page: int64(req.Page), HitsPerPage: int64(req.PerPage), } var filters []string if req.Scope != 0 { filters = append(filters, fmt.Sprintf("is_dir = %v", req.Scope == 1)) } if req.Parent != "" && req.Parent != "/" { // use parent_path_hashes to filter descendants parentHash := hashPath(req.Parent) filters = append(filters, fmt.Sprintf("parent_path_hashes = '%s'", parentHash)) } if len(filters) > 0 { mReq.Filter = strings.Join(filters, " AND ") } search, err := m.Client.Index(m.IndexUid).SearchWithContext(ctx, req.Keywords, mReq) if err != nil { return nil, 0, err } nodes, err := utils.SliceConvert(search.Hits, func(src any) (model.SearchNode, error) { srcMap := src.(map[string]any) return model.SearchNode{ Parent: srcMap["parent"].(string), Name: srcMap["name"].(string), IsDir: srcMap["is_dir"].(bool), Size: int64(srcMap["size"].(float64)), }, nil }) if err != nil { return nil, 0, err } return nodes, search.TotalHits, nil } func (m *Meilisearch) Index(ctx context.Context, node model.SearchNode) error { return m.BatchIndex(ctx, []model.SearchNode{node}) } func (m *Meilisearch) BatchIndex(ctx context.Context, nodes []model.SearchNode) error { documents, err := utils.SliceConvert(nodes, func(src model.SearchNode) (*searchDocument, error) { parentHash := hashPath(src.Parent) nodePath := path.Join(src.Parent, src.Name) nodePathHash := hashPath(nodePath) parentPaths := utils.GetPathHierarchy(src.Parent) parentPathHashes, err := utils.SliceConvert(parentPaths, func(parentPath string) (string, error) { return hashPath(parentPath), nil }) if err != nil { return nil, err } return &searchDocument{ ID: nodePathHash, ParentHash: parentHash, ParentPathHashes: parentPathHashes, SearchNode: src, }, nil }) if err != nil { return err } // max up to 10,000 documents per batch to reduce error rate while uploading over the Internet _, err = m.Client.Index(m.IndexUid).AddDocumentsInBatchesWithContext(ctx, documents, 10000) if err != nil { return err } // documents were uploaded and enqueued for indexing, just return early //// Wait for the task to complete and check //forTask, err := m.Client.WaitForTask(task.TaskUID, meilisearch.WaitParams{ // Context: ctx, // Interval: time.Millisecond * 50, //}) //if err != nil { // return err //} //if forTask.Status != meilisearch.TaskStatusSucceeded { // return fmt.Errorf("BatchIndex failed, task status is %s", forTask.Status) //} return nil } func (m *Meilisearch) getDocumentsByParent(ctx context.Context, parent string) ([]*searchDocument, error) { var result meilisearch.DocumentsResult query := &meilisearch.DocumentsQuery{ Limit: int64(model.MaxInt), } if parent != "" && parent != "/" { // use parent_hash to filter direct children parentHash := hashPath(parent) query.Filter = fmt.Sprintf("parent_hash = '%s'", parentHash) } err := m.Client.Index(m.IndexUid).GetDocumentsWithContext(ctx, query, &result) if err != nil { return nil, err } return utils.SliceConvert(result.Results, func(src map[string]any) (*searchDocument, error) { return buildSearchDocumentFromResults(src), nil }) } func (m *Meilisearch) Get(ctx context.Context, parent string) ([]model.SearchNode, error) { result, err := m.getDocumentsByParent(ctx, parent) if err != nil { return nil, err } return utils.SliceConvert(result, func(src *searchDocument) (model.SearchNode, error) { return src.SearchNode, nil }) } func (m *Meilisearch) getDocumentInPath(ctx context.Context, parent string, name string) (*searchDocument, error) { var result searchDocument // join them and calculate the hash to exactly identify the node nodePath := path.Join(parent, name) nodePathHash := hashPath(nodePath) err := m.Client.Index(m.IndexUid).GetDocumentWithContext(ctx, nodePathHash, nil, &result) if err != nil { // return nil for documents that no exists if err.(*meilisearch.Error).StatusCode == 404 { return nil, nil } return nil, err } return &result, nil } func (m *Meilisearch) delDirChild(ctx context.Context, prefix string) error { prefix = hashPath(prefix) // use parent_path_hashes to filter descendants, // so no longer need to walk through the directories to get their IDs, // speeding up the deletion process with easy maintained codebase filter := fmt.Sprintf("parent_path_hashes = '%s'", prefix) _, err := m.Client.Index(m.IndexUid).DeleteDocumentsByFilterWithContext(ctx, filter) // task was enqueued (if succeed), no need to wait return err } func (m *Meilisearch) Del(ctx context.Context, prefix string) error { prefix = utils.FixAndCleanPath(prefix) dir, name := path.Split(prefix) if dir != "/" { dir = dir[:len(dir)-1] } document, err := m.getDocumentInPath(ctx, dir, name) if err != nil { return err } if document == nil { // Defensive programming. Document may be the folder, try deleting Child return m.delDirChild(ctx, prefix) } if document.IsDir { err = m.delDirChild(ctx, prefix) if err != nil { return err } } _, err = m.Client.Index(m.IndexUid).DeleteDocumentWithContext(ctx, document.ID) // task was enqueued (if succeed), no need to wait return err } func (m *Meilisearch) Release(ctx context.Context) error { if m.taskQueue != nil { m.taskQueue.Stop() } return nil } func (m *Meilisearch) Clear(ctx context.Context) error { _, err := m.Client.Index(m.IndexUid).DeleteAllDocumentsWithContext(ctx) // task was enqueued (if succeed), no need to wait return err } func (m *Meilisearch) getTaskStatus(ctx context.Context, taskUID int64) (meilisearch.TaskStatus, error) { forTask, err := m.Client.WaitForTaskWithContext(ctx, taskUID, time.Second) if err != nil { return meilisearch.TaskStatusUnknown, err } return forTask.Status, nil } // EnqueueUpdate enqueues an update task to the task queue func (m *Meilisearch) EnqueueUpdate(parent string, objs []model.Obj) { if m.taskQueue == nil { return } m.taskQueue.Enqueue(parent, objs) } // batchIndexWithTaskUID indexes documents and returns all taskUIDs func (m *Meilisearch) batchIndexWithTaskUID(ctx context.Context, nodes []model.SearchNode) ([]int64, error) { if len(nodes) == 0 { return nil, nil } documents, err := utils.SliceConvert(nodes, func(src model.SearchNode) (*searchDocument, error) { parentHash := hashPath(src.Parent) nodePath := path.Join(src.Parent, src.Name) nodePathHash := hashPath(nodePath) parentPaths := utils.GetPathHierarchy(src.Parent) parentPathHashes, err := utils.SliceConvert(parentPaths, func(parentPath string) (string, error) { return hashPath(parentPath), nil }) if err != nil { return nil, err } return &searchDocument{ ID: nodePathHash, ParentHash: parentHash, ParentPathHashes: parentPathHashes, SearchNode: src, }, nil }) if err != nil { return nil, err } // max up to 10,000 documents per batch to reduce error rate while uploading over the Internet tasks, err := m.Client.Index(m.IndexUid).AddDocumentsInBatchesWithContext(ctx, documents, 10000) if err != nil { return nil, err } // Return all task UIDs taskUIDs := make([]int64, 0, len(tasks)) for _, task := range tasks { taskUIDs = append(taskUIDs, task.TaskUID) } return taskUIDs, nil } // batchDeleteWithTaskUID deletes documents and returns all taskUIDs func (m *Meilisearch) batchDeleteWithTaskUID(ctx context.Context, paths []string) ([]int64, error) { if len(paths) == 0 { return nil, nil } // Deduplicate paths first pathSet := make(map[string]struct{}) uniquePaths := make([]string, 0, len(paths)) for _, p := range paths { p = utils.FixAndCleanPath(p) if _, exists := pathSet[p]; !exists { pathSet[p] = struct{}{} uniquePaths = append(uniquePaths, p) } } const batchSize = 100 // max paths per batch to avoid filter length limits var taskUIDs []int64 // Process in batches to avoid filter length limits for i := 0; i < len(uniquePaths); i += batchSize { end := i + batchSize if end > len(uniquePaths) { end = len(uniquePaths) } batch := uniquePaths[i:end] // Build combined filter to delete all children in one request // Format: parent_path_hashes = 'hash1' OR parent_path_hashes = 'hash2' OR ... var filters []string for _, p := range batch { pathHash := hashPath(p) filters = append(filters, fmt.Sprintf("parent_path_hashes = '%s'", pathHash)) } if len(filters) > 0 { combinedFilter := strings.Join(filters, " OR ") // Delete all children for all paths in one request task, err := m.Client.Index(m.IndexUid).DeleteDocumentsByFilterWithContext(ctx, combinedFilter) if err != nil { return nil, err } taskUIDs = append(taskUIDs, task.TaskUID) } // Convert paths to document IDs and batch delete documentIDs := make([]string, 0, len(batch)) for _, p := range batch { documentIDs = append(documentIDs, hashPath(p)) } // Use batch delete API task, err := m.Client.Index(m.IndexUid).DeleteDocumentsWithContext(ctx, documentIDs) if err != nil { return nil, err } taskUIDs = append(taskUIDs, task.TaskUID) } return taskUIDs, nil } ================================================ FILE: internal/search/meilisearch/task_queue.go ================================================ package meilisearch import ( "context" "path" "sort" "strings" "sync" "sync/atomic" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" mapset "github.com/deckarep/golang-set/v2" log "github.com/sirupsen/logrus" ) // QueuedTask represents a task in the queue type QueuedTask struct { Parent string Objs []model.Obj // current file system state Depth int // path depth for sorting EnqueueAt time.Time // enqueue time } // TaskQueueManager manages the task queue for async index operations type TaskQueueManager struct { queue map[string]*QueuedTask // parent -> task pendingTasks map[string][]int64 // parent -> all submitted taskUIDs mu sync.RWMutex ticker *time.Ticker stopCh chan struct{} m *Meilisearch consuming atomic.Bool // flag to prevent concurrent consumption } // NewTaskQueueManager creates a new task queue manager func NewTaskQueueManager(m *Meilisearch) *TaskQueueManager { return &TaskQueueManager{ queue: make(map[string]*QueuedTask), pendingTasks: make(map[string][]int64), stopCh: make(chan struct{}), m: m, } } // calculateDepth calculates the depth of a path func calculateDepth(path string) int { if path == "/" { return 0 } return strings.Count(strings.Trim(path, "/"), "/") + 1 } // Enqueue enqueues a task with current file system state func (tqm *TaskQueueManager) Enqueue(parent string, objs []model.Obj) { tqm.mu.Lock() defer tqm.mu.Unlock() // deduplicate: overwrite existing task with the same parent tqm.queue[parent] = &QueuedTask{ Parent: parent, Objs: objs, Depth: calculateDepth(parent), EnqueueAt: time.Now(), } log.Debugf("enqueued update task for parent: %s, depth: %d, objs: %d", parent, calculateDepth(parent), len(objs)) } // Start starts the task queue consumer func (tqm *TaskQueueManager) Start() { tqm.ticker = time.NewTicker(30 * time.Second) go func() { for { select { case <-tqm.ticker.C: tqm.consume() case <-tqm.stopCh: log.Info("task queue manager stopped") return } } }() log.Info("task queue manager started, will consume every 30 seconds") } // Stop stops the task queue consumer func (tqm *TaskQueueManager) Stop() { if tqm.ticker != nil { tqm.ticker.Stop() } close(tqm.stopCh) } // consume processes all tasks in the queue func (tqm *TaskQueueManager) consume() { // Prevent concurrent consumption if !tqm.consuming.CompareAndSwap(false, true) { log.Warn("previous consume still running, skip this round") return } defer tqm.consuming.Store(false) tqm.mu.Lock() // extract all tasks tasks := make([]*QueuedTask, 0, len(tqm.queue)) for _, task := range tqm.queue { tasks = append(tasks, task) } // clear queue tqm.queue = make(map[string]*QueuedTask) tqm.mu.Unlock() if len(tasks) == 0 { return } log.Infof("consuming task queue: %d tasks", len(tasks)) // sort tasks: shallow paths first, then by enqueue time sort.Slice(tasks, func(i, j int) bool { if tasks[i].Depth != tasks[j].Depth { return tasks[i].Depth < tasks[j].Depth } return tasks[i].EnqueueAt.Before(tasks[j].EnqueueAt) }) ctx := context.Background() // execute tasks in order for _, task := range tasks { // Check if there are pending tasks for this parent tqm.mu.RLock() pendingTaskUIDs, hasPending := tqm.pendingTasks[task.Parent] tqm.mu.RUnlock() if hasPending && len(pendingTaskUIDs) > 0 { // Check all pending task statuses allCompleted := true for _, taskUID := range pendingTaskUIDs { taskStatus, err := tqm.m.getTaskStatus(ctx, taskUID) if err != nil { log.Errorf("failed to get task status for parent %s (taskUID: %d): %v", task.Parent, taskUID, err) // If we can't get status, assume it's done and continue checking continue } // Check if task is still running if taskStatus == "enqueued" || taskStatus == "processing" { log.Warnf("skipping task for parent %s: previous task %d still %s", task.Parent, taskUID, taskStatus) allCompleted = false break // No need to check remaining tasks } } if !allCompleted { // Re-enqueue the task if not already in queue (avoid overwriting newer snapshots) tqm.mu.Lock() if _, exists := tqm.queue[task.Parent]; !exists { tqm.queue[task.Parent] = task log.Debugf("re-enqueued skipped task for parent %s due to pending tasks", task.Parent) } else { log.Debugf("skipped task for parent %s not re-enqueued (newer task already in queue)", task.Parent) } tqm.mu.Unlock() continue // Skip this task, some previous tasks are still running } // All tasks are in terminal state, remove from pending log.Debugf("all previous tasks for parent %s are completed, proceeding with new task", task.Parent) tqm.mu.Lock() delete(tqm.pendingTasks, task.Parent) tqm.mu.Unlock() } // Execute the task tqm.executeTask(ctx, task) } log.Infof("task queue consumption completed") } // executeTask executes a single task func (tqm *TaskQueueManager) executeTask(ctx context.Context, task *QueuedTask) { parent := task.Parent currentObjs := task.Objs // Query index to get old state nodes, err := tqm.m.Get(ctx, parent) if err != nil { log.Errorf("failed to get indexed nodes for parent %s: %v", parent, err) return } // Calculate diff based on current index state now := mapset.NewSet[string]() for i := range currentObjs { now.Add(currentObjs[i].GetName()) } old := mapset.NewSet[string]() for i := range nodes { old.Add(nodes[i].Name) } toDelete := old.Difference(now) toAdd := now.Difference(old) // Collect paths to delete var pathsToDelete []string for i := range nodes { if toDelete.Contains(nodes[i].Name) && !op.HasStorage(path.Join(parent, nodes[i].Name)) { pathsToDelete = append(pathsToDelete, path.Join(parent, nodes[i].Name)) } } var allTaskUIDs []int64 // Execute delete first if len(pathsToDelete) > 0 { log.Debugf("executing delete for parent %s: %d paths", parent, len(pathsToDelete)) taskUIDs, err := tqm.m.batchDeleteWithTaskUID(ctx, pathsToDelete) if err != nil { log.Errorf("failed to batch delete for parent %s: %v", parent, err) // Continue to add even if delete fails } else { allTaskUIDs = append(allTaskUIDs, taskUIDs...) } } // Collect objects to add var nodesToAdd []model.SearchNode for i := range currentObjs { if toAdd.Contains(currentObjs[i].GetName()) { log.Debugf("will add index: %s", path.Join(parent, currentObjs[i].GetName())) nodesToAdd = append(nodesToAdd, model.SearchNode{ Parent: parent, Name: currentObjs[i].GetName(), IsDir: currentObjs[i].IsDir(), Size: currentObjs[i].GetSize(), }) } } // Execute add if len(nodesToAdd) > 0 { log.Debugf("executing add for parent %s: %d nodes", parent, len(nodesToAdd)) taskUIDs, err := tqm.m.batchIndexWithTaskUID(ctx, nodesToAdd) if err != nil { log.Errorf("failed to batch index for parent %s: %v", parent, err) } else { allTaskUIDs = append(allTaskUIDs, taskUIDs...) } } // Record all task UIDs for this parent if len(allTaskUIDs) > 0 { tqm.mu.Lock() tqm.pendingTasks[parent] = allTaskUIDs tqm.mu.Unlock() log.Debugf("recorded %d taskUIDs for parent %s", len(allTaskUIDs), parent) } } ================================================ FILE: internal/search/meilisearch/utils.go ================================================ package meilisearch import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) // hashPath hashes a path with SHA-1. // Path-relative exact matching should use hash, // because filtering strings on meilisearch is case-insensitive. func hashPath(path string) string { return utils.HashData(utils.SHA1, []byte(path)) } func buildSearchDocumentFromResults(results map[string]any) *searchDocument { document := &searchDocument{} // use assertion test to avoid panic document.SearchNode.Parent, _ = results["parent"].(string) document.SearchNode.Name, _ = results["name"].(string) document.SearchNode.IsDir, _ = results["is_dir"].(bool) // JSON numbers are typically float64, not int64 if size, ok := results["size"].(float64); ok { document.SearchNode.Size = int64(size) } document.ID, _ = results["id"].(string) document.ParentHash, _ = results["parent_hash"].(string) document.ParentPathHashes, _ = results["parent_path_hashes"].([]string) return document } ================================================ FILE: internal/search/search.go ================================================ package search import ( "context" "fmt" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/search/searcher" log "github.com/sirupsen/logrus" ) var instance searcher.Searcher = nil // Init or reset index func Init(mode string) error { if instance != nil { // unchanged, do nothing if instance.Config().Name == mode { return nil } err := instance.Release(context.Background()) if err != nil { log.Errorf("release instance err: %+v", err) } instance = nil } if Running() { return fmt.Errorf("index is running") } if mode == "none" { log.Warnf("not enable search") return nil } s, ok := searcher.NewMap[mode] if !ok { return fmt.Errorf("not support index: %s", mode) } i, err := s() if err != nil { log.Errorf("init searcher error: %+v", err) } else { instance = i } return err } func Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { return instance.Search(ctx, req) } func Index(ctx context.Context, parent string, obj model.Obj) error { if instance == nil { return errs.SearchNotAvailable } return instance.Index(ctx, model.SearchNode{ Parent: parent, Name: obj.GetName(), IsDir: obj.IsDir(), Size: obj.GetSize(), }) } type ObjWithParent struct { Parent string model.Obj } func BatchIndex(ctx context.Context, objs []ObjWithParent) error { if instance == nil { return errs.SearchNotAvailable } if len(objs) == 0 { return nil } var searchNodes []model.SearchNode for i := range objs { searchNodes = append(searchNodes, model.SearchNode{ Parent: objs[i].Parent, Name: objs[i].GetName(), IsDir: objs[i].IsDir(), Size: objs[i].GetSize(), }) } return instance.BatchIndex(ctx, searchNodes) } func init() { op.RegisterSettingItemHook(conf.SearchIndex, func(item *model.SettingItem) error { log.Debugf("searcher init, mode: %s", item.Value) return Init(item.Value) }) } ================================================ FILE: internal/search/searcher/manage.go ================================================ package searcher type New func() (Searcher, error) var NewMap = map[string]New{} func RegisterSearcher(config Config, searcher New) { NewMap[config.Name] = searcher } ================================================ FILE: internal/search/searcher/searcher.go ================================================ package searcher import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/model" ) type Config struct { Name string AutoUpdate bool } type Searcher interface { // Config of the searcher Config() Config // Search specific keywords in specific path Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) // Index obj with parent Index(ctx context.Context, node model.SearchNode) error // BatchIndex obj with parent BatchIndex(ctx context.Context, nodes []model.SearchNode) error // Get by parent Get(ctx context.Context, parent string) ([]model.SearchNode, error) // Del with prefix Del(ctx context.Context, prefix string) error // Release resource Release(ctx context.Context) error // Clear all index Clear(ctx context.Context) error } ================================================ FILE: internal/search/util.go ================================================ package search import ( "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/drivers/openlist" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" ) func Progress() (*model.IndexProgress, error) { p := setting.GetStr(conf.IndexProgress) var progress model.IndexProgress err := utils.Json.UnmarshalFromString(p, &progress) return &progress, err } func WriteProgress(progress *model.IndexProgress) { p, err := utils.Json.MarshalToString(progress) if err != nil { log.Errorf("marshal progress error: %+v", err) } err = op.SaveSettingItem(&model.SettingItem{ Key: conf.IndexProgress, Value: p, Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE, }) if err != nil { log.Errorf("save progress error: %+v", err) } } func updateIgnorePaths(customIgnorePaths string) { storages := op.GetAllStorages() ignorePaths := make([]string, 0) var skipDrivers = []string{"OpenList", "Virtual"} v3Visited := make(map[string]bool) for _, storage := range storages { if utils.SliceContains(skipDrivers, storage.Config().Name) { if storage.Config().Name == "OpenList" { addition := storage.GetAddition().(*openlist.Addition) allowIndexed, visited := v3Visited[addition.Address] if !visited { url := addition.Address + "/api/public/settings" res, err := base.RestyClient.R().Get(url) if err == nil { log.Debugf("allow_indexed body: %+v", res.String()) allowIndexed = utils.Json.Get(res.Body(), "data", conf.AllowIndexed).ToString() == "true" v3Visited[addition.Address] = allowIndexed } } log.Debugf("%s allow_indexed: %v", addition.Address, allowIndexed) if !allowIndexed { ignorePaths = append(ignorePaths, storage.GetStorage().MountPath) } } else { ignorePaths = append(ignorePaths, storage.GetStorage().MountPath) } } } if customIgnorePaths != "" { ignorePaths = append(ignorePaths, strings.Split(customIgnorePaths, "\n")...) } conf.SlicesMap[conf.IgnorePaths] = ignorePaths } func isIgnorePath(path string) bool { for _, ignorePath := range conf.SlicesMap[conf.IgnorePaths] { if strings.HasPrefix(path, ignorePath) { return true } } return false } func init() { op.RegisterSettingItemHook(conf.IgnorePaths, func(item *model.SettingItem) error { updateIgnorePaths(item.Value) return nil }) op.RegisterStorageHook(func(typ string, storage driver.Driver) { var skipDrivers = []string{"OpenList", "Virtual"} if utils.SliceContains(skipDrivers, storage.Config().Name) { updateIgnorePaths(setting.GetStr(conf.IgnorePaths)) } }) } ================================================ FILE: internal/setting/setting.go ================================================ package setting import ( "strconv" "github.com/OpenListTeam/OpenList/v4/internal/op" ) func GetStr(key string, defaultValue ...string) string { val, _ := op.GetSettingItemByKey(key) if val == nil { if len(defaultValue) > 0 { return defaultValue[0] } return "" } return val.Value } func GetInt(key string, defaultVal int) int { i, err := strconv.Atoi(GetStr(key)) if err != nil { return defaultVal } return i } func GetBool(key string) bool { return GetStr(key) == "true" || GetStr(key) == "1" } func GetFloat(key string, defaultVal float64) float64 { f, err := strconv.ParseFloat(GetStr(key), 64) if err != nil { return defaultVal } return f } ================================================ FILE: internal/sharing/archive.go ================================================ package sharing import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" ) func archiveMeta(ctx context.Context, sid, path string, args model.SharingArchiveMetaArgs) (*model.Sharing, *model.ArchiveMetaProvider, error) { sharing, err := op.GetSharingById(sid, args.Refresh) if err != nil { return nil, nil, errors.WithStack(errs.SharingNotFound) } if !sharing.Valid() { return sharing, nil, errors.WithStack(errs.InvalidSharing) } if !sharing.Verify(args.Pwd) { return sharing, nil, errors.WithStack(errs.WrongShareCode) } path = utils.FixAndCleanPath(path) if len(sharing.Files) == 1 || path != "/" { unwrapPath, err := op.GetSharingUnwrapPath(sharing, path) if err != nil { return nil, nil, errors.WithMessage(err, "failed get sharing unwrap path") } storage, actualPath, err := op.GetStorageAndActualPath(unwrapPath) if err != nil { return nil, nil, errors.WithMessage(err, "failed get sharing file") } obj, err := op.GetArchiveMeta(ctx, storage, actualPath, args.ArchiveMetaArgs) return sharing, obj, err } return nil, nil, errors.New("cannot get sharing root archive meta") } func archiveList(ctx context.Context, sid, path string, args model.SharingArchiveListArgs) (*model.Sharing, []model.Obj, error) { sharing, err := op.GetSharingById(sid, args.Refresh) if err != nil { return nil, nil, errors.WithStack(errs.SharingNotFound) } if !sharing.Valid() { return sharing, nil, errors.WithStack(errs.InvalidSharing) } if !sharing.Verify(args.Pwd) { return sharing, nil, errors.WithStack(errs.WrongShareCode) } path = utils.FixAndCleanPath(path) if len(sharing.Files) == 1 || path != "/" { unwrapPath, err := op.GetSharingUnwrapPath(sharing, path) if err != nil { return nil, nil, errors.WithMessage(err, "failed get sharing unwrap path") } storage, actualPath, err := op.GetStorageAndActualPath(unwrapPath) if err != nil { return nil, nil, errors.WithMessage(err, "failed get sharing file") } obj, err := op.ListArchive(ctx, storage, actualPath, args.ArchiveListArgs) return sharing, obj, err } return nil, nil, errors.New("cannot get sharing root archive list") } ================================================ FILE: internal/sharing/get.go ================================================ package sharing import ( "context" stdpath "path" "time" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" ) func get(ctx context.Context, sid, path string, args model.SharingListArgs) (*model.Sharing, model.Obj, error) { sharing, err := op.GetSharingById(sid, args.Refresh) if err != nil { return nil, nil, errors.WithStack(errs.SharingNotFound) } if !sharing.Valid() { return sharing, nil, errors.WithStack(errs.InvalidSharing) } if !sharing.Verify(args.Pwd) { return sharing, nil, errors.WithStack(errs.WrongShareCode) } path = utils.FixAndCleanPath(path) if len(sharing.Files) == 1 || path != "/" { unwrapPath, err := op.GetSharingUnwrapPath(sharing, path) if err != nil { return nil, nil, errors.WithMessage(err, "failed get sharing unwrap path") } if unwrapPath != "/" { virtualFiles := op.GetStorageVirtualFilesByPath(stdpath.Dir(unwrapPath)) for _, f := range virtualFiles { if f.GetName() == stdpath.Base(unwrapPath) { return sharing, f, nil } } } else { return sharing, &model.Object{ Name: sid, Size: 0, Modified: time.Time{}, IsFolder: true, }, nil } storage, actualPath, err := op.GetStorageAndActualPath(unwrapPath) if err != nil { return nil, nil, errors.WithMessage(err, "failed get sharing file") } obj, err := op.Get(ctx, storage, actualPath) return sharing, obj, err } return sharing, &model.Object{ Name: sid, Size: 0, Modified: time.Time{}, IsFolder: true, }, nil } ================================================ FILE: internal/sharing/link.go ================================================ package sharing import ( "context" "strings" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" ) func link(ctx context.Context, sid, path string, args *LinkArgs) (*model.Sharing, *model.Link, model.Obj, error) { sharing, err := op.GetSharingById(sid, args.SharingListArgs.Refresh) if err != nil { return nil, nil, nil, errors.WithStack(errs.SharingNotFound) } if !sharing.Valid() { return sharing, nil, nil, errors.WithStack(errs.InvalidSharing) } if !sharing.Verify(args.Pwd) { return sharing, nil, nil, errors.WithStack(errs.WrongShareCode) } path = utils.FixAndCleanPath(path) if len(sharing.Files) == 1 || path != "/" { unwrapPath, err := op.GetSharingUnwrapPath(sharing, path) if err != nil { return nil, nil, nil, errors.WithMessage(err, "failed get sharing unwrap path") } storage, actualPath, err := op.GetStorageAndActualPath(unwrapPath) if err != nil { return nil, nil, nil, errors.WithMessage(err, "failed get sharing link") } l, obj, err := op.Link(ctx, storage, actualPath, args.LinkArgs) if err != nil { return nil, nil, nil, errors.WithMessage(err, "failed get sharing link") } if l.URL != "" && !strings.HasPrefix(l.URL, "http://") && !strings.HasPrefix(l.URL, "https://") { l.URL = common.GetApiUrl(ctx) + l.URL } return sharing, l, obj, nil } return nil, nil, nil, errors.New("cannot get sharing root link") } ================================================ FILE: internal/sharing/list.go ================================================ package sharing import ( "context" stdpath "path" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" ) func list(ctx context.Context, sid, path string, args model.SharingListArgs) (*model.Sharing, []model.Obj, error) { sharing, err := op.GetSharingById(sid, args.Refresh) if err != nil { return nil, nil, errors.WithStack(errs.SharingNotFound) } if !sharing.Valid() { return sharing, nil, errors.WithStack(errs.InvalidSharing) } if !sharing.Verify(args.Pwd) { return sharing, nil, errors.WithStack(errs.WrongShareCode) } path = utils.FixAndCleanPath(path) if len(sharing.Files) == 1 || path != "/" { unwrapPath, err := op.GetSharingUnwrapPath(sharing, path) if err != nil { return nil, nil, errors.WithMessage(err, "failed get sharing unwrap path") } virtualFiles := op.GetStorageVirtualFilesByPath(unwrapPath) storage, actualPath, err := op.GetStorageAndActualPath(unwrapPath) if err != nil && len(virtualFiles) == 0 { return nil, nil, errors.WithMessage(err, "failed list sharing") } var objs []model.Obj if storage != nil { objs, err = op.List(ctx, storage, actualPath, model.ListArgs{ Refresh: args.Refresh, ReqPath: stdpath.Join(sid, path), }) if err != nil && len(virtualFiles) == 0 { return nil, nil, errors.WithMessage(err, "failed list sharing") } } om := model.NewObjMerge() objs = om.Merge(objs, virtualFiles...) model.SortFiles(objs, sharing.OrderBy, sharing.OrderDirection) model.ExtractFolder(objs, sharing.ExtractFolder) return sharing, objs, nil } objs := make([]model.Obj, 0, len(sharing.Files)) for _, f := range sharing.Files { if f != "/" { isVf := false virtualFiles := op.GetStorageVirtualFilesByPath(stdpath.Dir(f)) for _, vf := range virtualFiles { if vf.GetName() == stdpath.Base(f) { objs = append(objs, vf) isVf = true break } } if isVf { continue } } else { continue } storage, actualPath, err := op.GetStorageAndActualPath(f) if err != nil { continue } obj, err := op.Get(ctx, storage, actualPath) if err != nil { continue } objs = append(objs, obj) } model.SortFiles(objs, sharing.OrderBy, sharing.OrderDirection) model.ExtractFolder(objs, sharing.ExtractFolder) return sharing, objs, nil } ================================================ FILE: internal/sharing/sharing.go ================================================ package sharing import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/model" log "github.com/sirupsen/logrus" ) func List(ctx context.Context, sid, path string, args model.SharingListArgs) (*model.Sharing, []model.Obj, error) { sharing, res, err := list(ctx, sid, path, args) if err != nil { log.Errorf("failed list sharing %s/%s: %+v", sid, path, err) return nil, nil, err } return sharing, res, nil } func Get(ctx context.Context, sid, path string, args model.SharingListArgs) (*model.Sharing, model.Obj, error) { sharing, res, err := get(ctx, sid, path, args) if err != nil { log.Warnf("failed get sharing %s/%s: %s", sid, path, err) return nil, nil, err } return sharing, res, nil } func ArchiveMeta(ctx context.Context, sid, path string, args model.SharingArchiveMetaArgs) (*model.Sharing, *model.ArchiveMetaProvider, error) { sharing, res, err := archiveMeta(ctx, sid, path, args) if err != nil { log.Warnf("failed get sharing archive meta %s/%s: %s", sid, path, err) return nil, nil, err } return sharing, res, nil } func ArchiveList(ctx context.Context, sid, path string, args model.SharingArchiveListArgs) (*model.Sharing, []model.Obj, error) { sharing, res, err := archiveList(ctx, sid, path, args) if err != nil { log.Warnf("failed list sharing archive %s/%s: %s", sid, path, err) return nil, nil, err } return sharing, res, nil } type LinkArgs struct { model.SharingListArgs model.LinkArgs } func Link(ctx context.Context, sid, path string, args *LinkArgs) (*model.Sharing, *model.Link, model.Obj, error) { sharing, res, file, err := link(ctx, sid, path, args) if err != nil { log.Errorf("failed get sharing link %s/%s: %+v", sid, path, err) return nil, nil, nil, err } return sharing, res, file, nil } ================================================ FILE: internal/sign/archive.go ================================================ package sign import ( "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/sign" ) var onceArchive sync.Once var instanceArchive sign.Sign func SignArchive(data string) string { expire := setting.GetInt(conf.LinkExpiration, 0) if expire == 0 { return NotExpiredArchive(data) } else { return WithDurationArchive(data, time.Duration(expire)*time.Hour) } } func WithDurationArchive(data string, d time.Duration) string { onceArchive.Do(InstanceArchive) return instanceArchive.Sign(data, time.Now().Add(d).Unix()) } func NotExpiredArchive(data string) string { onceArchive.Do(InstanceArchive) return instanceArchive.Sign(data, 0) } func VerifyArchive(data string, sign string) error { onceArchive.Do(InstanceArchive) return instanceArchive.Verify(data, sign) } func InstanceArchive() { instanceArchive = sign.NewHMACSign([]byte(setting.GetStr(conf.Token) + "-archive")) } ================================================ FILE: internal/sign/sign.go ================================================ package sign import ( "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/sign" ) var once sync.Once var instance sign.Sign func Sign(data string) string { expire := setting.GetInt(conf.LinkExpiration, 0) if expire == 0 { return NotExpired(data) } else { return WithDuration(data, time.Duration(expire)*time.Hour) } } func WithDuration(data string, d time.Duration) string { once.Do(Instance) return instance.Sign(data, time.Now().Add(d).Unix()) } func NotExpired(data string) string { once.Do(Instance) return instance.Sign(data, 0) } func Verify(data string, sign string) error { once.Do(Instance) return instance.Verify(data, sign) } func Instance() { instance = sign.NewHMACSign([]byte(setting.GetStr(conf.Token))) } ================================================ FILE: internal/stream/limit.go ================================================ package stream import ( "context" "io" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "golang.org/x/time/rate" ) type Limiter interface { Limit() rate.Limit Burst() int TokensAt(time.Time) float64 Tokens() float64 Allow() bool AllowN(time.Time, int) bool Reserve() *rate.Reservation ReserveN(time.Time, int) *rate.Reservation Wait(context.Context) error WaitN(context.Context, int) error SetLimit(rate.Limit) SetLimitAt(time.Time, rate.Limit) SetBurst(int) SetBurstAt(time.Time, int) } var ( ClientDownloadLimit Limiter ClientUploadLimit Limiter ServerDownloadLimit Limiter ServerUploadLimit Limiter ) type RateLimitReader struct { io.Reader Limiter Limiter Ctx context.Context } func (r *RateLimitReader) Read(p []byte) (n int, err error) { if err = r.Ctx.Err(); err != nil { return 0, err } n, err = r.Reader.Read(p) if err != nil { return } if r.Limiter != nil { err = r.Limiter.WaitN(r.Ctx, n) } return } func (r *RateLimitReader) Close() error { if c, ok := r.Reader.(io.Closer); ok { return c.Close() } return nil } type RateLimitWriter struct { io.Writer Limiter Limiter Ctx context.Context } func (w *RateLimitWriter) Write(p []byte) (n int, err error) { if err = w.Ctx.Err(); err != nil { return 0, err } n, err = w.Writer.Write(p) if err != nil { return } if w.Limiter != nil { err = w.Limiter.WaitN(w.Ctx, n) } return } func (w *RateLimitWriter) Close() error { if c, ok := w.Writer.(io.Closer); ok { return c.Close() } return nil } type RateLimitFile struct { model.File Limiter Limiter Ctx context.Context } func (r *RateLimitFile) Read(p []byte) (n int, err error) { if err = r.Ctx.Err(); err != nil { return 0, err } n, err = r.File.Read(p) if err != nil { return } if r.Limiter != nil { err = r.Limiter.WaitN(r.Ctx, n) } return } func (r *RateLimitFile) ReadAt(p []byte, off int64) (n int, err error) { if err = r.Ctx.Err(); err != nil { return 0, err } n, err = r.File.ReadAt(p, off) if err != nil { return } if r.Limiter != nil { err = r.Limiter.WaitN(r.Ctx, n) } return } func (r *RateLimitFile) Close() error { if c, ok := r.File.(io.Closer); ok { return c.Close() } return nil } type RateLimitRangeReaderFunc RangeReaderFunc func (f RateLimitRangeReaderFunc) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { if ServerDownloadLimit == nil { return f(ctx, httpRange) } rc, err := f(ctx, httpRange) if err != nil { return nil, err } return &RateLimitReader{ Ctx: ctx, Reader: rc, Limiter: ServerDownloadLimit, }, nil } ================================================ FILE: internal/stream/stream.go ================================================ package stream import ( "context" "errors" "fmt" "io" "math" "os" "sync" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/buffer" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/rclone/rclone/lib/mmap" "go4.org/readerutil" ) type FileStream struct { Ctx context.Context model.Obj io.Reader Mimetype string WebPutAsTask bool ForceStreamUpload bool Exist model.Obj //the file existed in the destination, we can reuse some info since we wil overwrite it utils.Closers size int64 peekBuff *buffer.Reader oriReader io.Reader // the original reader, used for caching } func (f *FileStream) GetSize() int64 { if f.size > 0 { return f.size } return f.Obj.GetSize() } func (f *FileStream) GetMimetype() string { return f.Mimetype } func (f *FileStream) NeedStore() bool { return f.WebPutAsTask } func (f *FileStream) IsForceStreamUpload() bool { return f.ForceStreamUpload } func (f *FileStream) Close() error { if f.peekBuff != nil { f.peekBuff.Reset() f.oriReader = nil f.peekBuff = nil } return f.Closers.Close() } func (f *FileStream) GetExist() model.Obj { return f.Exist } func (f *FileStream) SetExist(obj model.Obj) { f.Exist = obj } // CacheFullAndWriter save all data into tmpFile or memory. // It's not thread-safe! func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writer) (model.File, error) { if cache := f.GetFile(); cache != nil { _, err := cache.Seek(0, io.SeekStart) if err != nil { return nil, err } if writer == nil { return cache, nil } reader := f.Reader if up != nil { cacheProgress := model.UpdateProgressWithRange(*up, 0, 50) *up = model.UpdateProgressWithRange(*up, 50, 100) reader = &ReaderUpdatingProgress{ Reader: &SimpleReaderWithSize{ Reader: reader, Size: f.GetSize(), }, UpdateProgress: cacheProgress, } } _, err = utils.CopyWithBuffer(writer, reader) if err == nil { _, err = cache.Seek(0, io.SeekStart) } if err != nil { return nil, err } return cache, nil } reader := f.Reader if f.peekBuff != nil { f.peekBuff.Seek(0, io.SeekStart) if writer != nil { _, err := utils.CopyWithBuffer(writer, f.peekBuff) if err != nil { return nil, err } f.peekBuff.Seek(0, io.SeekStart) } reader = f.oriReader } if writer != nil { reader = io.TeeReader(reader, writer) } if f.GetSize() < 0 { if f.peekBuff == nil { f.peekBuff = &buffer.Reader{} } // 检查是否有数据 buf := []byte{0} n, err := io.ReadFull(reader, buf) if n > 0 { f.peekBuff.Append(buf[:n]) } if err == io.ErrUnexpectedEOF { f.size = f.peekBuff.Size() f.Reader = f.peekBuff return f.peekBuff, nil } else if err != nil { return nil, err } if conf.MaxBufferLimit-n > conf.MmapThreshold && conf.MmapThreshold > 0 { m, err := mmap.Alloc(conf.MaxBufferLimit - n) if err == nil { f.Add(utils.CloseFunc(func() error { return mmap.Free(m) })) n, err = io.ReadFull(reader, m) if n > 0 { f.peekBuff.Append(m[:n]) } if err == io.ErrUnexpectedEOF { f.size = f.peekBuff.Size() f.Reader = f.peekBuff return f.peekBuff, nil } else if err != nil { return nil, err } } } tmpF, err := utils.CreateTempFile(reader, 0) if err != nil { return nil, err } f.Add(utils.CloseFunc(func() error { return errors.Join(tmpF.Close(), os.RemoveAll(tmpF.Name())) })) peekF, err := buffer.NewPeekFile(f.peekBuff, tmpF) if err != nil { return nil, err } f.size = peekF.Size() f.Reader = peekF return peekF, nil } if up != nil { cacheProgress := model.UpdateProgressWithRange(*up, 0, 50) *up = model.UpdateProgressWithRange(*up, 50, 100) size := f.GetSize() if f.peekBuff != nil { peekSize := f.peekBuff.Size() cacheProgress(float64(peekSize) / float64(size) * 100) size -= peekSize } reader = &ReaderUpdatingProgress{ Reader: &SimpleReaderWithSize{ Reader: reader, Size: size, }, UpdateProgress: cacheProgress, } } if f.peekBuff != nil { f.oriReader = reader } else { f.Reader = reader } return f.cache(f.GetSize()) } func (f *FileStream) GetFile() model.File { if file, ok := f.Reader.(model.File); ok { return file } return nil } // 从流读取指定范围的一块数据,并且不消耗流。 // 当读取的边界超过内部设置大小后会缓存整个流。 // 流未缓存时线程不完全 func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { if httpRange.Length < 0 || httpRange.Start+httpRange.Length > f.GetSize() { httpRange.Length = f.GetSize() - httpRange.Start } if f.GetFile() != nil { return io.NewSectionReader(f.GetFile(), httpRange.Start, httpRange.Length), nil } cache, err := f.cache(httpRange.Start + httpRange.Length) if err != nil { return nil, err } return io.NewSectionReader(cache, httpRange.Start, httpRange.Length), nil } // *旧笔记 // 使用bytes.Buffer作为io.CopyBuffer的写入对象，CopyBuffer会调用Buffer.ReadFrom // 即使被写入的数据量与Buffer.Cap一致，Buffer也会扩大 // 确保指定大小的数据被缓存 func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { if maxCacheSize > int64(conf.MaxBufferLimit) { size := f.GetSize() reader := f.Reader if f.peekBuff != nil { size -= f.peekBuff.Size() reader = f.oriReader } tmpF, err := utils.CreateTempFile(reader, size) if err != nil { return nil, err } f.Add(utils.CloseFunc(func() error { return errors.Join(tmpF.Close(), os.RemoveAll(tmpF.Name())) })) if f.peekBuff != nil { peekF, err := buffer.NewPeekFile(f.peekBuff, tmpF) if err != nil { return nil, err } f.Reader = peekF return peekF, nil } f.Reader = tmpF return tmpF, nil } if f.peekBuff == nil { f.peekBuff = &buffer.Reader{} f.oriReader = f.Reader f.Reader = io.MultiReader(f.peekBuff, f.oriReader) } bufSize := maxCacheSize - f.peekBuff.Size() if bufSize <= 0 { return f.peekBuff, nil } var buf []byte if conf.MmapThreshold > 0 && bufSize >= int64(conf.MmapThreshold) { m, err := mmap.Alloc(int(bufSize)) if err == nil { f.Add(utils.CloseFunc(func() error { return mmap.Free(m) })) buf = m } } if buf == nil { buf = make([]byte, bufSize) } n, err := io.ReadFull(f.oriReader, buf) if bufSize != int64(n) { return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", bufSize, n, err) } f.peekBuff.Append(buf) if f.peekBuff.Size() >= f.GetSize() { f.Reader = f.peekBuff } return f.peekBuff, nil } var _ model.FileStreamer = (*SeekableStream)(nil) var _ model.FileStreamer = (*FileStream)(nil) type SeekableStream struct { *FileStream // should have one of belows to support rangeRead rangeReader model.RangeReaderIF } // NewSeekableStream create a SeekableStream from FileStream and Link // if FileStream.Reader is not nil, use it directly // else create RangeReader from Link func NewSeekableStream(fs *FileStream, link *model.Link) (*SeekableStream, error) { if len(fs.Mimetype) == 0 { fs.Mimetype = utils.GetMimeType(fs.Obj.GetName()) } if fs.Reader != nil { fs.Add(link) return &SeekableStream{FileStream: fs}, nil } if link != nil { size := link.ContentLength if size <= 0 { size = fs.GetSize() } rr, err := GetRangeReaderFromLink(size, link) if err != nil { return nil, err } if _, ok := rr.(*model.FileRangeReader); ok { var rc io.ReadCloser rc, err = rr.RangeRead(fs.Ctx, http_range.Range{Length: -1}) if err != nil { return nil, err } fs.Reader = rc fs.Add(rc) } fs.size = size fs.Add(link) return &SeekableStream{FileStream: fs, rangeReader: rr}, nil } return nil, fmt.Errorf("illegal seekableStream") } // 如果使用缓存或者rangeReader读取指定范围的数据，是线程安全的 // 其他特性继承自FileStream.RangeRead func (ss *SeekableStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { if ss.GetFile() == nil && ss.rangeReader != nil { rc, err := ss.rangeReader.RangeRead(ss.Ctx, httpRange) if err != nil { return nil, err } ss.Add(rc) return rc, nil } return ss.FileStream.RangeRead(httpRange) } // only provide Reader as full stream when it's demanded. in rapid-upload, we can skip this to save memory func (ss *SeekableStream) Read(p []byte) (n int, err error) { if err := ss.generateReader(); err != nil { return 0, err } return ss.FileStream.Read(p) } func (ss *SeekableStream) generateReader() error { if ss.Reader == nil { if ss.rangeReader == nil { return fmt.Errorf("illegal seekableStream") } rc, err := ss.rangeReader.RangeRead(ss.Ctx, http_range.Range{Length: -1}) if err != nil { return err } ss.Add(rc) ss.Reader = rc } return nil } func (ss *SeekableStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writer) (model.File, error) { if err := ss.generateReader(); err != nil { return nil, err } return ss.FileStream.CacheFullAndWriter(up, writer) } type ReaderWithSize interface { io.Reader GetSize() int64 } type SimpleReaderWithSize struct { io.Reader Size int64 } func (r *SimpleReaderWithSize) GetSize() int64 { return r.Size } func (r *SimpleReaderWithSize) Close() error { if c, ok := r.Reader.(io.Closer); ok { return c.Close() } return nil } type ReaderUpdatingProgress struct { Reader ReaderWithSize model.UpdateProgress offset int } func (r *ReaderUpdatingProgress) Read(p []byte) (n int, err error) { n, err = r.Reader.Read(p) r.offset += n r.UpdateProgress(math.Min(100.0, float64(r.offset)/float64(r.Reader.GetSize())*100.0)) return n, err } func (r *ReaderUpdatingProgress) Close() error { if c, ok := r.Reader.(io.Closer); ok { return c.Close() } return nil } type RangeReadReadAtSeeker struct { ss *SeekableStream masterOff int64 readerMap sync.Map headCache *headCache } type headCache struct { reader io.Reader bufs [][]byte } func (c *headCache) head(p []byte) (int, error) { n := 0 for _, buf := range c.bufs { n += copy(p[n:], buf) if n == len(p) { return n, nil } } nn, err := io.ReadFull(c.reader, p[n:]) if nn > 0 { buf := make([]byte, nn) copy(buf, p[n:]) c.bufs = append(c.bufs, buf) n += nn if err == io.ErrUnexpectedEOF { err = io.EOF } } return n, err } func (r *headCache) Close() error { clear(r.bufs) r.bufs = nil return nil } func (r *RangeReadReadAtSeeker) InitHeadCache() { if r.masterOff == 0 { value, _ := r.readerMap.LoadAndDelete(int64(0)) r.headCache = &headCache{reader: value.(io.Reader)} r.ss.Closers.Add(r.headCache) } } func NewReadAtSeeker(ss *SeekableStream, offset int64, forceRange ...bool) (model.File, error) { if cache := ss.GetFile(); cache != nil { _, err := cache.Seek(offset, io.SeekStart) if err != nil { return nil, err } return cache, nil } r := &RangeReadReadAtSeeker{ ss: ss, masterOff: offset, } if offset != 0 || utils.IsBool(forceRange...) { if offset < 0 || offset > ss.GetSize() { return nil, errors.New("offset out of range") } reader, err := r.getReaderAtOffset(offset) if err != nil { return nil, err } r.readerMap.Store(int64(offset), reader) } else { r.readerMap.Store(int64(offset), ss) } return r, nil } func NewMultiReaderAt(ss []*SeekableStream) (readerutil.SizeReaderAt, error) { readers := make([]readerutil.SizeReaderAt, 0, len(ss)) for _, s := range ss { ra, err := NewReadAtSeeker(s, 0) if err != nil { return nil, err } readers = append(readers, io.NewSectionReader(ra, 0, s.GetSize())) } return readerutil.NewMultiReaderAt(readers...), nil } func (r *RangeReadReadAtSeeker) getReaderAtOffset(off int64) (io.Reader, error) { for { var cur int64 = -1 r.readerMap.Range(func(key, value any) bool { k := key.(int64) if off == k { cur = k return false } if off > k && off-k <= 4*utils.MB && k > cur { cur = k } return true }) if cur < 0 { break } v, ok := r.readerMap.LoadAndDelete(int64(cur)) if !ok { continue } rr := v.(io.Reader) if off == int64(cur) { // logrus.Debugf("getReaderAtOffset match_%d", off) return rr, nil } n, _ := utils.CopyWithBufferN(io.Discard, rr, off-cur) cur += n if cur == off { // logrus.Debugf("getReaderAtOffset old_%d", off) return rr, nil } break } // logrus.Debugf("getReaderAtOffset new_%d", off) reader, err := r.ss.RangeRead(http_range.Range{Start: off, Length: -1}) if err != nil { return nil, err } return reader, nil } func (r *RangeReadReadAtSeeker) ReadAt(p []byte, off int64) (n int, err error) { if off < 0 || off >= r.ss.GetSize() { return 0, io.EOF } if off == 0 && r.headCache != nil { return r.headCache.head(p) } var rr io.Reader rr, err = r.getReaderAtOffset(off) if err != nil { return 0, err } n, err = io.ReadFull(rr, p) if n > 0 { off += int64(n) switch err { case nil: r.readerMap.Store(int64(off), rr) case io.ErrUnexpectedEOF: err = io.EOF } } return n, err } func (r *RangeReadReadAtSeeker) Seek(offset int64, whence int) (int64, error) { switch whence { case io.SeekStart: case io.SeekCurrent: offset += r.masterOff case io.SeekEnd: offset += r.ss.GetSize() default: return 0, errors.New("Seek: invalid whence") } if offset < 0 || offset > r.ss.GetSize() { return 0, errors.New("Seek: invalid offset") } r.masterOff = offset return offset, nil } func (r *RangeReadReadAtSeeker) Read(p []byte) (n int, err error) { n, err = r.ReadAt(p, r.masterOff) if n > 0 { r.masterOff += int64(n) } return n, err } ================================================ FILE: internal/stream/stream_test.go ================================================ package stream import ( "bytes" "errors" "fmt" "io" "testing" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) func TestFileStream_RangeRead(t *testing.T) { type args struct { httpRange http_range.Range } buf := []byte("github.com/OpenListTeam/OpenList") f := &FileStream{ Obj: &model.Object{ Size: int64(len(buf)), }, Reader: io.NopCloser(bytes.NewReader(buf)), } tests := []struct { name string f *FileStream args args want func(f *FileStream, got io.Reader, err error) error }{ { name: "range 11-12", f: f, args: args{ httpRange: http_range.Range{Start: 11, Length: 12}, }, want: func(f *FileStream, got io.Reader, err error) error { if f.GetFile() != nil { return errors.New("cached") } b, _ := io.ReadAll(got) if !bytes.Equal(buf[11:11+12], b) { return fmt.Errorf("=%s ,want =%s", b, buf[11:11+12]) } return nil }, }, { name: "range 11-21", f: f, args: args{ httpRange: http_range.Range{Start: 11, Length: 21}, }, want: func(f *FileStream, got io.Reader, err error) error { if f.GetFile() == nil { return errors.New("not cached") } b, _ := io.ReadAll(got) if !bytes.Equal(buf[11:11+21], b) { return fmt.Errorf("=%s ,want =%s", b, buf[11:11+21]) } return nil }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := tt.f.RangeRead(tt.args.httpRange) if err := tt.want(tt.f, got, err); err != nil { t.Errorf("FileStream.RangeRead() %v", err) } }) } if f.GetFile() == nil { t.Error("not cached") } buf2 := make([]byte, len(buf)) if _, err := io.ReadFull(f, buf2); err != nil { t.Errorf("FileStream.Read() error = %v", err) } if !bytes.Equal(buf, buf2) { t.Errorf("FileStream.Read() = %s, want %s", buf2, buf) } } func TestFileStream_With_PreHash(t *testing.T) { buf := []byte("github.com/OpenListTeam/OpenList") f := &FileStream{ Obj: &model.Object{ Size: int64(len(buf)), }, Reader: io.NopCloser(bytes.NewReader(buf)), } const hashSize int64 = 20 reader, _ := f.RangeRead(http_range.Range{Start: 0, Length: hashSize}) preHash, _ := utils.HashReader(utils.SHA1, reader) if preHash == "" { t.Error("preHash is empty") } tmpF, fullHash, _ := CacheFullAndHash(f, nil, utils.SHA1) fmt.Println(fullHash) fileFullHash, _ := utils.HashFile(utils.SHA1, tmpF) fmt.Println(fileFullHash) if fullHash != fileFullHash { t.Errorf("fullHash and fileFullHash should match: fullHash=%s fileFullHash=%s", fullHash, fileFullHash) } } ================================================ FILE: internal/stream/util.go ================================================ package stream import ( "bytes" "context" "encoding/hex" "errors" "fmt" "io" "net/http" "os" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/pool" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/rclone/rclone/lib/mmap" log "github.com/sirupsen/logrus" ) type RangeReaderFunc func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) func (f RangeReaderFunc) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { return f(ctx, httpRange) } func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, error) { if link.RangeReader != nil { if link.Concurrency < 1 && link.PartSize < 1 { return link.RangeReader, nil } down := net.NewDownloader(func(d *net.Downloader) { d.Concurrency = link.Concurrency d.PartSize = link.PartSize d.HttpClient = net.GetRangeReaderHttpRequestFunc(link.RangeReader) }) rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { return down.Download(ctx, &net.HttpRequestParams{ Range: httpRange, Size: size, }) } // RangeReader只能在驱动限速 return RangeReaderFunc(rangeReader), nil } if len(link.URL) == 0 { return nil, errors.New("invalid link: must have at least one of URL or RangeReader") } if link.Concurrency > 0 || link.PartSize > 0 { down := net.NewDownloader(func(d *net.Downloader) { d.Concurrency = link.Concurrency d.PartSize = link.PartSize d.HttpClient = func(ctx context.Context, params *net.HttpRequestParams) (*http.Response, error) { if ServerDownloadLimit == nil { return net.DefaultHttpRequestFunc(ctx, params) } resp, err := net.DefaultHttpRequestFunc(ctx, params) if err == nil && resp.Body != nil { resp.Body = &RateLimitReader{ Ctx: ctx, Reader: resp.Body, Limiter: ServerDownloadLimit, } } return resp, err } }) rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { requestHeader, _ := ctx.Value(conf.RequestHeaderKey).(http.Header) header := net.ProcessHeader(requestHeader, link.Header) return down.Download(ctx, &net.HttpRequestParams{ Range: httpRange, Size: size, URL: link.URL, HeaderRef: header, }) } return RangeReaderFunc(rangeReader), nil } rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { if httpRange.Length < 0 || httpRange.Start+httpRange.Length > size { httpRange.Length = size - httpRange.Start } requestHeader, _ := ctx.Value(conf.RequestHeaderKey).(http.Header) header := net.ProcessHeader(requestHeader, link.Header) header = http_range.ApplyRangeToHttpHeader(httpRange, header) response, err := net.RequestHttp(ctx, "GET", header, link.URL) if err != nil { if _, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { return nil, err } return nil, fmt.Errorf("http request failure, err:%w", err) } if ServerDownloadLimit != nil { response.Body = &RateLimitReader{ Ctx: ctx, Reader: response.Body, Limiter: ServerDownloadLimit, } } if httpRange.Start == 0 && httpRange.Length == size || response.StatusCode == http.StatusPartialContent || checkContentRange(&response.Header, httpRange.Start) { return response.Body, nil } else if response.StatusCode == http.StatusOK { log.Warnf("remote http server not supporting range request, expect low perfromace!") readCloser, err := net.GetRangedHttpReader(response.Body, httpRange.Start, httpRange.Length) if err != nil { return nil, err } return readCloser, nil } return response.Body, nil } return RangeReaderFunc(rangeReader), nil } func GetRangeReaderFromMFile(size int64, file model.File) *model.FileRangeReader { return &model.FileRangeReader{ RangeReaderIF: RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { length := httpRange.Length if length < 0 || httpRange.Start+length > size { length = size - httpRange.Start } return &model.FileCloser{File: io.NewSectionReader(file, httpRange.Start, length)}, nil }), } } // 139 cloud does not properly return 206 http status code, add a hack here func checkContentRange(header *http.Header, offset int64) bool { start, _, err := http_range.ParseContentRange(header.Get("Content-Range")) if err != nil { log.Warnf("exception trying to parse Content-Range, will ignore,err=%s", err) } if start == offset { return true } return false } type ReaderWithCtx struct { io.Reader Ctx context.Context } func (r *ReaderWithCtx) Read(p []byte) (n int, err error) { if utils.IsCanceled(r.Ctx) { return 0, r.Ctx.Err() } return r.Reader.Read(p) } func (r *ReaderWithCtx) Close() error { if c, ok := r.Reader.(io.Closer); ok { return c.Close() } return nil } func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashType *utils.HashType, hashParams ...any) (model.File, string, error) { h := hashType.NewFunc(hashParams...) tmpF, err := stream.CacheFullAndWriter(up, h) if err != nil { return nil, "", err } return tmpF, hex.EncodeToString(h.Sum(nil)), nil } type StreamSectionReaderIF interface { // 线程不安全 GetSectionReader(off, length int64) (io.ReadSeeker, error) FreeSectionReader(sr io.ReadSeeker) // 线程不安全 DiscardSection(off int64, length int64) error } func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *model.UpdateProgress) (StreamSectionReaderIF, error) { if file.GetFile() != nil { return &cachedSectionReader{file.GetFile()}, nil } maxBufferSize = min(maxBufferSize, int(file.GetSize())) if maxBufferSize > conf.MaxBufferLimit { f, err := os.CreateTemp(conf.Conf.TempDir, "file-*") if err != nil { return nil, err } if f.Truncate(file.GetSize()) != nil { // fallback to full cache _, _ = f.Close(), os.Remove(f.Name()) cache, err := file.CacheFullAndWriter(up, nil) if err != nil { return nil, err } return &cachedSectionReader{cache}, nil } ss := &fileSectionReader{file: file, temp: f} ss.bufPool = &pool.Pool[*offsetWriterWithBase]{ New: func() *offsetWriterWithBase { base := ss.tempOffset ss.tempOffset += int64(maxBufferSize) return &offsetWriterWithBase{io.NewOffsetWriter(ss.temp, base), base} }, } file.Add(utils.CloseFunc(func() error { ss.bufPool.Reset() return errors.Join(ss.temp.Close(), os.Remove(ss.temp.Name())) })) return ss, nil } ss := &directSectionReader{file: file} if conf.MmapThreshold > 0 && maxBufferSize >= conf.MmapThreshold { ss.bufPool = &pool.Pool[[]byte]{ New: func() []byte { buf, err := mmap.Alloc(maxBufferSize) if err == nil { file.Add(utils.CloseFunc(func() error { return mmap.Free(buf) })) } else { buf = make([]byte, maxBufferSize) } return buf }, } } else { ss.bufPool = &pool.Pool[[]byte]{ New: func() []byte { return make([]byte, maxBufferSize) }, } } file.Add(utils.CloseFunc(func() error { ss.bufPool.Reset() return nil })) return ss, nil } type cachedSectionReader struct { cache io.ReaderAt } func (*cachedSectionReader) DiscardSection(off int64, length int64) error { return nil } func (s *cachedSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { return io.NewSectionReader(s.cache, off, length), nil } func (*cachedSectionReader) FreeSectionReader(sr io.ReadSeeker) {} type fileSectionReader struct { file model.FileStreamer fileOffset int64 temp *os.File tempOffset int64 bufPool *pool.Pool[*offsetWriterWithBase] } type offsetWriterWithBase struct { *io.OffsetWriter base int64 } // 线程不安全 func (ss *fileSectionReader) DiscardSection(off int64, length int64) error { if off != ss.fileOffset { return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } n, err := utils.CopyWithBufferN(io.Discard, ss.file, length) ss.fileOffset += n if err != nil { return fmt.Errorf("failed to skip data: (expect =%d, actual =%d) %w", length, n, err) } return nil } type fileBufferSectionReader struct { io.ReadSeeker fileBuf *offsetWriterWithBase } // 线程不安全 func (ss *fileSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { if off != ss.fileOffset { return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } fileBuf := ss.bufPool.Get() _, _ = fileBuf.Seek(0, io.SeekStart) n, err := utils.CopyWithBufferN(fileBuf, ss.file, length) ss.fileOffset += n if err != nil { return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) } return &fileBufferSectionReader{io.NewSectionReader(ss.temp, fileBuf.base, length), fileBuf}, nil } func (ss *fileSectionReader) FreeSectionReader(rs io.ReadSeeker) { if sr, ok := rs.(*fileBufferSectionReader); ok { ss.bufPool.Put(sr.fileBuf) sr.fileBuf = nil sr.ReadSeeker = nil } } type directSectionReader struct { file model.FileStreamer fileOffset int64 bufPool *pool.Pool[[]byte] } // 线程不安全 func (ss *directSectionReader) DiscardSection(off int64, length int64) error { if off != ss.fileOffset { return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } n, err := utils.CopyWithBufferN(io.Discard, ss.file, length) ss.fileOffset += n if err != nil { return fmt.Errorf("failed to skip data: (expect =%d, actual =%d) %w", length, n, err) } return nil } type bufferSectionReader struct { io.ReadSeeker buf []byte } // 线程不安全 func (ss *directSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { if off != ss.fileOffset { return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } tempBuf := ss.bufPool.Get() buf := tempBuf[:length] n, err := io.ReadFull(ss.file, buf) ss.fileOffset += int64(n) if int64(n) != length { return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) } return &bufferSectionReader{bytes.NewReader(buf), buf}, nil } func (ss *directSectionReader) FreeSectionReader(rs io.ReadSeeker) { if sr, ok := rs.(*bufferSectionReader); ok { ss.bufPool.Put(sr.buf[0:cap(sr.buf)]) sr.buf = nil sr.ReadSeeker = nil } } ================================================ FILE: internal/task/base.go ================================================ package task import ( "context" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/tache" ) type TaskExtension struct { tache.Base Creator *model.User startTime *time.Time endTime *time.Time TotalBytes int64 ApiUrl string } func (t *TaskExtension) SetCtx(ctx context.Context) { if t.Creator != nil { ctx = context.WithValue(ctx, conf.UserKey, t.Creator) } if len(t.ApiUrl) > 0 { ctx = context.WithValue(ctx, conf.ApiUrlKey, t.ApiUrl) } t.Base.SetCtx(ctx) } func (t *TaskExtension) SetCreator(creator *model.User) { t.Creator = creator t.Persist() } func (t *TaskExtension) GetCreator() *model.User { return t.Creator } func (t *TaskExtension) SetStartTime(startTime time.Time) { t.startTime = &startTime } func (t *TaskExtension) GetStartTime() *time.Time { return t.startTime } func (t *TaskExtension) SetEndTime(endTime time.Time) { t.endTime = &endTime } func (t *TaskExtension) GetEndTime() *time.Time { return t.endTime } func (t *TaskExtension) ClearEndTime() { t.endTime = nil } func (t *TaskExtension) SetTotalBytes(totalBytes int64) { t.TotalBytes = totalBytes } func (t *TaskExtension) GetTotalBytes() int64 { return t.TotalBytes } func (t *TaskExtension) SetRetry(retry int, maxRetry int) { t.Base.SetRetry(retry, maxRetry) if retry > 0 || !conf.Conf.Tasks.AllowRetryCanceled || t.Ctx() == nil { return } select { case <-t.Ctx().Done(): ctx, cancel := context.WithCancel(context.Background()) t.SetCtx(ctx) t.SetCancelFunc(cancel) default: } } type TaskExtensionInfo interface { tache.TaskWithInfo GetCreator() *model.User GetStartTime() *time.Time GetEndTime() *time.Time GetTotalBytes() int64 } ================================================ FILE: internal/task/manager.go ================================================ package task import ( "github.com/OpenListTeam/tache" ) type Manager[T tache.Task] interface { Add(task T) Cancel(id string) CancelAll() CancelByCondition(condition func(task T) bool) GetAll() []T GetByID(id string) (T, bool) GetByState(state ...tache.State) []T GetByCondition(condition func(task T) bool) []T Remove(id string) RemoveAll() RemoveByState(state ...tache.State) RemoveByCondition(condition func(task T) bool) Retry(id string) RetryAllFailed() } ================================================ FILE: internal/task_group/group.go ================================================ package task_group import ( "context" "sync" "github.com/sirupsen/logrus" ) type OnCompletionFunc func(ctx context.Context, groupID string, payloads ...any) type TaskGroupCoordinator struct { name string mu sync.Mutex groupPayloads map[string][]any groupStates map[string]groupState onCompletion OnCompletionFunc } type groupState struct { pending int hasSuccess bool } func NewTaskGroupCoordinator(name string, f OnCompletionFunc) *TaskGroupCoordinator { return &TaskGroupCoordinator{ name: name, groupPayloads: map[string][]any{}, groupStates: map[string]groupState{}, onCompletion: f, } } // payload可为nil func (tgc *TaskGroupCoordinator) AddTask(groupID string, payload any) { tgc.mu.Lock() defer tgc.mu.Unlock() state := tgc.groupStates[groupID] state.pending++ tgc.groupStates[groupID] = state logrus.Debugf("AddTask:%s ,count=%+v", groupID, state) if payload == nil { return } tgc.groupPayloads[groupID] = append(tgc.groupPayloads[groupID], payload) } func (tgc *TaskGroupCoordinator) AppendPayload(groupID string, payload any) { if payload == nil { return } tgc.mu.Lock() defer tgc.mu.Unlock() tgc.groupPayloads[groupID] = append(tgc.groupPayloads[groupID], payload) } func (tgc *TaskGroupCoordinator) Done(ctx context.Context, groupID string, success bool) { tgc.mu.Lock() defer tgc.mu.Unlock() state, ok := tgc.groupStates[groupID] if !ok || state.pending == 0 { return } if success { state.hasSuccess = true } logrus.Debugf("Done:%s ,state=%+v", groupID, state) if state.pending == 1 { payloads := tgc.groupPayloads[groupID] delete(tgc.groupStates, groupID) delete(tgc.groupPayloads, groupID) if tgc.onCompletion != nil && state.hasSuccess { logrus.Debugf("OnCompletion:%s", groupID) tgc.mu.Unlock() tgc.onCompletion(ctx, groupID, payloads...) tgc.mu.Lock() } return } state.pending-- tgc.groupStates[groupID] = state } ================================================ FILE: internal/task_group/transfer.go ================================================ package task_group import ( "context" "fmt" "path" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) type SrcPathToRemove string // ActualPath type DstPathToHook string func HookAndRemove(ctx context.Context, dstPath string, payloads ...any) { dstStorage, dstActualPath, err := op.GetStorageAndActualPath(dstPath) if err != nil { log.Error(errors.WithMessage(err, "failed get dst storage")) return } dstNeedHandleHook := setting.GetBool(conf.HandleHookAfterWriting) dstHandleHookLimit := setting.GetFloat(conf.HandleHookRateLimit, .0) var listLimiter *rate.Limiter if dstNeedHandleHook && dstHandleHookLimit > .0 { listLimiter = rate.NewLimiter(rate.Limit(dstHandleHookLimit), 1) } hookedPaths := make(map[string]struct{}) handleHook := func(actualPath string) { if _, ok := hookedPaths[actualPath]; ok { return } if listLimiter != nil { _ = listLimiter.Wait(ctx) } files, e := op.List(ctx, dstStorage, actualPath, model.ListArgs{SkipHook: true}) if e != nil { log.Errorf("failed handle objs update hook: %v", e) } else { op.HandleObjsUpdateHook(ctx, utils.GetFullPath(dstStorage.GetStorage().MountPath, actualPath), files) hookedPaths[actualPath] = struct{}{} } } if dstNeedHandleHook { handleHook(dstActualPath) } for _, payload := range payloads { switch p := payload.(type) { case DstPathToHook: if dstNeedHandleHook { handleHook(string(p)) } case SrcPathToRemove: srcStorage, srcActualPath, err := op.GetStorageAndActualPath(string(p)) if err != nil { log.Error(errors.WithMessage(err, "failed get src storage")) continue } err = verifyAndRemove(ctx, srcStorage, dstStorage, srcActualPath, dstActualPath) if err != nil { log.Error(err) } } } } func verifyAndRemove(ctx context.Context, srcStorage, dstStorage driver.Driver, srcPath, dstPath string) error { srcObj, err := op.GetUnwrap(ctx, srcStorage, srcPath) if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", path.Join(srcStorage.GetStorage().MountPath, srcPath)) } dstObjPath := path.Join(dstPath, srcObj.GetName()) dstObj, err := op.GetUnwrap(ctx, dstStorage, dstObjPath) if err != nil { return errors.WithMessagef(err, "failed get dst [%s] file", path.Join(dstStorage.GetStorage().MountPath, dstObjPath)) } if !dstObj.IsDir() { err = op.Remove(ctx, srcStorage, srcPath) if err != nil { return fmt.Errorf("failed remove %s: %+v", path.Join(srcStorage.GetStorage().MountPath, srcPath), err) } return nil } // Verify directory srcObjs, err := op.List(ctx, srcStorage, srcPath, model.ListArgs{}) if err != nil { return errors.WithMessagef(err, "failed list src [%s] objs", path.Join(srcStorage.GetStorage().MountPath, srcPath)) } hasErr := false for _, obj := range srcObjs { srcSubPath := path.Join(srcPath, obj.GetName()) err := verifyAndRemove(ctx, srcStorage, dstStorage, srcSubPath, dstObjPath) if err != nil { log.Error(err) hasErr = true } } if hasErr { return errors.Errorf("some subitems of [%s] failed to verify and remove", path.Join(srcStorage.GetStorage().MountPath, srcPath)) } err = op.Remove(ctx, srcStorage, srcPath) if err != nil { return fmt.Errorf("failed remove %s: %+v", path.Join(srcStorage.GetStorage().MountPath, srcPath), err) } return nil } var TransferCoordinator *TaskGroupCoordinator = NewTaskGroupCoordinator("HookAndRemove", HookAndRemove) ================================================ FILE: main.go ================================================ package main import "github.com/OpenListTeam/OpenList/v4/cmd" func main() { cmd.Execute() } ================================================ FILE: pkg/aria2/rpc/README.md ================================================ # PACKAGE DOCUMENTATION **package rpc** import "github.com/matzoe/argo/rpc" ## FUNCTIONS ``` func Call(address, method string, params, reply interface{}) error ``` ## TYPES ``` type Client struct { // contains filtered or unexported fields } ``` ``` func New(uri string) *Client ``` ``` func (id *Client) AddMetalink(uri string, options ...interface{}) (gid string, err error) ``` `aria2.addMetalink(metalink[, options[, position]])` This method adds Metalink download by uploading ".metalink" file. `metalink` is of type base64 which contains Base64-encoded ".metalink" file. `options` is of type struct and its members are a pair of option name and value. See Options below for more details. If `position` is given as an integer starting from 0, the new download is inserted at `position` in the waiting queue. If `position` is not given or `position` is larger than the size of the queue, it is appended at the end of the queue. This method returns array of GID of registered download. If `--rpc-save-upload-metadata` is true, the uploaded data is saved as a file named hex string of SHA-1 hash of data plus ".metalink" in the directory specified by `--dir` option. The example of filename is 0a3893293e27ac0490424c06de4d09242215f0a6.metalink. If same file already exists, it is overwritten. If the file cannot be saved successfully or `--rpc-save-upload-metadata` is false, the downloads added by this method are not saved by `--save-session`. ``` func (id *Client) AddTorrent(filename string, options ...interface{}) (gid string, err error) ``` `aria2.addTorrent(torrent[, uris[, options[, position]]])` This method adds BitTorrent download by uploading ".torrent" file. If you want to add BitTorrent Magnet URI, use `aria2.addUri()` method instead. torrent is of type base64 which contains Base64-encoded ".torrent" file. `uris` is of type array and its element is URI which is of type string. `uris` is used for Web-seeding. For single file torrents, URI can be a complete URI pointing to the resource or if URI ends with /, name in torrent file is added. For multi-file torrents, name and path in torrent are added to form a URI for each file. options is of type struct and its members are a pair of option name and value. See Options below for more details. If `position` is given as an integer starting from 0, the new download is inserted at `position` in the waiting queue. If `position` is not given or `position` is larger than the size of the queue, it is appended at the end of the queue. This method returns GID of registered download. If `--rpc-save-upload-metadata` is true, the uploaded data is saved as a file named hex string of SHA-1 hash of data plus ".torrent" in the directory specified by `--dir` option. The example of filename is 0a3893293e27ac0490424c06de4d09242215f0a6.torrent. If same file already exists, it is overwritten. If the file cannot be saved successfully or `--rpc-save-upload-metadata` is false, the downloads added by this method are not saved by -`-save-session`. ``` func (id *Client) AddUri(uri string, options ...interface{}) (gid string, err error) ``` `aria2.addUri(uris[, options[, position]])` This method adds new HTTP(S)/FTP/BitTorrent Magnet URI. `uris` is of type array and its element is URI which is of type string. For BitTorrent Magnet URI, `uris` must have only one element and it should be BitTorrent Magnet URI. URIs in uris must point to the same file. If you mix other URIs which point to another file, aria2 does not complain but download may fail. `options` is of type struct and its members are a pair of option name and value. See Options below for more details. If `position` is given as an integer starting from 0, the new download is inserted at position in the waiting queue. If `position` is not given or `position` is larger than the size of the queue, it is appended at the end of the queue. This method returns GID of registered download. ``` func (id *Client) ChangeGlobalOption(options map[string]interface{}) (g string, err error) ``` `aria2.changeGlobalOption(options)` This method changes global options dynamically. `options` is of type struct. The following `options` are available: download-result log log-level max-concurrent-downloads max-download-result max-overall-download-limit max-overall-upload-limit save-cookies save-session server-stat-of In addition to them, options listed in Input File subsection are available, except for following options: `checksum`, `index-out`, `out`, `pause` and `select-file`. Using `log` option, you can dynamically start logging or change log file. To stop logging, give empty string("") as a parameter value. Note that log file is always opened in append mode. This method returns OK for success. ``` func (id *Client) ChangeOption(gid string, options map[string]interface{}) (g string, err error) ``` `aria2.changeOption(gid, options)` This method changes options of the download denoted by `gid` dynamically. `gid` is of type string. `options` is of type struct. The following `options` are available for active downloads: bt-max-peers bt-request-peer-speed-limit bt-remove-unselected-file force-save max-download-limit max-upload-limit For waiting or paused downloads, in addition to the above options, options listed in Input File subsection are available, except for following options: dry-run, metalink-base-uri, parameterized-uri, pause, piece-length and rpc-save-upload-metadata option. This method returns OK for success. ``` func (id *Client) ChangePosition(gid string, pos int, how string) (p int, err error) ``` `aria2.changePosition(gid, pos, how)` This method changes the position of the download denoted by `gid`. `pos` is of type integer. `how` is of type string. If `how` is `POS_SET`, it moves the download to a position relative to the beginning of the queue. If `how` is `POS_CUR`, it moves the download to a position relative to the current position. If `how` is `POS_END`, it moves the download to a position relative to the end of the queue. If the destination position is less than 0 or beyond the end of the queue, it moves the download to the beginning or the end of the queue respectively. The response is of type integer and it is the destination position. ``` func (id *Client) ChangeUri(gid string, fileindex int, delUris []string, addUris []string, position ...int) (p []int, err error) ``` `aria2.changeUri(gid, fileIndex, delUris, addUris[, position])` This method removes URIs in `delUris` from and appends URIs in `addUris` to download denoted by gid. `delUris` and `addUris` are list of string. A download can contain multiple files and URIs are attached to each file. `fileIndex` is used to select which file to remove/attach given URIs. `fileIndex` is 1-based. `position` is used to specify where URIs are inserted in the existing waiting URI list. `position` is 0-based. When `position` is omitted, URIs are appended to the back of the list. This method first execute removal and then addition. `position` is the `position` after URIs are removed, not the `position` when this method is called. When removing URI, if same URIs exist in download, only one of them is removed for each URI in delUris. In other words, there are three URIs http://example.org/aria2 and you want remove them all, you have to specify (at least) 3 http://example.org/aria2 in delUris. This method returns a list which contains 2 integers. The first integer is the number of URIs deleted. The second integer is the number of URIs added. ``` func (id *Client) ForcePause(gid string) (g string, err error) ``` `aria2.forcePause(pid)` This method pauses the download denoted by `gid`. This method behaves just like aria2.pause() except that this method pauses download without any action which takes time such as contacting BitTorrent tracker. ``` func (id *Client) ForcePauseAll() (g string, err error) ``` `aria2.forcePauseAll()` This method is equal to calling `aria2.forcePause()` for every active/waiting download. This methods returns OK for success. ``` func (id *Client) ForceRemove(gid string) (g string, err error) ``` `aria2.forceRemove(gid)` This method removes the download denoted by `gid`. This method behaves just like aria2.remove() except that this method removes download without any action which takes time such as contacting BitTorrent tracker. ``` func (id *Client) ForceShutdown() (g string, err error) ``` `aria2.forceShutdown()` This method shutdowns aria2. This method behaves like `aria2.shutdown()` except that any actions which takes time such as contacting BitTorrent tracker are skipped. This method returns OK. ``` func (id *Client) GetFiles(gid string) (m map[string]interface{}, err error) ``` `aria2.getFiles(gid)` This method returns file list of the download denoted by `gid`. `gid` is of type string. ``` func (id *Client) GetGlobalOption() (m map[string]interface{}, err error) ``` `aria2.getGlobalOption()` This method returns global options. The response is of type struct. Its key is the name of option. The value type is string. Note that this method does not return options which have no default value and have not been set by the command-line options, configuration files or RPC methods. Because global options are used as a template for the options of newly added download, the response contains keys returned by `aria2.getOption()` method. ``` func (id *Client) GetGlobalStat() (m map[string]interface{}, err error) ``` `aria2.getGlobalStat()` This method returns global statistics such as overall download and upload speed. ``` func (id *Client) GetOption(gid string) (m map[string]interface{}, err error) ``` `aria2.getOption(gid)` This method returns options of the download denoted by `gid`. The response is of type struct. Its key is the name of option. The value type is string. Note that this method does not return options which have no default value and have not been set by the command-line options, configuration files or RPC methods. ``` func (id *Client) GetPeers(gid string) (m []map[string]interface{}, err error) ``` `aria2.getPeers(gid)` This method returns peer list of the download denoted by `gid`. `gid` is of type string. This method is for BitTorrent only. ``` func (id *Client) GetServers(gid string) (m []map[string]interface{}, err error) ``` `aria2.getServers(gid)` This method returns currently connected HTTP(S)/FTP servers of the download denoted by `gid`. `gid` is of type string. ``` func (id *Client) GetSessionInfo() (m map[string]interface{}, err error) ``` `aria2.getSessionInfo()` This method returns session information. ``` func (id *Client) GetUris(gid string) (m map[string]interface{}, err error) ``` `aria2.getUris(gid)` This method returns URIs used in the download denoted by `gid`. `gid` is of type string. ``` func (id *Client) GetVersion() (m map[string]interface{}, err error) ``` `aria2.getVersion()` This method returns version of the program and the list of enabled features. ``` func (id *Client) Multicall(methods []map[string]interface{}) (r []interface{}, err error) ``` `system.multicall(methods)` This method encapsulates multiple method calls in a single request. `methods` is of type array and its element is struct. The struct contains two keys: `methodName` and `params`. `methodName` is the method name to call and `params` is array containing parameters to the method. This method returns array of responses. The element of array will either be a one-item array containing the return value of each method call or struct of fault element if an encapsulated method call fails. ``` func (id *Client) Pause(gid string) (g string, err error) ``` `aria2.pause(gid)` This method pauses the download denoted by `gid`. `gid` is of type string. The status of paused download becomes paused. If the download is active, the download is placed on the first position of waiting queue. As long as the status is paused, the download is not started. To change status to waiting, use `aria2.unpause()` method. This method returns GID of paused download. ``` func (id *Client) PauseAll() (g string, err error) ``` `aria2.pauseAll()` This method is equal to calling `aria2.pause()` for every active/waiting download. This methods returns OK for success. ``` func (id *Client) PurgeDownloadResult() (g string, err error) ``` `aria2.purgeDownloadResult()` This method purges completed/error/removed downloads to free memory. This method returns OK. ``` func (id *Client) Remove(gid string) (g string, err error) ``` `aria2.remove(gid)` This method removes the download denoted by gid. `gid` is of type string. If specified download is in progress, it is stopped at first. The status of removed download becomes removed. This method returns GID of removed download. ``` func (id *Client) RemoveDownloadResult(gid string) (g string, err error) ``` `aria2.removeDownloadResult(gid)` This method removes completed/error/removed download denoted by `gid` from memory. This method returns OK for success. ``` func (id *Client) Shutdown() (g string, err error) ``` `aria2.shutdown()` This method shutdowns aria2. This method returns OK. ``` func (id *Client) TellActive(keys ...string) (m []map[string]interface{}, err error) ``` `aria2.tellActive([keys])` This method returns the list of active downloads. The response is of type array and its element is the same struct returned by `aria2.tellStatus()` method. For `keys` parameter, please refer to `aria2.tellStatus()` method. ``` func (id *Client) TellStatus(gid string, keys ...string) (m map[string]interface{}, err error) ``` `aria2.tellStatus(gid[, keys])` This method returns download progress of the download denoted by `gid`. `gid` is of type string. `keys` is array of string. If it is specified, the response contains only keys in `keys` array. If `keys` is empty or not specified, the response contains all keys. This is useful when you just want specific keys and avoid unnecessary transfers. For example, `aria2.tellStatus("2089b05ecca3d829", ["gid", "status"])` returns `gid` and `status` key. ``` func (id *Client) TellStopped(offset, num int, keys ...string) (m []map[string]interface{}, err error) ``` `aria2.tellStopped(offset, num[, keys])` This method returns the list of stopped download. `offset` is of type integer and specifies the `offset` from the oldest download. `num` is of type integer and specifies the number of downloads to be returned. For keys parameter, please refer to `aria2.tellStatus()` method. `offset` and `num` have the same semantics as `aria2.tellWaiting()` method. The response is of type array and its element is the same struct returned by `aria2.tellStatus()` method. ``` func (id *Client) TellWaiting(offset, num int, keys ...string) (m []map[string]interface{}, err error) ``` `aria2.tellWaiting(offset, num[, keys])` This method returns the list of waiting download, including paused downloads. `offset` is of type integer and specifies the `offset` from the download waiting at the front. num is of type integer and specifies the number of downloads to be returned. For keys parameter, please refer to aria2.tellStatus() method. If `offset` is a positive integer, this method returns downloads in the range of `[offset, offset + num)`. `offset` can be a negative integer. `offset == -1` points last download in the waiting queue and `offset == -2` points the download before the last download, and so on. The downloads in the response are in reversed order. For example, imagine that three downloads "A","B" and "C" are waiting in this order. aria2.tellWaiting(0, 1) returns ["A"]. aria2.tellWaiting(1, 2) returns ["B", "C"]. aria2.tellWaiting(-1, 2) returns ["C", "B"]. The response is of type array and its element is the same struct returned by `aria2.tellStatus()` method. ``` func (id *Client) Unpause(gid string) (g string, err error) ``` `aria2.unpause(gid)` This method changes the status of the download denoted by `gid` from paused to waiting. This makes the download eligible to restart. `gid` is of type string. This method returns GID of unpaused download. ``` func (id *Client) UnpauseAll() (g string, err error) ``` `aria2.unpauseAll()` This method is equal to calling `aria2.unpause()` for every active/waiting download. This methods returns OK for success. ================================================ FILE: pkg/aria2/rpc/call.go ================================================ package rpc import ( "context" "errors" "net" "net/http" "net/url" "sync" "sync/atomic" "time" "github.com/gorilla/websocket" log "github.com/sirupsen/logrus" ) type caller interface { // Call sends a request of rpc to aria2 daemon Call(method string, params, reply interface{}) (err error) Close() error } type httpCaller struct { uri string c *http.Client cancel context.CancelFunc wg *sync.WaitGroup once sync.Once } func newHTTPCaller(ctx context.Context, u *url.URL, timeout time.Duration, notifier Notifier) *httpCaller { c := &http.Client{ Transport: &http.Transport{ MaxIdleConnsPerHost: 1, MaxConnsPerHost: 1, // TLSClientConfig: tlsConfig, Dial: (&net.Dialer{ Timeout: timeout, KeepAlive: 60 * time.Second, }).Dial, TLSHandshakeTimeout: 3 * time.Second, ResponseHeaderTimeout: timeout, }, } var wg sync.WaitGroup ctx, cancel := context.WithCancel(ctx) h := &httpCaller{uri: u.String(), c: c, cancel: cancel, wg: &wg} if notifier != nil { h.setNotifier(ctx, *u, notifier) } return h } func (h *httpCaller) Close() (err error) { h.once.Do(func() { h.cancel() h.wg.Wait() }) return } func (h *httpCaller) setNotifier(ctx context.Context, u url.URL, notifier Notifier) (err error) { u.Scheme = "ws" conn, _, err := websocket.DefaultDialer.Dial(u.String(), nil) if err != nil { return } h.wg.Add(1) go func() { defer h.wg.Done() defer conn.Close() <-ctx.Done() conn.SetWriteDeadline(time.Now().Add(time.Second)) if err := conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")); err != nil { log.Printf("sending websocket close message: %v", err) } }() h.wg.Add(1) go func() { defer h.wg.Done() var request websocketResponse var err error for { select { case <-ctx.Done(): return default: } if err = conn.ReadJSON(&request); err != nil { select { case <-ctx.Done(): return default: } log.Printf("conn.ReadJSON|err:%v", err.Error()) return } switch request.Method { case "aria2.onDownloadStart": notifier.OnDownloadStart(request.Params) case "aria2.onDownloadPause": notifier.OnDownloadPause(request.Params) case "aria2.onDownloadStop": notifier.OnDownloadStop(request.Params) case "aria2.onDownloadComplete": notifier.OnDownloadComplete(request.Params) case "aria2.onDownloadError": notifier.OnDownloadError(request.Params) case "aria2.onBtDownloadComplete": notifier.OnBtDownloadComplete(request.Params) default: log.Printf("unexpected notification: %s", request.Method) } } }() return } func (h *httpCaller) Call(method string, params, reply interface{}) (err error) { payload, err := EncodeClientRequest(method, params) if err != nil { return } r, err := h.c.Post(h.uri, "application/json", payload) if err != nil { return } err = DecodeClientResponse(r.Body, &reply) r.Body.Close() return } type websocketCaller struct { conn *websocket.Conn sendChan chan *sendRequest cancel context.CancelFunc wg *sync.WaitGroup once sync.Once timeout time.Duration } func newWebsocketCaller(ctx context.Context, uri string, timeout time.Duration, notifier Notifier) (*websocketCaller, error) { var header = http.Header{} conn, _, err := websocket.DefaultDialer.Dial(uri, header) if err != nil { return nil, err } sendChan := make(chan *sendRequest, 16) var wg sync.WaitGroup ctx, cancel := context.WithCancel(ctx) w := &websocketCaller{conn: conn, wg: &wg, cancel: cancel, sendChan: sendChan, timeout: timeout} processor := NewResponseProcessor() wg.Add(1) go func() { // routine:recv defer wg.Done() defer cancel() for { select { case <-ctx.Done(): return default: } var resp websocketResponse if err := conn.ReadJSON(&resp); err != nil { select { case <-ctx.Done(): return default: } log.Printf("conn.ReadJSON|err:%v", err.Error()) return } if resp.Id == nil { // RPC notifications if notifier != nil { switch resp.Method { case "aria2.onDownloadStart": notifier.OnDownloadStart(resp.Params) case "aria2.onDownloadPause": notifier.OnDownloadPause(resp.Params) case "aria2.onDownloadStop": notifier.OnDownloadStop(resp.Params) case "aria2.onDownloadComplete": notifier.OnDownloadComplete(resp.Params) case "aria2.onDownloadError": notifier.OnDownloadError(resp.Params) case "aria2.onBtDownloadComplete": notifier.OnBtDownloadComplete(resp.Params) default: log.Printf("unexpected notification: %s", resp.Method) } } continue } processor.Process(resp.clientResponse) } }() wg.Add(1) go func() { // routine:send defer wg.Done() defer cancel() defer w.conn.Close() for { select { case <-ctx.Done(): if err := w.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")); err != nil { log.Printf("sending websocket close message: %v", err) } return case req := <-sendChan: processor.Add(req.request.Id, func(resp clientResponse) error { err := resp.decode(req.reply) req.cancel() return err }) w.conn.SetWriteDeadline(time.Now().Add(timeout)) w.conn.WriteJSON(req.request) } } }() return w, nil } func (w *websocketCaller) Close() (err error) { w.once.Do(func() { w.cancel() w.wg.Wait() }) return } func (w *websocketCaller) Call(method string, params, reply interface{}) (err error) { ctx, cancel := context.WithTimeout(context.Background(), w.timeout) defer cancel() select { case w.sendChan <- &sendRequest{cancel: cancel, request: &clientRequest{ Version: "2.0", Method: method, Params: params, Id: reqid(), }, reply: reply}: default: return errors.New("sending channel blocking") } <-ctx.Done() if err := ctx.Err(); err == context.DeadlineExceeded { return err } return } type sendRequest struct { cancel context.CancelFunc request *clientRequest reply interface{} } var reqid = func() func() uint64 { var id = uint64(time.Now().UnixNano()) return func() uint64 { return atomic.AddUint64(&id, 1) } }() ================================================ FILE: pkg/aria2/rpc/call_test.go ================================================ package rpc import ( "context" "testing" "time" ) func TestWebsocketCaller(t *testing.T) { time.Sleep(time.Second) c, err := newWebsocketCaller(context.Background(), "ws://localhost:6800/jsonrpc", time.Second, &DummyNotifier{}) if err != nil { t.Fatal(err.Error()) } defer c.Close() var info VersionInfo if err := c.Call(aria2GetVersion, []interface{}{}, &info); err != nil { t.Error(err.Error()) } else { println(info.Version) } } ================================================ FILE: pkg/aria2/rpc/client.go ================================================ package rpc import ( "context" "encoding/base64" "errors" "net/url" "os" "time" ) // Option is a container for specifying Call parameters and returning results type Option map[string]interface{} type Client interface { Protocol Close() error } type client struct { caller url *url.URL token string } var ( errInvalidParameter = errors.New("invalid parameter") errNotImplemented = errors.New("not implemented") errConnTimeout = errors.New("connect to aria2 daemon timeout") ) // New returns an instance of Client func New(ctx context.Context, uri string, token string, timeout time.Duration, notifier Notifier) (Client, error) { u, err := url.Parse(uri) if err != nil { return nil, err } var caller caller switch u.Scheme { case "http", "https": caller = newHTTPCaller(ctx, u, timeout, notifier) case "ws", "wss": caller, err = newWebsocketCaller(ctx, u.String(), timeout, notifier) if err != nil { return nil, err } default: return nil, errInvalidParameter } c := &client{caller: caller, url: u, token: token} return c, nil } // `aria2.addUri([secret, ]uris[, options[, position]])` // This method adds a new download. uris is an array of HTTP/FTP/SFTP/BitTorrent URIs (strings) pointing to the same resource. // If you mix URIs pointing to different resources, then the download may fail or be corrupted without aria2 complaining. // When adding BitTorrent Magnet URIs, uris must have only one element and it should be BitTorrent Magnet URI. // options is a struct and its members are pairs of option name and value. // If position is given, it must be an integer starting from 0. // The new download will be inserted at position in the waiting queue. // If position is omitted or position is larger than the current size of the queue, the new download is appended to the end of the queue. // This method returns the GID of the newly registered download. func (c *client) AddURI(uris []string, options ...interface{}) (gid string, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, uris) if options != nil { params = append(params, options...) } err = c.Call(aria2AddURI, params, &gid) return } // `aria2.addTorrent([secret, ]torrent[, uris[, options[, position]]])` // This method adds a BitTorrent download by uploading a ".torrent" file. // If you want to add a BitTorrent Magnet URI, use the aria2.addUri() method instead. // torrent must be a base64-encoded string containing the contents of the ".torrent" file. // uris is an array of URIs (string). uris is used for Web-seeding. // For single file torrents, the URI can be a complete URI pointing to the resource; if URI ends with /, name in torrent file is added. // For multi-file torrents, name and path in torrent are added to form a URI for each file. options is a struct and its members are pairs of option name and value. // If position is given, it must be an integer starting from 0. // The new download will be inserted at position in the waiting queue. // If position is omitted or position is larger than the current size of the queue, the new download is appended to the end of the queue. // This method returns the GID of the newly registered download. // If --rpc-save-upload-metadata is true, the uploaded data is saved as a file named as the hex string of SHA-1 hash of data plus ".torrent" in the directory specified by --dir option. // E.g. a file name might be 0a3893293e27ac0490424c06de4d09242215f0a6.torrent. // If a file with the same name already exists, it is overwritten! // If the file cannot be saved successfully or --rpc-save-upload-metadata is false, the downloads added by this method are not saved by --save-session. func (c *client) AddTorrent(filename string, options ...interface{}) (gid string, err error) { co, err := os.ReadFile(filename) if err != nil { return } file := base64.StdEncoding.EncodeToString(co) params := make([]interface{}, 0, 3) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, file) params = append(params, []interface{}{}) if options != nil { params = append(params, options...) } err = c.Call(aria2AddTorrent, params, &gid) return } // `aria2.addMetalink([secret, ]metalink[, options[, position]])` // This method adds a Metalink download by uploading a ".metalink" file. // metalink is a base64-encoded string which contains the contents of the ".metalink" file. // options is a struct and its members are pairs of option name and value. // If position is given, it must be an integer starting from 0. // The new download will be inserted at position in the waiting queue. // If position is omitted or position is larger than the current size of the queue, the new download is appended to the end of the queue. // This method returns an array of GIDs of newly registered downloads. // If --rpc-save-upload-metadata is true, the uploaded data is saved as a file named hex string of SHA-1 hash of data plus ".metalink" in the directory specified by --dir option. // E.g. a file name might be 0a3893293e27ac0490424c06de4d09242215f0a6.metalink. // If a file with the same name already exists, it is overwritten! // If the file cannot be saved successfully or --rpc-save-upload-metadata is false, the downloads added by this method are not saved by --save-session. func (c *client) AddMetalink(filename string, options ...interface{}) (gid []string, err error) { co, err := os.ReadFile(filename) if err != nil { return } file := base64.StdEncoding.EncodeToString(co) params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, file) if options != nil { params = append(params, options...) } err = c.Call(aria2AddMetalink, params, &gid) return } // `aria2.remove([secret, ]gid)` // This method removes the download denoted by gid (string). // If the specified download is in progress, it is first stopped. // The status of the removed download becomes removed. // This method returns GID of removed download. func (c *client) Remove(gid string) (g string, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) err = c.Call(aria2Remove, params, &g) return } // `aria2.forceRemove([secret, ]gid)` // This method removes the download denoted by gid. // This method behaves just like aria2.remove() except that this method removes the download without performing any actions which take time, such as contacting BitTorrent trackers to unregister the download first. func (c *client) ForceRemove(gid string) (g string, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) err = c.Call(aria2ForceRemove, params, &g) return } // `aria2.pause([secret, ]gid)` // This method pauses the download denoted by gid (string). // The status of paused download becomes paused. // If the download was active, the download is placed in the front of waiting queue. // While the status is paused, the download is not started. // To change status to waiting, use the aria2.unpause() method. // This method returns GID of paused download. func (c *client) Pause(gid string) (g string, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) err = c.Call(aria2Pause, params, &g) return } // `aria2.pauseAll([secret])` // This method is equal to calling aria2.pause() for every active/waiting download. // This methods returns OK. func (c *client) PauseAll() (ok string, err error) { params := []string{} if c.token != "" { params = append(params, "token:"+c.token) } err = c.Call(aria2PauseAll, params, &ok) return } // `aria2.forcePause([secret, ]gid)` // This method pauses the download denoted by gid. // This method behaves just like aria2.pause() except that this method pauses downloads without performing any actions which take time, such as contacting BitTorrent trackers to unregister the download first. func (c *client) ForcePause(gid string) (g string, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) err = c.Call(aria2ForcePause, params, &g) return } // `aria2.forcePauseAll([secret])` // This method is equal to calling aria2.forcePause() for every active/waiting download. // This methods returns OK. func (c *client) ForcePauseAll() (ok string, err error) { params := []string{} if c.token != "" { params = append(params, "token:"+c.token) } err = c.Call(aria2ForcePauseAll, params, &ok) return } // `aria2.unpause([secret, ]gid)` // This method changes the status of the download denoted by gid (string) from paused to waiting, making the download eligible to be restarted. // This method returns the GID of the unpaused download. func (c *client) Unpause(gid string) (g string, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) err = c.Call(aria2Unpause, params, &g) return } // `aria2.unpauseAll([secret])` // This method is equal to calling aria2.unpause() for every active/waiting download. // This methods returns OK. func (c *client) UnpauseAll() (ok string, err error) { params := []string{} if c.token != "" { params = append(params, "token:"+c.token) } err = c.Call(aria2UnpauseAll, params, &ok) return } // `aria2.tellStatus([secret, ]gid[, keys])` // This method returns the progress of the download denoted by gid (string). // keys is an array of strings. // If specified, the response contains only keys in the keys array. // If keys is empty or omitted, the response contains all keys. // This is useful when you just want specific keys and avoid unnecessary transfers. // For example, aria2.tellStatus("2089b05ecca3d829", ["gid", "status"]) returns the gid and status keys only. // The response is a struct and contains following keys. Values are strings. // https://aria2.github.io/manual/en/html/aria2c.html#aria2.tellStatus func (c *client) TellStatus(gid string, keys ...string) (info StatusInfo, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) if keys != nil { params = append(params, keys) } err = c.Call(aria2TellStatus, params, &info) return } // `aria2.getUris([secret, ]gid)` // This method returns the URIs used in the download denoted by gid (string). // The response is an array of structs and it contains following keys. Values are string. // // uri URI // status 'used' if the URI is in use. 'waiting' if the URI is still waiting in the queue. func (c *client) GetURIs(gid string) (infos []URIInfo, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) err = c.Call(aria2GetURIs, params, &infos) return } // `aria2.getFiles([secret, ]gid)` // This method returns the file list of the download denoted by gid (string). // The response is an array of structs which contain following keys. Values are strings. // https://aria2.github.io/manual/en/html/aria2c.html#aria2.getFiles func (c *client) GetFiles(gid string) (infos []FileInfo, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) err = c.Call(aria2GetFiles, params, &infos) return } // `aria2.getPeers([secret, ]gid)` // This method returns a list peers of the download denoted by gid (string). // This method is for BitTorrent only. // The response is an array of structs and contains the following keys. Values are strings. // https://aria2.github.io/manual/en/html/aria2c.html#aria2.getPeers func (c *client) GetPeers(gid string) (infos []PeerInfo, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) err = c.Call(aria2GetPeers, params, &infos) return } // `aria2.getServers([secret, ]gid)` // This method returns currently connected HTTP(S)/FTP/SFTP servers of the download denoted by gid (string). // The response is an array of structs and contains the following keys. Values are strings. // https://aria2.github.io/manual/en/html/aria2c.html#aria2.getServers func (c *client) GetServers(gid string) (infos []ServerInfo, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) err = c.Call(aria2GetServers, params, &infos) return } // `aria2.tellActive([secret][, keys])` // This method returns a list of active downloads. // The response is an array of the same structs as returned by the aria2.tellStatus() method. // For the keys parameter, please refer to the aria2.tellStatus() method. func (c *client) TellActive(keys ...string) (infos []StatusInfo, err error) { params := make([]interface{}, 0, 1) if c.token != "" { params = append(params, "token:"+c.token) } if keys != nil { params = append(params, keys) } err = c.Call(aria2TellActive, params, &infos) return } // `aria2.tellWaiting([secret, ]offset, num[, keys])` // This method returns a list of waiting downloads, including paused ones. // offset is an integer and specifies the offset from the download waiting at the front. // num is an integer and specifies the max. number of downloads to be returned. // For the keys parameter, please refer to the aria2.tellStatus() method. // If offset is a positive integer, this method returns downloads in the range of [offset, offset + num). // offset can be a negative integer. offset == -1 points last download in the waiting queue and offset == -2 points the download before the last download, and so on. // Downloads in the response are in reversed order then. // For example, imagine three downloads "A","B" and "C" are waiting in this order. // aria2.tellWaiting(0, 1) returns ["A"]. // aria2.tellWaiting(1, 2) returns ["B", "C"]. // aria2.tellWaiting(-1, 2) returns ["C", "B"]. // The response is an array of the same structs as returned by aria2.tellStatus() method. func (c *client) TellWaiting(offset, num int, keys ...string) (infos []StatusInfo, err error) { params := make([]interface{}, 0, 3) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, offset) params = append(params, num) if keys != nil { params = append(params, keys) } err = c.Call(aria2TellWaiting, params, &infos) return } // `aria2.tellStopped([secret, ]offset, num[, keys])` // This method returns a list of stopped downloads. // offset is an integer and specifies the offset from the least recently stopped download. // num is an integer and specifies the max. number of downloads to be returned. // For the keys parameter, please refer to the aria2.tellStatus() method. // offset and num have the same semantics as described in the aria2.tellWaiting() method. // The response is an array of the same structs as returned by the aria2.tellStatus() method. func (c *client) TellStopped(offset, num int, keys ...string) (infos []StatusInfo, err error) { params := make([]interface{}, 0, 3) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, offset) params = append(params, num) if keys != nil { params = append(params, keys) } err = c.Call(aria2TellStopped, params, &infos) return } // `aria2.changePosition([secret, ]gid, pos, how)` // This method changes the position of the download denoted by gid in the queue. // pos is an integer. how is a string. // If how is POS_SET, it moves the download to a position relative to the beginning of the queue. // If how is POS_CUR, it moves the download to a position relative to the current position. // If how is POS_END, it moves the download to a position relative to the end of the queue. // If the destination position is less than 0 or beyond the end of the queue, it moves the download to the beginning or the end of the queue respectively. // The response is an integer denoting the resulting position. // For example, if GID#2089b05ecca3d829 is currently in position 3, aria2.changePosition('2089b05ecca3d829', -1, 'POS_CUR') will change its position to 2. Additionally aria2.changePosition('2089b05ecca3d829', 0, 'POS_SET') will change its position to 0 (the beginning of the queue). func (c *client) ChangePosition(gid string, pos int, how string) (p int, err error) { params := make([]interface{}, 0, 3) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) params = append(params, pos) params = append(params, how) err = c.Call(aria2ChangePosition, params, &p) return } // `aria2.changeUri([secret, ]gid, fileIndex, delUris, addUris[, position])` // This method removes the URIs in delUris from and appends the URIs in addUris to download denoted by gid. // delUris and addUris are lists of strings. // A download can contain multiple files and URIs are attached to each file. // fileIndex is used to select which file to remove/attach given URIs. fileIndex is 1-based. // position is used to specify where URIs are inserted in the existing waiting URI list. position is 0-based. // When position is omitted, URIs are appended to the back of the list. // This method first executes the removal and then the addition. // position is the position after URIs are removed, not the position when this method is called. // When removing an URI, if the same URIs exist in download, only one of them is removed for each URI in delUris. // In other words, if there are three URIs http://example.org/aria2 and you want remove them all, you have to specify (at least) 3 http://example.org/aria2 in delUris. // This method returns a list which contains two integers. // The first integer is the number of URIs deleted. // The second integer is the number of URIs added. func (c *client) ChangeURI(gid string, fileindex int, delUris []string, addUris []string, position ...int) (p []int, err error) { params := make([]interface{}, 0, 5) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) params = append(params, fileindex) params = append(params, delUris) params = append(params, addUris) if position != nil { params = append(params, position[0]) } err = c.Call(aria2ChangeURI, params, &p) return } // `aria2.getOption([secret, ]gid)` // This method returns options of the download denoted by gid. // The response is a struct where keys are the names of options. // The values are strings. // Note that this method does not return options which have no default value and have not been set on the command-line, in configuration files or RPC methods. func (c *client) GetOption(gid string) (m Option, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) err = c.Call(aria2GetOption, params, &m) return } // `aria2.changeOption([secret, ]gid, options)` // This method changes options of the download denoted by gid (string) dynamically. options is a struct. // The following options are available for active downloads: // // bt-max-peers // bt-request-peer-speed-limit // bt-remove-unselected-file // force-save // max-download-limit // max-upload-limit // // For waiting or paused downloads, in addition to the above options, options listed in Input File subsection are available, except for following options: dry-run, metalink-base-uri, parameterized-uri, pause, piece-length and rpc-save-upload-metadata option. // This method returns OK for success. func (c *client) ChangeOption(gid string, option Option) (ok string, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) if option != nil { params = append(params, option) } err = c.Call(aria2ChangeOption, params, &ok) return } // `aria2.getGlobalOption([secret])` // This method returns the global options. // The response is a struct. // Its keys are the names of options. // Values are strings. // Note that this method does not return options which have no default value and have not been set on the command-line, in configuration files or RPC methods. Because global options are used as a template for the options of newly added downloads, the response contains keys returned by the aria2.getOption() method. func (c *client) GetGlobalOption() (m Option, err error) { params := []string{} if c.token != "" { params = append(params, "token:"+c.token) } err = c.Call(aria2GetGlobalOption, params, &m) return } // `aria2.changeGlobalOption([secret, ]options)` // This method changes global options dynamically. // options is a struct. // The following options are available: // // bt-max-open-files // download-result // log // log-level // max-concurrent-downloads // max-download-result // max-overall-download-limit // max-overall-upload-limit // save-cookies // save-session // server-stat-of // // In addition, options listed in the Input File subsection are available, except for following options: checksum, index-out, out, pause and select-file. // With the log option, you can dynamically start logging or change log file. // To stop logging, specify an empty string("") as the parameter value. // Note that log file is always opened in append mode. // This method returns OK for success. func (c *client) ChangeGlobalOption(options Option) (ok string, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, options) err = c.Call(aria2ChangeGlobalOption, params, &ok) return } // `aria2.getGlobalStat([secret])` // This method returns global statistics such as the overall download and upload speeds. // The response is a struct and contains the following keys. Values are strings. // // downloadSpeed Overall download speed (byte/sec). // uploadSpeed Overall upload speed(byte/sec). // numActive The number of active downloads. // numWaiting The number of waiting downloads. // numStopped The number of stopped downloads in the current session. // This value is capped by the --max-download-result option. // numStoppedTotal The number of stopped downloads in the current session and not capped by the --max-download-result option. func (c *client) GetGlobalStat() (info GlobalStatInfo, err error) { params := []string{} if c.token != "" { params = append(params, "token:"+c.token) } err = c.Call(aria2GetGlobalStat, params, &info) return } // `aria2.purgeDownloadResult([secret])` // This method purges completed/error/removed downloads to free memory. // This method returns OK. func (c *client) PurgeDownloadResult() (ok string, err error) { params := []string{} if c.token != "" { params = append(params, "token:"+c.token) } err = c.Call(aria2PurgeDownloadResult, params, &ok) return } // `aria2.removeDownloadResult([secret, ]gid)` // This method removes a completed/error/removed download denoted by gid from memory. // This method returns OK for success. func (c *client) RemoveDownloadResult(gid string) (ok string, err error) { params := make([]interface{}, 0, 2) if c.token != "" { params = append(params, "token:"+c.token) } params = append(params, gid) err = c.Call(aria2RemoveDownloadResult, params, &ok) return } // `aria2.getVersion([secret])` // This method returns the version of aria2 and the list of enabled features. // The response is a struct and contains following keys. // // version Version number of aria2 as a string. // enabledFeatures List of enabled features. Each feature is given as a string. func (c *client) GetVersion() (info VersionInfo, err error) { params := []string{} if c.token != "" { params = append(params, "token:"+c.token) } err = c.Call(aria2GetVersion, params, &info) return } // `aria2.getSessionInfo([secret])` // This method returns session information. // The response is a struct and contains following key. // // sessionId Session ID, which is generated each time when aria2 is invoked. func (c *client) GetSessionInfo() (info SessionInfo, err error) { params := []string{} if c.token != "" { params = append(params, "token:"+c.token) } err = c.Call(aria2GetSessionInfo, params, &info) return } // `aria2.shutdown([secret])` // This method shutdowns aria2. // This method returns OK. func (c *client) Shutdown() (ok string, err error) { params := []string{} if c.token != "" { params = append(params, "token:"+c.token) } err = c.Call(aria2Shutdown, params, &ok) return } // `aria2.forceShutdown([secret])` // This method shuts down aria2(). // This method behaves like :func:'aria2.shutdown` without performing any actions which take time, such as contacting BitTorrent trackers to unregister downloads first. // This method returns OK. func (c *client) ForceShutdown() (ok string, err error) { params := []string{} if c.token != "" { params = append(params, "token:"+c.token) } err = c.Call(aria2ForceShutdown, params, &ok) return } // `aria2.saveSession([secret])` // This method saves the current session to a file specified by the --save-session option. // This method returns OK if it succeeds. func (c *client) SaveSession() (ok string, err error) { params := []string{} if c.token != "" { params = append(params, "token:"+c.token) } err = c.Call(aria2SaveSession, params, &ok) return } // `system.multicall(methods)` // This methods encapsulates multiple method calls in a single request. // methods is an array of structs. // The structs contain two keys: methodName and params. // methodName is the method name to call and params is array containing parameters to the method call. // This method returns an array of responses. // The elements will be either a one-item array containing the return value of the method call or a struct of fault element if an encapsulated method call fails. func (c *client) Multicall(methods []Method) (r []interface{}, err error) { if len(methods) == 0 { err = errInvalidParameter return } err = c.Call(aria2Multicall, []interface{}{methods}, &r) return } // `system.listMethods()` // This method returns the all available RPC methods in an array of string. // Unlike other methods, this method does not require secret token. // This is safe because this method just returns the available method names. func (c *client) ListMethods() (methods []string, err error) { err = c.Call(aria2ListMethods, []string{}, &methods) return } ================================================ FILE: pkg/aria2/rpc/client_test.go ================================================ package rpc import ( "context" "testing" "time" ) func TestHTTPAll(t *testing.T) { const targetURL = "https://nodejs.org/dist/index.json" rpc, err := New(context.Background(), "http://localhost:6800/jsonrpc", "", time.Second, &DummyNotifier{}) if err != nil { t.Fatal(err) } defer rpc.Close() g, err := rpc.AddURI([]string{targetURL}) if err != nil { t.Fatal(err) } println(g) if _, err = rpc.TellActive(); err != nil { t.Error(err) } if _, err = rpc.PauseAll(); err != nil { t.Error(err) } if _, err = rpc.TellStatus(g); err != nil { t.Error(err) } if _, err = rpc.GetURIs(g); err != nil { t.Error(err) } if _, err = rpc.GetFiles(g); err != nil { t.Error(err) } if _, err = rpc.GetPeers(g); err != nil { t.Error(err) } if _, err = rpc.TellActive(); err != nil { t.Error(err) } if _, err = rpc.TellWaiting(0, 1); err != nil { t.Error(err) } if _, err = rpc.TellStopped(0, 1); err != nil { t.Error(err) } if _, err = rpc.GetOption(g); err != nil { t.Error(err) } if _, err = rpc.GetGlobalOption(); err != nil { t.Error(err) } if _, err = rpc.GetGlobalStat(); err != nil { t.Error(err) } if _, err = rpc.GetSessionInfo(); err != nil { t.Error(err) } if _, err = rpc.Remove(g); err != nil { t.Error(err) } if _, err = rpc.TellActive(); err != nil { t.Error(err) } } func TestWebsocketAll(t *testing.T) { const targetURL = "https://nodejs.org/dist/index.json" rpc, err := New(context.Background(), "ws://localhost:6800/jsonrpc", "", time.Second, &DummyNotifier{}) if err != nil { t.Fatal(err) } defer rpc.Close() g, err := rpc.AddURI([]string{targetURL}) if err != nil { t.Fatal(err) } println(g) if _, err = rpc.TellActive(); err != nil { t.Error(err) } if _, err = rpc.PauseAll(); err != nil { t.Error(err) } if _, err = rpc.TellStatus(g); err != nil { t.Error(err) } if _, err = rpc.GetURIs(g); err != nil { t.Error(err) } if _, err = rpc.GetFiles(g); err != nil { t.Error(err) } if _, err = rpc.GetPeers(g); err != nil { t.Error(err) } if _, err = rpc.TellActive(); err != nil { t.Error(err) } if _, err = rpc.TellWaiting(0, 1); err != nil { t.Error(err) } if _, err = rpc.TellStopped(0, 1); err != nil { t.Error(err) } if _, err = rpc.GetOption(g); err != nil { t.Error(err) } if _, err = rpc.GetGlobalOption(); err != nil { t.Error(err) } if _, err = rpc.GetGlobalStat(); err != nil { t.Error(err) } if _, err = rpc.GetSessionInfo(); err != nil { t.Error(err) } if _, err = rpc.Remove(g); err != nil { t.Error(err) } if _, err = rpc.TellActive(); err != nil { t.Error(err) } } ================================================ FILE: pkg/aria2/rpc/const.go ================================================ package rpc const ( aria2AddURI = "aria2.addUri" aria2AddTorrent = "aria2.addTorrent" aria2AddMetalink = "aria2.addMetalink" aria2Remove = "aria2.remove" aria2ForceRemove = "aria2.forceRemove" aria2Pause = "aria2.pause" aria2PauseAll = "aria2.pauseAll" aria2ForcePause = "aria2.forcePause" aria2ForcePauseAll = "aria2.forcePauseAll" aria2Unpause = "aria2.unpause" aria2UnpauseAll = "aria2.unpauseAll" aria2TellStatus = "aria2.tellStatus" aria2GetURIs = "aria2.getUris" aria2GetFiles = "aria2.getFiles" aria2GetPeers = "aria2.getPeers" aria2GetServers = "aria2.getServers" aria2TellActive = "aria2.tellActive" aria2TellWaiting = "aria2.tellWaiting" aria2TellStopped = "aria2.tellStopped" aria2ChangePosition = "aria2.changePosition" aria2ChangeURI = "aria2.changeUri" aria2GetOption = "aria2.getOption" aria2ChangeOption = "aria2.changeOption" aria2GetGlobalOption = "aria2.getGlobalOption" aria2ChangeGlobalOption = "aria2.changeGlobalOption" aria2GetGlobalStat = "aria2.getGlobalStat" aria2PurgeDownloadResult = "aria2.purgeDownloadResult" aria2RemoveDownloadResult = "aria2.removeDownloadResult" aria2GetVersion = "aria2.getVersion" aria2GetSessionInfo = "aria2.getSessionInfo" aria2Shutdown = "aria2.shutdown" aria2ForceShutdown = "aria2.forceShutdown" aria2SaveSession = "aria2.saveSession" aria2Multicall = "system.multicall" aria2ListMethods = "system.listMethods" ) ================================================ FILE: pkg/aria2/rpc/json2.go ================================================ package rpc // based on "github.com/gorilla/rpc/v2/json2" // Copyright 2009 The Go Authors. All rights reserved. // Copyright 2012 The Gorilla Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. import ( "bytes" "encoding/json" "errors" "io" ) // ---------------------------------------------------------------------------- // Request and Response // ---------------------------------------------------------------------------- // clientRequest represents a JSON-RPC request sent by a client. type clientRequest struct { // JSON-RPC protocol. Version string `json:"jsonrpc"` // A String containing the name of the method to be invoked. Method string `json:"method"` // Object to pass as request parameter to the method. Params interface{} `json:"params"` // The request id. This can be of any type. It is used to match the // response with the request that it is replying to. Id uint64 `json:"id"` } // clientResponse represents a JSON-RPC response returned to a client. type clientResponse struct { Version string `json:"jsonrpc"` Result *json.RawMessage `json:"result"` Error *json.RawMessage `json:"error"` Id *uint64 `json:"id"` } // EncodeClientRequest encodes parameters for a JSON-RPC client request. func EncodeClientRequest(method string, args interface{}) (*bytes.Buffer, error) { var buf bytes.Buffer c := &clientRequest{ Version: "2.0", Method: method, Params: args, Id: reqid(), } if err := json.NewEncoder(&buf).Encode(c); err != nil { return nil, err } return &buf, nil } func (c clientResponse) decode(reply interface{}) error { if c.Error != nil { jsonErr := &Error{} if err := json.Unmarshal(*c.Error, jsonErr); err != nil { return &Error{ Code: E_SERVER, Message: string(*c.Error), } } return jsonErr } if c.Result == nil { return ErrNullResult } return json.Unmarshal(*c.Result, reply) } // DecodeClientResponse decodes the response body of a client request into // the interface reply. func DecodeClientResponse(r io.Reader, reply interface{}) error { var c clientResponse if err := json.NewDecoder(r).Decode(&c); err != nil { return err } return c.decode(reply) } type ErrorCode int const ( E_PARSE ErrorCode = -32700 E_INVALID_REQ ErrorCode = -32600 E_NO_METHOD ErrorCode = -32601 E_BAD_PARAMS ErrorCode = -32602 E_INTERNAL ErrorCode = -32603 E_SERVER ErrorCode = -32000 ) var ErrNullResult = errors.New("result is null") type Error struct { // A Number that indicates the error type that occurred. Code ErrorCode `json:"code"` /* required */ // A String providing a short description of the error. // The message SHOULD be limited to a concise single sentence. Message string `json:"message"` /* required */ // A Primitive or Structured value that contains additional information about the error. Data interface{} `json:"data"` /* optional */ } func (e *Error) Error() string { return e.Message } ================================================ FILE: pkg/aria2/rpc/notification.go ================================================ package rpc import ( log "github.com/sirupsen/logrus" ) type Event struct { Gid string `json:"gid"` // GID of the download } // The RPC server might send notifications to the client. // Notifications is unidirectional, therefore the client which receives the notification must not respond to it. // The method signature of a notification is much like a normal method request but lacks the id key type websocketResponse struct { clientResponse Method string `json:"method"` Params []Event `json:"params"` } // Notifier handles rpc notification from aria2 server type Notifier interface { // OnDownloadStart will be sent when a download is started. OnDownloadStart([]Event) // OnDownloadPause will be sent when a download is paused. OnDownloadPause([]Event) // OnDownloadStop will be sent when a download is stopped by the user. OnDownloadStop([]Event) // OnDownloadComplete will be sent when a download is complete. For BitTorrent downloads, this notification is sent when the download is complete and seeding is over. OnDownloadComplete([]Event) // OnDownloadError will be sent when a download is stopped due to an error. OnDownloadError([]Event) // OnBtDownloadComplete will be sent when a torrent download is complete but seeding is still going on. OnBtDownloadComplete([]Event) } type DummyNotifier struct{} func (DummyNotifier) OnDownloadStart(events []Event) { log.Printf("%s started.", events) } func (DummyNotifier) OnDownloadPause(events []Event) { log.Printf("%s paused.", events) } func (DummyNotifier) OnDownloadStop(events []Event) { log.Printf("%s stopped.", events) } func (DummyNotifier) OnDownloadComplete(events []Event) { log.Printf("%s completed.", events) } func (DummyNotifier) OnDownloadError(events []Event) { log.Printf("%s error.", events) } func (DummyNotifier) OnBtDownloadComplete(events []Event) { log.Printf("bt %s completed.", events) } ================================================ FILE: pkg/aria2/rpc/proc.go ================================================ package rpc import "sync" type ResponseProcFn func(resp clientResponse) error type ResponseProcessor struct { cbs map[uint64]ResponseProcFn mu *sync.RWMutex } func NewResponseProcessor() *ResponseProcessor { return &ResponseProcessor{ make(map[uint64]ResponseProcFn), &sync.RWMutex{}, } } func (r *ResponseProcessor) Add(id uint64, fn ResponseProcFn) { r.mu.Lock() r.cbs[id] = fn r.mu.Unlock() } func (r *ResponseProcessor) remove(id uint64) { r.mu.Lock() delete(r.cbs, id) r.mu.Unlock() } // Process called by recv routine func (r *ResponseProcessor) Process(resp clientResponse) error { id := *resp.Id r.mu.RLock() fn, ok := r.cbs[id] r.mu.RUnlock() if ok && fn != nil { defer r.remove(id) return fn(resp) } return nil } ================================================ FILE: pkg/aria2/rpc/proto.go ================================================ package rpc // Protocol is a set of rpc methods that aria2 daemon supports type Protocol interface { AddURI(uris []string, options ...interface{}) (gid string, err error) AddTorrent(filename string, options ...interface{}) (gid string, err error) AddMetalink(filename string, options ...interface{}) (gid []string, err error) Remove(gid string) (g string, err error) ForceRemove(gid string) (g string, err error) Pause(gid string) (g string, err error) PauseAll() (ok string, err error) ForcePause(gid string) (g string, err error) ForcePauseAll() (ok string, err error) Unpause(gid string) (g string, err error) UnpauseAll() (ok string, err error) TellStatus(gid string, keys ...string) (info StatusInfo, err error) GetURIs(gid string) (infos []URIInfo, err error) GetFiles(gid string) (infos []FileInfo, err error) GetPeers(gid string) (infos []PeerInfo, err error) GetServers(gid string) (infos []ServerInfo, err error) TellActive(keys ...string) (infos []StatusInfo, err error) TellWaiting(offset, num int, keys ...string) (infos []StatusInfo, err error) TellStopped(offset, num int, keys ...string) (infos []StatusInfo, err error) ChangePosition(gid string, pos int, how string) (p int, err error) ChangeURI(gid string, fileindex int, delUris []string, addUris []string, position ...int) (p []int, err error) GetOption(gid string) (m Option, err error) ChangeOption(gid string, option Option) (ok string, err error) GetGlobalOption() (m Option, err error) ChangeGlobalOption(options Option) (ok string, err error) GetGlobalStat() (info GlobalStatInfo, err error) PurgeDownloadResult() (ok string, err error) RemoveDownloadResult(gid string) (ok string, err error) GetVersion() (info VersionInfo, err error) GetSessionInfo() (info SessionInfo, err error) Shutdown() (ok string, err error) ForceShutdown() (ok string, err error) SaveSession() (ok string, err error) Multicall(methods []Method) (r []interface{}, err error) ListMethods() (methods []string, err error) } ================================================ FILE: pkg/aria2/rpc/resp.go ================================================ //go:generate easyjson -all package rpc // StatusInfo represents response of aria2.tellStatus type StatusInfo struct { Gid string `json:"gid"` // GID of the download. Status string `json:"status"` // active for currently downloading/seeding downloads. waiting for downloads in the queue; download is not started. paused for paused downloads. error for downloads that were stopped because of error. complete for stopped and completed downloads. removed for the downloads removed by user. TotalLength string `json:"totalLength"` // Total length of the download in bytes. CompletedLength string `json:"completedLength"` // Completed length of the download in bytes. UploadLength string `json:"uploadLength"` // Uploaded length of the download in bytes. BitField string `json:"bitfield"` // Hexadecimal representation of the download progress. The highest bit corresponds to the piece at index 0. Any set bits indicate loaded pieces, while unset bits indicate not yet loaded and/or missing pieces. Any overflow bits at the end are set to zero. When the download was not started yet, this key will not be included in the response. DownloadSpeed string `json:"downloadSpeed"` // Download speed of this download measured in bytes/sec. UploadSpeed string `json:"uploadSpeed"` // Upload speed of this download measured in bytes/sec. InfoHash string `json:"infoHash"` // InfoHash. BitTorrent only. NumSeeders string `json:"numSeeders"` // The number of seeders aria2 has connected to. BitTorrent only. Seeder string `json:"seeder"` // true if the local endpoint is a seeder. Otherwise, false. BitTorrent only. PieceLength string `json:"pieceLength"` // Piece length in bytes. NumPieces string `json:"numPieces"` // The number of pieces. Connections string `json:"connections"` // The number of peers/servers aria2 has connected to. ErrorCode string `json:"errorCode"` // The code of the last error for this item, if any. The value is a string. The error codes are defined in the EXIT STATUS section. This value is only available for stopped/completed downloads. ErrorMessage string `json:"errorMessage"` // The (hopefully) human-readable error message associated to errorCode. FollowedBy []string `json:"followedBy"` // List of GIDs which are generated as the result of this download. For example, when aria2 downloads a Metalink file, it generates downloads described in the Metalink (see the --follow-metalink option). This value is useful to track auto-generated downloads. If there are no such downloads, this key will not be included in the response. BelongsTo string `json:"belongsTo"` // GID of a parent download. Some downloads are a part of another download. For example, if a file in a Metalink has BitTorrent resources, the downloads of ".torrent" files are parts of that parent. If this download has no parent, this key will not be included in the response. Dir string `json:"dir"` // Directory to save files. Files []FileInfo `json:"files"` // Returns the list of files. The elements of this list are the same structs used in aria2.getFiles() method. BitTorrent struct { AnnounceList [][]string `json:"announceList"` // List of lists of announce URIs. If the torrent contains announce and no announce-list, announce is converted to the announce-list format. Comment string `json:"comment"` // The comment of the torrent. comment.utf-8 is used if available. CreationDate int64 `json:"creationDate"` // The creation time of the torrent. The value is an integer since the epoch, measured in seconds. Mode string `json:"mode"` // File mode of the torrent. The value is either single or multi. Info struct { Name string `json:"name"` // name in info dictionary. name.utf-8 is used if available. } `json:"info"` // Struct which contains data from Info dictionary. It contains following keys. } `json:"bittorrent"` // Struct which contains information retrieved from the .torrent (file). BitTorrent only. It contains following keys. } // URIInfo represents an element of response of aria2.getUris type URIInfo struct { URI string `json:"uri"` // URI Status string `json:"status"` // 'used' if the URI is in use. 'waiting' if the URI is still waiting in the queue. } // FileInfo represents an element of response of aria2.getFiles type FileInfo struct { Index string `json:"index"` // Index of the file, starting at 1, in the same order as files appear in the multi-file torrent. Path string `json:"path"` // File path. Length string `json:"length"` // File size in bytes. CompletedLength string `json:"completedLength"` // Completed length of this file in bytes. Please note that it is possible that sum of completedLength is less than the completedLength returned by the aria2.tellStatus() method. This is because completedLength in aria2.getFiles() only includes completed pieces. On the other hand, completedLength in aria2.tellStatus() also includes partially completed pieces. Selected string `json:"selected"` // true if this file is selected by --select-file option. If --select-file is not specified or this is single-file torrent or not a torrent download at all, this value is always true. Otherwise false. URIs []URIInfo `json:"uris"` // Returns a list of URIs for this file. The element type is the same struct used in the aria2.getUris() method. } // PeerInfo represents an element of response of aria2.getPeers type PeerInfo struct { PeerId string `json:"peerId"` // Percent-encoded peer ID. IP string `json:"ip"` // IP address of the peer. Port string `json:"port"` // Port number of the peer. BitField string `json:"bitfield"` // Hexadecimal representation of the download progress of the peer. The highest bit corresponds to the piece at index 0. Set bits indicate the piece is available and unset bits indicate the piece is missing. Any spare bits at the end are set to zero. AmChoking string `json:"amChoking"` // true if aria2 is choking the peer. Otherwise false. PeerChoking string `json:"peerChoking"` // true if the peer is choking aria2. Otherwise false. DownloadSpeed string `json:"downloadSpeed"` // Download speed (byte/sec) that this client obtains from the peer. UploadSpeed string `json:"uploadSpeed"` // Upload speed(byte/sec) that this client uploads to the peer. Seeder string `json:"seeder"` // true if this peer is a seeder. Otherwise false. } // ServerInfo represents an element of response of aria2.getServers type ServerInfo struct { Index string `json:"index"` // Index of the file, starting at 1, in the same order as files appear in the multi-file metalink. Servers []struct { URI string `json:"uri"` // Original URI. CurrentURI string `json:"currentUri"` // This is the URI currently used for downloading. If redirection is involved, currentUri and uri may differ. DownloadSpeed string `json:"downloadSpeed"` // Download speed (byte/sec) } `json:"servers"` // A list of structs which contain the following keys. } // GlobalStatInfo represents response of aria2.getGlobalStat type GlobalStatInfo struct { DownloadSpeed string `json:"downloadSpeed"` // Overall download speed (byte/sec). UploadSpeed string `json:"uploadSpeed"` // Overall upload speed(byte/sec). NumActive string `json:"numActive"` // The number of active downloads. NumWaiting string `json:"numWaiting"` // The number of waiting downloads. NumStopped string `json:"numStopped"` // The number of stopped downloads in the current session. This value is capped by the --max-download-result option. NumStoppedTotal string `json:"numStoppedTotal"` // The number of stopped downloads in the current session and not capped by the --max-download-result option. } // VersionInfo represents response of aria2.getVersion type VersionInfo struct { Version string `json:"version"` // Version number of aria2 as a string. Features []string `json:"enabledFeatures"` // List of enabled features. Each feature is given as a string. } // SessionInfo represents response of aria2.getSessionInfo type SessionInfo struct { Id string `json:"sessionId"` // Session ID, which is generated each time when aria2 is invoked. } // Method is an element of parameters used in system.multicall type Method struct { Name string `json:"methodName"` // Method name to call Params []interface{} `json:"params"` // Array containing parameters to the method call } ================================================ FILE: pkg/buffer/bytes.go ================================================ package buffer import ( "errors" "io" ) // 用于存储不复用的[]byte type Reader struct { bufs [][]byte size int64 offset int64 } func (r *Reader) Size() int64 { return r.size } func (r *Reader) Append(buf []byte) { r.size += int64(len(buf)) r.bufs = append(r.bufs, buf) } func (r *Reader) Read(p []byte) (int, error) { n, err := r.ReadAt(p, r.offset) if n > 0 { r.offset += int64(n) } return n, err } func (r *Reader) ReadAt(p []byte, off int64) (int, error) { if off < 0 || off >= r.size { return 0, io.EOF } n := 0 readFrom := false for _, buf := range r.bufs { if readFrom { nn := copy(p[n:], buf) n += nn if n == len(p) { return n, nil } } else if newOff := off - int64(len(buf)); newOff >= 0 { off = newOff } else { nn := copy(p, buf[off:]) if nn == len(p) { return nn, nil } n += nn readFrom = true } } return n, io.EOF } func (r *Reader) Seek(offset int64, whence int) (int64, error) { switch whence { case io.SeekStart: case io.SeekCurrent: offset = r.offset + offset case io.SeekEnd: offset = r.size + offset default: return 0, errors.New("Seek: invalid whence") } if offset < 0 || offset > r.size { return 0, errors.New("Seek: invalid offset") } r.offset = offset return offset, nil } func (r *Reader) Reset() { clear(r.bufs) r.bufs = nil r.size = 0 r.offset = 0 } func NewReader(buf ...[]byte) *Reader { b := &Reader{ bufs: make([][]byte, 0, len(buf)), } for _, b1 := range buf { b.Append(b1) } return b } ================================================ FILE: pkg/buffer/bytes_test.go ================================================ package buffer import ( "errors" "io" "testing" ) func TestReader_ReadAt(t *testing.T) { type args struct { p []byte off int64 } bs := &Reader{} bs.Append([]byte("github.com")) bs.Append([]byte("/OpenList")) bs.Append([]byte("Team/")) bs.Append([]byte("OpenList")) tests := []struct { name string b *Reader args args want func(a args, n int, err error) error }{ { name: "readAt len 10 offset 0", b: bs, args: args{ p: make([]byte, 10), off: 0, }, want: func(a args, n int, err error) error { if n != len(a.p) { return errors.New("read length not match") } if string(a.p) != "github.com" { return errors.New("read content not match") } if err != nil { return err } return nil }, }, { name: "readAt len 12 offset 11", b: bs, args: args{ p: make([]byte, 12), off: 11, }, want: func(a args, n int, err error) error { if n != len(a.p) { return errors.New("read length not match") } if string(a.p) != "OpenListTeam" { return errors.New("read content not match") } if err != nil { return err } return nil }, }, { name: "readAt len 50 offset 24", b: bs, args: args{ p: make([]byte, 50), off: 24, }, want: func(a args, n int, err error) error { if n != int(bs.Size()-a.off) { return errors.New("read length not match") } if string(a.p[:n]) != "OpenList" { return errors.New("read content not match") } if err != io.EOF { return errors.New("expect eof") } return nil }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := tt.b.ReadAt(tt.args.p, tt.args.off) if err := tt.want(tt.args, got, err); err != nil { t.Errorf("Bytes.ReadAt() error = %v", err) } }) } } ================================================ FILE: pkg/buffer/file.go ================================================ package buffer import ( "errors" "io" "os" ) type PeekFile struct { peek *Reader file *os.File offset int64 size int64 } func (p *PeekFile) Read(b []byte) (n int, err error) { n, err = p.ReadAt(b, p.offset) if n > 0 { p.offset += int64(n) } return n, err } func (p *PeekFile) ReadAt(b []byte, off int64) (n int, err error) { if off < p.peek.Size() { n, err = p.peek.ReadAt(b, off) if err == nil || n == len(b) { return n, nil } // EOF } var nn int nn, err = p.file.ReadAt(b[n:], off+int64(n)-p.peek.Size()) return n + nn, err } func (p *PeekFile) Seek(offset int64, whence int) (int64, error) { switch whence { case io.SeekStart: case io.SeekCurrent: if offset == 0 { return p.offset, nil } offset = p.offset + offset case io.SeekEnd: offset = p.size + offset default: return 0, errors.New("Seek: invalid whence") } if offset < 0 || offset > p.size { return 0, errors.New("Seek: invalid offset") } if offset <= p.peek.Size() { _, err := p.peek.Seek(offset, io.SeekStart) if err != nil { return 0, err } _, err = p.file.Seek(0, io.SeekStart) if err != nil { return 0, err } } else { _, err := p.peek.Seek(p.peek.Size(), io.SeekStart) if err != nil { return 0, err } _, err = p.file.Seek(offset-p.peek.Size(), io.SeekStart) if err != nil { return 0, err } } p.offset = offset return offset, nil } func (p *PeekFile) Size() int64 { return p.size } func NewPeekFile(peek *Reader, file *os.File) (*PeekFile, error) { stat, err := file.Stat() if err == nil { return &PeekFile{peek: peek, file: file, size: stat.Size() + peek.Size()}, nil } return nil, err } ================================================ FILE: pkg/chanio/chanio.go ================================================ package chanio import ( "io" "sync/atomic" ) type ChanIO struct { cl atomic.Bool c chan []byte buf []byte } func New() *ChanIO { return &ChanIO{ cl: atomic.Bool{}, c: make(chan []byte), buf: make([]byte, 0), } } func (c *ChanIO) Read(p []byte) (int, error) { if c.cl.Load() { if len(c.buf) == 0 { return 0, io.EOF } n := copy(p, c.buf) if len(c.buf) > n { c.buf = c.buf[n:] } else { c.buf = make([]byte, 0) } return n, nil } for len(c.buf) < len(p) && !c.cl.Load() { c.buf = append(c.buf, <-c.c...) } n := copy(p, c.buf) if len(c.buf) > n { c.buf = c.buf[n:] } else { c.buf = make([]byte, 0) } return n, nil } func (c *ChanIO) Write(p []byte) (int, error) { if c.cl.Load() { return 0, io.ErrClosedPipe } c.c <- p return len(p), nil } func (c *ChanIO) Close() error { if c.cl.Load() { return io.ErrClosedPipe } c.cl.Store(true) close(c.c) return nil } ================================================ FILE: pkg/cookie/cookie.go ================================================ package cookie import ( "net/http" "strings" ) func Parse(str string) []*http.Cookie { header := http.Header{} header.Add("Cookie", str) request := http.Request{Header: header} return request.Cookies() } func ToString(cookies []*http.Cookie) string { if cookies == nil { return "" } cookieStrings := make([]string, len(cookies)) for i, cookie := range cookies { cookieStrings[i] = cookie.String() } return strings.Join(cookieStrings, ";") } func SetCookie(cookies []*http.Cookie, name, value string) []*http.Cookie { for i, cookie := range cookies { if cookie.Name == name { cookies[i].Value = value return cookies } } cookies = append(cookies, &http.Cookie{Name: name, Value: value}) return cookies } func GetCookie(cookies []*http.Cookie, name string) *http.Cookie { for _, cookie := range cookies { if cookie.Name == name { return cookie } } return nil } func SetStr(cookiesStr, name, value string) string { cookies := Parse(cookiesStr) cookies = SetCookie(cookies, name, value) return ToString(cookies) } func GetStr(cookiesStr, name string) string { cookies := Parse(cookiesStr) cookie := GetCookie(cookies, name) if cookie == nil { return "" } return cookie.Value } ================================================ FILE: pkg/cron/cron.go ================================================ package cron import "time" type Cron struct { d time.Duration ch chan struct{} } func NewCron(d time.Duration) *Cron { return &Cron{ d: d, ch: make(chan struct{}), } } func (c *Cron) Do(f func()) { go func() { ticker := time.NewTicker(c.d) defer ticker.Stop() for { select { case <-ticker.C: f() case <-c.ch: return } } }() } func (c *Cron) Stop() { select { case _, _ = <-c.ch: default: c.ch <- struct{}{} close(c.ch) } } ================================================ FILE: pkg/cron/cron_test.go ================================================ package cron import ( "testing" "time" ) func TestCron(t *testing.T) { c := NewCron(time.Second) c.Do(func() { t.Logf("cron log") }) time.Sleep(time.Second * 3) c.Stop() c.Stop() } ================================================ FILE: pkg/errgroup/errgroup.go ================================================ package errgroup import ( "context" "fmt" "sync" "sync/atomic" "github.com/avast/retry-go" ) type token struct{} type Group struct { cancel func(error) ctx context.Context opts []retry.Option success uint64 wg sync.WaitGroup sem chan token startChan chan token } func NewGroupWithContext(ctx context.Context, limit int, retryOpts ...retry.Option) (*Group, context.Context) { ctx, cancel := context.WithCancelCause(ctx) return (&Group{cancel: cancel, ctx: ctx, opts: append(retryOpts, retry.Context(ctx))}).SetLimit(limit), ctx } // OrderedGroup // 使得Lifecycle.Before是有序且线程安全 func NewOrderedGroupWithContext(ctx context.Context, limit int, retryOpts ...retry.Option) (*Group, context.Context) { group, ctx := NewGroupWithContext(ctx, limit, retryOpts...) group.startChan = make(chan token, 1) return group, ctx } func (g *Group) done() { if g.sem != nil { <-g.sem } g.wg.Done() atomic.AddUint64(&g.success, 1) } func (g *Group) Wait() error { g.wg.Wait() return context.Cause(g.ctx) } func (g *Group) Go(do func(ctx context.Context) error) { g.GoWithLifecycle(Lifecycle{Do: do}) } type Lifecycle struct { // Before在OrderedGroup是有序且线程安全的 // 只会被调用一次 Before func(ctx context.Context) (err error) // 如果Before返回err就不调用Do Do func(ctx context.Context) (err error) // 最后调用一次After After func(err error) } func (g *Group) GoWithLifecycle(lifecycle Lifecycle) { if g.startChan != nil { select { case <-g.ctx.Done(): return case g.startChan <- token{}: } } if g.sem != nil { select { case <-g.ctx.Done(): return case g.sem <- token{}: } } g.wg.Add(1) go func() { defer g.done() var err error if lifecycle.Before != nil { err = lifecycle.Before(g.ctx) } if err == nil { if g.startChan != nil { <-g.startChan } err = retry.Do(func() error { return lifecycle.Do(g.ctx) }, g.opts...) } if lifecycle.After != nil { lifecycle.After(err) } if err != nil { select { case <-g.ctx.Done(): return default: g.cancel(err) } } }() } func (g *Group) TryGo(f func(ctx context.Context) error) bool { if g.sem != nil { select { case g.sem <- token{}: default: return false } } g.wg.Add(1) go func() { defer g.done() if err := retry.Do(func() error { return f(g.ctx) }, g.opts...); err != nil { g.cancel(err) } }() return true } func (g *Group) SetLimit(n int) *Group { if len(g.sem) != 0 { panic(fmt.Errorf("errgroup: modify limit while %v goroutines in the group are still active", len(g.sem))) } if n > 0 { g.sem = make(chan token, n) } else { g.sem = nil } return g } func (g *Group) Success() uint64 { return atomic.LoadUint64(&g.success) } func (g *Group) Err() error { return context.Cause(g.ctx) } ================================================ FILE: pkg/generic/queue.go ================================================ package generic type Queue[T any] struct { queue []T } func NewQueue[T any]() *Queue[T] { return &Queue[T]{queue: make([]T, 0)} } func (q *Queue[T]) Push(v T) { q.queue = append(q.queue, v) } func (q *Queue[T]) Pop() T { v := q.queue[0] q.queue = q.queue[1:] return v } func (q *Queue[T]) Len() int { return len(q.queue) } func (q *Queue[T]) IsEmpty() bool { return len(q.queue) == 0 } func (q *Queue[T]) Clear() { q.queue = nil } func (q *Queue[T]) Peek() T { return q.queue[0] } func (q *Queue[T]) PeekN(n int) []T { return q.queue[:n] } func (q *Queue[T]) PopN(n int) []T { v := q.queue[:n] q.queue = q.queue[n:] return v } func (q *Queue[T]) PopAll() []T { v := q.queue q.queue = nil return v } func (q *Queue[T]) PopWhile(f func(T) bool) []T { var i int for i = 0; i < len(q.queue); i++ { if !f(q.queue[i]) { break } } v := q.queue[:i] q.queue = q.queue[i:] return v } func (q *Queue[T]) PopUntil(f func(T) bool) []T { var i int for i = 0; i < len(q.queue); i++ { if f(q.queue[i]) { break } } v := q.queue[:i] q.queue = q.queue[i:] return v } ================================================ FILE: pkg/generic_sync/map.go ================================================ // Copyright 2016 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package generic_sync import ( "sync" "sync/atomic" "unsafe" ) // MapOf is like a Go map[interface{}]interface{} but is safe for concurrent use // by multiple goroutines without additional locking or coordination. // Loads, stores, and deletes run in amortized constant time. // // The MapOf type is specialized. Most code should use a plain Go map instead, // with separate locking or coordination, for better type safety and to make it // easier to maintain other invariants along with the map content. // // The MapOf type is optimized for two common use cases: (1) when the entry for a given // key is only ever written once but read many times, as in caches that only grow, // or (2) when multiple goroutines read, write, and overwrite entries for disjoint // sets of keys. In these two cases, use of a MapOf may significantly reduce lock // contention compared to a Go map paired with a separate Mutex or RWMutex. // // The zero MapOf is empty and ready for use. A MapOf must not be copied after first use. type MapOf[K comparable, V any] struct { mu sync.Mutex // read contains the portion of the map's contents that are safe for // concurrent access (with or without mu held). // // The read field itself is always safe to load, but must only be stored with // mu held. // // Entries stored in read may be updated concurrently without mu, but updating // a previously-expunged entry requires that the entry be copied to the dirty // map and unexpunged with mu held. read atomic.Value // readOnly // dirty contains the portion of the map's contents that require mu to be // held. To ensure that the dirty map can be promoted to the read map quickly, // it also includes all of the non-expunged entries in the read map. // // Expunged entries are not stored in the dirty map. An expunged entry in the // clean map must be unexpunged and added to the dirty map before a new value // can be stored to it. // // If the dirty map is nil, the next write to the map will initialize it by // making a shallow copy of the clean map, omitting stale entries. dirty map[K]*entry[V] // misses counts the number of loads since the read map was last updated that // needed to lock mu to determine whether the key was present. // // Once enough misses have occurred to cover the cost of copying the dirty // map, the dirty map will be promoted to the read map (in the unamended // state) and the next store to the map will make a new dirty copy. misses int } // readOnly is an immutable struct stored atomically in the MapOf.read field. type readOnly[K comparable, V any] struct { m map[K]*entry[V] amended bool // true if the dirty map contains some key not in m. } // expunged is an arbitrary pointer that marks entries which have been deleted // from the dirty map. var expunged = unsafe.Pointer(new(interface{})) // An entry is a slot in the map corresponding to a particular key. type entry[V any] struct { // p points to the interface{} value stored for the entry. // // If p == nil, the entry has been deleted and m.dirty == nil. // // If p == expunged, the entry has been deleted, m.dirty != nil, and the entry // is missing from m.dirty. // // Otherwise, the entry is valid and recorded in m.read.m[key] and, if m.dirty // != nil, in m.dirty[key]. // // An entry can be deleted by atomic replacement with nil: when m.dirty is // next created, it will atomically replace nil with expunged and leave // m.dirty[key] unset. // // An entry's associated value can be updated by atomic replacement, provided // p != expunged. If p == expunged, an entry's associated value can be updated // only after first setting m.dirty[key] = e so that lookups using the dirty // map find the entry. p unsafe.Pointer // *interface{} } func newEntry[V any](i V) *entry[V] { return &entry[V]{p: unsafe.Pointer(&i)} } // Load returns the value stored in the map for a key, or nil if no // value is present. // The ok result indicates whether value was found in the map. func (m *MapOf[K, V]) Load(key K) (value V, ok bool) { read, _ := m.read.Load().(readOnly[K, V]) e, ok := read.m[key] if !ok && read.amended { m.mu.Lock() // Avoid reporting a spurious miss if m.dirty got promoted while we were // blocked on m.mu. (If further loads of the same key will not miss, it's // not worth copying the dirty map for this key.) read, _ = m.read.Load().(readOnly[K, V]) e, ok = read.m[key] if !ok && read.amended { e, ok = m.dirty[key] // Regardless of whether the entry was present, record a miss: this key // will take the slow path until the dirty map is promoted to the read // map. m.missLocked() } m.mu.Unlock() } if !ok { return value, false } return e.load() } func (m *MapOf[K, V]) Has(key K) bool { _, ok := m.Load(key) return ok } func (e *entry[V]) load() (value V, ok bool) { p := atomic.LoadPointer(&e.p) if p == nil || p == expunged { return value, false } return *(*V)(p), true } // Store sets the value for a key. func (m *MapOf[K, V]) Store(key K, value V) { read, _ := m.read.Load().(readOnly[K, V]) if e, ok := read.m[key]; ok && e.tryStore(&value) { return } m.mu.Lock() read, _ = m.read.Load().(readOnly[K, V]) if e, ok := read.m[key]; ok { if e.unexpungeLocked() { // The entry was previously expunged, which implies that there is a // non-nil dirty map and this entry is not in it. m.dirty[key] = e } e.storeLocked(&value) } else if e, ok := m.dirty[key]; ok { e.storeLocked(&value) } else { if !read.amended { // We're adding the first new key to the dirty map. // Make sure it is allocated and mark the read-only map as incomplete. m.dirtyLocked() m.read.Store(readOnly[K, V]{m: read.m, amended: true}) } m.dirty[key] = newEntry(value) } m.mu.Unlock() } // tryStore stores a value if the entry has not been expunged. // // If the entry is expunged, tryStore returns false and leaves the entry // unchanged. func (e *entry[V]) tryStore(i *V) bool { for { p := atomic.LoadPointer(&e.p) if p == expunged { return false } if atomic.CompareAndSwapPointer(&e.p, p, unsafe.Pointer(i)) { return true } } } // unexpungeLocked ensures that the entry is not marked as expunged. // // If the entry was previously expunged, it must be added to the dirty map // before m.mu is unlocked. func (e *entry[V]) unexpungeLocked() (wasExpunged bool) { return atomic.CompareAndSwapPointer(&e.p, expunged, nil) } // storeLocked unconditionally stores a value to the entry. // // The entry must be known not to be expunged. func (e *entry[V]) storeLocked(i *V) { atomic.StorePointer(&e.p, unsafe.Pointer(i)) } // LoadOrStore returns the existing value for the key if present. // Otherwise, it stores and returns the given value. // The loaded result is true if the value was loaded, false if stored. func (m *MapOf[K, V]) LoadOrStore(key K, value V) (actual V, loaded bool) { // Avoid locking if it's a clean hit. read, _ := m.read.Load().(readOnly[K, V]) if e, ok := read.m[key]; ok { actual, loaded, ok := e.tryLoadOrStore(value) if ok { return actual, loaded } } m.mu.Lock() read, _ = m.read.Load().(readOnly[K, V]) if e, ok := read.m[key]; ok { if e.unexpungeLocked() { m.dirty[key] = e } actual, loaded, _ = e.tryLoadOrStore(value) } else if e, ok := m.dirty[key]; ok { actual, loaded, _ = e.tryLoadOrStore(value) m.missLocked() } else { if !read.amended { // We're adding the first new key to the dirty map. // Make sure it is allocated and mark the read-only map as incomplete. m.dirtyLocked() m.read.Store(readOnly[K, V]{m: read.m, amended: true}) } m.dirty[key] = newEntry(value) actual, loaded = value, false } m.mu.Unlock() return actual, loaded } // tryLoadOrStore atomically loads or stores a value if the entry is not // expunged. // // If the entry is expunged, tryLoadOrStore leaves the entry unchanged and // returns with ok==false. func (e *entry[V]) tryLoadOrStore(i V) (actual V, loaded, ok bool) { p := atomic.LoadPointer(&e.p) if p == expunged { return actual, false, false } if p != nil { return *(*V)(p), true, true } // Copy the interface after the first load to make this method more amenable // to escape analysis: if we hit the "load" path or the entry is expunged, we // shouldn'V bother heap-allocating. ic := i for { if atomic.CompareAndSwapPointer(&e.p, nil, unsafe.Pointer(&ic)) { return i, false, true } p = atomic.LoadPointer(&e.p) if p == expunged { return actual, false, false } if p != nil { return *(*V)(p), true, true } } } // Delete deletes the value for a key. func (m *MapOf[K, V]) Delete(key K) { read, _ := m.read.Load().(readOnly[K, V]) e, ok := read.m[key] if !ok && read.amended { m.mu.Lock() read, _ = m.read.Load().(readOnly[K, V]) e, ok = read.m[key] if !ok && read.amended { delete(m.dirty, key) } m.mu.Unlock() } if ok { e.delete() } } func (e *entry[V]) delete() (hadValue bool) { for { p := atomic.LoadPointer(&e.p) if p == nil || p == expunged { return false } if atomic.CompareAndSwapPointer(&e.p, p, nil) { return true } } } // Range calls f sequentially for each key and value present in the map. // If f returns false, range stops the iteration. // // Range does not necessarily correspond to any consistent snapshot of the MapOf's // contents: no key will be visited more than once, but if the value for any key // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. // // Range may be O(N) with the number of elements in the map even if f returns // false after a constant number of calls. func (m *MapOf[K, V]) Range(f func(key K, value V) bool) { // We need to be able to iterate over all of the keys that were already // present at the start of the call to Range. // If read.amended is false, then read.m satisfies that property without // requiring us to hold m.mu for a long time. read, _ := m.read.Load().(readOnly[K, V]) if read.amended { // m.dirty contains keys not in read.m. Fortunately, Range is already O(N) // (assuming the caller does not break out early), so a call to Range // amortizes an entire copy of the map: we can promote the dirty copy // immediately! m.mu.Lock() read, _ = m.read.Load().(readOnly[K, V]) if read.amended { read = readOnly[K, V]{m: m.dirty} m.read.Store(read) m.dirty = nil m.misses = 0 } m.mu.Unlock() } for k, e := range read.m { v, ok := e.load() if !ok { continue } if !f(k, v) { break } } } // Values returns a slice of the values in the map. func (m *MapOf[K, V]) Values() []V { var values []V m.Range(func(key K, value V) bool { values = append(values, value) return true }) return values } func (m *MapOf[K, V]) Count() int { return len(m.dirty) } func (m *MapOf[K, V]) Empty() bool { return m.Count() == 0 } func (m *MapOf[K, V]) ToMap() map[K]V { ans := make(map[K]V) m.Range(func(key K, value V) bool { ans[key] = value return true }) return ans } func (m *MapOf[K, V]) Clear() { m.Range(func(key K, value V) bool { m.Delete(key) return true }) } func (m *MapOf[K, V]) missLocked() { m.misses++ if m.misses < len(m.dirty) { return } m.read.Store(readOnly[K, V]{m: m.dirty}) m.dirty = nil m.misses = 0 } func (m *MapOf[K, V]) dirtyLocked() { if m.dirty != nil { return } read, _ := m.read.Load().(readOnly[K, V]) m.dirty = make(map[K]*entry[V], len(read.m)) for k, e := range read.m { if !e.tryExpungeLocked() { m.dirty[k] = e } } } func (e *entry[V]) tryExpungeLocked() (isExpunged bool) { p := atomic.LoadPointer(&e.p) for p == nil { if atomic.CompareAndSwapPointer(&e.p, nil, expunged) { return true } p = atomic.LoadPointer(&e.p) } return p == expunged } ================================================ FILE: pkg/generic_sync/map_test.go ================================================ // Copyright 2016 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package generic_sync_test import ( "math/rand" "runtime" "sync" "testing" "github.com/OpenListTeam/OpenList/v4/pkg/generic_sync" ) func TestConcurrentRange(t *testing.T) { const mapSize = 1 << 10 m := new(generic_sync.MapOf[int64, int64]) for n := int64(1); n <= mapSize; n++ { m.Store(n, int64(n)) } done := make(chan struct{}) var wg sync.WaitGroup defer func() { close(done) wg.Wait() }() for g := int64(runtime.GOMAXPROCS(0)); g > 0; g-- { r := rand.New(rand.NewSource(g)) wg.Add(1) go func(g int64) { defer wg.Done() for i := int64(0); ; i++ { select { case <-done: return default: } for n := int64(1); n < mapSize; n++ { if r.Int63n(mapSize) == 0 { m.Store(n, n*i*g) } else { m.Load(n) } } } }(g) } iters := 1 << 10 if testing.Short() { iters = 16 } for n := iters; n > 0; n-- { seen := make(map[int64]bool, mapSize) m.Range(func(k, v int64) bool { if v%k != 0 { t.Fatalf("while Storing multiples of %v, Range saw value %v", k, v) } if seen[k] { t.Fatalf("Range visited key %v twice", k) } seen[k] = true return true }) if len(seen) != mapSize { t.Fatalf("Range visited %v elements of %v-element MapOf", len(seen), mapSize) } } } ================================================ FILE: pkg/http_range/range.go ================================================ // Package http_range implements http range parsing. package http_range import ( "errors" "fmt" "net/http" "net/textproto" "strconv" "strings" ) // Range specifies the byte range to be sent to the client. type Range struct { Start int64 Length int64 // limit of bytes to read, -1 for unlimited } // ContentRange returns Content-Range header value. func (r Range) ContentRange(size int64) string { return fmt.Sprintf("bytes %d-%d/%d", r.Start, r.Start+r.Length-1, size) } var ( // ErrNoOverlap is returned by ParseRange if first-byte-pos of // all the byte-range-spec values is greater than the content size. ErrNoOverlap = errors.New("invalid range: failed to overlap") // ErrInvalid is returned by ParseRange on invalid input. ErrInvalid = errors.New("invalid range") ) // ParseRange parses a Range header string as per RFC 7233. // ErrNoOverlap is returned if none of the ranges overlap. // ErrInvalid is returned if s is invalid range. func ParseRange(s string, size int64) ([]Range, error) { // nolint:gocognit if s == "" { return nil, nil // header not present } const b = "bytes=" if !strings.HasPrefix(s, b) { return nil, ErrInvalid } var ranges []Range noOverlap := false for _, ra := range strings.Split(s[len(b):], ",") { ra = textproto.TrimString(ra) if ra == "" { continue } i := strings.Index(ra, "-") if i < 0 { return nil, ErrInvalid } start, end := textproto.TrimString(ra[:i]), textproto.TrimString(ra[i+1:]) var r Range if start == "" { // If no start is specified, end specifies the // range start relative to the end of the file, // and we are dealing with // which has to be a non-negative integer as per // RFC 7233 Section 2.1 "Byte-Ranges". if end == "" || end[0] == '-' { return nil, ErrInvalid } i, err := strconv.ParseInt(end, 10, 64) if i < 0 || err != nil { return nil, ErrInvalid } if i > size { i = size } r.Start = size - i r.Length = size - r.Start } else { i, err := strconv.ParseInt(start, 10, 64) if err != nil || i < 0 { return nil, ErrInvalid } if i >= size { // If the range begins after the size of the content, // then it does not overlap. noOverlap = true continue } r.Start = i if end == "" { // If no end is specified, range extends to end of the file. r.Length = size - r.Start } else { i, err := strconv.ParseInt(end, 10, 64) if err != nil || r.Start > i { return nil, ErrInvalid } if i >= size { i = size - 1 } r.Length = i - r.Start + 1 } } ranges = append(ranges, r) } if noOverlap && len(ranges) == 0 { // The specified ranges did not overlap with the content. return nil, ErrNoOverlap } return ranges, nil } // ParseContentRange this function parse content-range in http response func ParseContentRange(s string) (start, end int64, err error) { if s == "" { return 0, 0, ErrInvalid } const b = "bytes " if !strings.HasPrefix(s, b) { return 0, 0, ErrInvalid } p1 := strings.Index(s, "-") p2 := strings.Index(s, "/") if p1 < 0 || p2 < 0 { return 0, 0, ErrInvalid } startStr, endStr := textproto.TrimString(s[len(b):p1]), textproto.TrimString(s[p1+1:p2]) start, startErr := strconv.ParseInt(startStr, 10, 64) end, endErr := strconv.ParseInt(endStr, 10, 64) return start, end, errors.Join(startErr, endErr) } func (r Range) MimeHeader(contentType string, size int64) textproto.MIMEHeader { return textproto.MIMEHeader{ "Content-Range": {r.ContentRange(size)}, "Content-Type": {contentType}, } } // ApplyRangeToHttpHeader for http request header func ApplyRangeToHttpHeader(p Range, headerRef http.Header) http.Header { header := headerRef if header == nil { header = http.Header{} } if p.Start == 0 && p.Length < 0 { header.Del("Range") } else { end := "" if p.Length >= 0 { end = strconv.FormatInt(p.Start+p.Length-1, 10) } header.Set("Range", fmt.Sprintf("bytes=%v-%v", p.Start, end)) } return header } ================================================ FILE: pkg/mq/mq.go ================================================ package mq import ( "sync" "github.com/OpenListTeam/OpenList/v4/pkg/generic" ) type Message[T any] struct { Content T } type BasicConsumer[T any] func(Message[T]) type AllConsumer[T any] func([]Message[T]) type MQ[T any] interface { Publish(Message[T]) Consume(BasicConsumer[T]) ConsumeAll(AllConsumer[T]) Clear() Len() int } type inMemoryMQ[T any] struct { queue generic.Queue[Message[T]] sync.Mutex } func NewInMemoryMQ[T any]() MQ[T] { return &inMemoryMQ[T]{queue: *generic.NewQueue[Message[T]]()} } func (mq *inMemoryMQ[T]) Publish(msg Message[T]) { mq.Lock() defer mq.Unlock() mq.queue.Push(msg) } func (mq *inMemoryMQ[T]) Consume(consumer BasicConsumer[T]) { mq.Lock() defer mq.Unlock() for !mq.queue.IsEmpty() { consumer(mq.queue.Pop()) } } func (mq *inMemoryMQ[T]) ConsumeAll(consumer AllConsumer[T]) { mq.Lock() defer mq.Unlock() consumer(mq.queue.PopAll()) } func (mq *inMemoryMQ[T]) Clear() { mq.Lock() defer mq.Unlock() mq.queue.Clear() } func (mq *inMemoryMQ[T]) Len() int { mq.Lock() defer mq.Unlock() return mq.queue.Len() } ================================================ FILE: pkg/pool/pool.go ================================================ package pool import "sync" type Pool[T any] struct { New func() T MaxCap int cache []T mu sync.Mutex } func (p *Pool[T]) Get() T { p.mu.Lock() defer p.mu.Unlock() if len(p.cache) == 0 { return p.New() } item := p.cache[len(p.cache)-1] p.cache = p.cache[:len(p.cache)-1] return item } func (p *Pool[T]) Put(item T) { p.mu.Lock() defer p.mu.Unlock() if p.MaxCap == 0 || len(p.cache) < int(p.MaxCap) { p.cache = append(p.cache, item) } } func (p *Pool[T]) Reset() { p.mu.Lock() defer p.mu.Unlock() clear(p.cache) p.cache = nil } ================================================ FILE: pkg/qbittorrent/client.go ================================================ package qbittorrent import ( "bytes" "errors" "io" "mime/multipart" "net/http" "net/http/cookiejar" "net/url" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type Client interface { AddFromLink(link string, savePath string, id string) error GetInfo(id string) (TorrentInfo, error) GetFiles(id string) ([]FileInfo, error) Delete(id string, deleteFiles bool) error } type client struct { url *url.URL client http.Client Client } func New(webuiUrl string) (Client, error) { u, err := url.Parse(webuiUrl) if err != nil { return nil, err } jar, err := cookiejar.New(nil) if err != nil { return nil, err } transport := &http.Transport{ MaxIdleConns: 10, MaxIdleConnsPerHost: 2, IdleConnTimeout: 30 * time.Second, DisableKeepAlives: false, // Enable connection reuse } var c = &client{ url: u, client: http.Client{ Jar: jar, Transport: transport, Timeout: 30 * time.Second, // Set overall timeout }, } err = c.checkAuthorization() if err != nil { return nil, err } return c, nil } func (c *client) checkAuthorization() error { // check authorization if c.authorized() { return nil } // check authorization after logging in err := c.login() if err != nil { return err } if c.authorized() { return nil } return errors.New("unauthorized qbittorrent url") } func (c *client) authorized() bool { resp, err := c.post("/api/v2/app/version", nil) if err != nil { return false } defer resp.Body.Close() return resp.StatusCode == 200 // the status code will be 403 if not authorized } func (c *client) login() error { // prepare HTTP request v := url.Values{} v.Set("username", c.url.User.Username()) passwd, _ := c.url.User.Password() v.Set("password", passwd) resp, err := c.post("/api/v2/auth/login", v) if err != nil { return err } defer resp.Body.Close() // check result body := make([]byte, 2) _, err = resp.Body.Read(body) if err != nil { return err } if string(body) != "Ok" { return errors.New("failed to login into qBittorrent webui with url: " + c.url.String()) } return nil } func (c *client) post(path string, data url.Values) (*http.Response, error) { u := c.url.JoinPath(path) u.User = nil // remove userinfo for requests req, err := http.NewRequest(http.MethodPost, u.String(), bytes.NewReader([]byte(data.Encode()))) if err != nil { return nil, err } if data != nil { req.Header.Add("Content-Type", "application/x-www-form-urlencoded") } resp, err := c.client.Do(req) if err != nil { return nil, err } if resp.Cookies() != nil { c.client.Jar.SetCookies(u, resp.Cookies()) } return resp, nil } func (c *client) AddFromLink(link string, savePath string, id string) error { err := c.checkAuthorization() if err != nil { return err } buf := new(bytes.Buffer) writer := multipart.NewWriter(buf) addField := func(name string, value string) { if err != nil { return } err = writer.WriteField(name, value) } addField("urls", link) addField("savepath", savePath) addField("tags", "openlist-"+id) addField("autoTMM", "false") if err != nil { return err } err = writer.Close() if err != nil { return err } u := c.url.JoinPath("/api/v2/torrents/add") u.User = nil // remove userinfo for requests req, err := http.NewRequest(http.MethodPost, u.String(), buf) if err != nil { return err } req.Header.Add("Content-Type", writer.FormDataContentType()) resp, err := c.client.Do(req) if err != nil { return err } defer resp.Body.Close() // check result body := make([]byte, 2) _, err = resp.Body.Read(body) if err != nil { return err } if resp.StatusCode != 200 || string(body) != "Ok" { return errors.New("failed to add qBittorrent task: " + link) } return nil } type TorrentStatus string const ( ERROR TorrentStatus = "error" MISSINGFILES TorrentStatus = "missingFiles" UPLOADING TorrentStatus = "uploading" PAUSEDUP TorrentStatus = "pausedUP" QUEUEDUP TorrentStatus = "queuedUP" STALLEDUP TorrentStatus = "stalledUP" CHECKINGUP TorrentStatus = "checkingUP" FORCEDUP TorrentStatus = "forcedUP" ALLOCATING TorrentStatus = "allocating" DOWNLOADING TorrentStatus = "downloading" METADL TorrentStatus = "metaDL" PAUSEDDL TorrentStatus = "pausedDL" QUEUEDDL TorrentStatus = "queuedDL" STALLEDDL TorrentStatus = "stalledDL" CHECKINGDL TorrentStatus = "checkingDL" FORCEDDL TorrentStatus = "forcedDL" CHECKINGRESUMEDATA TorrentStatus = "checkingResumeData" MOVING TorrentStatus = "moving" UNKNOWN TorrentStatus = "unknown" ) // https://github.com/DGuang21/PTGo/blob/main/app/client/client_distributer.go type TorrentInfo struct { AddedOn int `json:"added_on"` // 将 torrent 添加到客户端的时间（Unix Epoch） AmountLeft int64 `json:"amount_left"` // 剩余大小（字节） AutoTmm bool `json:"auto_tmm"` // 此 torrent 是否由 Automatic Torrent Management 管理 Availability float64 `json:"availability"` // 当前百分比 Category string `json:"category"` // Completed int64 `json:"completed"` // 完成的传输数据量（字节） CompletionOn int `json:"completion_on"` // Torrent 完成的时间（Unix Epoch） ContentPath string `json:"content_path"` // torrent 内容的绝对路径（多文件 torrent 的根路径，单文件 torrent 的绝对文件路径） DlLimit int `json:"dl_limit"` // Torrent 下载速度限制（字节/秒） Dlspeed int `json:"dlspeed"` // Torrent 下载速度（字节/秒） Downloaded int64 `json:"downloaded"` // 已经下载大小 DownloadedSession int64 `json:"downloaded_session"` // 此会话下载的数据量 Eta int `json:"eta"` // FLPiecePrio bool `json:"f_l_piece_prio"` // 如果第一个最后一块被优先考虑，则为true ForceStart bool `json:"force_start"` // 如果为此 torrent 启用了强制启动，则为true Hash string `json:"hash"` // LastActivity int `json:"last_activity"` // 上次活跃的时间（Unix Epoch） MagnetURI string `json:"magnet_uri"` // 与此 torrent 对应的 Magnet URI MaxRatio float64 `json:"max_ratio"` // 种子/上传停止种子前的最大共享比率 MaxSeedingTime int `json:"max_seeding_time"` // 停止种子种子前的最长种子时间（秒） Name string `json:"name"` // NumComplete int `json:"num_complete"` // NumIncomplete int `json:"num_incomplete"` // NumLeechs int `json:"num_leechs"` // 连接到的 leechers 的数量 NumSeeds int `json:"num_seeds"` // 连接到的种子数 Priority int `json:"priority"` // 速度优先。如果队列被禁用或 torrent 处于种子模式，则返回 -1 Progress float64 `json:"progress"` // 进度 Ratio float64 `json:"ratio"` // Torrent 共享比率 RatioLimit int `json:"ratio_limit"` // SavePath string `json:"save_path"` SeedingTime int `json:"seeding_time"` // Torrent 完成用时（秒） SeedingTimeLimit int `json:"seeding_time_limit"` // max_seeding_time SeenComplete int `json:"seen_complete"` // 上次 torrent 完成的时间 SeqDl bool `json:"seq_dl"` // 如果启用顺序下载，则为true Size int64 `json:"size"` // State TorrentStatus `json:"state"` // 参见https://github.com/qbittorrent/qBittorrent/wiki/WebUI-API-(qBittorrent-4.1)#get-torrent-list SuperSeeding bool `json:"super_seeding"` // 如果启用超级播种，则为true Tags string `json:"tags"` // Torrent 的逗号连接标签列表 TimeActive int `json:"time_active"` // 总活动时间（秒） TotalSize int64 `json:"total_size"` // 此 torrent 中所有文件的总大小（字节）（包括未选择的文件） Tracker string `json:"tracker"` // 第一个具有工作状态的tracker。如果没有tracker在工作，则返回空字符串。 TrackersCount int `json:"trackers_count"` // UpLimit int `json:"up_limit"` // 上传限制 Uploaded int64 `json:"uploaded"` // 累计上传 UploadedSession int64 `json:"uploaded_session"` // 当前session累计上传 Upspeed int `json:"upspeed"` // 上传速度（字节/秒） } type InfoNotFoundError struct { Id string Err error } func (i InfoNotFoundError) Error() string { return "there should be exactly one task with tag \"openlist-" + i.Id + "\"" } func NewInfoNotFoundError(id string) InfoNotFoundError { return InfoNotFoundError{Id: id} } func (c *client) GetInfo(id string) (TorrentInfo, error) { var infos []TorrentInfo err := c.checkAuthorization() if err != nil { return TorrentInfo{}, err } v := url.Values{} v.Set("tag", "openlist-"+id) response, err := c.post("/api/v2/torrents/info", v) if err != nil { return TorrentInfo{}, err } defer response.Body.Close() body, err := io.ReadAll(response.Body) if err != nil { return TorrentInfo{}, err } err = utils.Json.Unmarshal(body, &infos) if err != nil { return TorrentInfo{}, err } if len(infos) != 1 { return TorrentInfo{}, NewInfoNotFoundError(id) } return infos[0], nil } type FileInfo struct { Index int `json:"index"` Name string `json:"name"` Size int64 `json:"size"` Progress float32 `json:"progress"` Priority int `json:"priority"` IsSeed bool `json:"is_seed"` PieceRange []int `json:"piece_range"` Availability float32 `json:"availability"` } func (c *client) GetFiles(id string) ([]FileInfo, error) { var infos []FileInfo err := c.checkAuthorization() if err != nil { return []FileInfo{}, err } tInfo, err := c.GetInfo(id) if err != nil { return []FileInfo{}, err } v := url.Values{} v.Set("hash", tInfo.Hash) response, err := c.post("/api/v2/torrents/files", v) if err != nil { return []FileInfo{}, err } defer response.Body.Close() body, err := io.ReadAll(response.Body) if err != nil { return []FileInfo{}, err } err = utils.Json.Unmarshal(body, &infos) if err != nil { return []FileInfo{}, err } return infos, nil } func (c *client) Delete(id string, deleteFiles bool) error { err := c.checkAuthorization() if err != nil { return err } info, err := c.GetInfo(id) if err != nil { return err } v := url.Values{} v.Set("hashes", info.Hash) if deleteFiles { v.Set("deleteFiles", "true") } else { v.Set("deleteFiles", "false") } deleteResp, err := c.post("/api/v2/torrents/delete", v) if err != nil { return err } defer deleteResp.Body.Close() if deleteResp.StatusCode != 200 { return errors.New("failed to delete qbittorrent task") } v = url.Values{} v.Set("tags", "openlist-"+id) deleteTagsResp, err := c.post("/api/v2/torrents/deleteTags", v) if err != nil { return err } defer deleteTagsResp.Body.Close() if deleteTagsResp.StatusCode != 200 { return errors.New("failed to delete qbittorrent tag") } return nil } ================================================ FILE: pkg/sign/hmac.go ================================================ package sign import ( "crypto/hmac" "crypto/sha256" "encoding/base64" "io" "strconv" "strings" "time" ) type HMACSign struct { SecretKey []byte } func (s HMACSign) Sign(data string, expire int64) string { h := hmac.New(sha256.New, s.SecretKey) expireTimeStamp := strconv.FormatInt(expire, 10) _, err := io.WriteString(h, data+":"+expireTimeStamp) if err != nil { return "" } return base64.URLEncoding.EncodeToString(h.Sum(nil)) + ":" + expireTimeStamp } func (s HMACSign) Verify(data, sign string) error { signSlice := strings.Split(sign, ":") // check whether contains expire time if signSlice[len(signSlice)-1] == "" { return ErrExpireMissing } // check whether expire time is expired expires, err := strconv.ParseInt(signSlice[len(signSlice)-1], 10, 64) if err != nil { return ErrExpireInvalid } // if expire time is expired, return error if expires < time.Now().Unix() && expires != 0 { return ErrSignExpired } // verify sign if s.Sign(data, expires) != sign { return ErrSignInvalid } return nil } func NewHMACSign(secret []byte) Sign { return HMACSign{SecretKey: secret} } ================================================ FILE: pkg/sign/sign.go ================================================ package sign import "errors" type Sign interface { Sign(data string, expire int64) string Verify(data, sign string) error } var ( ErrSignExpired = errors.New("sign expired") ErrSignInvalid = errors.New("sign invalid") ErrExpireInvalid = errors.New("expire invalid") ErrExpireMissing = errors.New("expire missing") ) ================================================ FILE: pkg/singleflight/signleflight_test.go ================================================ // Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package singleflight // import "golang.org/x/sync/singleflight" import ( "bytes" "errors" "fmt" "os" "os/exec" "runtime" "runtime/debug" "strings" "sync" "sync/atomic" "testing" "time" ) type errValue struct{} func (err *errValue) Error() string { return "error value" } func TestPanicErrorUnwrap(t *testing.T) { t.Parallel() testCases := []struct { name string panicValue any wrappedErrorType bool }{ { name: "panicError wraps non-error type", panicValue: &panicError{value: "string value"}, wrappedErrorType: false, }, { name: "panicError wraps error type", panicValue: &panicError{value: new(errValue)}, wrappedErrorType: false, }, } for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() var recovered any group := &Group[any]{} func() { defer func() { recovered = recover() t.Logf("after panic(%#v) in group.Do, recovered %#v", tc.panicValue, recovered) }() _, _, _ = group.Do(tc.name, func() (any, error) { panic(tc.panicValue) }) }() if recovered == nil { t.Fatal("expected a non-nil panic value") } err, ok := recovered.(error) if !ok { t.Fatalf("recovered non-error type: %T", recovered) } if !errors.Is(err, new(errValue)) && tc.wrappedErrorType { t.Errorf("unexpected wrapped error type %T; want %T", err, new(errValue)) } }) } } func TestDo(t *testing.T) { var g Group[string] v, err, _ := g.Do("key", func() (string, error) { return "bar", nil }) if got, want := fmt.Sprintf("%v (%T)", v, v), "bar (string)"; got != want { t.Errorf("Do = %v; want %v", got, want) } if err != nil { t.Errorf("Do error = %v", err) } } func TestDoErr(t *testing.T) { var g Group[any] someErr := errors.New("Some error") v, err, _ := g.Do("key", func() (any, error) { return nil, someErr }) if err != someErr { t.Errorf("Do error = %v; want someErr %v", err, someErr) } if v != nil { t.Errorf("unexpected non-nil value %#v", v) } } func TestDoDupSuppress(t *testing.T) { var g Group[string] var wg1, wg2 sync.WaitGroup c := make(chan string, 1) var calls int32 fn := func() (string, error) { if atomic.AddInt32(&calls, 1) == 1 { // First invocation. wg1.Done() } v := <-c c <- v // pump; make available for any future calls time.Sleep(10 * time.Millisecond) // let more goroutines enter Do return v, nil } const n = 10 wg1.Add(1) for i := 0; i < n; i++ { wg1.Add(1) wg2.Add(1) go func() { defer wg2.Done() wg1.Done() v, err, _ := g.Do("key", fn) if err != nil { t.Errorf("Do error: %v", err) return } if v != "bar" { t.Errorf("Do = %T %v; want %q", v, v, "bar") } }() } wg1.Wait() // At least one goroutine is in fn now and all of them have at // least reached the line before the Do. c <- "bar" wg2.Wait() if got := atomic.LoadInt32(&calls); got <= 0 || got >= n { t.Errorf("number of calls = %d; want over 0 and less than %d", got, n) } } // Test that singleflight behaves correctly after Forget called. // See https://github.com/golang/go/issues/31420 func TestForget(t *testing.T) { var g Group[int] var ( firstStarted = make(chan struct{}) unblockFirst = make(chan struct{}) firstFinished = make(chan struct{}) ) go func() { g.Do("key", func() (i int, e error) { close(firstStarted) <-unblockFirst close(firstFinished) return }) }() <-firstStarted g.Forget("key") unblockSecond := make(chan struct{}) secondResult := g.DoChan("key", func() (i int, e error) { <-unblockSecond return 2, nil }) close(unblockFirst) <-firstFinished thirdResult := g.DoChan("key", func() (i int, e error) { return 3, nil }) close(unblockSecond) <-secondResult r := <-thirdResult if r.Val != 2 { t.Errorf("We should receive result produced by second call, expected: 2, got %d", r.Val) } } func TestDoChan(t *testing.T) { var g Group[string] ch := g.DoChan("key", func() (string, error) { return "bar", nil }) res := <-ch v := res.Val err := res.Err if got, want := fmt.Sprintf("%v (%T)", v, v), "bar (string)"; got != want { t.Errorf("Do = %v; want %v", got, want) } if err != nil { t.Errorf("Do error = %v", err) } } // Test singleflight behaves correctly after Do panic. // See https://github.com/golang/go/issues/41133 func TestPanicDo(t *testing.T) { var g Group[any] fn := func() (any, error) { panic("invalid memory address or nil pointer dereference") } const n = 5 waited := int32(n) panicCount := int32(0) done := make(chan struct{}) for i := 0; i < n; i++ { go func() { defer func() { if err := recover(); err != nil { t.Logf("Got panic: %v\n%s", err, debug.Stack()) atomic.AddInt32(&panicCount, 1) } if atomic.AddInt32(&waited, -1) == 0 { close(done) } }() g.Do("key", fn) }() } select { case <-done: if panicCount != n { t.Errorf("Expect %d panic, but got %d", n, panicCount) } case <-time.After(time.Second): t.Fatalf("Do hangs") } } func TestGoexitDo(t *testing.T) { var g Group[any] fn := func() (any, error) { runtime.Goexit() return nil, nil } const n = 5 waited := int32(n) done := make(chan struct{}) for i := 0; i < n; i++ { go func() { var err error defer func() { if err != nil { t.Errorf("Error should be nil, but got: %v", err) } if atomic.AddInt32(&waited, -1) == 0 { close(done) } }() _, err, _ = g.Do("key", fn) }() } select { case <-done: case <-time.After(time.Second): t.Fatalf("Do hangs") } } func executable(t testing.TB) string { exe, err := os.Executable() if err != nil { t.Skipf("skipping: test executable not found") } // Control case: check whether exec.Command works at all. // (For example, it might fail with a permission error on iOS.) cmd := exec.Command(exe, "-test.list=^$") cmd.Env = []string{} if err := cmd.Run(); err != nil { t.Skipf("skipping: exec appears not to work on %s: %v", runtime.GOOS, err) } return exe } func TestPanicDoChan(t *testing.T) { if os.Getenv("TEST_PANIC_DOCHAN") != "" { defer func() { recover() }() g := new(Group[any]) ch := g.DoChan("", func() (any, error) { panic("Panicking in DoChan") }) <-ch t.Fatalf("DoChan unexpectedly returned") } t.Parallel() cmd := exec.Command(executable(t), "-test.run="+t.Name(), "-test.v") cmd.Env = append(os.Environ(), "TEST_PANIC_DOCHAN=1") out := new(bytes.Buffer) cmd.Stdout = out cmd.Stderr = out if err := cmd.Start(); err != nil { t.Fatal(err) } err := cmd.Wait() t.Logf("%s:\n%s", strings.Join(cmd.Args, " "), out) if err == nil { t.Errorf("Test subprocess passed; want a crash due to panic in DoChan") } if bytes.Contains(out.Bytes(), []byte("DoChan unexpectedly")) { t.Errorf("Test subprocess failed with an unexpected failure mode.") } if !bytes.Contains(out.Bytes(), []byte("Panicking in DoChan")) { t.Errorf("Test subprocess failed, but the crash isn't caused by panicking in DoChan") } } func TestPanicDoSharedByDoChan(t *testing.T) { if os.Getenv("TEST_PANIC_DOCHAN") != "" { blocked := make(chan struct{}) unblock := make(chan struct{}) g := new(Group[any]) go func() { defer func() { recover() }() g.Do("", func() (any, error) { close(blocked) <-unblock panic("Panicking in Do") }) }() <-blocked ch := g.DoChan("", func() (any, error) { panic("DoChan unexpectedly executed callback") }) close(unblock) <-ch t.Fatalf("DoChan unexpectedly returned") } t.Parallel() cmd := exec.Command(executable(t), "-test.run="+t.Name(), "-test.v") cmd.Env = append(os.Environ(), "TEST_PANIC_DOCHAN=1") out := new(bytes.Buffer) cmd.Stdout = out cmd.Stderr = out if err := cmd.Start(); err != nil { t.Fatal(err) } err := cmd.Wait() t.Logf("%s:\n%s", strings.Join(cmd.Args, " "), out) if err == nil { t.Errorf("Test subprocess passed; want a crash due to panic in Do shared by DoChan") } if bytes.Contains(out.Bytes(), []byte("DoChan unexpectedly")) { t.Errorf("Test subprocess failed with an unexpected failure mode.") } if !bytes.Contains(out.Bytes(), []byte("Panicking in Do")) { t.Errorf("Test subprocess failed, but the crash isn't caused by panicking in Do") } } func ExampleGroup() { g := new(Group[string]) block := make(chan struct{}) res1c := g.DoChan("key", func() (string, error) { <-block return "func 1", nil }) res2c := g.DoChan("key", func() (string, error) { <-block return "func 2", nil }) close(block) res1 := <-res1c res2 := <-res2c // Results are shared by functions executed with duplicate keys. fmt.Println("Shared:", res2.Shared) // Only the first function is executed: it is registered and started with "key", // and doesn't complete before the second function is registered with a duplicate key. fmt.Println("Equal results:", res1.Val == res2.Val) fmt.Println("Result:", res1.Val) // Output: // Shared: true // Equal results: true // Result: func 1 } ================================================ FILE: pkg/singleflight/singleflight.go ================================================ // Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package singleflight provides a duplicate function call suppression // mechanism. package singleflight // import "golang.org/x/sync/singleflight" import ( "bytes" "errors" "fmt" "runtime" "runtime/debug" "sync" ) // errGoexit indicates the runtime.Goexit was called in // the user given function. var errGoexit = errors.New("runtime.Goexit was called") // A panicError is an arbitrary value recovered from a panic // with the stack trace during the execution of given function. type panicError struct { value any stack []byte } // Error implements error interface. func (p *panicError) Error() string { return fmt.Sprintf("%v\n\n%s", p.value, p.stack) } func (p *panicError) Unwrap() error { err, ok := p.value.(error) if !ok { return nil } return err } func newPanicError(v any) error { stack := debug.Stack() // The first line of the stack trace is of the form "goroutine N [status]:" // but by the time the panic reaches Do the goroutine may no longer exist // and its status will have changed. Trim out the misleading line. if line := bytes.IndexByte(stack[:], '\n'); line >= 0 { stack = stack[line+1:] } return &panicError{value: v, stack: stack} } // call is an in-flight or completed singleflight.Do call type call[T any] struct { wg sync.WaitGroup // These fields are written once before the WaitGroup is done // and are only read after the WaitGroup is done. val T err error // These fields are read and written with the singleflight // mutex held before the WaitGroup is done, and are read but // not written after the WaitGroup is done. dups int chans []chan<- Result[T] } // Group represents a class of work and forms a namespace in // which units of work can be executed with duplicate suppression. type Group[T any] struct { mu sync.Mutex // protects m m map[string]*call[T] // lazily initialized } // Result holds the results of Do, so they can be passed // on a channel. type Result[T any] struct { Val T Err error Shared bool } // Do executes and returns the results of the given function, making // sure that only one execution is in-flight for a given key at a // time. If a duplicate comes in, the duplicate caller waits for the // original to complete and receives the same results. // The return value shared indicates whether v was given to multiple callers. func (g *Group[T]) Do(key string, fn func() (T, error)) (v T, err error, shared bool) { g.mu.Lock() if g.m == nil { g.m = make(map[string]*call[T]) } if c, ok := g.m[key]; ok { c.dups++ g.mu.Unlock() c.wg.Wait() if e, ok := c.err.(*panicError); ok { panic(e) } else if c.err == errGoexit { runtime.Goexit() } return c.val, c.err, true } c := new(call[T]) c.wg.Add(1) g.m[key] = c g.mu.Unlock() g.doCall(c, key, fn) return c.val, c.err, c.dups > 0 } // DoChan is like Do but returns a channel that will receive the // results when they are ready. // // The returned channel will not be closed. func (g *Group[T]) DoChan(key string, fn func() (T, error)) <-chan Result[T] { ch := make(chan Result[T], 1) g.mu.Lock() if g.m == nil { g.m = make(map[string]*call[T]) } if c, ok := g.m[key]; ok { c.dups++ c.chans = append(c.chans, ch) g.mu.Unlock() return ch } c := &call[T]{chans: []chan<- Result[T]{ch}} c.wg.Add(1) g.m[key] = c g.mu.Unlock() go g.doCall(c, key, fn) return ch } // doCall handles the single call for a key. func (g *Group[T]) doCall(c *call[T], key string, fn func() (T, error)) { normalReturn := false recovered := false // use double-defer to distinguish panic from runtime.Goexit, // more details see https://golang.org/cl/134395 defer func() { // the given function invoked runtime.Goexit if !normalReturn && !recovered { c.err = errGoexit } g.mu.Lock() defer g.mu.Unlock() c.wg.Done() if g.m[key] == c { delete(g.m, key) } if e, ok := c.err.(*panicError); ok { // In order to prevent the waiting channels from being blocked forever, // needs to ensure that this panic cannot be recovered. if len(c.chans) > 0 { go panic(e) select {} // Keep this goroutine around so that it will appear in the crash dump. } else { panic(e) } } else if c.err == errGoexit { // Already in the process of goexit, no need to call again } else { // Normal return for _, ch := range c.chans { ch <- Result[T]{c.val, c.err, c.dups > 0} } } }() func() { defer func() { if !normalReturn { // Ideally, we would wait to take a stack trace until we've determined // whether this is a panic or a runtime.Goexit. // // Unfortunately, the only way we can distinguish the two is to see // whether the recover stopped the goroutine from terminating, and by // the time we know that, the part of the stack trace relevant to the // panic has been discarded. if r := recover(); r != nil { c.err = newPanicError(r) } } }() c.val, c.err = fn() normalReturn = true }() if !normalReturn { recovered = true } } // Forget tells the singleflight to forget about a key. Future calls // to Do for this key will call the function rather than waiting for // an earlier call to complete. func (g *Group[T]) Forget(key string) { g.mu.Lock() delete(g.m, key) g.mu.Unlock() } ================================================ FILE: pkg/singleflight/var.go ================================================ package singleflight var AnyGroup Group[any] ================================================ FILE: pkg/task/errors.go ================================================ package task import "errors" var ( ErrTaskNotFound = errors.New("task not found") ErrTaskRunning = errors.New("task is running") ) ================================================ FILE: pkg/task/manager.go ================================================ package task import ( "github.com/OpenListTeam/OpenList/v4/pkg/generic_sync" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) type Manager[K comparable] struct { curID K workerC chan struct{} updateID func(*K) tasks generic_sync.MapOf[K, *Task[K]] } func (tm *Manager[K]) Submit(task *Task[K]) K { if tm.updateID != nil { tm.updateID(&tm.curID) task.ID = tm.curID } tm.tasks.Store(task.ID, task) tm.do(task) return task.ID } func (tm *Manager[K]) do(task *Task[K]) { go func() { log.Debugf("task [%s] waiting for worker", task.Name) select { case <-tm.workerC: log.Debugf("task [%s] starting", task.Name) task.run() log.Debugf("task [%s] ended", task.Name) case <-task.Ctx.Done(): log.Debugf("task [%s] canceled", task.Name) return } // return worker tm.workerC <- struct{}{} }() } func (tm *Manager[K]) GetAll() []*Task[K] { return tm.tasks.Values() } func (tm *Manager[K]) Get(tid K) (*Task[K], bool) { return tm.tasks.Load(tid) } func (tm *Manager[K]) MustGet(tid K) *Task[K] { task, _ := tm.Get(tid) return task } func (tm *Manager[K]) Retry(tid K) error { t, ok := tm.Get(tid) if !ok { return errors.WithStack(ErrTaskNotFound) } tm.do(t) return nil } func (tm *Manager[K]) Cancel(tid K) error { t, ok := tm.Get(tid) if !ok { return errors.WithStack(ErrTaskNotFound) } t.Cancel() return nil } func (tm *Manager[K]) Remove(tid K) error { t, ok := tm.Get(tid) if !ok { return errors.WithStack(ErrTaskNotFound) } if !t.Done() { return errors.WithStack(ErrTaskRunning) } tm.tasks.Delete(tid) return nil } // RemoveAll removes all tasks from the manager, this maybe shouldn't be used // because the task maybe still running. func (tm *Manager[K]) RemoveAll() { tm.tasks.Clear() } func (tm *Manager[K]) RemoveByStates(states ...string) { tasks := tm.GetAll() for _, task := range tasks { if utils.SliceContains(states, task.GetState()) { _ = tm.Remove(task.ID) } } } func (tm *Manager[K]) GetByStates(states ...string) []*Task[K] { var tasks []*Task[K] tm.tasks.Range(func(key K, value *Task[K]) bool { if utils.SliceContains(states, value.GetState()) { tasks = append(tasks, value) } return true }) return tasks } func (tm *Manager[K]) ListUndone() []*Task[K] { return tm.GetByStates(PENDING, RUNNING, CANCELING) } func (tm *Manager[K]) ListDone() []*Task[K] { return tm.GetByStates(SUCCEEDED, CANCELED, ERRORED) } func (tm *Manager[K]) ClearDone() { tm.RemoveByStates(SUCCEEDED, CANCELED, ERRORED) } func (tm *Manager[K]) ClearSucceeded() { tm.RemoveByStates(SUCCEEDED) } func (tm *Manager[K]) RawTasks() *generic_sync.MapOf[K, *Task[K]] { return &tm.tasks } func NewTaskManager[K comparable](maxWorker int, updateID ...func(*K)) *Manager[K] { tm := &Manager[K]{ tasks: generic_sync.MapOf[K, *Task[K]]{}, workerC: make(chan struct{}, maxWorker), } for i := 0; i < maxWorker; i++ { tm.workerC <- struct{}{} } if len(updateID) > 0 { tm.updateID = updateID[0] } return tm } ================================================ FILE: pkg/task/task.go ================================================ // Package task manage task, such as file upload, file copy between storages, offline download, etc. package task import ( "context" "runtime" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) var ( PENDING = "pending" RUNNING = "running" SUCCEEDED = "succeeded" CANCELING = "canceling" CANCELED = "canceled" ERRORED = "errored" ) type Func[K comparable] func(task *Task[K]) error type Callback[K comparable] func(task *Task[K]) type Task[K comparable] struct { ID K Name string state string // pending, running, finished, canceling, canceled, errored status string progress float64 Error error Func Func[K] callback Callback[K] Ctx context.Context cancel context.CancelFunc } func (t *Task[K]) SetStatus(status string) { t.status = status } func (t *Task[K]) SetProgress(percentage float64) { t.progress = percentage } func (t Task[K]) GetProgress() float64 { return t.progress } func (t Task[K]) GetState() string { return t.state } func (t Task[K]) GetStatus() string { return t.status } func (t Task[K]) GetErrMsg() string { if t.Error == nil { return "" } return t.Error.Error() } func getCurrentGoroutineStack() string { buf := make([]byte, 1<<16) n := runtime.Stack(buf, false) return string(buf[:n]) } func (t *Task[K]) run() { t.state = RUNNING defer func() { if err := recover(); err != nil { log.Errorf("error [%s] while run task [%s],stack trace:\n%s", err, t.Name, getCurrentGoroutineStack()) t.Error = errors.Errorf("panic: %+v", err) t.state = ERRORED } }() t.Error = t.Func(t) if t.Error != nil { log.Errorf("error [%+v] while run task [%s]", t.Error, t.Name) } if errors.Is(t.Ctx.Err(), context.Canceled) { t.state = CANCELED } else if t.Error != nil { t.state = ERRORED } else { t.state = SUCCEEDED t.SetProgress(100) if t.callback != nil { t.callback(t) } } } func (t *Task[K]) retry() { t.run() } func (t *Task[K]) Done() bool { return t.state == SUCCEEDED || t.state == CANCELED || t.state == ERRORED } func (t *Task[K]) Cancel() { if t.state == SUCCEEDED || t.state == CANCELED { return } if t.cancel != nil { t.cancel() } // maybe can't cancel t.state = CANCELING } func WithCancelCtx[K comparable](task *Task[K]) *Task[K] { ctx, cancel := context.WithCancel(context.Background()) task.Ctx = ctx task.cancel = cancel task.state = PENDING return task } ================================================ FILE: pkg/task/task_test.go ================================================ package task import ( "sync/atomic" "testing" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" ) func TestTask_Manager(t *testing.T) { tm := NewTaskManager(3, func(id *uint64) { atomic.AddUint64(id, 1) }) id := tm.Submit(WithCancelCtx(&Task[uint64]{ Name: "test", Func: func(task *Task[uint64]) error { time.Sleep(time.Millisecond * 500) return nil }, })) task, ok := tm.Get(id) if !ok { t.Fatal("task not found") } time.Sleep(time.Millisecond * 100) if task.state != RUNNING { t.Errorf("task status not running: %s", task.state) } time.Sleep(time.Second) if task.state != SUCCEEDED { t.Errorf("task status not finished: %s", task.state) } } func TestTask_Cancel(t *testing.T) { tm := NewTaskManager(3, func(id *uint64) { atomic.AddUint64(id, 1) }) id := tm.Submit(WithCancelCtx(&Task[uint64]{ Name: "test", Func: func(task *Task[uint64]) error { for { if utils.IsCanceled(task.Ctx) { return nil } else { t.Logf("task is running") } } }, })) task, ok := tm.Get(id) if !ok { t.Fatal("task not found") } time.Sleep(time.Microsecond * 50) task.Cancel() time.Sleep(time.Millisecond) if task.state != CANCELED { t.Errorf("task status not canceled: %s", task.state) } } func TestTask_Retry(t *testing.T) { tm := NewTaskManager(3, func(id *uint64) { atomic.AddUint64(id, 1) }) num := 0 id := tm.Submit(WithCancelCtx(&Task[uint64]{ Name: "test", Func: func(task *Task[uint64]) error { num++ if num&1 == 1 { return errors.New("test error") } return nil }, })) task, ok := tm.Get(id) if !ok { t.Fatal("task not found") } time.Sleep(time.Millisecond) if task.Error == nil { t.Error(task.state) t.Fatal("task error is nil, but expected error") } else { t.Logf("task error: %s", task.Error) } task.retry() time.Sleep(time.Millisecond) if task.Error != nil { t.Errorf("task error: %+v, but expected nil", task.Error) } } ================================================ FILE: pkg/utils/balance.go ================================================ package utils import "strings" var balance = ".balance" func IsBalance(str string) bool { return strings.Contains(str, balance) } // GetActualMountPath remove balance suffix func GetActualMountPath(mountPath string) string { bIndex := strings.LastIndex(mountPath, ".balance") if bIndex != -1 { mountPath = mountPath[:bIndex] } return mountPath } ================================================ FILE: pkg/utils/bool.go ================================================ package utils func IsBool(bs ...bool) bool { return len(bs) > 0 && bs[0] } ================================================ FILE: pkg/utils/ctx.go ================================================ package utils import ( "context" ) func IsCanceled(ctx context.Context) bool { select { case <-ctx.Done(): return true default: return false } } ================================================ FILE: pkg/utils/email.go ================================================ package utils import "regexp" func IsEmailFormat(email string) bool { pattern := `^[0-9a-z][_.0-9a-z-]{0,31}@([0-9a-z][0-9a-z-]{0,30}[0-9a-z]\.){1,4}[a-z]{2,4}$` reg := regexp.MustCompile(pattern) return reg.MatchString(email) } ================================================ FILE: pkg/utils/file.go ================================================ package utils import ( "fmt" "io" "mime" "os" "path" "path/filepath" "strings" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/conf" log "github.com/sirupsen/logrus" ) // CopyFile File copies a single file from src to dst func CopyFile(src, dst string) error { var err error var srcfd *os.File var dstfd *os.File var srcinfo os.FileInfo if srcfd, err = os.Open(src); err != nil { return err } defer srcfd.Close() if dstfd, err = CreateNestedFile(dst); err != nil { return err } defer dstfd.Close() if _, err = CopyWithBuffer(dstfd, srcfd); err != nil { return err } if srcinfo, err = os.Stat(src); err != nil { return err } return os.Chmod(dst, srcinfo.Mode()) } // CopyDir Dir copies a whole directory recursively func CopyDir(src, dst string) error { var err error var fds []os.DirEntry var srcinfo os.FileInfo if srcinfo, err = os.Stat(src); err != nil { return err } if err = os.MkdirAll(dst, srcinfo.Mode()); err != nil { return err } if fds, err = os.ReadDir(src); err != nil { return err } for _, fd := range fds { srcfp := path.Join(src, fd.Name()) dstfp := path.Join(dst, fd.Name()) if fd.IsDir() { if err = CopyDir(srcfp, dstfp); err != nil { fmt.Println(err) } } else { if err = CopyFile(srcfp, dstfp); err != nil { fmt.Println(err) } } } return nil } // SymlinkOrCopyFile symlinks a file or copy if symlink failed func SymlinkOrCopyFile(src, dst string) error { if err := CreateNestedDirectory(filepath.Dir(dst)); err != nil { return err } if err := os.Symlink(src, dst); err != nil { return CopyFile(src, dst) } return nil } // Exists determine whether the file exists func Exists(name string) bool { if _, err := os.Stat(name); err != nil { if os.IsNotExist(err) { return false } } return true } // CreateNestedDirectory create nested directory func CreateNestedDirectory(path string) error { err := os.MkdirAll(path, 0700) if err != nil { log.Errorf("can't create folder, %s", err) } return err } // CreateNestedFile create nested file func CreateNestedFile(path string) (*os.File, error) { basePath := filepath.Dir(path) if err := CreateNestedDirectory(basePath); err != nil { return nil, err } return os.Create(path) } // CreateTempFile create temp file from io.ReadCloser, and seek to 0 func CreateTempFile(r io.Reader, size int64) (*os.File, error) { if f, ok := r.(*os.File); ok { return f, nil } f, err := os.CreateTemp(conf.Conf.TempDir, "file-*") if err != nil { return nil, err } readBytes, err := CopyWithBuffer(f, r) if err != nil { _ = os.Remove(f.Name()) return nil, errs.NewErr(err, "CreateTempFile failed") } if size > 0 && readBytes != size { _ = os.Remove(f.Name()) return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", readBytes, size) } _, err = f.Seek(0, io.SeekStart) if err != nil { _ = os.Remove(f.Name()) return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") } return f, nil } // GetFileType get file type func GetFileType(filename string) int { ext := strings.ToLower(Ext(filename)) if SliceContains(conf.SlicesMap[conf.AudioTypes], ext) { return conf.AUDIO } if SliceContains(conf.SlicesMap[conf.VideoTypes], ext) { return conf.VIDEO } if SliceContains(conf.SlicesMap[conf.ImageTypes], ext) { return conf.IMAGE } if SliceContains(conf.SlicesMap[conf.TextTypes], ext) { return conf.TEXT } return conf.UNKNOWN } func GetObjType(filename string, isDir bool) int { if isDir { return conf.FOLDER } return GetFileType(filename) } var extraMimeTypes = map[string]string{ ".apk": "application/vnd.android.package-archive", } func GetMimeType(name string) string { ext := path.Ext(name) if m, ok := extraMimeTypes[ext]; ok { return m } m := mime.TypeByExtension(ext) if m != "" { return m } return "application/octet-stream" } const ( KB = 1 << (10 * (iota + 1)) MB GB TB ) // IsSystemFile checks if a filename is a common system file that should be ignored // Returns true for files like .DS_Store, desktop.ini, Thumbs.db, and Apple Double files (._*) func IsSystemFile(filename string) bool { // Common system files switch filename { case ".DS_Store", "desktop.ini", "Thumbs.db", "@eaDir": return true } // Apple Double files (._*) if strings.HasPrefix(filename, "._") { return true } return false } ================================================ FILE: pkg/utils/file_test.go ================================================ package utils import ( "testing" ) func TestIsSystemFile(t *testing.T) { testCases := []struct { filename string expected bool }{ // System files that should be filtered {".DS_Store", true}, {"desktop.ini", true}, {"Thumbs.db", true}, {"._test.txt", true}, {"._", true}, {"._somefile", true}, {"._folder_name", true}, {"@eaDir", true}, // Regular files that should not be filtered {"test.txt", false}, {"file.pdf", false}, {"document.docx", false}, {".gitignore", false}, {".env", false}, {"_underscore.txt", false}, {"normal_file.txt", false}, {"", false}, {".hidden", false}, {"..special", false}, } for _, tc := range testCases { t.Run(tc.filename, func(t *testing.T) { result := IsSystemFile(tc.filename) if result != tc.expected { t.Errorf("IsSystemFile(%q) = %v, want %v", tc.filename, result, tc.expected) } }) } } ================================================ FILE: pkg/utils/hash/gcid.go ================================================ package hash_extend import ( "crypto/sha1" "encoding" "fmt" "hash" "strconv" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) var GCID = utils.RegisterHashWithParam("gcid", "GCID", 40, func(a ...any) hash.Hash { var ( size int64 err error ) if len(a) > 0 { size, err = strconv.ParseInt(fmt.Sprint(a[0]), 10, 64) if err != nil { panic(err) } } return NewGcid(size) }) func NewGcid(size int64) hash.Hash { calcBlockSize := func(j int64) int64 { var psize int64 = 0x40000 for float64(j)/float64(psize) > 0x200 && psize < 0x200000 { psize = psize << 1 } return psize } return &gcid{ hash: sha1.New(), hashState: sha1.New(), blockSize: int(calcBlockSize(size)), } } type gcid struct { hash hash.Hash hashState hash.Hash blockSize int offset int } func (h *gcid) Write(p []byte) (n int, err error) { n = len(p) for len(p) > 0 { if h.offset < h.blockSize { var lastSize = h.blockSize - h.offset if lastSize > len(p) { lastSize = len(p) } h.hashState.Write(p[:lastSize]) h.offset += lastSize p = p[lastSize:] } if h.offset >= h.blockSize { h.hash.Write(h.hashState.Sum(nil)) h.hashState.Reset() h.offset = 0 } } return } func (h *gcid) Sum(b []byte) []byte { if h.offset != 0 { if hashm, ok := h.hash.(encoding.BinaryMarshaler); ok { if hashum, ok := h.hash.(encoding.BinaryUnmarshaler); ok { tempData, _ := hashm.MarshalBinary() defer hashum.UnmarshalBinary(tempData) h.hash.Write(h.hashState.Sum(nil)) } } } return h.hash.Sum(b) } func (h *gcid) Reset() { h.hash.Reset() h.hashState.Reset() } func (h *gcid) Size() int { return h.hash.Size() } func (h *gcid) BlockSize() int { return h.blockSize } ================================================ FILE: pkg/utils/hash.go ================================================ package utils import ( "crypto/md5" "crypto/sha1" "crypto/sha256" "encoding" "encoding/hex" "encoding/json" "errors" "hash" "io" "iter" "github.com/OpenListTeam/OpenList/v4/internal/errs" log "github.com/sirupsen/logrus" ) func GetMD5EncodeStr(data string) string { return HashData(MD5, []byte(data)) } //inspired by "github.com/rclone/rclone/fs/hash" // ErrUnsupported should be returned by filesystem, // if it is requested to deliver an unsupported hash type. var ErrUnsupported = errors.New("hash type not supported") // HashType indicates a standard hashing algorithm type HashType struct { Width int Name string Alias string NewFunc func(...any) hash.Hash } func (ht *HashType) MarshalJSON() ([]byte, error) { return []byte(`"` + ht.Name + `"`), nil } func (ht *HashType) MarshalText() (text []byte, err error) { return []byte(ht.Name), nil } var ( _ json.Marshaler = (*HashType)(nil) //_ json.Unmarshaler = (*HashType)(nil) // read/write from/to json keys _ encoding.TextMarshaler = (*HashType)(nil) //_ encoding.TextUnmarshaler = (*HashType)(nil) ) var ( name2hash = map[string]*HashType{} alias2hash = map[string]*HashType{} Supported []*HashType ) func GetHashByName(name string) (ht *HashType, ok bool) { ht, ok = name2hash[name] return } // RegisterHash adds a new Hash to the list and returns its Type func RegisterHash(name, alias string, width int, newFunc func() hash.Hash) *HashType { return RegisterHashWithParam(name, alias, width, func(a ...any) hash.Hash { return newFunc() }) } func RegisterHashWithParam(name, alias string, width int, newFunc func(...any) hash.Hash) *HashType { newType := &HashType{ Name: name, Alias: alias, Width: width, NewFunc: newFunc, } name2hash[name] = newType alias2hash[alias] = newType Supported = append(Supported, newType) return newType } var ( // MD5 indicates MD5 support MD5 = RegisterHash("md5", "MD5", 32, md5.New) // SHA1 indicates SHA-1 support SHA1 = RegisterHash("sha1", "SHA-1", 40, sha1.New) // SHA256 indicates SHA-256 support SHA256 = RegisterHash("sha256", "SHA-256", 64, sha256.New) ) // HashData get hash of one hashType func HashData(hashType *HashType, data []byte, params ...any) string { h := hashType.NewFunc(params...) h.Write(data) return hex.EncodeToString(h.Sum(nil)) } // HashReader get hash of one hashType from a reader func HashReader(hashType *HashType, reader io.Reader, params ...any) (string, error) { h := hashType.NewFunc(params...) _, err := CopyWithBuffer(h, reader) if err != nil { return "", errs.NewErr(err, "HashReader error") } return hex.EncodeToString(h.Sum(nil)), nil } // HashFile get hash of one hashType from a model.File func HashFile(hashType *HashType, file io.ReadSeeker, params ...any) (string, error) { str, err := HashReader(hashType, file, params...) if err != nil { return "", err } if _, err = file.Seek(0, io.SeekStart); err != nil { return str, err } return str, nil } // fromTypes will return hashers for all the requested types. func fromTypes(types []*HashType) map[*HashType]hash.Hash { hashers := map[*HashType]hash.Hash{} for _, t := range types { hashers[t] = t.NewFunc() } return hashers } // toMultiWriter will return a set of hashers into a // single multiwriter, where one write will update all // the hashers. func toMultiWriter(h map[*HashType]hash.Hash) io.Writer { // Convert to to slice var w = make([]io.Writer, 0, len(h)) for _, v := range h { w = append(w, v) } return io.MultiWriter(w...) } // A MultiHasher will construct various hashes on all incoming writes. type MultiHasher struct { w io.Writer size int64 h map[*HashType]hash.Hash // Hashes } // NewMultiHasher will return a hash writer that will write // the requested hash types. func NewMultiHasher(types []*HashType) *MultiHasher { hashers := fromTypes(types) m := MultiHasher{h: hashers, w: toMultiWriter(hashers)} return &m } func (m *MultiHasher) Write(p []byte) (n int, err error) { n, err = m.w.Write(p) m.size += int64(n) return n, err } func (m *MultiHasher) GetHashInfo() *HashInfo { dst := make(map[*HashType]string) for k, v := range m.h { dst[k] = hex.EncodeToString(v.Sum(nil)) } return &HashInfo{h: dst} } // Sum returns the specified hash from the multihasher func (m *MultiHasher) Sum(hashType *HashType) ([]byte, error) { h, ok := m.h[hashType] if !ok { return nil, ErrUnsupported } return h.Sum(nil), nil } // Size returns the number of bytes written func (m *MultiHasher) Size() int64 { return m.size } // A HashInfo contains hash string for one or more hashType type HashInfo struct { h map[*HashType]string `json:"hashInfo"` } func NewHashInfoByMap(h map[*HashType]string) HashInfo { return HashInfo{h} } func NewHashInfo(ht *HashType, str string) HashInfo { m := make(map[*HashType]string) if ht != nil { m[ht] = str } return HashInfo{h: m} } func (hi HashInfo) String() string { result, err := json.Marshal(hi.h) if err != nil { return "" } return string(result) } func FromString(str string) HashInfo { hi := NewHashInfo(nil, "") var tmp map[string]string err := json.Unmarshal([]byte(str), &tmp) if err != nil { log.Warnf("failed to unmarsh HashInfo from string=%s", str) } else { for k, v := range tmp { if name2hash[k] != nil && len(v) > 0 { hi.h[name2hash[k]] = v } } } return hi } func (hi HashInfo) GetHash(ht *HashType) string { return hi.h[ht] } func (hi HashInfo) Export() map[*HashType]string { return hi.h } func (hi HashInfo) All() iter.Seq2[*HashType, string] { return func(yield func(*HashType, string) bool) { for hashType, hashValue := range hi.h { if !yield(hashType, hashValue) { return } } } } ================================================ FILE: pkg/utils/hash_test.go ================================================ package utils import ( "bytes" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "testing" ) type hashTest struct { input []byte output map[*HashType]string } var hashTestSet = []hashTest{ { input: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, output: map[*HashType]string{ MD5: "bf13fc19e5151ac57d4252e0e0f87abe", SHA1: "3ab6543c08a75f292a5ecedac87ec41642d12166", SHA256: "c839e57675862af5c21bd0a15413c3ec579e0d5522dab600bc6c3489b05b8f54", }, }, // Empty data set { input: []byte{}, output: map[*HashType]string{ MD5: "d41d8cd98f00b204e9800998ecf8427e", SHA1: "da39a3ee5e6b4b0d3255bfef95601890afd80709", SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, }, } func TestMultiHasher(t *testing.T) { for _, test := range hashTestSet { mh := NewMultiHasher([]*HashType{MD5, SHA1, SHA256}) n, err := CopyWithBuffer(mh, bytes.NewBuffer(test.input)) require.NoError(t, err) assert.Len(t, test.input, int(n)) hashInfo := mh.GetHashInfo() for k, v := range hashInfo.h { expect, ok := test.output[k] require.True(t, ok, "test output for hash not found") assert.Equal(t, expect, v) } // Test that all are present for k, v := range test.output { expect, ok := hashInfo.h[k] require.True(t, ok, "test output for hash not found") assert.Equal(t, expect, v) } for k, v := range test.output { expect := hashInfo.GetHash(k) require.True(t, len(expect) > 0, "test output for hash not found") assert.Equal(t, expect, v) } expect := hashInfo.GetHash(nil) require.True(t, len(expect) == 0, "unknown type should return empty string") str := hashInfo.String() Log.Info("str=" + str) newHi := FromString(str) assert.Equal(t, newHi.h, hashInfo.h) } } ================================================ FILE: pkg/utils/html.go ================================================ package utils import "github.com/microcosm-cc/bluemonday" var htmlSanitizePolicy = bluemonday.StrictPolicy() func SanitizeHTML(s string) string { return htmlSanitizePolicy.Sanitize(s) } ================================================ FILE: pkg/utils/http.go ================================================ package utils import ( "fmt" "net/url" "strings" ) // GenerateContentDisposition 生成符合RFC 5987标准的Content-Disposition头部 func GenerateContentDisposition(fileName string) string { // 按照RFC 2047进行编码，用于filename部分 encodedName := urlEncode(fileName) // 按照RFC 5987进行编码，用于filename*部分 encodedNameRFC5987 := encodeRFC5987(fileName) return fmt.Sprintf("attachment; filename=\"%s\"; filename*=utf-8''%s", encodedName, encodedNameRFC5987) } // encodeRFC5987 按照RFC 5987规范编码字符串，适用于HTTP头部参数中的非ASCII字符 func encodeRFC5987(s string) string { var buf strings.Builder for _, r := range []byte(s) { // 根据RFC 5987，只有字母、数字和部分特殊符号可以不编码 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '-' || r == '.' || r == '_' || r == '~' { buf.WriteByte(r) } else { // 其他字符都需要百分号编码 fmt.Fprintf(&buf, "%%%02X", r) } } return buf.String() } func urlEncode(s string) string { s = url.QueryEscape(s) s = strings.ReplaceAll(s, "+", "%20") return s } ================================================ FILE: pkg/utils/io.go ================================================ package utils import ( "bytes" "context" "errors" "fmt" "io" "math" "sync" "sync/atomic" "time" log "github.com/sirupsen/logrus" ) // here is some syntaxic sugar inspired by the Tomas Senart's video, // it allows me to inline the Reader interface type readerFunc func(p []byte) (n int, err error) func (rf readerFunc) Read(p []byte) (n int, err error) { return rf(p) } // CopyWithCtx slightly modified function signature: // - context has been added in order to propagate cancellation // - I do not return the number of bytes written, has it is not useful in my use case func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, progress func(percentage float64)) error { // Copy will call the Reader and Writer interface multiple time, in order // to copy by chunk (avoiding loading the whole file in memory). // I insert the ability to cancel before read time as it is the earliest // possible in the call process. var finish int64 = 0 s := size / 100 _, err := CopyWithBuffer(out, readerFunc(func(p []byte) (int, error) { // golang non-blocking channel: https://gobyexample.com/non-blocking-channel-operations select { // if context has been canceled case <-ctx.Done(): // stop process and propagate "context canceled" error return 0, ctx.Err() default: // otherwise just run default io.Reader implementation n, err := in.Read(p) if s > 0 && (err == nil || err == io.EOF) { finish += int64(n) progress(float64(finish) / float64(s)) } return n, err } })) return err } type limitWriter struct { w io.Writer limit int64 } func (l *limitWriter) Write(p []byte) (n int, err error) { lp := len(p) if l.limit > 0 { if int64(lp) > l.limit { p = p[:l.limit] } l.limit -= int64(len(p)) _, err = l.w.Write(p) } return lp, err } func LimitWriter(w io.Writer, limit int64) io.Writer { return &limitWriter{w: w, limit: limit} } type ReadCloser struct { io.Reader io.Closer } type CloseFunc func() error func (c CloseFunc) Close() error { return c() } func NewReadCloser(reader io.Reader, close CloseFunc) io.ReadCloser { return ReadCloser{ Reader: reader, Closer: close, } } func NewLimitReadCloser(reader io.Reader, close CloseFunc, limit int64) io.ReadCloser { return NewReadCloser(io.LimitReader(reader, limit), close) } type MultiReadable struct { originReader io.Reader reader io.Reader cache *bytes.Buffer } func NewMultiReadable(reader io.Reader) *MultiReadable { return &MultiReadable{ originReader: reader, reader: reader, } } func (mr *MultiReadable) Read(p []byte) (int, error) { n, err := mr.reader.Read(p) if _, ok := mr.reader.(io.Seeker); !ok && n > 0 { if mr.cache == nil { mr.cache = &bytes.Buffer{} } mr.cache.Write(p[:n]) } return n, err } func (mr *MultiReadable) Reset() error { if seeker, ok := mr.reader.(io.Seeker); ok { _, err := seeker.Seek(0, io.SeekStart) return err } if mr.cache != nil && mr.cache.Len() > 0 { mr.reader = io.MultiReader(mr.cache, mr.reader) mr.cache = nil } return nil } func (mr *MultiReadable) Close() error { if closer, ok := mr.originReader.(io.Closer); ok { return closer.Close() } return nil } func Retry(attempts int, sleep time.Duration, f func() error) (err error) { for i := 0; i < attempts; i++ { //fmt.Println("This is attempt number", i) if i > 0 { log.Println("retrying after error:", err) time.Sleep(sleep) sleep *= 2 } err = f() if err == nil { return nil } } return fmt.Errorf("after %d attempts, last error: %s", attempts, err) } type ClosersIF interface { io.Closer Add(closer io.Closer) AddIfCloser(a any) } type Closers []io.Closer func (c *Closers) Close() error { var errs []error for _, closer := range *c { if closer != nil { errs = append(errs, closer.Close()) } } clear(*c) *c = (*c)[:0] return errors.Join(errs...) } func (c *Closers) Add(closer io.Closer) { if closer != nil { *c = append(*c, closer) } } func (c *Closers) AddIfCloser(a any) { if closer, ok := a.(io.Closer); ok { *c = append(*c, closer) } } var _ ClosersIF = (*Closers)(nil) func NewClosers(c ...io.Closer) Closers { return Closers(c) } type SyncClosers struct { closers []io.Closer ref int32 } // if closed, return false func (c *SyncClosers) AcquireReference() bool { ref := atomic.AddInt32(&c.ref, 1) if ref > 0 { // log.Debugf("AcquireReference %p: %d", c, ref) return true } atomic.StoreInt32(&c.ref, closersClosed) return false } const closersClosed = math.MinInt32 func (c *SyncClosers) Close() error { for { ref := atomic.LoadInt32(&c.ref) if ref < 0 { return nil } if ref > 1 { if atomic.CompareAndSwapInt32(&c.ref, ref, ref-1) { // log.Debugf("ReleaseReference %p: %d", c, ref) return nil } } else if atomic.CompareAndSwapInt32(&c.ref, ref, closersClosed) { break } } // log.Debugf("FinalClose %p", c) var errs []error for _, closer := range c.closers { if closer != nil { errs = append(errs, closer.Close()) } } clear(c.closers) c.closers = nil return errors.Join(errs...) } func (c *SyncClosers) Add(closer io.Closer) { if closer != nil { if atomic.LoadInt32(&c.ref) < 0 { panic("Not reusable") } c.closers = append(c.closers, closer) } } func (c *SyncClosers) AddIfCloser(a any) { if closer, ok := a.(io.Closer); ok { if atomic.LoadInt32(&c.ref) < 0 { panic("Not reusable") } c.closers = append(c.closers, closer) } } var _ ClosersIF = (*SyncClosers)(nil) // 实现cache.Expirable接口 func (c *SyncClosers) Expired() bool { return atomic.LoadInt32(&c.ref) < 0 } func (c *SyncClosers) Length() int { return len(c.closers) } func NewSyncClosers(c ...io.Closer) SyncClosers { return SyncClosers{closers: c} } type Ordered interface { ~int | ~int8 | ~int16 | ~int32 | ~int64 | ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr | ~float32 | ~float64 | ~string } func Min[T Ordered](a, b T) T { if a < b { return a } return b } func Max[T Ordered](a, b T) T { if a < b { return b } return a } var IoBuffPool = &sync.Pool{ New: func() interface{} { return make([]byte, 32*1024*2) // Two times of size in io package }, } func CopyWithBuffer(dst io.Writer, src io.Reader) (written int64, err error) { buff := IoBuffPool.Get().([]byte) defer IoBuffPool.Put(buff) return io.CopyBuffer(dst, src, buff) } func CopyWithBufferN(dst io.Writer, src io.Reader, n int64) (written int64, err error) { written, err = CopyWithBuffer(dst, io.LimitReader(src, n)) if written == n { return n, nil } if written < n && err == nil { // src stopped early; must have been EOF. err = io.EOF } return } ================================================ FILE: pkg/utils/ip.go ================================================ package utils import ( "net" "net/http" "strings" ) func ClientIP(r *http.Request) string { xForwardedFor := r.Header.Get("X-Forwarded-For") ip := strings.TrimSpace(strings.Split(xForwardedFor, ",")[0]) if ip != "" { return ip } ip = strings.TrimSpace(r.Header.Get("X-Real-Ip")) if ip != "" { return ip } if ip, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr)); err == nil { return ip } return "" } func IsLocalIPAddr(ip string) bool { return IsLocalIP(net.ParseIP(ip)) } func IsLocalIP(ip net.IP) bool { if ip == nil { return false } if ip.IsLoopback() { return true } ip4 := ip.To4() if ip4 == nil { return false } return ip4[0] == 10 || // 10.0.0.0/8 (ip4[0] == 172 && ip4[1] >= 16 && ip4[1] <= 31) || // 172.16.0.0/12 (ip4[0] == 169 && ip4[1] == 254) || // 169.254.0.0/16 (ip4[0] == 192 && ip4[1] == 168) // 192.168.0.0/16 } ================================================ FILE: pkg/utils/json.go ================================================ package utils import ( stdjson "encoding/json" "os" json "github.com/json-iterator/go" log "github.com/sirupsen/logrus" ) var Json = json.ConfigCompatibleWithStandardLibrary // WriteJsonToFile write struct to json file func WriteJsonToFile(dst string, data interface{}, std ...bool) bool { str, err := json.MarshalIndent(data, "", " ") if len(std) > 0 && std[0] { str, err = stdjson.MarshalIndent(data, "", " ") } if err != nil { log.Errorf("failed convert Conf to []byte:%s", err.Error()) return false } err = os.WriteFile(dst, str, 0777) if err != nil { log.Errorf("failed to write json file:%s", err.Error()) return false } return true } ================================================ FILE: pkg/utils/log.go ================================================ package utils import ( log "github.com/sirupsen/logrus" ) var Log = log.New() ================================================ FILE: pkg/utils/map.go ================================================ package utils func MergeMap(mObj ...map[string]interface{}) map[string]interface{} { newObj := map[string]interface{}{} for _, m := range mObj { for k, v := range m { newObj[k] = v } } return newObj } ================================================ FILE: pkg/utils/oauth2.go ================================================ package utils import "golang.org/x/oauth2" type tokenSource struct { fn func() (*oauth2.Token, error) } func (t *tokenSource) Token() (*oauth2.Token, error) { return t.fn() } func TokenSource(fn func() (*oauth2.Token, error)) oauth2.TokenSource { return &tokenSource{fn} } ================================================ FILE: pkg/utils/path.go ================================================ package utils import ( "net/url" stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/errs" ) // FixAndCleanPath // The upper layer of the root directory is still the root directory. // So ".." And "." will be cleared // for example // 1. ".." or "." => "/" // 2. "../..." or "./..." => "/..." // 3. "../.x." or "./.x." => "/.x." // 4. "x//\\y" = > "/z/x" func FixAndCleanPath(path string) string { path = strings.ReplaceAll(path, "\\", "/") if !strings.HasPrefix(path, "/") { path = "/" + path } return stdpath.Clean(path) } // PathAddSeparatorSuffix Add path '/' suffix // for example /root => /root/ func PathAddSeparatorSuffix(path string) string { if !strings.HasSuffix(path, "/") { path = path + "/" } return path } // PathEqual judge path is equal func PathEqual(path1, path2 string) bool { return FixAndCleanPath(path1) == FixAndCleanPath(path2) } func IsSubPath(path string, subPath string) bool { path, subPath = FixAndCleanPath(path), FixAndCleanPath(subPath) return path == subPath || strings.HasPrefix(subPath, PathAddSeparatorSuffix(path)) } func Ext(path string) string { return strings.ToLower(SourceExt(path)) } func SourceExt(path string) string { ext := stdpath.Ext(path) if len(ext) > 0 && ext[0] == '.' { ext = ext[1:] } return ext } func EncodePath(path string, all ...bool) string { seg := strings.Split(path, "/") toReplace := []struct { Src string Dst string }{ {Src: "%", Dst: "%25"}, {"%", "%25"}, {"?", "%3F"}, {"#", "%23"}, } for i := range seg { if len(all) > 0 && all[0] { seg[i] = url.PathEscape(seg[i]) } else { for j := range toReplace { seg[i] = strings.ReplaceAll(seg[i], toReplace[j].Src, toReplace[j].Dst) } } } return strings.Join(seg, "/") } func JoinBasePath(basePath, reqPath string) (string, error) { isRelativePath := strings.Contains(reqPath, "..") reqPath = FixAndCleanPath(reqPath) if isRelativePath && !strings.Contains(reqPath, "..") { return "", errs.RelativePath } return stdpath.Join(FixAndCleanPath(basePath), reqPath), nil } func GetFullPath(mountPath, path string) string { return stdpath.Join(GetActualMountPath(mountPath), path) } // GetPathHierarchy generates a hierarchy of paths from the given path. // // Example: // 1. "/" => {"/"} // 2. "" => {"/"} // 3. "/a/b/c" => {"/", "/a", "/a/b", "/a/b/c"} // 4. "/a/b/c/d/e.txt" => {"/", "/a", "/a/b", "/a/b/c", "/a/b/c/d", "/a/b/c/d/e.txt"} // 5. "./a/b///c" => {"/", "/a", "/a/b", "/a/b/c"} func GetPathHierarchy(path string) []string { if path == "" || path == "/" { return []string{"/"} } path = FixAndCleanPath(path) if !strings.HasPrefix(path, "/") { path = "/" + path } hierarchy := []string{"/"} parts := strings.Split(path, "/") currentPath := "" for _, part := range parts { if part == "" { continue } currentPath += "/" + part hierarchy = append(hierarchy, currentPath) } return hierarchy } ================================================ FILE: pkg/utils/path_test.go ================================================ package utils import ( "reflect" "testing" ) func TestEncodePath(t *testing.T) { t.Log(EncodePath("http://localhost:5244/d/123#.png")) } func TestFixAndCleanPath(t *testing.T) { datas := map[string]string{ "": "/", ".././": "/", "../../.../": "/...", "x//\\y/": "/x/y", ".././.x/.y/.//..x../..y..": "/.x/.y/..x../..y..", } for key, value := range datas { if FixAndCleanPath(key) != value { t.Logf("raw %s fix fail", key) } } } func TestGetPathHierarchy(t *testing.T) { testCases := map[string][]string{ "": {"/"}, "/": {"/"}, "/home": {"/", "/home"}, "/home/user": {"/", "/home", "/home/user"}, "/home/user/documents": {"/", "/home", "/home/user", "/home/user/documents"}, "/home/user/documents/files/test.txt": {"/", "/home", "/home/user", "/home/user/documents", "/home/user/documents/files", "/home/user/documents/files/test.txt"}, "home": {"/", "/home"}, "home/user": {"/", "/home", "/home/user"}, "./home/": {"/", "/home"}, "..//home//user/../././": {"/", "/home"}, "/home///user///documents///": {"/", "/home", "/home/user", "/home/user/documents"}, "/home/user with spaces/doc": {"/", "/home", "/home/user with spaces", "/home/user with spaces/doc"}, "/home/user@domain.com/files": {"/", "/home", "/home/user@domain.com", "/home/user@domain.com/files"}, "/home/.hidden/.config": {"/", "/home", "/home/.hidden", "/home/.hidden/.config"}, } for input, expected := range testCases { t.Run(input, func(t *testing.T) { result := GetPathHierarchy(input) if !reflect.DeepEqual(result, expected) { t.Errorf("GetPathHierarchy(%q) = %v, want %v", input, result, expected) } }) } } ================================================ FILE: pkg/utils/random/random.go ================================================ package random import ( "crypto/rand" "math/big" mathRand "math/rand" "time" "github.com/google/uuid" ) var Rand *mathRand.Rand const letterBytes = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" func String(n int) string { b := make([]byte, n) letterLen := big.NewInt(int64(len(letterBytes))) for i := range b { idx, err := rand.Int(rand.Reader, letterLen) if err != nil { panic(err) } b[i] = letterBytes[idx.Int64()] } return string(b) } func Token() string { return "openlist-" + uuid.NewString() + String(64) } func RangeInt64(left, right int64) int64 { return mathRand.Int63n(left+right) - left } func init() { s := mathRand.NewSource(time.Now().UnixNano()) Rand = mathRand.New(s) } ================================================ FILE: pkg/utils/slice.go ================================================ package utils import ( "strings" "github.com/pkg/errors" ) // SliceEqual check if two slices are equal func SliceEqual[T comparable](a, b []T) bool { if len(a) != len(b) { return false } for i, v := range a { if v != b[i] { return false } } return true } // SliceContains check if slice contains element func SliceContains[T comparable](arr []T, v T) bool { for _, vv := range arr { if vv == v { return true } } return false } // SliceAllContains check if slice all contains elements func SliceAllContains[T comparable](arr []T, vs ...T) bool { vsMap := make(map[T]struct{}) for _, v := range arr { vsMap[v] = struct{}{} } for _, v := range vs { if _, ok := vsMap[v]; !ok { return false } } return true } // SliceConvert convert slice to another type slice func SliceConvert[S any, D any](srcS []S, convert func(src S) (D, error)) ([]D, error) { res := make([]D, 0, len(srcS)) for i := range srcS { dst, err := convert(srcS[i]) if err != nil { return nil, err } res = append(res, dst) } return res, nil } func MustSliceConvert[S any, D any](srcS []S, convert func(src S) D) []D { res := make([]D, 0, len(srcS)) for i := range srcS { dst := convert(srcS[i]) res = append(res, dst) } return res } func MergeErrors(errs ...error) error { errStr := strings.Join(MustSliceConvert(errs, func(err error) string { return err.Error() }), "\n") if errStr != "" { return errors.New(errStr) } return nil } func SliceMeet[T1, T2 any](arr []T1, v T2, meet func(item T1, v T2) bool) bool { for _, item := range arr { if meet(item, v) { return true } } return false } func SliceFilter[T any](arr []T, filter func(src T) bool) []T { res := make([]T, 0, len(arr)) for _, src := range arr { if filter(src) { res = append(res, src) } } return res } func SliceReplace[T any](arr []T, replace func(src T) T) { for i, src := range arr { arr[i] = replace(src) } } ================================================ FILE: pkg/utils/str.go ================================================ package utils import ( "encoding/base64" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" ) func MappingName(name string) string { for k, v := range conf.FilenameCharMap { name = strings.ReplaceAll(name, k, v) } return name } var DEC = map[string]string{ "-": "+", "_": "/", ".": "=", } func SafeAtob(data string) (string, error) { for k, v := range DEC { data = strings.ReplaceAll(data, k, v) } bytes, err := base64.StdEncoding.DecodeString(data) if err != nil { return "", err } return string(bytes), err } // GetNoneEmpty returns the first non-empty string, return empty if all empty func GetNoneEmpty(strArr ...string) string { for _, s := range strArr { if len(s) > 0 { return s } } return "" } ================================================ FILE: pkg/utils/time.go ================================================ package utils import ( "sync" "time" ) var CNLoc = time.FixedZone("UTC", 8*60*60) func MustParseCNTime(str string) time.Time { lastOpTime, _ := time.ParseInLocation("2006-01-02 15:04:05 -07", str+" +08", CNLoc) return lastOpTime } func NewDebounce(interval time.Duration) func(f func()) { var timer *time.Timer var lock sync.Mutex return func(f func()) { lock.Lock() defer lock.Unlock() if timer != nil { timer.Stop() } timer = time.AfterFunc(interval, f) } } func NewDebounce2(interval time.Duration, f func()) func() { var timer *time.Timer var lock sync.Mutex return func() { lock.Lock() defer lock.Unlock() if timer == nil { timer = time.AfterFunc(interval, f) } timer.Reset(interval) } } func NewThrottle(interval time.Duration) func(func()) { var lastCall time.Time var lock sync.Mutex return func(fn func()) { lock.Lock() defer lock.Unlock() now := time.Now() if now.Sub(lastCall) >= interval { lastCall = now go fn() } } } func NewThrottle2(interval time.Duration, fn func()) func() { var lastCall time.Time var lock sync.Mutex return func() { lock.Lock() defer lock.Unlock() now := time.Now() if now.Sub(lastCall) >= interval { lastCall = now go fn() } } } ================================================ FILE: pkg/utils/url.go ================================================ package utils import ( "net/url" ) func InjectQuery(raw string, query url.Values) (string, error) { param := query.Encode() if param == "" { return raw, nil } u, err := url.Parse(raw) if err != nil { return "", err } joiner := "?" if u.RawQuery != "" { joiner = "&" } return raw + joiner + param, nil } ================================================ FILE: public/public.go ================================================ package public import "embed" //go:embed all:dist var Public embed.FS ================================================ FILE: server/common/auth.go ================================================ package common import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/go-cache" "github.com/golang-jwt/jwt/v4" "github.com/pkg/errors" ) var SecretKey []byte type UserClaims struct { Username string `json:"username"` PwdTS int64 `json:"pwd_ts"` jwt.RegisteredClaims } var validTokenCache = cache.NewMemCache[bool]() func GenerateToken(user *model.User) (tokenString string, err error) { claim := UserClaims{ Username: user.Username, PwdTS: user.PwdTS, RegisteredClaims: jwt.RegisteredClaims{ ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Duration(conf.Conf.TokenExpiresIn) * time.Hour)), IssuedAt: jwt.NewNumericDate(time.Now()), NotBefore: jwt.NewNumericDate(time.Now()), }} token := jwt.NewWithClaims(jwt.SigningMethodHS256, claim) tokenString, err = token.SignedString(SecretKey) if err != nil { return "", err } validTokenCache.Set(tokenString, true) return tokenString, err } func ParseToken(tokenString string) (*UserClaims, error) { token, err := jwt.ParseWithClaims(tokenString, &UserClaims{}, func(token *jwt.Token) (interface{}, error) { return SecretKey, nil }) if IsTokenInvalidated(tokenString) { return nil, errors.New("token is invalidated") } if err != nil { if ve, ok := err.(*jwt.ValidationError); ok { if ve.Errors&jwt.ValidationErrorMalformed != 0 { return nil, errors.New("that's not even a token") } else if ve.Errors&jwt.ValidationErrorExpired != 0 { return nil, errors.New("token is expired") } else if ve.Errors&jwt.ValidationErrorNotValidYet != 0 { return nil, errors.New("token not active yet") } else { return nil, errors.New("couldn't handle this token") } } } if claims, ok := token.Claims.(*UserClaims); ok && token.Valid { return claims, nil } return nil, errors.New("couldn't handle this token") } func InvalidateToken(tokenString string) error { if tokenString == "" { return nil // don't invalidate empty guest token } validTokenCache.Del(tokenString) return nil } func IsTokenInvalidated(tokenString string) bool { _, ok := validTokenCache.Get(tokenString) return !ok } ================================================ FILE: server/common/base.go ================================================ package common import ( "context" "fmt" "net/http" stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" ) func GetApiUrlFromRequest(r *http.Request) string { api := conf.Conf.SiteURL if strings.HasPrefix(api, "http") { return strings.TrimSuffix(api, "/") } if r != nil { protocol := "http" if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" { protocol = "https" } host := r.Header.Get("X-Forwarded-Host") if host == "" { host = r.Host } api = fmt.Sprintf("%s://%s", protocol, stdpath.Join(host, api)) } api = strings.TrimSuffix(api, "/") return api } func GetApiUrl(ctx context.Context) string { api, _ := ctx.Value(conf.ApiUrlKey).(string) return api } ================================================ FILE: server/common/check.go ================================================ package common import ( "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/dlclark/regexp2" ) func IsStorageSignEnabled(rawPath string) bool { storage := op.GetBalancedStorage(rawPath) return storage != nil && storage.GetStorage().EnableSign } func CanWrite(meta *model.Meta, path string) bool { if meta == nil || !meta.Write { return false } return meta.WSub || meta.Path == path } func IsApply(metaPath, reqPath string, applySub bool) bool { if utils.PathEqual(metaPath, reqPath) { return true } return utils.IsSubPath(metaPath, reqPath) && applySub } func CanAccess(user *model.User, meta *model.Meta, reqPath string, password string) bool { // if the reqPath is in hide (only can check the nearest meta) and user can't see hides, can't access if meta != nil && !user.CanSeeHides() && meta.Hide != "" && IsApply(meta.Path, path.Dir(reqPath), meta.HSub) { // the meta should apply to the parent of current path for _, hide := range strings.Split(meta.Hide, "\n") { re := regexp2.MustCompile(hide, regexp2.None) if isMatch, _ := re.MatchString(path.Base(reqPath)); isMatch { return false } } } // if is not guest and can access without password if user.CanAccessWithoutPassword() { return true } // if meta is nil or password is empty, can access if meta == nil || meta.Password == "" { return true } // if meta doesn't apply to sub_folder, can access if !utils.PathEqual(meta.Path, reqPath) && !meta.PSub { return true } // validate password return meta.Password == password } // ShouldProxy TODO need optimize // when should be proxy? // 1. config.MustProxy() // 2. storage.WebProxy // 3. proxy_types func ShouldProxy(storage driver.Driver, filename string) bool { if storage.Config().MustProxy() || storage.GetStorage().WebProxy { return true } if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) { return true } return false } ================================================ FILE: server/common/check_test.go ================================================ package common import "testing" func TestIsApply(t *testing.T) { datas := []struct { metaPath string reqPath string applySub bool result bool }{ { metaPath: "/", reqPath: "/test", applySub: true, result: true, }, } for i, data := range datas { if IsApply(data.metaPath, data.reqPath, data.applySub) != data.result { t.Errorf("TestIsApply %d failed", i) } } } ================================================ FILE: server/common/common.go ================================================ package common import ( "context" "fmt" "html" "net/http" "strings" "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) func hidePrivacy(msg string) string { for _, r := range conf.PrivacyReg { msg = r.ReplaceAllStringFunc(msg, func(s string) string { return strings.Repeat("*", len(s)) }) } return msg } // ErrorResp is used to return error response // @param l: if true, log error func ErrorResp(c *gin.Context, err error, code int, l ...bool) { ErrorWithDataResp(c, err, code, nil, l...) //if len(l) > 0 && l[0] { // if flags.Debug || flags.Dev { // log.Errorf("%+v", err) // } else { // log.Errorf("%v", err) // } //} //c.JSON(200, Resp[interface{}]{ // Code: code, // Message: hidePrivacy(err.Error()), // Data: nil, //}) //c.Abort() } // ErrorPage is used to return error page HTML. // It also returns standard HTTP status code. // @param l: if true, log error func ErrorPage(c *gin.Context, err error, code int, l ...bool) { if len(l) > 0 && l[0] { if flags.Debug || flags.Dev { log.Errorf("%+v", err) } else { log.Errorf("%v", err) } } codes := fmt.Sprintf("%d %s", code, http.StatusText(code)) html := fmt.Sprintf(` %s

%s

%s

`, codes, codes, html.EscapeString(hidePrivacy(err.Error()))) c.Data(code, "text/html; charset=utf-8", []byte(html)) c.Abort() } func ErrorWithDataResp(c *gin.Context, err error, code int, data interface{}, l ...bool) { if len(l) > 0 && l[0] { if flags.Debug || flags.Dev { log.Errorf("%+v", err) } else { log.Errorf("%v", err) } } c.JSON(200, Resp[interface{}]{ Code: code, Message: hidePrivacy(err.Error()), Data: data, }) c.Abort() } func ErrorStrResp(c *gin.Context, str string, code int, l ...bool) { if len(l) != 0 && l[0] { log.Error(str) } c.JSON(200, Resp[interface{}]{ Code: code, Message: hidePrivacy(str), Data: nil, }) c.Abort() } func SuccessResp(c *gin.Context, data ...interface{}) { SuccessWithMsgResp(c, "success", data...) } func SuccessWithMsgResp(c *gin.Context, msg string, data ...interface{}) { var respData interface{} if len(data) > 0 { respData = data[0] } c.JSON(200, Resp[interface{}]{ Code: 200, Message: msg, Data: respData, }) } func Pluralize(count int, singular, plural string) string { if count == 1 { return singular } return plural } func GinWithValue(c *gin.Context, keyAndValue ...any) { c.Request = c.Request.WithContext( ContentWithValue(c.Request.Context(), keyAndValue...), ) } func ContentWithValue(ctx context.Context, keyAndValue ...any) context.Context { if len(keyAndValue) < 1 || len(keyAndValue)%2 != 0 { panic("keyAndValue must be an even number of arguments (key, value, ...)") } for len(keyAndValue) > 0 { ctx = context.WithValue(ctx, keyAndValue[0], keyAndValue[1]) keyAndValue = keyAndValue[2:] } return ctx } ================================================ FILE: server/common/hide_privacy_test.go ================================================ package common import ( "regexp" "testing" "github.com/OpenListTeam/OpenList/v4/internal/conf" ) func TestHidePrivacy(t *testing.T) { reg, err := regexp.Compile("(?U)access_token=(.*)&") if err != nil { t.Fatal(err) } conf.PrivacyReg = []*regexp.Regexp{reg} res := hidePrivacy(`Get "https://pan.baidu.com/rest/2.0/xpan/file?access_token=121.d1f66e95acfa40274920079396a51c48.Y2aP2vQDq90hLBE3PAbVije59uTcn7GiWUfw8LCM_olw&dir=%2F&limit=200&method=list&order=name&start=0&web=web " : net/http: TLS handshake timeout`) t.Log(res) } ================================================ FILE: server/common/ldap.go ================================================ package common import ( "crypto/tls" "fmt" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/ldap.v3" ) var ErrFailedLdapAuth = errors.New("failed to auth") func HandleLdapLogin(username, password string) error { // Auth start ldapServer := setting.GetStr(conf.LdapServer) skipTlsVerify := setting.GetBool(conf.LdapSkipTlsVerify) ldapManagerDN := setting.GetStr(conf.LdapManagerDN) ldapManagerPassword := setting.GetStr(conf.LdapManagerPassword) ldapUserSearchBase := setting.GetStr(conf.LdapUserSearchBase) ldapUserSearchFilter := setting.GetStr(conf.LdapUserSearchFilter) // (uid=%s) // Connect to LdapServer l, err := dial(ldapServer, skipTlsVerify) if err != nil { return errors.WithMessagef(err, "failed to connect to LDAP") } defer l.Close() // First bind with a read only user if ldapManagerDN != "" && ldapManagerPassword != "" { err = l.Bind(ldapManagerDN, ldapManagerPassword) if err != nil { return errors.WithMessagef(err, "failed to bind to LDAP") } } // Search for the given username searchRequest := ldap.NewSearchRequest( ldapUserSearchBase, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, fmt.Sprintf(ldapUserSearchFilter, ldap.EscapeFilter(username)), []string{"dn"}, nil, ) sr, err := l.Search(searchRequest) if err != nil { return errors.WithMessagef(err, "failed login ldap: LDAP search failed") } if len(sr.Entries) != 1 { return errors.New("failed login ldap: user does not exist or too many entries returned") } userDN := sr.Entries[0].DN // Bind as the user to verify their password err = l.Bind(userDN, password) if err != nil { return errors.WithMessagef(ErrFailedLdapAuth, "%v", err) } log.Infof("LDAP auth successful for %s", username) // Auth finished return nil } func LdapRegister(username string) (*model.User, error) { if username == "" { return nil, errors.New("cannot get username from ldap provider") } user := &model.User{ Username: username, Password: "", Authn: "[]", Permission: int32(setting.GetInt(conf.LdapDefaultPermission, 0)), BasePath: setting.GetStr(conf.LdapDefaultDir), Role: 0, Disabled: false, AllowLdap: true, } user.SetPassword(random.String(16)) if err := op.CreateUser(user); err != nil { return nil, err } return user, nil } func dial(ldapServer string, skipTlsVerify ...bool) (*ldap.Conn, error) { tlsEnabled := false if strings.HasPrefix(ldapServer, "ldaps://") { tlsEnabled = true ldapServer = strings.TrimPrefix(ldapServer, "ldaps://") } else if strings.HasPrefix(ldapServer, "ldap://") { ldapServer = strings.TrimPrefix(ldapServer, "ldap://") } if tlsEnabled { return ldap.DialTLS("tcp", ldapServer, &tls.Config{InsecureSkipVerify: utils.IsBool(skipTlsVerify...)}) } else { return ldap.Dial("tcp", ldapServer) } } ================================================ FILE: server/common/proxy.go ================================================ package common import ( "context" "fmt" "io" "net/http" "strings" "maps" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error { // if link.MFile != nil { // attachHeader(w, file, link) // http.ServeContent(w, r, file.GetName(), file.ModTime(), link.MFile) // return nil // } if link.Concurrency > 0 || link.PartSize > 0 { attachHeader(w, file, link) size := link.ContentLength if size <= 0 { size = file.GetSize() } rrf, _ := stream.GetRangeReaderFromLink(size, link) if link.RangeReader == nil { r = r.WithContext(context.WithValue(r.Context(), conf.RequestHeaderKey, r.Header)) } return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), size, &model.RangeReadCloser{ RangeReader: rrf, }) } if link.RangeReader != nil { attachHeader(w, file, link) size := link.ContentLength if size <= 0 { size = file.GetSize() } return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), size, &model.RangeReadCloser{ RangeReader: link.RangeReader, }) } //transparent proxy header := net.ProcessHeader(r.Header, link.Header) res, err := net.RequestHttp(r.Context(), r.Method, header, link.URL) if err != nil { return err } defer res.Body.Close() maps.Copy(w.Header(), res.Header) w.WriteHeader(res.StatusCode) if r.Method == http.MethodHead { return nil } _, err = utils.CopyWithBuffer(w, &stream.RateLimitReader{ Reader: res.Body, Limiter: stream.ServerDownloadLimit, Ctx: r.Context(), }) return err } func attachHeader(w http.ResponseWriter, file model.Obj, link *model.Link) { fileName := file.GetName() w.Header().Set("Content-Disposition", utils.GenerateContentDisposition(fileName)) w.Header().Set("Content-Type", utils.GetMimeType(fileName)) size := link.ContentLength if size <= 0 { size = file.GetSize() } w.Header().Set("Etag", GetEtag(file, size)) contentType := link.Header.Get("Content-Type") if len(contentType) > 0 { w.Header().Set("Content-Type", contentType) } else { w.Header().Set("Content-Type", utils.GetMimeType(fileName)) } } func GetEtag(file model.Obj, size int64) string { hash := "" for _, v := range file.GetHash().Export() { if v > hash { hash = v } } if len(hash) > 0 { return fmt.Sprintf(`"%s"`, hash) } // 参考nginx return fmt.Sprintf(`"%x-%x"`, file.ModTime().Unix(), size) } func ProxyRange(ctx context.Context, link *model.Link, size int64) *model.Link { if link.RangeReader == nil && !strings.HasPrefix(link.URL, GetApiUrl(ctx)+"/") { if link.ContentLength > 0 { size = link.ContentLength } rrf, err := stream.GetRangeReaderFromLink(size, link) if err == nil { return &model.Link{ RangeReader: rrf, ContentLength: size, } } } return link } type InterceptResponseWriter struct { http.ResponseWriter io.Writer } func (iw *InterceptResponseWriter) Write(p []byte) (int, error) { return iw.Writer.Write(p) } type WrittenResponseWriter struct { http.ResponseWriter written bool } func (ww *WrittenResponseWriter) Write(p []byte) (int, error) { n, err := ww.ResponseWriter.Write(p) if !ww.written && n > 0 { ww.written = true } return n, err } func (ww *WrittenResponseWriter) IsWritten() bool { return ww.written } func GenerateDownProxyURL(storage *model.Storage, reqPath string) string { if storage.DownProxyURL == "" { return "" } query := "" if !storage.DisableProxySign { query = "?sign=" + sign.Sign(reqPath) } return fmt.Sprintf("%s%s%s", strings.Split(storage.DownProxyURL, "\n")[0], utils.EncodePath(reqPath, true), query, ) } ================================================ FILE: server/common/resp.go ================================================ package common type Resp[T any] struct { Code int `json:"code"` Message string `json:"message"` Data T `json:"data"` } type PageResp struct { Content interface{} `json:"content"` Total int64 `json:"total"` } ================================================ FILE: server/common/sign.go ================================================ package common import ( stdpath "path" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/sign" ) func Sign(obj model.Obj, parent string, encrypt bool) string { if obj.IsDir() || (!encrypt && !setting.GetBool(conf.SignAll)) { return "" } return sign.Sign(stdpath.Join(parent, obj.GetName())) } ================================================ FILE: server/debug.go ================================================ package server import ( "net/http" _ "net/http/pprof" "runtime" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/server/middlewares" "github.com/gin-gonic/gin" ) func _pprof(g *gin.RouterGroup) { g.Any("/*name", gin.WrapH(http.DefaultServeMux)) } func debug(g *gin.RouterGroup) { g.GET("/path/*path", middlewares.Down(sign.Verify), func(c *gin.Context) { rawPath := c.Request.Context().Value(conf.PathKey).(string) c.JSON(200, gin.H{ "path": rawPath, }) }) g.GET("/hide_privacy", func(c *gin.Context) { common.ErrorStrResp(c, "This is ip: 1.1.1.1", 400) }) g.GET("/gc", func(c *gin.Context) { runtime.GC() c.String(http.StatusOK, "ok") }) _pprof(g.Group("/pprof")) } ================================================ FILE: server/ftp/afero.go ================================================ package ftp import ( "context" "errors" "fmt" "io" "os" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" ftpserver "github.com/fclairamb/ftpserverlib" "github.com/spf13/afero" ) type AferoAdapter struct { ctx context.Context nextFileSize int64 } func NewAferoAdapter(ctx context.Context) *AferoAdapter { return &AferoAdapter{ctx: ctx} } func (a *AferoAdapter) Create(_ string) (afero.File, error) { // See also GetHandle return nil, errs.NotImplement } func (a *AferoAdapter) Mkdir(name string, _ os.FileMode) error { return Mkdir(a.ctx, name) } func (a *AferoAdapter) MkdirAll(path string, perm os.FileMode) error { return a.Mkdir(path, perm) } func (a *AferoAdapter) Open(_ string) (afero.File, error) { // See also GetHandle and ReadDir return nil, errs.NotImplement } func (a *AferoAdapter) OpenFile(_ string, _ int, _ os.FileMode) (afero.File, error) { // See also GetHandle return nil, errs.NotImplement } func (a *AferoAdapter) Remove(name string) error { return Remove(a.ctx, name) } func (a *AferoAdapter) RemoveAll(path string) error { return a.Remove(path) } func (a *AferoAdapter) Rename(oldName, newName string) error { return Rename(a.ctx, oldName, newName) } func (a *AferoAdapter) Stat(name string) (os.FileInfo, error) { return Stat(a.ctx, name) } func (a *AferoAdapter) Name() string { return "OpenList FTP Endpoint" } func (a *AferoAdapter) Chmod(_ string, _ os.FileMode) error { return errs.NotSupport } func (a *AferoAdapter) Chown(_ string, _, _ int) error { return errs.NotSupport } func (a *AferoAdapter) Chtimes(_ string, _ time.Time, _ time.Time) error { return errs.NotSupport } func (a *AferoAdapter) ReadDir(name string) ([]os.FileInfo, error) { return List(a.ctx, name) } func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserver.FileTransfer, error) { fileSize := a.nextFileSize a.nextFileSize = 0 if (flags & os.O_SYNC) != 0 { return nil, errs.NotSupport } if (flags & os.O_APPEND) != 0 { return nil, errs.NotSupport } user := a.ctx.Value(conf.UserKey).(*model.User) path, err := user.JoinPath(name) if err != nil { return nil, err } if f, err := Borrow(a.ctx, path); !errors.Is(err, errs.ObjectNotFound) { if err != nil { return nil, err } if (flags & os.O_EXCL) != 0 { return nil, errs.ObjectAlreadyExists } if (flags & os.O_WRONLY) != 0 { return nil, errors.New("cannot write to uploading file") } _, err = f.Seek(offset, io.SeekStart) if err != nil { _ = f.Close() return nil, fmt.Errorf("failed seek borrow: %+v", err) } return f, nil } _, err = fs.Get(a.ctx, path, &fs.GetArgs{}) exists := err == nil if (flags&os.O_CREATE) == 0 && !exists { return nil, errs.ObjectNotFound } if (flags&os.O_EXCL) != 0 && exists { return nil, errs.ObjectAlreadyExists } if (flags & os.O_WRONLY) != 0 { if offset != 0 { return nil, errs.NotSupport } trunc := (flags & os.O_TRUNC) != 0 if fileSize > 0 { return OpenUploadWithLength(a.ctx, path, trunc, fileSize) } else { return OpenUpload(a.ctx, path, trunc) } } return OpenDownload(a.ctx, path, offset) } func (a *AferoAdapter) Site(param string) *ftpserver.AnswerCommand { spl := strings.SplitN(param, " ", 2) cmd := strings.ToUpper(spl[0]) var params string if len(spl) > 1 { params = spl[1] } else { params = "" } switch cmd { case "SIZE": code, msg := HandleSIZE(params, a) return &ftpserver.AnswerCommand{ Code: code, Message: msg, } } return nil } func (a *AferoAdapter) SetNextFileSize(size int64) { a.nextFileSize = size } ================================================ FILE: server/ftp/fsmanage.go ================================================ package ftp import ( "context" stdpath "path" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" ) func Mkdir(ctx context.Context, path string) error { user := ctx.Value(conf.UserKey).(*model.User) reqPath, err := user.JoinPath(path) if err != nil { return err } if !user.CanWrite() || !user.CanFTPManage() { meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { return err } } if !common.CanWrite(meta, reqPath) { return errs.PermissionDenied } } return fs.MakeDir(ctx, reqPath) } func Remove(ctx context.Context, path string) error { user := ctx.Value(conf.UserKey).(*model.User) if !user.CanRemove() || !user.CanFTPManage() { return errs.PermissionDenied } reqPath, err := user.JoinPath(path) if err != nil { return err } if err = RemoveStage(reqPath); !errors.Is(err, errs.ObjectNotFound) { return err } return fs.Remove(ctx, reqPath) } func Rename(ctx context.Context, oldPath, newPath string) error { user := ctx.Value(conf.UserKey).(*model.User) srcPath, err := user.JoinPath(oldPath) if err != nil { return err } dstPath, err := user.JoinPath(newPath) if err != nil { return err } srcDir, srcBase := stdpath.Split(srcPath) dstDir, dstBase := stdpath.Split(dstPath) if srcDir == dstDir { if !user.CanRename() || !user.CanFTPManage() { return errs.PermissionDenied } if err = MoveStage(srcPath, dstPath); !errors.Is(err, errs.ObjectNotFound) { return err } return fs.Rename(ctx, srcPath, dstBase) } else { if !user.CanFTPManage() || !user.CanMove() || (srcBase != dstBase && !user.CanRename()) { return errs.PermissionDenied } if err = MoveStage(srcPath, dstPath); !errors.Is(err, errs.ObjectNotFound) { return err } if srcBase != dstBase { err = fs.Rename(ctx, srcPath, dstBase, true) if err != nil { return err } } _, err = fs.Move(ctx, stdpath.Join(srcDir, dstBase), dstDir) return err } } ================================================ FILE: server/ftp/fsread.go ================================================ package ftp import ( "context" "io" fs2 "io/fs" "net/http" "os" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" ) type FileDownloadProxy struct { model.File io.Closer ctx context.Context } func OpenDownload(ctx context.Context, reqPath string, offset int64) (*FileDownloadProxy, error) { user := ctx.Value(conf.UserKey).(*model.User) meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { return nil, err } } ctx = context.WithValue(ctx, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, ctx.Value(conf.MetaPassKey).(string)) { return nil, errs.PermissionDenied } // directly use proxy header, _ := ctx.Value(conf.ProxyHeaderKey).(http.Header) ip, _ := ctx.Value(conf.ClientIPKey).(string) link, obj, err := fs.Link(ctx, reqPath, model.LinkArgs{IP: ip, Header: header}) if err != nil { return nil, err } ss, err := stream.NewSeekableStream(&stream.FileStream{ Obj: obj, Ctx: ctx, }, link) if err != nil { _ = link.Close() return nil, err } reader, err := stream.NewReadAtSeeker(ss, offset) if err != nil { _ = ss.Close() return nil, err } return &FileDownloadProxy{File: reader, Closer: ss, ctx: ctx}, nil } func (f *FileDownloadProxy) Read(p []byte) (n int, err error) { n, err = f.File.Read(p) if err != nil { return n, err } err = stream.ClientDownloadLimit.WaitN(f.ctx, n) return n, err } func (f *FileDownloadProxy) ReadAt(p []byte, off int64) (n int, err error) { n, err = f.File.ReadAt(p, off) if err != nil { return n, err } err = stream.ClientDownloadLimit.WaitN(f.ctx, n) return n, err } func (f *FileDownloadProxy) Write(p []byte) (n int, err error) { return 0, errs.NotSupport } type OsFileInfoAdapter struct { obj model.Obj } func (o *OsFileInfoAdapter) Name() string { return o.obj.GetName() } func (o *OsFileInfoAdapter) Size() int64 { return o.obj.GetSize() } func (o *OsFileInfoAdapter) Mode() fs2.FileMode { var mode fs2.FileMode = 0o755 if o.IsDir() { mode |= fs2.ModeDir } return mode } func (o *OsFileInfoAdapter) ModTime() time.Time { return o.obj.ModTime() } func (o *OsFileInfoAdapter) IsDir() bool { return o.obj.IsDir() } func (o *OsFileInfoAdapter) Sys() any { return o.obj } func Stat(ctx context.Context, path string) (os.FileInfo, error) { user := ctx.Value(conf.UserKey).(*model.User) reqPath, err := user.JoinPath(path) if err != nil { return nil, err } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { return nil, err } } ctx = context.WithValue(ctx, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, ctx.Value(conf.MetaPassKey).(string)) { return nil, errs.PermissionDenied } if ret, err := StatStage(reqPath); !errors.Is(err, errs.ObjectNotFound) { return ret, err } obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) if err != nil { return nil, err } return &OsFileInfoAdapter{obj: obj}, nil } func List(ctx context.Context, path string) ([]os.FileInfo, error) { user := ctx.Value(conf.UserKey).(*model.User) reqPath, err := user.JoinPath(path) if err != nil { return nil, err } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { return nil, err } } ctx = context.WithValue(ctx, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, ctx.Value(conf.MetaPassKey).(string)) { return nil, errs.PermissionDenied } objs, err := fs.List(ctx, reqPath, &fs.ListArgs{}) if err != nil { return nil, err } uploading := ListStage(reqPath) for _, o := range objs { delete(uploading, o.GetName()) } for _, u := range uploading { objs = append(objs, u) } ret := make([]os.FileInfo, len(objs)) for i, obj := range objs { ret[i] = &OsFileInfoAdapter{obj: obj} } return ret, nil } ================================================ FILE: server/ftp/fsup.go ================================================ package ftp import ( "bytes" "context" "fmt" "io" "net/http" "os" stdpath "path" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" ftpserver "github.com/fclairamb/ftpserverlib" "github.com/pkg/errors" ) type FileUploadProxy struct { ftpserver.FileTransfer buffer *os.File path string ctx context.Context trunc bool } func uploadAuth(ctx context.Context, path string) error { user := ctx.Value(conf.UserKey).(*model.User) meta, err := op.GetNearestMeta(stdpath.Dir(path)) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { return err } } if !(common.CanAccess(user, meta, path, ctx.Value(conf.MetaPassKey).(string)) && ((user.CanFTPManage() && user.CanWrite()) || common.CanWrite(meta, stdpath.Dir(path)))) { return errs.PermissionDenied } return nil } func OpenUpload(ctx context.Context, path string, trunc bool) (*FileUploadProxy, error) { err := uploadAuth(ctx, path) if err != nil { return nil, err } // Check if system file should be ignored _, name := stdpath.Split(path) if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(name) { return nil, errs.IgnoredSystemFile } tmpFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*") if err != nil { return nil, err } return &FileUploadProxy{buffer: tmpFile, path: path, ctx: ctx, trunc: trunc}, nil } func (f *FileUploadProxy) Read(p []byte) (n int, err error) { return 0, errs.NotSupport } func (f *FileUploadProxy) Write(p []byte) (n int, err error) { n, err = f.buffer.Write(p) if err != nil { return n, err } err = stream.ClientUploadLimit.WaitN(f.ctx, n) return n, err } func (f *FileUploadProxy) Seek(offset int64, whence int) (int64, error) { return f.buffer.Seek(offset, whence) } func (f *FileUploadProxy) Close() error { dir, name := stdpath.Split(f.path) size, err := f.buffer.Seek(0, io.SeekCurrent) if err != nil { return err } if _, err := f.buffer.Seek(0, io.SeekStart); err != nil { return err } arr := make([]byte, 512) if _, err := f.buffer.Read(arr); err != nil { return err } contentType := http.DetectContentType(arr) if _, err := f.buffer.Seek(0, io.SeekStart); err != nil { return err } user := f.ctx.Value(conf.UserKey).(*model.User) sf, borrow, err := MakeStage(f.ctx, f.buffer, size, f.path, func(target string) { ctx := context.WithValue(context.Background(), conf.UserKey, user) dstDir, dstBase := stdpath.Split(target) if dir == dstDir { _ = fs.Rename(ctx, f.path, dstBase) } else { if name != dstBase { e := fs.Rename(ctx, f.path, dstBase, true) if e != nil { return } } _, _ = fs.Move(ctx, stdpath.Join(dir, dstBase), dstDir) } }) if err != nil { return fmt.Errorf("failed make stage for [%s]: %+v", f.path, err) } if f.trunc { _ = fs.Remove(f.ctx, f.path) } s := &stream.FileStream{ Obj: &model.Object{ Name: name, Size: size, Modified: time.Now(), }, Mimetype: contentType, WebPutAsTask: true, Reader: f.buffer, } s.Add(borrow) task, err := fs.PutAsTask(f.ctx, dir, s) if err != nil { _ = s.Close() return err } sf.SetRemoveCallback(func() { fs.UploadTaskManager.Cancel(task.GetID()) }) return nil } type FileUploadWithLengthProxy struct { ftpserver.FileTransfer ctx context.Context path string length int64 first512Bytes [512]byte pFirst int pipeWriter io.WriteCloser errChan chan error } func OpenUploadWithLength(ctx context.Context, path string, trunc bool, length int64) (*FileUploadWithLengthProxy, error) { err := uploadAuth(ctx, path) if err != nil { return nil, err } // Check if system file should be ignored _, name := stdpath.Split(path) if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(name) { return nil, errs.IgnoredSystemFile } if trunc { _ = fs.Remove(ctx, path) } return &FileUploadWithLengthProxy{ctx: ctx, path: path, length: length}, nil } func (f *FileUploadWithLengthProxy) Read(p []byte) (n int, err error) { return 0, errs.NotSupport } func (f *FileUploadWithLengthProxy) write(p []byte) (n int, err error) { if f.pipeWriter != nil { select { case e := <-f.errChan: return 0, e default: return f.pipeWriter.Write(p) } } else if len(p) < 512-f.pFirst { copy(f.first512Bytes[f.pFirst:], p) f.pFirst += len(p) return len(p), nil } else { copy(f.first512Bytes[f.pFirst:], p[:512-f.pFirst]) contentType := http.DetectContentType(f.first512Bytes[:]) dir, name := stdpath.Split(f.path) reader, writer := io.Pipe() f.errChan = make(chan error, 1) s := &stream.FileStream{ Obj: &model.Object{ Name: name, Size: f.length, Modified: time.Now(), }, Mimetype: contentType, WebPutAsTask: false, Reader: reader, } go func() { e := fs.PutDirectly(f.ctx, dir, s, true) f.errChan <- e close(f.errChan) }() f.pipeWriter = writer n, err = writer.Write(f.first512Bytes[:]) if err != nil { return n, err } n1, err := writer.Write(p[512-f.pFirst:]) if err != nil { return n1 + 512 - f.pFirst, err } f.pFirst = 512 return len(p), nil } } func (f *FileUploadWithLengthProxy) Write(p []byte) (n int, err error) { n, err = f.write(p) if err != nil { return n, err } err = stream.ClientUploadLimit.WaitN(f.ctx, n) return n, err } func (f *FileUploadWithLengthProxy) Seek(offset int64, whence int) (int64, error) { return 0, errs.NotSupport } func (f *FileUploadWithLengthProxy) Close() error { if f.pipeWriter != nil { err := f.pipeWriter.Close() if err != nil { return err } err = <-f.errChan return err } else { data := f.first512Bytes[:f.pFirst] contentType := http.DetectContentType(data) dir, name := stdpath.Split(f.path) s := &stream.FileStream{ Obj: &model.Object{ Name: name, Size: int64(f.pFirst), Modified: time.Now(), }, Mimetype: contentType, WebPutAsTask: false, Reader: bytes.NewReader(data), } return fs.PutDirectly(f.ctx, dir, s) } } ================================================ FILE: server/ftp/site.go ================================================ package ftp import ( "fmt" "strconv" ftpserver "github.com/fclairamb/ftpserverlib" ) func HandleSIZE(param string, client ftpserver.ClientDriver) (int, string) { fs, ok := client.(*AferoAdapter) if !ok { return ftpserver.StatusNotLoggedIn, "Unexpected exception (driver is nil)" } size, err := strconv.ParseInt(param, 10, 64) if err != nil { return ftpserver.StatusSyntaxErrorParameters, fmt.Sprintf( "Couldn't parse file size, given: %s, err: %v", param, err) } fs.SetNextFileSize(size) return ftpserver.StatusOK, "Accepted next file size" } ================================================ FILE: server/ftp/upload_stage.go ================================================ package ftp import ( "context" "errors" "fmt" "os" "strings" "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" log "github.com/sirupsen/logrus" "github.com/tchap/go-patricia/v2/patricia" ) var ( stage *patricia.Trie stageMutex = sync.Mutex{} ErrStagePathConflict = errors.New("upload path conflict") ErrStageMoved = errors.New("uploading file has been moved") ) func InitStage() { if stage != nil { return } stage = patricia.NewTrie(patricia.MaxPrefixPerNode(16)) } type UploadingFile struct { name string size int64 modTime time.Time refCount int currentPath string softLinks []patricia.Prefix mvCallback func(string) rmCallback func() } func (u *UploadingFile) SetRemoveCallback(rm func()) { stageMutex.Lock() defer stageMutex.Unlock() u.rmCallback = rm } type softLink struct { target *UploadingFile } func MakeStage(ctx context.Context, buffer *os.File, size int64, path string, mv func(string)) (*UploadingFile, *BorrowedFile, error) { stageMutex.Lock() defer stageMutex.Unlock() prefix := patricia.Prefix(path) f := &UploadingFile{ name: buffer.Name(), size: size, modTime: time.Now(), refCount: 1, currentPath: path, softLinks: []patricia.Prefix{}, mvCallback: mv, } if !stage.Insert(prefix, f) { return nil, nil, ErrStagePathConflict } log.Debugf("[ftp-stage] succeed to make [%s] stage", buffer.Name()) return f, &BorrowedFile{ file: buffer, path: prefix, ctx: ctx, }, nil } func Borrow(ctx context.Context, path string) (*BorrowedFile, error) { stageMutex.Lock() defer stageMutex.Unlock() prefix := patricia.Prefix(path) v := stage.Get(prefix) if v == nil { return nil, errs.ObjectNotFound } s, ok := v.(*UploadingFile) if !ok { s = v.(*softLink).target } if s.currentPath != path { return nil, ErrStageMoved } borrowed, err := os.OpenFile(s.name, os.O_RDONLY, 0o644) if err != nil { return nil, fmt.Errorf("failed borrow [%s]: %+v", s.name, err) } s.refCount++ log.Debugf("[ftp-stage] borrow [%s] succeed", s.name) return &BorrowedFile{ file: borrowed, path: prefix, ctx: ctx, }, nil } func drop(path patricia.Prefix) { stageMutex.Lock() defer stageMutex.Unlock() v := stage.Get(path) if v == nil { return } s, ok := v.(*UploadingFile) if !ok { s = v.(*softLink).target } s.refCount-- log.Debugf("[ftp-stage] dropped [%s]", s.name) if s.refCount == 0 { log.Debugf("[ftp-stage] there is no more reference to [%s], removing temp file", s.name) err := os.RemoveAll(s.name) if err != nil { log.Errorf("[ftp-stage] failed to remove stage file [%s]: %+v", s.name, err) } for _, sl := range s.softLinks { stage.Delete(sl) } stage.Delete(path) if s.currentPath != string(path) { if s.currentPath != "" { go s.mvCallback(s.currentPath) } } } } func ListStage(path string) map[string]model.Obj { stageMutex.Lock() defer stageMutex.Unlock() path = path + "/" prefix := patricia.Prefix(path) ret := make(map[string]model.Obj) _ = stage.VisitSubtree(prefix, func(prefix patricia.Prefix, item patricia.Item) error { visit := string(prefix) visitSub := strings.TrimPrefix(visit, path) name, _, nonDirect := strings.Cut(visitSub, "/") if nonDirect { return nil } f, ok := item.(*UploadingFile) if !ok { f = item.(*softLink).target } if f.currentPath == visit { ret[name] = &model.Object{ Path: visit, Name: name, Size: f.size, Modified: f.modTime, IsFolder: false, } } return nil }) return ret } func StatStage(path string) (os.FileInfo, error) { stageMutex.Lock() defer stageMutex.Unlock() prefix := patricia.Prefix(path) v := stage.Get(prefix) if v == nil { return nil, errs.ObjectNotFound } s, ok := v.(*UploadingFile) if !ok { s = v.(*softLink).target } if s.currentPath != path { return nil, ErrStageMoved } return os.Stat(s.name) } func MoveStage(from, to string) error { stageMutex.Lock() defer stageMutex.Unlock() prefix := patricia.Prefix(from) v := stage.Get(prefix) if v == nil { return errs.ObjectNotFound } s, ok := v.(*UploadingFile) if !ok { s = v.(*softLink).target } if s.currentPath != from { return ErrStageMoved } slPrefix := patricia.Prefix(to) sl := &softLink{target: s} if !stage.Insert(slPrefix, sl) { return ErrStagePathConflict } s.currentPath = to s.softLinks = append(s.softLinks, slPrefix) return nil } func RemoveStage(path string) error { stageMutex.Lock() defer stageMutex.Unlock() prefix := patricia.Prefix(path) v := stage.Get(prefix) if v == nil { return errs.ObjectNotFound } s, ok := v.(*UploadingFile) if !ok { s = v.(*softLink).target } if s.currentPath != path { return ErrStageMoved } s.currentPath = "" if s.rmCallback != nil { s.rmCallback() } return nil } type BorrowedFile struct { file *os.File path patricia.Prefix ctx context.Context } func (f *BorrowedFile) Read(p []byte) (n int, err error) { n, err = f.file.Read(p) if err != nil { return n, err } err = stream.ClientDownloadLimit.WaitN(f.ctx, n) return n, err } func (f *BorrowedFile) ReadAt(p []byte, off int64) (n int, err error) { n, err = f.file.ReadAt(p, off) if err != nil { return n, err } err = stream.ClientDownloadLimit.WaitN(f.ctx, n) return n, err } func (f *BorrowedFile) Seek(offset int64, whence int) (int64, error) { return f.file.Seek(offset, whence) } func (f *BorrowedFile) Write(_ []byte) (n int, err error) { return 0, errs.NotSupport } func (f *BorrowedFile) Close() error { err := f.file.Close() drop(f.path) return err } ================================================ FILE: server/ftp.go ================================================ package server import ( "context" "crypto/tls" "errors" "fmt" "math/rand" "net" "net/http" "os" "strconv" "strings" "sync" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/server/ftp" ftpserver "github.com/fclairamb/ftpserverlib" ) type FtpMainDriver struct { settings *ftpserver.Settings proxyHeader http.Header clients map[uint32]ftpserver.ClientContext shutdownLock sync.RWMutex isShutdown bool tlsConfig *tls.Config } func NewMainDriver() (*FtpMainDriver, error) { ftp.InitStage() transferType := ftpserver.TransferTypeASCII if conf.Conf.FTP.DefaultTransferBinary { transferType = ftpserver.TransferTypeBinary } activeConnCheck := ftpserver.IPMatchDisabled if conf.Conf.FTP.EnableActiveConnIPCheck { activeConnCheck = ftpserver.IPMatchRequired } pasvConnCheck := ftpserver.IPMatchDisabled if conf.Conf.FTP.EnablePasvConnIPCheck { pasvConnCheck = ftpserver.IPMatchRequired } tlsRequired := ftpserver.ClearOrEncrypted if setting.GetBool(conf.FTPImplicitTLS) { tlsRequired = ftpserver.ImplicitEncryption } else if setting.GetBool(conf.FTPMandatoryTLS) { tlsRequired = ftpserver.MandatoryEncryption } tlsConf, err := getTlsConf(setting.GetStr(conf.FTPTLSPrivateKeyPath), setting.GetStr(conf.FTPTLSPublicCertPath)) if err != nil && tlsRequired != ftpserver.ClearOrEncrypted { return nil, fmt.Errorf("FTP mandatory TLS has been enabled, but the certificate failed to load: %w", err) } return &FtpMainDriver{ settings: &ftpserver.Settings{ ListenAddr: conf.Conf.FTP.Listen, PublicHost: lookupIP(setting.GetStr(conf.FTPPublicHost)), PassiveTransferPortRange: newPortMapper(setting.GetStr(conf.FTPPasvPortMap)), ActiveTransferPortNon20: conf.Conf.FTP.ActiveTransferPortNon20, IdleTimeout: conf.Conf.FTP.IdleTimeout, ConnectionTimeout: conf.Conf.FTP.ConnectionTimeout, DisableMLSD: false, DisableMLST: false, DisableMFMT: true, Banner: setting.GetStr(conf.Announcement), TLSRequired: tlsRequired, DisableLISTArgs: false, DisableSite: false, DisableActiveMode: conf.Conf.FTP.DisableActiveMode, EnableHASH: false, DisableSTAT: false, DisableSYST: false, EnableCOMB: false, DefaultTransferType: transferType, ActiveConnectionsCheck: activeConnCheck, PasvConnectionsCheck: pasvConnCheck, }, proxyHeader: http.Header{ "User-Agent": {base.UserAgent}, }, clients: make(map[uint32]ftpserver.ClientContext), shutdownLock: sync.RWMutex{}, isShutdown: false, tlsConfig: tlsConf, }, nil } func (d *FtpMainDriver) GetSettings() (*ftpserver.Settings, error) { return d.settings, nil } func (d *FtpMainDriver) ClientConnected(cc ftpserver.ClientContext) (string, error) { if d.isShutdown || !d.shutdownLock.TryRLock() { return "", errors.New("server has shutdown") } defer d.shutdownLock.RUnlock() d.clients[cc.ID()] = cc return "OpenList FTP Endpoint", nil } func (d *FtpMainDriver) ClientDisconnected(cc ftpserver.ClientContext) { err := cc.Close() if err != nil { utils.Log.Errorf("failed to close client: %v", err) } delete(d.clients, cc.ID()) } func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) (ftpserver.ClientDriver, error) { ip := cc.RemoteAddr().String() count, ok := model.LoginCache.Get(ip) if ok && count >= model.DefaultMaxAuthRetries { model.LoginCache.Expire(ip, model.DefaultLockDuration) return nil, errors.New("Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later.") } var userObj *model.User var err error if user == "anonymous" || user == "guest" { userObj, err = op.GetGuest() if err != nil { return nil, err } } else { userObj, err = op.GetUserByName(user) if err == nil { err = userObj.ValidateRawPassword(pass) if err != nil && setting.GetBool(conf.LdapLoginEnabled) && userObj.AllowLdap { err = common.HandleLdapLogin(user, pass) } } else if setting.GetBool(conf.LdapLoginEnabled) && model.CanFTPAccess(int32(setting.GetInt(conf.LdapDefaultPermission, 0))) { userObj, err = tryLdapLoginAndRegister(user, pass) } if err != nil { model.LoginCache.Set(ip, count+1) return nil, err } } if userObj.Disabled || !userObj.CanFTPAccess() { model.LoginCache.Set(ip, count+1) return nil, errors.New("user is not allowed to access via FTP") } model.LoginCache.Del(ip) ctx := context.Background() ctx = context.WithValue(ctx, conf.UserKey, userObj) if user == "anonymous" || user == "guest" { ctx = context.WithValue(ctx, conf.MetaPassKey, pass) } else { ctx = context.WithValue(ctx, conf.MetaPassKey, "") } ctx = context.WithValue(ctx, conf.ClientIPKey, ip) ctx = context.WithValue(ctx, conf.ProxyHeaderKey, d.proxyHeader) return ftp.NewAferoAdapter(ctx), nil } func (d *FtpMainDriver) GetTLSConfig() (*tls.Config, error) { if d.tlsConfig == nil { return nil, errors.New("TLS config not provided") } return d.tlsConfig, nil } func (d *FtpMainDriver) Stop() { d.isShutdown = true d.shutdownLock.Lock() defer d.shutdownLock.Unlock() for _, value := range d.clients { _ = value.Close() } } func lookupIP(host string) string { if host == "" || net.ParseIP(host) != nil { return host } ips, err := net.LookupIP(host) if err != nil || len(ips) == 0 { utils.Log.Errorf("given FTP public host is invalid, and the default value will be used: %v", err) return "" } for _, ip := range ips { if ip.To4() != nil { return ip.String() } } v6 := ips[0].String() utils.Log.Warnf("no IPv4 record looked up, %s will be used as public host, and it might do not work.", v6) return v6 } type group struct { ExposedStart int ListenedStart int Length int } type pasvPortGetter struct { groups []group totalLength int } func (m *pasvPortGetter) FetchNext() (int, int, bool) { idxPort := rand.Intn(m.totalLength) for _, g := range m.groups { if idxPort >= g.Length { idxPort -= g.Length } else { return g.ExposedStart + idxPort, g.ListenedStart + idxPort, true } } // unreachable return 0, 0, false } func (m *pasvPortGetter) NumberAttempts() int { return conf.Conf.FTP.FindPasvPortAttempts } func newPortMapper(str string) ftpserver.PasvPortGetter { if str == "" { return nil } pasvPortMappers := strings.Split(strings.Replace(str, "\n", ",", -1), ",") groups := make([]group, len(pasvPortMappers)) totalLength := 0 convertToPorts := func(str string) (int, int, error) { start, end, multi := strings.Cut(str, "-") if multi { si, err := strconv.Atoi(start) if err != nil { return 0, 0, err } ei, err := strconv.Atoi(end) if err != nil { return 0, 0, err } if ei < si || ei < 1024 || si < 1024 || ei > 65535 || si > 65535 { return 0, 0, errors.New("invalid port") } return si, ei - si + 1, nil } else { ret, err := strconv.Atoi(str) if err != nil { return 0, 0, err } else { return ret, 1, nil } } } for i, mapper := range pasvPortMappers { var err error exposed, listened, mapped := strings.Cut(mapper, ":") for { if mapped { var es, ls, el, ll int es, el, err = convertToPorts(exposed) if err != nil { break } ls, ll, err = convertToPorts(listened) if err != nil { break } if el != ll { err = errors.New("the number of exposed ports and listened ports does not match") break } groups[i].ExposedStart = es groups[i].ListenedStart = ls groups[i].Length = el totalLength += el } else { var start, length int start, length, err = convertToPorts(mapper) groups[i].ExposedStart = start groups[i].ListenedStart = start groups[i].Length = length totalLength += length } break } if err != nil { utils.Log.Errorf("failed to convert FTP PASV port mapper %s: %v, the port mapper will be ignored.", mapper, err) return nil } } return &pasvPortGetter{groups: groups, totalLength: totalLength} } func getTlsConf(keyPath, certPath string) (*tls.Config, error) { if keyPath == "" || certPath == "" { return nil, errors.New("private key or certificate is not provided") } cert, err := os.ReadFile(certPath) if err != nil { return nil, err } key, err := os.ReadFile(keyPath) if err != nil { return nil, err } tlsCert, err := tls.X509KeyPair(cert, key) if err != nil { return nil, err } return &tls.Config{Certificates: []tls.Certificate{tlsCert}}, nil } ================================================ FILE: server/handles/archive.go ================================================ package handles import ( "fmt" "io" stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) type ArchiveMetaReq struct { Path string `json:"path" form:"path"` Password string `json:"password" form:"password"` Refresh bool `json:"refresh" form:"refresh"` ArchivePass string `json:"archive_pass" form:"archive_pass"` } type ArchiveMetaResp struct { Comment string `json:"comment"` IsEncrypted bool `json:"encrypted"` Content []ArchiveContentResp `json:"content"` Sort *model.Sort `json:"sort,omitempty"` RawURL string `json:"raw_url"` Sign string `json:"sign"` } type ArchiveContentResp struct { ObjResp Children []ArchiveContentResp `json:"children"` } func toObjsRespWithoutSignAndThumb(obj model.Obj) ObjResp { return ObjResp{ Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), Modified: obj.ModTime(), Created: obj.CreateTime(), HashInfoStr: obj.GetHash().String(), HashInfo: obj.GetHash().Export(), Sign: "", Thumb: "", Type: utils.GetObjType(obj.GetName(), obj.IsDir()), } } func toContentResp(objs []model.ObjTree) []ArchiveContentResp { if objs == nil { return nil } ret, _ := utils.SliceConvert(objs, func(src model.ObjTree) (ArchiveContentResp, error) { return ArchiveContentResp{ ObjResp: toObjsRespWithoutSignAndThumb(src), Children: toContentResp(src.GetChildren()), }, nil }) return ret } func FsArchiveMetaSplit(c *gin.Context) { var req ArchiveMetaReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if strings.HasPrefix(req.Path, "/@s") { req.Path = strings.TrimPrefix(req.Path, "/@s") SharingArchiveMeta(c, &req) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() && user.Disabled { common.ErrorStrResp(c, "Guest user is disabled, login please", 401) return } FsArchiveMeta(c, &req, user) } func FsArchiveMeta(c *gin.Context, req *ArchiveMetaReq, user *model.User) { if !user.CanReadArchives() { common.ErrorResp(c, errs.PermissionDenied, 403) return } reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { common.ErrorResp(c, err, 500, true) return } } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } archiveArgs := model.ArchiveArgs{ LinkArgs: model.LinkArgs{ Header: c.Request.Header, Type: c.Query("type"), }, Password: req.ArchivePass, } ret, err := fs.ArchiveMeta(c.Request.Context(), reqPath, model.ArchiveMetaArgs{ ArchiveArgs: archiveArgs, Refresh: req.Refresh, }) if err != nil { if errors.Is(err, errs.WrongArchivePassword) { common.ErrorResp(c, err, 202) } else { common.ErrorResp(c, err, 500) } return } s := "" if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { s = sign.SignArchive(reqPath) } api := "/ae" if ret.DriverProviding { api = "/ad" } common.SuccessResp(c, ArchiveMetaResp{ Comment: ret.GetComment(), IsEncrypted: ret.IsEncrypted(), Content: toContentResp(ret.GetTree()), Sort: ret.Sort, RawURL: fmt.Sprintf("%s%s%s", common.GetApiUrl(c), api, utils.EncodePath(reqPath, true)), Sign: s, }) } type ArchiveListReq struct { ArchiveMetaReq model.PageReq InnerPath string `json:"inner_path" form:"inner_path"` } func FsArchiveListSplit(c *gin.Context) { var req ArchiveListReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } req.Validate() if strings.HasPrefix(req.Path, "/@s") { req.Path = strings.TrimPrefix(req.Path, "/@s") SharingArchiveList(c, &req) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() && user.Disabled { common.ErrorStrResp(c, "Guest user is disabled, login please", 401) return } FsArchiveList(c, &req, user) } func FsArchiveList(c *gin.Context, req *ArchiveListReq, user *model.User) { if !user.CanReadArchives() { common.ErrorResp(c, errs.PermissionDenied, 403) return } reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { common.ErrorResp(c, err, 500, true) return } } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } objs, err := fs.ArchiveList(c.Request.Context(), reqPath, model.ArchiveListArgs{ ArchiveInnerArgs: model.ArchiveInnerArgs{ ArchiveArgs: model.ArchiveArgs{ LinkArgs: model.LinkArgs{ Header: c.Request.Header, Type: c.Query("type"), }, Password: req.ArchivePass, }, InnerPath: utils.FixAndCleanPath(req.InnerPath), }, Refresh: req.Refresh, }) if err != nil { if errors.Is(err, errs.WrongArchivePassword) { common.ErrorResp(c, err, 202) } else { common.ErrorResp(c, err, 500) } return } total, objs := pagination(objs, &req.PageReq) ret, _ := utils.SliceConvert(objs, func(src model.Obj) (ObjResp, error) { return toObjsRespWithoutSignAndThumb(src), nil }) common.SuccessResp(c, common.PageResp{ Content: ret, Total: int64(total), }) } type ArchiveDecompressReq struct { SrcDir string `json:"src_dir" form:"src_dir"` DstDir string `json:"dst_dir" form:"dst_dir"` Names []string `json:"name" form:"name"` ArchivePass string `json:"archive_pass" form:"archive_pass"` InnerPath string `json:"inner_path" form:"inner_path"` CacheFull bool `json:"cache_full" form:"cache_full"` PutIntoNewDir bool `json:"put_into_new_dir" form:"put_into_new_dir"` Overwrite bool `json:"overwrite" form:"overwrite"` } func FsArchiveDecompress(c *gin.Context) { var req ArchiveDecompressReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if !user.CanDecompress() { common.ErrorResp(c, errs.PermissionDenied, 403) return } srcPaths := make([]string, 0, len(req.Names)) for _, name := range req.Names { srcPath, err := user.JoinPath(stdpath.Join(req.SrcDir, name)) if err != nil { common.ErrorResp(c, err, 403) return } srcPaths = append(srcPaths, srcPath) } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } tasks := make([]task.TaskExtensionInfo, 0, len(srcPaths)) for _, srcPath := range srcPaths { t, e := fs.ArchiveDecompress(c.Request.Context(), srcPath, dstDir, model.ArchiveDecompressArgs{ ArchiveInnerArgs: model.ArchiveInnerArgs{ ArchiveArgs: model.ArchiveArgs{ LinkArgs: model.LinkArgs{ Header: c.Request.Header, Type: c.Query("type"), }, Password: req.ArchivePass, }, InnerPath: utils.FixAndCleanPath(req.InnerPath), }, CacheFull: req.CacheFull, PutIntoNewDir: req.PutIntoNewDir, Overwrite: req.Overwrite, }) if e != nil { if errors.Is(e, errs.WrongArchivePassword) { common.ErrorResp(c, e, 202) } else { common.ErrorResp(c, e, 500) } return } if t != nil { tasks = append(tasks, t) } } common.SuccessResp(c, gin.H{ "task": getTaskInfos(tasks), }) } func ArchiveDown(c *gin.Context) { archiveRawPath := c.Request.Context().Value(conf.PathKey).(string) innerPath := utils.FixAndCleanPath(c.Query("inner")) password := c.Query("pass") filename := stdpath.Base(innerPath) storage, err := fs.GetStorage(archiveRawPath, &fs.GetStoragesArgs{}) if err != nil { common.ErrorPage(c, err, 500) return } if common.ShouldProxy(storage, filename) { ArchiveProxy(c) return } else { link, _, err := fs.ArchiveDriverExtract(c.Request.Context(), archiveRawPath, model.ArchiveInnerArgs{ ArchiveArgs: model.ArchiveArgs{ LinkArgs: model.LinkArgs{ IP: c.ClientIP(), Header: c.Request.Header, Type: c.Query("type"), Redirect: true, }, Password: password, }, InnerPath: innerPath, }) if err != nil { common.ErrorPage(c, err, 500) return } redirect(c, link) } } func ArchiveProxy(c *gin.Context) { archiveRawPath := c.Request.Context().Value(conf.PathKey).(string) innerPath := utils.FixAndCleanPath(c.Query("inner")) password := c.Query("pass") filename := stdpath.Base(innerPath) storage, err := fs.GetStorage(archiveRawPath, &fs.GetStoragesArgs{}) if err != nil { common.ErrorPage(c, err, 500) return } if canProxy(storage, filename) { // TODO: Support external download proxy URL link, file, err := fs.ArchiveDriverExtract(c.Request.Context(), archiveRawPath, model.ArchiveInnerArgs{ ArchiveArgs: model.ArchiveArgs{ LinkArgs: model.LinkArgs{ Header: c.Request.Header, Type: c.Query("type"), }, Password: password, }, InnerPath: innerPath, }) if err != nil { common.ErrorPage(c, err, 500) return } proxy(c, link, file, storage.GetStorage().ProxyRange) } else { common.ErrorPage(c, errors.New("proxy not allowed"), 403) return } } func proxyInternalExtract(c *gin.Context, rc io.ReadCloser, size int64, fileName string) { defer func() { if err := rc.Close(); err != nil { log.Errorf("failed to close file streamer, %v", err) } }() headers := map[string]string{ "Referrer-Policy": "no-referrer", "Cache-Control": "max-age=0, no-cache, no-store, must-revalidate", } headers["Content-Disposition"] = utils.GenerateContentDisposition(fileName) contentType := c.Request.Header.Get("Content-Type") if contentType == "" { contentType = utils.GetMimeType(fileName) } c.DataFromReader(200, size, contentType, rc, headers) } func ArchiveInternalExtract(c *gin.Context) { archiveRawPath := c.Request.Context().Value(conf.PathKey).(string) innerPath := utils.FixAndCleanPath(c.Query("inner")) password := c.Query("pass") rc, size, err := fs.ArchiveInternalExtract(c.Request.Context(), archiveRawPath, model.ArchiveInnerArgs{ ArchiveArgs: model.ArchiveArgs{ LinkArgs: model.LinkArgs{ Header: c.Request.Header, Type: c.Query("type"), }, Password: password, }, InnerPath: innerPath, }) if err != nil { common.ErrorPage(c, err, 500) return } fileName := stdpath.Base(innerPath) proxyInternalExtract(c, rc, size, fileName) } func ArchiveExtensions(c *gin.Context) { var ext []string for key := range tool.Tools { ext = append(ext, key) } common.SuccessResp(c, ext) } ================================================ FILE: server/handles/auth.go ================================================ package handles import ( "bytes" "encoding/base64" "image/png" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pquerna/otp/totp" ) type LoginReq struct { Username string `json:"username" binding:"required"` Password string `json:"password"` OtpCode string `json:"otp_code"` } // Login Deprecated func Login(c *gin.Context) { var req LoginReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } req.Password = model.StaticHash(req.Password) loginHash(c, &req) } // LoginHash login with password hashed by sha256 func LoginHash(c *gin.Context) { var req LoginReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } loginHash(c, &req) } func loginHash(c *gin.Context, req *LoginReq) { // check count of login ip := c.ClientIP() count, ok := model.LoginCache.Get(ip) if ok && count >= model.DefaultMaxAuthRetries { common.ErrorStrResp(c, model.TooManyAttempts, 429) model.LoginCache.Expire(ip, model.DefaultLockDuration) return } // check username user, err := op.GetUserByName(req.Username) if err != nil { common.ErrorStrResp(c, model.InvalidUsernameOrPassword, 401) model.LoginCache.Set(ip, count+1) return } // validate password hash if err := user.ValidatePwdStaticHash(req.Password); err != nil { common.ErrorStrResp(c, model.InvalidUsernameOrPassword, 401) model.LoginCache.Set(ip, count+1) return } // check 2FA if user.OtpSecret != "" { if !totp.Validate(req.OtpCode, user.OtpSecret) { // 402 - need opt common.ErrorStrResp(c, model.Invalid2FACode, 402) model.LoginCache.Set(ip, count+1) return } } // generate token token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c, gin.H{"token": token}) model.LoginCache.Del(ip) } type UserResp struct { model.User Otp bool `json:"otp"` } // CurrentUser get current user by token // if token is empty, return guest user func CurrentUser(c *gin.Context) { user := c.Request.Context().Value(conf.UserKey).(*model.User) userResp := UserResp{ User: *user, } userResp.Password = "" if userResp.OtpSecret != "" { userResp.Otp = true } common.SuccessResp(c, userResp) } func UpdateCurrent(c *gin.Context) { var req model.User if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { common.ErrorStrResp(c, model.GuestCannotUpdateProfile, 403) return } user.Username = req.Username if req.Password != "" { user.SetPassword(req.Password) } user.SsoID = req.SsoID if err := op.UpdateUser(user); err != nil { common.ErrorResp(c, err, 500) } else { common.SuccessResp(c) } } func Generate2FA(c *gin.Context) { user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { common.ErrorStrResp(c, model.GuestCannotGenerate2FA, 403) return } key, err := totp.Generate(totp.GenerateOpts{ Issuer: "OpenList", AccountName: user.Username, }) if err != nil { common.ErrorResp(c, err, 500) return } img, err := key.Image(400, 400) if err != nil { common.ErrorResp(c, err, 500) return } // to base64 var buf bytes.Buffer png.Encode(&buf, img) b64 := base64.StdEncoding.EncodeToString(buf.Bytes()) common.SuccessResp(c, gin.H{ "qr": "data:image/png;base64," + b64, "secret": key.Secret(), }) } type Verify2FAReq struct { Code string `json:"code" binding:"required"` Secret string `json:"secret" binding:"required"` } func Verify2FA(c *gin.Context) { var req Verify2FAReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { common.ErrorStrResp(c, model.GuestCannotGenerate2FA, 403) return } if !totp.Validate(req.Code, req.Secret) { common.ErrorStrResp(c, model.Invalid2FACode, 400) return } user.OtpSecret = req.Secret if err := op.UpdateUser(user); err != nil { common.ErrorResp(c, err, 500) } else { common.SuccessResp(c) } } func LogOut(c *gin.Context) { err := common.InvalidateToken(c.GetHeader("Authorization")) if err != nil { common.ErrorResp(c, err, 500) } else { common.SuccessResp(c) } } ================================================ FILE: server/handles/const.go ================================================ package handles const ( CANCEL = "cancel" OVERWRITE = "overwrite" SKIP = "skip" ) ================================================ FILE: server/handles/direct_upload.go ================================================ package handles import ( "net/url" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" ) type FsGetDirectUploadInfoReq struct { Path string `json:"path" form:"path"` FileName string `json:"file_name" form:"file_name"` FileSize int64 `json:"file_size" form:"file_size"` Tool string `json:"tool" form:"tool"` } // FsGetDirectUploadInfo returns the direct upload info if supported by the driver // If the driver does not support direct upload, returns null for upload_info func FsGetDirectUploadInfo(c *gin.Context) { var req FsGetDirectUploadInfoReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } // Decode path path, err := url.PathUnescape(req.Path) if err != nil { common.ErrorResp(c, err, 400) return } // Get user and join path user := c.Request.Context().Value(conf.UserKey).(*model.User) path, err = user.JoinPath(path) if err != nil { common.ErrorResp(c, err, 403) return } overwrite := c.GetHeader("Overwrite") != "false" if !overwrite { if res, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true}); res != nil { common.ErrorStrResp(c, "file exists", 403) return } } directUploadInfo, err := fs.GetDirectUploadInfo(c, req.Tool, path, req.FileName, req.FileSize) if err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, directUploadInfo) } ================================================ FILE: server/handles/down.go ================================================ package handles import ( "bytes" "errors" "fmt" stdpath "path" "strconv" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/microcosm-cc/bluemonday" log "github.com/sirupsen/logrus" "github.com/yuin/goldmark" ) func Down(c *gin.Context) { rawPath := c.Request.Context().Value(conf.PathKey).(string) filename := stdpath.Base(rawPath) storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{}) if err != nil { common.ErrorPage(c, err, 500) return } if common.ShouldProxy(storage, filename) { Proxy(c) return } else { link, _, err := fs.Link(c.Request.Context(), rawPath, model.LinkArgs{ IP: c.ClientIP(), Header: c.Request.Header, Type: c.Query("type"), Redirect: true, }) if err != nil { common.ErrorPage(c, err, 500) return } redirect(c, link) } } func Proxy(c *gin.Context) { rawPath := c.Request.Context().Value(conf.PathKey).(string) filename := stdpath.Base(rawPath) storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{}) if err != nil { common.ErrorPage(c, err, 500) return } if canProxy(storage, filename) { if _, ok := c.GetQuery("d"); !ok { if url := common.GenerateDownProxyURL(storage.GetStorage(), rawPath); url != "" { c.Redirect(302, url) return } } link, file, err := fs.Link(c.Request.Context(), rawPath, model.LinkArgs{ Header: c.Request.Header, Type: c.Query("type"), }) if err != nil { common.ErrorPage(c, err, 500) return } proxy(c, link, file, storage.GetStorage().ProxyRange) } else { common.ErrorPage(c, errors.New("proxy not allowed"), 403) return } } func redirect(c *gin.Context, link *model.Link) { defer link.Close() var err error c.Header("Referrer-Policy", "no-referrer") c.Header("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") if setting.GetBool(conf.ForwardDirectLinkParams) { query := c.Request.URL.Query() for _, v := range conf.SlicesMap[conf.IgnoreDirectLinkParams] { query.Del(v) } link.URL, err = utils.InjectQuery(link.URL, query) if err != nil { common.ErrorPage(c, err, 500) return } } c.Redirect(302, link.URL) } func proxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange bool) { defer link.Close() var err error if link.URL != "" && setting.GetBool(conf.ForwardDirectLinkParams) { query := c.Request.URL.Query() for _, v := range conf.SlicesMap[conf.IgnoreDirectLinkParams] { query.Del(v) } link.URL, err = utils.InjectQuery(link.URL, query) if err != nil { common.ErrorPage(c, err, 500) return } } if proxyRange { link = common.ProxyRange(c, link, file.GetSize()) } Writer := &common.WrittenResponseWriter{ResponseWriter: c.Writer} raw, _ := strconv.ParseBool(c.DefaultQuery("raw", "false")) if utils.Ext(file.GetName()) == "md" && setting.GetBool(conf.FilterReadMeScripts) && !raw { buf := bytes.NewBuffer(make([]byte, 0, file.GetSize())) w := &common.InterceptResponseWriter{ResponseWriter: Writer, Writer: buf} err = common.Proxy(w, c.Request, link, file) if err == nil && buf.Len() > 0 { if c.Writer.Status() < 200 || c.Writer.Status() > 300 { c.Writer.Write(buf.Bytes()) return } var html bytes.Buffer if err = goldmark.Convert(buf.Bytes(), &html); err != nil { err = fmt.Errorf("markdown conversion failed: %w", err) } else { buf.Reset() err = bluemonday.UGCPolicy().SanitizeReaderToWriter(&html, buf) if err == nil { Writer.Header().Set("Content-Length", strconv.FormatInt(int64(buf.Len()), 10)) Writer.Header().Set("Content-Type", "text/html; charset=utf-8") _, err = utils.CopyWithBuffer(Writer, buf) } } } } else { err = common.Proxy(Writer, c.Request, link, file) } if err == nil { return } if Writer.IsWritten() { log.Errorf("%s %s local proxy error: %+v", c.Request.Method, c.Request.URL.Path, err) } else { if statusCode, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { common.ErrorPage(c, err, int(statusCode), true) } else { common.ErrorPage(c, err, 500, true) } } } // TODO need optimize // when can be proxy? // 1. text file // 2. config.MustProxy() // 3. storage.WebProxy // 4. proxy_types // solution: text_file + shouldProxy() func canProxy(storage driver.Driver, filename string) bool { if storage.Config().MustProxy() || storage.GetStorage().WebProxy || storage.GetStorage().WebdavProxyURL() { return true } if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) { return true } if utils.SliceContains(conf.SlicesMap[conf.TextTypes], utils.Ext(filename)) { return true } return false } ================================================ FILE: server/handles/driver.go ================================================ package handles import ( "fmt" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" ) func ListDriverInfo(c *gin.Context) { common.SuccessResp(c, op.GetDriverInfoMap()) } func ListDriverNames(c *gin.Context) { common.SuccessResp(c, op.GetDriverNames()) } func GetDriverInfo(c *gin.Context) { driverName := c.Query("driver") infoMap := op.GetDriverInfoMap() items, ok := infoMap[driverName] if !ok { common.ErrorStrResp(c, fmt.Sprintf("driver [%s] not found", driverName), 404) return } common.SuccessResp(c, items) } ================================================ FILE: server/handles/fsbatch.go ================================================ package handles import ( "fmt" "regexp" "slices" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/generic" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" ) type RecursiveMoveReq struct { SrcDir string `json:"src_dir"` DstDir string `json:"dst_dir"` ConflictPolicy string `json:"conflict_policy"` } func FsRecursiveMove(c *gin.Context) { var req RecursiveMoveReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if !user.CanMove() { common.ErrorResp(c, errs.PermissionDenied, 403) return } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } meta, err := op.GetNearestMeta(srcDir) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { common.ErrorResp(c, err, 500, true) return } } common.GinWithValue(c, conf.MetaKey, meta) rootFiles, err := fs.List(c.Request.Context(), srcDir, &fs.ListArgs{}) if err != nil { common.ErrorResp(c, err, 500) return } var existingFileNames []string if req.ConflictPolicy != OVERWRITE { dstFiles, err := fs.List(c.Request.Context(), dstDir, &fs.ListArgs{}) if err != nil { common.ErrorResp(c, err, 500) return } existingFileNames = make([]string, 0, len(dstFiles)) for _, dstFile := range dstFiles { existingFileNames = append(existingFileNames, dstFile.GetName()) } } // record the file path filePathMap := make(map[model.Obj]string) movingFiles := generic.NewQueue[model.Obj]() movingFileNames := make([]string, 0, len(rootFiles)) for _, file := range rootFiles { movingFiles.Push(file) filePathMap[file] = srcDir } for !movingFiles.IsEmpty() { movingFile := movingFiles.Pop() movingFilePath := filePathMap[movingFile] movingFileName := fmt.Sprintf("%s/%s", movingFilePath, movingFile.GetName()) if movingFile.IsDir() { // directory, recursive move subFilePath := movingFileName subFiles, err := fs.List(c.Request.Context(), movingFileName, &fs.ListArgs{Refresh: true}) if err != nil { common.ErrorResp(c, err, 500) return } for _, subFile := range subFiles { movingFiles.Push(subFile) filePathMap[subFile] = subFilePath } } else { if movingFilePath == dstDir { // same directory, don't move continue } if slices.Contains(existingFileNames, movingFile.GetName()) { if req.ConflictPolicy == CANCEL { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", movingFile.GetName()), 403) return } else if req.ConflictPolicy == SKIP { continue } } else if req.ConflictPolicy != OVERWRITE { existingFileNames = append(existingFileNames, movingFile.GetName()) } movingFileNames = append(movingFileNames, movingFileName) } } var count = 0 for i, fileName := range movingFileNames { // move _, err := fs.Move(c.Request.Context(), fileName, dstDir, len(movingFileNames) > i+1) if err != nil { common.ErrorResp(c, err, 500) return } count++ } common.SuccessWithMsgResp(c, fmt.Sprintf("Successfully moved %d %s", count, common.Pluralize(count, "file", "files"))) } type BatchRenameReq struct { SrcDir string `json:"src_dir"` RenameObjects []struct { SrcName string `json:"src_name"` NewName string `json:"new_name"` } `json:"rename_objects"` } func FsBatchRename(c *gin.Context) { var req BatchRenameReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if !user.CanRename() { common.ErrorResp(c, errs.PermissionDenied, 403) return } reqPath, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { common.ErrorResp(c, err, 500, true) return } } common.GinWithValue(c, conf.MetaKey, meta) for _, renameObject := range req.RenameObjects { if renameObject.SrcName == "" || renameObject.NewName == "" { continue } err = checkRelativePath(renameObject.NewName) if err != nil { common.ErrorResp(c, err, 403) return } filePath := fmt.Sprintf("%s/%s", reqPath, renameObject.SrcName) if err := fs.Rename(c.Request.Context(), filePath, renameObject.NewName); err != nil { common.ErrorResp(c, err, 500) return } } common.SuccessResp(c) } type RegexRenameReq struct { SrcDir string `json:"src_dir"` SrcNameRegex string `json:"src_name_regex"` NewNameRegex string `json:"new_name_regex"` } func FsRegexRename(c *gin.Context) { var req RegexRenameReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if !user.CanRename() { common.ErrorResp(c, errs.PermissionDenied, 403) return } reqPath, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { common.ErrorResp(c, err, 500, true) return } } common.GinWithValue(c, conf.MetaKey, meta) srcRegexp, err := regexp.Compile(req.SrcNameRegex) if err != nil { common.ErrorResp(c, err, 500) return } files, err := fs.List(c.Request.Context(), reqPath, &fs.ListArgs{}) if err != nil { common.ErrorResp(c, err, 500) return } for _, file := range files { if srcRegexp.MatchString(file.GetName()) { newFileName := srcRegexp.ReplaceAllString(file.GetName(), req.NewNameRegex) err := checkRelativePath(newFileName) if err != nil { common.ErrorResp(c, err, 403) return } filePath := fmt.Sprintf("%s/%s", reqPath, file.GetName()) if err := fs.Rename(c.Request.Context(), filePath, newFileName); err != nil { common.ErrorResp(c, err, 500) return } } } common.SuccessResp(c) } ================================================ FILE: server/handles/fsmanage.go ================================================ package handles import ( "fmt" stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/pkg/generic" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) type MkdirOrLinkReq struct { Path string `json:"path" form:"path"` } func FsMkdir(c *gin.Context) { var req MkdirOrLinkReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } if !user.CanWrite() { meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { common.ErrorResp(c, err, 500, true) return } } if !common.CanWrite(meta, reqPath) { common.ErrorResp(c, errs.PermissionDenied, 403) return } } if err := fs.MakeDir(c.Request.Context(), reqPath); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c) } type MoveCopyReq struct { SrcDir string `json:"src_dir"` DstDir string `json:"dst_dir"` Names []string `json:"names"` Overwrite bool `json:"overwrite"` SkipExisting bool `json:"skip_existing"` Merge bool `json:"merge"` } func FsMove(c *gin.Context) { var req MoveCopyReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if len(req.Names) == 0 { common.ErrorStrResp(c, "Empty file names", 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if !user.CanMove() { common.ErrorResp(c, errs.PermissionDenied, 403) return } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } validPaths := make([]string, 0, len(req.Names)) for _, name := range req.Names { // ensure req.Names is not a relative path srcPath := stdpath.Join(req.SrcDir, name) srcPath, err = user.JoinPath(srcPath) if err != nil { common.ErrorResp(c, err, 403) return } if !req.Overwrite { base := stdpath.Base(srcPath) if base == "." || base == "/" { common.ErrorStrResp(c, fmt.Sprintf("invalid file name [%s]", name), 400) return } if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, base), &fs.GetArgs{NoLog: true}); res != nil { if !req.SkipExisting { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return } else { continue } } } validPaths = append(validPaths, srcPath) } // Create all tasks immediately without any synchronous validation // All validation will be done asynchronously in the background var addedTasks []task.TaskExtensionInfo for i, p := range validPaths { t, err := fs.Move(c.Request.Context(), p, dstDir, len(validPaths) > i+1) if t != nil { addedTasks = append(addedTasks, t) } if err != nil { common.ErrorResp(c, err, 500) return } } // Return immediately with task information if len(addedTasks) > 0 { common.SuccessResp(c, gin.H{ "message": fmt.Sprintf("Successfully created %d move task(s)", len(addedTasks)), "tasks": getTaskInfos(addedTasks), }) } else { common.SuccessResp(c, gin.H{ "message": "Move operations completed immediately", }) } } func FsCopy(c *gin.Context) { var req MoveCopyReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if len(req.Names) == 0 { common.ErrorStrResp(c, "Empty file names", 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if !user.CanCopy() { common.ErrorResp(c, errs.PermissionDenied, 403) return } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } validPaths := make([]string, 0, len(req.Names)) for _, name := range req.Names { // ensure req.Names is not a relative path srcPath := stdpath.Join(req.SrcDir, name) srcPath, err = user.JoinPath(srcPath) if err != nil { common.ErrorResp(c, err, 403) return } if !req.Overwrite { base := stdpath.Base(srcPath) if base == "." || base == "/" { common.ErrorStrResp(c, fmt.Sprintf("invalid file name [%s]", name), 400) return } if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, base), &fs.GetArgs{NoLog: true}); res != nil { if !req.SkipExisting && !req.Merge { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return } else if !req.Merge || !res.IsDir() { continue } } } validPaths = append(validPaths, srcPath) } // Create all tasks immediately without any synchronous validation // All validation will be done asynchronously in the background var addedTasks []task.TaskExtensionInfo for i, p := range validPaths { var t task.TaskExtensionInfo if req.Merge { t, err = fs.Merge(c.Request.Context(), p, dstDir, len(validPaths) > i+1) } else { t, err = fs.Copy(c.Request.Context(), p, dstDir, len(validPaths) > i+1) } if t != nil { addedTasks = append(addedTasks, t) } if err != nil { common.ErrorResp(c, err, 500) return } } // Return immediately with task information if len(addedTasks) > 0 { common.SuccessResp(c, gin.H{ "message": fmt.Sprintf("Successfully created %d copy task(s)", len(addedTasks)), "tasks": getTaskInfos(addedTasks), }) } else { common.SuccessResp(c, gin.H{ "message": "Copy operations completed immediately", }) } } type RenameReq struct { Path string `json:"path"` Name string `json:"name"` Overwrite bool `json:"overwrite"` } func FsRename(c *gin.Context) { var req RenameReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if !user.CanRename() { common.ErrorResp(c, errs.PermissionDenied, 403) return } reqPath, err := user.JoinPath(req.Path) if err == nil { err = checkRelativePath(req.Name) } if err != nil { common.ErrorResp(c, err, 403) return } if !req.Overwrite { dstPath := stdpath.Join(stdpath.Dir(reqPath), req.Name) if dstPath != reqPath { if res, _ := fs.Get(c.Request.Context(), dstPath, &fs.GetArgs{NoLog: true}); res != nil { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", req.Name), 403) return } } } if err := fs.Rename(c.Request.Context(), reqPath, req.Name); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c) } func checkRelativePath(path string) error { if strings.ContainsAny(path, "/\\") || path == "" || path == "." || path == ".." { return errs.RelativePath } return nil } type RemoveReq struct { Dir string `json:"dir"` Names []string `json:"names"` } func FsRemove(c *gin.Context) { var req RemoveReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if len(req.Names) == 0 { common.ErrorStrResp(c, "Empty file names", 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if !user.CanRemove() { common.ErrorResp(c, errs.PermissionDenied, 403) return } for i, name := range req.Names { if strings.TrimSpace(utils.FixAndCleanPath(name)) == "/" { log.Warnf("FsRemove: invalid item skipped: %s (parent directory: %s)\n", name, req.Dir) req.Names[i] = "" continue } // ensure req.Names is not a relative path var err error req.Names[i], err = user.JoinPath(stdpath.Join(req.Dir, name)) if err != nil { common.ErrorResp(c, err, 403) return } } for _, path := range req.Names { if path == "" { continue } err := fs.Remove(c.Request.Context(), path) if err != nil { common.ErrorResp(c, err, 500) return } } //fs.ClearCache(req.Dir) common.SuccessResp(c) } type RemoveEmptyDirectoryReq struct { SrcDir string `json:"src_dir"` } func FsRemoveEmptyDirectory(c *gin.Context) { var req RemoveEmptyDirectoryReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if !user.CanRemove() { common.ErrorResp(c, errs.PermissionDenied, 403) return } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } meta, err := op.GetNearestMeta(srcDir) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { common.ErrorResp(c, err, 500, true) return } } common.GinWithValue(c, conf.MetaKey, meta) rootFiles, err := fs.List(c.Request.Context(), srcDir, &fs.ListArgs{}) if err != nil { common.ErrorResp(c, err, 500) return } // record the file path filePathMap := make(map[model.Obj]string) // record the parent file fileParentMap := make(map[model.Obj]model.Obj) // removing files removingFiles := generic.NewQueue[model.Obj]() // removed files removedFiles := make(map[string]bool) for _, file := range rootFiles { if !file.IsDir() { continue } removingFiles.Push(file) filePathMap[file] = srcDir } for !removingFiles.IsEmpty() { removingFile := removingFiles.Pop() removingFilePath := fmt.Sprintf("%s/%s", filePathMap[removingFile], removingFile.GetName()) if removedFiles[removingFilePath] { continue } subFiles, err := fs.List(c.Request.Context(), removingFilePath, &fs.ListArgs{Refresh: true}) if err != nil { common.ErrorResp(c, err, 500) return } if len(subFiles) == 0 { // remove empty directory err = fs.Remove(c.Request.Context(), removingFilePath) removedFiles[removingFilePath] = true if err != nil { common.ErrorResp(c, err, 500) return } // recheck parent folder parentFile, exist := fileParentMap[removingFile] if exist { removingFiles.Push(parentFile) } } else { // recursive remove for _, subFile := range subFiles { if !subFile.IsDir() { continue } removingFiles.Push(subFile) filePathMap[subFile] = removingFilePath fileParentMap[subFile] = removingFile } } } common.SuccessResp(c) } // Link return real link, just for proxy program, it may contain cookie, so just allowed for admin func Link(c *gin.Context) { var req MkdirOrLinkReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } //user := c.Request.Context().Value(conf.UserKey).(*model.User) //rawPath := stdpath.Join(user.BasePath, req.Path) // why need not join base_path? because it's always the full path rawPath := req.Path storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{}) if err != nil { common.ErrorResp(c, err, 500) return } if storage.Config().NoLinkURL { common.SuccessResp(c, model.Link{ URL: fmt.Sprintf("%s/p%s?d&sign=%s", common.GetApiUrl(c), utils.EncodePath(rawPath, true), sign.Sign(rawPath)), }) return } link, _, err := fs.Link(c.Request.Context(), rawPath, model.LinkArgs{IP: c.ClientIP(), Header: c.Request.Header, Redirect: true}) if err != nil { common.ErrorResp(c, err, 500) return } defer link.Close() common.SuccessResp(c, link) } ================================================ FILE: server/handles/fsread.go ================================================ package handles import ( "fmt" stdpath "path" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" ) type ListReq struct { model.PageReq Path string `json:"path" form:"path"` Password string `json:"password" form:"password"` Refresh bool `json:"refresh"` } type DirReq struct { Path string `json:"path" form:"path"` Password string `json:"password" form:"password"` ForceRoot bool `json:"force_root" form:"force_root"` } type ObjResp struct { Name string `json:"name"` Size int64 `json:"size"` IsDir bool `json:"is_dir"` Modified time.Time `json:"modified"` Created time.Time `json:"created"` Sign string `json:"sign"` Thumb string `json:"thumb"` Type int `json:"type"` HashInfoStr string `json:"hashinfo"` HashInfo map[*utils.HashType]string `json:"hash_info"` MountDetails *model.StorageDetails `json:"mount_details,omitempty"` } type FsListResp struct { Content []ObjResp `json:"content"` Total int64 `json:"total"` Readme string `json:"readme"` Header string `json:"header"` Write bool `json:"write"` Provider string `json:"provider"` DirectUploadTools []string `json:"direct_upload_tools,omitempty"` } func FsListSplit(c *gin.Context) { var req ListReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } req.Validate() if strings.HasPrefix(req.Path, "/@s") { req.Path = strings.TrimPrefix(req.Path, "/@s") SharingList(c, &req) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() && user.Disabled { common.ErrorStrResp(c, "Guest user is disabled, login please", 401) return } FsList(c, &req, user) } func FsList(c *gin.Context, req *ListReq, user *model.User) { reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { common.ErrorResp(c, err, 500, true) return } } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } if !user.CanWrite() && !common.CanWrite(meta, reqPath) && req.Refresh { common.ErrorStrResp(c, "Refresh without permission", 403) return } objs, err := fs.List(c.Request.Context(), reqPath, &fs.ListArgs{ Refresh: req.Refresh, WithStorageDetails: !user.IsGuest() && !setting.GetBool(conf.HideStorageDetails), }) if err != nil { common.ErrorResp(c, err, 500) return } total, objs := pagination(objs, &req.PageReq) provider := "unknown" var directUploadTools []string if user.CanWrite() { if storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}); err == nil { directUploadTools = op.GetDirectUploadTools(storage) } } common.SuccessResp(c, FsListResp{ Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), Total: int64(total), Readme: getReadme(meta, reqPath), Header: getHeader(meta, reqPath), Write: user.CanWrite() || common.CanWrite(meta, reqPath), Provider: provider, DirectUploadTools: directUploadTools, }) } func FsDirs(c *gin.Context) { var req DirReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) reqPath := req.Path if req.ForceRoot { if !user.IsAdmin() { common.ErrorStrResp(c, "Permission denied", 403) return } } else { tmp, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } reqPath = tmp } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { common.ErrorResp(c, err, 500, true) return } } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } objs, err := fs.List(c.Request.Context(), reqPath, &fs.ListArgs{}) if err != nil { common.ErrorResp(c, err, 500) return } dirs := filterDirs(objs) common.SuccessResp(c, dirs) } type DirResp struct { Name string `json:"name"` Modified time.Time `json:"modified"` } func filterDirs(objs []model.Obj) []DirResp { var dirs []DirResp for _, obj := range objs { if obj.IsDir() { dirs = append(dirs, DirResp{ Name: obj.GetName(), Modified: obj.ModTime(), }) } } return dirs } func getReadme(meta *model.Meta, path string) string { if meta != nil && (utils.PathEqual(meta.Path, path) || meta.RSub) { return meta.Readme } return "" } func getHeader(meta *model.Meta, path string) string { if meta != nil && (utils.PathEqual(meta.Path, path) || meta.HeaderSub) { return meta.Header } return "" } func isEncrypt(meta *model.Meta, path string) bool { if common.IsStorageSignEnabled(path) { return true } if meta == nil || meta.Password == "" { return false } if !utils.PathEqual(meta.Path, path) && !meta.PSub { return false } return true } func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { pageIndex, pageSize := req.Page, req.PerPage total := len(objs) start := (pageIndex - 1) * pageSize if start > total { return total, []model.Obj{} } end := start + pageSize if end > total { end = total } return total, objs[start:end] } func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { var resp []ObjResp for _, obj := range objs { thumb, _ := model.GetThumb(obj) mountDetails, _ := model.GetStorageDetails(obj) resp = append(resp, ObjResp{ Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), Modified: obj.ModTime(), Created: obj.CreateTime(), HashInfoStr: obj.GetHash().String(), HashInfo: obj.GetHash().Export(), Sign: common.Sign(obj, parent, encrypt), Thumb: thumb, Type: utils.GetObjType(obj.GetName(), obj.IsDir()), MountDetails: mountDetails, }) } return resp } type FsGetReq struct { Path string `json:"path" form:"path"` Password string `json:"password" form:"password"` } type FsGetResp struct { ObjResp RawURL string `json:"raw_url"` Readme string `json:"readme"` Header string `json:"header"` Provider string `json:"provider"` Related []ObjResp `json:"related"` } func FsGetSplit(c *gin.Context) { var req FsGetReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if strings.HasPrefix(req.Path, "/@s") { req.Path = strings.TrimPrefix(req.Path, "/@s") SharingGet(c, &req) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() && user.Disabled { common.ErrorStrResp(c, "Guest user is disabled, login please", 401) return } FsGet(c, &req, user) } func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { common.ErrorResp(c, err, 500) return } } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } obj, err := fs.Get(c.Request.Context(), reqPath, &fs.GetArgs{ WithStorageDetails: !user.IsGuest() && !setting.GetBool(conf.HideStorageDetails), }) if err != nil { common.ErrorResp(c, err, 500) return } var rawURL string storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) provider, ok := model.GetProvider(obj) if !ok && err == nil { provider = storage.Config().Name } if !obj.IsDir() { if err != nil { common.ErrorResp(c, err, 500) return } if storage.Config().MustProxy() || storage.GetStorage().WebProxy { rawURL = common.GenerateDownProxyURL(storage.GetStorage(), reqPath) if rawURL == "" { query := "" if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { query = "?sign=" + sign.Sign(reqPath) } rawURL = fmt.Sprintf("%s/p%s%s", common.GetApiUrl(c), utils.EncodePath(reqPath, true), query) } } else { // file have raw url if url, ok := model.GetUrl(obj); ok { rawURL = url } else { // if storage is not proxy, use raw url by fs.Link link, _, err := fs.Link(c.Request.Context(), reqPath, model.LinkArgs{ IP: c.ClientIP(), Header: c.Request.Header, Redirect: true, }) if err != nil { common.ErrorResp(c, err, 500) return } defer link.Close() rawURL = link.URL } } } var related []model.Obj parentPath := stdpath.Dir(reqPath) sameLevelFiles, err := fs.List(c.Request.Context(), parentPath, &fs.ListArgs{}) if err == nil { related = filterRelated(sameLevelFiles, obj) } parentMeta, _ := op.GetNearestMeta(parentPath) thumb, _ := model.GetThumb(obj) mountDetails, _ := model.GetStorageDetails(obj) common.SuccessResp(c, FsGetResp{ ObjResp: ObjResp{ Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), Modified: obj.ModTime(), Created: obj.CreateTime(), HashInfoStr: obj.GetHash().String(), HashInfo: obj.GetHash().Export(), Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), Type: utils.GetFileType(obj.GetName()), Thumb: thumb, MountDetails: mountDetails, }, RawURL: rawURL, Readme: getReadme(meta, reqPath), Header: getHeader(meta, reqPath), Provider: provider, Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath)), }) } func filterRelated(objs []model.Obj, obj model.Obj) []model.Obj { var related []model.Obj nameWithoutExt := strings.TrimSuffix(obj.GetName(), stdpath.Ext(obj.GetName())) for _, o := range objs { if o.GetName() == obj.GetName() { continue } if strings.HasPrefix(o.GetName(), nameWithoutExt) { related = append(related, o) } } return related } type FsOtherReq struct { model.FsOtherArgs Password string `json:"password" form:"password"` } func FsOther(c *gin.Context) { var req FsOtherReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) var err error req.Path, err = user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } meta, err := op.GetNearestMeta(req.Path) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { common.ErrorResp(c, err, 500) return } } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, req.Path, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } res, err := fs.Other(c.Request.Context(), req.FsOtherArgs) if err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, res) } ================================================ FILE: server/handles/fsup.go ================================================ package handles import ( "io" "net/url" stdpath "path" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" ) func getLastModified(c *gin.Context) time.Time { now := time.Now() lastModifiedStr := c.GetHeader("Last-Modified") lastModifiedMillisecond, err := strconv.ParseInt(lastModifiedStr, 10, 64) if err != nil { return now } lastModified := time.UnixMilli(lastModifiedMillisecond) return lastModified } // shouldIgnoreSystemFile checks if the filename should be ignored based on settings func shouldIgnoreSystemFile(filename string) bool { if setting.GetBool(conf.IgnoreSystemFiles) { return utils.IsSystemFile(filename) } return false } func FsStream(c *gin.Context) { defer func() { if n, _ := io.ReadFull(c.Request.Body, []byte{0}); n == 1 { _, _ = utils.CopyWithBuffer(io.Discard, c.Request.Body) } _ = c.Request.Body.Close() }() path := c.GetHeader("File-Path") path, err := url.PathUnescape(path) if err != nil { common.ErrorResp(c, err, 400) return } asTask := c.GetHeader("As-Task") == "true" overwrite := c.GetHeader("Overwrite") != "false" user := c.Request.Context().Value(conf.UserKey).(*model.User) path, err = user.JoinPath(path) if err != nil { common.ErrorResp(c, err, 403) return } if !overwrite { if res, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true}); res != nil { common.ErrorStrResp(c, "file exists", 403) return } } dir, name := stdpath.Split(path) // Check if system file should be ignored if shouldIgnoreSystemFile(name) { common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403) return } // 如果请求头 Content-Length 和 X-File-Size 都没有，则 size=-1，表示未知大小的流式上传 size := c.Request.ContentLength if size < 0 { sizeStr := c.GetHeader("X-File-Size") if sizeStr != "" { size, err = strconv.ParseInt(sizeStr, 10, 64) if err != nil { common.ErrorResp(c, err, 400) return } } } h := make(map[*utils.HashType]string) if md5 := c.GetHeader("X-File-Md5"); md5 != "" { h[utils.MD5] = md5 } if sha1 := c.GetHeader("X-File-Sha1"); sha1 != "" { h[utils.SHA1] = sha1 } if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { h[utils.SHA256] = sha256 } mimetype := c.GetHeader("Content-Type") if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) } s := &stream.FileStream{ Obj: &model.Object{ Name: name, Size: size, Modified: getLastModified(c), HashInfo: utils.NewHashInfoByMap(h), }, Reader: c.Request.Body, Mimetype: mimetype, WebPutAsTask: asTask, } var t task.TaskExtensionInfo if asTask { t, err = fs.PutAsTask(c.Request.Context(), dir, s) } else { err = fs.PutDirectly(c.Request.Context(), dir, s) } if err != nil { common.ErrorResp(c, err, 500) return } if t == nil { common.SuccessResp(c) return } common.SuccessResp(c, gin.H{ "task": getTaskInfo(t), }) } func FsForm(c *gin.Context) { defer func() { if n, _ := io.ReadFull(c.Request.Body, []byte{0}); n == 1 { _, _ = utils.CopyWithBuffer(io.Discard, c.Request.Body) } _ = c.Request.Body.Close() }() path := c.GetHeader("File-Path") path, err := url.PathUnescape(path) if err != nil { common.ErrorResp(c, err, 400) return } asTask := c.GetHeader("As-Task") == "true" overwrite := c.GetHeader("Overwrite") != "false" user := c.Request.Context().Value(conf.UserKey).(*model.User) path, err = user.JoinPath(path) if err != nil { common.ErrorResp(c, err, 403) return } if !overwrite { if res, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true}); res != nil { common.ErrorStrResp(c, "file exists", 403) return } } storage, err := fs.GetStorage(path, &fs.GetStoragesArgs{}) if err != nil { common.ErrorResp(c, err, 400) return } if storage.Config().NoUpload { common.ErrorStrResp(c, "Current storage doesn't support upload", 405) return } file, err := c.FormFile("file") if err != nil { common.ErrorResp(c, err, 500) return } f, err := file.Open() if err != nil { common.ErrorResp(c, err, 500) return } defer f.Close() dir, name := stdpath.Split(path) // Check if system file should be ignored if shouldIgnoreSystemFile(name) { common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403) return } h := make(map[*utils.HashType]string) if md5 := c.GetHeader("X-File-Md5"); md5 != "" { h[utils.MD5] = md5 } if sha1 := c.GetHeader("X-File-Sha1"); sha1 != "" { h[utils.SHA1] = sha1 } if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { h[utils.SHA256] = sha256 } mimetype := file.Header.Get("Content-Type") if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) } s := &stream.FileStream{ Obj: &model.Object{ Name: name, Size: file.Size, Modified: getLastModified(c), HashInfo: utils.NewHashInfoByMap(h), }, Reader: f, Mimetype: mimetype, WebPutAsTask: asTask, } var t task.TaskExtensionInfo if asTask { s.Reader = struct { io.Reader }{f} t, err = fs.PutAsTask(c.Request.Context(), dir, s) } else { err = fs.PutDirectly(c.Request.Context(), dir, s) } if err != nil { common.ErrorResp(c, err, 500) return } if t == nil { common.SuccessResp(c) return } common.SuccessResp(c, gin.H{ "task": getTaskInfo(t), }) } ================================================ FILE: server/handles/helper.go ================================================ package handles import ( "fmt" "html" "net/url" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" ) func Favicon(c *gin.Context) { c.Redirect(302, setting.GetStr(conf.Favicon)) } func Robots(c *gin.Context) { c.String(200, setting.GetStr(conf.RobotsTxt)) } func Plist(c *gin.Context) { linkNameB64 := strings.TrimSuffix(c.Param("link_name"), ".plist") linkName, err := utils.SafeAtob(linkNameB64) if err != nil { common.ErrorResp(c, err, 400) return } linkNameSplit := strings.Split(linkName, "/") if len(linkNameSplit) != 2 { common.ErrorStrResp(c, "malformed link", 400) return } linkEncode := linkNameSplit[0] linkStr, err := url.PathUnescape(linkEncode) if err != nil { common.ErrorResp(c, err, 400) return } link, err := url.Parse(linkStr) if err != nil { common.ErrorResp(c, err, 400) return } nameEncode := linkNameSplit[1] fullName, err := url.PathUnescape(nameEncode) if err != nil { common.ErrorResp(c, err, 400) return } name := fullName identifier := fmt.Sprintf("org.oplist.%s", fullName) if strings.Contains(fullName, "@") { ss := strings.Split(fullName, "@") name = strings.Join(ss[:len(ss)-1], "@") identifier = ss[len(ss)-1] } Url := link.String() Url = strings.ReplaceAll(Url, "<", "<") Url = strings.ReplaceAll(Url, ">", ">") name = html.EscapeString(name) identifier = html.EscapeString(identifier) plist := fmt.Sprintf(` items assets kind software-package url metadata bundle-identifier %s bundle-version 4.4 kind software title %s `, Url, identifier, name) c.Header("Content-Type", "application/xml;charset=utf-8") c.Status(200) _, _ = c.Writer.WriteString(plist) } ================================================ FILE: server/handles/index.go ================================================ package handles import ( "context" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/search" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) type UpdateIndexReq struct { Paths []string `json:"paths"` MaxDepth int `json:"max_depth"` //IgnorePaths []string `json:"ignore_paths"` } func BuildIndex(c *gin.Context) { if search.Running() { common.ErrorStrResp(c, "index is running", 400) return } go func() { ctx := context.Background() err := search.Clear(ctx) if err != nil { log.Errorf("clear index error: %+v", err) return } err = search.BuildIndex(context.Background(), []string{"/"}, conf.SlicesMap[conf.IgnorePaths], setting.GetInt(conf.MaxIndexDepth, 20), true) if err != nil { log.Errorf("build index error: %+v", err) } }() common.SuccessResp(c) } func UpdateIndex(c *gin.Context) { var req UpdateIndexReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if search.Running() { common.ErrorStrResp(c, "index is running", 400) return } if !search.Config(c).AutoUpdate { common.ErrorStrResp(c, "update is not supported for current index", 400) return } go func() { ctx := context.Background() for _, path := range req.Paths { err := search.Del(ctx, path) if err != nil { log.Errorf("delete index on %s error: %+v", path, err) return } } err := search.BuildIndex(context.Background(), req.Paths, conf.SlicesMap[conf.IgnorePaths], req.MaxDepth, false) if err != nil { log.Errorf("update index error: %+v", err) } }() common.SuccessResp(c) } func StopIndex(c *gin.Context) { quit := search.Quit.Load() if quit == nil { common.ErrorStrResp(c, "index is not running", 400) return } select { case *quit <- struct{}{}: default: } common.SuccessResp(c) } func ClearIndex(c *gin.Context) { if search.Running() { common.ErrorStrResp(c, "index is running", 400) return } search.Clear(c) search.WriteProgress(&model.IndexProgress{ ObjCount: 0, IsDone: true, LastDoneTime: nil, Error: "", }) common.SuccessResp(c) } func GetProgress(c *gin.Context) { progress, err := search.Progress() if err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, progress) } ================================================ FILE: server/handles/ldap_login.go ================================================ package handles import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" ) func LoginLdap(c *gin.Context) { var req LoginReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } enabled := setting.GetBool(conf.LdapLoginEnabled) if !enabled { common.ErrorStrResp(c, "ldap is not enabled", 403) return } user, err := op.GetUserByName(req.Username) if err == nil && !user.AllowLdap { common.ErrorStrResp(c, "login via ldap is not allowed", 403) return } // check count of login ip := c.ClientIP() count, ok := model.LoginCache.Get(ip) if ok && count >= model.DefaultMaxAuthRetries { common.ErrorStrResp(c, "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later.", 429) model.LoginCache.Expire(ip, model.DefaultLockDuration) return } err = common.HandleLdapLogin(req.Username, req.Password) if err != nil { if errors.Is(err, common.ErrFailedLdapAuth) { model.LoginCache.Set(ip, count+1) common.ErrorResp(c, err, 400) } else { common.ErrorResp(c, err, 500) } return } if user == nil { user, err = common.LdapRegister(req.Username) if err != nil { common.ErrorResp(c, err, 400) model.LoginCache.Set(ip, count+1) return } } // generate token token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400, true) return } common.SuccessResp(c, gin.H{"token": token}) model.LoginCache.Del(ip) } ================================================ FILE: server/handles/meta.go ================================================ package handles import ( "fmt" "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/dlclark/regexp2" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) func ListMetas(c *gin.Context) { var req model.PageReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } req.Validate() log.Debugf("%+v", req) metas, total, err := op.GetMetas(req.Page, req.PerPage) if err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c, common.PageResp{ Content: metas, Total: total, }) } func CreateMeta(c *gin.Context) { var req model.Meta if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } r, err := validHide(req.Hide) if err != nil { common.ErrorStrResp(c, fmt.Sprintf("%s is illegal: %s", r, err.Error()), 400) return } if err := op.CreateMeta(&req); err != nil { common.ErrorResp(c, err, 500, true) } else { common.SuccessResp(c) } } func UpdateMeta(c *gin.Context) { var req model.Meta if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } r, err := validHide(req.Hide) if err != nil { common.ErrorStrResp(c, fmt.Sprintf("%s is illegal: %s", r, err.Error()), 400) return } if err := op.UpdateMeta(&req); err != nil { common.ErrorResp(c, err, 500, true) } else { common.SuccessResp(c) } } func validHide(hide string) (string, error) { rs := strings.Split(hide, "\n") for _, r := range rs { _, err := regexp2.Compile(r, regexp2.None) if err != nil { return r, err } } return "", nil } func DeleteMeta(c *gin.Context) { idStr := c.Query("id") id, err := strconv.Atoi(idStr) if err != nil { common.ErrorResp(c, err, 400) return } if err := op.DeleteMetaById(uint(id)); err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c) } func GetMeta(c *gin.Context) { idStr := c.Query("id") id, err := strconv.Atoi(idStr) if err != nil { common.ErrorResp(c, err, 400) return } meta, err := op.GetMetaById(uint(id)) if err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c, meta) } ================================================ FILE: server/handles/offline_download.go ================================================ package handles import ( "strings" _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" "github.com/OpenListTeam/OpenList/v4/drivers/thunder" "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" ) type SetAria2Req struct { Uri string `json:"uri" form:"uri"` Secret string `json:"secret" form:"secret"` } func SetAria2(c *gin.Context) { var req SetAria2Req if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } items := []model.SettingItem{ {Key: conf.Aria2Uri, Value: req.Uri, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.Aria2Secret, Value: req.Secret, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } if err := op.SaveSettingItems(items); err != nil { common.ErrorResp(c, err, 500) return } _tool, err := tool.Tools.Get("aria2") if err != nil { common.ErrorResp(c, err, 500) return } version, err := _tool.Init() if err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, version) } type SetQbittorrentReq struct { Url string `json:"url" form:"url"` Seedtime string `json:"seedtime" form:"seedtime"` } func SetQbittorrent(c *gin.Context) { var req SetQbittorrentReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } items := []model.SettingItem{ {Key: conf.QbittorrentUrl, Value: req.Url, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.QbittorrentSeedtime, Value: req.Seedtime, Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } if err := op.SaveSettingItems(items); err != nil { common.ErrorResp(c, err, 500) return } _tool, err := tool.Tools.Get("qBittorrent") if err != nil { common.ErrorResp(c, err, 500) return } if _, err := _tool.Init(); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, "ok") } type SetTransmissionReq struct { Uri string `json:"uri" form:"uri"` Seedtime string `json:"seedtime" form:"seedtime"` } func SetTransmission(c *gin.Context) { var req SetTransmissionReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } items := []model.SettingItem{ {Key: conf.TransmissionUri, Value: req.Uri, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.TransmissionSeedtime, Value: req.Seedtime, Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } if err := op.SaveSettingItems(items); err != nil { common.ErrorResp(c, err, 500) return } _tool, err := tool.Tools.Get("Transmission") if err != nil { common.ErrorResp(c, err, 500) return } if _, err := _tool.Init(); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, "ok") } type Set115Req struct { TempDir string `json:"temp_dir" form:"temp_dir"` } func Set115(c *gin.Context) { var req Set115Req if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if req.TempDir != "" { storage, _, err := op.GetStorageAndActualPath(req.TempDir) if err != nil { common.ErrorStrResp(c, "storage does not exists", 400) return } if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) return } if _, ok := storage.(*_115.Pan115); !ok { common.ErrorStrResp(c, "unsupported storage driver for offline download, only 115 Cloud is supported", 400) return } } items := []model.SettingItem{ {Key: conf.Pan115TempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } if err := op.SaveSettingItems(items); err != nil { common.ErrorResp(c, err, 500) return } _tool, err := tool.Tools.Get("115 Cloud") if err != nil { common.ErrorResp(c, err, 500) return } if _, err := _tool.Init(); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, "ok") } type Set115OpenReq struct { TempDir string `json:"temp_dir" form:"temp_dir"` } func Set115Open(c *gin.Context) { var req Set115OpenReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if req.TempDir != "" { storage, _, err := op.GetStorageAndActualPath(req.TempDir) if err != nil { common.ErrorStrResp(c, "storage does not exists", 400) return } if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) return } if _, ok := storage.(*_115_open.Open115); !ok { common.ErrorStrResp(c, "unsupported storage driver for offline download, only 115 Open is supported", 400) return } } items := []model.SettingItem{ {Key: conf.Pan115OpenTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } if err := op.SaveSettingItems(items); err != nil { common.ErrorResp(c, err, 500) return } _tool, err := tool.Tools.Get("115 Open") if err != nil { common.ErrorResp(c, err, 500) return } if _, err := _tool.Init(); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, "ok") } type Set123PanReq struct { TempDir string `json:"temp_dir" form:"temp_dir"` } func Set123Pan(c *gin.Context) { var req Set123PanReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if req.TempDir != "" { storage, _, err := op.GetStorageAndActualPath(req.TempDir) if err != nil { common.ErrorStrResp(c, "storage does not exists", 400) return } if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) return } if _, ok := storage.(*_123.Pan123); !ok { common.ErrorStrResp(c, "unsupported storage driver for offline download, only 123Pan is supported", 400) return } } items := []model.SettingItem{ {Key: conf.Pan123TempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } if err := op.SaveSettingItems(items); err != nil { common.ErrorResp(c, err, 500) return } _tool, err := tool.Tools.Get("123Pan") if err != nil { common.ErrorResp(c, err, 500) return } if _, err := _tool.Init(); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, "ok") } type Set123OpenReq struct { TempDir string `json:"temp_dir" form:"temp_dir"` CallbackUrl string `json:"callback_url" form:"callback_url"` } func Set123Open(c *gin.Context) { var req Set123OpenReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if req.TempDir != "" { storage, _, err := op.GetStorageAndActualPath(req.TempDir) if err != nil { common.ErrorStrResp(c, "storage does not exists", 400) return } if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) return } if _, ok := storage.(*_123_open.Open123); !ok { common.ErrorStrResp(c, "unsupported storage driver for offline download, only 123 Open is supported", 400) return } } items := []model.SettingItem{ {Key: conf.Pan123OpenTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.Pan123OpenOfflineDownloadCallbackUrl, Value: req.CallbackUrl, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } if err := op.SaveSettingItems(items); err != nil { common.ErrorResp(c, err, 500) return } _tool, err := tool.Tools.Get("123 Open") if err != nil { common.ErrorResp(c, err, 500) return } if _, err := _tool.Init(); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, "ok") } type SetPikPakReq struct { TempDir string `json:"temp_dir" form:"temp_dir"` } func SetPikPak(c *gin.Context) { var req SetPikPakReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if req.TempDir != "" { storage, _, err := op.GetStorageAndActualPath(req.TempDir) if err != nil { common.ErrorStrResp(c, "storage does not exists", 400) return } if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) return } if _, ok := storage.(*pikpak.PikPak); !ok { common.ErrorStrResp(c, "unsupported storage driver for offline download, only PikPak is supported", 400) return } } items := []model.SettingItem{ {Key: conf.PikPakTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } if err := op.SaveSettingItems(items); err != nil { common.ErrorResp(c, err, 500) return } _tool, err := tool.Tools.Get("PikPak") if err != nil { common.ErrorResp(c, err, 500) return } if _, err := _tool.Init(); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, "ok") } type SetThunderReq struct { TempDir string `json:"temp_dir" form:"temp_dir"` } func SetThunder(c *gin.Context) { var req SetThunderReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if req.TempDir != "" { storage, _, err := op.GetStorageAndActualPath(req.TempDir) if err != nil { common.ErrorStrResp(c, "storage does not exists", 400) return } if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) return } if _, ok := storage.(*thunder.Thunder); !ok { common.ErrorStrResp(c, "unsupported storage driver for offline download, only Thunder is supported", 400) return } } items := []model.SettingItem{ {Key: conf.ThunderTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } if err := op.SaveSettingItems(items); err != nil { common.ErrorResp(c, err, 500) return } _tool, err := tool.Tools.Get("Thunder") if err != nil { common.ErrorResp(c, err, 500) return } if _, err := _tool.Init(); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, "ok") } type SetThunderXReq struct { TempDir string `json:"temp_dir" form:"temp_dir"` } func SetThunderX(c *gin.Context) { var req SetThunderXReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if req.TempDir != "" { storage, _, err := op.GetStorageAndActualPath(req.TempDir) if err != nil { common.ErrorStrResp(c, "storage does not exists", 400) return } if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) return } if _, ok := storage.(*thunderx.ThunderX); !ok { common.ErrorStrResp(c, "unsupported storage driver for offline download, only ThunderX is supported", 400) return } } items := []model.SettingItem{ {Key: conf.ThunderXTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } if err := op.SaveSettingItems(items); err != nil { common.ErrorResp(c, err, 500) return } _tool, err := tool.Tools.Get("ThunderX") if err != nil { common.ErrorResp(c, err, 500) return } if _, err := _tool.Init(); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, "ok") } type SetThunderBrowserReq struct { TempDir string `json:"temp_dir" form:"temp_dir"` } func SetThunderBrowser(c *gin.Context) { var req SetThunderBrowserReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if req.TempDir != "" { storage, _, err := op.GetStorageAndActualPath(req.TempDir) if err != nil { common.ErrorStrResp(c, "storage does not exists", 400) return } if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) return } switch storage.(type) { case *thunder_browser.ThunderBrowser, *thunder_browser.ThunderBrowserExpert: default: common.ErrorStrResp(c, "unsupported storage driver for offline download, only ThunderBrowser is supported", 400) return } } items := []model.SettingItem{ {Key: conf.ThunderBrowserTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } if err := op.SaveSettingItems(items); err != nil { common.ErrorResp(c, err, 500) return } _tool, err := tool.Tools.Get("ThunderBrowser") if err != nil { common.ErrorResp(c, err, 500) return } if _, err := _tool.Init(); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, "ok") } func OfflineDownloadTools(c *gin.Context) { tools := tool.Tools.Names() common.SuccessResp(c, tools) } type AddOfflineDownloadReq struct { Urls []string `json:"urls"` Path string `json:"path"` Tool string `json:"tool"` DeletePolicy string `json:"delete_policy"` } func AddOfflineDownload(c *gin.Context) { user := c.Request.Context().Value(conf.UserKey).(*model.User) if !user.CanAddOfflineDownloadTasks() { common.ErrorStrResp(c, "permission denied", 403) return } var req AddOfflineDownloadReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } var tasks []task.TaskExtensionInfo for _, url := range req.Urls { // Filter out empty lines and whitespace-only strings trimmedUrl := strings.TrimSpace(url) if trimmedUrl == "" { continue } t, err := tool.AddURL(c, &tool.AddURLArgs{ URL: trimmedUrl, DstDirPath: reqPath, Tool: req.Tool, DeletePolicy: tool.DeletePolicy(req.DeletePolicy), }) if err != nil { common.ErrorResp(c, err, 500) return } if t != nil { tasks = append(tasks, t) } } common.SuccessResp(c, gin.H{ "tasks": getTaskInfos(tasks), }) } ================================================ FILE: server/handles/scan.go ================================================ package handles import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" ) type ManualScanReq struct { Path string `json:"path"` Limit float64 `json:"limit"` } func StartManualScan(c *gin.Context) { var req ManualScanReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if err := op.BeginManualScan(req.Path, req.Limit); err != nil { common.ErrorResp(c, err, 400) return } common.SuccessResp(c) } func StopManualScan(c *gin.Context) { if !op.ManualScanRunning() { common.ErrorStrResp(c, "manual scan is not running", 400) return } op.StopManualScan() common.SuccessResp(c) } type ManualScanResp struct { ObjCount uint64 `json:"obj_count"` IsDone bool `json:"is_done"` } func GetManualScanProgress(c *gin.Context) { ret := ManualScanResp{ ObjCount: op.ScannedCount.Load(), IsDone: !op.ManualScanRunning(), } common.SuccessResp(c, ret) } ================================================ FILE: server/handles/search.go ================================================ package handles import ( "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/search" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" ) type SearchReq struct { model.SearchReq Password string `json:"password"` } type SearchResp struct { model.SearchNode Type int `json:"type"` } func Search(c *gin.Context) { var ( req SearchReq err error ) if err = c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) req.Parent, err = user.JoinPath(req.Parent) if err != nil { common.ErrorResp(c, err, 400) return } if err := req.Validate(); err != nil { common.ErrorResp(c, err, 400) return } nodes, total, err := search.Search(c, req.SearchReq) if err != nil { common.ErrorResp(c, err, 500) return } var filteredNodes []model.SearchNode for _, node := range nodes { if !strings.HasPrefix(node.Parent, user.BasePath) { continue } meta, err := op.GetNearestMeta(node.Parent) if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { continue } if !common.CanAccess(user, meta, path.Join(node.Parent, node.Name), req.Password) { continue } filteredNodes = append(filteredNodes, node) } common.SuccessResp(c, common.PageResp{ Content: utils.MustSliceConvert(filteredNodes, nodeToSearchResp), Total: total, }) } func nodeToSearchResp(node model.SearchNode) SearchResp { return SearchResp{ SearchNode: node, Type: utils.GetObjType(node.Name, node.IsDir), } } ================================================ FILE: server/handles/setting.go ================================================ package handles import ( "sort" "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/data" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/server/static" "github.com/gin-gonic/gin" ) func ResetToken(c *gin.Context) { token := random.Token() item := model.SettingItem{Key: "token", Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} if err := op.SaveSettingItem(&item); err != nil { common.ErrorResp(c, err, 500) return } sign.Instance() common.SuccessResp(c, token) } func GetSetting(c *gin.Context) { key := c.Query("key") keys := c.Query("keys") if key != "" { item, err := op.GetSettingItemByKey(key) if err != nil { common.ErrorResp(c, err, 400) return } common.SuccessResp(c, item) } else { items, err := op.GetSettingItemInKeys(strings.Split(keys, ",")) if err != nil { common.ErrorResp(c, err, 400) return } common.SuccessResp(c, items) } } func SaveSettings(c *gin.Context) { var req []model.SettingItem if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if err := op.SaveSettingItems(req); err != nil { common.ErrorResp(c, err, 500) } else { common.SuccessResp(c) static.UpdateIndex() } } func ListSettings(c *gin.Context) { groupStr := c.Query("group") groupsStr := c.Query("groups") var settings []model.SettingItem var err error if groupsStr == "" && groupStr == "" { settings, err = op.GetSettingItems() } else { var groupStrings []string if groupsStr != "" { groupStrings = strings.Split(groupsStr, ",") } else { groupStrings = append(groupStrings, groupStr) } var groups []int for _, str := range groupStrings { group, err := strconv.Atoi(str) if err != nil { common.ErrorResp(c, err, 400) return } groups = append(groups, group) } settings, err = op.GetSettingItemsInGroups(groups) } if err != nil { common.ErrorResp(c, err, 400) return } common.SuccessResp(c, settings) } func DefaultSettings(c *gin.Context) { groupStr := c.Query("group") groupsStr := c.Query("groups") settings := data.InitialSettings() if groupsStr == "" && groupStr == "" { for i := range settings { (&settings[i]).Index = uint(i) } common.SuccessResp(c, settings) } else { var groupStrings []string if groupsStr != "" { groupStrings = strings.Split(groupsStr, ",") } else { groupStrings = append(groupStrings, groupStr) } var groups []int for _, str := range groupStrings { group, err := strconv.Atoi(str) if err != nil { common.ErrorResp(c, err, 400) return } groups = append(groups, group) } sort.Ints(groups) var resultItems []model.SettingItem for _, group := range groups { for i := range settings { item := settings[i] if group == item.Group { item.Index = uint(i) resultItems = append(resultItems, item) } } } common.SuccessResp(c, resultItems) } } func DeleteSetting(c *gin.Context) { key := c.Query("key") if err := op.DeleteSettingItemByKey(key); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c) } func PublicSettings(c *gin.Context) { common.SuccessResp(c, op.GetPublicSettingsMap()) } ================================================ FILE: server/handles/sharing.go ================================================ package handles import ( "fmt" stdpath "path" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/sharing" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/go-cache" "github.com/gin-gonic/gin" "github.com/pkg/errors" ) func SharingGet(c *gin.Context, req *FsGetReq) { sid, path, _ := strings.Cut(strings.TrimPrefix(req.Path, "/"), "/") if sid == "" { common.ErrorStrResp(c, "invalid share id", 400) return } s, obj, err := sharing.Get(c.Request.Context(), sid, path, model.SharingListArgs{ Refresh: false, Pwd: req.Password, }) if dealError(c, err) { return } _ = countAccess(c.ClientIP(), s) url := "" if !obj.IsDir() { fakePath := fmt.Sprintf("/%s/%s", sid, path) url = fmt.Sprintf("%s/sd%s", common.GetApiUrl(c), utils.EncodePath(fakePath, true)) if s.Pwd != "" { url += "?pwd=" + s.Pwd } } thumb, _ := model.GetThumb(obj) common.SuccessResp(c, FsGetResp{ ObjResp: ObjResp{ Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), Modified: obj.ModTime(), Created: obj.CreateTime(), HashInfoStr: obj.GetHash().String(), HashInfo: obj.GetHash().Export(), Sign: "", Type: utils.GetFileType(obj.GetName()), Thumb: thumb, }, RawURL: url, Readme: s.Readme, Header: s.Header, Provider: "unknown", Related: nil, }) } func SharingList(c *gin.Context, req *ListReq) { sid, path, _ := strings.Cut(strings.TrimPrefix(req.Path, "/"), "/") if sid == "" { common.ErrorStrResp(c, "invalid share id", 400) return } s, objs, err := sharing.List(c.Request.Context(), sid, path, model.SharingListArgs{ Refresh: req.Refresh, Pwd: req.Password, }) if dealError(c, err) { return } _ = countAccess(c.ClientIP(), s) total, objs := pagination(objs, &req.PageReq) common.SuccessResp(c, FsListResp{ Content: utils.MustSliceConvert(objs, func(obj model.Obj) ObjResp { thumb, _ := model.GetThumb(obj) return ObjResp{ Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), Modified: obj.ModTime(), Created: obj.CreateTime(), HashInfoStr: obj.GetHash().String(), HashInfo: obj.GetHash().Export(), Sign: "", Thumb: thumb, Type: utils.GetObjType(obj.GetName(), obj.IsDir()), } }), Total: int64(total), Readme: s.Readme, Header: s.Header, Write: false, Provider: "unknown", }) } func SharingArchiveMeta(c *gin.Context, req *ArchiveMetaReq) { if !setting.GetBool(conf.ShareArchivePreview) { common.ErrorStrResp(c, "sharing archives previewing is not allowed", 403) return } sid, path, _ := strings.Cut(strings.TrimPrefix(req.Path, "/"), "/") if sid == "" { common.ErrorStrResp(c, "invalid share id", 400) return } archiveArgs := model.ArchiveArgs{ LinkArgs: model.LinkArgs{ Header: c.Request.Header, Type: c.Query("type"), }, Password: req.ArchivePass, } s, ret, err := sharing.ArchiveMeta(c.Request.Context(), sid, path, model.SharingArchiveMetaArgs{ ArchiveMetaArgs: model.ArchiveMetaArgs{ ArchiveArgs: archiveArgs, Refresh: req.Refresh, }, Pwd: req.Password, }) if dealError(c, err) { return } _ = countAccess(c.ClientIP(), s) fakePath := fmt.Sprintf("/%s/%s", sid, path) url := fmt.Sprintf("%s/sad%s", common.GetApiUrl(c), utils.EncodePath(fakePath, true)) if s.Pwd != "" { url += "?pwd=" + s.Pwd } common.SuccessResp(c, ArchiveMetaResp{ Comment: ret.GetComment(), IsEncrypted: ret.IsEncrypted(), Content: toContentResp(ret.GetTree()), Sort: ret.Sort, RawURL: url, Sign: "", }) } func SharingArchiveList(c *gin.Context, req *ArchiveListReq) { if !setting.GetBool(conf.ShareArchivePreview) { common.ErrorStrResp(c, "sharing archives previewing is not allowed", 403) return } sid, path, _ := strings.Cut(strings.TrimPrefix(req.Path, "/"), "/") if sid == "" { common.ErrorStrResp(c, "invalid share id", 400) return } innerArgs := model.ArchiveInnerArgs{ ArchiveArgs: model.ArchiveArgs{ LinkArgs: model.LinkArgs{ Header: c.Request.Header, Type: c.Query("type"), }, Password: req.ArchivePass, }, InnerPath: utils.FixAndCleanPath(req.InnerPath), } s, objs, err := sharing.ArchiveList(c.Request.Context(), sid, path, model.SharingArchiveListArgs{ ArchiveListArgs: model.ArchiveListArgs{ ArchiveInnerArgs: innerArgs, Refresh: req.Refresh, }, Pwd: req.Password, }) if dealError(c, err) { return } _ = countAccess(c.ClientIP(), s) total, objs := pagination(objs, &req.PageReq) ret, _ := utils.SliceConvert(objs, func(src model.Obj) (ObjResp, error) { return toObjsRespWithoutSignAndThumb(src), nil }) common.SuccessResp(c, common.PageResp{ Content: ret, Total: int64(total), }) } func SharingDown(c *gin.Context) { sid := c.Request.Context().Value(conf.SharingIDKey).(string) path := c.Request.Context().Value(conf.PathKey).(string) path = utils.FixAndCleanPath(path) pwd := c.Query("pwd") s, err := op.GetSharingById(sid) if err == nil { if !s.Valid() { err = errs.InvalidSharing } else if !s.Verify(pwd) { err = errs.WrongShareCode } else if len(s.Files) != 1 && path == "/" { err = errors.New("cannot get sharing root link") } } if dealErrorPage(c, err) { return } unwrapPath, err := op.GetSharingUnwrapPath(s, path) if err != nil { common.ErrorPage(c, errors.New("failed get sharing unwrap path"), 500) return } storage, actualPath, err := op.GetStorageAndActualPath(unwrapPath) if dealErrorPage(c, err) { return } if setting.GetBool(conf.ShareForceProxy) || common.ShouldProxy(storage, stdpath.Base(actualPath)) { if _, ok := c.GetQuery("d"); !ok { if url := common.GenerateDownProxyURL(storage.GetStorage(), unwrapPath); url != "" { c.Redirect(302, url) _ = countAccess(c.ClientIP(), s) return } } link, obj, err := op.Link(c.Request.Context(), storage, actualPath, model.LinkArgs{ Header: c.Request.Header, Type: c.Query("type"), }) if err != nil { common.ErrorPage(c, errors.WithMessage(err, "failed get sharing link"), 500) return } _ = countAccess(c.ClientIP(), s) proxy(c, link, obj, storage.GetStorage().ProxyRange) } else { link, _, err := op.Link(c.Request.Context(), storage, actualPath, model.LinkArgs{ IP: c.ClientIP(), Header: c.Request.Header, Type: c.Query("type"), Redirect: true, }) if err != nil { common.ErrorPage(c, errors.WithMessage(err, "failed get sharing link"), 500) return } _ = countAccess(c.ClientIP(), s) redirect(c, link) } } func SharingArchiveExtract(c *gin.Context) { if !setting.GetBool(conf.ShareArchivePreview) { common.ErrorPage(c, errors.New("sharing archives previewing is not allowed"), 403) return } sid := c.Request.Context().Value(conf.SharingIDKey).(string) path := c.Request.Context().Value(conf.PathKey).(string) path = utils.FixAndCleanPath(path) pwd := c.Query("pwd") innerPath := utils.FixAndCleanPath(c.Query("inner")) archivePass := c.Query("pass") s, err := op.GetSharingById(sid) if err == nil { if !s.Valid() { err = errs.InvalidSharing } else if !s.Verify(pwd) { err = errs.WrongShareCode } else if len(s.Files) != 1 && path == "/" { err = errors.New("cannot extract sharing root") } } if dealErrorPage(c, err) { return } unwrapPath, err := op.GetSharingUnwrapPath(s, path) if err != nil { common.ErrorPage(c, errors.New("failed get sharing unwrap path"), 500) return } storage, actualPath, err := op.GetStorageAndActualPath(unwrapPath) if dealErrorPage(c, err) { return } args := model.ArchiveInnerArgs{ ArchiveArgs: model.ArchiveArgs{ LinkArgs: model.LinkArgs{ Header: c.Request.Header, Type: c.Query("type"), }, Password: archivePass, }, InnerPath: innerPath, } if _, ok := storage.(driver.ArchiveReader); ok { if setting.GetBool(conf.ShareForceProxy) || common.ShouldProxy(storage, stdpath.Base(actualPath)) { link, obj, err := op.DriverExtract(c.Request.Context(), storage, actualPath, args) if dealErrorPage(c, err) { return } proxy(c, link, obj, storage.GetStorage().ProxyRange) } else { args.Redirect = true link, _, err := op.DriverExtract(c.Request.Context(), storage, actualPath, args) if dealErrorPage(c, err) { return } redirect(c, link) } } else { rc, size, err := op.InternalExtract(c.Request.Context(), storage, actualPath, args) if dealErrorPage(c, err) { return } fileName := stdpath.Base(innerPath) proxyInternalExtract(c, rc, size, fileName) } } func dealError(c *gin.Context, err error) bool { if err == nil { return false } else if errors.Is(err, errs.SharingNotFound) { common.ErrorStrResp(c, "the share does not exist", 500) } else if errors.Is(err, errs.InvalidSharing) { common.ErrorStrResp(c, "the share has expired or is no longer valid", 500) } else if errors.Is(err, errs.WrongShareCode) { common.ErrorResp(c, err, 403) } else if errors.Is(err, errs.WrongArchivePassword) { common.ErrorResp(c, err, 202) } else { common.ErrorResp(c, err, 500) } return true } func dealErrorPage(c *gin.Context, err error) bool { if err == nil { return false } else if errors.Is(err, errs.SharingNotFound) { common.ErrorPage(c, errors.New("the share does not exist"), 500) } else if errors.Is(err, errs.InvalidSharing) { common.ErrorPage(c, errors.New("the share has expired or is no longer valid"), 500) } else if errors.Is(err, errs.WrongShareCode) { common.ErrorPage(c, err, 403) } else if errors.Is(err, errs.WrongArchivePassword) { common.ErrorPage(c, err, 202) } else { common.ErrorPage(c, err, 500) } return true } type SharingResp struct { *model.Sharing CreatorName string `json:"creator"` CreatorRole int `json:"creator_role"` } func GetSharing(c *gin.Context) { sid := c.Query("id") user := c.Request.Context().Value(conf.UserKey).(*model.User) s, err := op.GetSharingById(sid) if err != nil || (!user.IsAdmin() && s.Creator.ID != user.ID) { common.ErrorStrResp(c, "sharing not found", 404) return } common.SuccessResp(c, SharingResp{ Sharing: s, CreatorName: s.Creator.Username, CreatorRole: s.Creator.Role, }) } func ListSharings(c *gin.Context) { var req model.PageReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } req.Validate() user := c.Request.Context().Value(conf.UserKey).(*model.User) var sharings []model.Sharing var total int64 var err error if user.IsAdmin() { sharings, total, err = op.GetSharings(req.Page, req.PerPage) } else { sharings, total, err = op.GetSharingsByCreatorId(user.ID, req.Page, req.PerPage) } if err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c, common.PageResp{ Content: utils.MustSliceConvert(sharings, func(s model.Sharing) SharingResp { return SharingResp{ Sharing: &s, CreatorName: s.Creator.Username, CreatorRole: s.Creator.Role, } }), Total: total, }) } type UpdateSharingReq struct { Files []string `json:"files"` Expires *time.Time `json:"expires"` Pwd string `json:"pwd"` MaxAccessed int `json:"max_accessed"` Disabled bool `json:"disabled"` Remark string `json:"remark"` Readme string `json:"readme"` Header string `json:"header"` model.Sort CreatorName string `json:"creator"` Accessed int `json:"accessed"` ID string `json:"id"` } func UpdateSharing(c *gin.Context) { var req UpdateSharingReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if len(req.Files) == 0 || (len(req.Files) == 1 && req.Files[0] == "") { common.ErrorStrResp(c, "must add at least 1 object", 400) return } var user *model.User var err error reqUser := c.Request.Context().Value(conf.UserKey).(*model.User) if reqUser.IsAdmin() && req.CreatorName != "" { user, err = op.GetUserByName(req.CreatorName) if err != nil { common.ErrorStrResp(c, "no such a user", 400) return } } else { user = reqUser if !user.CanShare() { common.ErrorStrResp(c, "permission denied", 403) return } } for i, s := range req.Files { s = utils.FixAndCleanPath(s) req.Files[i] = s if !reqUser.IsAdmin() && !strings.HasPrefix(s, user.BasePath) { common.ErrorStrResp(c, fmt.Sprintf("permission denied to share path [%s]", s), 500) return } } s, err := op.GetSharingById(req.ID) if err != nil || (!reqUser.IsAdmin() && s.CreatorId != user.ID) { common.ErrorStrResp(c, "sharing not found", 404) return } if reqUser.IsAdmin() && req.CreatorName == "" { user = s.Creator } s.Files = req.Files s.Expires = req.Expires s.Pwd = req.Pwd s.Accessed = req.Accessed s.MaxAccessed = req.MaxAccessed s.Disabled = req.Disabled s.Sort = req.Sort s.Header = req.Header s.Readme = req.Readme s.Remark = req.Remark s.Creator = user if err = op.UpdateSharing(s); err != nil { common.ErrorResp(c, err, 500) } else { common.SuccessResp(c, SharingResp{ Sharing: s, CreatorName: s.Creator.Username, CreatorRole: s.Creator.Role, }) } } func CreateSharing(c *gin.Context) { var req UpdateSharingReq var err error if err = c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if len(req.Files) == 0 || (len(req.Files) == 1 && req.Files[0] == "") { common.ErrorStrResp(c, "must add at least 1 object", 400) return } var user *model.User reqUser := c.Request.Context().Value(conf.UserKey).(*model.User) if reqUser.IsAdmin() && req.CreatorName != "" { user, err = op.GetUserByName(req.CreatorName) if err != nil { common.ErrorStrResp(c, "no such a user", 400) return } } else { user = reqUser if !user.CanShare() || (!user.IsAdmin() && req.ID != "") { common.ErrorStrResp(c, "permission denied", 403) return } } for i, s := range req.Files { s = utils.FixAndCleanPath(s) req.Files[i] = s if !reqUser.IsAdmin() && !strings.HasPrefix(s, user.BasePath) { common.ErrorStrResp(c, fmt.Sprintf("permission denied to share path [%s]", s), 500) return } } s := &model.Sharing{ SharingDB: &model.SharingDB{ ID: req.ID, Expires: req.Expires, Pwd: req.Pwd, Accessed: req.Accessed, MaxAccessed: req.MaxAccessed, Disabled: req.Disabled, Sort: req.Sort, Remark: req.Remark, Readme: req.Readme, Header: req.Header, }, Files: req.Files, Creator: user, } var id string if id, err = op.CreateSharing(s); err != nil { common.ErrorResp(c, err, 500) } else { s.ID = id common.SuccessResp(c, SharingResp{ Sharing: s, CreatorName: s.Creator.Username, CreatorRole: s.Creator.Role, }) } } func DeleteSharing(c *gin.Context) { sid := c.Query("id") user := c.Request.Context().Value(conf.UserKey).(*model.User) s, err := op.GetSharingById(sid) if err != nil || (!user.IsAdmin() && s.CreatorId != user.ID) { common.ErrorResp(c, err, 404) return } if err = op.DeleteSharing(sid); err != nil { common.ErrorResp(c, err, 500) } else { common.SuccessResp(c) } } func SetEnableSharing(disable bool) func(ctx *gin.Context) { return func(c *gin.Context) { sid := c.Query("id") user := c.Request.Context().Value(conf.UserKey).(*model.User) s, err := op.GetSharingById(sid) if err != nil || (!user.IsAdmin() && s.CreatorId != user.ID) { common.ErrorStrResp(c, "sharing not found", 404) return } s.Disabled = disable if err = op.UpdateSharing(s, true); err != nil { common.ErrorResp(c, err, 500) } else { common.SuccessResp(c) } } } var ( AccessCache = cache.NewMemCache[interface{}]() AccessCountDelay = 30 * time.Minute ) func countAccess(ip string, s *model.Sharing) error { key := fmt.Sprintf("%s:%s", s.ID, ip) _, ok := AccessCache.Get(key) if !ok { AccessCache.Set(key, struct{}{}, cache.WithEx[interface{}](AccessCountDelay)) s.Accessed += 1 return op.UpdateSharing(s, true) } return nil } ================================================ FILE: server/handles/sshkey.go ================================================ package handles import ( "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" ) type SSHKeyAddReq struct { Title string `json:"title" binding:"required"` Key string `json:"key" binding:"required"` } func AddMyPublicKey(c *gin.Context) { userObj, ok := c.Request.Context().Value(conf.UserKey).(*model.User) if !ok || userObj.IsGuest() { common.ErrorStrResp(c, "user invalid", 401) return } var req SSHKeyAddReq if err := c.ShouldBind(&req); err != nil { common.ErrorStrResp(c, "request invalid", 400) return } if req.Title == "" { common.ErrorStrResp(c, "request invalid", 400) return } key := &model.SSHPublicKey{ Title: req.Title, KeyStr: strings.TrimSpace(req.Key), UserId: userObj.ID, } err, parsed := op.CreateSSHPublicKey(key) if !parsed { common.ErrorStrResp(c, "provided key invalid", 400) return } else if err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c) } func ListMyPublicKey(c *gin.Context) { userObj, ok := c.Request.Context().Value(conf.UserKey).(*model.User) if !ok || userObj.IsGuest() { common.ErrorStrResp(c, "user invalid", 401) return } list(c, userObj) } func DeleteMyPublicKey(c *gin.Context) { userObj, ok := c.Request.Context().Value(conf.UserKey).(*model.User) if !ok || userObj.IsGuest() { common.ErrorStrResp(c, "user invalid", 401) return } keyId, err := strconv.Atoi(c.Query("id")) if err != nil { common.ErrorStrResp(c, "id format invalid", 400) return } key, err := op.GetSSHPublicKeyByIdAndUserId(uint(keyId), userObj.ID) if err != nil { common.ErrorStrResp(c, "failed to get public key", 404) return } err = op.DeleteSSHPublicKeyById(key.ID) if err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c) } func ListPublicKeys(c *gin.Context) { userId, err := strconv.Atoi(c.Query("uid")) if err != nil { common.ErrorStrResp(c, "user id format invalid", 400) return } userObj, err := op.GetUserById(uint(userId)) if err != nil { common.ErrorStrResp(c, "user invalid", 404) return } list(c, userObj) } func DeletePublicKey(c *gin.Context) { keyId, err := strconv.Atoi(c.Query("id")) if err != nil { common.ErrorStrResp(c, "id format invalid", 400) return } err = op.DeleteSSHPublicKeyById(uint(keyId)) if err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c) } func list(c *gin.Context, userObj *model.User) { var req model.PageReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } req.Validate() keys, total, err := op.GetSSHPublicKeyByUserId(userObj.ID, req.Page, req.PerPage) if err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c, common.PageResp{ Content: keys, Total: total, }) } ================================================ FILE: server/handles/ssologin.go ================================================ package handles import ( "encoding/base64" "errors" "fmt" "net/http" "net/url" "path" "strings" "time" "github.com/OpenListTeam/go-cache" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/coreos/go-oidc" "github.com/gin-gonic/gin" "github.com/go-resty/resty/v2" "golang.org/x/oauth2" "gorm.io/gorm" ) const stateLength = 16 const stateExpire = time.Minute * 5 var stateCache = cache.NewMemCache[string](cache.WithShards[string](stateLength)) func _keyState(clientID, state string) string { return fmt.Sprintf("%s_%s", clientID, state) } func generateState(clientID, ip string) string { state := random.String(stateLength) stateCache.Set(_keyState(clientID, state), ip, cache.WithEx[string](stateExpire)) return state } func verifyState(clientID, ip, state string) bool { value, ok := stateCache.Get(_keyState(clientID, state)) return ok && value == ip } func ssoRedirectUri(c *gin.Context, useCompatibility bool, method string) string { if useCompatibility { return common.GetApiUrl(c) + "/api/auth/" + method } else { return common.GetApiUrl(c) + "/api/auth/sso_callback" + "?method=" + method } } func SSOLoginRedirect(c *gin.Context) { method := c.Query("method") useCompatibility := setting.GetBool(conf.SSOCompatibilityMode) enabled := setting.GetBool(conf.SSOLoginEnabled) clientId := setting.GetStr(conf.SSOClientId) platform := setting.GetStr(conf.SSOLoginPlatform) var rUrl string if !enabled { common.ErrorStrResp(c, "Single sign-on is not enabled", 403) return } urlValues := url.Values{} if method == "" { common.ErrorStrResp(c, "no method provided", 400) return } redirectUri := ssoRedirectUri(c, useCompatibility, method) urlValues.Add("response_type", "code") urlValues.Add("redirect_uri", redirectUri) urlValues.Add("client_id", clientId) switch platform { case "Github": rUrl = "https://github.com/login/oauth/authorize?" urlValues.Add("scope", "read:user") case "Microsoft": rUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/authorize?" urlValues.Add("scope", "user.read") urlValues.Add("response_mode", "query") case "Google": rUrl = "https://accounts.google.com/o/oauth2/v2/auth?" urlValues.Add("scope", "https://www.googleapis.com/auth/userinfo.profile") case "Dingtalk": rUrl = "https://login.dingtalk.com/oauth2/auth?" urlValues.Add("scope", "openid") urlValues.Add("prompt", "consent") urlValues.Add("response_type", "code") case "Casdoor": endpoint := strings.TrimSuffix(setting.GetStr(conf.SSOEndpointName), "/") rUrl = endpoint + "/login/oauth/authorize?" urlValues.Add("scope", "profile") urlValues.Add("state", endpoint) case "OIDC": oauth2Config, err := GetOIDCClient(c, useCompatibility, redirectUri, method) if err != nil { common.ErrorStrResp(c, err.Error(), 400) return } state := generateState(clientId, c.ClientIP()) c.Redirect(http.StatusFound, oauth2Config.AuthCodeURL(state)) return default: common.ErrorStrResp(c, "invalid platform", 400) return } c.Redirect(302, rUrl+urlValues.Encode()) } var ssoClient = resty.New().SetRetryCount(3) func GetOIDCClient(c *gin.Context, useCompatibility bool, redirectUri, method string) (*oauth2.Config, error) { if redirectUri == "" { redirectUri = ssoRedirectUri(c, useCompatibility, method) } endpoint := setting.GetStr(conf.SSOEndpointName) provider, err := oidc.NewProvider(c, endpoint) if err != nil { return nil, err } clientId := setting.GetStr(conf.SSOClientId) clientSecret := setting.GetStr(conf.SSOClientSecret) extraScopes := []string{} if setting.GetStr(conf.SSOExtraScopes) != "" { extraScopes = strings.Split(setting.GetStr(conf.SSOExtraScopes), " ") } return &oauth2.Config{ ClientID: clientId, ClientSecret: clientSecret, RedirectURL: redirectUri, // Discovery returns the OAuth2 endpoints. Endpoint: provider.Endpoint(), // "openid" is a required scope for OpenID Connect flows. Scopes: append([]string{oidc.ScopeOpenID, "profile"}, extraScopes...), }, nil } func autoRegister(username, userID string, err error) (*model.User, error) { if !errors.Is(err, gorm.ErrRecordNotFound) || !setting.GetBool(conf.SSOAutoRegister) { return nil, err } if username == "" { return nil, errors.New("cannot get username from SSO provider") } user := &model.User{ ID: 0, Username: username, Password: random.String(16), Permission: int32(setting.GetInt(conf.SSODefaultPermission, 0)), BasePath: setting.GetStr(conf.SSODefaultDir), Role: 0, Disabled: false, SsoID: userID, } if err = db.CreateUser(user); err != nil { if strings.HasPrefix(err.Error(), "UNIQUE constraint failed") && strings.HasSuffix(err.Error(), "username") { user.Username = user.Username + "_" + userID if err = db.CreateUser(user); err != nil { return nil, err } } else { return nil, err } } return user, nil } func parseJWT(p string) ([]byte, error) { parts := strings.Split(p, ".") if len(parts) < 2 { return nil, fmt.Errorf("oidc: malformed jwt, expected 3 parts got %d", len(parts)) } payload, err := base64.RawURLEncoding.DecodeString(parts[1]) if err != nil { return nil, fmt.Errorf("oidc: malformed jwt payload: %v", err) } return payload, nil } func OIDCLoginCallback(c *gin.Context) { useCompatibility := setting.GetBool(conf.SSOCompatibilityMode) method := c.Query("method") if useCompatibility { method = path.Base(c.Request.URL.Path) } clientId := setting.GetStr(conf.SSOClientId) endpoint := setting.GetStr(conf.SSOEndpointName) provider, err := oidc.NewProvider(c, endpoint) if err != nil { common.ErrorResp(c, err, 400) return } oauth2Config, err := GetOIDCClient(c, useCompatibility, "", method) if err != nil { common.ErrorResp(c, err, 400) return } if !verifyState(clientId, c.ClientIP(), c.Query("state")) { common.ErrorStrResp(c, "incorrect or expired state parameter", 400) return } oauth2Token, err := oauth2Config.Exchange(c, c.Query("code")) if err != nil { common.ErrorResp(c, err, 400) return } rawIDToken, ok := oauth2Token.Extra("id_token").(string) if !ok { common.ErrorStrResp(c, "no id_token found in oauth2 token", 400) return } verifier := provider.Verifier(&oidc.Config{ ClientID: clientId, }) _, err = verifier.Verify(c, rawIDToken) if err != nil { common.ErrorResp(c, err, 400) return } payload, err := parseJWT(rawIDToken) if err != nil { common.ErrorResp(c, err, 400) return } userID := utils.Json.Get(payload, setting.GetStr(conf.SSOOIDCUsernameKey, "name")).ToString() if userID == "" { common.ErrorStrResp(c, "cannot get username from OIDC provider", 400) return } if method == "get_sso_id" { if useCompatibility { c.Redirect(302, common.GetApiUrl(c)+"/@manage?sso_id="+userID) return } html := fmt.Sprintf(` `, userID) c.Data(200, "text/html; charset=utf-8", []byte(html)) return } if method == "sso_get_token" { user, err := db.GetUserBySSOID(userID) if err != nil { user, err = autoRegister(userID, userID, err) if err != nil { common.ErrorResp(c, err, 400) return } } token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400) return } if useCompatibility { c.Redirect(302, common.GetApiUrl(c)+"/@login?token="+token) return } html := fmt.Sprintf(` `, token) c.Data(200, "text/html; charset=utf-8", []byte(html)) return } } func SSOLoginCallback(c *gin.Context) { enabled := setting.GetBool(conf.SSOLoginEnabled) usecompatibility := setting.GetBool(conf.SSOCompatibilityMode) if !enabled { common.ErrorResp(c, errors.New("sso login is disabled"), 500) return } argument := c.Query("method") if usecompatibility { argument = path.Base(c.Request.URL.Path) } if !utils.SliceContains([]string{"get_sso_id", "sso_get_token"}, argument) { common.ErrorResp(c, errors.New("invalid request"), 500) return } clientId := setting.GetStr(conf.SSOClientId) platform := setting.GetStr(conf.SSOLoginPlatform) clientSecret := setting.GetStr(conf.SSOClientSecret) var tokenUrl, userUrl, scope, authField, idField, usernameField string additionalForm := make(map[string]string) switch platform { case "Github": tokenUrl = "https://github.com/login/oauth/access_token" userUrl = "https://api.github.com/user" authField = "code" scope = "read:user" idField = "id" usernameField = "login" case "Microsoft": tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token" userUrl = "https://graph.microsoft.com/v1.0/me" additionalForm["grant_type"] = "authorization_code" scope = "user.read" authField = "code" idField = "id" usernameField = "displayName" case "Google": tokenUrl = "https://oauth2.googleapis.com/token" userUrl = "https://www.googleapis.com/oauth2/v1/userinfo" additionalForm["grant_type"] = "authorization_code" scope = "https://www.googleapis.com/auth/userinfo.profile" authField = "code" idField = "id" usernameField = "name" case "Dingtalk": tokenUrl = "https://api.dingtalk.com/v1.0/oauth2/userAccessToken" userUrl = "https://api.dingtalk.com/v1.0/contact/users/me" authField = "authCode" idField = "unionId" usernameField = "nick" case "Casdoor": endpoint := strings.TrimSuffix(setting.GetStr(conf.SSOEndpointName), "/") tokenUrl = endpoint + "/api/login/oauth/access_token" userUrl = endpoint + "/api/userinfo" additionalForm["grant_type"] = "authorization_code" scope = "profile" authField = "code" idField = "sub" usernameField = "preferred_username" case "OIDC": OIDCLoginCallback(c) return default: common.ErrorStrResp(c, "invalid platform", 400) return } callbackCode := c.Query(authField) if callbackCode == "" { common.ErrorStrResp(c, "No code provided", 400) return } var resp *resty.Response var err error if platform == "Dingtalk" { resp, err = ssoClient.R().SetHeader("content-type", "application/json").SetHeader("Accept", "application/json"). SetBody(map[string]string{ "clientId": clientId, "clientSecret": clientSecret, "code": callbackCode, "grantType": "authorization_code", }). Post(tokenUrl) } else { var redirect_uri string if usecompatibility { redirect_uri = common.GetApiUrl(c) + "/api/auth/" + argument } else { redirect_uri = common.GetApiUrl(c) + "/api/auth/sso_callback" + "?method=" + argument } resp, err = ssoClient.R().SetHeader("Accept", "application/json"). SetFormData(map[string]string{ "client_id": clientId, "client_secret": clientSecret, "code": callbackCode, "redirect_uri": redirect_uri, "scope": scope, }).SetFormData(additionalForm).Post(tokenUrl) } if err != nil { common.ErrorResp(c, err, 400) return } if platform == "Dingtalk" { accessToken := utils.Json.Get(resp.Body(), "accessToken").ToString() resp, err = ssoClient.R().SetHeader("x-acs-dingtalk-access-token", accessToken). Get(userUrl) } else { accessToken := utils.Json.Get(resp.Body(), "access_token").ToString() resp, err = ssoClient.R().SetHeader("Authorization", "Bearer "+accessToken). Get(userUrl) } if err != nil { common.ErrorResp(c, err, 400) return } userID := utils.Json.Get(resp.Body(), idField).ToString() if utils.SliceContains([]string{"", "0"}, userID) { common.ErrorResp(c, errors.New("error occurred"), 400) return } if argument == "get_sso_id" { if usecompatibility { c.Redirect(302, common.GetApiUrl(c)+"/@manage?sso_id="+userID) return } html := fmt.Sprintf(` `, userID) c.Data(200, "text/html; charset=utf-8", []byte(html)) return } username := utils.Json.Get(resp.Body(), usernameField).ToString() user, err := db.GetUserBySSOID(userID) if err != nil { user, err = autoRegister(username, userID, err) if err != nil { common.ErrorResp(c, err, 400) return } } token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400) return } if usecompatibility { c.Redirect(302, common.GetApiUrl(c)+"/@login?token="+token) return } html := fmt.Sprintf(` `, token) c.Data(200, "text/html; charset=utf-8", []byte(html)) } ================================================ FILE: server/handles/storage.go ================================================ package handles import ( "context" "errors" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) type StorageResp struct { model.Storage MountDetails *model.StorageDetails `json:"mount_details,omitempty"` } type detailWithIndex struct { idx int val *model.StorageDetails } func makeStorageResp(ctx *gin.Context, storages []model.Storage) []*StorageResp { ret := make([]*StorageResp, len(storages)) detailsChan := make(chan detailWithIndex, len(storages)) workerCount := 0 for i, s := range storages { ret[i] = &StorageResp{ Storage: s, MountDetails: nil, } if setting.GetBool(conf.HideStorageDetailsInManagePage) { continue } d, err := op.GetStorageByMountPath(s.MountPath) if err != nil { continue } _, ok := d.(driver.WithDetails) if !ok { continue } workerCount++ go func(dri driver.Driver, idx int) { details, e := op.GetStorageDetails(ctx, dri) if e != nil { if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) { log.Errorf("failed get %s details: %+v", dri.GetStorage().MountPath, e) } } detailsChan <- detailWithIndex{idx: idx, val: details} }(d, i) } for workerCount > 0 { select { case r := <-detailsChan: ret[r.idx].MountDetails = r.val workerCount-- case <-time.After(time.Second * 3): workerCount = 0 } } return ret } func ListStorages(c *gin.Context) { var req model.PageReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } req.Validate() log.Debugf("%+v", req) storages, total, err := db.GetStorages(req.Page, req.PerPage) if err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c, common.PageResp{ Content: makeStorageResp(c, storages), Total: total, }) } func CreateStorage(c *gin.Context) { var req model.Storage if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if id, err := op.CreateStorage(c.Request.Context(), req); err != nil { common.ErrorWithDataResp(c, err, 500, gin.H{ "id": id, }, true) } else { common.SuccessResp(c, gin.H{ "id": id, }) } } func UpdateStorage(c *gin.Context) { var req model.Storage if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if err := op.UpdateStorage(c.Request.Context(), req); err != nil { common.ErrorResp(c, err, 500, true) } else { common.SuccessResp(c) } } func DeleteStorage(c *gin.Context) { idStr := c.Query("id") id, err := strconv.Atoi(idStr) if err != nil { common.ErrorResp(c, err, 400) return } if err := op.DeleteStorageById(c.Request.Context(), uint(id)); err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c) } func DisableStorage(c *gin.Context) { idStr := c.Query("id") id, err := strconv.Atoi(idStr) if err != nil { common.ErrorResp(c, err, 400) return } if err := op.DisableStorage(c.Request.Context(), uint(id)); err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c) } func EnableStorage(c *gin.Context) { idStr := c.Query("id") id, err := strconv.Atoi(idStr) if err != nil { common.ErrorResp(c, err, 400) return } if err := op.EnableStorage(c.Request.Context(), uint(id)); err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c) } func GetStorage(c *gin.Context) { idStr := c.Query("id") id, err := strconv.Atoi(idStr) if err != nil { common.ErrorResp(c, err, 400) return } storage, err := db.GetStorageById(uint(id)) if err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c, storage) } func LoadAllStorages(c *gin.Context) { storages, err := db.GetEnabledStorages() if err != nil { log.Errorf("failed get enabled storages: %+v", err) common.ErrorResp(c, err, 500, true) return } conf.ResetStoragesLoadSignal() go func(storages []model.Storage) { for _, storage := range storages { storageDriver, err := op.GetStorageByMountPath(storage.MountPath) if err != nil { log.Errorf("failed get storage driver: %+v", err) continue } // drop the storage in the driver if err := storageDriver.Drop(context.Background()); err != nil { log.Errorf("failed drop storage: %+v", err) continue } if err := op.LoadStorage(context.Background(), storage); err != nil { log.Errorf("failed get enabled storages: %+v", err) continue } log.Infof("success load storage: [%s], driver: [%s]", storage.MountPath, storage.Driver) } conf.SendStoragesLoadedSignal() }(storages) common.SuccessResp(c) } ================================================ FILE: server/handles/task.go ================================================ package handles import ( "math" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" "github.com/gin-gonic/gin" ) type TaskInfo struct { ID string `json:"id"` Name string `json:"name"` Creator string `json:"creator"` CreatorRole int `json:"creator_role"` State tache.State `json:"state"` Status string `json:"status"` Progress float64 `json:"progress"` StartTime *time.Time `json:"start_time"` EndTime *time.Time `json:"end_time"` TotalBytes int64 `json:"total_bytes"` Error string `json:"error"` } func getTaskInfo[T task.TaskExtensionInfo](task T) TaskInfo { errMsg := "" if task.GetErr() != nil { errMsg = task.GetErr().Error() } progress := task.GetProgress() // if progress is NaN, set it to 100 if math.IsNaN(progress) { progress = 100 } creatorName := "" creatorRole := -1 if task.GetCreator() != nil { creatorName = task.GetCreator().Username creatorRole = task.GetCreator().Role } return TaskInfo{ ID: task.GetID(), Name: task.GetName(), Creator: creatorName, CreatorRole: creatorRole, State: task.GetState(), Status: task.GetStatus(), Progress: progress, StartTime: task.GetStartTime(), EndTime: task.GetEndTime(), TotalBytes: task.GetTotalBytes(), Error: errMsg, } } func getTaskInfos[T task.TaskExtensionInfo](tasks []T) []TaskInfo { return utils.MustSliceConvert(tasks, getTaskInfo[T]) } func argsContains[T comparable](v T, slice ...T) bool { return utils.SliceContains(slice, v) } func getUserInfo(c *gin.Context) (bool, uint, bool) { if user, ok := c.Request.Context().Value(conf.UserKey).(*model.User); ok { return user.IsAdmin(), user.ID, true } else { return false, 0, false } } func getTargetedHandler[T task.TaskExtensionInfo](manager task.Manager[T], callback func(c *gin.Context, task T)) gin.HandlerFunc { return func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) if !ok { // if there is no bug, here is unreachable common.ErrorStrResp(c, "user invalid", 401) return } t, ok := manager.GetByID(c.Query("tid")) if !ok { common.ErrorStrResp(c, "task not found", 404) return } if !isAdmin && uid != t.GetCreator().ID { // to avoid an attacker using error messages to guess valid TID, return a 404 rather than a 403 common.ErrorStrResp(c, "task not found", 404) return } callback(c, t) } } func getBatchHandler[T task.TaskExtensionInfo](manager task.Manager[T], callback func(task T)) gin.HandlerFunc { return func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) if !ok { common.ErrorStrResp(c, "user invalid", 401) return } var tids []string if err := c.ShouldBind(&tids); err != nil { common.ErrorStrResp(c, "invalid request format", 400) return } retErrs := make(map[string]string) for _, tid := range tids { t, ok := manager.GetByID(tid) if !ok || (!isAdmin && uid != t.GetCreator().ID) { retErrs[tid] = "task not found" continue } callback(t) } common.SuccessResp(c, retErrs) } } func taskRoute[T task.TaskExtensionInfo](g *gin.RouterGroup, manager task.Manager[T]) { g.GET("/undone", func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) if !ok { // if there is no bug, here is unreachable common.ErrorStrResp(c, "user invalid", 401) return } common.SuccessResp(c, getTaskInfos(manager.GetByCondition(func(task T) bool { // avoid directly passing the user object into the function to reduce closure size return (isAdmin || uid == task.GetCreator().ID) && argsContains(task.GetState(), tache.StatePending, tache.StateRunning, tache.StateCanceling, tache.StateErrored, tache.StateFailing, tache.StateWaitingRetry, tache.StateBeforeRetry) }))) }) g.GET("/done", func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) if !ok { // if there is no bug, here is unreachable common.ErrorStrResp(c, "user invalid", 401) return } common.SuccessResp(c, getTaskInfos(manager.GetByCondition(func(task T) bool { return (isAdmin || uid == task.GetCreator().ID) && argsContains(task.GetState(), tache.StateCanceled, tache.StateFailed, tache.StateSucceeded) }))) }) g.POST("/info", getTargetedHandler(manager, func(c *gin.Context, task T) { common.SuccessResp(c, getTaskInfo(task)) })) g.POST("/cancel", getTargetedHandler(manager, func(c *gin.Context, task T) { manager.Cancel(task.GetID()) common.SuccessResp(c) })) g.POST("/delete", getTargetedHandler(manager, func(c *gin.Context, task T) { manager.Remove(task.GetID()) common.SuccessResp(c) })) g.POST("/retry", getTargetedHandler(manager, func(c *gin.Context, task T) { manager.Retry(task.GetID()) common.SuccessResp(c) })) g.POST("/cancel_some", getBatchHandler(manager, func(task T) { manager.Cancel(task.GetID()) })) g.POST("/delete_some", getBatchHandler(manager, func(task T) { manager.Remove(task.GetID()) })) g.POST("/retry_some", getBatchHandler(manager, func(task T) { manager.Retry(task.GetID()) })) g.POST("/clear_done", func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) if !ok { // if there is no bug, here is unreachable common.ErrorStrResp(c, "user invalid", 401) return } manager.RemoveByCondition(func(task T) bool { return (isAdmin || uid == task.GetCreator().ID) && argsContains(task.GetState(), tache.StateCanceled, tache.StateFailed, tache.StateSucceeded) }) common.SuccessResp(c) }) g.POST("/clear_succeeded", func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) if !ok { // if there is no bug, here is unreachable common.ErrorStrResp(c, "user invalid", 401) return } manager.RemoveByCondition(func(task T) bool { return (isAdmin || uid == task.GetCreator().ID) && task.GetState() == tache.StateSucceeded }) common.SuccessResp(c) }) g.POST("/retry_failed", func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) if !ok { // if there is no bug, here is unreachable common.ErrorStrResp(c, "user invalid", 401) return } tasks := manager.GetByCondition(func(task T) bool { return (isAdmin || uid == task.GetCreator().ID) && task.GetState() == tache.StateFailed }) for _, t := range tasks { manager.Retry(t.GetID()) } common.SuccessResp(c) }) } func SetupTaskRoute(g *gin.RouterGroup) { taskRoute(g.Group("/upload"), fs.UploadTaskManager) taskRoute(g.Group("/copy"), fs.CopyTaskManager) taskRoute(g.Group("/move"), fs.MoveTaskManager) taskRoute(g.Group("/offline_download"), tool.DownloadTaskManager) taskRoute(g.Group("/offline_download_transfer"), tool.TransferTaskManager) taskRoute(g.Group("/decompress"), fs.ArchiveDownloadTaskManager) taskRoute(g.Group("/decompress_upload"), fs.ArchiveContentUploadTaskManager) } ================================================ FILE: server/handles/user.go ================================================ package handles import ( "strconv" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) func ListUsers(c *gin.Context) { var req model.PageReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } req.Validate() log.Debugf("%+v", req) users, total, err := op.GetUsers(req.Page, req.PerPage) if err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c, common.PageResp{ Content: users, Total: total, }) } func CreateUser(c *gin.Context) { var req model.User if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } if req.IsAdmin() || req.IsGuest() { common.ErrorStrResp(c, "admin or guest user can not be created", 400, true) return } req.SetPassword(req.Password) req.Password = "" req.Authn = "[]" if err := op.CreateUser(&req); err != nil { common.ErrorResp(c, err, 500, true) } else { common.SuccessResp(c) } } func UpdateUser(c *gin.Context) { var req model.User if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } user, err := op.GetUserById(req.ID) if err != nil { common.ErrorResp(c, err, 500) return } if user.Role != req.Role { common.ErrorStrResp(c, "role can not be changed", 400) return } if req.Password == "" { req.PwdHash = user.PwdHash req.Salt = user.Salt } else { req.SetPassword(req.Password) req.Password = "" } if req.OtpSecret == "" { req.OtpSecret = user.OtpSecret } if req.Disabled && req.IsAdmin() { common.ErrorStrResp(c, "admin user can not be disabled", 400) return } if err := op.UpdateUser(&req); err != nil { common.ErrorResp(c, err, 500) } else { common.SuccessResp(c) } } func DeleteUser(c *gin.Context) { idStr := c.Query("id") id, err := strconv.Atoi(idStr) if err != nil { common.ErrorResp(c, err, 400) return } if err := op.DeleteUserById(uint(id)); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c) } func GetUser(c *gin.Context) { idStr := c.Query("id") id, err := strconv.Atoi(idStr) if err != nil { common.ErrorResp(c, err, 400) return } user, err := op.GetUserById(uint(id)) if err != nil { common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c, user) } func Cancel2FAById(c *gin.Context) { idStr := c.Query("id") id, err := strconv.Atoi(idStr) if err != nil { common.ErrorResp(c, err, 400) return } if err := op.Cancel2FAById(uint(id)); err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c) } func DelUserCache(c *gin.Context) { username := c.Query("username") err := op.DelUserCache(username) if err != nil { common.ErrorResp(c, err, 500) return } common.SuccessResp(c) } ================================================ FILE: server/handles/webauthn.go ================================================ package handles import ( "encoding/base64" "encoding/binary" "encoding/json" "fmt" "github.com/OpenListTeam/OpenList/v4/internal/authn" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/go-webauthn/webauthn/protocol" "github.com/go-webauthn/webauthn/webauthn" ) func BeginAuthnLogin(c *gin.Context) { enabled := setting.GetBool(conf.WebauthnLoginEnabled) if !enabled { common.ErrorStrResp(c, "WebAuthn is not enabled", 403) return } authnInstance, err := authn.NewAuthnInstance(c) if err != nil { common.ErrorResp(c, err, 400) return } var ( options *protocol.CredentialAssertion sessionData *webauthn.SessionData ) if username := c.Query("username"); username != "" { var user *model.User user, err = db.GetUserByName(username) if err == nil { options, sessionData, err = authnInstance.BeginLogin(user) } } else { // client-side discoverable login options, sessionData, err = authnInstance.BeginDiscoverableLogin() } if err != nil { common.ErrorResp(c, err, 400) return } val, err := json.Marshal(sessionData) if err != nil { common.ErrorResp(c, err, 400) return } common.SuccessResp(c, gin.H{ "options": options, "session": val, }) } func FinishAuthnLogin(c *gin.Context) { enabled := setting.GetBool(conf.WebauthnLoginEnabled) if !enabled { common.ErrorStrResp(c, "WebAuthn is not enabled", 403) return } authnInstance, err := authn.NewAuthnInstance(c) if err != nil { common.ErrorResp(c, err, 400) return } sessionDataString := c.GetHeader("session") sessionDataBytes, err := base64.StdEncoding.DecodeString(sessionDataString) if err != nil { common.ErrorResp(c, err, 400) return } var sessionData webauthn.SessionData if err := json.Unmarshal(sessionDataBytes, &sessionData); err != nil { common.ErrorResp(c, err, 400) return } var user *model.User if username := c.Query("username"); username != "" { user, err = db.GetUserByName(username) if err != nil { common.ErrorResp(c, err, 400) return } _, err = authnInstance.FinishLogin(user, sessionData, c.Request) } else { // client-side discoverable login _, err = authnInstance.FinishDiscoverableLogin(func(_, userHandle []byte) (webauthn.User, error) { // first param `rawID` in this callback function is equal to ID in webauthn.Credential, // but it's unnnecessary to check it. // userHandle param is equal to (User).WebAuthnID(). userID := uint(binary.LittleEndian.Uint64(userHandle)) user, err = db.GetUserById(userID) if err != nil { return nil, err } return user, nil }, sessionData, c.Request) } if err != nil { common.ErrorResp(c, err, 400) return } token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400, true) return } common.SuccessResp(c, gin.H{"token": token}) } func BeginAuthnRegistration(c *gin.Context) { enabled := setting.GetBool(conf.WebauthnLoginEnabled) if !enabled { common.ErrorStrResp(c, "WebAuthn is not enabled", 403) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) authnInstance, err := authn.NewAuthnInstance(c) if err != nil { common.ErrorResp(c, err, 400) return } options, sessionData, err := authnInstance.BeginRegistration(user) if err != nil { common.ErrorResp(c, err, 400) return } val, err := json.Marshal(sessionData) if err != nil { common.ErrorResp(c, err, 400) return } common.SuccessResp(c, gin.H{ "options": options, "session": val, }) } func FinishAuthnRegistration(c *gin.Context) { enabled := setting.GetBool(conf.WebauthnLoginEnabled) if !enabled { common.ErrorStrResp(c, "WebAuthn is not enabled", 403) return } user := c.Request.Context().Value(conf.UserKey).(*model.User) sessionDataString := c.GetHeader("Session") authnInstance, err := authn.NewAuthnInstance(c) if err != nil { common.ErrorResp(c, err, 400) return } sessionDataBytes, err := base64.StdEncoding.DecodeString(sessionDataString) if err != nil { common.ErrorResp(c, err, 400) return } var sessionData webauthn.SessionData if err := json.Unmarshal(sessionDataBytes, &sessionData); err != nil { common.ErrorResp(c, err, 400) return } credential, err := authnInstance.FinishRegistration(user, sessionData, c.Request) if err != nil { common.ErrorResp(c, err, 400) return } err = db.RegisterAuthn(user, credential) if err != nil { common.ErrorResp(c, err, 400) return } err = op.DelUserCache(user.Username) if err != nil { common.ErrorResp(c, err, 400) return } common.SuccessResp(c, "Registered Successfully") } func DeleteAuthnLogin(c *gin.Context) { user := c.Request.Context().Value(conf.UserKey).(*model.User) type DeleteAuthnReq struct { ID string `json:"id"` } var req DeleteAuthnReq err := c.ShouldBind(&req) if err != nil { common.ErrorResp(c, err, 400) return } err = db.RemoveAuthn(user, req.ID) if err != nil { common.ErrorResp(c, err, 400) return } err = op.DelUserCache(user.Username) if err != nil { common.ErrorResp(c, err, 400) return } common.SuccessResp(c, "Deleted Successfully") } func GetAuthnCredentials(c *gin.Context) { type WebAuthnCredentials struct { ID []byte `json:"id"` FingerPrint string `json:"fingerprint"` } user := c.Request.Context().Value(conf.UserKey).(*model.User) credentials := user.WebAuthnCredentials() res := make([]WebAuthnCredentials, 0, len(credentials)) for _, v := range credentials { credential := WebAuthnCredentials{ ID: v.ID, FingerPrint: fmt.Sprintf("% X", v.Authenticator.AAGUID), } res = append(res, credential) } common.SuccessResp(c, res) } ================================================ FILE: server/middlewares/auth.go ================================================ package middlewares import ( "crypto/subtle" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) // Auth is a middleware that checks if the user is logged in. // if token is empty, set user to guest func Auth(allowDisabledGuest bool) func(c *gin.Context) { return func(c *gin.Context) { token := c.GetHeader("Authorization") if subtle.ConstantTimeCompare([]byte(token), []byte(setting.GetStr(conf.Token))) == 1 { admin, err := op.GetAdmin() if err != nil { common.ErrorResp(c, err, 500) c.Abort() return } common.GinWithValue(c, conf.UserKey, admin) log.Debugf("use admin token: %+v", admin) c.Next() return } if token == "" { guest, err := op.GetGuest() if err != nil { common.ErrorResp(c, err, 500) c.Abort() return } if !allowDisabledGuest && guest.Disabled { common.ErrorStrResp(c, "Guest user is disabled, login please", 401) c.Abort() return } common.GinWithValue(c, conf.UserKey, guest) log.Debugf("use empty token: %+v", guest) c.Next() return } userClaims, err := common.ParseToken(token) if err != nil { common.ErrorResp(c, err, 401) c.Abort() return } user, err := op.GetUserByName(userClaims.Username) if err != nil { common.ErrorResp(c, err, 401) c.Abort() return } // validate password timestamp if userClaims.PwdTS != user.PwdTS { common.ErrorStrResp(c, "Password has been changed, login please", 401) c.Abort() return } if user.Disabled { common.ErrorStrResp(c, "Current user is disabled, replace please", 401) c.Abort() return } common.GinWithValue(c, conf.UserKey, user) log.Debugf("use login token: %+v", user) c.Next() } } func Authn(c *gin.Context) { token := c.GetHeader("Authorization") if subtle.ConstantTimeCompare([]byte(token), []byte(setting.GetStr(conf.Token))) == 1 { admin, err := op.GetAdmin() if err != nil { common.ErrorResp(c, err, 500) c.Abort() return } common.GinWithValue(c, conf.UserKey, admin) log.Debugf("use admin token: %+v", admin) c.Next() return } if token == "" { guest, err := op.GetGuest() if err != nil { common.ErrorResp(c, err, 500) c.Abort() return } common.GinWithValue(c, conf.UserKey, guest) log.Debugf("use empty token: %+v", guest) c.Next() return } userClaims, err := common.ParseToken(token) if err != nil { common.ErrorResp(c, err, 401) c.Abort() return } user, err := op.GetUserByName(userClaims.Username) if err != nil { common.ErrorResp(c, err, 401) c.Abort() return } // validate password timestamp if userClaims.PwdTS != user.PwdTS { common.ErrorStrResp(c, "Password has been changed, login please", 401) c.Abort() return } if user.Disabled { common.ErrorStrResp(c, "Current user is disabled, replace please", 401) c.Abort() return } common.GinWithValue(c, conf.UserKey, user) log.Debugf("use login token: %+v", user) c.Next() } func AuthNotGuest(c *gin.Context) { user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { common.ErrorStrResp(c, "You are a guest", 403) c.Abort() } else { c.Next() } } func AuthAdmin(c *gin.Context) { user := c.Request.Context().Value(conf.UserKey).(*model.User) if !user.IsAdmin() { common.ErrorStrResp(c, "You are not an admin", 403) c.Abort() } else { c.Next() } } ================================================ FILE: server/middlewares/check.go ================================================ package middlewares import ( "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" ) func StoragesLoaded(c *gin.Context) { if !conf.StoragesLoaded { if utils.SliceContains([]string{"", "/", "/favicon.ico"}, c.Request.URL.Path) { c.Next() return } paths := []string{"/assets", "/images", "/streamer", "/static"} for _, path := range paths { if strings.HasPrefix(c.Request.URL.Path, path) { c.Next() return } } select { case <-conf.StoragesLoadSignal(): case <-c.Request.Context().Done(): c.Abort() return } } common.GinWithValue(c, conf.ApiUrlKey, common.GetApiUrlFromRequest(c.Request), ) c.Next() } ================================================ FILE: server/middlewares/down.go ================================================ package middlewares import ( "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" ) func PathParse(c *gin.Context) { rawPath := parsePath(c.Param("path")) common.GinWithValue(c, conf.PathKey, rawPath) c.Next() } func Down(verifyFunc func(string, string) error) func(c *gin.Context) { return func(c *gin.Context) { rawPath := c.Request.Context().Value(conf.PathKey).(string) meta, err := op.GetNearestMeta(rawPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { common.ErrorPage(c, err, 500, true) return } } common.GinWithValue(c, conf.MetaKey, meta) // verify sign if needSign(meta, rawPath) { s := c.Query("sign") err = verifyFunc(rawPath, strings.TrimSuffix(s, "/")) if err != nil { common.ErrorPage(c, err, 401) c.Abort() return } } c.Next() } } // TODO: implement // path maybe contains # ? etc. func parsePath(path string) string { return utils.FixAndCleanPath(path) } func needSign(meta *model.Meta, path string) bool { if setting.GetBool(conf.SignAll) { return true } if common.IsStorageSignEnabled(path) { return true } if meta == nil || meta.Password == "" { return false } if !meta.PSub && path != meta.Path { return false } return true } ================================================ FILE: server/middlewares/filtered_logger.go ================================================ package middlewares import ( "net/netip" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) type filter struct { CIDR *netip.Prefix `json:"cidr,omitempty"` Path *string `json:"path,omitempty"` Method *string `json:"method,omitempty"` } var filterList []*filter func initFilterList() { for _, s := range conf.Conf.Log.Filter.Filters { f := new(filter) if s.CIDR != "" { cidr, err := netip.ParsePrefix(s.CIDR) if err != nil { log.Errorf("failed to parse CIDR %s: %v", s.CIDR, err) continue } f.CIDR = &cidr } if s.Path != "" { f.Path = &s.Path } if s.Method != "" { f.Method = &s.Method } if f.CIDR == nil && f.Path == nil && f.Method == nil { log.Warnf("filter %s is empty, skipping", s) continue } filterList = append(filterList, f) log.Debugf("added filter: %+v", f) } log.Infof("Loaded %d log filters.", len(filterList)) } func skiperDecider(c *gin.Context) bool { // every filter need metch all condithon as filter match // so if any condithon not metch, skip this filter // all filters misatch, log this request for _, f := range filterList { if f.CIDR != nil { cip := netip.MustParseAddr(c.ClientIP()) if !f.CIDR.Contains(cip) { continue } } if f.Path != nil { if (*f.Path)[0] == '/' { // match path as prefix/exact path if !strings.HasPrefix(c.Request.URL.Path, *f.Path) { continue } } else { // match path as relative path if !strings.Contains(c.Request.URL.Path, "/"+*f.Path) { continue } } } if f.Method != nil { if *f.Method != c.Request.Method { continue } } return true } return false } func FilteredLogger() gin.HandlerFunc { initFilterList() return gin.LoggerWithConfig(gin.LoggerConfig{ Output: log.StandardLogger().Out, Skip: skiperDecider, }) } ================================================ FILE: server/middlewares/fsup.go ================================================ package middlewares import ( "net/url" stdpath "path" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" ) func FsUp(c *gin.Context) { path := c.GetHeader("File-Path") password := c.GetHeader("Password") path, err := url.PathUnescape(path) if err != nil { common.ErrorResp(c, err, 400) c.Abort() return } user := c.Request.Context().Value(conf.UserKey).(*model.User) path, err = user.JoinPath(path) if err != nil { common.ErrorResp(c, err, 403) return } meta, err := op.GetNearestMeta(stdpath.Dir(path)) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { common.ErrorResp(c, err, 500, true) c.Abort() return } } if !(common.CanAccess(user, meta, path, password) && (user.CanWrite() || common.CanWrite(meta, stdpath.Dir(path)))) { common.ErrorResp(c, errs.PermissionDenied, 403) c.Abort() return } c.Next() } ================================================ FILE: server/middlewares/https.go ================================================ package middlewares import ( "fmt" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/gin-gonic/gin" ) func ForceHttps(c *gin.Context) { if c.Request.TLS == nil { host := c.Request.Host // change port to https port host = strings.Replace(host, fmt.Sprintf(":%d", conf.Conf.Scheme.HttpPort), fmt.Sprintf(":%d", conf.Conf.Scheme.HttpsPort), 1) c.Redirect(302, "https://"+host+c.Request.RequestURI) c.Abort() return } c.Next() } ================================================ FILE: server/middlewares/limit.go ================================================ package middlewares import ( "io" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/gin-gonic/gin" ) func MaxAllowed(n int) gin.HandlerFunc { sem := make(chan struct{}, n) acquire := func() { sem <- struct{}{} } release := func() { <-sem } return func(c *gin.Context) { acquire() defer release() c.Next() } } func UploadRateLimiter(limiter stream.Limiter) gin.HandlerFunc { return func(c *gin.Context) { c.Request.Body = &stream.RateLimitReader{ Reader: c.Request.Body, Limiter: limiter, Ctx: c, } c.Next() } } type ResponseWriterWrapper struct { gin.ResponseWriter WrapWriter io.Writer } func (w *ResponseWriterWrapper) Write(p []byte) (n int, err error) { return w.WrapWriter.Write(p) } func DownloadRateLimiter(limiter stream.Limiter) gin.HandlerFunc { return func(c *gin.Context) { c.Writer = &ResponseWriterWrapper{ ResponseWriter: c.Writer, WrapWriter: &stream.RateLimitWriter{ Writer: c.Writer, Limiter: limiter, Ctx: c, }, } c.Next() } } ================================================ FILE: server/middlewares/search.go ================================================ package middlewares import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" ) func SearchIndex(c *gin.Context) { mode := setting.GetStr(conf.SearchIndex) if mode == "none" { common.ErrorResp(c, errs.SearchNotAvailable, 404) c.Abort() } else { c.Next() } } ================================================ FILE: server/middlewares/sharing.go ================================================ package middlewares import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" ) func SharingIdParse(c *gin.Context) { sid := c.Param("sid") common.GinWithValue(c, conf.SharingIDKey, sid) c.Next() } func EmptyPathParse(c *gin.Context) { common.GinWithValue(c, conf.PathKey, "/") c.Next() } ================================================ FILE: server/router.go ================================================ package server import ( "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/message" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/server/handles" "github.com/OpenListTeam/OpenList/v4/server/middlewares" "github.com/OpenListTeam/OpenList/v4/server/static" "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" ) func Init(e *gin.Engine) { e.ContextWithFallback = true if !utils.SliceContains([]string{"", "/"}, conf.URL.Path) { e.GET("/", func(c *gin.Context) { c.Redirect(302, conf.URL.Path) }) } Cors(e) g := e.Group(conf.URL.Path) if conf.Conf.Scheme.HttpPort != -1 && conf.Conf.Scheme.HttpsPort != -1 && conf.Conf.Scheme.ForceHttps { e.Use(middlewares.ForceHttps) } g.Any("/ping", func(c *gin.Context) { c.String(200, "pong") }) g.GET("/favicon.ico", handles.Favicon) g.GET("/robots.txt", handles.Robots) g.GET("/manifest.json", static.ManifestJSON) g.GET("/i/:link_name", handles.Plist) common.SecretKey = []byte(conf.Conf.JwtSecret) g.Use(middlewares.StoragesLoaded) if conf.Conf.MaxConnections > 0 { g.Use(middlewares.MaxAllowed(conf.Conf.MaxConnections)) } WebDav(g.Group("/dav")) S3(g.Group("/s3")) downloadLimiter := middlewares.DownloadRateLimiter(stream.ClientDownloadLimit) signCheck := middlewares.Down(sign.Verify) g.GET("/d/*path", middlewares.PathParse, signCheck, downloadLimiter, handles.Down) g.GET("/p/*path", middlewares.PathParse, signCheck, downloadLimiter, handles.Proxy) g.HEAD("/d/*path", middlewares.PathParse, signCheck, handles.Down) g.HEAD("/p/*path", middlewares.PathParse, signCheck, handles.Proxy) archiveSignCheck := middlewares.Down(sign.VerifyArchive) g.GET("/ad/*path", middlewares.PathParse, archiveSignCheck, downloadLimiter, handles.ArchiveDown) g.GET("/ap/*path", middlewares.PathParse, archiveSignCheck, downloadLimiter, handles.ArchiveProxy) g.GET("/ae/*path", middlewares.PathParse, archiveSignCheck, downloadLimiter, handles.ArchiveInternalExtract) g.HEAD("/ad/*path", middlewares.PathParse, archiveSignCheck, handles.ArchiveDown) g.HEAD("/ap/*path", middlewares.PathParse, archiveSignCheck, handles.ArchiveProxy) g.HEAD("/ae/*path", middlewares.PathParse, archiveSignCheck, handles.ArchiveInternalExtract) g.GET("/sd/:sid", middlewares.EmptyPathParse, middlewares.SharingIdParse, downloadLimiter, handles.SharingDown) g.GET("/sd/:sid/*path", middlewares.PathParse, middlewares.SharingIdParse, downloadLimiter, handles.SharingDown) g.HEAD("/sd/:sid", middlewares.EmptyPathParse, middlewares.SharingIdParse, handles.SharingDown) g.HEAD("/sd/:sid/*path", middlewares.PathParse, middlewares.SharingIdParse, handles.SharingDown) g.GET("/sad/:sid", middlewares.EmptyPathParse, middlewares.SharingIdParse, downloadLimiter, handles.SharingArchiveExtract) g.GET("/sad/:sid/*path", middlewares.PathParse, middlewares.SharingIdParse, downloadLimiter, handles.SharingArchiveExtract) g.HEAD("/sad/:sid", middlewares.EmptyPathParse, middlewares.SharingIdParse, handles.SharingArchiveExtract) g.HEAD("/sad/:sid/*path", middlewares.PathParse, middlewares.SharingIdParse, handles.SharingArchiveExtract) api := g.Group("/api") auth := api.Group("", middlewares.Auth(false)) webauthn := api.Group("/authn", middlewares.Authn) api.POST("/auth/login", handles.Login) api.POST("/auth/login/hash", handles.LoginHash) api.POST("/auth/login/ldap", handles.LoginLdap) auth.GET("/me", handles.CurrentUser) auth.POST("/me/update", handles.UpdateCurrent) auth.GET("/me/sshkey/list", handles.ListMyPublicKey) auth.POST("/me/sshkey/add", handles.AddMyPublicKey) auth.POST("/me/sshkey/delete", handles.DeleteMyPublicKey) auth.POST("/auth/2fa/generate", handles.Generate2FA) auth.POST("/auth/2fa/verify", handles.Verify2FA) auth.GET("/auth/logout", handles.LogOut) // auth api.GET("/auth/sso", handles.SSOLoginRedirect) api.GET("/auth/sso_callback", handles.SSOLoginCallback) api.GET("/auth/get_sso_id", handles.SSOLoginCallback) api.GET("/auth/sso_get_token", handles.SSOLoginCallback) // webauthn api.GET("/authn/webauthn_begin_login", handles.BeginAuthnLogin) api.POST("/authn/webauthn_finish_login", handles.FinishAuthnLogin) webauthn.GET("/webauthn_begin_registration", handles.BeginAuthnRegistration) webauthn.POST("/webauthn_finish_registration", handles.FinishAuthnRegistration) webauthn.POST("/delete_authn", handles.DeleteAuthnLogin) webauthn.GET("/getcredentials", handles.GetAuthnCredentials) // no need auth public := api.Group("/public") public.Any("/settings", handles.PublicSettings) public.Any("/offline_download_tools", handles.OfflineDownloadTools) public.Any("/archive_extensions", handles.ArchiveExtensions) _fs(auth.Group("/fs")) fsAndShare(api.Group("/fs", middlewares.Auth(true))) _task(auth.Group("/task", middlewares.AuthNotGuest)) _sharing(auth.Group("/share", middlewares.AuthNotGuest)) admin(auth.Group("/admin", middlewares.AuthAdmin)) if flags.Debug || flags.Dev { debug(g.Group("/debug")) } static.Static(g, func(handlers ...gin.HandlerFunc) { e.NoRoute(handlers...) }) } func admin(g *gin.RouterGroup) { meta := g.Group("/meta") meta.GET("/list", handles.ListMetas) meta.GET("/get", handles.GetMeta) meta.POST("/create", handles.CreateMeta) meta.POST("/update", handles.UpdateMeta) meta.POST("/delete", handles.DeleteMeta) user := g.Group("/user") user.GET("/list", handles.ListUsers) user.GET("/get", handles.GetUser) user.POST("/create", handles.CreateUser) user.POST("/update", handles.UpdateUser) user.POST("/cancel_2fa", handles.Cancel2FAById) user.POST("/delete", handles.DeleteUser) user.POST("/del_cache", handles.DelUserCache) user.GET("/sshkey/list", handles.ListPublicKeys) user.POST("/sshkey/delete", handles.DeletePublicKey) storage := g.Group("/storage") storage.GET("/list", handles.ListStorages) storage.GET("/get", handles.GetStorage) storage.POST("/create", handles.CreateStorage) storage.POST("/update", handles.UpdateStorage) storage.POST("/delete", handles.DeleteStorage) storage.POST("/enable", handles.EnableStorage) storage.POST("/disable", handles.DisableStorage) storage.POST("/load_all", handles.LoadAllStorages) driver := g.Group("/driver") driver.GET("/list", handles.ListDriverInfo) driver.GET("/names", handles.ListDriverNames) driver.GET("/info", handles.GetDriverInfo) setting := g.Group("/setting") setting.GET("/get", handles.GetSetting) setting.GET("/list", handles.ListSettings) setting.POST("/save", handles.SaveSettings) setting.POST("/delete", handles.DeleteSetting) setting.POST("/default", handles.DefaultSettings) setting.POST("/reset_token", handles.ResetToken) setting.POST("/set_aria2", handles.SetAria2) setting.POST("/set_qbit", handles.SetQbittorrent) setting.POST("/set_transmission", handles.SetTransmission) setting.POST("/set_115", handles.Set115) setting.POST("/set_115_open", handles.Set115Open) setting.POST("/set_123_pan", handles.Set123Pan) setting.POST("/set_123_open", handles.Set123Open) setting.POST("/set_pikpak", handles.SetPikPak) setting.POST("/set_thunder", handles.SetThunder) setting.POST("/set_thunderx", handles.SetThunderX) setting.POST("/set_thunder_browser", handles.SetThunderBrowser) // retain /admin/task API to ensure compatibility with legacy automation scripts _task(g.Group("/task")) ms := g.Group("/message") ms.POST("/get", message.HttpInstance.GetHandle) ms.POST("/send", message.HttpInstance.SendHandle) index := g.Group("/index") index.POST("/build", middlewares.SearchIndex, handles.BuildIndex) index.POST("/update", middlewares.SearchIndex, handles.UpdateIndex) index.POST("/stop", middlewares.SearchIndex, handles.StopIndex) index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex) index.GET("/progress", middlewares.SearchIndex, handles.GetProgress) scan := g.Group("/scan") scan.POST("/start", handles.StartManualScan) scan.POST("/stop", handles.StopManualScan) scan.GET("/progress", handles.GetManualScanProgress) } func fsAndShare(g *gin.RouterGroup) { g.Any("/list", handles.FsListSplit) g.Any("/get", handles.FsGetSplit) a := g.Group("/archive") a.Any("/meta", handles.FsArchiveMetaSplit) a.Any("/list", handles.FsArchiveListSplit) } func _fs(g *gin.RouterGroup) { g.Any("/search", middlewares.SearchIndex, handles.Search) g.Any("/other", handles.FsOther) g.Any("/dirs", handles.FsDirs) g.POST("/mkdir", handles.FsMkdir) g.POST("/rename", handles.FsRename) g.POST("/batch_rename", handles.FsBatchRename) g.POST("/regex_rename", handles.FsRegexRename) g.POST("/move", handles.FsMove) g.POST("/recursive_move", handles.FsRecursiveMove) g.POST("/copy", handles.FsCopy) g.POST("/remove", handles.FsRemove) g.POST("/remove_empty_directory", handles.FsRemoveEmptyDirectory) uploadLimiter := middlewares.UploadRateLimiter(stream.ClientUploadLimit) g.PUT("/put", middlewares.FsUp, uploadLimiter, handles.FsStream) g.PUT("/form", middlewares.FsUp, uploadLimiter, handles.FsForm) g.POST("/link", middlewares.AuthAdmin, handles.Link) // g.POST("/add_aria2", handles.AddOfflineDownload) // g.POST("/add_qbit", handles.AddQbittorrent) // g.POST("/add_transmission", handles.SetTransmission) g.POST("/add_offline_download", handles.AddOfflineDownload) g.POST("/archive/decompress", handles.FsArchiveDecompress) // Direct upload (client-side upload to storage) g.POST("/get_direct_upload_info", middlewares.FsUp, handles.FsGetDirectUploadInfo) } func _task(g *gin.RouterGroup) { handles.SetupTaskRoute(g) } func _sharing(g *gin.RouterGroup) { g.Any("/list", handles.ListSharings) g.GET("/get", handles.GetSharing) g.POST("/create", handles.CreateSharing) g.POST("/update", handles.UpdateSharing) g.POST("/delete", handles.DeleteSharing) g.POST("/enable", handles.SetEnableSharing(false)) g.POST("/disable", handles.SetEnableSharing(true)) } func Cors(r *gin.Engine) { config := cors.DefaultConfig() // config.AllowAllOrigins = true config.AllowOrigins = conf.Conf.Cors.AllowOrigins config.AllowHeaders = conf.Conf.Cors.AllowHeaders config.AllowMethods = conf.Conf.Cors.AllowMethods r.Use(cors.New(config)) } func InitS3(e *gin.Engine) { Cors(e) S3Server(e.Group("/")) } ================================================ FILE: server/s3/backend.go ================================================ // Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 // Package s3 implements a fake s3 server for openlist package s3 import ( "context" "encoding/hex" "fmt" "io" "path" "strings" "sync" "time" "github.com/pkg/errors" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/itsHenry35/gofakes3" "github.com/ncw/swift/v2" log "github.com/sirupsen/logrus" ) var ( emptyPrefix = &gofakes3.Prefix{} timeFormat = "Mon, 2 Jan 2006 15:04:05 GMT" ) // s3Backend implements the gofacess3.Backend interface to make an S3 // backend for gofakes3 type s3Backend struct { meta *sync.Map } // newBackend creates a new SimpleBucketBackend. func newBackend() gofakes3.Backend { return &s3Backend{ meta: new(sync.Map), } } // ListBuckets always returns the default bucket. func (b *s3Backend) ListBuckets(ctx context.Context) ([]gofakes3.BucketInfo, error) { buckets, err := getAndParseBuckets() if err != nil { return nil, err } var response []gofakes3.BucketInfo for _, b := range buckets { node, _ := fs.Get(ctx, b.Path, &fs.GetArgs{}) response = append(response, gofakes3.BucketInfo{ // Name: gofakes3.URLEncode(b.Name), Name: b.Name, CreationDate: gofakes3.NewContentTime(node.ModTime()), }) } return response, nil } // ListBucket lists the objects in the given bucket. func (b *s3Backend) ListBucket(ctx context.Context, bucketName string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) { bucket, err := getBucketByName(bucketName) if err != nil { return nil, err } bucketPath := bucket.Path if prefix == nil { prefix = emptyPrefix } // workaround if strings.TrimSpace(prefix.Prefix) == "" { prefix.HasPrefix = false } if strings.TrimSpace(prefix.Delimiter) == "" { prefix.HasDelimiter = false } response := gofakes3.NewObjectList() path, remaining := prefixParser(prefix) err = b.entryListR(bucketPath, path, remaining, prefix.HasDelimiter, response) if err == gofakes3.ErrNoSuchKey { // AWS just returns an empty list response = gofakes3.NewObjectList() } else if err != nil { return nil, err } return b.pager(response, page) } // HeadObject returns the fileinfo for the given object name. // // Note that the metadata is not supported yet. func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName string) (*gofakes3.Object, error) { bucket, err := getBucketByName(bucketName) if err != nil { return nil, err } bucketPath := bucket.Path fp := path.Join(bucketPath, objectName) fmeta, _ := op.GetNearestMeta(fp) node, err := fs.Get(context.WithValue(ctx, conf.MetaKey, fmeta), fp, &fs.GetArgs{}) if err != nil { return nil, gofakes3.KeyNotFound(objectName) } if node.IsDir() { return nil, gofakes3.KeyNotFound(objectName) } size := node.GetSize() // hash := getFileHashByte(fobj) meta := map[string]string{ "Last-Modified": node.ModTime().Format(timeFormat), "Content-Type": utils.GetMimeType(fp), } if val, ok := b.meta.Load(fp); ok { metaMap := val.(map[string]string) for k, v := range metaMap { meta[k] = v } } return &gofakes3.Object{ Name: objectName, // Hash: hash, Metadata: meta, Size: size, Contents: noOpReadCloser{}, }, nil } // GetObject fetchs the object from the filesystem. func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (s3Obj *gofakes3.Object, err error) { bucket, err := getBucketByName(bucketName) if err != nil { return nil, err } bucketPath := bucket.Path fp := path.Join(bucketPath, objectName) fmeta, _ := op.GetNearestMeta(fp) node, err := fs.Get(context.WithValue(ctx, conf.MetaKey, fmeta), fp, &fs.GetArgs{}) if err != nil { return nil, gofakes3.KeyNotFound(objectName) } if node.IsDir() { return nil, gofakes3.KeyNotFound(objectName) } link, file, err := fs.Link(ctx, fp, model.LinkArgs{}) if err != nil { return nil, err } defer func() { if s3Obj == nil { _ = link.Close() } }() size := link.ContentLength if size <= 0 { size = file.GetSize() } rnge, err := rangeRequest.Range(size) if err != nil { return nil, err } rrf, err := stream.GetRangeReaderFromLink(size, link) if err != nil { return nil, fmt.Errorf("the remote storage driver need to be enhanced to support s3") } var rd io.Reader if rnge != nil { rd, err = rrf.RangeRead(ctx, http_range.Range(*rnge)) } else { rd, err = rrf.RangeRead(ctx, http_range.Range{Length: -1}) } if err != nil { return nil, err } meta := map[string]string{ "Last-Modified": node.ModTime().Format(timeFormat), "Content-Disposition": utils.GenerateContentDisposition(file.GetName()), "Content-Type": utils.GetMimeType(fp), } if val, ok := b.meta.Load(fp); ok { metaMap := val.(map[string]string) for k, v := range metaMap { meta[k] = v } } return &gofakes3.Object{ // Name: gofakes3.URLEncode(objectName), Name: objectName, // Hash: "", Metadata: meta, Size: size, Range: rnge, Contents: utils.ReadCloser{Reader: rd, Closer: link}, }, nil } // TouchObject creates or updates meta on specified object. func (b *s3Backend) TouchObject(ctx context.Context, fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) { //TODO: implement return result, gofakes3.ErrNotImplemented } // PutObject creates or overwrites the object with the given name. func (b *s3Backend) PutObject( ctx context.Context, bucketName, objectName string, meta map[string]string, input io.Reader, size int64, ) (result gofakes3.PutObjectResult, err error) { bucket, err := getBucketByName(bucketName) if err != nil { return result, err } bucketPath := bucket.Path isDir := strings.HasSuffix(objectName, "/") log.Debugf("isDir: %v", isDir) fp := path.Join(bucketPath, objectName) log.Debugf("fp: %s, bucketPath: %s, objectName: %s", fp, bucketPath, objectName) var reqPath string if isDir { reqPath = fp + "/" } else { reqPath = path.Dir(fp) } log.Debugf("reqPath: %s", reqPath) fmeta, _ := op.GetNearestMeta(fp) ctx = context.WithValue(ctx, conf.MetaKey, fmeta) _, err = fs.Get(ctx, reqPath, &fs.GetArgs{}) if err != nil { if errs.IsObjectNotFound(err) && strings.Contains(objectName, "/") { log.Debugf("reqPath: %s not found and objectName contains /, need to makeDir", reqPath) err = fs.MakeDir(ctx, reqPath) if err != nil { return result, errors.WithMessagef(err, "failed to makeDir, reqPath: %s", reqPath) } } else { return result, gofakes3.KeyNotFound(objectName) } } if isDir { return result, nil } var ti time.Time if val, ok := meta["X-Amz-Meta-Mtime"]; ok { ti, _ = swift.FloatStringToTime(val) } if val, ok := meta["mtime"]; ok { ti, _ = swift.FloatStringToTime(val) } // If Modified is not set, use current time if ti.IsZero() { ti = time.Now() } obj := model.Object{ Name: path.Base(fp), Size: size, Modified: ti, Ctime: time.Now(), } // Check if system file should be ignored if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) { return result, errs.IgnoredSystemFile } stream := &stream.FileStream{ Obj: &obj, Reader: input, Mimetype: meta["Content-Type"], } err = fs.PutDirectly(ctx, reqPath, stream) if err != nil { return result, err } // if err := stream.Close(); err != nil { // // remove file when close error occurred (FsPutErr) // _ = fs.Remove(ctx, fp) // return result, err // } b.meta.Store(fp, meta) return result, nil } // DeleteMulti deletes multiple objects in a single request. func (b *s3Backend) DeleteMulti(ctx context.Context, bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) { for _, object := range objects { if err := b.deleteObject(ctx, bucketName, object); err != nil { log.Errorf("delete object failed: %v", err) result.Error = append(result.Error, gofakes3.ErrorResult{ Code: gofakes3.ErrInternal, Message: gofakes3.ErrInternal.Message(), Key: object, }) } else { result.Deleted = append(result.Deleted, gofakes3.ObjectID{ Key: object, }) } } return result, nil } // DeleteObject deletes the object with the given name. func (b *s3Backend) DeleteObject(ctx context.Context, bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) { return result, b.deleteObject(ctx, bucketName, objectName) } // deleteObject deletes the object from the filesystem. func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName string) error { bucket, err := getBucketByName(bucketName) if err != nil { return err } bucketPath := bucket.Path fp := path.Join(bucketPath, objectName) fmeta, _ := op.GetNearestMeta(fp) // S3 does not report an error when attemping to delete a key that does not exist, so // we need to skip IsNotExist errors. if _, err := fs.Get(context.WithValue(ctx, conf.MetaKey, fmeta), fp, &fs.GetArgs{}); err != nil && !errs.IsObjectNotFound(err) { return err } fs.Remove(ctx, fp) return nil } // CreateBucket creates a new bucket. func (b *s3Backend) CreateBucket(ctx context.Context, name string) error { return gofakes3.ErrNotImplemented } // DeleteBucket deletes the bucket with the given name. func (b *s3Backend) DeleteBucket(ctx context.Context, name string) error { return gofakes3.ErrNotImplemented } // BucketExists checks if the bucket exists. func (b *s3Backend) BucketExists(ctx context.Context, name string) (exists bool, err error) { buckets, err := getAndParseBuckets() if err != nil { return false, err } for _, b := range buckets { if b.Name == name { return true, nil } } return false, nil } // CopyObject copy specified object from srcKey to dstKey. func (b *s3Backend) CopyObject(ctx context.Context, srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) { if srcBucket == dstBucket && srcKey == dstKey { //TODO: update meta return result, nil } srcB, err := getBucketByName(srcBucket) if err != nil { return result, err } srcBucketPath := srcB.Path srcFp := path.Join(srcBucketPath, srcKey) fmeta, _ := op.GetNearestMeta(srcFp) srcNode, err := fs.Get(context.WithValue(ctx, conf.MetaKey, fmeta), srcFp, &fs.GetArgs{}) c, err := b.GetObject(ctx, srcBucket, srcKey, nil) if err != nil { return } defer func() { _ = c.Contents.Close() }() for k, v := range c.Metadata { if _, found := meta[k]; !found && k != "X-Amz-Acl" { meta[k] = v } } if _, ok := meta["mtime"]; !ok { meta["mtime"] = swift.TimeToFloatString(srcNode.ModTime()) } _, err = b.PutObject(ctx, dstBucket, dstKey, meta, c.Contents, c.Size) if err != nil { return } return gofakes3.CopyObjectResult{ ETag: `"` + hex.EncodeToString(c.Hash) + `"`, LastModified: gofakes3.NewContentTime(srcNode.ModTime()), }, nil } ================================================ FILE: server/s3/ioutils.go ================================================ // Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 // Package s3 implements a fake s3 server for openlist package s3 import "io" type noOpReadCloser struct{} type readerWithCloser struct { io.Reader closer func() error } var _ io.ReadCloser = &readerWithCloser{} func (d noOpReadCloser) Read(b []byte) (n int, err error) { return 0, io.EOF } func (d noOpReadCloser) Close() error { return nil } func limitReadCloser(rdr io.Reader, closer func() error, sz int64) io.ReadCloser { return &readerWithCloser{ Reader: io.LimitReader(rdr, sz), closer: closer, } } func (rwc *readerWithCloser) Close() error { if rwc.closer != nil { return rwc.closer() } return nil } ================================================ FILE: server/s3/list.go ================================================ // Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 // Package s3 implements a fake s3 server for openlist package s3 import ( "path" "strings" "time" "github.com/itsHenry35/gofakes3" log "github.com/sirupsen/logrus" ) func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error { fp := path.Join(bucket, fdPath) dirEntries, err := getDirEntries(fp) if err != nil { return err } // workaround as s3 can't have empty files in directories, useful in deletions if len(dirEntries) == 0 { item := &gofakes3.Content{ // Key: gofakes3.URLEncode(path.Join(fdPath, emptyObjectName)), Key: path.Join(fdPath, emptyObjectName), LastModified: gofakes3.NewContentTime(time.Now()), ETag: getFileHash(nil), // No entry, so no hash Size: 0, StorageClass: gofakes3.StorageStandard, } response.Add(item) log.Debugf("Adding empty object %s to response", item.Key) return nil } for _, entry := range dirEntries { object := entry.GetName() // workround for control-chars detect objectPath := path.Join(fdPath, object) if !strings.HasPrefix(object, name) { continue } if entry.IsDir() { if addPrefix { // response.AddPrefix(gofakes3.URLEncode(objectPath)) response.AddPrefix(objectPath) continue } err := b.entryListR(bucket, path.Join(fdPath, object), "", false, response) if err != nil { return err } } else { item := &gofakes3.Content{ // Key: gofakes3.URLEncode(objectPath), Key: objectPath, LastModified: gofakes3.NewContentTime(entry.ModTime()), ETag: getFileHash(entry), Size: entry.GetSize(), StorageClass: gofakes3.StorageStandard, } response.Add(item) } } return nil } ================================================ FILE: server/s3/logger.go ================================================ // Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 // Package s3 implements a fake s3 server for openlist package s3 import ( "fmt" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/itsHenry35/gofakes3" ) // logger output formatted message type logger struct{} // print log message func (l logger) Print(level gofakes3.LogLevel, v ...interface{}) { switch level { default: fallthrough case gofakes3.LogErr: utils.Log.Errorf("serve s3: %s", fmt.Sprintln(v...)) case gofakes3.LogWarn: utils.Log.Infof("serve s3: %s", fmt.Sprintln(v...)) case gofakes3.LogInfo: utils.Log.Debugf("serve s3: %s", fmt.Sprintln(v...)) } } ================================================ FILE: server/s3/pager.go ================================================ // Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 // Package s3 implements a fake s3 server for openlist package s3 import ( "sort" "github.com/itsHenry35/gofakes3" ) // pager splits the object list into smulitply pages. func (db *s3Backend) pager(list *gofakes3.ObjectList, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) { // sort by alphabet sort.Slice(list.CommonPrefixes, func(i, j int) bool { return list.CommonPrefixes[i].Prefix < list.CommonPrefixes[j].Prefix }) // sort by modtime sort.Slice(list.Contents, func(i, j int) bool { return list.Contents[i].LastModified.Before(list.Contents[j].LastModified.Time) }) tokens := page.MaxKeys if tokens == 0 { tokens = 1000 } if page.HasMarker { for i, obj := range list.Contents { if obj.Key == page.Marker { list.Contents = list.Contents[i+1:] break } } for i, obj := range list.CommonPrefixes { if obj.Prefix == page.Marker { list.CommonPrefixes = list.CommonPrefixes[i+1:] break } } } response := gofakes3.NewObjectList() for _, obj := range list.CommonPrefixes { if tokens <= 0 { break } response.AddPrefix(obj.Prefix) tokens-- } for _, obj := range list.Contents { if tokens <= 0 { break } response.Add(obj) tokens-- } if len(list.CommonPrefixes)+len(list.Contents) > int(page.MaxKeys) { response.IsTruncated = true if len(response.Contents) > 0 { response.NextMarker = response.Contents[len(response.Contents)-1].Key } else { response.NextMarker = response.CommonPrefixes[len(response.CommonPrefixes)-1].Prefix } } return response, nil } ================================================ FILE: server/s3/server.go ================================================ // Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 // Package s3 implements a fake s3 server for openlist package s3 import ( "context" "math/rand" "net/http" "github.com/itsHenry35/gofakes3" ) // Make a new S3 Server to serve the remote func NewServer(ctx context.Context) (h http.Handler, err error) { var newLogger logger faker := gofakes3.New( newBackend(), // gofakes3.WithHostBucket(!opt.pathBucketMode), gofakes3.WithLogger(newLogger), gofakes3.WithRequestID(rand.Uint64()), gofakes3.WithoutVersioning(), gofakes3.WithV4Auth(authlistResolver()), gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied ) return faker.Server(), nil } ================================================ FILE: server/s3/utils.go ================================================ // Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 // Package s3 implements a fake s3 server for openlist package s3 import ( "context" "encoding/json" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/itsHenry35/gofakes3" ) type Bucket struct { Name string `json:"name"` Path string `json:"path"` } const emptyObjectName = "ThisIsAnEmptyFolderInTheS3Bucket" func getAndParseBuckets() ([]Bucket, error) { var res []Bucket err := json.Unmarshal([]byte(setting.GetStr(conf.S3Buckets)), &res) return res, err } func getBucketByName(name string) (Bucket, error) { buckets, err := getAndParseBuckets() if err != nil { return Bucket{}, err } for _, b := range buckets { if b.Name == name { return b, nil } } return Bucket{}, gofakes3.BucketNotFound(name) } func getDirEntries(path string) ([]model.Obj, error) { ctx := context.Background() meta, _ := op.GetNearestMeta(path) fi, err := fs.Get(context.WithValue(ctx, conf.MetaKey, meta), path, &fs.GetArgs{}) if errs.IsNotFoundError(err) { return nil, gofakes3.ErrNoSuchKey } else if err != nil { return nil, gofakes3.ErrNoSuchKey } if !fi.IsDir() { return nil, gofakes3.ErrNoSuchKey } dirEntries, err := fs.List(context.WithValue(ctx, conf.MetaKey, meta), path, &fs.ListArgs{}) if err != nil { return nil, err } return dirEntries, nil } // func getFileHashByte(node interface{}) []byte { // b, err := hex.DecodeString(getFileHash(node)) // if err != nil { // return nil // } // return b // } func getFileHash(node interface{}) string { // var o fs.Object // switch b := node.(type) { // case vfs.Node: // fsObj, ok := b.DirEntry().(fs.Object) // if !ok { // fs.Debugf("serve s3", "File uploading - reading hash from VFS cache") // in, err := b.Open(os.O_RDONLY) // if err != nil { // return "" // } // defer func() { // _ = in.Close() // }() // h, err := hash.NewMultiHasherTypes(hash.NewHashSet(hash.MD5)) // if err != nil { // return "" // } // _, err = io.Copy(h, in) // if err != nil { // return "" // } // return h.Sums()[hash.MD5] // } // o = fsObj // case fs.Object: // o = b // } // hash, err := o.Hash(context.Background(), hash.MD5) // if err != nil { // return "" // } // return hash return "" } func prefixParser(p *gofakes3.Prefix) (path, remaining string) { idx := strings.LastIndexByte(p.Prefix, '/') if idx < 0 { return "", p.Prefix } return p.Prefix[:idx], p.Prefix[idx+1:] } // // FIXME this could be implemented by VFS.MkdirAll() // func mkdirRecursive(path string, VFS *vfs.VFS) error { // path = strings.Trim(path, "/") // dirs := strings.Split(path, "/") // dir := "" // for _, d := range dirs { // dir += "/" + d // if _, err := VFS.Stat(dir); err != nil { // err := VFS.Mkdir(dir, 0777) // if err != nil { // return err // } // } // } // return nil // } // func rmdirRecursive(p string, VFS *vfs.VFS) { // dir := path.Dir(p) // if !strings.ContainsAny(dir, "/\\") { // // might be bucket(root) // return // } // if _, err := VFS.Stat(dir); err == nil { // err := VFS.Remove(dir) // if err != nil { // return // } // rmdirRecursive(dir, VFS) // } // } func authlistResolver() map[string]string { s3accesskeyid := setting.GetStr(conf.S3AccessKeyId) s3secretaccesskey := setting.GetStr(conf.S3SecretAccessKey) if s3accesskeyid == "" && s3secretaccesskey == "" { return nil } authList := make(map[string]string) authList[s3accesskeyid] = s3secretaccesskey return authList } ================================================ FILE: server/s3.go ================================================ package server import ( "context" "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/server/s3" "github.com/gin-gonic/gin" ) func S3(g *gin.RouterGroup) { if !conf.Conf.S3.Enable { g.Any("/*path", func(c *gin.Context) { common.ErrorStrResp(c, "S3 server is not enabled", 403) }) return } if conf.Conf.S3.Port != -1 { g.Any("/*path", func(c *gin.Context) { common.ErrorStrResp(c, "S3 server bound to single port", 403) }) return } h, _ := s3.NewServer(context.Background()) g.Any("/*path", func(c *gin.Context) { adjustedPath := strings.TrimPrefix(c.Request.URL.Path, path.Join(conf.URL.Path, "/s3")) c.Request.URL.Path = adjustedPath gin.WrapH(h)(c) }) } func S3Server(g *gin.RouterGroup) { h, _ := s3.NewServer(context.Background()) g.Any("/*path", gin.WrapH(h)) } ================================================ FILE: server/sftp/const.go ================================================ package sftp // From leffss/sftpd const ( SSH_FXF_READ = 0x00000001 SSH_FXF_WRITE = 0x00000002 SSH_FXF_APPEND = 0x00000004 SSH_FXF_CREAT = 0x00000008 SSH_FXF_TRUNC = 0x00000010 SSH_FXF_EXCL = 0x00000020 ) ================================================ FILE: server/sftp/hostkey.go ================================================ package sftp import ( "crypto/rand" "crypto/rsa" "crypto/x509" "encoding/pem" "fmt" "os" "path/filepath" "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "golang.org/x/crypto/ssh" ) var SSHSigners []ssh.Signer func InitHostKey() { if SSHSigners != nil { return } sshPath := filepath.Join(flags.DataDir, "ssh") if !utils.Exists(sshPath) { err := utils.CreateNestedDirectory(sshPath) if err != nil { utils.Log.Errorf("failed to create ssh directory: %+v", err) return } } SSHSigners = make([]ssh.Signer, 0, 4) if rsaKey, ok := LoadOrGenerateRSAHostKey(sshPath); ok { SSHSigners = append(SSHSigners, rsaKey) } // TODO Add keys for other encryption algorithms } func LoadOrGenerateRSAHostKey(parentDir string) (ssh.Signer, bool) { privateKeyPath := filepath.Join(parentDir, "ssh_host_rsa_key") publicKeyPath := filepath.Join(parentDir, "ssh_host_rsa_key.pub") privateKeyBytes, err := os.ReadFile(privateKeyPath) if err == nil { var privateKey *rsa.PrivateKey privateKey, err = rsaDecodePrivateKey(privateKeyBytes) if err == nil { var ret ssh.Signer ret, err = ssh.NewSignerFromKey(privateKey) if err == nil { return ret, true } } } _ = os.Remove(privateKeyPath) _ = os.Remove(publicKeyPath) privateKey, err := rsa.GenerateKey(rand.Reader, 4096) if err != nil { utils.Log.Errorf("failed to generate RSA private key: %+v", err) return nil, false } publicKey, err := ssh.NewPublicKey(&privateKey.PublicKey) if err != nil { utils.Log.Errorf("failed to generate RSA public key: %+v", err) return nil, false } ret, err := ssh.NewSignerFromKey(privateKey) if err != nil { utils.Log.Errorf("failed to generate RSA signer: %+v", err) return nil, false } privateBytes := rsaEncodePrivateKey(privateKey) publicBytes := ssh.MarshalAuthorizedKey(publicKey) err = os.WriteFile(privateKeyPath, privateBytes, 0600) if err != nil { utils.Log.Errorf("failed to write RSA private key to file: %+v", err) return nil, false } err = os.WriteFile(publicKeyPath, publicBytes, 0644) if err != nil { _ = os.Remove(privateKeyPath) utils.Log.Errorf("failed to write RSA public key to file: %+v", err) return nil, false } return ret, true } func rsaEncodePrivateKey(privateKey *rsa.PrivateKey) []byte { privateKeyBytes := x509.MarshalPKCS1PrivateKey(privateKey) privateBlock := &pem.Block{ Type: "RSA PRIVATE KEY", Headers: nil, Bytes: privateKeyBytes, } return pem.EncodeToMemory(privateBlock) } func rsaDecodePrivateKey(bytes []byte) (*rsa.PrivateKey, error) { block, _ := pem.Decode(bytes) if block == nil { return nil, fmt.Errorf("failed to parse PEM block containing the key") } privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) if err != nil { return nil, err } return privateKey, nil } ================================================ FILE: server/sftp/sftp.go ================================================ package sftp import ( "os" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/ftp" "github.com/OpenListTeam/sftpd-openlist" ) type DriverAdapter struct { FtpDriver *ftp.AferoAdapter } func (s *DriverAdapter) OpenFile(_ string, _ uint32, _ *sftpd.Attr) (sftpd.File, error) { // See also GetHandle return nil, errs.NotImplement } func (s *DriverAdapter) OpenDir(_ string) (sftpd.Dir, error) { // See also GetHandle return nil, errs.NotImplement } func (s *DriverAdapter) Remove(name string) error { return s.FtpDriver.Remove(name) } func (s *DriverAdapter) Rename(old, new string, _ uint32) error { return s.FtpDriver.Rename(old, new) } func (s *DriverAdapter) Mkdir(name string, attr *sftpd.Attr) error { return s.FtpDriver.Mkdir(name, attr.Mode) } func (s *DriverAdapter) Rmdir(name string) error { return s.Remove(name) } func (s *DriverAdapter) Stat(name string, _ bool) (*sftpd.Attr, error) { stat, err := s.FtpDriver.Stat(name) if err != nil { return nil, err } return fileInfoToSftpAttr(stat), nil } func (s *DriverAdapter) SetStat(_ string, _ *sftpd.Attr) error { return errs.NotSupport } func (s *DriverAdapter) ReadLink(_ string) (string, error) { return "", errs.NotSupport } func (s *DriverAdapter) CreateLink(_, _ string, _ uint32) error { return errs.NotSupport } func (s *DriverAdapter) RealPath(path string) (string, error) { return utils.FixAndCleanPath(path), nil } func (s *DriverAdapter) GetHandle(name string, flags uint32, _ *sftpd.Attr, offset uint64) (sftpd.FileTransfer, error) { return s.FtpDriver.GetHandle(name, sftpFlagToOpenMode(flags), int64(offset)) } func (s *DriverAdapter) ReadDir(name string) ([]sftpd.NamedAttr, error) { dir, err := s.FtpDriver.ReadDir(name) if err != nil { return nil, err } ret := make([]sftpd.NamedAttr, len(dir)) for i, d := range dir { ret[i] = *fileInfoToSftpNamedAttr(d) } return ret, nil } // From leffss/sftpd func sftpFlagToOpenMode(flags uint32) int { mode := 0 if (flags & SSH_FXF_READ) != 0 { mode |= os.O_RDONLY } if (flags & SSH_FXF_WRITE) != 0 { mode |= os.O_WRONLY } if (flags & SSH_FXF_APPEND) != 0 { mode |= os.O_APPEND } if (flags & SSH_FXF_CREAT) != 0 { mode |= os.O_CREATE } if (flags & SSH_FXF_TRUNC) != 0 { mode |= os.O_TRUNC } if (flags & SSH_FXF_EXCL) != 0 { mode |= os.O_EXCL } return mode } func fileInfoToSftpAttr(stat os.FileInfo) *sftpd.Attr { ret := &sftpd.Attr{} ret.Flags |= sftpd.ATTR_SIZE ret.Size = uint64(stat.Size()) ret.Flags |= sftpd.ATTR_MODE ret.Mode = stat.Mode() ret.Flags |= sftpd.ATTR_TIME ret.ATime = stat.Sys().(model.Obj).CreateTime() ret.MTime = stat.ModTime() return ret } func fileInfoToSftpNamedAttr(stat os.FileInfo) *sftpd.NamedAttr { return &sftpd.NamedAttr{ Name: stat.Name(), Attr: *fileInfoToSftpAttr(stat), } } ================================================ FILE: server/sftp.go ================================================ package server import ( "context" "net/http" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/server/ftp" "github.com/OpenListTeam/OpenList/v4/server/sftp" "github.com/OpenListTeam/sftpd-openlist" "github.com/pkg/errors" "golang.org/x/crypto/ssh" ) type SftpDriver struct { proxyHeader http.Header config *sftpd.Config } func NewSftpDriver() (*SftpDriver, error) { ftp.InitStage() sftp.InitHostKey() return &SftpDriver{ proxyHeader: http.Header{ "User-Agent": {base.UserAgent}, }, }, nil } func (d *SftpDriver) GetConfig() *sftpd.Config { if d.config != nil { return d.config } var pwdAuth func(conn ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) = nil if !setting.GetBool(conf.SFTPDisablePasswordLogin) { pwdAuth = d.PasswordAuth } serverConfig := ssh.ServerConfig{ NoClientAuth: true, NoClientAuthCallback: d.NoClientAuth, PasswordCallback: pwdAuth, PublicKeyCallback: d.PublicKeyAuth, AuthLogCallback: d.AuthLogCallback, BannerCallback: d.GetBanner, } for _, k := range sftp.SSHSigners { serverConfig.AddHostKey(k) } d.config = &sftpd.Config{ ServerConfig: serverConfig, HostPort: conf.Conf.SFTP.Listen, ErrorLogFunc: utils.Log.Error, // DebugLogFunc: utils.Log.Debugf, } return d.config } func (d *SftpDriver) GetFileSystem(sc *ssh.ServerConn) (sftpd.FileSystem, error) { userObj, err := op.GetUserByName(sc.User()) if err != nil { return nil, err } ctx := context.Background() ctx = context.WithValue(ctx, conf.UserKey, userObj) ctx = context.WithValue(ctx, conf.MetaPassKey, "") ctx = context.WithValue(ctx, conf.ClientIPKey, sc.RemoteAddr().String()) ctx = context.WithValue(ctx, conf.ProxyHeaderKey, d.proxyHeader) return &sftp.DriverAdapter{FtpDriver: ftp.NewAferoAdapter(ctx)}, nil } func (d *SftpDriver) Close() { } func (d *SftpDriver) NoClientAuth(conn ssh.ConnMetadata) (*ssh.Permissions, error) { if conn.User() != "guest" { return nil, errors.New("only guest is allowed to login without authorization") } guest, err := op.GetGuest() if err != nil { return nil, err } if guest.Disabled || !guest.CanFTPAccess() { return nil, errors.New("user is not allowed to access via SFTP") } return nil, nil } func (d *SftpDriver) PasswordAuth(conn ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) { ip := conn.RemoteAddr().String() count, ok := model.LoginCache.Get(ip) if ok && count >= model.DefaultMaxAuthRetries { model.LoginCache.Expire(ip, model.DefaultLockDuration) return nil, errors.New("Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later.") } pass := string(password) userObj, err := op.GetUserByName(conn.User()) if err == nil { err = userObj.ValidateRawPassword(pass) if err != nil && setting.GetBool(conf.LdapLoginEnabled) && userObj.AllowLdap { err = common.HandleLdapLogin(conn.User(), pass) } } else if setting.GetBool(conf.LdapLoginEnabled) && model.CanFTPAccess(int32(setting.GetInt(conf.LdapDefaultPermission, 0))) { userObj, err = tryLdapLoginAndRegister(conn.User(), pass) } if err != nil { model.LoginCache.Set(ip, count+1) return nil, err } if userObj.Disabled || !userObj.CanFTPAccess() { model.LoginCache.Set(ip, count+1) return nil, errors.New("user is not allowed to access via SFTP") } model.LoginCache.Del(ip) return nil, nil } func (d *SftpDriver) PublicKeyAuth(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) { userObj, err := op.GetUserByName(conn.User()) if err != nil { return nil, err } if userObj.Disabled || !userObj.CanFTPAccess() { return nil, errors.New("user is not allowed to access via SFTP") } keys, _, err := op.GetSSHPublicKeyByUserId(userObj.ID, 1, -1) if err != nil { return nil, err } marshal := string(key.Marshal()) for _, sk := range keys { if marshal != sk.KeyStr { pubKey, _, _, _, e := ssh.ParseAuthorizedKey([]byte(sk.KeyStr)) if e != nil || marshal != string(pubKey.Marshal()) { continue } } sk.LastUsedTime = time.Now() _ = op.UpdateSSHPublicKey(&sk) return nil, nil } return nil, errors.New("public key refused") } func (d *SftpDriver) AuthLogCallback(conn ssh.ConnMetadata, method string, err error) { ip := conn.RemoteAddr().String() if err == nil { utils.Log.Infof("[SFTP] %s(%s) logged in via %s", conn.User(), ip, method) } else if method != "none" { utils.Log.Infof("[SFTP] %s(%s) tries logging in via %s but with error: %s", conn.User(), ip, method, err) } } func (d *SftpDriver) GetBanner(_ ssh.ConnMetadata) string { return setting.GetStr(conf.Announcement) } ================================================ FILE: server/static/config.go ================================================ package static import ( "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type SiteConfig struct { BasePath string Cdn string } func getSiteConfig() SiteConfig { siteConfig := SiteConfig{ BasePath: conf.URL.Path, Cdn: strings.ReplaceAll(strings.TrimSuffix(conf.Conf.Cdn, "/"), "$version", strings.TrimPrefix(conf.WebVersion, "v")), } if siteConfig.BasePath != "" { siteConfig.BasePath = utils.FixAndCleanPath(siteConfig.BasePath) // Keep consistent with frontend: trim trailing slash unless it's root if siteConfig.BasePath != "/" && strings.HasSuffix(siteConfig.BasePath, "/") { siteConfig.BasePath = strings.TrimSuffix(siteConfig.BasePath, "/") } } if siteConfig.BasePath == "" { siteConfig.BasePath = "/" } if siteConfig.Cdn == "" { siteConfig.Cdn = strings.TrimSuffix(siteConfig.BasePath, "/") } return siteConfig } ================================================ FILE: server/static/static.go ================================================ package static import ( "encoding/json" "errors" "fmt" "io" "io/fs" "net/http" "os" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/public" "github.com/gin-gonic/gin" ) type ManifestIcon struct { Src string `json:"src"` Sizes string `json:"sizes"` Type string `json:"type"` } type Manifest struct { Display string `json:"display"` Scope string `json:"scope"` StartURL string `json:"start_url"` Name string `json:"name"` Icons []ManifestIcon `json:"icons"` } var static fs.FS func initStatic() { utils.Log.Debug("Initializing static file system...") if conf.Conf.DistDir == "" { dist, err := fs.Sub(public.Public, "dist") if err != nil { utils.Log.Fatalf("failed to read dist dir: %v", err) } static = dist utils.Log.Debug("Using embedded dist directory") return } static = os.DirFS(conf.Conf.DistDir) utils.Log.Infof("Using custom dist directory: %s", conf.Conf.DistDir) } func replaceStrings(content string, replacements map[string]string) string { for old, new := range replacements { content = strings.Replace(content, old, new, 1) } return content } func initIndex(siteConfig SiteConfig) { utils.Log.Debug("Initializing index.html...") // dist_dir is empty and cdn is not empty, and web_version is empty or beta or dev or rolling if conf.Conf.DistDir == "" && conf.Conf.Cdn != "" && (conf.WebVersion == "" || conf.WebVersion == "beta" || conf.WebVersion == "dev" || conf.WebVersion == "rolling") { utils.Log.Infof("Fetching index.html from CDN: %s/index.html...", siteConfig.Cdn) resp, err := base.RestyClient.R(). SetHeader("Accept", "text/html"). Get(fmt.Sprintf("%s/index.html", siteConfig.Cdn)) if err != nil { utils.Log.Fatalf("failed to fetch index.html from CDN: %v", err) } if resp.StatusCode() != http.StatusOK { utils.Log.Fatalf("failed to fetch index.html from CDN, status code: %d", resp.StatusCode()) } conf.RawIndexHtml = string(resp.Body()) utils.Log.Info("Successfully fetched index.html from CDN") } else { utils.Log.Debug("Reading index.html from static files system...") indexFile, err := static.Open("index.html") if err != nil { if errors.Is(err, fs.ErrNotExist) { utils.Log.Fatalf("index.html not exist, you may forget to put dist of frontend to public/dist") } utils.Log.Fatalf("failed to read index.html: %v", err) } defer func() { _ = indexFile.Close() }() index, err := io.ReadAll(indexFile) if err != nil { utils.Log.Fatalf("failed to read dist/index.html") } conf.RawIndexHtml = string(index) utils.Log.Debug("Successfully read index.html from static files system") } utils.Log.Debug("Replacing placeholders in index.html...") // Construct the correct manifest path based on basePath manifestPath := "/manifest.json" if siteConfig.BasePath != "/" { manifestPath = siteConfig.BasePath + "/manifest.json" } replaceMap := map[string]string{ "cdn: undefined": fmt.Sprintf("cdn: '%s'", siteConfig.Cdn), "base_path: undefined": fmt.Sprintf("base_path: '%s'", siteConfig.BasePath), `href="/manifest.json"`: fmt.Sprintf(`href="%s"`, manifestPath), } conf.RawIndexHtml = replaceStrings(conf.RawIndexHtml, replaceMap) UpdateIndex() } func UpdateIndex() { utils.Log.Debug("Updating index.html with settings...") favicon := setting.GetStr(conf.Favicon) logo := strings.Split(setting.GetStr(conf.Logo), "\n")[0] title := setting.GetStr(conf.SiteTitle) customizeHead := setting.GetStr(conf.CustomizeHead) customizeBody := setting.GetStr(conf.CustomizeBody) mainColor := setting.GetStr(conf.MainColor) utils.Log.Debug("Applying replacements for default pages...") replaceMap1 := map[string]string{ "https://res.oplist.org/logo/logo.svg": favicon, "https://res.oplist.org/logo/logo.png": logo, "Loading...": title, "main_color: undefined": fmt.Sprintf("main_color: '%s'", mainColor), } conf.ManageHtml = replaceStrings(conf.RawIndexHtml, replaceMap1) utils.Log.Debug("Applying replacements for manage pages...") replaceMap2 := map[string]string{ "": customizeHead, "": customizeBody, } conf.IndexHtml = replaceStrings(conf.ManageHtml, replaceMap2) utils.Log.Debug("Index.html update completed") } func ManifestJSON(c *gin.Context) { // Get site configuration to ensure consistent base path handling siteConfig := getSiteConfig() // Get site title from settings siteTitle := setting.GetStr(conf.SiteTitle) // Get logo from settings, use the first line (light theme logo) logoSetting := setting.GetStr(conf.Logo) logoUrl := strings.Split(logoSetting, "\n")[0] // Use base path from site config for consistency basePath := siteConfig.BasePath // Determine scope and start_url // PWA scope and start_url should always point to our application's base path // regardless of whether static resources come from CDN or local server scope := basePath startURL := basePath manifest := Manifest{ Display: "standalone", Scope: scope, StartURL: startURL, Name: siteTitle, Icons: []ManifestIcon{ { Src: logoUrl, Sizes: "512x512", Type: "image/png", }, }, } c.Header("Content-Type", "application/json") c.Header("Cache-Control", "public, max-age=3600") // cache for 1 hour if err := json.NewEncoder(c.Writer).Encode(manifest); err != nil { utils.Log.Errorf("Failed to encode manifest.json: %v", err) c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to generate manifest"}) return } } func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { utils.Log.Debug("Setting up static routes...") siteConfig := getSiteConfig() initStatic() initIndex(siteConfig) folders := []string{"assets", "images", "streamer", "static"} if conf.Conf.Cdn == "" { utils.Log.Debug("Setting up static file serving...") r.Use(func(c *gin.Context) { for _, folder := range folders { if strings.HasPrefix(c.Request.RequestURI, fmt.Sprintf("/%s/", folder)) { c.Header("Cache-Control", "public, max-age=15552000") } } }) for _, folder := range folders { sub, err := fs.Sub(static, folder) if err != nil { utils.Log.Fatalf("can't find folder: %s", folder) } utils.Log.Debugf("Setting up route for folder: %s", folder) r.StaticFS(fmt.Sprintf("/%s/", folder), http.FS(sub)) } } else { // Ensure static file redirected to CDN for _, folder := range folders { r.GET(fmt.Sprintf("/%s/*filepath", folder), func(c *gin.Context) { filepath := c.Param("filepath") c.Redirect(http.StatusFound, fmt.Sprintf("%s/%s%s", siteConfig.Cdn, folder, filepath)) }) } } utils.Log.Debug("Setting up catch-all route...") noRoute(func(c *gin.Context) { if c.Request.Method != "GET" && c.Request.Method != "POST" { c.Status(405) return } c.Header("Content-Type", "text/html") c.Status(200) if strings.HasPrefix(c.Request.URL.Path, "/@manage") { _, _ = c.Writer.WriteString(conf.ManageHtml) } else { _, _ = c.Writer.WriteString(conf.IndexHtml) } c.Writer.Flush() c.Writer.WriteHeaderNow() }) } ================================================ FILE: server/utils.go ================================================ package server import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/server/common" ) func tryLdapLoginAndRegister(user, pass string) (*model.User, error) { err := common.HandleLdapLogin(user, pass) if err != nil { return nil, err } return common.LdapRegister(user) } ================================================ FILE: server/webdav/buffered_response_writer.go ================================================ package webdav import ( "net/http" ) type bufferedResponseWriter struct { statusCode int data []byte header http.Header } func (w *bufferedResponseWriter) Header() http.Header { if w.header == nil { w.header = make(http.Header) } return w.header } func (w *bufferedResponseWriter) Write(bytes []byte) (int, error) { w.data = append(w.data, bytes...) return len(bytes), nil } func (w *bufferedResponseWriter) WriteHeader(statusCode int) { if w.statusCode == 0 { w.statusCode = statusCode } } func (w *bufferedResponseWriter) WriteToResponse(rw http.ResponseWriter) (int, error) { h := rw.Header() for k, vs := range w.header { for _, v := range vs { h.Add(k, v) } } rw.WriteHeader(w.statusCode) return rw.Write(w.data) } func newBufferedResponseWriter() *bufferedResponseWriter { return &bufferedResponseWriter{ statusCode: 0, } } ================================================ FILE: server/webdav/file.go ================================================ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package webdav import ( "context" "net/http" "path" "path/filepath" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" ) // slashClean is equivalent to but slightly more efficient than // path.Clean("/" + name). func slashClean(name string) string { if name == "" || name[0] != '/' { name = "/" + name } return path.Clean(name) } // moveFiles moves files and/or directories from src to dst. // // See section 9.9.4 for when various HTTP status codes apply. func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int, err error) { srcDir := path.Dir(src) dstDir := path.Dir(dst) srcName := path.Base(src) dstName := path.Base(dst) user := ctx.Value(conf.UserKey).(*model.User) if srcDir != dstDir && !user.CanMove() { return http.StatusForbidden, nil } if srcName != dstName && !user.CanRename() { return http.StatusForbidden, nil } if srcDir == dstDir { err = fs.Rename(ctx, src, dstName) } else { _, err = fs.Move(context.WithValue(ctx, conf.NoTaskKey, struct{}{}), src, dstDir) if err != nil { return http.StatusInternalServerError, err } if srcName != dstName { err = fs.Rename(ctx, path.Join(dstDir, srcName), dstName) } } if err != nil { return http.StatusInternalServerError, err } // TODO if there are no files copy, should return 204 return http.StatusCreated, nil } // copyFiles copies files and/or directories from src to dst. // // See section 9.8.5 for when various HTTP status codes apply. func copyFiles(ctx context.Context, src, dst string, overwrite bool) (status int, err error) { dstDir := path.Dir(dst) _, err = fs.Copy(context.WithValue(ctx, conf.NoTaskKey, struct{}{}), src, dstDir) if err != nil { return http.StatusInternalServerError, err } // TODO if there are no files copy, should return 204 return http.StatusCreated, nil } // walkFS traverses filesystem fs starting at name up to depth levels. // // Allowed values for depth are 0, 1 or infiniteDepth. For each visited node, // walkFS calls walkFn. If a visited file system node is a directory and // walkFn returns path.SkipDir, walkFS will skip traversal of this node. func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn func(reqPath string, info model.Obj, err error) error) error { // This implementation is based on Walk's code in the standard path/path package. err := walkFn(name, info, nil) if err != nil { if info.IsDir() && err == filepath.SkipDir { return nil } return err } if !info.IsDir() || depth == 0 { return nil } if depth == 1 { depth = 0 } meta, _ := op.GetNearestMeta(name) // Read directory names. objs, err := fs.List(context.WithValue(ctx, conf.MetaKey, meta), name, &fs.ListArgs{}) //f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) //if err != nil { // return walkFn(name, info, err) //} //fileInfos, err := f.Readdir(0) //f.Close() if err != nil { return walkFn(name, info, err) } for _, fileInfo := range objs { filename := path.Join(name, fileInfo.GetName()) if err != nil { if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { return err } } else { err = walkFS(ctx, depth, filename, fileInfo, walkFn) if err != nil { if !fileInfo.IsDir() || err != filepath.SkipDir { return err } } } } return nil } ================================================ FILE: server/webdav/if.go ================================================ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package webdav // The If header is covered by Section 10.4. // http://www.webdav.org/specs/rfc4918.html#HEADER_If import ( "strings" ) // ifHeader is a disjunction (OR) of ifLists. type ifHeader struct { lists []ifList } // ifList is a conjunction (AND) of Conditions, and an optional resource tag. type ifList struct { resourceTag string conditions []Condition } // parseIfHeader parses the "If: foo bar" HTTP header. The httpHeader string // should omit the "If:" prefix and have any "\r\n"s collapsed to a " ", as is // returned by req.Header.Get("If") for a http.Request req. func parseIfHeader(httpHeader string) (h ifHeader, ok bool) { s := strings.TrimSpace(httpHeader) switch tokenType, _, _ := lex(s); tokenType { case '(': return parseNoTagLists(s) case angleTokenType: return parseTaggedLists(s) default: return ifHeader{}, false } } func parseNoTagLists(s string) (h ifHeader, ok bool) { for { l, remaining, ok := parseList(s) if !ok { return ifHeader{}, false } h.lists = append(h.lists, l) if remaining == "" { return h, true } s = remaining } } func parseTaggedLists(s string) (h ifHeader, ok bool) { resourceTag, n := "", 0 for first := true; ; first = false { tokenType, tokenStr, remaining := lex(s) switch tokenType { case angleTokenType: if !first && n == 0 { return ifHeader{}, false } resourceTag, n = tokenStr, 0 s = remaining case '(': n++ l, remaining, ok := parseList(s) if !ok { return ifHeader{}, false } l.resourceTag = resourceTag h.lists = append(h.lists, l) if remaining == "" { return h, true } s = remaining default: return ifHeader{}, false } } } func parseList(s string) (l ifList, remaining string, ok bool) { tokenType, _, s := lex(s) if tokenType != '(' { return ifList{}, "", false } for { tokenType, _, remaining = lex(s) if tokenType == ')' { if len(l.conditions) == 0 { return ifList{}, "", false } return l, remaining, true } c, remaining, ok := parseCondition(s) if !ok { return ifList{}, "", false } l.conditions = append(l.conditions, c) s = remaining } } func parseCondition(s string) (c Condition, remaining string, ok bool) { tokenType, tokenStr, s := lex(s) if tokenType == notTokenType { c.Not = true tokenType, tokenStr, s = lex(s) } switch tokenType { case strTokenType, angleTokenType: c.Token = tokenStr case squareTokenType: c.ETag = tokenStr default: return Condition{}, "", false } return c, s, true } // Single-rune tokens like '(' or ')' have a token type equal to their rune. // All other tokens have a negative token type. const ( errTokenType = rune(-1) eofTokenType = rune(-2) strTokenType = rune(-3) notTokenType = rune(-4) angleTokenType = rune(-5) squareTokenType = rune(-6) ) func lex(s string) (tokenType rune, tokenStr string, remaining string) { // The net/textproto Reader that parses the HTTP header will collapse // Linear White Space that spans multiple "\r\n" lines to a single " ", // so we don't need to look for '\r' or '\n'. for len(s) > 0 && (s[0] == '\t' || s[0] == ' ') { s = s[1:] } if len(s) == 0 { return eofTokenType, "", "" } i := 0 loop: for ; i < len(s); i++ { switch s[i] { case '\t', ' ', '(', ')', '<', '>', '[', ']': break loop } } if i != 0 { tokenStr, remaining = s[:i], s[i:] if tokenStr == "Not" { return notTokenType, "", remaining } return strTokenType, tokenStr, remaining } j := 0 switch s[0] { case '<': j, tokenType = strings.IndexByte(s, '>'), angleTokenType case '[': j, tokenType = strings.IndexByte(s, ']'), squareTokenType default: return rune(s[0]), "", s[1:] } if j < 0 { return errTokenType, "", "" } return tokenType, s[1:j], s[j+1:] } ================================================ FILE: server/webdav/internal/xml/README ================================================ This is a fork of the encoding/xml package at ca1d6c4, the last commit before https://go.googlesource.com/go/+/c0d6d33 "encoding/xml: restore Go 1.4 name space behavior" made late in the lead-up to the Go 1.5 release. The list of encoding/xml changes is at https://go.googlesource.com/go/+log/master/src/encoding/xml This fork is temporary, and I (nigeltao) expect to revert it after Go 1.6 is released. See http://golang.org/issue/11841 ================================================ FILE: server/webdav/internal/xml/atom_test.go ================================================ // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package xml import "time" var atomValue = &Feed{ XMLName: Name{"http://www.w3.org/2005/Atom", "feed"}, Title: "Example Feed", Link: []Link{{Href: "http://example.org/"}}, Updated: ParseTime("2003-12-13T18:30:02Z"), Author: Person{Name: "John Doe"}, Id: "urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6", Entry: []Entry{ { Title: "Atom-Powered Robots Run Amok", Link: []Link{{Href: "http://example.org/2003/12/13/atom03"}}, Id: "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a", Updated: ParseTime("2003-12-13T18:30:02Z"), Summary: NewText("Some text."), }, }, } var atomXml = `` + `` + `Example Feed` + `urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6` + `` + `John Doe` + `` + `Atom-Powered Robots Run Amok` + `urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a` + `` + `2003-12-13T18:30:02Z` + `` + `Some text.` + `` + `` func ParseTime(str string) time.Time { t, err := time.Parse(time.RFC3339, str) if err != nil { panic(err) } return t } func NewText(text string) Text { return Text{ Body: text, } } ================================================ FILE: server/webdav/internal/xml/example_test.go ================================================ // Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package xml_test import ( "encoding/xml" "fmt" "os" ) func ExampleMarshalIndent() { type Address struct { City, State string } type Person struct { XMLName xml.Name `xml:"person"` Id int `xml:"id,attr"` FirstName string `xml:"name>first"` LastName string `xml:"name>last"` Age int `xml:"age"` Height float32 `xml:"height,omitempty"` Married bool Address Comment string `xml:",comment"` } v := &Person{Id: 13, FirstName: "John", LastName: "Doe", Age: 42} v.Comment = " Need more details. " v.Address = Address{"Hanga Roa", "Easter Island"} output, err := xml.MarshalIndent(v, " ", " ") if err != nil { fmt.Printf("error: %v\n", err) } os.Stdout.Write(output) // Output: // // // John // Doe // // 42 // false // Hanga Roa // Easter Island // // } func ExampleEncoder() { type Address struct { City, State string } type Person struct { XMLName xml.Name `xml:"person"` Id int `xml:"id,attr"` FirstName string `xml:"name>first"` LastName string `xml:"name>last"` Age int `xml:"age"` Height float32 `xml:"height,omitempty"` Married bool Address Comment string `xml:",comment"` } v := &Person{Id: 13, FirstName: "John", LastName: "Doe", Age: 42} v.Comment = " Need more details. " v.Address = Address{"Hanga Roa", "Easter Island"} enc := xml.NewEncoder(os.Stdout) enc.Indent(" ", " ") if err := enc.Encode(v); err != nil { fmt.Printf("error: %v\n", err) } // Output: // // // John // Doe // // 42 // false // Hanga Roa // Easter Island // // } // This example demonstrates unmarshaling an XML excerpt into a value with // some preset fields. Note that the Phone field isn't modified and that // the XML element is ignored. Also, the Groups field is assigned // considering the element path provided in its tag. func ExampleUnmarshal() { type Email struct { Where string `xml:"where,attr"` Addr string } type Address struct { City, State string } type Result struct { XMLName xml.Name `xml:"Person"` Name string `xml:"FullName"` Phone string Email []Email Groups []string `xml:"Group>Value"` Address } v := Result{Name: "none", Phone: "none"} data := ` Grace R. Emlin Example Inc. gre@example.com gre@work.com Friends Squash Hanga Roa Easter Island ` err := xml.Unmarshal([]byte(data), &v) if err != nil { fmt.Printf("error: %v", err) return } fmt.Printf("XMLName: %#v\n", v.XMLName) fmt.Printf("Name: %q\n", v.Name) fmt.Printf("Phone: %q\n", v.Phone) fmt.Printf("Email: %v\n", v.Email) fmt.Printf("Groups: %v\n", v.Groups) fmt.Printf("Address: %v\n", v.Address) // Output: // XMLName: xml.Name{Space:"", Local:"Person"} // Name: "Grace R. Emlin" // Phone: "none" // Email: [{home gre@example.com} {work gre@work.com}] // Groups: [Friends Squash] // Address: {Hanga Roa Easter Island} } ================================================ FILE: server/webdav/internal/xml/marshal.go ================================================ // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package xml import ( "bufio" "bytes" "encoding" "fmt" "io" "reflect" "strconv" "strings" ) const ( // A generic XML header suitable for use with the output of Marshal. // This is not automatically added to any output of this package, // it is provided as a convenience. Header = `` + "\n" ) // Marshal returns the XML encoding of v. // // Marshal handles an array or slice by marshalling each of the elements. // Marshal handles a pointer by marshalling the value it points at or, if the // pointer is nil, by writing nothing. Marshal handles an interface value by // marshalling the value it contains or, if the interface value is nil, by // writing nothing. Marshal handles all other data by writing one or more XML // elements containing the data. // // The name for the XML elements is taken from, in order of preference: // - the tag on the XMLName field, if the data is a struct // - the value of the XMLName field of type xml.Name // - the tag of the struct field used to obtain the data // - the name of the struct field used to obtain the data // - the name of the marshalled type // // The XML element for a struct contains marshalled elements for each of the // exported fields of the struct, with these exceptions: // - the XMLName field, described above, is omitted. // - a field with tag "-" is omitted. // - a field with tag "name,attr" becomes an attribute with // the given name in the XML element. // - a field with tag ",attr" becomes an attribute with the // field name in the XML element. // - a field with tag ",chardata" is written as character data, // not as an XML element. // - a field with tag ",innerxml" is written verbatim, not subject // to the usual marshalling procedure. // - a field with tag ",comment" is written as an XML comment, not // subject to the usual marshalling procedure. It must not contain // the "--" string within it. // - a field with a tag including the "omitempty" option is omitted // if the field value is empty. The empty values are false, 0, any // nil pointer or interface value, and any array, slice, map, or // string of length zero. // - an anonymous struct field is handled as if the fields of its // value were part of the outer struct. // // If a field uses a tag "a>b>c", then the element c will be nested inside // parent elements a and b. Fields that appear next to each other that name // the same parent will be enclosed in one XML element. // // See MarshalIndent for an example. // // Marshal will return an error if asked to marshal a channel, function, or map. func Marshal(v interface{}) ([]byte, error) { var b bytes.Buffer if err := NewEncoder(&b).Encode(v); err != nil { return nil, err } return b.Bytes(), nil } // Marshaler is the interface implemented by objects that can marshal // themselves into valid XML elements. // // MarshalXML encodes the receiver as zero or more XML elements. // By convention, arrays or slices are typically encoded as a sequence // of elements, one per entry. // Using start as the element tag is not required, but doing so // will enable Unmarshal to match the XML elements to the correct // struct field. // One common implementation strategy is to construct a separate // value with a layout corresponding to the desired XML and then // to encode it using e.EncodeElement. // Another common strategy is to use repeated calls to e.EncodeToken // to generate the XML output one token at a time. // The sequence of encoded tokens must make up zero or more valid // XML elements. type Marshaler interface { MarshalXML(e *Encoder, start StartElement) error } // MarshalerAttr is the interface implemented by objects that can marshal // themselves into valid XML attributes. // // MarshalXMLAttr returns an XML attribute with the encoded value of the receiver. // Using name as the attribute name is not required, but doing so // will enable Unmarshal to match the attribute to the correct // struct field. // If MarshalXMLAttr returns the zero attribute Attr{}, no attribute // will be generated in the output. // MarshalXMLAttr is used only for struct fields with the // "attr" option in the field tag. type MarshalerAttr interface { MarshalXMLAttr(name Name) (Attr, error) } // MarshalIndent works like Marshal, but each XML element begins on a new // indented line that starts with prefix and is followed by one or more // copies of indent according to the nesting depth. func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { var b bytes.Buffer enc := NewEncoder(&b) enc.Indent(prefix, indent) if err := enc.Encode(v); err != nil { return nil, err } return b.Bytes(), nil } // An Encoder writes XML data to an output stream. type Encoder struct { p printer } // NewEncoder returns a new encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { e := &Encoder{printer{Writer: bufio.NewWriter(w)}} e.p.encoder = e return e } // Indent sets the encoder to generate XML in which each element // begins on a new indented line that starts with prefix and is followed by // one or more copies of indent according to the nesting depth. func (enc *Encoder) Indent(prefix, indent string) { enc.p.prefix = prefix enc.p.indent = indent } // Encode writes the XML encoding of v to the stream. // // See the documentation for Marshal for details about the conversion // of Go values to XML. // // Encode calls Flush before returning. func (enc *Encoder) Encode(v interface{}) error { err := enc.p.marshalValue(reflect.ValueOf(v), nil, nil) if err != nil { return err } return enc.p.Flush() } // EncodeElement writes the XML encoding of v to the stream, // using start as the outermost tag in the encoding. // // See the documentation for Marshal for details about the conversion // of Go values to XML. // // EncodeElement calls Flush before returning. func (enc *Encoder) EncodeElement(v interface{}, start StartElement) error { err := enc.p.marshalValue(reflect.ValueOf(v), nil, &start) if err != nil { return err } return enc.p.Flush() } var ( begComment = []byte("") endProcInst = []byte("?>") endDirective = []byte(">") ) // EncodeToken writes the given XML token to the stream. // It returns an error if StartElement and EndElement tokens are not // properly matched. // // EncodeToken does not call Flush, because usually it is part of a // larger operation such as Encode or EncodeElement (or a custom // Marshaler's MarshalXML invoked during those), and those will call // Flush when finished. Callers that create an Encoder and then invoke // EncodeToken directly, without using Encode or EncodeElement, need to // call Flush when finished to ensure that the XML is written to the // underlying writer. // // EncodeToken allows writing a ProcInst with Target set to "xml" only // as the first token in the stream. // // When encoding a StartElement holding an XML namespace prefix // declaration for a prefix that is not already declared, contained // elements (including the StartElement itself) will use the declared // prefix when encoding names with matching namespace URIs. func (enc *Encoder) EncodeToken(t Token) error { p := &enc.p switch t := t.(type) { case StartElement: if err := p.writeStart(&t); err != nil { return err } case EndElement: if err := p.writeEnd(t.Name); err != nil { return err } case CharData: escapeText(p, t, false) case Comment: if bytes.Contains(t, endComment) { return fmt.Errorf("xml: EncodeToken of Comment containing --> marker") } p.WriteString("") return p.cachedWriteError() case ProcInst: // First token to be encoded which is also a ProcInst with target of xml // is the xml declaration. The only ProcInst where target of xml is allowed. if t.Target == "xml" && p.Buffered() != 0 { return fmt.Errorf("xml: EncodeToken of ProcInst xml target only valid for xml declaration, first token encoded") } if !isNameString(t.Target) { return fmt.Errorf("xml: EncodeToken of ProcInst with invalid Target") } if bytes.Contains(t.Inst, endProcInst) { return fmt.Errorf("xml: EncodeToken of ProcInst containing ?> marker") } p.WriteString(" 0 { p.WriteByte(' ') p.Write(t.Inst) } p.WriteString("?>") case Directive: if !isValidDirective(t) { return fmt.Errorf("xml: EncodeToken of Directive containing wrong < or > markers") } p.WriteString("") default: return fmt.Errorf("xml: EncodeToken of invalid token type") } return p.cachedWriteError() } // isValidDirective reports whether dir is a valid directive text, // meaning angle brackets are matched, ignoring comments and strings. func isValidDirective(dir Directive) bool { var ( depth int inquote uint8 incomment bool ) for i, c := range dir { switch { case incomment: if c == '>' { if n := 1 + i - len(endComment); n >= 0 && bytes.Equal(dir[n:i+1], endComment) { incomment = false } } // Just ignore anything in comment case inquote != 0: if c == inquote { inquote = 0 } // Just ignore anything within quotes case c == '\'' || c == '"': inquote = c case c == '<': if i+len(begComment) < len(dir) && bytes.Equal(dir[i:i+len(begComment)], begComment) { incomment = true } else { depth++ } case c == '>': if depth == 0 { return false } depth-- } } return depth == 0 && inquote == 0 && !incomment } // Flush flushes any buffered XML to the underlying writer. // See the EncodeToken documentation for details about when it is necessary. func (enc *Encoder) Flush() error { return enc.p.Flush() } type printer struct { *bufio.Writer encoder *Encoder seq int indent string prefix string depth int indentedIn bool putNewline bool defaultNS string attrNS map[string]string // map prefix -> name space attrPrefix map[string]string // map name space -> prefix prefixes []printerPrefix tags []Name } // printerPrefix holds a namespace undo record. // When an element is popped, the prefix record // is set back to the recorded URL. The empty // prefix records the URL for the default name space. // // The start of an element is recorded with an element // that has mark=true. type printerPrefix struct { prefix string url string mark bool } func (p *printer) prefixForNS(url string, isAttr bool) string { // The "http://www.w3.org/XML/1998/namespace" name space is predefined as "xml" // and must be referred to that way. // (The "http://www.w3.org/2000/xmlns/" name space is also predefined as "xmlns", // but users should not be trying to use that one directly - that's our job.) if url == xmlURL { return "xml" } if !isAttr && url == p.defaultNS { // We can use the default name space. return "" } return p.attrPrefix[url] } // defineNS pushes any namespace definition found in the given attribute. // If ignoreNonEmptyDefault is true, an xmlns="nonempty" // attribute will be ignored. func (p *printer) defineNS(attr Attr, ignoreNonEmptyDefault bool) error { var prefix string if attr.Name.Local == "xmlns" { if attr.Name.Space != "" && attr.Name.Space != "xml" && attr.Name.Space != xmlURL { return fmt.Errorf("xml: cannot redefine xmlns attribute prefix") } } else if attr.Name.Space == "xmlns" && attr.Name.Local != "" { prefix = attr.Name.Local if attr.Value == "" { // Technically, an empty XML namespace is allowed for an attribute. // From http://www.w3.org/TR/xml-names11/#scoping-defaulting: // // The attribute value in a namespace declaration for a prefix may be // empty. This has the effect, within the scope of the declaration, of removing // any association of the prefix with a namespace name. // // However our namespace prefixes here are used only as hints. There's // no need to respect the removal of a namespace prefix, so we ignore it. return nil } } else { // Ignore: it's not a namespace definition return nil } if prefix == "" { if attr.Value == p.defaultNS { // No need for redefinition. return nil } if attr.Value != "" && ignoreNonEmptyDefault { // We have an xmlns="..." value but // it can't define a name space in this context, // probably because the element has an empty // name space. In this case, we just ignore // the name space declaration. return nil } } else if _, ok := p.attrPrefix[attr.Value]; ok { // There's already a prefix for the given name space, // so use that. This prevents us from // having two prefixes for the same name space // so attrNS and attrPrefix can remain bijective. return nil } p.pushPrefix(prefix, attr.Value) return nil } // createNSPrefix creates a name space prefix attribute // to use for the given name space, defining a new prefix // if necessary. // If isAttr is true, the prefix is to be created for an attribute // prefix, which means that the default name space cannot // be used. func (p *printer) createNSPrefix(url string, isAttr bool) { if _, ok := p.attrPrefix[url]; ok { // We already have a prefix for the given URL. return } switch { case !isAttr && url == p.defaultNS: // We can use the default name space. return case url == "": // The only way we can encode names in the empty // name space is by using the default name space, // so we must use that. if p.defaultNS != "" { // The default namespace is non-empty, so we // need to set it to empty. p.pushPrefix("", "") } return case url == xmlURL: return } // TODO If the URL is an existing prefix, we could // use it as is. That would enable the // marshaling of elements that had been unmarshaled // and with a name space prefix that was not found. // although technically it would be incorrect. // Pick a name. We try to use the final element of the path // but fall back to _. prefix := strings.TrimRight(url, "/") if i := strings.LastIndex(prefix, "/"); i >= 0 { prefix = prefix[i+1:] } if prefix == "" || !isName([]byte(prefix)) || strings.Contains(prefix, ":") { prefix = "_" } if strings.HasPrefix(prefix, "xml") { // xmlanything is reserved. prefix = "_" + prefix } if p.attrNS[prefix] != "" { // Name is taken. Find a better one. for p.seq++; ; p.seq++ { if id := prefix + "_" + strconv.Itoa(p.seq); p.attrNS[id] == "" { prefix = id break } } } p.pushPrefix(prefix, url) } // writeNamespaces writes xmlns attributes for all the // namespace prefixes that have been defined in // the current element. func (p *printer) writeNamespaces() { for i := len(p.prefixes) - 1; i >= 0; i-- { prefix := p.prefixes[i] if prefix.mark { return } p.WriteString(" ") if prefix.prefix == "" { // Default name space. p.WriteString(`xmlns="`) } else { p.WriteString("xmlns:") p.WriteString(prefix.prefix) p.WriteString(`="`) } EscapeText(p, []byte(p.nsForPrefix(prefix.prefix))) p.WriteString(`"`) } } // pushPrefix pushes a new prefix on the prefix stack // without checking to see if it is already defined. func (p *printer) pushPrefix(prefix, url string) { p.prefixes = append(p.prefixes, printerPrefix{ prefix: prefix, url: p.nsForPrefix(prefix), }) p.setAttrPrefix(prefix, url) } // nsForPrefix returns the name space for the given // prefix. Note that this is not valid for the // empty attribute prefix, which always has an empty // name space. func (p *printer) nsForPrefix(prefix string) string { if prefix == "" { return p.defaultNS } return p.attrNS[prefix] } // markPrefix marks the start of an element on the prefix // stack. func (p *printer) markPrefix() { p.prefixes = append(p.prefixes, printerPrefix{ mark: true, }) } // popPrefix pops all defined prefixes for the current // element. func (p *printer) popPrefix() { for len(p.prefixes) > 0 { prefix := p.prefixes[len(p.prefixes)-1] p.prefixes = p.prefixes[:len(p.prefixes)-1] if prefix.mark { break } p.setAttrPrefix(prefix.prefix, prefix.url) } } // setAttrPrefix sets an attribute name space prefix. // If url is empty, the attribute is removed. // If prefix is empty, the default name space is set. func (p *printer) setAttrPrefix(prefix, url string) { if prefix == "" { p.defaultNS = url return } if url == "" { delete(p.attrPrefix, p.attrNS[prefix]) delete(p.attrNS, prefix) return } if p.attrPrefix == nil { // Need to define a new name space. p.attrPrefix = make(map[string]string) p.attrNS = make(map[string]string) } // Remove any old prefix value. This is OK because we maintain a // strict one-to-one mapping between prefix and URL (see // defineNS) delete(p.attrPrefix, p.attrNS[prefix]) p.attrPrefix[url] = prefix p.attrNS[prefix] = url } var ( marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() marshalerAttrType = reflect.TypeOf((*MarshalerAttr)(nil)).Elem() textMarshalerType = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem() ) // marshalValue writes one or more XML elements representing val. // If val was obtained from a struct field, finfo must have its details. func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo, startTemplate *StartElement) error { if startTemplate != nil && startTemplate.Name.Local == "" { return fmt.Errorf("xml: EncodeElement of StartElement with missing name") } if !val.IsValid() { return nil } if finfo != nil && finfo.flags&fOmitEmpty != 0 && isEmptyValue(val) { return nil } // Drill into interfaces and pointers. // This can turn into an infinite loop given a cyclic chain, // but it matches the Go 1 behavior. for val.Kind() == reflect.Interface || val.Kind() == reflect.Ptr { if val.IsNil() { return nil } val = val.Elem() } kind := val.Kind() typ := val.Type() // Check for marshaler. if val.CanInterface() && typ.Implements(marshalerType) { return p.marshalInterface(val.Interface().(Marshaler), p.defaultStart(typ, finfo, startTemplate)) } if val.CanAddr() { pv := val.Addr() if pv.CanInterface() && pv.Type().Implements(marshalerType) { return p.marshalInterface(pv.Interface().(Marshaler), p.defaultStart(pv.Type(), finfo, startTemplate)) } } // Check for text marshaler. if val.CanInterface() && typ.Implements(textMarshalerType) { return p.marshalTextInterface(val.Interface().(encoding.TextMarshaler), p.defaultStart(typ, finfo, startTemplate)) } if val.CanAddr() { pv := val.Addr() if pv.CanInterface() && pv.Type().Implements(textMarshalerType) { return p.marshalTextInterface(pv.Interface().(encoding.TextMarshaler), p.defaultStart(pv.Type(), finfo, startTemplate)) } } // Slices and arrays iterate over the elements. They do not have an enclosing tag. if (kind == reflect.Slice || kind == reflect.Array) && typ.Elem().Kind() != reflect.Uint8 { for i, n := 0, val.Len(); i < n; i++ { if err := p.marshalValue(val.Index(i), finfo, startTemplate); err != nil { return err } } return nil } tinfo, err := getTypeInfo(typ) if err != nil { return err } // Create start element. // Precedence for the XML element name is: // 0. startTemplate // 1. XMLName field in underlying struct; // 2. field name/tag in the struct field; and // 3. type name var start StartElement // explicitNS records whether the element's name space has been // explicitly set (for example an XMLName field). explicitNS := false if startTemplate != nil { start.Name = startTemplate.Name explicitNS = true start.Attr = append(start.Attr, startTemplate.Attr...) } else if tinfo.xmlname != nil { xmlname := tinfo.xmlname if xmlname.name != "" { start.Name.Space, start.Name.Local = xmlname.xmlns, xmlname.name } else if v, ok := xmlname.value(val).Interface().(Name); ok && v.Local != "" { start.Name = v } explicitNS = true } if start.Name.Local == "" && finfo != nil { start.Name.Local = finfo.name if finfo.xmlns != "" { start.Name.Space = finfo.xmlns explicitNS = true } } if start.Name.Local == "" { name := typ.Name() if name == "" { return &UnsupportedTypeError{typ} } start.Name.Local = name } // defaultNS records the default name space as set by a xmlns="..." // attribute. We don't set p.defaultNS because we want to let // the attribute writing code (in p.defineNS) be solely responsible // for maintaining that. defaultNS := p.defaultNS // Attributes for i := range tinfo.fields { finfo := &tinfo.fields[i] if finfo.flags&fAttr == 0 { continue } attr, err := p.fieldAttr(finfo, val) if err != nil { return err } if attr.Name.Local == "" { continue } start.Attr = append(start.Attr, attr) if attr.Name.Space == "" && attr.Name.Local == "xmlns" { defaultNS = attr.Value } } if !explicitNS { // Historic behavior: elements use the default name space // they are contained in by default. start.Name.Space = defaultNS } // Historic behaviour: an element that's in a namespace sets // the default namespace for all elements contained within it. start.setDefaultNamespace() if err := p.writeStart(&start); err != nil { return err } if val.Kind() == reflect.Struct { err = p.marshalStruct(tinfo, val) } else { s, b, err1 := p.marshalSimple(typ, val) if err1 != nil { err = err1 } else if b != nil { EscapeText(p, b) } else { p.EscapeString(s) } } if err != nil { return err } if err := p.writeEnd(start.Name); err != nil { return err } return p.cachedWriteError() } // fieldAttr returns the attribute of the given field. // If the returned attribute has an empty Name.Local, // it should not be used. // The given value holds the value containing the field. func (p *printer) fieldAttr(finfo *fieldInfo, val reflect.Value) (Attr, error) { fv := finfo.value(val) name := Name{Space: finfo.xmlns, Local: finfo.name} if finfo.flags&fOmitEmpty != 0 && isEmptyValue(fv) { return Attr{}, nil } if fv.Kind() == reflect.Interface && fv.IsNil() { return Attr{}, nil } if fv.CanInterface() && fv.Type().Implements(marshalerAttrType) { attr, err := fv.Interface().(MarshalerAttr).MarshalXMLAttr(name) return attr, err } if fv.CanAddr() { pv := fv.Addr() if pv.CanInterface() && pv.Type().Implements(marshalerAttrType) { attr, err := pv.Interface().(MarshalerAttr).MarshalXMLAttr(name) return attr, err } } if fv.CanInterface() && fv.Type().Implements(textMarshalerType) { text, err := fv.Interface().(encoding.TextMarshaler).MarshalText() if err != nil { return Attr{}, err } return Attr{name, string(text)}, nil } if fv.CanAddr() { pv := fv.Addr() if pv.CanInterface() && pv.Type().Implements(textMarshalerType) { text, err := pv.Interface().(encoding.TextMarshaler).MarshalText() if err != nil { return Attr{}, err } return Attr{name, string(text)}, nil } } // Dereference or skip nil pointer, interface values. switch fv.Kind() { case reflect.Ptr, reflect.Interface: if fv.IsNil() { return Attr{}, nil } fv = fv.Elem() } s, b, err := p.marshalSimple(fv.Type(), fv) if err != nil { return Attr{}, err } if b != nil { s = string(b) } return Attr{name, s}, nil } // defaultStart returns the default start element to use, // given the reflect type, field info, and start template. func (p *printer) defaultStart(typ reflect.Type, finfo *fieldInfo, startTemplate *StartElement) StartElement { var start StartElement // Precedence for the XML element name is as above, // except that we do not look inside structs for the first field. if startTemplate != nil { start.Name = startTemplate.Name start.Attr = append(start.Attr, startTemplate.Attr...) } else if finfo != nil && finfo.name != "" { start.Name.Local = finfo.name start.Name.Space = finfo.xmlns } else if typ.Name() != "" { start.Name.Local = typ.Name() } else { // Must be a pointer to a named type, // since it has the Marshaler methods. start.Name.Local = typ.Elem().Name() } // Historic behaviour: elements use the name space of // the element they are contained in by default. if start.Name.Space == "" { start.Name.Space = p.defaultNS } start.setDefaultNamespace() return start } // marshalInterface marshals a Marshaler interface value. func (p *printer) marshalInterface(val Marshaler, start StartElement) error { // Push a marker onto the tag stack so that MarshalXML // cannot close the XML tags that it did not open. p.tags = append(p.tags, Name{}) n := len(p.tags) err := val.MarshalXML(p.encoder, start) if err != nil { return err } // Make sure MarshalXML closed all its tags. p.tags[n-1] is the mark. if len(p.tags) > n { return fmt.Errorf("xml: %s.MarshalXML wrote invalid XML: <%s> not closed", receiverType(val), p.tags[len(p.tags)-1].Local) } p.tags = p.tags[:n-1] return nil } // marshalTextInterface marshals a TextMarshaler interface value. func (p *printer) marshalTextInterface(val encoding.TextMarshaler, start StartElement) error { if err := p.writeStart(&start); err != nil { return err } text, err := val.MarshalText() if err != nil { return err } EscapeText(p, text) return p.writeEnd(start.Name) } // writeStart writes the given start element. func (p *printer) writeStart(start *StartElement) error { if start.Name.Local == "" { return fmt.Errorf("xml: start tag with no name") } p.tags = append(p.tags, start.Name) p.markPrefix() // Define any name spaces explicitly declared in the attributes. // We do this as a separate pass so that explicitly declared prefixes // will take precedence over implicitly declared prefixes // regardless of the order of the attributes. ignoreNonEmptyDefault := start.Name.Space == "" for _, attr := range start.Attr { if err := p.defineNS(attr, ignoreNonEmptyDefault); err != nil { return err } } // Define any new name spaces implied by the attributes. for _, attr := range start.Attr { name := attr.Name // From http://www.w3.org/TR/xml-names11/#defaulting // "Default namespace declarations do not apply directly // to attribute names; the interpretation of unprefixed // attributes is determined by the element on which they // appear." // This means we don't need to create a new namespace // when an attribute name space is empty. if name.Space != "" && !name.isNamespace() { p.createNSPrefix(name.Space, true) } } p.createNSPrefix(start.Name.Space, false) p.writeIndent(1) p.WriteByte('<') p.writeName(start.Name, false) p.writeNamespaces() for _, attr := range start.Attr { name := attr.Name if name.Local == "" || name.isNamespace() { // Namespaces have already been written by writeNamespaces above. continue } p.WriteByte(' ') p.writeName(name, true) p.WriteString(`="`) p.EscapeString(attr.Value) p.WriteByte('"') } p.WriteByte('>') return nil } // writeName writes the given name. It assumes // that p.createNSPrefix(name) has already been called. func (p *printer) writeName(name Name, isAttr bool) { if prefix := p.prefixForNS(name.Space, isAttr); prefix != "" { p.WriteString(prefix) p.WriteByte(':') } p.WriteString(name.Local) } func (p *printer) writeEnd(name Name) error { if name.Local == "" { return fmt.Errorf("xml: end tag with no name") } if len(p.tags) == 0 || p.tags[len(p.tags)-1].Local == "" { return fmt.Errorf("xml: end tag without start tag", name.Local) } if top := p.tags[len(p.tags)-1]; top != name { if top.Local != name.Local { return fmt.Errorf("xml: end tag does not match start tag <%s>", name.Local, top.Local) } return fmt.Errorf("xml: end tag in namespace %s does not match start tag <%s> in namespace %s", name.Local, name.Space, top.Local, top.Space) } p.tags = p.tags[:len(p.tags)-1] p.writeIndent(-1) p.WriteByte('<') p.WriteByte('/') p.writeName(name, false) p.WriteByte('>') p.popPrefix() return nil } func (p *printer) marshalSimple(typ reflect.Type, val reflect.Value) (string, []byte, error) { switch val.Kind() { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return strconv.FormatInt(val.Int(), 10), nil, nil case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: return strconv.FormatUint(val.Uint(), 10), nil, nil case reflect.Float32, reflect.Float64: return strconv.FormatFloat(val.Float(), 'g', -1, val.Type().Bits()), nil, nil case reflect.String: return val.String(), nil, nil case reflect.Bool: return strconv.FormatBool(val.Bool()), nil, nil case reflect.Array: if typ.Elem().Kind() != reflect.Uint8 { break } // [...]byte var bytes []byte if val.CanAddr() { bytes = val.Slice(0, val.Len()).Bytes() } else { bytes = make([]byte, val.Len()) reflect.Copy(reflect.ValueOf(bytes), val) } return "", bytes, nil case reflect.Slice: if typ.Elem().Kind() != reflect.Uint8 { break } // []byte return "", val.Bytes(), nil } return "", nil, &UnsupportedTypeError{typ} } var ddBytes = []byte("--") func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { s := parentStack{p: p} for i := range tinfo.fields { finfo := &tinfo.fields[i] if finfo.flags&fAttr != 0 { continue } vf := finfo.value(val) // Dereference or skip nil pointer, interface values. switch vf.Kind() { case reflect.Ptr, reflect.Interface: if !vf.IsNil() { vf = vf.Elem() } } switch finfo.flags & fMode { case fCharData: if err := s.setParents(&noField, reflect.Value{}); err != nil { return err } if vf.CanInterface() && vf.Type().Implements(textMarshalerType) { data, err := vf.Interface().(encoding.TextMarshaler).MarshalText() if err != nil { return err } Escape(p, data) continue } if vf.CanAddr() { pv := vf.Addr() if pv.CanInterface() && pv.Type().Implements(textMarshalerType) { data, err := pv.Interface().(encoding.TextMarshaler).MarshalText() if err != nil { return err } Escape(p, data) continue } } var scratch [64]byte switch vf.Kind() { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: Escape(p, strconv.AppendInt(scratch[:0], vf.Int(), 10)) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: Escape(p, strconv.AppendUint(scratch[:0], vf.Uint(), 10)) case reflect.Float32, reflect.Float64: Escape(p, strconv.AppendFloat(scratch[:0], vf.Float(), 'g', -1, vf.Type().Bits())) case reflect.Bool: Escape(p, strconv.AppendBool(scratch[:0], vf.Bool())) case reflect.String: if err := EscapeText(p, []byte(vf.String())); err != nil { return err } case reflect.Slice: if elem, ok := vf.Interface().([]byte); ok { if err := EscapeText(p, elem); err != nil { return err } } } continue case fComment: if err := s.setParents(&noField, reflect.Value{}); err != nil { return err } k := vf.Kind() if !(k == reflect.String || k == reflect.Slice && vf.Type().Elem().Kind() == reflect.Uint8) { return fmt.Errorf("xml: bad type for comment field of %s", val.Type()) } if vf.Len() == 0 { continue } p.writeIndent(0) p.WriteString("" is invalid grammar. Make it "- -->" p.WriteByte(' ') } p.WriteString("-->") continue case fInnerXml: iface := vf.Interface() switch raw := iface.(type) { case []byte: p.Write(raw) continue case string: p.WriteString(raw) continue } case fElement, fElement | fAny: if err := s.setParents(finfo, vf); err != nil { return err } } if err := p.marshalValue(vf, finfo, nil); err != nil { return err } } if err := s.setParents(&noField, reflect.Value{}); err != nil { return err } return p.cachedWriteError() } var noField fieldInfo // return the bufio Writer's cached write error func (p *printer) cachedWriteError() error { _, err := p.Write(nil) return err } func (p *printer) writeIndent(depthDelta int) { if len(p.prefix) == 0 && len(p.indent) == 0 { return } if depthDelta < 0 { p.depth-- if p.indentedIn { p.indentedIn = false return } p.indentedIn = false } if p.putNewline { p.WriteByte('\n') } else { p.putNewline = true } if len(p.prefix) > 0 { p.WriteString(p.prefix) } if len(p.indent) > 0 { for i := 0; i < p.depth; i++ { p.WriteString(p.indent) } } if depthDelta > 0 { p.depth++ p.indentedIn = true } } type parentStack struct { p *printer xmlns string parents []string } // setParents sets the stack of current parents to those found in finfo. // It only writes the start elements if vf holds a non-nil value. // If finfo is &noField, it pops all elements. func (s *parentStack) setParents(finfo *fieldInfo, vf reflect.Value) error { xmlns := s.p.defaultNS if finfo.xmlns != "" { xmlns = finfo.xmlns } commonParents := 0 if xmlns == s.xmlns { for ; commonParents < len(finfo.parents) && commonParents < len(s.parents); commonParents++ { if finfo.parents[commonParents] != s.parents[commonParents] { break } } } // Pop off any parents that aren't in common with the previous field. for i := len(s.parents) - 1; i >= commonParents; i-- { if err := s.p.writeEnd(Name{ Space: s.xmlns, Local: s.parents[i], }); err != nil { return err } } s.parents = finfo.parents s.xmlns = xmlns if commonParents >= len(s.parents) { // No new elements to push. return nil } if (vf.Kind() == reflect.Ptr || vf.Kind() == reflect.Interface) && vf.IsNil() { // The element is nil, so no need for the start elements. s.parents = s.parents[:commonParents] return nil } // Push any new parents required. for _, name := range s.parents[commonParents:] { start := &StartElement{ Name: Name{ Space: s.xmlns, Local: name, }, } // Set the default name space for parent elements // to match what we do with other elements. if s.xmlns != s.p.defaultNS { start.setDefaultNamespace() } if err := s.p.writeStart(start); err != nil { return err } } return nil } // A MarshalXMLError is returned when Marshal encounters a type // that cannot be converted into XML. type UnsupportedTypeError struct { Type reflect.Type } func (e *UnsupportedTypeError) Error() string { return "xml: unsupported type: " + e.Type.String() } func isEmptyValue(v reflect.Value) bool { switch v.Kind() { case reflect.Array, reflect.Map, reflect.Slice, reflect.String: return v.Len() == 0 case reflect.Bool: return !v.Bool() case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return v.Int() == 0 case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: return v.Uint() == 0 case reflect.Float32, reflect.Float64: return v.Float() == 0 case reflect.Interface, reflect.Ptr: return v.IsNil() } return false } ================================================ FILE: server/webdav/internal/xml/marshal_test.go ================================================ // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package xml import ( "bytes" "errors" "fmt" "io" "reflect" "strconv" "strings" "sync" "testing" "time" ) type DriveType int const ( HyperDrive DriveType = iota ImprobabilityDrive ) type Passenger struct { Name []string `xml:"name"` Weight float32 `xml:"weight"` } type Ship struct { XMLName struct{} `xml:"spaceship"` Name string `xml:"name,attr"` Pilot string `xml:"pilot,attr"` Drive DriveType `xml:"drive"` Age uint `xml:"age"` Passenger []*Passenger `xml:"passenger"` secret string } type NamedType string type Port struct { XMLName struct{} `xml:"port"` Type string `xml:"type,attr,omitempty"` Comment string `xml:",comment"` Number string `xml:",chardata"` } type Domain struct { XMLName struct{} `xml:"domain"` Country string `xml:",attr,omitempty"` Name []byte `xml:",chardata"` Comment []byte `xml:",comment"` } type Book struct { XMLName struct{} `xml:"book"` Title string `xml:",chardata"` } type Event struct { XMLName struct{} `xml:"event"` Year int `xml:",chardata"` } type Movie struct { XMLName struct{} `xml:"movie"` Length uint `xml:",chardata"` } type Pi struct { XMLName struct{} `xml:"pi"` Approximation float32 `xml:",chardata"` } type Universe struct { XMLName struct{} `xml:"universe"` Visible float64 `xml:",chardata"` } type Particle struct { XMLName struct{} `xml:"particle"` HasMass bool `xml:",chardata"` } type Departure struct { XMLName struct{} `xml:"departure"` When time.Time `xml:",chardata"` } type SecretAgent struct { XMLName struct{} `xml:"agent"` Handle string `xml:"handle,attr"` Identity string Obfuscate string `xml:",innerxml"` } type NestedItems struct { XMLName struct{} `xml:"result"` Items []string `xml:">item"` Item1 []string `xml:"Items>item1"` } type NestedOrder struct { XMLName struct{} `xml:"result"` Field1 string `xml:"parent>c"` Field2 string `xml:"parent>b"` Field3 string `xml:"parent>a"` } type MixedNested struct { XMLName struct{} `xml:"result"` A string `xml:"parent1>a"` B string `xml:"b"` C string `xml:"parent1>parent2>c"` D string `xml:"parent1>d"` } type NilTest struct { A interface{} `xml:"parent1>parent2>a"` B interface{} `xml:"parent1>b"` C interface{} `xml:"parent1>parent2>c"` } type Service struct { XMLName struct{} `xml:"service"` Domain *Domain `xml:"host>domain"` Port *Port `xml:"host>port"` Extra1 interface{} Extra2 interface{} `xml:"host>extra2"` } var nilStruct *Ship type EmbedA struct { EmbedC EmbedB EmbedB FieldA string } type EmbedB struct { FieldB string *EmbedC } type EmbedC struct { FieldA1 string `xml:"FieldA>A1"` FieldA2 string `xml:"FieldA>A2"` FieldB string FieldC string } type NameCasing struct { XMLName struct{} `xml:"casing"` Xy string XY string XyA string `xml:"Xy,attr"` XYA string `xml:"XY,attr"` } type NamePrecedence struct { XMLName Name `xml:"Parent"` FromTag XMLNameWithoutTag `xml:"InTag"` FromNameVal XMLNameWithoutTag FromNameTag XMLNameWithTag InFieldName string } type XMLNameWithTag struct { XMLName Name `xml:"InXMLNameTag"` Value string `xml:",chardata"` } type XMLNameWithNSTag struct { XMLName Name `xml:"ns InXMLNameWithNSTag"` Value string `xml:",chardata"` } type XMLNameWithoutTag struct { XMLName Name Value string `xml:",chardata"` } type NameInField struct { Foo Name `xml:"ns foo"` } type AttrTest struct { Int int `xml:",attr"` Named int `xml:"int,attr"` Float float64 `xml:",attr"` Uint8 uint8 `xml:",attr"` Bool bool `xml:",attr"` Str string `xml:",attr"` Bytes []byte `xml:",attr"` } type OmitAttrTest struct { Int int `xml:",attr,omitempty"` Named int `xml:"int,attr,omitempty"` Float float64 `xml:",attr,omitempty"` Uint8 uint8 `xml:",attr,omitempty"` Bool bool `xml:",attr,omitempty"` Str string `xml:",attr,omitempty"` Bytes []byte `xml:",attr,omitempty"` } type OmitFieldTest struct { Int int `xml:",omitempty"` Named int `xml:"int,omitempty"` Float float64 `xml:",omitempty"` Uint8 uint8 `xml:",omitempty"` Bool bool `xml:",omitempty"` Str string `xml:",omitempty"` Bytes []byte `xml:",omitempty"` Ptr *PresenceTest `xml:",omitempty"` } type AnyTest struct { XMLName struct{} `xml:"a"` Nested string `xml:"nested>value"` AnyField AnyHolder `xml:",any"` } type AnyOmitTest struct { XMLName struct{} `xml:"a"` Nested string `xml:"nested>value"` AnyField *AnyHolder `xml:",any,omitempty"` } type AnySliceTest struct { XMLName struct{} `xml:"a"` Nested string `xml:"nested>value"` AnyField []AnyHolder `xml:",any"` } type AnyHolder struct { XMLName Name XML string `xml:",innerxml"` } type RecurseA struct { A string B *RecurseB } type RecurseB struct { A *RecurseA B string } type PresenceTest struct { Exists *struct{} } type IgnoreTest struct { PublicSecret string `xml:"-"` } type MyBytes []byte type Data struct { Bytes []byte Attr []byte `xml:",attr"` Custom MyBytes } type Plain struct { V interface{} } type MyInt int type EmbedInt struct { MyInt } type Strings struct { X []string `xml:"A>B,omitempty"` } type PointerFieldsTest struct { XMLName Name `xml:"dummy"` Name *string `xml:"name,attr"` Age *uint `xml:"age,attr"` Empty *string `xml:"empty,attr"` Contents *string `xml:",chardata"` } type ChardataEmptyTest struct { XMLName Name `xml:"test"` Contents *string `xml:",chardata"` } type MyMarshalerTest struct { } var _ Marshaler = (*MyMarshalerTest)(nil) func (m *MyMarshalerTest) MarshalXML(e *Encoder, start StartElement) error { e.EncodeToken(start) e.EncodeToken(CharData([]byte("hello world"))) e.EncodeToken(EndElement{start.Name}) return nil } type MyMarshalerAttrTest struct{} var _ MarshalerAttr = (*MyMarshalerAttrTest)(nil) func (m *MyMarshalerAttrTest) MarshalXMLAttr(name Name) (Attr, error) { return Attr{name, "hello world"}, nil } type MyMarshalerValueAttrTest struct{} var _ MarshalerAttr = MyMarshalerValueAttrTest{} func (m MyMarshalerValueAttrTest) MarshalXMLAttr(name Name) (Attr, error) { return Attr{name, "hello world"}, nil } type MarshalerStruct struct { Foo MyMarshalerAttrTest `xml:",attr"` } type MarshalerValueStruct struct { Foo MyMarshalerValueAttrTest `xml:",attr"` } type InnerStruct struct { XMLName Name `xml:"testns outer"` } type OuterStruct struct { InnerStruct IntAttr int `xml:"int,attr"` } type OuterNamedStruct struct { InnerStruct XMLName Name `xml:"outerns test"` IntAttr int `xml:"int,attr"` } type OuterNamedOrderedStruct struct { XMLName Name `xml:"outerns test"` InnerStruct IntAttr int `xml:"int,attr"` } type OuterOuterStruct struct { OuterStruct } type NestedAndChardata struct { AB []string `xml:"A>B"` Chardata string `xml:",chardata"` } type NestedAndComment struct { AB []string `xml:"A>B"` Comment string `xml:",comment"` } type XMLNSFieldStruct struct { Ns string `xml:"xmlns,attr"` Body string } type NamedXMLNSFieldStruct struct { XMLName struct{} `xml:"testns test"` Ns string `xml:"xmlns,attr"` Body string } type XMLNSFieldStructWithOmitEmpty struct { Ns string `xml:"xmlns,attr,omitempty"` Body string } type NamedXMLNSFieldStructWithEmptyNamespace struct { XMLName struct{} `xml:"test"` Ns string `xml:"xmlns,attr"` Body string } type RecursiveXMLNSFieldStruct struct { Ns string `xml:"xmlns,attr"` Body *RecursiveXMLNSFieldStruct `xml:",omitempty"` Text string `xml:",omitempty"` } func ifaceptr(x interface{}) interface{} { return &x } var ( nameAttr = "Sarah" ageAttr = uint(12) contentsAttr = "lorem ipsum" ) // Unless explicitly stated as such (or *Plain), all of the // tests below are two-way tests. When introducing new tests, // please try to make them two-way as well to ensure that // marshalling and unmarshalling are as symmetrical as feasible. var marshalTests = []struct { Value interface{} ExpectXML string MarshalOnly bool UnmarshalOnly bool }{ // Test nil marshals to nothing {Value: nil, ExpectXML: ``, MarshalOnly: true}, {Value: nilStruct, ExpectXML: ``, MarshalOnly: true}, // Test value types {Value: &Plain{true}, ExpectXML: `true`}, {Value: &Plain{false}, ExpectXML: `false`}, {Value: &Plain{int(42)}, ExpectXML: `42`}, {Value: &Plain{int8(42)}, ExpectXML: `42`}, {Value: &Plain{int16(42)}, ExpectXML: `42`}, {Value: &Plain{int32(42)}, ExpectXML: `42`}, {Value: &Plain{uint(42)}, ExpectXML: `42`}, {Value: &Plain{uint8(42)}, ExpectXML: `42`}, {Value: &Plain{uint16(42)}, ExpectXML: `42`}, {Value: &Plain{uint32(42)}, ExpectXML: `42`}, {Value: &Plain{float32(1.25)}, ExpectXML: `1.25`}, {Value: &Plain{float64(1.25)}, ExpectXML: `1.25`}, {Value: &Plain{uintptr(0xFFDD)}, ExpectXML: `65501`}, {Value: &Plain{"gopher"}, ExpectXML: `gopher`}, {Value: &Plain{[]byte("gopher")}, ExpectXML: `gopher`}, {Value: &Plain{""}, ExpectXML: `</>`}, {Value: &Plain{[]byte("")}, ExpectXML: `</>`}, {Value: &Plain{[3]byte{'<', '/', '>'}}, ExpectXML: `</>`}, {Value: &Plain{NamedType("potato")}, ExpectXML: `potato`}, {Value: &Plain{[]int{1, 2, 3}}, ExpectXML: `123`}, {Value: &Plain{[3]int{1, 2, 3}}, ExpectXML: `123`}, {Value: ifaceptr(true), MarshalOnly: true, ExpectXML: `true`}, // Test time. { Value: &Plain{time.Unix(1e9, 123456789).UTC()}, ExpectXML: `2001-09-09T01:46:40.123456789Z`, }, // A pointer to struct{} may be used to test for an element's presence. { Value: &PresenceTest{new(struct{})}, ExpectXML: ``, }, { Value: &PresenceTest{}, ExpectXML: ``, }, // A pointer to struct{} may be used to test for an element's presence. { Value: &PresenceTest{new(struct{})}, ExpectXML: ``, }, { Value: &PresenceTest{}, ExpectXML: ``, }, // A []byte field is only nil if the element was not found. { Value: &Data{}, ExpectXML: ``, UnmarshalOnly: true, }, { Value: &Data{Bytes: []byte{}, Custom: MyBytes{}, Attr: []byte{}}, ExpectXML: ``, UnmarshalOnly: true, }, // Check that []byte works, including named []byte types. { Value: &Data{Bytes: []byte("ab"), Custom: MyBytes("cd"), Attr: []byte{'v'}}, ExpectXML: `abcd`, }, // Test innerxml { Value: &SecretAgent{ Handle: "007", Identity: "James Bond", Obfuscate: "", }, ExpectXML: `James Bond`, MarshalOnly: true, }, { Value: &SecretAgent{ Handle: "007", Identity: "James Bond", Obfuscate: "James Bond", }, ExpectXML: `James Bond`, UnmarshalOnly: true, }, // Test structs {Value: &Port{Type: "ssl", Number: "443"}, ExpectXML: `443`}, {Value: &Port{Number: "443"}, ExpectXML: `443`}, {Value: &Port{Type: ""}, ExpectXML: ``}, {Value: &Port{Number: "443", Comment: "https"}, ExpectXML: `443`}, {Value: &Port{Number: "443", Comment: "add space-"}, ExpectXML: `443`, MarshalOnly: true}, {Value: &Domain{Name: []byte("google.com&friends")}, ExpectXML: `google.com&friends`}, {Value: &Domain{Name: []byte("google.com"), Comment: []byte(" &friends ")}, ExpectXML: `google.com`}, {Value: &Book{Title: "Pride & Prejudice"}, ExpectXML: `Pride & Prejudice`}, {Value: &Event{Year: -3114}, ExpectXML: `-3114`}, {Value: &Movie{Length: 13440}, ExpectXML: `13440`}, {Value: &Pi{Approximation: 3.14159265}, ExpectXML: `3.1415927`}, {Value: &Universe{Visible: 9.3e13}, ExpectXML: `9.3e+13`}, {Value: &Particle{HasMass: true}, ExpectXML: `true`}, {Value: &Departure{When: ParseTime("2013-01-09T00:15:00-09:00")}, ExpectXML: `2013-01-09T00:15:00-09:00`}, {Value: atomValue, ExpectXML: atomXml}, { Value: &Ship{ Name: "Heart of Gold", Pilot: "Computer", Age: 1, Drive: ImprobabilityDrive, Passenger: []*Passenger{ { Name: []string{"Zaphod", "Beeblebrox"}, Weight: 7.25, }, { Name: []string{"Trisha", "McMillen"}, Weight: 5.5, }, { Name: []string{"Ford", "Prefect"}, Weight: 7, }, { Name: []string{"Arthur", "Dent"}, Weight: 6.75, }, }, }, ExpectXML: `` + `` + strconv.Itoa(int(ImprobabilityDrive)) + `` + `1` + `` + `Zaphod` + `Beeblebrox` + `7.25` + `` + `` + `Trisha` + `McMillen` + `5.5` + `` + `` + `Ford` + `Prefect` + `7` + `` + `` + `Arthur` + `Dent` + `6.75` + `` + ``, }, // Test a>b { Value: &NestedItems{Items: nil, Item1: nil}, ExpectXML: `` + `` + `` + ``, }, { Value: &NestedItems{Items: []string{}, Item1: []string{}}, ExpectXML: `` + `` + `` + ``, MarshalOnly: true, }, { Value: &NestedItems{Items: nil, Item1: []string{"A"}}, ExpectXML: `` + `` + `A` + `` + ``, }, { Value: &NestedItems{Items: []string{"A", "B"}, Item1: nil}, ExpectXML: `` + `` + `A` + `B` + `` + ``, }, { Value: &NestedItems{Items: []string{"A", "B"}, Item1: []string{"C"}}, ExpectXML: `` + `` + `A` + `B` + `C` + `` + ``, }, { Value: &NestedOrder{Field1: "C", Field2: "B", Field3: "A"}, ExpectXML: `` + `` + `C` + `B` + `A` + `` + ``, }, { Value: &NilTest{A: "A", B: nil, C: "C"}, ExpectXML: `` + `` + `A` + `C` + `` + ``, MarshalOnly: true, // Uses interface{} }, { Value: &MixedNested{A: "A", B: "B", C: "C", D: "D"}, ExpectXML: `` + `A` + `B` + `` + `C` + `D` + `` + ``, }, { Value: &Service{Port: &Port{Number: "80"}}, ExpectXML: `80`, }, { Value: &Service{}, ExpectXML: ``, }, { Value: &Service{Port: &Port{Number: "80"}, Extra1: "A", Extra2: "B"}, ExpectXML: `` + `80` + `A` + `B` + ``, MarshalOnly: true, }, { Value: &Service{Port: &Port{Number: "80"}, Extra2: "example"}, ExpectXML: `` + `80` + `example` + ``, MarshalOnly: true, }, { Value: &struct { XMLName struct{} `xml:"space top"` A string `xml:"x>a"` B string `xml:"x>b"` C string `xml:"space x>c"` C1 string `xml:"space1 x>c"` D1 string `xml:"space1 x>d"` E1 string `xml:"x>e"` }{ A: "a", B: "b", C: "c", C1: "c1", D1: "d1", E1: "e1", }, ExpectXML: `` + `abc` + `` + `c1` + `d1` + `` + `` + `e1` + `` + ``, }, { Value: &struct { XMLName Name A string `xml:"x>a"` B string `xml:"x>b"` C string `xml:"space x>c"` C1 string `xml:"space1 x>c"` D1 string `xml:"space1 x>d"` }{ XMLName: Name{ Space: "space0", Local: "top", }, A: "a", B: "b", C: "c", C1: "c1", D1: "d1", }, ExpectXML: `` + `ab` + `c` + `` + `c1` + `d1` + `` + ``, }, { Value: &struct { XMLName struct{} `xml:"top"` B string `xml:"space x>b"` B1 string `xml:"space1 x>b"` }{ B: "b", B1: "b1", }, ExpectXML: `` + `b` + `b1` + ``, }, // Test struct embedding { Value: &EmbedA{ EmbedC: EmbedC{ FieldA1: "", // Shadowed by A.A FieldA2: "", // Shadowed by A.A FieldB: "A.C.B", FieldC: "A.C.C", }, EmbedB: EmbedB{ FieldB: "A.B.B", EmbedC: &EmbedC{ FieldA1: "A.B.C.A1", FieldA2: "A.B.C.A2", FieldB: "", // Shadowed by A.B.B FieldC: "A.B.C.C", }, }, FieldA: "A.A", }, ExpectXML: `` + `A.C.B` + `A.C.C` + `` + `A.B.B` + `` + `A.B.C.A1` + `A.B.C.A2` + `` + `A.B.C.C` + `` + `A.A` + ``, }, // Test that name casing matters { Value: &NameCasing{Xy: "mixed", XY: "upper", XyA: "mixedA", XYA: "upperA"}, ExpectXML: `mixedupper`, }, // Test the order in which the XML element name is chosen { Value: &NamePrecedence{ FromTag: XMLNameWithoutTag{Value: "A"}, FromNameVal: XMLNameWithoutTag{XMLName: Name{Local: "InXMLName"}, Value: "B"}, FromNameTag: XMLNameWithTag{Value: "C"}, InFieldName: "D", }, ExpectXML: `` + `A` + `B` + `C` + `D` + ``, MarshalOnly: true, }, { Value: &NamePrecedence{ XMLName: Name{Local: "Parent"}, FromTag: XMLNameWithoutTag{XMLName: Name{Local: "InTag"}, Value: "A"}, FromNameVal: XMLNameWithoutTag{XMLName: Name{Local: "FromNameVal"}, Value: "B"}, FromNameTag: XMLNameWithTag{XMLName: Name{Local: "InXMLNameTag"}, Value: "C"}, InFieldName: "D", }, ExpectXML: `` + `A` + `B` + `C` + `D` + ``, UnmarshalOnly: true, }, // xml.Name works in a plain field as well. { Value: &NameInField{Name{Space: "ns", Local: "foo"}}, ExpectXML: ``, }, { Value: &NameInField{Name{Space: "ns", Local: "foo"}}, ExpectXML: ``, UnmarshalOnly: true, }, // Marshaling zero xml.Name uses the tag or field name. { Value: &NameInField{}, ExpectXML: ``, MarshalOnly: true, }, // Test attributes { Value: &AttrTest{ Int: 8, Named: 9, Float: 23.5, Uint8: 255, Bool: true, Str: "str", Bytes: []byte("byt"), }, ExpectXML: ``, }, { Value: &AttrTest{Bytes: []byte{}}, ExpectXML: ``, }, { Value: &OmitAttrTest{ Int: 8, Named: 9, Float: 23.5, Uint8: 255, Bool: true, Str: "str", Bytes: []byte("byt"), }, ExpectXML: ``, }, { Value: &OmitAttrTest{}, ExpectXML: ``, }, // pointer fields { Value: &PointerFieldsTest{Name: &nameAttr, Age: &ageAttr, Contents: &contentsAttr}, ExpectXML: `lorem ipsum`, MarshalOnly: true, }, // empty chardata pointer field { Value: &ChardataEmptyTest{}, ExpectXML: ``, MarshalOnly: true, }, // omitempty on fields { Value: &OmitFieldTest{ Int: 8, Named: 9, Float: 23.5, Uint8: 255, Bool: true, Str: "str", Bytes: []byte("byt"), Ptr: &PresenceTest{}, }, ExpectXML: `` + `8` + `9` + `23.5` + `255` + `true` + `str` + `byt` + `` + ``, }, { Value: &OmitFieldTest{}, ExpectXML: ``, }, // Test ",any" { ExpectXML: `knownunknown`, Value: &AnyTest{ Nested: "known", AnyField: AnyHolder{ XMLName: Name{Local: "other"}, XML: "unknown", }, }, }, { Value: &AnyTest{Nested: "known", AnyField: AnyHolder{ XML: "", XMLName: Name{Local: "AnyField"}, }, }, ExpectXML: `known`, }, { ExpectXML: `b`, Value: &AnyOmitTest{ Nested: "b", }, }, { ExpectXML: `bei`, Value: &AnySliceTest{ Nested: "b", AnyField: []AnyHolder{ { XMLName: Name{Local: "c"}, XML: "e", }, { XMLName: Name{Space: "f", Local: "g"}, XML: "i", }, }, }, }, { ExpectXML: `b`, Value: &AnySliceTest{ Nested: "b", }, }, // Test recursive types. { Value: &RecurseA{ A: "a1", B: &RecurseB{ A: &RecurseA{"a2", nil}, B: "b1", }, }, ExpectXML: `a1a2b1`, }, // Test ignoring fields via "-" tag { ExpectXML: ``, Value: &IgnoreTest{}, }, { ExpectXML: ``, Value: &IgnoreTest{PublicSecret: "can't tell"}, MarshalOnly: true, }, { ExpectXML: `ignore me`, Value: &IgnoreTest{}, UnmarshalOnly: true, }, // Test escaping. { ExpectXML: `dquote: "; squote: '; ampersand: &; less: <; greater: >;`, Value: &AnyTest{ Nested: `dquote: "; squote: '; ampersand: &; less: <; greater: >;`, AnyField: AnyHolder{XMLName: Name{Local: "empty"}}, }, }, { ExpectXML: `newline: ; cr: ; tab: ;`, Value: &AnyTest{ Nested: "newline: \n; cr: \r; tab: \t;", AnyField: AnyHolder{XMLName: Name{Local: "AnyField"}}, }, }, { ExpectXML: "1\r2\r\n3\n\r4\n5", Value: &AnyTest{ Nested: "1\n2\n3\n\n4\n5", }, UnmarshalOnly: true, }, { ExpectXML: `42`, Value: &EmbedInt{ MyInt: 42, }, }, // Test omitempty with parent chain; see golang.org/issue/4168. { ExpectXML: ``, Value: &Strings{}, }, // Custom marshalers. { ExpectXML: `hello world`, Value: &MyMarshalerTest{}, }, { ExpectXML: ``, Value: &MarshalerStruct{}, }, { ExpectXML: ``, Value: &MarshalerValueStruct{}, }, { ExpectXML: ``, Value: &OuterStruct{IntAttr: 10}, }, { ExpectXML: ``, Value: &OuterNamedStruct{XMLName: Name{Space: "outerns", Local: "test"}, IntAttr: 10}, }, { ExpectXML: ``, Value: &OuterNamedOrderedStruct{XMLName: Name{Space: "outerns", Local: "test"}, IntAttr: 10}, }, { ExpectXML: ``, Value: &OuterOuterStruct{OuterStruct{IntAttr: 10}}, }, { ExpectXML: `test`, Value: &NestedAndChardata{AB: make([]string, 2), Chardata: "test"}, }, { ExpectXML: ``, Value: &NestedAndComment{AB: make([]string, 2), Comment: "test"}, }, { ExpectXML: `hello world`, Value: &XMLNSFieldStruct{Ns: "http://example.com/ns", Body: "hello world"}, }, { ExpectXML: `hello world`, Value: &NamedXMLNSFieldStruct{Ns: "http://example.com/ns", Body: "hello world"}, }, { ExpectXML: `hello world`, Value: &NamedXMLNSFieldStruct{Ns: "", Body: "hello world"}, }, { ExpectXML: `hello world`, Value: &XMLNSFieldStructWithOmitEmpty{Body: "hello world"}, }, { // The xmlns attribute must be ignored because the // element is in the empty namespace, so it's not possible // to set the default namespace to something non-empty. ExpectXML: `hello world`, Value: &NamedXMLNSFieldStructWithEmptyNamespace{Ns: "foo", Body: "hello world"}, MarshalOnly: true, }, { ExpectXML: `hello world`, Value: &RecursiveXMLNSFieldStruct{ Ns: "foo", Body: &RecursiveXMLNSFieldStruct{ Text: "hello world", }, }, }, } func TestMarshal(t *testing.T) { for idx, test := range marshalTests { if test.UnmarshalOnly { continue } data, err := Marshal(test.Value) if err != nil { t.Errorf("#%d: marshal(%#v): %s", idx, test.Value, err) continue } if got, want := string(data), test.ExpectXML; got != want { if strings.Contains(want, "\n") { t.Errorf("#%d: marshal(%#v):\nHAVE:\n%s\nWANT:\n%s", idx, test.Value, got, want) } else { t.Errorf("#%d: marshal(%#v):\nhave %#q\nwant %#q", idx, test.Value, got, want) } } } } type AttrParent struct { X string `xml:"X>Y,attr"` } type BadAttr struct { Name []string `xml:"name,attr"` } var marshalErrorTests = []struct { Value interface{} Err string Kind reflect.Kind }{ { Value: make(chan bool), Err: "xml: unsupported type: chan bool", Kind: reflect.Chan, }, { Value: map[string]string{ "question": "What do you get when you multiply six by nine?", "answer": "42", }, Err: "xml: unsupported type: map[string]string", Kind: reflect.Map, }, { Value: map[*Ship]bool{nil: false}, Err: "xml: unsupported type: map[*xml.Ship]bool", Kind: reflect.Map, }, { Value: &Domain{Comment: []byte("f--bar")}, Err: `xml: comments must not contain "--"`, }, // Reject parent chain with attr, never worked; see golang.org/issue/5033. { Value: &AttrParent{}, Err: `xml: X>Y chain not valid with attr flag`, }, { Value: BadAttr{[]string{"X", "Y"}}, Err: `xml: unsupported type: []string`, }, } var marshalIndentTests = []struct { Value interface{} Prefix string Indent string ExpectXML string }{ { Value: &SecretAgent{ Handle: "007", Identity: "James Bond", Obfuscate: "", }, Prefix: "", Indent: "\t", ExpectXML: fmt.Sprintf("\n\tJames Bond\n"), }, } func TestMarshalErrors(t *testing.T) { for idx, test := range marshalErrorTests { data, err := Marshal(test.Value) if err == nil { t.Errorf("#%d: marshal(%#v) = [success] %q, want error %v", idx, test.Value, data, test.Err) continue } if err.Error() != test.Err { t.Errorf("#%d: marshal(%#v) = [error] %v, want %v", idx, test.Value, err, test.Err) } if test.Kind != reflect.Invalid { if kind := err.(*UnsupportedTypeError).Type.Kind(); kind != test.Kind { t.Errorf("#%d: marshal(%#v) = [error kind] %s, want %s", idx, test.Value, kind, test.Kind) } } } } // Do invertibility testing on the various structures that we test func TestUnmarshal(t *testing.T) { for i, test := range marshalTests { if test.MarshalOnly { continue } if _, ok := test.Value.(*Plain); ok { continue } vt := reflect.TypeOf(test.Value) dest := reflect.New(vt.Elem()).Interface() err := Unmarshal([]byte(test.ExpectXML), dest) switch fix := dest.(type) { case *Feed: fix.Author.InnerXML = "" for i := range fix.Entry { fix.Entry[i].Author.InnerXML = "" } } if err != nil { t.Errorf("#%d: unexpected error: %#v", i, err) } else if got, want := dest, test.Value; !reflect.DeepEqual(got, want) { t.Errorf("#%d: unmarshal(%q):\nhave %#v\nwant %#v", i, test.ExpectXML, got, want) } } } func TestMarshalIndent(t *testing.T) { for i, test := range marshalIndentTests { data, err := MarshalIndent(test.Value, test.Prefix, test.Indent) if err != nil { t.Errorf("#%d: Error: %s", i, err) continue } if got, want := string(data), test.ExpectXML; got != want { t.Errorf("#%d: MarshalIndent:\nGot:%s\nWant:\n%s", i, got, want) } } } type limitedBytesWriter struct { w io.Writer remain int // until writes fail } func (lw *limitedBytesWriter) Write(p []byte) (n int, err error) { if lw.remain <= 0 { println("error") return 0, errors.New("write limit hit") } if len(p) > lw.remain { p = p[:lw.remain] n, _ = lw.w.Write(p) lw.remain = 0 return n, errors.New("write limit hit") } n, err = lw.w.Write(p) lw.remain -= n return n, err } func TestMarshalWriteErrors(t *testing.T) { var buf bytes.Buffer const writeCap = 1024 w := &limitedBytesWriter{&buf, writeCap} enc := NewEncoder(w) var err error var i int const n = 4000 for i = 1; i <= n; i++ { err = enc.Encode(&Passenger{ Name: []string{"Alice", "Bob"}, Weight: 5, }) if err != nil { break } } if err == nil { t.Error("expected an error") } if i == n { t.Errorf("expected to fail before the end") } if buf.Len() != writeCap { t.Errorf("buf.Len() = %d; want %d", buf.Len(), writeCap) } } func TestMarshalWriteIOErrors(t *testing.T) { enc := NewEncoder(errWriter{}) expectErr := "unwritable" err := enc.Encode(&Passenger{}) if err == nil || err.Error() != expectErr { t.Errorf("EscapeTest = [error] %v, want %v", err, expectErr) } } func TestMarshalFlush(t *testing.T) { var buf bytes.Buffer enc := NewEncoder(&buf) if err := enc.EncodeToken(CharData("hello world")); err != nil { t.Fatalf("enc.EncodeToken: %v", err) } if buf.Len() > 0 { t.Fatalf("enc.EncodeToken caused actual write: %q", buf.Bytes()) } if err := enc.Flush(); err != nil { t.Fatalf("enc.Flush: %v", err) } if buf.String() != "hello world" { t.Fatalf("after enc.Flush, buf.String() = %q, want %q", buf.String(), "hello world") } } var encodeElementTests = []struct { desc string value interface{} start StartElement expectXML string }{{ desc: "simple string", value: "hello", start: StartElement{ Name: Name{Local: "a"}, }, expectXML: `hello`, }, { desc: "string with added attributes", value: "hello", start: StartElement{ Name: Name{Local: "a"}, Attr: []Attr{{ Name: Name{Local: "x"}, Value: "y", }, { Name: Name{Local: "foo"}, Value: "bar", }}, }, expectXML: `hello`, }, { desc: "start element with default name space", value: struct { Foo XMLNameWithNSTag }{ Foo: XMLNameWithNSTag{ Value: "hello", }, }, start: StartElement{ Name: Name{Space: "ns", Local: "a"}, Attr: []Attr{{ Name: Name{Local: "xmlns"}, // "ns" is the name space defined in XMLNameWithNSTag Value: "ns", }}, }, expectXML: `hello`, }, { desc: "start element in name space with different default name space", value: struct { Foo XMLNameWithNSTag }{ Foo: XMLNameWithNSTag{ Value: "hello", }, }, start: StartElement{ Name: Name{Space: "ns2", Local: "a"}, Attr: []Attr{{ Name: Name{Local: "xmlns"}, // "ns" is the name space defined in XMLNameWithNSTag Value: "ns", }}, }, expectXML: `hello`, }, { desc: "XMLMarshaler with start element with default name space", value: &MyMarshalerTest{}, start: StartElement{ Name: Name{Space: "ns2", Local: "a"}, Attr: []Attr{{ Name: Name{Local: "xmlns"}, // "ns" is the name space defined in XMLNameWithNSTag Value: "ns", }}, }, expectXML: `hello world`, }} func TestEncodeElement(t *testing.T) { for idx, test := range encodeElementTests { var buf bytes.Buffer enc := NewEncoder(&buf) err := enc.EncodeElement(test.value, test.start) if err != nil { t.Fatalf("enc.EncodeElement: %v", err) } err = enc.Flush() if err != nil { t.Fatalf("enc.Flush: %v", err) } if got, want := buf.String(), test.expectXML; got != want { t.Errorf("#%d(%s): EncodeElement(%#v, %#v):\nhave %#q\nwant %#q", idx, test.desc, test.value, test.start, got, want) } } } func BenchmarkMarshal(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { Marshal(atomValue) } } func BenchmarkUnmarshal(b *testing.B) { b.ReportAllocs() xml := []byte(atomXml) for i := 0; i < b.N; i++ { Unmarshal(xml, &Feed{}) } } // golang.org/issue/6556 func TestStructPointerMarshal(t *testing.T) { type A struct { XMLName string `xml:"a"` B []interface{} } type C struct { XMLName Name Value string `xml:"value"` } a := new(A) a.B = append(a.B, &C{ XMLName: Name{Local: "c"}, Value: "x", }) b, err := Marshal(a) if err != nil { t.Fatal(err) } if x := string(b); x != "x" { t.Fatal(x) } var v A err = Unmarshal(b, &v) if err != nil { t.Fatal(err) } } var encodeTokenTests = []struct { desc string toks []Token want string err string }{{ desc: "start element with name space", toks: []Token{ StartElement{Name{"space", "local"}, nil}, }, want: ``, }, { desc: "start element with no name", toks: []Token{ StartElement{Name{"space", ""}, nil}, }, err: "xml: start tag with no name", }, { desc: "end element with no name", toks: []Token{ EndElement{Name{"space", ""}}, }, err: "xml: end tag with no name", }, { desc: "char data", toks: []Token{ CharData("foo"), }, want: `foo`, }, { desc: "char data with escaped chars", toks: []Token{ CharData(" \t\n"), }, want: " \n", }, { desc: "comment", toks: []Token{ Comment("foo"), }, want: ``, }, { desc: "comment with invalid content", toks: []Token{ Comment("foo-->"), }, err: "xml: EncodeToken of Comment containing --> marker", }, { desc: "proc instruction", toks: []Token{ ProcInst{"Target", []byte("Instruction")}, }, want: ``, }, { desc: "proc instruction with empty target", toks: []Token{ ProcInst{"", []byte("Instruction")}, }, err: "xml: EncodeToken of ProcInst with invalid Target", }, { desc: "proc instruction with bad content", toks: []Token{ ProcInst{"", []byte("Instruction?>")}, }, err: "xml: EncodeToken of ProcInst with invalid Target", }, { desc: "directive", toks: []Token{ Directive("foo"), }, want: ``, }, { desc: "more complex directive", toks: []Token{ Directive("DOCTYPE doc [ '> ]"), }, want: `'> ]>`, }, { desc: "directive instruction with bad name", toks: []Token{ Directive("foo>"), }, err: "xml: EncodeToken of Directive containing wrong < or > markers", }, { desc: "end tag without start tag", toks: []Token{ EndElement{Name{"foo", "bar"}}, }, err: "xml: end tag without start tag", }, { desc: "mismatching end tag local name", toks: []Token{ StartElement{Name{"", "foo"}, nil}, EndElement{Name{"", "bar"}}, }, err: "xml: end tag does not match start tag ", want: ``, }, { desc: "mismatching end tag namespace", toks: []Token{ StartElement{Name{"space", "foo"}, nil}, EndElement{Name{"another", "foo"}}, }, err: "xml: end tag in namespace another does not match start tag in namespace space", want: ``, }, { desc: "start element with explicit namespace", toks: []Token{ StartElement{Name{"space", "local"}, []Attr{ {Name{"xmlns", "x"}, "space"}, {Name{"space", "foo"}, "value"}, }}, }, want: ``, }, { desc: "start element with explicit namespace and colliding prefix", toks: []Token{ StartElement{Name{"space", "local"}, []Attr{ {Name{"xmlns", "x"}, "space"}, {Name{"space", "foo"}, "value"}, {Name{"x", "bar"}, "other"}, }}, }, want: ``, }, { desc: "start element using previously defined namespace", toks: []Token{ StartElement{Name{"", "local"}, []Attr{ {Name{"xmlns", "x"}, "space"}, }}, StartElement{Name{"space", "foo"}, []Attr{ {Name{"space", "x"}, "y"}, }}, }, want: ``, }, { desc: "nested name space with same prefix", toks: []Token{ StartElement{Name{"", "foo"}, []Attr{ {Name{"xmlns", "x"}, "space1"}, }}, StartElement{Name{"", "foo"}, []Attr{ {Name{"xmlns", "x"}, "space2"}, }}, StartElement{Name{"", "foo"}, []Attr{ {Name{"space1", "a"}, "space1 value"}, {Name{"space2", "b"}, "space2 value"}, }}, EndElement{Name{"", "foo"}}, EndElement{Name{"", "foo"}}, StartElement{Name{"", "foo"}, []Attr{ {Name{"space1", "a"}, "space1 value"}, {Name{"space2", "b"}, "space2 value"}, }}, }, want: ``, }, { desc: "start element defining several prefixes for the same name space", toks: []Token{ StartElement{Name{"space", "foo"}, []Attr{ {Name{"xmlns", "a"}, "space"}, {Name{"xmlns", "b"}, "space"}, {Name{"space", "x"}, "value"}, }}, }, want: ``, }, { desc: "nested element redefines name space", toks: []Token{ StartElement{Name{"", "foo"}, []Attr{ {Name{"xmlns", "x"}, "space"}, }}, StartElement{Name{"space", "foo"}, []Attr{ {Name{"xmlns", "y"}, "space"}, {Name{"space", "a"}, "value"}, }}, }, want: ``, }, { desc: "nested element creates alias for default name space", toks: []Token{ StartElement{Name{"space", "foo"}, []Attr{ {Name{"", "xmlns"}, "space"}, }}, StartElement{Name{"space", "foo"}, []Attr{ {Name{"xmlns", "y"}, "space"}, {Name{"space", "a"}, "value"}, }}, }, want: ``, }, { desc: "nested element defines default name space with existing prefix", toks: []Token{ StartElement{Name{"", "foo"}, []Attr{ {Name{"xmlns", "x"}, "space"}, }}, StartElement{Name{"space", "foo"}, []Attr{ {Name{"", "xmlns"}, "space"}, {Name{"space", "a"}, "value"}, }}, }, want: ``, }, { desc: "nested element uses empty attribute name space when default ns defined", toks: []Token{ StartElement{Name{"space", "foo"}, []Attr{ {Name{"", "xmlns"}, "space"}, }}, StartElement{Name{"space", "foo"}, []Attr{ {Name{"", "attr"}, "value"}, }}, }, want: ``, }, { desc: "redefine xmlns", toks: []Token{ StartElement{Name{"", "foo"}, []Attr{ {Name{"foo", "xmlns"}, "space"}, }}, }, err: `xml: cannot redefine xmlns attribute prefix`, }, { desc: "xmlns with explicit name space #1", toks: []Token{ StartElement{Name{"space", "foo"}, []Attr{ {Name{"xml", "xmlns"}, "space"}, }}, }, want: ``, }, { desc: "xmlns with explicit name space #2", toks: []Token{ StartElement{Name{"space", "foo"}, []Attr{ {Name{xmlURL, "xmlns"}, "space"}, }}, }, want: ``, }, { desc: "empty name space declaration is ignored", toks: []Token{ StartElement{Name{"", "foo"}, []Attr{ {Name{"xmlns", "foo"}, ""}, }}, }, want: ``, }, { desc: "attribute with no name is ignored", toks: []Token{ StartElement{Name{"", "foo"}, []Attr{ {Name{"", ""}, "value"}, }}, }, want: ``, }, { desc: "namespace URL with non-valid name", toks: []Token{ StartElement{Name{"/34", "foo"}, []Attr{ {Name{"/34", "x"}, "value"}, }}, }, want: `<_:foo xmlns:_="/34" _:x="value">`, }, { desc: "nested element resets default namespace to empty", toks: []Token{ StartElement{Name{"space", "foo"}, []Attr{ {Name{"", "xmlns"}, "space"}, }}, StartElement{Name{"", "foo"}, []Attr{ {Name{"", "xmlns"}, ""}, {Name{"", "x"}, "value"}, {Name{"space", "x"}, "value"}, }}, }, want: ``, }, { desc: "nested element requires empty default name space", toks: []Token{ StartElement{Name{"space", "foo"}, []Attr{ {Name{"", "xmlns"}, "space"}, }}, StartElement{Name{"", "foo"}, nil}, }, want: ``, }, { desc: "attribute uses name space from xmlns", toks: []Token{ StartElement{Name{"some/space", "foo"}, []Attr{ {Name{"", "attr"}, "value"}, {Name{"some/space", "other"}, "other value"}, }}, }, want: ``, }, { desc: "default name space should not be used by attributes", toks: []Token{ StartElement{Name{"space", "foo"}, []Attr{ {Name{"", "xmlns"}, "space"}, {Name{"xmlns", "bar"}, "space"}, {Name{"space", "baz"}, "foo"}, }}, StartElement{Name{"space", "baz"}, nil}, EndElement{Name{"space", "baz"}}, EndElement{Name{"space", "foo"}}, }, want: ``, }, { desc: "default name space not used by attributes, not explicitly defined", toks: []Token{ StartElement{Name{"space", "foo"}, []Attr{ {Name{"", "xmlns"}, "space"}, {Name{"space", "baz"}, "foo"}, }}, StartElement{Name{"space", "baz"}, nil}, EndElement{Name{"space", "baz"}}, EndElement{Name{"space", "foo"}}, }, want: ``, }, { desc: "impossible xmlns declaration", toks: []Token{ StartElement{Name{"", "foo"}, []Attr{ {Name{"", "xmlns"}, "space"}, }}, StartElement{Name{"space", "bar"}, []Attr{ {Name{"space", "attr"}, "value"}, }}, }, want: ``, }} func TestEncodeToken(t *testing.T) { loop: for i, tt := range encodeTokenTests { var buf bytes.Buffer enc := NewEncoder(&buf) var err error for j, tok := range tt.toks { err = enc.EncodeToken(tok) if err != nil && j < len(tt.toks)-1 { t.Errorf("#%d %s token #%d: %v", i, tt.desc, j, err) continue loop } } errorf := func(f string, a ...interface{}) { t.Errorf("#%d %s token #%d:%s", i, tt.desc, len(tt.toks)-1, fmt.Sprintf(f, a...)) } switch { case tt.err != "" && err == nil: errorf(" expected error; got none") continue case tt.err == "" && err != nil: errorf(" got error: %v", err) continue case tt.err != "" && err != nil && tt.err != err.Error(): errorf(" error mismatch; got %v, want %v", err, tt.err) continue } if err := enc.Flush(); err != nil { errorf(" %v", err) continue } if got := buf.String(); got != tt.want { errorf("\ngot %v\nwant %v", got, tt.want) continue } } } func TestProcInstEncodeToken(t *testing.T) { var buf bytes.Buffer enc := NewEncoder(&buf) if err := enc.EncodeToken(ProcInst{"xml", []byte("Instruction")}); err != nil { t.Fatalf("enc.EncodeToken: expected to be able to encode xml target ProcInst as first token, %s", err) } if err := enc.EncodeToken(ProcInst{"Target", []byte("Instruction")}); err != nil { t.Fatalf("enc.EncodeToken: expected to be able to add non-xml target ProcInst") } if err := enc.EncodeToken(ProcInst{"xml", []byte("Instruction")}); err == nil { t.Fatalf("enc.EncodeToken: expected to not be allowed to encode xml target ProcInst when not first token") } } func TestDecodeEncode(t *testing.T) { var in, out bytes.Buffer in.WriteString(` `) dec := NewDecoder(&in) enc := NewEncoder(&out) for tok, err := dec.Token(); err == nil; tok, err = dec.Token() { err = enc.EncodeToken(tok) if err != nil { t.Fatalf("enc.EncodeToken: Unable to encode token (%#v), %v", tok, err) } } } // Issue 9796. Used to fail with GORACE="halt_on_error=1" -race. func TestRace9796(t *testing.T) { type A struct{} type B struct { C []A `xml:"X>Y"` } var wg sync.WaitGroup for i := 0; i < 2; i++ { wg.Add(1) go func() { Marshal(B{[]A{{}}}) wg.Done() }() } wg.Wait() } func TestIsValidDirective(t *testing.T) { testOK := []string{ "<>", "< < > >", "' '>' >", " ]>", " '<' ' doc ANY> ]>", ">>> a < comment --> [ ] >", } testKO := []string{ "<", ">", "", "< > > < < >", " -->", "", "'", "", } for _, s := range testOK { if !isValidDirective(Directive(s)) { t.Errorf("Directive %q is expected to be valid", s) } } for _, s := range testKO { if isValidDirective(Directive(s)) { t.Errorf("Directive %q is expected to be invalid", s) } } } // Issue 11719. EncodeToken used to silently eat tokens with an invalid type. func TestSimpleUseOfEncodeToken(t *testing.T) { var buf bytes.Buffer enc := NewEncoder(&buf) if err := enc.EncodeToken(&StartElement{Name: Name{"", "object1"}}); err == nil { t.Errorf("enc.EncodeToken: pointer type should be rejected") } if err := enc.EncodeToken(&EndElement{Name: Name{"", "object1"}}); err == nil { t.Errorf("enc.EncodeToken: pointer type should be rejected") } if err := enc.EncodeToken(StartElement{Name: Name{"", "object2"}}); err != nil { t.Errorf("enc.EncodeToken: StartElement %s", err) } if err := enc.EncodeToken(EndElement{Name: Name{"", "object2"}}); err != nil { t.Errorf("enc.EncodeToken: EndElement %s", err) } if err := enc.EncodeToken(Universe{}); err == nil { t.Errorf("enc.EncodeToken: invalid type not caught") } if err := enc.Flush(); err != nil { t.Errorf("enc.Flush: %s", err) } if buf.Len() == 0 { t.Errorf("enc.EncodeToken: empty buffer") } want := "" if buf.String() != want { t.Errorf("enc.EncodeToken: expected %q; got %q", want, buf.String()) } } ================================================ FILE: server/webdav/internal/xml/read.go ================================================ // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package xml import ( "bytes" "encoding" "errors" "fmt" "reflect" "strconv" "strings" ) // BUG(rsc): Mapping between XML elements and data structures is inherently flawed: // an XML element is an order-dependent collection of anonymous // values, while a data structure is an order-independent collection // of named values. // See package json for a textual representation more suitable // to data structures. // Unmarshal parses the XML-encoded data and stores the result in // the value pointed to by v, which must be an arbitrary struct, // slice, or string. Well-formed data that does not fit into v is // discarded. // // Because Unmarshal uses the reflect package, it can only assign // to exported (upper case) fields. Unmarshal uses a case-sensitive // comparison to match XML element names to tag values and struct // field names. // // Unmarshal maps an XML element to a struct using the following rules. // In the rules, the tag of a field refers to the value associated with the // key 'xml' in the struct field's tag (see the example above). // // - If the struct has a field of type []byte or string with tag // ",innerxml", Unmarshal accumulates the raw XML nested inside the // element in that field. The rest of the rules still apply. // // - If the struct has a field named XMLName of type xml.Name, // Unmarshal records the element name in that field. // // - If the XMLName field has an associated tag of the form // "name" or "namespace-URL name", the XML element must have // the given name (and, optionally, name space) or else Unmarshal // returns an error. // // - If the XML element has an attribute whose name matches a // struct field name with an associated tag containing ",attr" or // the explicit name in a struct field tag of the form "name,attr", // Unmarshal records the attribute value in that field. // // - If the XML element contains character data, that data is // accumulated in the first struct field that has tag ",chardata". // The struct field may have type []byte or string. // If there is no such field, the character data is discarded. // // - If the XML element contains comments, they are accumulated in // the first struct field that has tag ",comment". The struct // field may have type []byte or string. If there is no such // field, the comments are discarded. // // - If the XML element contains a sub-element whose name matches // the prefix of a tag formatted as "a" or "a>b>c", unmarshal // will descend into the XML structure looking for elements with the // given names, and will map the innermost elements to that struct // field. A tag starting with ">" is equivalent to one starting // with the field name followed by ">". // // - If the XML element contains a sub-element whose name matches // a struct field's XMLName tag and the struct field has no // explicit name tag as per the previous rule, unmarshal maps // the sub-element to that struct field. // // - If the XML element contains a sub-element whose name matches a // field without any mode flags (",attr", ",chardata", etc), Unmarshal // maps the sub-element to that struct field. // // - If the XML element contains a sub-element that hasn't matched any // of the above rules and the struct has a field with tag ",any", // unmarshal maps the sub-element to that struct field. // // - An anonymous struct field is handled as if the fields of its // value were part of the outer struct. // // - A struct field with tag "-" is never unmarshalled into. // // Unmarshal maps an XML element to a string or []byte by saving the // concatenation of that element's character data in the string or // []byte. The saved []byte is never nil. // // Unmarshal maps an attribute value to a string or []byte by saving // the value in the string or slice. // // Unmarshal maps an XML element to a slice by extending the length of // the slice and mapping the element to the newly created value. // // Unmarshal maps an XML element or attribute value to a bool by // setting it to the boolean value represented by the string. // // Unmarshal maps an XML element or attribute value to an integer or // floating-point field by setting the field to the result of // interpreting the string value in decimal. There is no check for // overflow. // // Unmarshal maps an XML element to an xml.Name by recording the // element name. // // Unmarshal maps an XML element to a pointer by setting the pointer // to a freshly allocated value and then mapping the element to that value. func Unmarshal(data []byte, v interface{}) error { return NewDecoder(bytes.NewReader(data)).Decode(v) } // Decode works like xml.Unmarshal, except it reads the decoder // stream to find the start element. func (d *Decoder) Decode(v interface{}) error { return d.DecodeElement(v, nil) } // DecodeElement works like xml.Unmarshal except that it takes // a pointer to the start XML element to decode into v. // It is useful when a client reads some raw XML tokens itself // but also wants to defer to Unmarshal for some elements. func (d *Decoder) DecodeElement(v interface{}, start *StartElement) error { val := reflect.ValueOf(v) if val.Kind() != reflect.Ptr { return errors.New("non-pointer passed to Unmarshal") } return d.unmarshal(val.Elem(), start) } // An UnmarshalError represents an error in the unmarshalling process. type UnmarshalError string func (e UnmarshalError) Error() string { return string(e) } // Unmarshaler is the interface implemented by objects that can unmarshal // an XML element description of themselves. // // UnmarshalXML decodes a single XML element // beginning with the given start element. // If it returns an error, the outer call to Unmarshal stops and // returns that error. // UnmarshalXML must consume exactly one XML element. // One common implementation strategy is to unmarshal into // a separate value with a layout matching the expected XML // using d.DecodeElement, and then to copy the data from // that value into the receiver. // Another common strategy is to use d.Token to process the // XML object one token at a time. // UnmarshalXML may not use d.RawToken. type Unmarshaler interface { UnmarshalXML(d *Decoder, start StartElement) error } // UnmarshalerAttr is the interface implemented by objects that can unmarshal // an XML attribute description of themselves. // // UnmarshalXMLAttr decodes a single XML attribute. // If it returns an error, the outer call to Unmarshal stops and // returns that error. // UnmarshalXMLAttr is used only for struct fields with the // "attr" option in the field tag. type UnmarshalerAttr interface { UnmarshalXMLAttr(attr Attr) error } // receiverType returns the receiver type to use in an expression like "%s.MethodName". func receiverType(val interface{}) string { t := reflect.TypeOf(val) if t.Name() != "" { return t.String() } return "(" + t.String() + ")" } // unmarshalInterface unmarshals a single XML element into val. // start is the opening tag of the element. func (p *Decoder) unmarshalInterface(val Unmarshaler, start *StartElement) error { // Record that decoder must stop at end tag corresponding to start. p.pushEOF() p.unmarshalDepth++ err := val.UnmarshalXML(p, *start) p.unmarshalDepth-- if err != nil { p.popEOF() return err } if !p.popEOF() { return fmt.Errorf("xml: %s.UnmarshalXML did not consume entire <%s> element", receiverType(val), start.Name.Local) } return nil } // unmarshalTextInterface unmarshals a single XML element into val. // The chardata contained in the element (but not its children) // is passed to the text unmarshaler. func (p *Decoder) unmarshalTextInterface(val encoding.TextUnmarshaler, start *StartElement) error { var buf []byte depth := 1 for depth > 0 { t, err := p.Token() if err != nil { return err } switch t := t.(type) { case CharData: if depth == 1 { buf = append(buf, t...) } case StartElement: depth++ case EndElement: depth-- } } return val.UnmarshalText(buf) } // unmarshalAttr unmarshals a single XML attribute into val. func (p *Decoder) unmarshalAttr(val reflect.Value, attr Attr) error { if val.Kind() == reflect.Ptr { if val.IsNil() { val.Set(reflect.New(val.Type().Elem())) } val = val.Elem() } if val.CanInterface() && val.Type().Implements(unmarshalerAttrType) { // This is an unmarshaler with a non-pointer receiver, // so it's likely to be incorrect, but we do what we're told. return val.Interface().(UnmarshalerAttr).UnmarshalXMLAttr(attr) } if val.CanAddr() { pv := val.Addr() if pv.CanInterface() && pv.Type().Implements(unmarshalerAttrType) { return pv.Interface().(UnmarshalerAttr).UnmarshalXMLAttr(attr) } } // Not an UnmarshalerAttr; try encoding.TextUnmarshaler. if val.CanInterface() && val.Type().Implements(textUnmarshalerType) { // This is an unmarshaler with a non-pointer receiver, // so it's likely to be incorrect, but we do what we're told. return val.Interface().(encoding.TextUnmarshaler).UnmarshalText([]byte(attr.Value)) } if val.CanAddr() { pv := val.Addr() if pv.CanInterface() && pv.Type().Implements(textUnmarshalerType) { return pv.Interface().(encoding.TextUnmarshaler).UnmarshalText([]byte(attr.Value)) } } copyValue(val, []byte(attr.Value)) return nil } var ( unmarshalerType = reflect.TypeOf((*Unmarshaler)(nil)).Elem() unmarshalerAttrType = reflect.TypeOf((*UnmarshalerAttr)(nil)).Elem() textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() ) // Unmarshal a single XML element into val. func (p *Decoder) unmarshal(val reflect.Value, start *StartElement) error { // Find start element if we need it. if start == nil { for { tok, err := p.Token() if err != nil { return err } if t, ok := tok.(StartElement); ok { start = &t break } } } // Load value from interface, but only if the result will be // usefully addressable. if val.Kind() == reflect.Interface && !val.IsNil() { e := val.Elem() if e.Kind() == reflect.Ptr && !e.IsNil() { val = e } } if val.Kind() == reflect.Ptr { if val.IsNil() { val.Set(reflect.New(val.Type().Elem())) } val = val.Elem() } if val.CanInterface() && val.Type().Implements(unmarshalerType) { // This is an unmarshaler with a non-pointer receiver, // so it's likely to be incorrect, but we do what we're told. return p.unmarshalInterface(val.Interface().(Unmarshaler), start) } if val.CanAddr() { pv := val.Addr() if pv.CanInterface() && pv.Type().Implements(unmarshalerType) { return p.unmarshalInterface(pv.Interface().(Unmarshaler), start) } } if val.CanInterface() && val.Type().Implements(textUnmarshalerType) { return p.unmarshalTextInterface(val.Interface().(encoding.TextUnmarshaler), start) } if val.CanAddr() { pv := val.Addr() if pv.CanInterface() && pv.Type().Implements(textUnmarshalerType) { return p.unmarshalTextInterface(pv.Interface().(encoding.TextUnmarshaler), start) } } var ( data []byte saveData reflect.Value comment []byte saveComment reflect.Value saveXML reflect.Value saveXMLIndex int saveXMLData []byte saveAny reflect.Value sv reflect.Value tinfo *typeInfo err error ) switch v := val; v.Kind() { default: return errors.New("unknown type " + v.Type().String()) case reflect.Interface: // TODO: For now, simply ignore the field. In the near // future we may choose to unmarshal the start // element on it, if not nil. return p.Skip() case reflect.Slice: typ := v.Type() if typ.Elem().Kind() == reflect.Uint8 { // []byte saveData = v break } // Slice of element values. // Grow slice. n := v.Len() if n >= v.Cap() { ncap := 2 * n if ncap < 4 { ncap = 4 } new := reflect.MakeSlice(typ, n, ncap) reflect.Copy(new, v) v.Set(new) } v.SetLen(n + 1) // Recur to read element into slice. if err := p.unmarshal(v.Index(n), start); err != nil { v.SetLen(n) return err } return nil case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, reflect.String: saveData = v case reflect.Struct: typ := v.Type() if typ == nameType { v.Set(reflect.ValueOf(start.Name)) break } sv = v tinfo, err = getTypeInfo(typ) if err != nil { return err } // Validate and assign element name. if tinfo.xmlname != nil { finfo := tinfo.xmlname if finfo.name != "" && finfo.name != start.Name.Local { return UnmarshalError("expected element type <" + finfo.name + "> but have <" + start.Name.Local + ">") } if finfo.xmlns != "" && finfo.xmlns != start.Name.Space { e := "expected element <" + finfo.name + "> in name space " + finfo.xmlns + " but have " if start.Name.Space == "" { e += "no name space" } else { e += start.Name.Space } return UnmarshalError(e) } fv := finfo.value(sv) if _, ok := fv.Interface().(Name); ok { fv.Set(reflect.ValueOf(start.Name)) } } // Assign attributes. // Also, determine whether we need to save character data or comments. for i := range tinfo.fields { finfo := &tinfo.fields[i] switch finfo.flags & fMode { case fAttr: strv := finfo.value(sv) // Look for attribute. for _, a := range start.Attr { if a.Name.Local == finfo.name && (finfo.xmlns == "" || finfo.xmlns == a.Name.Space) { if err := p.unmarshalAttr(strv, a); err != nil { return err } break } } case fCharData: if !saveData.IsValid() { saveData = finfo.value(sv) } case fComment: if !saveComment.IsValid() { saveComment = finfo.value(sv) } case fAny, fAny | fElement: if !saveAny.IsValid() { saveAny = finfo.value(sv) } case fInnerXml: if !saveXML.IsValid() { saveXML = finfo.value(sv) if p.saved == nil { saveXMLIndex = 0 p.saved = new(bytes.Buffer) } else { saveXMLIndex = p.savedOffset() } } } } } // Find end element. // Process sub-elements along the way. Loop: for { var savedOffset int if saveXML.IsValid() { savedOffset = p.savedOffset() } tok, err := p.Token() if err != nil { return err } switch t := tok.(type) { case StartElement: consumed := false if sv.IsValid() { consumed, err = p.unmarshalPath(tinfo, sv, nil, &t) if err != nil { return err } if !consumed && saveAny.IsValid() { consumed = true if err := p.unmarshal(saveAny, &t); err != nil { return err } } } if !consumed { if err := p.Skip(); err != nil { return err } } case EndElement: if saveXML.IsValid() { saveXMLData = p.saved.Bytes()[saveXMLIndex:savedOffset] if saveXMLIndex == 0 { p.saved = nil } } break Loop case CharData: if saveData.IsValid() { data = append(data, t...) } case Comment: if saveComment.IsValid() { comment = append(comment, t...) } } } if saveData.IsValid() && saveData.CanInterface() && saveData.Type().Implements(textUnmarshalerType) { if err := saveData.Interface().(encoding.TextUnmarshaler).UnmarshalText(data); err != nil { return err } saveData = reflect.Value{} } if saveData.IsValid() && saveData.CanAddr() { pv := saveData.Addr() if pv.CanInterface() && pv.Type().Implements(textUnmarshalerType) { if err := pv.Interface().(encoding.TextUnmarshaler).UnmarshalText(data); err != nil { return err } saveData = reflect.Value{} } } if err := copyValue(saveData, data); err != nil { return err } switch t := saveComment; t.Kind() { case reflect.String: t.SetString(string(comment)) case reflect.Slice: t.Set(reflect.ValueOf(comment)) } switch t := saveXML; t.Kind() { case reflect.String: t.SetString(string(saveXMLData)) case reflect.Slice: t.Set(reflect.ValueOf(saveXMLData)) } return nil } func copyValue(dst reflect.Value, src []byte) (err error) { dst0 := dst if dst.Kind() == reflect.Ptr { if dst.IsNil() { dst.Set(reflect.New(dst.Type().Elem())) } dst = dst.Elem() } // Save accumulated data. switch dst.Kind() { case reflect.Invalid: // Probably a comment. default: return errors.New("cannot unmarshal into " + dst0.Type().String()) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: itmp, err := strconv.ParseInt(string(src), 10, dst.Type().Bits()) if err != nil { return err } dst.SetInt(itmp) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: utmp, err := strconv.ParseUint(string(src), 10, dst.Type().Bits()) if err != nil { return err } dst.SetUint(utmp) case reflect.Float32, reflect.Float64: ftmp, err := strconv.ParseFloat(string(src), dst.Type().Bits()) if err != nil { return err } dst.SetFloat(ftmp) case reflect.Bool: value, err := strconv.ParseBool(strings.TrimSpace(string(src))) if err != nil { return err } dst.SetBool(value) case reflect.String: dst.SetString(string(src)) case reflect.Slice: if len(src) == 0 { // non-nil to flag presence src = []byte{} } dst.SetBytes(src) } return nil } // unmarshalPath walks down an XML structure looking for wanted // paths, and calls unmarshal on them. // The consumed result tells whether XML elements have been consumed // from the Decoder until start's matching end element, or if it's // still untouched because start is uninteresting for sv's fields. func (p *Decoder) unmarshalPath(tinfo *typeInfo, sv reflect.Value, parents []string, start *StartElement) (consumed bool, err error) { recurse := false Loop: for i := range tinfo.fields { finfo := &tinfo.fields[i] if finfo.flags&fElement == 0 || len(finfo.parents) < len(parents) || finfo.xmlns != "" && finfo.xmlns != start.Name.Space { continue } for j := range parents { if parents[j] != finfo.parents[j] { continue Loop } } if len(finfo.parents) == len(parents) && finfo.name == start.Name.Local { // It's a perfect match, unmarshal the field. return true, p.unmarshal(finfo.value(sv), start) } if len(finfo.parents) > len(parents) && finfo.parents[len(parents)] == start.Name.Local { // It's a prefix for the field. Break and recurse // since it's not ok for one field path to be itself // the prefix for another field path. recurse = true // We can reuse the same slice as long as we // don't try to append to it. parents = finfo.parents[:len(parents)+1] break } } if !recurse { // We have no business with this element. return false, nil } // The element is not a perfect match for any field, but one // or more fields have the path to this element as a parent // prefix. Recurse and attempt to match these. for { var tok Token tok, err = p.Token() if err != nil { return true, err } switch t := tok.(type) { case StartElement: consumed2, err := p.unmarshalPath(tinfo, sv, parents, &t) if err != nil { return true, err } if !consumed2 { if err := p.Skip(); err != nil { return true, err } } case EndElement: return true, nil } } } // Skip reads tokens until it has consumed the end element // matching the most recent start element already consumed. // It recurs if it encounters a start element, so it can be used to // skip nested structures. // It returns nil if it finds an end element matching the start // element; otherwise it returns an error describing the problem. func (d *Decoder) Skip() error { for { tok, err := d.Token() if err != nil { return err } switch tok.(type) { case StartElement: if err := d.Skip(); err != nil { return err } case EndElement: return nil } } } ================================================ FILE: server/webdav/internal/xml/read_test.go ================================================ // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package xml import ( "bytes" "fmt" "io" "reflect" "strings" "testing" "time" ) // Stripped down Atom feed data structures. func TestUnmarshalFeed(t *testing.T) { var f Feed if err := Unmarshal([]byte(atomFeedString), &f); err != nil { t.Fatalf("Unmarshal: %s", err) } if !reflect.DeepEqual(f, atomFeed) { t.Fatalf("have %#v\nwant %#v", f, atomFeed) } } // hget http://codereview.appspot.com/rss/mine/rsc const atomFeedString = ` Code Review - My issueshttp://codereview.appspot.com/rietveld<>rietveld: an attempt at pubsubhubbub 2009-10-04T01:35:58+00:00email-address-removedurn:md5:134d9179c41f806be79b3a5f7877d19a An attempt at adding pubsubhubbub support to Rietveld. http://code.google.com/p/pubsubhubbub http://code.google.com/p/rietveld/issues/detail?id=155 The server side of the protocol is trivial: 1. add a &lt;link rel=&quot;hub&quot; href=&quot;hub-server&quot;&gt; tag to all feeds that will be pubsubhubbubbed. 2. every time one of those feeds changes, tell the hub with a simple POST request. I have tested this by adding debug prints to a local hub server and checking that the server got the right publish requests. I can&#39;t quite get the server to work, but I think the bug is not in my code. I think that the server expects to be able to grab the feed and see the feed&#39;s actual URL in the link rel=&quot;self&quot;, but the default value for that drops the :port from the URL, and I cannot for the life of me figure out how to get the Atom generator deep inside django not to do that, or even where it is doing that, or even what code is running to generate the Atom feed. (I thought I knew but I added some assert False statements and it kept running!) Ignoring that particular problem, I would appreciate feedback on the right way to get the two values at the top of feeds.py marked NOTE(rsc). rietveld: correct tab handling 2009-10-03T23:02:17+00:00email-address-removedurn:md5:0a2a4f19bb815101f0ba2904aed7c35a This fixes the buggy tab rendering that can be seen at http://codereview.appspot.com/116075/diff/1/2 The fundamental problem was that the tab code was not being told what column the text began in, so it didn&#39;t know where to put the tab stops. Another problem was that some of the code assumed that string byte offsets were the same as column offsets, which is only true if there are no tabs. In the process of fixing this, I cleaned up the arguments to Fold and ExpandTabs and renamed them Break and _ExpandTabs so that I could be sure that I found all the call sites. I also wanted to verify that ExpandTabs was not being used from outside intra_region_diff.py. ` type Feed struct { XMLName Name `xml:"http://www.w3.org/2005/Atom feed"` Title string `xml:"title"` Id string `xml:"id"` Link []Link `xml:"link"` Updated time.Time `xml:"updated,attr"` Author Person `xml:"author"` Entry []Entry `xml:"entry"` } type Entry struct { Title string `xml:"title"` Id string `xml:"id"` Link []Link `xml:"link"` Updated time.Time `xml:"updated"` Author Person `xml:"author"` Summary Text `xml:"summary"` } type Link struct { Rel string `xml:"rel,attr,omitempty"` Href string `xml:"href,attr"` } type Person struct { Name string `xml:"name"` URI string `xml:"uri"` Email string `xml:"email"` InnerXML string `xml:",innerxml"` } type Text struct { Type string `xml:"type,attr,omitempty"` Body string `xml:",chardata"` } var atomFeed = Feed{ XMLName: Name{"http://www.w3.org/2005/Atom", "feed"}, Title: "Code Review - My issues", Link: []Link{ {Rel: "alternate", Href: "http://codereview.appspot.com/"}, {Rel: "self", Href: "http://codereview.appspot.com/rss/mine/rsc"}, }, Id: "http://codereview.appspot.com/", Updated: ParseTime("2009-10-04T01:35:58+00:00"), Author: Person{ Name: "rietveld<>", InnerXML: "rietveld<>", }, Entry: []Entry{ { Title: "rietveld: an attempt at pubsubhubbub\n", Link: []Link{ {Rel: "alternate", Href: "http://codereview.appspot.com/126085"}, }, Updated: ParseTime("2009-10-04T01:35:58+00:00"), Author: Person{ Name: "email-address-removed", InnerXML: "email-address-removed", }, Id: "urn:md5:134d9179c41f806be79b3a5f7877d19a", Summary: Text{ Type: "html", Body: ` An attempt at adding pubsubhubbub support to Rietveld. http://code.google.com/p/pubsubhubbub http://code.google.com/p/rietveld/issues/detail?id=155 The server side of the protocol is trivial: 1. add a <link rel="hub" href="hub-server"> tag to all feeds that will be pubsubhubbubbed. 2. every time one of those feeds changes, tell the hub with a simple POST request. I have tested this by adding debug prints to a local hub server and checking that the server got the right publish requests. I can't quite get the server to work, but I think the bug is not in my code. I think that the server expects to be able to grab the feed and see the feed's actual URL in the link rel="self", but the default value for that drops the :port from the URL, and I cannot for the life of me figure out how to get the Atom generator deep inside django not to do that, or even where it is doing that, or even what code is running to generate the Atom feed. (I thought I knew but I added some assert False statements and it kept running!) Ignoring that particular problem, I would appreciate feedback on the right way to get the two values at the top of feeds.py marked NOTE(rsc). `, }, }, { Title: "rietveld: correct tab handling\n", Link: []Link{ {Rel: "alternate", Href: "http://codereview.appspot.com/124106"}, }, Updated: ParseTime("2009-10-03T23:02:17+00:00"), Author: Person{ Name: "email-address-removed", InnerXML: "email-address-removed", }, Id: "urn:md5:0a2a4f19bb815101f0ba2904aed7c35a", Summary: Text{ Type: "html", Body: ` This fixes the buggy tab rendering that can be seen at http://codereview.appspot.com/116075/diff/1/2 The fundamental problem was that the tab code was not being told what column the text began in, so it didn't know where to put the tab stops. Another problem was that some of the code assumed that string byte offsets were the same as column offsets, which is only true if there are no tabs. In the process of fixing this, I cleaned up the arguments to Fold and ExpandTabs and renamed them Break and _ExpandTabs so that I could be sure that I found all the call sites. I also wanted to verify that ExpandTabs was not being used from outside intra_region_diff.py. `, }, }, }, } const pathTestString = ` 1 A B C D <_> E 2 ` type PathTestItem struct { Value string } type PathTestA struct { Items []PathTestItem `xml:">Item1"` Before, After string } type PathTestB struct { Other []PathTestItem `xml:"Items>Item1"` Before, After string } type PathTestC struct { Values1 []string `xml:"Items>Item1>Value"` Values2 []string `xml:"Items>Item2>Value"` Before, After string } type PathTestSet struct { Item1 []PathTestItem } type PathTestD struct { Other PathTestSet `xml:"Items"` Before, After string } type PathTestE struct { Underline string `xml:"Items>_>Value"` Before, After string } var pathTests = []interface{}{ &PathTestA{Items: []PathTestItem{{"A"}, {"D"}}, Before: "1", After: "2"}, &PathTestB{Other: []PathTestItem{{"A"}, {"D"}}, Before: "1", After: "2"}, &PathTestC{Values1: []string{"A", "C", "D"}, Values2: []string{"B"}, Before: "1", After: "2"}, &PathTestD{Other: PathTestSet{Item1: []PathTestItem{{"A"}, {"D"}}}, Before: "1", After: "2"}, &PathTestE{Underline: "E", Before: "1", After: "2"}, } func TestUnmarshalPaths(t *testing.T) { for _, pt := range pathTests { v := reflect.New(reflect.TypeOf(pt).Elem()).Interface() if err := Unmarshal([]byte(pathTestString), v); err != nil { t.Fatalf("Unmarshal: %s", err) } if !reflect.DeepEqual(v, pt) { t.Fatalf("have %#v\nwant %#v", v, pt) } } } type BadPathTestA struct { First string `xml:"items>item1"` Other string `xml:"items>item2"` Second string `xml:"items"` } type BadPathTestB struct { Other string `xml:"items>item2>value"` First string `xml:"items>item1"` Second string `xml:"items>item1>value"` } type BadPathTestC struct { First string Second string `xml:"First"` } type BadPathTestD struct { BadPathEmbeddedA BadPathEmbeddedB } type BadPathEmbeddedA struct { First string } type BadPathEmbeddedB struct { Second string `xml:"First"` } var badPathTests = []struct { v, e interface{} }{ {&BadPathTestA{}, &TagPathError{reflect.TypeOf(BadPathTestA{}), "First", "items>item1", "Second", "items"}}, {&BadPathTestB{}, &TagPathError{reflect.TypeOf(BadPathTestB{}), "First", "items>item1", "Second", "items>item1>value"}}, {&BadPathTestC{}, &TagPathError{reflect.TypeOf(BadPathTestC{}), "First", "", "Second", "First"}}, {&BadPathTestD{}, &TagPathError{reflect.TypeOf(BadPathTestD{}), "First", "", "Second", "First"}}, } func TestUnmarshalBadPaths(t *testing.T) { for _, tt := range badPathTests { err := Unmarshal([]byte(pathTestString), tt.v) if !reflect.DeepEqual(err, tt.e) { t.Fatalf("Unmarshal with %#v didn't fail properly:\nhave %#v,\nwant %#v", tt.v, err, tt.e) } } } const OK = "OK" const withoutNameTypeData = ` ` type TestThree struct { XMLName Name `xml:"Test3"` Attr string `xml:",attr"` } func TestUnmarshalWithoutNameType(t *testing.T) { var x TestThree if err := Unmarshal([]byte(withoutNameTypeData), &x); err != nil { t.Fatalf("Unmarshal: %s", err) } if x.Attr != OK { t.Fatalf("have %v\nwant %v", x.Attr, OK) } } func TestUnmarshalAttr(t *testing.T) { type ParamVal struct { Int int `xml:"int,attr"` } type ParamPtr struct { Int *int `xml:"int,attr"` } type ParamStringPtr struct { Int *string `xml:"int,attr"` } x := []byte(``) p1 := &ParamPtr{} if err := Unmarshal(x, p1); err != nil { t.Fatalf("Unmarshal: %s", err) } if p1.Int == nil { t.Fatalf("Unmarshal failed in to *int field") } else if *p1.Int != 1 { t.Fatalf("Unmarshal with %s failed:\nhave %#v,\n want %#v", x, p1.Int, 1) } p2 := &ParamVal{} if err := Unmarshal(x, p2); err != nil { t.Fatalf("Unmarshal: %s", err) } if p2.Int != 1 { t.Fatalf("Unmarshal with %s failed:\nhave %#v,\n want %#v", x, p2.Int, 1) } p3 := &ParamStringPtr{} if err := Unmarshal(x, p3); err != nil { t.Fatalf("Unmarshal: %s", err) } if p3.Int == nil { t.Fatalf("Unmarshal failed in to *string field") } else if *p3.Int != "1" { t.Fatalf("Unmarshal with %s failed:\nhave %#v,\n want %#v", x, p3.Int, 1) } } type Tables struct { HTable string `xml:"http://www.w3.org/TR/html4/ table"` FTable string `xml:"http://www.w3schools.com/furniture table"` } var tables = []struct { xml string tab Tables ns string }{ { xml: `` + `hello
` + `world
` + ``, tab: Tables{"hello", "world"}, }, { xml: `` + `world
` + `hello
` + ``, tab: Tables{"hello", "world"}, }, { xml: `` + `world` + `hello` + ``, tab: Tables{"hello", "world"}, }, { xml: `` + `bogus
` + ``, tab: Tables{}, }, { xml: `` + `only
` + ``, tab: Tables{HTable: "only"}, ns: "http://www.w3.org/TR/html4/", }, { xml: `` + `only
` + ``, tab: Tables{FTable: "only"}, ns: "http://www.w3schools.com/furniture", }, { xml: `` + `only
` + ``, tab: Tables{}, ns: "something else entirely", }, } func TestUnmarshalNS(t *testing.T) { for i, tt := range tables { var dst Tables var err error if tt.ns != "" { d := NewDecoder(strings.NewReader(tt.xml)) d.DefaultSpace = tt.ns err = d.Decode(&dst) } else { err = Unmarshal([]byte(tt.xml), &dst) } if err != nil { t.Errorf("#%d: Unmarshal: %v", i, err) continue } want := tt.tab if dst != want { t.Errorf("#%d: dst=%+v, want %+v", i, dst, want) } } } func TestRoundTrip(t *testing.T) { // From issue 7535 const s = `` in := bytes.NewBufferString(s) for i := 0; i < 10; i++ { out := &bytes.Buffer{} d := NewDecoder(in) e := NewEncoder(out) for { t, err := d.Token() if err == io.EOF { break } if err != nil { fmt.Println("failed:", err) return } e.EncodeToken(t) } e.Flush() in = out } if got := in.String(); got != s { t.Errorf("have: %q\nwant: %q\n", got, s) } } func TestMarshalNS(t *testing.T) { dst := Tables{"hello", "world"} data, err := Marshal(&dst) if err != nil { t.Fatalf("Marshal: %v", err) } want := `hello
world
` str := string(data) if str != want { t.Errorf("have: %q\nwant: %q\n", str, want) } } type TableAttrs struct { TAttr TAttr } type TAttr struct { HTable string `xml:"http://www.w3.org/TR/html4/ table,attr"` FTable string `xml:"http://www.w3schools.com/furniture table,attr"` Lang string `xml:"http://www.w3.org/XML/1998/namespace lang,attr,omitempty"` Other1 string `xml:"http://golang.org/xml/ other,attr,omitempty"` Other2 string `xml:"http://golang.org/xmlfoo/ other,attr,omitempty"` Other3 string `xml:"http://golang.org/json/ other,attr,omitempty"` Other4 string `xml:"http://golang.org/2/json/ other,attr,omitempty"` } var tableAttrs = []struct { xml string tab TableAttrs ns string }{ { xml: ``, tab: TableAttrs{TAttr{HTable: "hello", FTable: "world"}}, }, { xml: ``, tab: TableAttrs{TAttr{HTable: "hello", FTable: "world"}}, }, { xml: ``, tab: TableAttrs{TAttr{HTable: "hello", FTable: "world"}}, }, { // Default space does not apply to attribute names. xml: ``, tab: TableAttrs{TAttr{HTable: "hello", FTable: ""}}, }, { // Default space does not apply to attribute names. xml: ``, tab: TableAttrs{TAttr{HTable: "", FTable: "world"}}, }, { xml: ``, tab: TableAttrs{}, }, { // Default space does not apply to attribute names. xml: ``, tab: TableAttrs{TAttr{HTable: "hello", FTable: ""}}, ns: "http://www.w3schools.com/furniture", }, { // Default space does not apply to attribute names. xml: ``, tab: TableAttrs{TAttr{HTable: "", FTable: "world"}}, ns: "http://www.w3.org/TR/html4/", }, { xml: ``, tab: TableAttrs{}, ns: "something else entirely", }, } func TestUnmarshalNSAttr(t *testing.T) { for i, tt := range tableAttrs { var dst TableAttrs var err error if tt.ns != "" { d := NewDecoder(strings.NewReader(tt.xml)) d.DefaultSpace = tt.ns err = d.Decode(&dst) } else { err = Unmarshal([]byte(tt.xml), &dst) } if err != nil { t.Errorf("#%d: Unmarshal: %v", i, err) continue } want := tt.tab if dst != want { t.Errorf("#%d: dst=%+v, want %+v", i, dst, want) } } } func TestMarshalNSAttr(t *testing.T) { src := TableAttrs{TAttr{"hello", "world", "en_US", "other1", "other2", "other3", "other4"}} data, err := Marshal(&src) if err != nil { t.Fatalf("Marshal: %v", err) } want := `` str := string(data) if str != want { t.Errorf("Marshal:\nhave: %#q\nwant: %#q\n", str, want) } var dst TableAttrs if err := Unmarshal(data, &dst); err != nil { t.Errorf("Unmarshal: %v", err) } if dst != src { t.Errorf("Unmarshal = %q, want %q", dst, src) } } type MyCharData struct { body string } func (m *MyCharData) UnmarshalXML(d *Decoder, start StartElement) error { for { t, err := d.Token() if err == io.EOF { // found end of element break } if err != nil { return err } if char, ok := t.(CharData); ok { m.body += string(char) } } return nil } var _ Unmarshaler = (*MyCharData)(nil) func (m *MyCharData) UnmarshalXMLAttr(attr Attr) error { panic("must not call") } type MyAttr struct { attr string } func (m *MyAttr) UnmarshalXMLAttr(attr Attr) error { m.attr = attr.Value return nil } var _ UnmarshalerAttr = (*MyAttr)(nil) type MyStruct struct { Data *MyCharData Attr *MyAttr `xml:",attr"` Data2 MyCharData Attr2 MyAttr `xml:",attr"` } func TestUnmarshaler(t *testing.T) { xml := ` hello world howdy world ` var m MyStruct if err := Unmarshal([]byte(xml), &m); err != nil { t.Fatal(err) } if m.Data == nil || m.Attr == nil || m.Data.body != "hello world" || m.Attr.attr != "attr1" || m.Data2.body != "howdy world" || m.Attr2.attr != "attr2" { t.Errorf("m=%#+v\n", m) } } type Pea struct { Cotelydon string } type Pod struct { Pea interface{} `xml:"Pea"` } // https://golang.org/issue/6836 func TestUnmarshalIntoInterface(t *testing.T) { pod := new(Pod) pod.Pea = new(Pea) xml := `Green stuff` err := Unmarshal([]byte(xml), pod) if err != nil { t.Fatalf("failed to unmarshal %q: %v", xml, err) } pea, ok := pod.Pea.(*Pea) if !ok { t.Fatalf("unmarshalled into wrong type: have %T want *Pea", pod.Pea) } have, want := pea.Cotelydon, "Green stuff" if have != want { t.Errorf("failed to unmarshal into interface, have %q want %q", have, want) } } ================================================ FILE: server/webdav/internal/xml/typeinfo.go ================================================ // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package xml import ( "fmt" "reflect" "strings" "sync" ) // typeInfo holds details for the xml representation of a type. type typeInfo struct { xmlname *fieldInfo fields []fieldInfo } // fieldInfo holds details for the xml representation of a single field. type fieldInfo struct { idx []int name string xmlns string flags fieldFlags parents []string } type fieldFlags int const ( fElement fieldFlags = 1 << iota fAttr fCharData fInnerXml fComment fAny fOmitEmpty fMode = fElement | fAttr | fCharData | fInnerXml | fComment | fAny ) var tinfoMap = make(map[reflect.Type]*typeInfo) var tinfoLock sync.RWMutex var nameType = reflect.TypeOf(Name{}) // getTypeInfo returns the typeInfo structure with details necessary // for marshalling and unmarshalling typ. func getTypeInfo(typ reflect.Type) (*typeInfo, error) { tinfoLock.RLock() tinfo, ok := tinfoMap[typ] tinfoLock.RUnlock() if ok { return tinfo, nil } tinfo = &typeInfo{} if typ.Kind() == reflect.Struct && typ != nameType { n := typ.NumField() for i := 0; i < n; i++ { f := typ.Field(i) if f.PkgPath != "" || f.Tag.Get("xml") == "-" { continue // Private field } // For embedded structs, embed its fields. if f.Anonymous { t := f.Type if t.Kind() == reflect.Ptr { t = t.Elem() } if t.Kind() == reflect.Struct { inner, err := getTypeInfo(t) if err != nil { return nil, err } if tinfo.xmlname == nil { tinfo.xmlname = inner.xmlname } for _, finfo := range inner.fields { finfo.idx = append([]int{i}, finfo.idx...) if err := addFieldInfo(typ, tinfo, &finfo); err != nil { return nil, err } } continue } } finfo, err := structFieldInfo(typ, &f) if err != nil { return nil, err } if f.Name == "XMLName" { tinfo.xmlname = finfo continue } // Add the field if it doesn't conflict with other fields. if err := addFieldInfo(typ, tinfo, finfo); err != nil { return nil, err } } } tinfoLock.Lock() tinfoMap[typ] = tinfo tinfoLock.Unlock() return tinfo, nil } // structFieldInfo builds and returns a fieldInfo for f. func structFieldInfo(typ reflect.Type, f *reflect.StructField) (*fieldInfo, error) { finfo := &fieldInfo{idx: f.Index} // Split the tag from the xml namespace if necessary. tag := f.Tag.Get("xml") if i := strings.Index(tag, " "); i >= 0 { finfo.xmlns, tag = tag[:i], tag[i+1:] } // Parse flags. tokens := strings.Split(tag, ",") if len(tokens) == 1 { finfo.flags = fElement } else { tag = tokens[0] for _, flag := range tokens[1:] { switch flag { case "attr": finfo.flags |= fAttr case "chardata": finfo.flags |= fCharData case "innerxml": finfo.flags |= fInnerXml case "comment": finfo.flags |= fComment case "any": finfo.flags |= fAny case "omitempty": finfo.flags |= fOmitEmpty } } // Validate the flags used. valid := true switch mode := finfo.flags & fMode; mode { case 0: finfo.flags |= fElement case fAttr, fCharData, fInnerXml, fComment, fAny: if f.Name == "XMLName" || tag != "" && mode != fAttr { valid = false } default: // This will also catch multiple modes in a single field. valid = false } if finfo.flags&fMode == fAny { finfo.flags |= fElement } if finfo.flags&fOmitEmpty != 0 && finfo.flags&(fElement|fAttr) == 0 { valid = false } if !valid { return nil, fmt.Errorf("xml: invalid tag in field %s of type %s: %q", f.Name, typ, f.Tag.Get("xml")) } } // Use of xmlns without a name is not allowed. if finfo.xmlns != "" && tag == "" { return nil, fmt.Errorf("xml: namespace without name in field %s of type %s: %q", f.Name, typ, f.Tag.Get("xml")) } if f.Name == "XMLName" { // The XMLName field records the XML element name. Don't // process it as usual because its name should default to // empty rather than to the field name. finfo.name = tag return finfo, nil } if tag == "" { // If the name part of the tag is completely empty, get // default from XMLName of underlying struct if feasible, // or field name otherwise. if xmlname := lookupXMLName(f.Type); xmlname != nil { finfo.xmlns, finfo.name = xmlname.xmlns, xmlname.name } else { finfo.name = f.Name } return finfo, nil } if finfo.xmlns == "" && finfo.flags&fAttr == 0 { // If it's an element no namespace specified, get the default // from the XMLName of enclosing struct if possible. if xmlname := lookupXMLName(typ); xmlname != nil { finfo.xmlns = xmlname.xmlns } } // Prepare field name and parents. parents := strings.Split(tag, ">") if parents[0] == "" { parents[0] = f.Name } if parents[len(parents)-1] == "" { return nil, fmt.Errorf("xml: trailing '>' in field %s of type %s", f.Name, typ) } finfo.name = parents[len(parents)-1] if len(parents) > 1 { if (finfo.flags & fElement) == 0 { return nil, fmt.Errorf("xml: %s chain not valid with %s flag", tag, strings.Join(tokens[1:], ",")) } finfo.parents = parents[:len(parents)-1] } // If the field type has an XMLName field, the names must match // so that the behavior of both marshalling and unmarshalling // is straightforward and unambiguous. if finfo.flags&fElement != 0 { ftyp := f.Type xmlname := lookupXMLName(ftyp) if xmlname != nil && xmlname.name != finfo.name { return nil, fmt.Errorf("xml: name %q in tag of %s.%s conflicts with name %q in %s.XMLName", finfo.name, typ, f.Name, xmlname.name, ftyp) } } return finfo, nil } // lookupXMLName returns the fieldInfo for typ's XMLName field // in case it exists and has a valid xml field tag, otherwise // it returns nil. func lookupXMLName(typ reflect.Type) (xmlname *fieldInfo) { for typ.Kind() == reflect.Ptr { typ = typ.Elem() } if typ.Kind() != reflect.Struct { return nil } for i, n := 0, typ.NumField(); i < n; i++ { f := typ.Field(i) if f.Name != "XMLName" { continue } finfo, err := structFieldInfo(typ, &f) if finfo.name != "" && err == nil { return finfo } // Also consider errors as a non-existent field tag // and let getTypeInfo itself report the error. break } return nil } func min(a, b int) int { if a <= b { return a } return b } // addFieldInfo adds finfo to tinfo.fields if there are no // conflicts, or if conflicts arise from previous fields that were // obtained from deeper embedded structures than finfo. In the latter // case, the conflicting entries are dropped. // A conflict occurs when the path (parent + name) to a field is // itself a prefix of another path, or when two paths match exactly. // It is okay for field paths to share a common, shorter prefix. func addFieldInfo(typ reflect.Type, tinfo *typeInfo, newf *fieldInfo) error { var conflicts []int Loop: // First, figure all conflicts. Most working code will have none. for i := range tinfo.fields { oldf := &tinfo.fields[i] if oldf.flags&fMode != newf.flags&fMode { continue } if oldf.xmlns != "" && newf.xmlns != "" && oldf.xmlns != newf.xmlns { continue } minl := min(len(newf.parents), len(oldf.parents)) for p := 0; p < minl; p++ { if oldf.parents[p] != newf.parents[p] { continue Loop } } if len(oldf.parents) > len(newf.parents) { if oldf.parents[len(newf.parents)] == newf.name { conflicts = append(conflicts, i) } } else if len(oldf.parents) < len(newf.parents) { if newf.parents[len(oldf.parents)] == oldf.name { conflicts = append(conflicts, i) } } else { if newf.name == oldf.name { conflicts = append(conflicts, i) } } } // Without conflicts, add the new field and return. if conflicts == nil { tinfo.fields = append(tinfo.fields, *newf) return nil } // If any conflict is shallower, ignore the new field. // This matches the Go field resolution on embedding. for _, i := range conflicts { if len(tinfo.fields[i].idx) < len(newf.idx) { return nil } } // Otherwise, if any of them is at the same depth level, it's an error. for _, i := range conflicts { oldf := &tinfo.fields[i] if len(oldf.idx) == len(newf.idx) { f1 := typ.FieldByIndex(oldf.idx) f2 := typ.FieldByIndex(newf.idx) return &TagPathError{typ, f1.Name, f1.Tag.Get("xml"), f2.Name, f2.Tag.Get("xml")} } } // Otherwise, the new field is shallower, and thus takes precedence, // so drop the conflicting fields from tinfo and append the new one. for c := len(conflicts) - 1; c >= 0; c-- { i := conflicts[c] copy(tinfo.fields[i:], tinfo.fields[i+1:]) tinfo.fields = tinfo.fields[:len(tinfo.fields)-1] } tinfo.fields = append(tinfo.fields, *newf) return nil } // A TagPathError represents an error in the unmarshalling process // caused by the use of field tags with conflicting paths. type TagPathError struct { Struct reflect.Type Field1, Tag1 string Field2, Tag2 string } func (e *TagPathError) Error() string { return fmt.Sprintf("%s field %q with tag %q conflicts with field %q with tag %q", e.Struct, e.Field1, e.Tag1, e.Field2, e.Tag2) } // value returns v's field value corresponding to finfo. // It's equivalent to v.FieldByIndex(finfo.idx), but initializes // and dereferences pointers as necessary. func (finfo *fieldInfo) value(v reflect.Value) reflect.Value { for i, x := range finfo.idx { if i > 0 { t := v.Type() if t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct { if v.IsNil() { v.Set(reflect.New(v.Type().Elem())) } v = v.Elem() } } v = v.Field(x) } return v } ================================================ FILE: server/webdav/internal/xml/xml.go ================================================ // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package xml implements a simple XML 1.0 parser that // understands XML name spaces. package xml // References: // Annotated XML spec: http://www.xml.com/axml/testaxml.htm // XML name spaces: http://www.w3.org/TR/REC-xml-names/ // TODO(rsc): // Test error handling. import ( "bufio" "bytes" "errors" "fmt" "io" "strconv" "strings" "unicode" "unicode/utf8" ) // A SyntaxError represents a syntax error in the XML input stream. type SyntaxError struct { Msg string Line int } func (e *SyntaxError) Error() string { return "XML syntax error on line " + strconv.Itoa(e.Line) + ": " + e.Msg } // A Name represents an XML name (Local) annotated with a name space // identifier (Space). In tokens returned by Decoder.Token, the Space // identifier is given as a canonical URL, not the short prefix used in // the document being parsed. // // As a special case, XML namespace declarations will use the literal // string "xmlns" for the Space field instead of the fully resolved URL. // See Encoder.EncodeToken for more information on namespace encoding // behaviour. type Name struct { Space, Local string } // isNamespace reports whether the name is a namespace-defining name. func (name Name) isNamespace() bool { return name.Local == "xmlns" || name.Space == "xmlns" } // An Attr represents an attribute in an XML element (Name=Value). type Attr struct { Name Name Value string } // A Token is an interface holding one of the token types: // StartElement, EndElement, CharData, Comment, ProcInst, or Directive. type Token interface{} // A StartElement represents an XML start element. type StartElement struct { Name Name Attr []Attr } func (e StartElement) Copy() StartElement { attrs := make([]Attr, len(e.Attr)) copy(attrs, e.Attr) e.Attr = attrs return e } // End returns the corresponding XML end element. func (e StartElement) End() EndElement { return EndElement{e.Name} } // setDefaultNamespace sets the namespace of the element // as the default for all elements contained within it. func (e *StartElement) setDefaultNamespace() { if e.Name.Space == "" { // If there's no namespace on the element, don't // set the default. Strictly speaking this might be wrong, as // we can't tell if the element had no namespace set // or was just using the default namespace. return } // Don't add a default name space if there's already one set. for _, attr := range e.Attr { if attr.Name.Space == "" && attr.Name.Local == "xmlns" { return } } e.Attr = append(e.Attr, Attr{ Name: Name{ Local: "xmlns", }, Value: e.Name.Space, }) } // An EndElement represents an XML end element. type EndElement struct { Name Name } // A CharData represents XML character data (raw text), // in which XML escape sequences have been replaced by // the characters they represent. type CharData []byte func makeCopy(b []byte) []byte { b1 := make([]byte, len(b)) copy(b1, b) return b1 } func (c CharData) Copy() CharData { return CharData(makeCopy(c)) } // A Comment represents an XML comment of the form . // The bytes do not include the comment markers. type Comment []byte func (c Comment) Copy() Comment { return Comment(makeCopy(c)) } // A ProcInst represents an XML processing instruction of the form type ProcInst struct { Target string Inst []byte } func (p ProcInst) Copy() ProcInst { p.Inst = makeCopy(p.Inst) return p } // A Directive represents an XML directive of the form . // The bytes do not include the markers. type Directive []byte func (d Directive) Copy() Directive { return Directive(makeCopy(d)) } // CopyToken returns a copy of a Token. func CopyToken(t Token) Token { switch v := t.(type) { case CharData: return v.Copy() case Comment: return v.Copy() case Directive: return v.Copy() case ProcInst: return v.Copy() case StartElement: return v.Copy() } return t } // A Decoder represents an XML parser reading a particular input stream. // The parser assumes that its input is encoded in UTF-8. type Decoder struct { // Strict defaults to true, enforcing the requirements // of the XML specification. // If set to false, the parser allows input containing common // mistakes: // * If an element is missing an end tag, the parser invents // end tags as necessary to keep the return values from Token // properly balanced. // * In attribute values and character data, unknown or malformed // character entities (sequences beginning with &) are left alone. // // Setting: // // d.Strict = false; // d.AutoClose = HTMLAutoClose; // d.Entity = HTMLEntity // // creates a parser that can handle typical HTML. // // Strict mode does not enforce the requirements of the XML name spaces TR. // In particular it does not reject name space tags using undefined prefixes. // Such tags are recorded with the unknown prefix as the name space URL. Strict bool // When Strict == false, AutoClose indicates a set of elements to // consider closed immediately after they are opened, regardless // of whether an end element is present. AutoClose []string // Entity can be used to map non-standard entity names to string replacements. // The parser behaves as if these standard mappings are present in the map, // regardless of the actual map content: // // "lt": "<", // "gt": ">", // "amp": "&", // "apos": "'", // "quot": `"`, Entity map[string]string // CharsetReader, if non-nil, defines a function to generate // charset-conversion readers, converting from the provided // non-UTF-8 charset into UTF-8. If CharsetReader is nil or // returns an error, parsing stops with an error. One of the // the CharsetReader's result values must be non-nil. CharsetReader func(charset string, input io.Reader) (io.Reader, error) // DefaultSpace sets the default name space used for unadorned tags, // as if the entire XML stream were wrapped in an element containing // the attribute xmlns="DefaultSpace". DefaultSpace string r io.ByteReader buf bytes.Buffer saved *bytes.Buffer stk *stack free *stack needClose bool toClose Name nextToken Token nextByte int ns map[string]string err error line int offset int64 unmarshalDepth int } // NewDecoder creates a new XML parser reading from r. // If r does not implement io.ByteReader, NewDecoder will // do its own buffering. func NewDecoder(r io.Reader) *Decoder { d := &Decoder{ ns: make(map[string]string), nextByte: -1, line: 1, Strict: true, } d.switchToReader(r) return d } // Token returns the next XML token in the input stream. // At the end of the input stream, Token returns nil, io.EOF. // // Slices of bytes in the returned token data refer to the // parser's internal buffer and remain valid only until the next // call to Token. To acquire a copy of the bytes, call CopyToken // or the token's Copy method. // // Token expands self-closing elements such as
// into separate start and end elements returned by successive calls. // // Token guarantees that the StartElement and EndElement // tokens it returns are properly nested and matched: // if Token encounters an unexpected end element, // it will return an error. // // Token implements XML name spaces as described by // http://www.w3.org/TR/REC-xml-names/. Each of the // Name structures contained in the Token has the Space // set to the URL identifying its name space when known. // If Token encounters an unrecognized name space prefix, // it uses the prefix as the Space rather than report an error. func (d *Decoder) Token() (t Token, err error) { if d.stk != nil && d.stk.kind == stkEOF { err = io.EOF return } if d.nextToken != nil { t = d.nextToken d.nextToken = nil } else if t, err = d.rawToken(); err != nil { return } if !d.Strict { if t1, ok := d.autoClose(t); ok { d.nextToken = t t = t1 } } switch t1 := t.(type) { case StartElement: // In XML name spaces, the translations listed in the // attributes apply to the element name and // to the other attribute names, so process // the translations first. for _, a := range t1.Attr { if a.Name.Space == "xmlns" { v, ok := d.ns[a.Name.Local] d.pushNs(a.Name.Local, v, ok) d.ns[a.Name.Local] = a.Value } if a.Name.Space == "" && a.Name.Local == "xmlns" { // Default space for untagged names v, ok := d.ns[""] d.pushNs("", v, ok) d.ns[""] = a.Value } } d.translate(&t1.Name, true) for i := range t1.Attr { d.translate(&t1.Attr[i].Name, false) } d.pushElement(t1.Name) t = t1 case EndElement: d.translate(&t1.Name, true) if !d.popElement(&t1) { return nil, d.err } t = t1 } return } const xmlURL = "http://www.w3.org/XML/1998/namespace" // Apply name space translation to name n. // The default name space (for Space=="") // applies only to element names, not to attribute names. func (d *Decoder) translate(n *Name, isElementName bool) { switch { case n.Space == "xmlns": return case n.Space == "" && !isElementName: return case n.Space == "xml": n.Space = xmlURL case n.Space == "" && n.Local == "xmlns": return } if v, ok := d.ns[n.Space]; ok { n.Space = v } else if n.Space == "" { n.Space = d.DefaultSpace } } func (d *Decoder) switchToReader(r io.Reader) { // Get efficient byte at a time reader. // Assume that if reader has its own // ReadByte, it's efficient enough. // Otherwise, use bufio. if rb, ok := r.(io.ByteReader); ok { d.r = rb } else { d.r = bufio.NewReader(r) } } // Parsing state - stack holds old name space translations // and the current set of open elements. The translations to pop when // ending a given tag are *below* it on the stack, which is // more work but forced on us by XML. type stack struct { next *stack kind int name Name ok bool } const ( stkStart = iota stkNs stkEOF ) func (d *Decoder) push(kind int) *stack { s := d.free if s != nil { d.free = s.next } else { s = new(stack) } s.next = d.stk s.kind = kind d.stk = s return s } func (d *Decoder) pop() *stack { s := d.stk if s != nil { d.stk = s.next s.next = d.free d.free = s } return s } // Record that after the current element is finished // (that element is already pushed on the stack) // Token should return EOF until popEOF is called. func (d *Decoder) pushEOF() { // Walk down stack to find Start. // It might not be the top, because there might be stkNs // entries above it. start := d.stk for start.kind != stkStart { start = start.next } // The stkNs entries below a start are associated with that // element too; skip over them. for start.next != nil && start.next.kind == stkNs { start = start.next } s := d.free if s != nil { d.free = s.next } else { s = new(stack) } s.kind = stkEOF s.next = start.next start.next = s } // Undo a pushEOF. // The element must have been finished, so the EOF should be at the top of the stack. func (d *Decoder) popEOF() bool { if d.stk == nil || d.stk.kind != stkEOF { return false } d.pop() return true } // Record that we are starting an element with the given name. func (d *Decoder) pushElement(name Name) { s := d.push(stkStart) s.name = name } // Record that we are changing the value of ns[local]. // The old value is url, ok. func (d *Decoder) pushNs(local string, url string, ok bool) { s := d.push(stkNs) s.name.Local = local s.name.Space = url s.ok = ok } // Creates a SyntaxError with the current line number. func (d *Decoder) syntaxError(msg string) error { return &SyntaxError{Msg: msg, Line: d.line} } // Record that we are ending an element with the given name. // The name must match the record at the top of the stack, // which must be a pushElement record. // After popping the element, apply any undo records from // the stack to restore the name translations that existed // before we saw this element. func (d *Decoder) popElement(t *EndElement) bool { s := d.pop() name := t.Name switch { case s == nil || s.kind != stkStart: d.err = d.syntaxError("unexpected end element ") return false case s.name.Local != name.Local: if !d.Strict { d.needClose = true d.toClose = t.Name t.Name = s.name return true } d.err = d.syntaxError("element <" + s.name.Local + "> closed by ") return false case s.name.Space != name.Space: d.err = d.syntaxError("element <" + s.name.Local + "> in space " + s.name.Space + "closed by in space " + name.Space) return false } // Pop stack until a Start or EOF is on the top, undoing the // translations that were associated with the element we just closed. for d.stk != nil && d.stk.kind != stkStart && d.stk.kind != stkEOF { s := d.pop() if s.ok { d.ns[s.name.Local] = s.name.Space } else { delete(d.ns, s.name.Local) } } return true } // If the top element on the stack is autoclosing and // t is not the end tag, invent the end tag. func (d *Decoder) autoClose(t Token) (Token, bool) { if d.stk == nil || d.stk.kind != stkStart { return nil, false } name := strings.ToLower(d.stk.name.Local) for _, s := range d.AutoClose { if strings.ToLower(s) == name { // This one should be auto closed if t doesn't close it. et, ok := t.(EndElement) if !ok || et.Name.Local != name { return EndElement{d.stk.name}, true } break } } return nil, false } var errRawToken = errors.New("xml: cannot use RawToken from UnmarshalXML method") // RawToken is like Token but does not verify that // start and end elements match and does not translate // name space prefixes to their corresponding URLs. func (d *Decoder) RawToken() (Token, error) { if d.unmarshalDepth > 0 { return nil, errRawToken } return d.rawToken() } func (d *Decoder) rawToken() (Token, error) { if d.err != nil { return nil, d.err } if d.needClose { // The last element we read was self-closing and // we returned just the StartElement half. // Return the EndElement half now. d.needClose = false return EndElement{d.toClose}, nil } b, ok := d.getc() if !ok { return nil, d.err } if b != '<' { // Text section. d.ungetc(b) data := d.text(-1, false) if data == nil { return nil, d.err } return CharData(data), nil } if b, ok = d.mustgetc(); !ok { return nil, d.err } switch b { case '/': // ' { d.err = d.syntaxError("invalid characters between ") return nil, d.err } return EndElement{name}, nil case '?': // ' { break } b0 = b } data := d.buf.Bytes() data = data[0 : len(data)-2] // chop ?> if target == "xml" { content := string(data) ver := procInst("version", content) if ver != "" && ver != "1.0" { d.err = fmt.Errorf("xml: unsupported version %q; only version 1.0 is supported", ver) return nil, d.err } enc := procInst("encoding", content) if enc != "" && enc != "utf-8" && enc != "UTF-8" { if d.CharsetReader == nil { d.err = fmt.Errorf("xml: encoding %q declared but Decoder.CharsetReader is nil", enc) return nil, d.err } newr, err := d.CharsetReader(enc, d.r.(io.Reader)) if err != nil { d.err = fmt.Errorf("xml: opening charset %q: %v", enc, err) return nil, d.err } if newr == nil { panic("CharsetReader returned a nil Reader for charset " + enc) } d.switchToReader(newr) } } return ProcInst{target, data}, nil case '!': // ' { break } b0, b1 = b1, b } data := d.buf.Bytes() data = data[0 : len(data)-3] // chop --> return Comment(data), nil case '[': // . data := d.text(-1, true) if data == nil { return nil, d.err } return CharData(data), nil } // Probably a directive: , , etc. // We don't care, but accumulate for caller. Quoted angle // brackets do not count for nesting. d.buf.Reset() d.buf.WriteByte(b) inquote := uint8(0) depth := 0 for { if b, ok = d.mustgetc(); !ok { return nil, d.err } if inquote == 0 && b == '>' && depth == 0 { break } HandleB: d.buf.WriteByte(b) switch { case b == inquote: inquote = 0 case inquote != 0: // in quotes, no special action case b == '\'' || b == '"': inquote = b case b == '>' && inquote == 0: depth-- case b == '<' && inquote == 0: // Look for ` var testEntity = map[string]string{"何": "What", "is-it": "is it?"} var rawTokens = []Token{ CharData("\n"), ProcInst{"xml", []byte(`version="1.0" encoding="UTF-8"`)}, CharData("\n"), Directive(`DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"`), CharData("\n"), StartElement{Name{"", "body"}, []Attr{{Name{"xmlns", "foo"}, "ns1"}, {Name{"", "xmlns"}, "ns2"}, {Name{"xmlns", "tag"}, "ns3"}}}, CharData("\n "), StartElement{Name{"", "hello"}, []Attr{{Name{"", "lang"}, "en"}}}, CharData("World <>'\" 白鵬翔"), EndElement{Name{"", "hello"}}, CharData("\n "), StartElement{Name{"", "query"}, []Attr{}}, CharData("What is it?"), EndElement{Name{"", "query"}}, CharData("\n "), StartElement{Name{"", "goodbye"}, []Attr{}}, EndElement{Name{"", "goodbye"}}, CharData("\n "), StartElement{Name{"", "outer"}, []Attr{{Name{"foo", "attr"}, "value"}, {Name{"xmlns", "tag"}, "ns4"}}}, CharData("\n "), StartElement{Name{"", "inner"}, []Attr{}}, EndElement{Name{"", "inner"}}, CharData("\n "), EndElement{Name{"", "outer"}}, CharData("\n "), StartElement{Name{"tag", "name"}, []Attr{}}, CharData("\n "), CharData("Some text here."), CharData("\n "), EndElement{Name{"tag", "name"}}, CharData("\n"), EndElement{Name{"", "body"}}, Comment(" missing final newline "), } var cookedTokens = []Token{ CharData("\n"), ProcInst{"xml", []byte(`version="1.0" encoding="UTF-8"`)}, CharData("\n"), Directive(`DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"`), CharData("\n"), StartElement{Name{"ns2", "body"}, []Attr{{Name{"xmlns", "foo"}, "ns1"}, {Name{"", "xmlns"}, "ns2"}, {Name{"xmlns", "tag"}, "ns3"}}}, CharData("\n "), StartElement{Name{"ns2", "hello"}, []Attr{{Name{"", "lang"}, "en"}}}, CharData("World <>'\" 白鵬翔"), EndElement{Name{"ns2", "hello"}}, CharData("\n "), StartElement{Name{"ns2", "query"}, []Attr{}}, CharData("What is it?"), EndElement{Name{"ns2", "query"}}, CharData("\n "), StartElement{Name{"ns2", "goodbye"}, []Attr{}}, EndElement{Name{"ns2", "goodbye"}}, CharData("\n "), StartElement{Name{"ns2", "outer"}, []Attr{{Name{"ns1", "attr"}, "value"}, {Name{"xmlns", "tag"}, "ns4"}}}, CharData("\n "), StartElement{Name{"ns2", "inner"}, []Attr{}}, EndElement{Name{"ns2", "inner"}}, CharData("\n "), EndElement{Name{"ns2", "outer"}}, CharData("\n "), StartElement{Name{"ns3", "name"}, []Attr{}}, CharData("\n "), CharData("Some text here."), CharData("\n "), EndElement{Name{"ns3", "name"}}, CharData("\n"), EndElement{Name{"ns2", "body"}}, Comment(" missing final newline "), } const testInputAltEncoding = ` VALUE` var rawTokensAltEncoding = []Token{ CharData("\n"), ProcInst{"xml", []byte(`version="1.0" encoding="x-testing-uppercase"`)}, CharData("\n"), StartElement{Name{"", "tag"}, []Attr{}}, CharData("value"), EndElement{Name{"", "tag"}}, } var xmlInput = []string{ // unexpected EOF cases "<", "", "", "", // "", // let the Token() caller handle "", "", "", "", " c;", "", "", "", // "", // let the Token() caller handle "", "", "cdata]]>", } func TestRawToken(t *testing.T) { d := NewDecoder(strings.NewReader(testInput)) d.Entity = testEntity testRawToken(t, d, testInput, rawTokens) } const nonStrictInput = ` non&entity &unknown;entity { &#zzz; &なまえ3; <-gt; &; &0a; ` var nonStringEntity = map[string]string{"": "oops!", "0a": "oops!"} var nonStrictTokens = []Token{ CharData("\n"), StartElement{Name{"", "tag"}, []Attr{}}, CharData("non&entity"), EndElement{Name{"", "tag"}}, CharData("\n"), StartElement{Name{"", "tag"}, []Attr{}}, CharData("&unknown;entity"), EndElement{Name{"", "tag"}}, CharData("\n"), StartElement{Name{"", "tag"}, []Attr{}}, CharData("{"), EndElement{Name{"", "tag"}}, CharData("\n"), StartElement{Name{"", "tag"}, []Attr{}}, CharData("&#zzz;"), EndElement{Name{"", "tag"}}, CharData("\n"), StartElement{Name{"", "tag"}, []Attr{}}, CharData("&なまえ3;"), EndElement{Name{"", "tag"}}, CharData("\n"), StartElement{Name{"", "tag"}, []Attr{}}, CharData("<-gt;"), EndElement{Name{"", "tag"}}, CharData("\n"), StartElement{Name{"", "tag"}, []Attr{}}, CharData("&;"), EndElement{Name{"", "tag"}}, CharData("\n"), StartElement{Name{"", "tag"}, []Attr{}}, CharData("&0a;"), EndElement{Name{"", "tag"}}, CharData("\n"), } func TestNonStrictRawToken(t *testing.T) { d := NewDecoder(strings.NewReader(nonStrictInput)) d.Strict = false testRawToken(t, d, nonStrictInput, nonStrictTokens) } type downCaser struct { t *testing.T r io.ByteReader } func (d *downCaser) ReadByte() (c byte, err error) { c, err = d.r.ReadByte() if c >= 'A' && c <= 'Z' { c += 'a' - 'A' } return } func (d *downCaser) Read(p []byte) (int, error) { d.t.Fatalf("unexpected Read call on downCaser reader") panic("unreachable") } func TestRawTokenAltEncoding(t *testing.T) { d := NewDecoder(strings.NewReader(testInputAltEncoding)) d.CharsetReader = func(charset string, input io.Reader) (io.Reader, error) { if charset != "x-testing-uppercase" { t.Fatalf("unexpected charset %q", charset) } return &downCaser{t, input.(io.ByteReader)}, nil } testRawToken(t, d, testInputAltEncoding, rawTokensAltEncoding) } func TestRawTokenAltEncodingNoConverter(t *testing.T) { d := NewDecoder(strings.NewReader(testInputAltEncoding)) token, err := d.RawToken() if token == nil { t.Fatalf("expected a token on first RawToken call") } if err != nil { t.Fatal(err) } token, err = d.RawToken() if token != nil { t.Errorf("expected a nil token; got %#v", token) } if err == nil { t.Fatalf("expected an error on second RawToken call") } const encoding = "x-testing-uppercase" if !strings.Contains(err.Error(), encoding) { t.Errorf("expected error to contain %q; got error: %v", encoding, err) } } func testRawToken(t *testing.T, d *Decoder, raw string, rawTokens []Token) { lastEnd := int64(0) for i, want := range rawTokens { start := d.InputOffset() have, err := d.RawToken() end := d.InputOffset() if err != nil { t.Fatalf("token %d: unexpected error: %s", i, err) } if !reflect.DeepEqual(have, want) { var shave, swant string if _, ok := have.(CharData); ok { shave = fmt.Sprintf("CharData(%q)", have) } else { shave = fmt.Sprintf("%#v", have) } if _, ok := want.(CharData); ok { swant = fmt.Sprintf("CharData(%q)", want) } else { swant = fmt.Sprintf("%#v", want) } t.Errorf("token %d = %s, want %s", i, shave, swant) } // Check that InputOffset returned actual token. switch { case start < lastEnd: t.Errorf("token %d: position [%d,%d) for %T is before previous token", i, start, end, have) case start >= end: // Special case: EndElement can be synthesized. if start == end && end == lastEnd { break } t.Errorf("token %d: position [%d,%d) for %T is empty", i, start, end, have) case end > int64(len(raw)): t.Errorf("token %d: position [%d,%d) for %T extends beyond input", i, start, end, have) default: text := raw[start:end] if strings.ContainsAny(text, "<>") && (!strings.HasPrefix(text, "<") || !strings.HasSuffix(text, ">")) { t.Errorf("token %d: misaligned raw token %#q for %T", i, text, have) } } lastEnd = end } } // Ensure that directives (specifically !DOCTYPE) include the complete // text of any nested directives, noting that < and > do not change // nesting depth if they are in single or double quotes. var nestedDirectivesInput = ` ]> ">]> ]> '>]> ]> '>]> ]> ` var nestedDirectivesTokens = []Token{ CharData("\n"), Directive(`DOCTYPE []`), CharData("\n"), Directive(`DOCTYPE [">]`), CharData("\n"), Directive(`DOCTYPE []`), CharData("\n"), Directive(`DOCTYPE ['>]`), CharData("\n"), Directive(`DOCTYPE []`), CharData("\n"), Directive(`DOCTYPE ['>]`), CharData("\n"), Directive(`DOCTYPE []`), CharData("\n"), } func TestNestedDirectives(t *testing.T) { d := NewDecoder(strings.NewReader(nestedDirectivesInput)) for i, want := range nestedDirectivesTokens { have, err := d.Token() if err != nil { t.Fatalf("token %d: unexpected error: %s", i, err) } if !reflect.DeepEqual(have, want) { t.Errorf("token %d = %#v want %#v", i, have, want) } } } func TestToken(t *testing.T) { d := NewDecoder(strings.NewReader(testInput)) d.Entity = testEntity for i, want := range cookedTokens { have, err := d.Token() if err != nil { t.Fatalf("token %d: unexpected error: %s", i, err) } if !reflect.DeepEqual(have, want) { t.Errorf("token %d = %#v want %#v", i, have, want) } } } func TestSyntax(t *testing.T) { for i := range xmlInput { d := NewDecoder(strings.NewReader(xmlInput[i])) var err error for _, err = d.Token(); err == nil; _, err = d.Token() { } if _, ok := err.(*SyntaxError); !ok { t.Fatalf(`xmlInput "%s": expected SyntaxError not received`, xmlInput[i]) } } } type allScalars struct { True1 bool True2 bool False1 bool False2 bool Int int Int8 int8 Int16 int16 Int32 int32 Int64 int64 Uint int Uint8 uint8 Uint16 uint16 Uint32 uint32 Uint64 uint64 Uintptr uintptr Float32 float32 Float64 float64 String string PtrString *string } var all = allScalars{ True1: true, True2: true, False1: false, False2: false, Int: 1, Int8: -2, Int16: 3, Int32: -4, Int64: 5, Uint: 6, Uint8: 7, Uint16: 8, Uint32: 9, Uint64: 10, Uintptr: 11, Float32: 13.0, Float64: 14.0, String: "15", PtrString: &sixteen, } var sixteen = "16" const testScalarsInput = ` true 1 false 0 1 -2 3 -4 5 6 7 8 9 10 11 12.0 13.0 14.0 15 16 ` func TestAllScalars(t *testing.T) { var a allScalars err := Unmarshal([]byte(testScalarsInput), &a) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(a, all) { t.Errorf("have %+v want %+v", a, all) } } type item struct { Field_a string } func TestIssue569(t *testing.T) { data := `abcd` var i item err := Unmarshal([]byte(data), &i) if err != nil || i.Field_a != "abcd" { t.Fatal("Expecting abcd") } } func TestUnquotedAttrs(t *testing.T) { data := "" d := NewDecoder(strings.NewReader(data)) d.Strict = false token, err := d.Token() if _, ok := err.(*SyntaxError); ok { t.Errorf("Unexpected error: %v", err) } if token.(StartElement).Name.Local != "tag" { t.Errorf("Unexpected tag name: %v", token.(StartElement).Name.Local) } attr := token.(StartElement).Attr[0] if attr.Value != "azAZ09:-_" { t.Errorf("Unexpected attribute value: %v", attr.Value) } if attr.Name.Local != "attr" { t.Errorf("Unexpected attribute name: %v", attr.Name.Local) } } func TestValuelessAttrs(t *testing.T) { tests := [][3]string{ {"

", "p", "nowrap"}, {"

", "p", "nowrap"}, {"", "input", "checked"}, {"", "input", "checked"}, } for _, test := range tests { d := NewDecoder(strings.NewReader(test[0])) d.Strict = false token, err := d.Token() if _, ok := err.(*SyntaxError); ok { t.Errorf("Unexpected error: %v", err) } if token.(StartElement).Name.Local != test[1] { t.Errorf("Unexpected tag name: %v", token.(StartElement).Name.Local) } attr := token.(StartElement).Attr[0] if attr.Value != test[2] { t.Errorf("Unexpected attribute value: %v", attr.Value) } if attr.Name.Local != test[2] { t.Errorf("Unexpected attribute name: %v", attr.Name.Local) } } } func TestCopyTokenCharData(t *testing.T) { data := []byte("same data") var tok1 Token = CharData(data) tok2 := CopyToken(tok1) if !reflect.DeepEqual(tok1, tok2) { t.Error("CopyToken(CharData) != CharData") } data[1] = 'o' if reflect.DeepEqual(tok1, tok2) { t.Error("CopyToken(CharData) uses same buffer.") } } func TestCopyTokenStartElement(t *testing.T) { elt := StartElement{Name{"", "hello"}, []Attr{{Name{"", "lang"}, "en"}}} var tok1 Token = elt tok2 := CopyToken(tok1) if tok1.(StartElement).Attr[0].Value != "en" { t.Error("CopyToken overwrote Attr[0]") } if !reflect.DeepEqual(tok1, tok2) { t.Error("CopyToken(StartElement) != StartElement") } tok1.(StartElement).Attr[0] = Attr{Name{"", "lang"}, "de"} if reflect.DeepEqual(tok1, tok2) { t.Error("CopyToken(CharData) uses same buffer.") } } func TestSyntaxErrorLineNum(t *testing.T) { testInput := "

Foo

\n\n

Bar\n" d := NewDecoder(strings.NewReader(testInput)) var err error for _, err = d.Token(); err == nil; _, err = d.Token() { } synerr, ok := err.(*SyntaxError) if !ok { t.Error("Expected SyntaxError.") } if synerr.Line != 3 { t.Error("SyntaxError didn't have correct line number.") } } func TestTrailingRawToken(t *testing.T) { input := ` ` d := NewDecoder(strings.NewReader(input)) var err error for _, err = d.RawToken(); err == nil; _, err = d.RawToken() { } if err != io.EOF { t.Fatalf("d.RawToken() = _, %v, want _, io.EOF", err) } } func TestTrailingToken(t *testing.T) { input := ` ` d := NewDecoder(strings.NewReader(input)) var err error for _, err = d.Token(); err == nil; _, err = d.Token() { } if err != io.EOF { t.Fatalf("d.Token() = _, %v, want _, io.EOF", err) } } func TestEntityInsideCDATA(t *testing.T) { input := `` d := NewDecoder(strings.NewReader(input)) var err error for _, err = d.Token(); err == nil; _, err = d.Token() { } if err != io.EOF { t.Fatalf("d.Token() = _, %v, want _, io.EOF", err) } } var characterTests = []struct { in string err string }{ {"\x12", "illegal character code U+0012"}, {"\x0b", "illegal character code U+000B"}, {"\xef\xbf\xbe", "illegal character code U+FFFE"}, {"\r\n\x07", "illegal character code U+0007"}, {"what's up", "expected attribute name in element"}, {"&abc\x01;", "invalid character entity &abc (no semicolon)"}, {"&\x01;", "invalid character entity & (no semicolon)"}, {"&\xef\xbf\xbe;", "invalid character entity &\uFFFE;"}, {"&hello;", "invalid character entity &hello;"}, } func TestDisallowedCharacters(t *testing.T) { for i, tt := range characterTests { d := NewDecoder(strings.NewReader(tt.in)) var err error for err == nil { _, err = d.Token() } synerr, ok := err.(*SyntaxError) if !ok { t.Fatalf("input %d d.Token() = _, %v, want _, *SyntaxError", i, err) } if synerr.Msg != tt.err { t.Fatalf("input %d synerr.Msg wrong: want %q, got %q", i, tt.err, synerr.Msg) } } } type procInstEncodingTest struct { expect, got string } var procInstTests = []struct { input string expect [2]string }{ {`version="1.0" encoding="utf-8"`, [2]string{"1.0", "utf-8"}}, {`version="1.0" encoding='utf-8'`, [2]string{"1.0", "utf-8"}}, {`version="1.0" encoding='utf-8' `, [2]string{"1.0", "utf-8"}}, {`version="1.0" encoding=utf-8`, [2]string{"1.0", ""}}, {`encoding="FOO" `, [2]string{"", "FOO"}}, } func TestProcInstEncoding(t *testing.T) { for _, test := range procInstTests { if got := procInst("version", test.input); got != test.expect[0] { t.Errorf("procInst(version, %q) = %q; want %q", test.input, got, test.expect[0]) } if got := procInst("encoding", test.input); got != test.expect[1] { t.Errorf("procInst(encoding, %q) = %q; want %q", test.input, got, test.expect[1]) } } } // Ensure that directives with comments include the complete // text of any nested directives. var directivesWithCommentsInput = ` ]> ]> --> --> []> ` var directivesWithCommentsTokens = []Token{ CharData("\n"), Directive(`DOCTYPE []`), CharData("\n"), Directive(`DOCTYPE []`), CharData("\n"), Directive(`DOCTYPE []`), CharData("\n"), } func TestDirectivesWithComments(t *testing.T) { d := NewDecoder(strings.NewReader(directivesWithCommentsInput)) for i, want := range directivesWithCommentsTokens { have, err := d.Token() if err != nil { t.Fatalf("token %d: unexpected error: %s", i, err) } if !reflect.DeepEqual(have, want) { t.Errorf("token %d = %#v want %#v", i, have, want) } } } // Writer whose Write method always returns an error. type errWriter struct{} func (errWriter) Write(p []byte) (n int, err error) { return 0, fmt.Errorf("unwritable") } func TestEscapeTextIOErrors(t *testing.T) { expectErr := "unwritable" err := EscapeText(errWriter{}, []byte{'A'}) if err == nil || err.Error() != expectErr { t.Errorf("have %v, want %v", err, expectErr) } } func TestEscapeTextInvalidChar(t *testing.T) { input := []byte("A \x00 terminated string.") expected := "A \uFFFD terminated string." buff := new(bytes.Buffer) if err := EscapeText(buff, input); err != nil { t.Fatalf("have %v, want nil", err) } text := buff.String() if text != expected { t.Errorf("have %v, want %v", text, expected) } } func TestIssue5880(t *testing.T) { type T []byte data, err := Marshal(T{192, 168, 0, 1}) if err != nil { t.Errorf("Marshal error: %v", err) } if !utf8.Valid(data) { t.Errorf("Marshal generated invalid UTF-8: %x", data) } } ================================================ FILE: server/webdav/litmus_test_server.go ================================================ // Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. //go:build ignore // +build ignore /* This program is a server for the WebDAV 'litmus' compliance test at http://www.webdav.org/neon/litmus/ To run the test: go run litmus_test_server.go and separately, from the downloaded litmus-xxx directory: make URL=http://localhost:9999/ check */ package main import ( "flag" "fmt" "log" "net/http" "net/url" "golang.org/x/net/webdav" ) var port = flag.Int("port", 9999, "server port") func main() { flag.Parse() log.SetFlags(0) h := &webdav.Handler{ FileSystem: webdav.NewMemFS(), LockSystem: webdav.NewMemLS(), Logger: func(r *http.Request, err error) { litmus := r.Header.Get("X-Litmus") if len(litmus) > 19 { litmus = litmus[:16] + "..." } switch r.Method { case "COPY", "MOVE": dst := "" if u, err := url.Parse(r.Header.Get("Destination")); err == nil { dst = u.Path } o := r.Header.Get("Overwrite") log.Printf("%-20s%-10s%-30s%-30so=%-2s%v", litmus, r.Method, r.URL.Path, dst, o, err) default: log.Printf("%-20s%-10s%-30s%v", litmus, r.Method, r.URL.Path, err) } }, } // The next line would normally be: // http.Handle("/", h) // but we wrap that HTTP handler h to cater for a special case. // // The propfind_invalid2 litmus test case expects an empty namespace prefix // declaration to be an error. The FAQ in the webdav litmus test says: // // "What does the "propfind_invalid2" test check for?... // // If a request was sent with an XML body which included an empty namespace // prefix declaration (xmlns:ns1=""), then the server must reject that with // a "400 Bad Request" response, as it is invalid according to the XML // Namespace specification." // // On the other hand, the Go standard library's encoding/xml package // accepts an empty xmlns namespace, as per the discussion at // https://github.com/golang/go/issues/8068 // // Empty namespaces seem disallowed in the second (2006) edition of the XML // standard, but allowed in a later edition. The grammar differs between // http://www.w3.org/TR/2006/REC-xml-names-20060816/#ns-decl and // http://www.w3.org/TR/REC-xml-names/#dt-prefix // // Thus, we assume that the propfind_invalid2 test is obsolete, and // hard-code the 400 Bad Request response that the test expects. http.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("X-Litmus") == "props: 3 (propfind_invalid2)" { http.Error(w, "400 Bad Request", http.StatusBadRequest) return } h.ServeHTTP(w, r) })) addr := fmt.Sprintf(":%d", *port) log.Printf("Serving %v", addr) log.Fatal(http.ListenAndServe(addr, nil)) } ================================================ FILE: server/webdav/lock.go ================================================ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package webdav import ( "container/heap" "errors" "strconv" "strings" "sync" "time" ) var ( // ErrConfirmationFailed is returned by a LockSystem's Confirm method. ErrConfirmationFailed = errors.New("webdav: confirmation failed") // ErrForbidden is returned by a LockSystem's Unlock method. ErrForbidden = errors.New("webdav: forbidden") // ErrLocked is returned by a LockSystem's Create, Refresh and Unlock methods. ErrLocked = errors.New("webdav: locked") // ErrNoSuchLock is returned by a LockSystem's Refresh and Unlock methods. ErrNoSuchLock = errors.New("webdav: no such lock") ) // Condition can match a WebDAV resource, based on a token or ETag. // Exactly one of Token and ETag should be non-empty. type Condition struct { Not bool Token string ETag string } // LockSystem manages access to a collection of named resources. The elements // in a lock name are separated by slash ('/', U+002F) characters, regardless // of host operating system convention. type LockSystem interface { // Confirm confirms that the caller can claim all of the locks specified by // the given conditions, and that holding the union of all of those locks // gives exclusive access to all of the named resources. Up to two resources // can be named. Empty names are ignored. // // Exactly one of release and err will be non-nil. If release is non-nil, // all of the requested locks are held until release is called. Calling // release does not unlock the lock, in the WebDAV UNLOCK sense, but once // Confirm has confirmed that a lock claim is valid, that lock cannot be // Confirmed again until it has been released. // // If Confirm returns ErrConfirmationFailed then the Handler will continue // to try any other set of locks presented (a WebDAV HTTP request can // present more than one set of locks). If it returns any other non-nil // error, the Handler will write a "500 Internal Server Error" HTTP status. Confirm(now time.Time, name0, name1 string, conditions ...Condition) (release func(), err error) // Create creates a lock with the given depth, duration, owner and root // (name). The depth will either be negative (meaning infinite) or zero. // // If Create returns ErrLocked then the Handler will write a "423 Locked" // HTTP status. If it returns any other non-nil error, the Handler will // write a "500 Internal Server Error" HTTP status. // // See http://www.webdav.org/specs/rfc4918.html#rfc.section.9.10.6 for // when to use each error. // // The token returned identifies the created lock. It should be an absolute // URI as defined by RFC 3986, Section 4.3. In particular, it should not // contain whitespace. Create(now time.Time, details LockDetails) (token string, err error) // Refresh refreshes the lock with the given token. // // If Refresh returns ErrLocked then the Handler will write a "423 Locked" // HTTP Status. If Refresh returns ErrNoSuchLock then the Handler will write // a "412 Precondition Failed" HTTP Status. If it returns any other non-nil // error, the Handler will write a "500 Internal Server Error" HTTP status. // // See http://www.webdav.org/specs/rfc4918.html#rfc.section.9.10.6 for // when to use each error. Refresh(now time.Time, token string, duration time.Duration) (LockDetails, error) // Unlock unlocks the lock with the given token. // // If Unlock returns ErrForbidden then the Handler will write a "403 // Forbidden" HTTP Status. If Unlock returns ErrLocked then the Handler // will write a "423 Locked" HTTP status. If Unlock returns ErrNoSuchLock // then the Handler will write a "409 Conflict" HTTP Status. If it returns // any other non-nil error, the Handler will write a "500 Internal Server // Error" HTTP status. // // See http://www.webdav.org/specs/rfc4918.html#rfc.section.9.11.1 for // when to use each error. Unlock(now time.Time, token string) error } // LockDetails are a lock's metadata. type LockDetails struct { // Root is the root resource name being locked. For a zero-depth lock, the // root is the only resource being locked. Root string // Duration is the lock timeout. A negative duration means infinite. Duration time.Duration // OwnerXML is the verbatim XML given in a LOCK HTTP request. // // TODO: does the "verbatim" nature play well with XML namespaces? // Does the OwnerXML field need to have more structure? See // https://codereview.appspot.com/175140043/#msg2 OwnerXML string // ZeroDepth is whether the lock has zero depth. If it does not have zero // depth, it has infinite depth. ZeroDepth bool } // NewMemLS returns a new in-memory LockSystem. func NewMemLS() LockSystem { return &memLS{ byName: make(map[string]*memLSNode), byToken: make(map[string]*memLSNode), gen: uint64(time.Now().Unix()), } } type memLS struct { mu sync.Mutex byName map[string]*memLSNode byToken map[string]*memLSNode gen uint64 // byExpiry only contains those nodes whose LockDetails have a finite // Duration and are yet to expire. byExpiry byExpiry } func (m *memLS) nextToken() string { m.gen++ return strconv.FormatUint(m.gen, 10) } func (m *memLS) collectExpiredNodes(now time.Time) { for len(m.byExpiry) > 0 { if now.Before(m.byExpiry[0].expiry) { break } m.remove(m.byExpiry[0]) } } func (m *memLS) Confirm(now time.Time, name0, name1 string, conditions ...Condition) (func(), error) { m.mu.Lock() defer m.mu.Unlock() m.collectExpiredNodes(now) var n0, n1 *memLSNode if name0 != "" { if n0 = m.lookup(slashClean(name0), conditions...); n0 == nil { return nil, ErrConfirmationFailed } } if name1 != "" { if n1 = m.lookup(slashClean(name1), conditions...); n1 == nil { return nil, ErrConfirmationFailed } } // Don't hold the same node twice. if n1 == n0 { n1 = nil } if n0 != nil { m.hold(n0) } if n1 != nil { m.hold(n1) } return func() { m.mu.Lock() defer m.mu.Unlock() if n1 != nil { m.unhold(n1) } if n0 != nil { m.unhold(n0) } }, nil } // lookup returns the node n that locks the named resource, provided that n // matches at least one of the given conditions and that lock isn't held by // another party. Otherwise, it returns nil. // // n may be a parent of the named resource, if n is an infinite depth lock. func (m *memLS) lookup(name string, conditions ...Condition) (n *memLSNode) { // TODO: support Condition.Not and Condition.ETag. for _, c := range conditions { n = m.byToken[c.Token] if n == nil || n.held { continue } if name == n.details.Root { return n } if n.details.ZeroDepth { continue } if n.details.Root == "/" || strings.HasPrefix(name, n.details.Root+"/") { return n } } return nil } func (m *memLS) hold(n *memLSNode) { if n.held { panic("webdav: memLS inconsistent held state") } n.held = true if n.details.Duration >= 0 && n.byExpiryIndex >= 0 { heap.Remove(&m.byExpiry, n.byExpiryIndex) } } func (m *memLS) unhold(n *memLSNode) { if !n.held { panic("webdav: memLS inconsistent held state") } n.held = false if n.details.Duration >= 0 { heap.Push(&m.byExpiry, n) } } func (m *memLS) Create(now time.Time, details LockDetails) (string, error) { m.mu.Lock() defer m.mu.Unlock() m.collectExpiredNodes(now) details.Root = slashClean(details.Root) if !m.canCreate(details.Root, details.ZeroDepth) { return "", ErrLocked } n := m.create(details.Root) n.token = m.nextToken() m.byToken[n.token] = n n.details = details if n.details.Duration >= 0 { n.expiry = now.Add(n.details.Duration) heap.Push(&m.byExpiry, n) } return n.token, nil } func (m *memLS) Refresh(now time.Time, token string, duration time.Duration) (LockDetails, error) { m.mu.Lock() defer m.mu.Unlock() m.collectExpiredNodes(now) n := m.byToken[token] if n == nil { return LockDetails{}, ErrNoSuchLock } if n.held { return LockDetails{}, ErrLocked } if n.byExpiryIndex >= 0 { heap.Remove(&m.byExpiry, n.byExpiryIndex) } n.details.Duration = duration if n.details.Duration >= 0 { n.expiry = now.Add(n.details.Duration) heap.Push(&m.byExpiry, n) } return n.details, nil } func (m *memLS) Unlock(now time.Time, token string) error { m.mu.Lock() defer m.mu.Unlock() m.collectExpiredNodes(now) n := m.byToken[token] if n == nil { return ErrNoSuchLock } if n.held { return ErrLocked } m.remove(n) return nil } func (m *memLS) canCreate(name string, zeroDepth bool) bool { return walkToRoot(name, func(name0 string, first bool) bool { n := m.byName[name0] if n == nil { return true } if first { if n.token != "" { // The target node is already locked. return false } if !zeroDepth { // The requested lock depth is infinite, and the fact that n exists // (n != nil) means that a descendent of the target node is locked. return false } } else if n.token != "" && !n.details.ZeroDepth { // An ancestor of the target node is locked with infinite depth. return false } return true }) } func (m *memLS) create(name string) (ret *memLSNode) { walkToRoot(name, func(name0 string, first bool) bool { n := m.byName[name0] if n == nil { n = &memLSNode{ details: LockDetails{ Root: name0, }, byExpiryIndex: -1, } m.byName[name0] = n } n.refCount++ if first { ret = n } return true }) return ret } func (m *memLS) remove(n *memLSNode) { delete(m.byToken, n.token) n.token = "" walkToRoot(n.details.Root, func(name0 string, first bool) bool { x := m.byName[name0] x.refCount-- if x.refCount == 0 { delete(m.byName, name0) } return true }) if n.byExpiryIndex >= 0 { heap.Remove(&m.byExpiry, n.byExpiryIndex) } } func walkToRoot(name string, f func(name0 string, first bool) bool) bool { for first := true; ; first = false { if !f(name, first) { return false } if name == "/" { break } name = name[:strings.LastIndex(name, "/")] if name == "" { name = "/" } } return true } type memLSNode struct { // details are the lock metadata. Even if this node's name is not explicitly locked, // details.Root will still equal the node's name. details LockDetails // token is the unique identifier for this node's lock. An empty token means that // this node is not explicitly locked. token string // refCount is the number of self-or-descendent nodes that are explicitly locked. refCount int // expiry is when this node's lock expires. expiry time.Time // byExpiryIndex is the index of this node in memLS.byExpiry. It is -1 // if this node does not expire, or has expired. byExpiryIndex int // held is whether this node's lock is actively held by a Confirm call. held bool } type byExpiry []*memLSNode func (b *byExpiry) Len() int { return len(*b) } func (b *byExpiry) Less(i, j int) bool { return (*b)[i].expiry.Before((*b)[j].expiry) } func (b *byExpiry) Swap(i, j int) { (*b)[i], (*b)[j] = (*b)[j], (*b)[i] (*b)[i].byExpiryIndex = i (*b)[j].byExpiryIndex = j } func (b *byExpiry) Push(x interface{}) { n := x.(*memLSNode) n.byExpiryIndex = len(*b) *b = append(*b, n) } func (b *byExpiry) Pop() interface{} { i := len(*b) - 1 n := (*b)[i] (*b)[i] = nil n.byExpiryIndex = -1 *b = (*b)[:i] return n } const infiniteTimeout = -1 // parseTimeout parses the Timeout HTTP header, as per section 10.7. If s is // empty, an infiniteTimeout is returned. func parseTimeout(s string) (time.Duration, error) { if s == "" { return infiniteTimeout, nil } if i := strings.IndexByte(s, ','); i >= 0 { s = s[:i] } s = strings.TrimSpace(s) if s == "Infinite" { return infiniteTimeout, nil } const pre = "Second-" if !strings.HasPrefix(s, pre) { return 0, errInvalidTimeout } s = s[len(pre):] if s == "" || s[0] < '0' || '9' < s[0] { return 0, errInvalidTimeout } n, err := strconv.ParseInt(s, 10, 64) if err != nil || 1<<32-1 < n { return 0, errInvalidTimeout } return time.Duration(n) * time.Second, nil } ================================================ FILE: server/webdav/lock_test.go ================================================ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package webdav import ( "fmt" "math/rand" "path" "reflect" "sort" "strconv" "strings" "testing" "time" ) func TestWalkToRoot(t *testing.T) { testCases := []struct { name string want []string }{{ "/a/b/c/d", []string{ "/a/b/c/d", "/a/b/c", "/a/b", "/a", "/", }, }, { "/a", []string{ "/a", "/", }, }, { "/", []string{ "/", }, }} for _, tc := range testCases { var got []string if !walkToRoot(tc.name, func(name0 string, first bool) bool { if first != (len(got) == 0) { t.Errorf("name=%q: first=%t but len(got)==%d", tc.name, first, len(got)) return false } got = append(got, name0) return true }) { continue } if !reflect.DeepEqual(got, tc.want) { t.Errorf("name=%q:\ngot %q\nwant %q", tc.name, got, tc.want) } } } var lockTestDurations = []time.Duration{ infiniteTimeout, // infiniteTimeout means to never expire. 0, // A zero duration means to expire immediately. 100 * time.Hour, // A very large duration will not expire in these tests. } // lockTestNames are the names of a set of mutually compatible locks. For each // name fragment: // - _ means no explicit lock. // - i means an infinite-depth lock, // - z means a zero-depth lock, var lockTestNames = []string{ "/_/_/_/_/z", "/_/_/i", "/_/z", "/_/z/i", "/_/z/z", "/_/z/_/i", "/_/z/_/z", "/i", "/z", "/z/_/i", "/z/_/z", } func lockTestZeroDepth(name string) bool { switch name[len(name)-1] { case 'i': return false case 'z': return true } panic(fmt.Sprintf("lock name %q did not end with 'i' or 'z'", name)) } func TestMemLSCanCreate(t *testing.T) { now := time.Unix(0, 0) m := NewMemLS().(*memLS) for _, name := range lockTestNames { _, err := m.Create(now, LockDetails{ Root: name, Duration: infiniteTimeout, ZeroDepth: lockTestZeroDepth(name), }) if err != nil { t.Fatalf("creating lock for %q: %v", name, err) } } wantCanCreate := func(name string, zeroDepth bool) bool { for _, n := range lockTestNames { switch { case n == name: // An existing lock has the same name as the proposed lock. return false case strings.HasPrefix(n, name): // An existing lock would be a child of the proposed lock, // which conflicts if the proposed lock has infinite depth. if !zeroDepth { return false } case strings.HasPrefix(name, n): // An existing lock would be an ancestor of the proposed lock, // which conflicts if the ancestor has infinite depth. if n[len(n)-1] == 'i' { return false } } } return true } var check func(int, string) check = func(recursion int, name string) { for _, zeroDepth := range []bool{false, true} { got := m.canCreate(name, zeroDepth) want := wantCanCreate(name, zeroDepth) if got != want { t.Errorf("canCreate name=%q zeroDepth=%t: got %t, want %t", name, zeroDepth, got, want) } } if recursion == 6 { return } if name != "/" { name += "/" } for _, c := range "_iz" { check(recursion+1, name+string(c)) } } check(0, "/") } func TestMemLSLookup(t *testing.T) { now := time.Unix(0, 0) m := NewMemLS().(*memLS) badToken := m.nextToken() t.Logf("badToken=%q", badToken) for _, name := range lockTestNames { token, err := m.Create(now, LockDetails{ Root: name, Duration: infiniteTimeout, ZeroDepth: lockTestZeroDepth(name), }) if err != nil { t.Fatalf("creating lock for %q: %v", name, err) } t.Logf("%-15q -> node=%p token=%q", name, m.byName[name], token) } baseNames := append([]string{"/a", "/b/c"}, lockTestNames...) for _, baseName := range baseNames { for _, suffix := range []string{"", "/0", "/1/2/3"} { name := baseName + suffix goodToken := "" base := m.byName[baseName] if base != nil && (suffix == "" || !lockTestZeroDepth(baseName)) { goodToken = base.token } for _, token := range []string{badToken, goodToken} { if token == "" { continue } got := m.lookup(name, Condition{Token: token}) want := base if token == badToken { want = nil } if got != want { t.Errorf("name=%-20qtoken=%q (bad=%t): got %p, want %p", name, token, token == badToken, got, want) } } } } } func TestMemLSConfirm(t *testing.T) { now := time.Unix(0, 0) m := NewMemLS().(*memLS) alice, err := m.Create(now, LockDetails{ Root: "/alice", Duration: infiniteTimeout, ZeroDepth: false, }) if err != nil { t.Fatalf("Create: %v", err) } tweedle, err := m.Create(now, LockDetails{ Root: "/tweedle", Duration: infiniteTimeout, ZeroDepth: false, }) if err != nil { t.Fatalf("Create: %v", err) } if err := m.consistent(); err != nil { t.Fatalf("Create: inconsistent state: %v", err) } // Test a mismatch between name and condition. _, err = m.Confirm(now, "/tweedle/dee", "", Condition{Token: alice}) if err != ErrConfirmationFailed { t.Fatalf("Confirm (mismatch): got %v, want ErrConfirmationFailed", err) } if err := m.consistent(); err != nil { t.Fatalf("Confirm (mismatch): inconsistent state: %v", err) } // Test two names (that fall under the same lock) in the one Confirm call. release, err := m.Confirm(now, "/tweedle/dee", "/tweedle/dum", Condition{Token: tweedle}) if err != nil { t.Fatalf("Confirm (twins): %v", err) } if err := m.consistent(); err != nil { t.Fatalf("Confirm (twins): inconsistent state: %v", err) } release() if err := m.consistent(); err != nil { t.Fatalf("release (twins): inconsistent state: %v", err) } // Test the same two names in overlapping Confirm / release calls. releaseDee, err := m.Confirm(now, "/tweedle/dee", "", Condition{Token: tweedle}) if err != nil { t.Fatalf("Confirm (sequence #0): %v", err) } if err := m.consistent(); err != nil { t.Fatalf("Confirm (sequence #0): inconsistent state: %v", err) } _, err = m.Confirm(now, "/tweedle/dum", "", Condition{Token: tweedle}) if err != ErrConfirmationFailed { t.Fatalf("Confirm (sequence #1): got %v, want ErrConfirmationFailed", err) } if err := m.consistent(); err != nil { t.Fatalf("Confirm (sequence #1): inconsistent state: %v", err) } releaseDee() if err := m.consistent(); err != nil { t.Fatalf("release (sequence #2): inconsistent state: %v", err) } releaseDum, err := m.Confirm(now, "/tweedle/dum", "", Condition{Token: tweedle}) if err != nil { t.Fatalf("Confirm (sequence #3): %v", err) } if err := m.consistent(); err != nil { t.Fatalf("Confirm (sequence #3): inconsistent state: %v", err) } // Test that you can't unlock a held lock. err = m.Unlock(now, tweedle) if err != ErrLocked { t.Fatalf("Unlock (sequence #4): got %v, want ErrLocked", err) } releaseDum() if err := m.consistent(); err != nil { t.Fatalf("release (sequence #5): inconsistent state: %v", err) } err = m.Unlock(now, tweedle) if err != nil { t.Fatalf("Unlock (sequence #6): %v", err) } if err := m.consistent(); err != nil { t.Fatalf("Unlock (sequence #6): inconsistent state: %v", err) } } func TestMemLSNonCanonicalRoot(t *testing.T) { now := time.Unix(0, 0) m := NewMemLS().(*memLS) token, err := m.Create(now, LockDetails{ Root: "/foo/./bar//", Duration: 1 * time.Second, }) if err != nil { t.Fatalf("Create: %v", err) } if err := m.consistent(); err != nil { t.Fatalf("Create: inconsistent state: %v", err) } if err := m.Unlock(now, token); err != nil { t.Fatalf("Unlock: %v", err) } if err := m.consistent(); err != nil { t.Fatalf("Unlock: inconsistent state: %v", err) } } func TestMemLSExpiry(t *testing.T) { m := NewMemLS().(*memLS) testCases := []string{ "setNow 0", "create /a.5", "want /a.5", "create /c.6", "want /a.5 /c.6", "create /a/b.7", "want /a.5 /a/b.7 /c.6", "setNow 4", "want /a.5 /a/b.7 /c.6", "setNow 5", "want /a/b.7 /c.6", "setNow 6", "want /a/b.7", "setNow 7", "want ", "setNow 8", "want ", "create /a.12", "create /b.13", "create /c.15", "create /a/d.16", "want /a.12 /a/d.16 /b.13 /c.15", "refresh /a.14", "want /a.14 /a/d.16 /b.13 /c.15", "setNow 12", "want /a.14 /a/d.16 /b.13 /c.15", "setNow 13", "want /a.14 /a/d.16 /c.15", "setNow 14", "want /a/d.16 /c.15", "refresh /a/d.20", "refresh /c.20", "want /a/d.20 /c.20", "setNow 20", "want ", } tokens := map[string]string{} zTime := time.Unix(0, 0) now := zTime for i, tc := range testCases { j := strings.IndexByte(tc, ' ') if j < 0 { t.Fatalf("test case #%d %q: invalid command", i, tc) } op, arg := tc[:j], tc[j+1:] switch op { default: t.Fatalf("test case #%d %q: invalid operation %q", i, tc, op) case "create", "refresh": parts := strings.Split(arg, ".") if len(parts) != 2 { t.Fatalf("test case #%d %q: invalid create", i, tc) } root := parts[0] d, err := strconv.Atoi(parts[1]) if err != nil { t.Fatalf("test case #%d %q: invalid duration", i, tc) } dur := time.Unix(0, 0).Add(time.Duration(d) * time.Second).Sub(now) switch op { case "create": token, err := m.Create(now, LockDetails{ Root: root, Duration: dur, ZeroDepth: true, }) if err != nil { t.Fatalf("test case #%d %q: Create: %v", i, tc, err) } tokens[root] = token case "refresh": token := tokens[root] if token == "" { t.Fatalf("test case #%d %q: no token for %q", i, tc, root) } got, err := m.Refresh(now, token, dur) if err != nil { t.Fatalf("test case #%d %q: Refresh: %v", i, tc, err) } want := LockDetails{ Root: root, Duration: dur, ZeroDepth: true, } if got != want { t.Fatalf("test case #%d %q:\ngot %v\nwant %v", i, tc, got, want) } } case "setNow": d, err := strconv.Atoi(arg) if err != nil { t.Fatalf("test case #%d %q: invalid duration", i, tc) } now = time.Unix(0, 0).Add(time.Duration(d) * time.Second) case "want": m.mu.Lock() m.collectExpiredNodes(now) got := make([]string, 0, len(m.byToken)) for _, n := range m.byToken { got = append(got, fmt.Sprintf("%s.%d", n.details.Root, n.expiry.Sub(zTime)/time.Second)) } m.mu.Unlock() sort.Strings(got) want := []string{} if arg != "" { want = strings.Split(arg, " ") } if !reflect.DeepEqual(got, want) { t.Fatalf("test case #%d %q:\ngot %q\nwant %q", i, tc, got, want) } } if err := m.consistent(); err != nil { t.Fatalf("test case #%d %q: inconsistent state: %v", i, tc, err) } } } func TestMemLS(t *testing.T) { now := time.Unix(0, 0) m := NewMemLS().(*memLS) rng := rand.New(rand.NewSource(0)) tokens := map[string]string{} nConfirm, nCreate, nRefresh, nUnlock := 0, 0, 0, 0 const N = 2000 for i := 0; i < N; i++ { name := lockTestNames[rng.Intn(len(lockTestNames))] duration := lockTestDurations[rng.Intn(len(lockTestDurations))] confirmed, unlocked := false, false // If the name was already locked, we randomly confirm/release, refresh // or unlock it. Otherwise, we create a lock. token := tokens[name] if token != "" { switch rng.Intn(3) { case 0: confirmed = true nConfirm++ release, err := m.Confirm(now, name, "", Condition{Token: token}) if err != nil { t.Fatalf("iteration #%d: Confirm %q: %v", i, name, err) } if err := m.consistent(); err != nil { t.Fatalf("iteration #%d: inconsistent state: %v", i, err) } release() case 1: nRefresh++ if _, err := m.Refresh(now, token, duration); err != nil { t.Fatalf("iteration #%d: Refresh %q: %v", i, name, err) } case 2: unlocked = true nUnlock++ if err := m.Unlock(now, token); err != nil { t.Fatalf("iteration #%d: Unlock %q: %v", i, name, err) } } } else { nCreate++ var err error token, err = m.Create(now, LockDetails{ Root: name, Duration: duration, ZeroDepth: lockTestZeroDepth(name), }) if err != nil { t.Fatalf("iteration #%d: Create %q: %v", i, name, err) } } if !confirmed { if duration == 0 || unlocked { // A zero-duration lock should expire immediately and is // effectively equivalent to being unlocked. tokens[name] = "" } else { tokens[name] = token } } if err := m.consistent(); err != nil { t.Fatalf("iteration #%d: inconsistent state: %v", i, err) } } if nConfirm < N/10 { t.Fatalf("too few Confirm calls: got %d, want >= %d", nConfirm, N/10) } if nCreate < N/10 { t.Fatalf("too few Create calls: got %d, want >= %d", nCreate, N/10) } if nRefresh < N/10 { t.Fatalf("too few Refresh calls: got %d, want >= %d", nRefresh, N/10) } if nUnlock < N/10 { t.Fatalf("too few Unlock calls: got %d, want >= %d", nUnlock, N/10) } } func (m *memLS) consistent() error { m.mu.Lock() defer m.mu.Unlock() // If m.byName is non-empty, then it must contain an entry for the root "/", // and its refCount should equal the number of locked nodes. if len(m.byName) > 0 { n := m.byName["/"] if n == nil { return fmt.Errorf(`non-empty m.byName does not contain the root "/"`) } if n.refCount != len(m.byToken) { return fmt.Errorf("root node refCount=%d, differs from len(m.byToken)=%d", n.refCount, len(m.byToken)) } } for name, n := range m.byName { // The map keys should be consistent with the node's copy of the key. if n.details.Root != name { return fmt.Errorf("node name %q != byName map key %q", n.details.Root, name) } // A name must be clean, and start with a "/". if len(name) == 0 || name[0] != '/' { return fmt.Errorf(`node name %q does not start with "/"`, name) } if name != path.Clean(name) { return fmt.Errorf(`node name %q is not clean`, name) } // A node's refCount should be positive. if n.refCount <= 0 { return fmt.Errorf("non-positive refCount for node at name %q", name) } // A node's refCount should be the number of self-or-descendents that // are locked (i.e. have a non-empty token). var list []string for name0, n0 := range m.byName { // All of lockTestNames' name fragments are one byte long: '_', 'i' or 'z', // so strings.HasPrefix is equivalent to self-or-descendent name match. // We don't have to worry about "/foo/bar" being a false positive match // for "/foo/b". if strings.HasPrefix(name0, name) && n0.token != "" { list = append(list, name0) } } if n.refCount != len(list) { sort.Strings(list) return fmt.Errorf("node at name %q has refCount %d but locked self-or-descendents are %q (len=%d)", name, n.refCount, list, len(list)) } // A node n is in m.byToken if it has a non-empty token. if n.token != "" { if _, ok := m.byToken[n.token]; !ok { return fmt.Errorf("node at name %q has token %q but not in m.byToken", name, n.token) } } // A node n is in m.byExpiry if it has a non-negative byExpiryIndex. if n.byExpiryIndex >= 0 { if n.byExpiryIndex >= len(m.byExpiry) { return fmt.Errorf("node at name %q has byExpiryIndex %d but m.byExpiry has length %d", name, n.byExpiryIndex, len(m.byExpiry)) } if n != m.byExpiry[n.byExpiryIndex] { return fmt.Errorf("node at name %q has byExpiryIndex %d but that indexes a different node", name, n.byExpiryIndex) } } } for token, n := range m.byToken { // The map keys should be consistent with the node's copy of the key. if n.token != token { return fmt.Errorf("node token %q != byToken map key %q", n.token, token) } // Every node in m.byToken is in m.byName. if _, ok := m.byName[n.details.Root]; !ok { return fmt.Errorf("node at name %q in m.byToken but not in m.byName", n.details.Root) } } for i, n := range m.byExpiry { // The slice indices should be consistent with the node's copy of the index. if n.byExpiryIndex != i { return fmt.Errorf("node byExpiryIndex %d != byExpiry slice index %d", n.byExpiryIndex, i) } // Every node in m.byExpiry is in m.byName. if _, ok := m.byName[n.details.Root]; !ok { return fmt.Errorf("node at name %q in m.byExpiry but not in m.byName", n.details.Root) } // No node in m.byExpiry should be held. if n.held { return fmt.Errorf("node at name %q in m.byExpiry is held", n.details.Root) } } return nil } func TestParseTimeout(t *testing.T) { testCases := []struct { s string want time.Duration wantErr error }{{ "", infiniteTimeout, nil, }, { "Infinite", infiniteTimeout, nil, }, { "Infinitesimal", 0, errInvalidTimeout, }, { "infinite", 0, errInvalidTimeout, }, { "Second-0", 0 * time.Second, nil, }, { "Second-123", 123 * time.Second, nil, }, { " Second-456 ", 456 * time.Second, nil, }, { "Second-4100000000", 4100000000 * time.Second, nil, }, { "junk", 0, errInvalidTimeout, }, { "Second-", 0, errInvalidTimeout, }, { "Second--1", 0, errInvalidTimeout, }, { "Second--123", 0, errInvalidTimeout, }, { "Second-+123", 0, errInvalidTimeout, }, { "Second-0x123", 0, errInvalidTimeout, }, { "second-123", 0, errInvalidTimeout, }, { "Second-4294967295", 4294967295 * time.Second, nil, }, { // Section 10.7 says that "The timeout value for TimeType "Second" // must not be greater than 2^32-1." "Second-4294967296", 0, errInvalidTimeout, }, { // This test case comes from section 9.10.9 of the spec. It says, // // "In this request, the client has specified that it desires an // infinite-length lock, if available, otherwise a timeout of 4.1 // billion seconds, if available." // // The Go WebDAV package always supports infinite length locks, // and ignores the fallback after the comma. "Infinite, Second-4100000000", infiniteTimeout, nil, }} for _, tc := range testCases { got, gotErr := parseTimeout(tc.s) if got != tc.want || gotErr != tc.wantErr { t.Errorf("parsing %q:\ngot %v, %v\nwant %v, %v", tc.s, got, gotErr, tc.want, tc.wantErr) } } } ================================================ FILE: server/webdav/prop.go ================================================ // Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package webdav import ( "bytes" "context" "encoding/xml" "errors" "fmt" "net/http" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" ) // Proppatch describes a property update instruction as defined in RFC 4918. // See http://www.webdav.org/specs/rfc4918.html#METHOD_PROPPATCH type Proppatch struct { // Remove specifies whether this patch removes properties. If it does not // remove them, it sets them. Remove bool // Props contains the properties to be set or removed. Props []Property } // Propstat describes a XML propstat element as defined in RFC 4918. // See http://www.webdav.org/specs/rfc4918.html#ELEMENT_propstat type Propstat struct { // Props contains the properties for which Status applies. Props []Property // Status defines the HTTP status code of the properties in Prop. // Allowed values include, but are not limited to the WebDAV status // code extensions for HTTP/1.1. // http://www.webdav.org/specs/rfc4918.html#status.code.extensions.to.http11 Status int // XMLError contains the XML representation of the optional error element. // XML content within this field must not rely on any predefined // namespace declarations or prefixes. If empty, the XML error element // is omitted. XMLError string // ResponseDescription contains the contents of the optional // responsedescription field. If empty, the XML element is omitted. ResponseDescription string } // makePropstats returns a slice containing those of x and y whose Props slice // is non-empty. If both are empty, it returns a slice containing an otherwise // zero Propstat whose HTTP status code is 200 OK. func makePropstats(x, y Propstat) []Propstat { pstats := make([]Propstat, 0, 2) if len(x.Props) != 0 { pstats = append(pstats, x) } if len(y.Props) != 0 { pstats = append(pstats, y) } if len(pstats) == 0 { pstats = append(pstats, Propstat{ Status: http.StatusOK, }) } return pstats } // DeadPropsHolder holds the dead properties of a resource. // // Dead properties are those properties that are explicitly defined. In // comparison, live properties, such as DAV:getcontentlength, are implicitly // defined by the underlying resource, and cannot be explicitly overridden or // removed. See the Terminology section of // http://www.webdav.org/specs/rfc4918.html#rfc.section.3 // // There is a whitelist of the names of live properties. This package handles // all live properties, and will only pass non-whitelisted names to the Patch // method of DeadPropsHolder implementations. type DeadPropsHolder interface { // DeadProps returns a copy of the dead properties held. DeadProps() (map[xml.Name]Property, error) // Patch patches the dead properties held. // // Patching is atomic; either all or no patches succeed. It returns (nil, // non-nil) if an internal server error occurred, otherwise the Propstats // collectively contain one Property for each proposed patch Property. If // all patches succeed, Patch returns a slice of length one and a Propstat // element with a 200 OK HTTP status code. If none succeed, for reasons // other than an internal server error, no Propstat has status 200 OK. // // For more details on when various HTTP status codes apply, see // http://www.webdav.org/specs/rfc4918.html#PROPPATCH-status Patch([]Proppatch) ([]Propstat, error) } // liveProps contains all supported properties. var liveProps = map[xml.Name]struct { // findFn implements the propfind function of this property. If nil, // it indicates a hidden property. findFn func(context.Context, LockSystem, string, model.Obj) (string, error) // dir is true if the property applies to directories. dir bool }{ {Space: "DAV:", Local: "resourcetype"}: { findFn: findResourceType, dir: true, }, {Space: "DAV:", Local: "displayname"}: { findFn: findDisplayName, dir: true, }, {Space: "DAV:", Local: "getcontentlength"}: { findFn: findContentLength, dir: false, }, {Space: "DAV:", Local: "getlastmodified"}: { findFn: findLastModified, // http://webdav.org/specs/rfc4918.html#PROPERTY_getlastmodified // suggests that getlastmodified should only apply to GETable // resources, and this package does not support GET on directories. // // Nonetheless, some WebDAV clients expect child directories to be // sortable by getlastmodified date, so this value is true, not false. // See golang.org/issue/15334. dir: true, }, {Space: "DAV:", Local: "creationdate"}: { findFn: findCreationDate, dir: true, }, {Space: "DAV:", Local: "getcontentlanguage"}: { findFn: nil, dir: false, }, {Space: "DAV:", Local: "getcontenttype"}: { findFn: findContentType, dir: false, }, {Space: "DAV:", Local: "getetag"}: { findFn: findETag, // findETag implements ETag as the concatenated hex values of a file's // modification time and size. This is not a reliable synchronization // mechanism for directories, so we do not advertise getetag for DAV // collections. dir: false, }, // TODO: The lockdiscovery property requires LockSystem to list the // active locks on a resource. {Space: "DAV:", Local: "lockdiscovery"}: {}, {Space: "DAV:", Local: "supportedlock"}: { findFn: findSupportedLock, dir: true, }, {Space: "http://owncloud.org/ns", Local: "checksums"}: { findFn: findChecksums, dir: false, }, } // TODO(nigeltao) merge props and allprop? // Props returns the status of the properties named pnames for resource name. // // Each Propstat has a unique status and each property name will only be part // of one Propstat element. func props(ctx context.Context, ls LockSystem, fi model.Obj, pnames []xml.Name) ([]Propstat, error) { //f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) //if err != nil { // return nil, err //} //defer f.Close() //fi, err := f.Stat() //if err != nil { // return nil, err //} isDir := fi.IsDir() var deadProps map[xml.Name]Property // ??? what is this for? //if dph, ok := f.(DeadPropsHolder); ok { // deadProps, err = dph.DeadProps() // if err != nil { // return nil, err // } //} pstatOK := Propstat{Status: http.StatusOK} pstatNotFound := Propstat{Status: http.StatusNotFound} for _, pn := range pnames { // If this file has dead properties, check if they contain pn. if dp, ok := deadProps[pn]; ok { pstatOK.Props = append(pstatOK.Props, dp) continue } // Otherwise, it must either be a live property or we don't know it. if prop := liveProps[pn]; prop.findFn != nil && (prop.dir || !isDir) { innerXML, err := prop.findFn(ctx, ls, fi.GetName(), fi) if err != nil { return nil, err } pstatOK.Props = append(pstatOK.Props, Property{ XMLName: pn, InnerXML: []byte(innerXML), }) } else { pstatNotFound.Props = append(pstatNotFound.Props, Property{ XMLName: pn, }) } } return makePropstats(pstatOK, pstatNotFound), nil } // Propnames returns the property names defined for resource name. func propnames(ctx context.Context, ls LockSystem, fi model.Obj) ([]xml.Name, error) { //f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) //if err != nil { // return nil, err //} //defer f.Close() //fi, err := f.Stat() //if err != nil { // return nil, err //} isDir := fi.IsDir() var deadProps map[xml.Name]Property // ??? what is this for? //if dph, ok := f.(DeadPropsHolder); ok { // deadProps, err = dph.DeadProps() // if err != nil { // return nil, err // } //} pnames := make([]xml.Name, 0, len(liveProps)+len(deadProps)) for pn, prop := range liveProps { if prop.findFn != nil && (prop.dir || !isDir) { pnames = append(pnames, pn) } } for pn := range deadProps { pnames = append(pnames, pn) } return pnames, nil } // Allprop returns the properties defined for resource name and the properties // named in include. // // Note that RFC 4918 defines 'allprop' to return the DAV: properties defined // within the RFC plus dead properties. Other live properties should only be // returned if they are named in 'include'. // // See http://www.webdav.org/specs/rfc4918.html#METHOD_PROPFIND func allprop(ctx context.Context, ls LockSystem, fi model.Obj, include []xml.Name) ([]Propstat, error) { pnames, err := propnames(ctx, ls, fi) if err != nil { return nil, err } // Add names from include if they are not already covered in pnames. nameset := make(map[xml.Name]bool) for _, pn := range pnames { nameset[pn] = true } for _, pn := range include { if !nameset[pn] { pnames = append(pnames, pn) } } return props(ctx, ls, fi, pnames) } // Patch patches the properties of resource name. The return values are // constrained in the same manner as DeadPropsHolder.Patch. func patch(ctx context.Context, ls LockSystem, name string, patches []Proppatch) ([]Propstat, error) { conflict := false loop: for _, patch := range patches { for _, p := range patch.Props { if _, ok := liveProps[p.XMLName]; ok { conflict = true break loop } } } if conflict { pstatForbidden := Propstat{ Status: http.StatusForbidden, XMLError: ``, } pstatFailedDep := Propstat{ Status: StatusFailedDependency, } for _, patch := range patches { for _, p := range patch.Props { if _, ok := liveProps[p.XMLName]; ok { pstatForbidden.Props = append(pstatForbidden.Props, Property{XMLName: p.XMLName}) } else { pstatFailedDep.Props = append(pstatFailedDep.Props, Property{XMLName: p.XMLName}) } } } return makePropstats(pstatForbidden, pstatFailedDep), nil } // ------------------------------------------------------------ //f, err := fs.OpenFile(ctx, name, os.O_RDWR, 0) //if err != nil { // return nil, err //} //defer f.Close() //if dph, ok := f.(DeadPropsHolder); ok { // ret, err := dph.Patch(patches) // if err != nil { // return nil, err // } // // http://www.webdav.org/specs/rfc4918.html#ELEMENT_propstat says that // // "The contents of the prop XML element must only list the names of // // properties to which the result in the status element applies." // for _, pstat := range ret { // for i, p := range pstat.Props { // pstat.Props[i] = Property{XMLName: p.XMLName} // } // } // return ret, nil //} // ------------------------------------------------------------ // The file doesn't implement the optional DeadPropsHolder interface, so // all patches are forbidden. pstat := Propstat{Status: http.StatusForbidden} for _, patch := range patches { for _, p := range patch.Props { pstat.Props = append(pstat.Props, Property{XMLName: p.XMLName}) } } return []Propstat{pstat}, nil } func escapeXML(s string) string { for i := 0; i < len(s); i++ { // As an optimization, if s contains only ASCII letters, digits or a // few special characters, the escaped value is s itself and we don't // need to allocate a buffer and convert between string and []byte. switch c := s[i]; { case c == ' ' || c == '_' || ('+' <= c && c <= '9') || // Digits as well as + , - . and / ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z'): continue } // Otherwise, go through the full escaping process. var buf bytes.Buffer xml.EscapeText(&buf, []byte(s)) return buf.String() } return s } func findResourceType(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { if fi.IsDir() { return ``, nil } return "", nil } func findDisplayName(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { if slashClean(name) == "/" { // Hide the real name of a possibly prefixed root directory. return "", nil } return escapeXML(fi.GetName()), nil } func findContentLength(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { return strconv.FormatInt(fi.GetSize(), 10), nil } func findLastModified(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { return fi.ModTime().UTC().Format(http.TimeFormat), nil } func findCreationDate(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { userAgent := ctx.Value(conf.UserAgentKey).(string) if strings.Contains(strings.ToLower(userAgent), "microsoft-webdav") { return fi.CreateTime().UTC().Format(http.TimeFormat), nil } return fi.CreateTime().UTC().Format(time.RFC3339), nil } // ErrNotImplemented should be returned by optional interfaces if they // want the original implementation to be used. var ErrNotImplemented = errors.New("not implemented") // ContentTyper is an optional interface for the os.FileInfo // objects returned by the FileSystem. // // If this interface is defined then it will be used to read the // content type from the object. // // If this interface is not defined the file will be opened and the // content type will be guessed from the initial contents of the file. type ContentTyper interface { // ContentType returns the content type for the file. // // If this returns error ErrNotImplemented then the error will // be ignored and the base implementation will be used // instead. ContentType(ctx context.Context) (string, error) } func findContentType(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { //if do, ok := fi.(ContentTyper); ok { // ctype, err := do.ContentType(ctx) // if err != ErrNotImplemented { // return ctype, err // } //} //f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) //if err != nil { // return "", err //} //defer f.Close() // This implementation is based on serveContent's code in the standard net/http package. ctype := utils.GetMimeType(name) return ctype, nil //if ctype != "" { // return ctype, nil //} //return "application/octet-stream", nil // Read a chunk to decide between utf-8 text and binary. //var buf [512]byte //n, err := io.ReadFull(f, buf[:]) //if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { // return "", err //} //ctype = http.DetectContentType(buf[:n]) //// Rewind file. //_, err = f.Seek(0, os.SEEK_SET) //return ctype, err } // ETager is an optional interface for the os.FileInfo objects // returned by the FileSystem. // // If this interface is defined then it will be used to read the ETag // for the object. // // If this interface is not defined an ETag will be computed using the // ModTime() and the Size() methods of the os.FileInfo object. type ETager interface { // ETag returns an ETag for the file. This should be of the // form "value" or W/"value" // // If this returns error ErrNotImplemented then the error will // be ignored and the base implementation will be used // instead. ETag(ctx context.Context) (string, error) } func findETag(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { if do, ok := fi.(ETager); ok { etag, err := do.ETag(ctx) if !errors.Is(err, ErrNotImplemented) { return etag, err } } return common.GetEtag(fi, fi.GetSize()), nil } func findSupportedLock(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { return `` + `` + `` + `` + ``, nil } func findChecksums(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { checksums := "" for hashType, hashValue := range fi.GetHash().All() { checksums += fmt.Sprintf("%s:%s", hashType.Name, hashValue) } return checksums, nil } ================================================ FILE: server/webdav/util.go ================================================ package webdav import ( log "github.com/sirupsen/logrus" "net/http" "strconv" "time" ) func (h *Handler) getModTime(r *http.Request) time.Time { return h.getHeaderTime(r, "X-OC-Mtime", "") } // owncloud/ nextcloud haven't impl this, but we can add the support since rclone may support this soon. // try ModTime if CreateTime not found in header func (h *Handler) getCreateTime(r *http.Request) time.Time { return h.getHeaderTime(r, "X-OC-Ctime", "X-OC-Mtime") } func (h *Handler) getHeaderTime(r *http.Request, header, alternative string) time.Time { hVal := r.Header.Get(header) // try alternative if hVal == "" && alternative != "" { hVal = r.Header.Get(alternative) } if hVal != "" { modTimeUnix, err := strconv.ParseInt(hVal, 10, 64) if err == nil { return time.Unix(modTimeUnix, 0) } log.Warnf("getModTime in Webdav, failed to parse %s, %s", header, err) } return time.Now() } ================================================ FILE: server/webdav/webdav.go ================================================ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package webdav provides a WebDAV server implementation. package webdav // import "golang.org/x/net/webdav" import ( "context" "errors" "fmt" "io" "net/http" "net/url" "os" "path" "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" ) type Handler struct { // Prefix is the URL path prefix to strip from WebDAV resource paths. Prefix string // LockSystem is the lock management system. LockSystem LockSystem // Logger is an optional error logger. If non-nil, it will be called // for all HTTP requests. Logger func(*http.Request, error) } func (h *Handler) stripPrefix(p string) (string, int, error) { if h.Prefix == "" { return p, http.StatusOK, nil } if r := strings.TrimPrefix(p, h.Prefix); len(r) < len(p) { return r, http.StatusOK, nil } return p, http.StatusNotFound, errPrefixMismatch } func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { status, err := http.StatusBadRequest, errUnsupportedMethod brw := newBufferedResponseWriter() useBufferedWriter := true if h.LockSystem == nil { status, err = http.StatusInternalServerError, errNoLockSystem } else { switch r.Method { case "OPTIONS": status, err = h.handleOptions(brw, r) case "GET", "HEAD", "POST": useBufferedWriter = false Writer := &common.WrittenResponseWriter{ResponseWriter: w} status, err = h.handleGetHeadPost(Writer, r) if status != 0 && Writer.IsWritten() { status = 0 } case "DELETE": status, err = h.handleDelete(brw, r) case "PUT": status, err = h.handlePut(brw, r) case "MKCOL": status, err = h.handleMkcol(brw, r) case "COPY", "MOVE": status, err = h.handleCopyMove(brw, r) case "LOCK": status, err = h.handleLock(brw, r) case "UNLOCK": status, err = h.handleUnlock(brw, r) case "PROPFIND": status, err = h.handlePropfind(brw, r) // if there is a error for PROPFIND, we should be as an empty folder to the client if err != nil { status = http.StatusNotFound } case "PROPPATCH": status, err = h.handleProppatch(brw, r) } } if status != 0 { w.WriteHeader(status) if status != http.StatusNoContent { w.Write([]byte(StatusText(status))) } } else if useBufferedWriter { brw.WriteToResponse(w) } if h.Logger != nil && err != nil { h.Logger(r, err) } } func (h *Handler) lock(now time.Time, root string) (token string, status int, err error) { token, err = h.LockSystem.Create(now, LockDetails{ Root: root, Duration: infiniteTimeout, ZeroDepth: true, }) if err != nil { if err == ErrLocked { return "", StatusLocked, err } return "", http.StatusInternalServerError, err } return token, 0, nil } func (h *Handler) confirmLocks(r *http.Request, src, dst string) (release func(), status int, err error) { hdr := r.Header.Get("If") if hdr == "" { // An empty If header means that the client hasn't previously created locks. // Even if this client doesn't care about locks, we still need to check that // the resources aren't locked by another client, so we create temporary // locks that would conflict with another client's locks. These temporary // locks are unlocked at the end of the HTTP request. now, srcToken, dstToken := time.Now(), "", "" if src != "" { srcToken, status, err = h.lock(now, src) if err != nil { return nil, status, err } } if dst != "" { dstToken, status, err = h.lock(now, dst) if err != nil { if srcToken != "" { h.LockSystem.Unlock(now, srcToken) } return nil, status, err } } return func() { if dstToken != "" { h.LockSystem.Unlock(now, dstToken) } if srcToken != "" { h.LockSystem.Unlock(now, srcToken) } }, 0, nil } ih, ok := parseIfHeader(hdr) if !ok { return nil, http.StatusBadRequest, errInvalidIfHeader } // ih is a disjunction (OR) of ifLists, so any ifList will do. for _, l := range ih.lists { lsrc := l.resourceTag if lsrc == "" { lsrc = src } else { u, err := url.Parse(lsrc) if err != nil { continue } if u.Host != r.Host { continue } lsrc, status, err = h.stripPrefix(u.Path) if err != nil { return nil, status, err } } release, err = h.LockSystem.Confirm(time.Now(), lsrc, dst, l.conditions...) if err == ErrConfirmationFailed { continue } if err != nil { return nil, http.StatusInternalServerError, err } return release, 0, nil } // Section 10.4.1 says that "If this header is evaluated and all state lists // fail, then the request must fail with a 412 (Precondition Failed) status." // We follow the spec even though the cond_put_corrupt_token test case from // the litmus test warns on seeing a 412 instead of a 423 (Locked). return nil, http.StatusPreconditionFailed, ErrLocked } func (h *Handler) handleOptions(w http.ResponseWriter, r *http.Request) (status int, err error) { reqPath, status, err := h.stripPrefix(r.URL.Path) if err != nil { return status, err } ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { return 403, err } allow := "OPTIONS, LOCK, PUT, MKCOL" if fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err == nil { if fi.IsDir() { allow = "OPTIONS, LOCK, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND" } else { allow = "OPTIONS, LOCK, GET, HEAD, POST, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND, PUT" } } w.Header().Set("Allow", allow) // http://www.webdav.org/specs/rfc4918.html#dav.compliance.classes w.Header().Set("DAV", "1, 2") // http://msdn.microsoft.com/en-au/library/cc250217.aspx w.Header().Set("MS-Author-Via", "DAV") return 0, nil } func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (status int, err error) { reqPath, status, err := h.stripPrefix(r.URL.Path) if err != nil { return status, err } // TODO: check locks for read-only access?? ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { return http.StatusForbidden, err } fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) if err != nil { return http.StatusNotFound, err } if fi.IsDir() { if r.Method == http.MethodHead { w.Header().Set("Content-Type", "httpd/unix-directory") w.Header().Set("Content-Length", "0") return http.StatusOK, nil } return http.StatusMethodNotAllowed, nil } // Let ServeContent determine the Content-Type header. storage, _ := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) if storage.GetStorage().Webdav302() { link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{IP: utils.ClientIP(r), Header: r.Header, Redirect: true}) if err != nil { return http.StatusInternalServerError, err } defer link.Close() http.Redirect(w, r, link.URL, http.StatusFound) return 0, nil } if storage.GetStorage().WebdavProxyURL() { if url := common.GenerateDownProxyURL(storage.GetStorage(), reqPath); url != "" { w.Header().Set("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") http.Redirect(w, r, url, http.StatusFound) return 0, nil } } link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{Header: r.Header}) if err != nil { return http.StatusInternalServerError, err } defer link.Close() if storage.GetStorage().ProxyRange { link = common.ProxyRange(ctx, link, fi.GetSize()) } err = common.Proxy(w, r, link, fi) if err != nil { if statusCode, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { return int(statusCode), err } return http.StatusInternalServerError, fmt.Errorf("webdav proxy error: %+v", err) } return 0, nil } func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status int, err error) { reqPath, status, err := h.stripPrefix(r.URL.Path) if err != nil { return status, err } release, status, err := h.confirmLocks(r, reqPath, "") if err != nil { return status, err } defer release() ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { return 403, err } // TODO: return MultiStatus where appropriate. // "godoc os RemoveAll" says that "If the path does not exist, RemoveAll // returns nil (no error)." WebDAV semantics are that it should return a // "404 Not Found". We therefore have to Stat before we RemoveAll. if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err != nil { if errs.IsObjectNotFound(err) { return http.StatusNotFound, err } return http.StatusMethodNotAllowed, err } if err := fs.Remove(ctx, reqPath); err != nil { return http.StatusMethodNotAllowed, err } //fs.ClearCache(path.Dir(reqPath)) return http.StatusNoContent, nil } func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, err error) { defer func() { if n, _ := io.ReadFull(r.Body, []byte{0}); n == 1 { _, _ = utils.CopyWithBuffer(io.Discard, r.Body) } _ = r.Body.Close() }() reqPath, status, err := h.stripPrefix(r.URL.Path) if err != nil { return status, err } if reqPath == "" { return http.StatusMethodNotAllowed, nil } release, status, err := h.confirmLocks(r, reqPath, "") if err != nil { return status, err } defer release() // TODO(rost): Support the If-Match, If-None-Match headers? See bradfitz' // comments in http.checkEtag. ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { return http.StatusForbidden, err } size := r.ContentLength if size < 0 { sizeStr := r.Header.Get("X-File-Size") if sizeStr != "" { size, err = strconv.ParseInt(sizeStr, 10, 64) if err != nil { return http.StatusBadRequest, err } } } obj := model.Object{ Name: path.Base(reqPath), Size: size, Modified: h.getModTime(r), Ctime: h.getCreateTime(r), } // Check if system file should be ignored if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) { return http.StatusForbidden, errs.IgnoredSystemFile } fsStream := &stream.FileStream{ Obj: &obj, Reader: r.Body, Mimetype: r.Header.Get("Content-Type"), } if fsStream.Mimetype == "" { fsStream.Mimetype = utils.GetMimeType(reqPath) } err = fs.PutDirectly(ctx, path.Dir(reqPath), fsStream) if errs.IsNotFoundError(err) { return http.StatusNotFound, err } // TODO(rost): Returning 405 Method Not Allowed might not be appropriate. if err != nil { return http.StatusMethodNotAllowed, err } fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) if err != nil { fi = &obj } etag, err := findETag(ctx, h.LockSystem, reqPath, fi) if err != nil { return http.StatusInternalServerError, err } w.Header().Set("Etag", etag) return http.StatusCreated, nil } func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status int, err error) { reqPath, status, err := h.stripPrefix(r.URL.Path) if err != nil { return status, err } release, status, err := h.confirmLocks(r, reqPath, "") if err != nil { return status, err } defer release() ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { return 403, err } if r.ContentLength > 0 { return http.StatusUnsupportedMediaType, nil } // RFC 4918 9.3.1 //405 (Method Not Allowed) - MKCOL can only be executed on an unmapped URL if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err == nil { return http.StatusMethodNotAllowed, err } // RFC 4918 9.3.1 // 409 (Conflict) The server MUST NOT create those intermediate collections automatically. reqDir := path.Dir(reqPath) if _, err := fs.Get(ctx, reqDir, &fs.GetArgs{}); err != nil { if errs.IsObjectNotFound(err) { return http.StatusConflict, err } return http.StatusMethodNotAllowed, err } if err := fs.MakeDir(ctx, reqPath); err != nil { if os.IsNotExist(err) { return http.StatusConflict, err } return http.StatusMethodNotAllowed, err } return http.StatusCreated, nil } func (h *Handler) handleCopyMove(w http.ResponseWriter, r *http.Request) (status int, err error) { hdr := r.Header.Get("Destination") if hdr == "" { return http.StatusBadRequest, errInvalidDestination } u, err := url.Parse(hdr) if err != nil { return http.StatusBadRequest, errInvalidDestination } if u.Host != "" && u.Host != r.Host { return http.StatusBadGateway, errInvalidDestination } src, status, err := h.stripPrefix(r.URL.Path) if err != nil { return status, err } dst, status, err := h.stripPrefix(u.Path) if err != nil { return status, err } if dst == "" { return http.StatusBadGateway, errInvalidDestination } if dst == src { return http.StatusForbidden, errDestinationEqualsSource } ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) src, err = user.JoinPath(src) if err != nil { return 403, err } dst, err = user.JoinPath(dst) if err != nil { return 403, err } if r.Method == "COPY" { // Section 7.5.1 says that a COPY only needs to lock the destination, // not both destination and source. Strictly speaking, this is racy, // even though a COPY doesn't modify the source, if a concurrent // operation modifies the source. However, the litmus test explicitly // checks that COPYing a locked-by-another source is OK. release, status, err := h.confirmLocks(r, "", dst) if err != nil { return status, err } defer release() // Section 9.8.3 says that "The COPY method on a collection without a Depth // header must act as if a Depth header with value "infinity" was included". depth := infiniteDepth if hdr := r.Header.Get("Depth"); hdr != "" { depth = parseDepth(hdr) if depth != 0 && depth != infiniteDepth { // Section 9.8.3 says that "A client may submit a Depth header on a // COPY on a collection with a value of "0" or "infinity"." return http.StatusBadRequest, errInvalidDepth } } return copyFiles(ctx, src, dst, r.Header.Get("Overwrite") != "F") } release, status, err := h.confirmLocks(r, src, dst) if err != nil { return status, err } defer release() // Section 9.9.2 says that "The MOVE method on a collection must act as if // a "Depth: infinity" header was used on it. A client must not submit a // Depth header on a MOVE on a collection with any value but "infinity"." if hdr := r.Header.Get("Depth"); hdr != "" { if parseDepth(hdr) != infiniteDepth { return http.StatusBadRequest, errInvalidDepth } } return moveFiles(ctx, src, dst, r.Header.Get("Overwrite") == "T") } func (h *Handler) handleLock(w http.ResponseWriter, r *http.Request) (retStatus int, retErr error) { duration, err := parseTimeout(r.Header.Get("Timeout")) if err != nil { return http.StatusBadRequest, err } li, status, err := readLockInfo(r.Body) if err != nil { return status, err } ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) token, ld, now, created := "", LockDetails{}, time.Now(), false if li == (lockInfo{}) { // An empty lockInfo means to refresh the lock. ih, ok := parseIfHeader(r.Header.Get("If")) if !ok { return http.StatusBadRequest, errInvalidIfHeader } if len(ih.lists) == 1 && len(ih.lists[0].conditions) == 1 { token = ih.lists[0].conditions[0].Token } if token == "" { return http.StatusBadRequest, errInvalidLockToken } ld, err = h.LockSystem.Refresh(now, token, duration) if err != nil { if err == ErrNoSuchLock { return http.StatusPreconditionFailed, err } return http.StatusInternalServerError, err } } else { // Section 9.10.3 says that "If no Depth header is submitted on a LOCK request, // then the request MUST act as if a "Depth:infinity" had been submitted." depth := infiniteDepth if hdr := r.Header.Get("Depth"); hdr != "" { depth = parseDepth(hdr) if depth != 0 && depth != infiniteDepth { // Section 9.10.3 says that "Values other than 0 or infinity must not be // used with the Depth header on a LOCK method". return http.StatusBadRequest, errInvalidDepth } } reqPath, status, err := h.stripPrefix(r.URL.Path) if err != nil { return status, err } reqPath, err = user.JoinPath(reqPath) if err != nil { return 403, err } ld = LockDetails{ Root: reqPath, Duration: duration, OwnerXML: li.Owner.InnerXML, ZeroDepth: depth == 0, } token, err = h.LockSystem.Create(now, ld) if err != nil { if err == ErrLocked { return StatusLocked, err } return http.StatusInternalServerError, err } defer func() { if retErr != nil { h.LockSystem.Unlock(now, token) } }() // ??? Why create resource here? //// Create the resource if it didn't previously exist. //if _, err := h.FileSystem.Stat(ctx, reqPath); err != nil { // f, err := h.FileSystem.OpenFile(ctx, reqPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) // if err != nil { // // TODO: detect missing intermediate dirs and return http.StatusConflict? // return http.StatusInternalServerError, err // } // f.Close() // created = true //} // http://www.webdav.org/specs/rfc4918.html#HEADER_Lock-Token says that the // Lock-Token value is a Coded-URL. We add angle brackets. w.Header().Set("Lock-Token", "<"+token+">") } w.Header().Set("Content-Type", "application/xml; charset=utf-8") if created { // This is "w.WriteHeader(http.StatusCreated)" and not "return // http.StatusCreated, nil" because we write our own (XML) response to w // and Handler.ServeHTTP would otherwise write "Created". w.WriteHeader(http.StatusCreated) } writeLockInfo(w, token, ld) return 0, nil } func (h *Handler) handleUnlock(w http.ResponseWriter, r *http.Request) (status int, err error) { // http://www.webdav.org/specs/rfc4918.html#HEADER_Lock-Token says that the // Lock-Token value is a Coded-URL. We strip its angle brackets. t := r.Header.Get("Lock-Token") if len(t) < 2 || t[0] != '<' || t[len(t)-1] != '>' { return http.StatusBadRequest, errInvalidLockToken } t = t[1 : len(t)-1] switch err = h.LockSystem.Unlock(time.Now(), t); err { case nil: return http.StatusNoContent, err case ErrForbidden: return http.StatusForbidden, err case ErrLocked: return StatusLocked, err case ErrNoSuchLock: return http.StatusConflict, err default: return http.StatusInternalServerError, err } } func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status int, err error) { reqPath, status, err := h.stripPrefix(r.URL.Path) if err != nil { return status, err } ctx := r.Context() userAgent := r.Header.Get("User-Agent") ctx = context.WithValue(ctx, conf.UserAgentKey, userAgent) user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { return 403, err } fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) if err != nil { if errs.IsNotFoundError(err) { return http.StatusNotFound, err } return http.StatusMethodNotAllowed, err } depth := infiniteDepth if hdr := r.Header.Get("Depth"); hdr != "" { depth = parseDepth(hdr) if depth == invalidDepth { return http.StatusBadRequest, errInvalidDepth } } pf, status, err := readPropfind(r.Body) if err != nil { return status, err } mw := multistatusWriter{w: w} walkFn := func(reqPath string, info model.Obj, err error) error { if err != nil { return err } var pstats []Propstat if pf.Propname != nil { pnames, err := propnames(ctx, h.LockSystem, info) if err != nil { return err } pstat := Propstat{Status: http.StatusOK} for _, xmlname := range pnames { pstat.Props = append(pstat.Props, Property{XMLName: xmlname}) } pstats = append(pstats, pstat) } else if pf.Allprop != nil { pstats, err = allprop(ctx, h.LockSystem, info, pf.Prop) } else { pstats, err = props(ctx, h.LockSystem, info, pf.Prop) } if err != nil { return err } href := path.Join(h.Prefix, strings.TrimPrefix(reqPath, user.BasePath)) if href != "/" && info.IsDir() { href += "/" } return mw.write(makePropstatResponse(href, pstats)) } walkErr := walkFS(ctx, depth, reqPath, fi, walkFn) closeErr := mw.close() if walkErr != nil { return http.StatusInternalServerError, walkErr } if closeErr != nil { return http.StatusInternalServerError, closeErr } return 0, nil } func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (status int, err error) { reqPath, status, err := h.stripPrefix(r.URL.Path) if err != nil { return status, err } release, status, err := h.confirmLocks(r, reqPath, "") if err != nil { return status, err } defer release() ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { return 403, err } if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err != nil { if errs.IsObjectNotFound(err) { return http.StatusNotFound, err } return http.StatusMethodNotAllowed, err } patches, status, err := readProppatch(r.Body) if err != nil { return status, err } pstats, err := patch(ctx, h.LockSystem, reqPath, patches) if err != nil { return http.StatusInternalServerError, err } mw := multistatusWriter{w: w} writeErr := mw.write(makePropstatResponse(r.URL.Path, pstats)) closeErr := mw.close() if writeErr != nil { return http.StatusInternalServerError, writeErr } if closeErr != nil { return http.StatusInternalServerError, closeErr } return 0, nil } func makePropstatResponse(href string, pstats []Propstat) *response { resp := response{ Href: []string{(&url.URL{Path: href}).EscapedPath()}, Propstat: make([]propstat, 0, len(pstats)), } for _, p := range pstats { var xmlErr *xmlError if p.XMLError != "" { xmlErr = &xmlError{InnerXML: []byte(p.XMLError)} } resp.Propstat = append(resp.Propstat, propstat{ Status: fmt.Sprintf("HTTP/1.1 %d %s", p.Status, StatusText(p.Status)), Prop: p.Props, ResponseDescription: p.ResponseDescription, Error: xmlErr, }) } return &resp } const ( infiniteDepth = -1 invalidDepth = -2 ) // parseDepth maps the strings "0", "1" and "infinity" to 0, 1 and // infiniteDepth. Parsing any other string returns invalidDepth. // // Different WebDAV methods have further constraints on valid depths: // - PROPFIND has no further restrictions, as per section 9.1. // - COPY accepts only "0" or "infinity", as per section 9.8.3. // - MOVE accepts only "infinity", as per section 9.9.2. // - LOCK accepts only "0" or "infinity", as per section 9.10.3. // // These constraints are enforced by the handleXxx methods. func parseDepth(s string) int { switch s { case "0": return 0 case "1": return 1 case "infinity": return infiniteDepth } return invalidDepth } // http://www.webdav.org/specs/rfc4918.html#status.code.extensions.to.http11 const ( StatusMulti = 207 StatusUnprocessableEntity = 422 StatusLocked = 423 StatusFailedDependency = 424 StatusInsufficientStorage = 507 ) func StatusText(code int) string { switch code { case StatusMulti: return "Multi-Status" case StatusUnprocessableEntity: return "Unprocessable Entity" case StatusLocked: return "Locked" case StatusFailedDependency: return "Failed Dependency" case StatusInsufficientStorage: return "Insufficient Storage" } return http.StatusText(code) } var ( errDestinationEqualsSource = errors.New("webdav: destination equals source") errDirectoryNotEmpty = errors.New("webdav: directory not empty") errInvalidDepth = errors.New("webdav: invalid depth") errInvalidDestination = errors.New("webdav: invalid destination") errInvalidIfHeader = errors.New("webdav: invalid If header") errInvalidLockInfo = errors.New("webdav: invalid lock info") errInvalidLockToken = errors.New("webdav: invalid lock token") errInvalidPropfind = errors.New("webdav: invalid propfind") errInvalidProppatch = errors.New("webdav: invalid proppatch") errInvalidResponse = errors.New("webdav: invalid response") errInvalidTimeout = errors.New("webdav: invalid timeout") errNoFileSystem = errors.New("webdav: no file system") errNoLockSystem = errors.New("webdav: no lock system") errNotADirectory = errors.New("webdav: not a directory") errPrefixMismatch = errors.New("webdav: prefix mismatch") errRecursionTooDeep = errors.New("webdav: recursion too deep") errUnsupportedLockInfo = errors.New("webdav: unsupported lock info") errUnsupportedMethod = errors.New("webdav: unsupported method") ) ================================================ FILE: server/webdav/xml.go ================================================ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package webdav // The XML encoding is covered by Section 14. // http://www.webdav.org/specs/rfc4918.html#xml.element.definitions import ( "bytes" "encoding/xml" "fmt" "io" "net/http" "time" // As of https://go-review.googlesource.com/#/c/12772/ which was submitted // in July 2015, this package uses an internal fork of the standard // library's encoding/xml package, due to changes in the way namespaces // were encoded. Such changes were introduced in the Go 1.5 cycle, but were // rolled back in response to https://github.com/golang/go/issues/11841 // // However, this package's exported API, specifically the Property and // DeadPropsHolder types, need to refer to the standard library's version // of the xml.Name type, as code that imports this package cannot refer to // the internal version. // // This file therefore imports both the internal and external versions, as // ixml and xml, and converts between them. // // In the long term, this package should use the standard library's version // only, and the internal fork deleted, once // https://github.com/golang/go/issues/13400 is resolved. ixml "github.com/OpenListTeam/OpenList/v4/server/webdav/internal/xml" ) // http://www.webdav.org/specs/rfc4918.html#ELEMENT_lockinfo type lockInfo struct { XMLName ixml.Name `xml:"lockinfo"` Exclusive *struct{} `xml:"lockscope>exclusive"` Shared *struct{} `xml:"lockscope>shared"` Write *struct{} `xml:"locktype>write"` Owner owner `xml:"owner"` } // http://www.webdav.org/specs/rfc4918.html#ELEMENT_owner type owner struct { InnerXML string `xml:",innerxml"` } func readLockInfo(r io.Reader) (li lockInfo, status int, err error) { c := &countingReader{r: r} if err = ixml.NewDecoder(c).Decode(&li); err != nil { if err == io.EOF { if c.n == 0 { // An empty body means to refresh the lock. // http://www.webdav.org/specs/rfc4918.html#refreshing-locks return lockInfo{}, 0, nil } err = errInvalidLockInfo } return lockInfo{}, http.StatusBadRequest, err } // We only support exclusive (non-shared) write locks. In practice, these are // the only types of locks that seem to matter. if li.Exclusive == nil || li.Shared != nil || li.Write == nil { return lockInfo{}, http.StatusNotImplemented, errUnsupportedLockInfo } return li, 0, nil } type countingReader struct { n int r io.Reader } func (c *countingReader) Read(p []byte) (int, error) { n, err := c.r.Read(p) c.n += n return n, err } func writeLockInfo(w io.Writer, token string, ld LockDetails) (int, error) { depth := "infinity" if ld.ZeroDepth { depth = "0" } timeout := ld.Duration / time.Second return fmt.Fprintf(w, "\n"+ "\n"+ " \n"+ " \n"+ " %s\n"+ " %s\n"+ " Second-%d\n"+ " %s\n"+ " %s\n"+ "", depth, ld.OwnerXML, timeout, escape(token), escape(ld.Root), ) } func escape(s string) string { for i := 0; i < len(s); i++ { switch s[i] { case '"', '&', '\'', '<', '>': b := bytes.NewBuffer(nil) ixml.EscapeText(b, []byte(s)) return b.String() } } return s } // Next returns the next token, if any, in the XML stream of d. // RFC 4918 requires to ignore comments, processing instructions // and directives. // http://www.webdav.org/specs/rfc4918.html#property_values // http://www.webdav.org/specs/rfc4918.html#xml-extensibility func next(d *ixml.Decoder) (ixml.Token, error) { for { t, err := d.Token() if err != nil { return t, err } switch t.(type) { case ixml.Comment, ixml.Directive, ixml.ProcInst: continue default: return t, nil } } } // http://www.webdav.org/specs/rfc4918.html#ELEMENT_prop (for propfind) type propfindProps []xml.Name // UnmarshalXML appends the property names enclosed within start to pn. // // It returns an error if start does not contain any properties or if // properties contain values. Character data between properties is ignored. func (pn *propfindProps) UnmarshalXML(d *ixml.Decoder, start ixml.StartElement) error { for { t, err := next(d) if err != nil { return err } switch t.(type) { case ixml.EndElement: if len(*pn) == 0 { return fmt.Errorf("%s must not be empty", start.Name.Local) } return nil case ixml.StartElement: name := t.(ixml.StartElement).Name t, err = next(d) if err != nil { return err } if _, ok := t.(ixml.EndElement); !ok { return fmt.Errorf("unexpected token %T", t) } *pn = append(*pn, xml.Name(name)) } } } // http://www.webdav.org/specs/rfc4918.html#ELEMENT_propfind type propfind struct { XMLName ixml.Name `xml:"DAV: propfind"` Allprop *struct{} `xml:"DAV: allprop"` Propname *struct{} `xml:"DAV: propname"` Prop propfindProps `xml:"DAV: prop"` Include propfindProps `xml:"DAV: include"` } func readPropfind(r io.Reader) (pf propfind, status int, err error) { c := countingReader{r: r} if err = ixml.NewDecoder(&c).Decode(&pf); err != nil { if err == io.EOF { if c.n == 0 { // An empty body means to propfind allprop. // http://www.webdav.org/specs/rfc4918.html#METHOD_PROPFIND return propfind{Allprop: new(struct{})}, 0, nil } err = errInvalidPropfind } return propfind{}, http.StatusBadRequest, err } if pf.Allprop == nil && pf.Include != nil { return propfind{}, http.StatusBadRequest, errInvalidPropfind } if pf.Allprop != nil && (pf.Prop != nil || pf.Propname != nil) { return propfind{}, http.StatusBadRequest, errInvalidPropfind } if pf.Prop != nil && pf.Propname != nil { return propfind{}, http.StatusBadRequest, errInvalidPropfind } if pf.Propname == nil && pf.Allprop == nil && pf.Prop == nil { return propfind{}, http.StatusBadRequest, errInvalidPropfind } return pf, 0, nil } // Property represents a single DAV resource property as defined in RFC 4918. // See http://www.webdav.org/specs/rfc4918.html#data.model.for.resource.properties type Property struct { // XMLName is the fully qualified name that identifies this property. XMLName xml.Name // Lang is an optional xml:lang attribute. Lang string `xml:"xml:lang,attr,omitempty"` // InnerXML contains the XML representation of the property value. // See http://www.webdav.org/specs/rfc4918.html#property_values // // Property values of complex type or mixed-content must have fully // expanded XML namespaces or be self-contained with according // XML namespace declarations. They must not rely on any XML // namespace declarations within the scope of the XML document, // even including the DAV: namespace. InnerXML []byte `xml:",innerxml"` } // ixmlProperty is the same as the Property type except it holds an ixml.Name // instead of an xml.Name. type ixmlProperty struct { XMLName ixml.Name Lang string `xml:"xml:lang,attr,omitempty"` InnerXML []byte `xml:",innerxml"` } // http://www.webdav.org/specs/rfc4918.html#ELEMENT_error // See multistatusWriter for the "D:" namespace prefix. type xmlError struct { XMLName ixml.Name `xml:"D:error"` InnerXML []byte `xml:",innerxml"` } // http://www.webdav.org/specs/rfc4918.html#ELEMENT_propstat // See multistatusWriter for the "D:" namespace prefix. type propstat struct { Prop []Property `xml:"D:prop>_ignored_"` Status string `xml:"D:status"` Error *xmlError `xml:"D:error"` ResponseDescription string `xml:"D:responsedescription,omitempty"` } // ixmlPropstat is the same as the propstat type except it holds an ixml.Name // instead of an xml.Name. type ixmlPropstat struct { Prop []ixmlProperty `xml:"D:prop>_ignored_"` Status string `xml:"D:status"` Error *xmlError `xml:"D:error"` ResponseDescription string `xml:"D:responsedescription,omitempty"` } // MarshalXML prepends the "D:" namespace prefix on properties in the DAV: namespace // before encoding. See multistatusWriter. func (ps propstat) MarshalXML(e *ixml.Encoder, start ixml.StartElement) error { // Convert from a propstat to an ixmlPropstat. ixmlPs := ixmlPropstat{ Prop: make([]ixmlProperty, len(ps.Prop)), Status: ps.Status, Error: ps.Error, ResponseDescription: ps.ResponseDescription, } for k, prop := range ps.Prop { ixmlPs.Prop[k] = ixmlProperty{ XMLName: ixml.Name(prop.XMLName), Lang: prop.Lang, InnerXML: prop.InnerXML, } } for k, prop := range ixmlPs.Prop { if prop.XMLName.Space == "DAV:" { prop.XMLName = ixml.Name{Space: "", Local: "D:" + prop.XMLName.Local} ixmlPs.Prop[k] = prop } } // Distinct type to avoid infinite recursion of MarshalXML. type newpropstat ixmlPropstat return e.EncodeElement(newpropstat(ixmlPs), start) } // http://www.webdav.org/specs/rfc4918.html#ELEMENT_response // See multistatusWriter for the "D:" namespace prefix. type response struct { XMLName ixml.Name `xml:"D:response"` Href []string `xml:"D:href"` Propstat []propstat `xml:"D:propstat"` Status string `xml:"D:status,omitempty"` Error *xmlError `xml:"D:error"` ResponseDescription string `xml:"D:responsedescription,omitempty"` } // MultistatusWriter marshals one or more Responses into a XML // multistatus response. // See http://www.webdav.org/specs/rfc4918.html#ELEMENT_multistatus // TODO(rsto, mpl): As a workaround, the "D:" namespace prefix, defined as // "DAV:" on this element, is prepended on the nested response, as well as on all // its nested elements. All property names in the DAV: namespace are prefixed as // well. This is because some versions of Mini-Redirector (on windows 7) ignore // elements with a default namespace (no prefixed namespace). A less intrusive fix // should be possible after golang.org/cl/11074. See https://golang.org/issue/11177 type multistatusWriter struct { // ResponseDescription contains the optional responsedescription // of the multistatus XML element. Only the latest content before // close will be emitted. Empty response descriptions are not // written. responseDescription string w http.ResponseWriter enc *ixml.Encoder } // Write validates and emits a DAV response as part of a multistatus response // element. // // It sets the HTTP status code of its underlying http.ResponseWriter to 207 // (Multi-Status) and populates the Content-Type header. If r is the // first, valid response to be written, Write prepends the XML representation // of r with a multistatus tag. Callers must call close after the last response // has been written. func (w *multistatusWriter) write(r *response) error { switch len(r.Href) { case 0: return errInvalidResponse case 1: if len(r.Propstat) > 0 != (r.Status == "") { return errInvalidResponse } default: if len(r.Propstat) > 0 || r.Status == "" { return errInvalidResponse } } err := w.writeHeader() if err != nil { return err } return w.enc.Encode(r) } // writeHeader writes a XML multistatus start element on w's underlying // http.ResponseWriter and returns the result of the write operation. // After the first write attempt, writeHeader becomes a no-op. func (w *multistatusWriter) writeHeader() error { if w.enc != nil { return nil } w.w.Header().Add("Content-Type", "text/xml; charset=utf-8") w.w.WriteHeader(StatusMulti) _, err := fmt.Fprintf(w.w, ``) if err != nil { return err } w.enc = ixml.NewEncoder(w.w) return w.enc.EncodeToken(ixml.StartElement{ Name: ixml.Name{ Space: "DAV:", Local: "multistatus", }, Attr: []ixml.Attr{{ Name: ixml.Name{Space: "xmlns", Local: "D"}, Value: "DAV:", }}, }) } // Close completes the marshalling of the multistatus response. It returns // an error if the multistatus response could not be completed. If both the // return value and field enc of w are nil, then no multistatus response has // been written. func (w *multistatusWriter) close() error { if w.enc == nil { return nil } var end []ixml.Token if w.responseDescription != "" { name := ixml.Name{Space: "DAV:", Local: "responsedescription"} end = append(end, ixml.StartElement{Name: name}, ixml.CharData(w.responseDescription), ixml.EndElement{Name: name}, ) } end = append(end, ixml.EndElement{ Name: ixml.Name{Space: "DAV:", Local: "multistatus"}, }) for _, t := range end { err := w.enc.EncodeToken(t) if err != nil { return err } } return w.enc.Flush() } var xmlLangName = ixml.Name{Space: "http://www.w3.org/XML/1998/namespace", Local: "lang"} func xmlLang(s ixml.StartElement, d string) string { for _, attr := range s.Attr { if attr.Name == xmlLangName { return attr.Value } } return d } type xmlValue []byte func (v *xmlValue) UnmarshalXML(d *ixml.Decoder, start ixml.StartElement) error { // The XML value of a property can be arbitrary, mixed-content XML. // To make sure that the unmarshalled value contains all required // namespaces, we encode all the property value XML tokens into a // buffer. This forces the encoder to redeclare any used namespaces. var b bytes.Buffer e := ixml.NewEncoder(&b) for { t, err := next(d) if err != nil { return err } if e, ok := t.(ixml.EndElement); ok && e.Name == start.Name { break } if err = e.EncodeToken(t); err != nil { return err } } err := e.Flush() if err != nil { return err } *v = b.Bytes() return nil } // http://www.webdav.org/specs/rfc4918.html#ELEMENT_prop (for proppatch) type proppatchProps []Property // UnmarshalXML appends the property names and values enclosed within start // to ps. // // An xml:lang attribute that is defined either on the DAV:prop or property // name XML element is propagated to the property's Lang field. // // UnmarshalXML returns an error if start does not contain any properties or if // property values contain syntactically incorrect XML. func (ps *proppatchProps) UnmarshalXML(d *ixml.Decoder, start ixml.StartElement) error { lang := xmlLang(start, "") for { t, err := next(d) if err != nil { return err } switch elem := t.(type) { case ixml.EndElement: if len(*ps) == 0 { return fmt.Errorf("%s must not be empty", start.Name.Local) } return nil case ixml.StartElement: p := Property{ XMLName: xml.Name(t.(ixml.StartElement).Name), Lang: xmlLang(t.(ixml.StartElement), lang), } err = d.DecodeElement(((*xmlValue)(&p.InnerXML)), &elem) if err != nil { return err } *ps = append(*ps, p) } } } // http://www.webdav.org/specs/rfc4918.html#ELEMENT_set // http://www.webdav.org/specs/rfc4918.html#ELEMENT_remove type setRemove struct { XMLName ixml.Name Lang string `xml:"xml:lang,attr,omitempty"` Prop proppatchProps `xml:"DAV: prop"` } // http://www.webdav.org/specs/rfc4918.html#ELEMENT_propertyupdate type propertyupdate struct { XMLName ixml.Name `xml:"DAV: propertyupdate"` Lang string `xml:"xml:lang,attr,omitempty"` SetRemove []setRemove `xml:",any"` } func readProppatch(r io.Reader) (patches []Proppatch, status int, err error) { var pu propertyupdate if err = ixml.NewDecoder(r).Decode(&pu); err != nil { return nil, http.StatusBadRequest, err } for _, op := range pu.SetRemove { remove := false switch op.XMLName { case ixml.Name{Space: "DAV:", Local: "set"}: // No-op. case ixml.Name{Space: "DAV:", Local: "remove"}: for _, p := range op.Prop { if len(p.InnerXML) > 0 { return nil, http.StatusBadRequest, errInvalidProppatch } } remove = true default: return nil, http.StatusBadRequest, errInvalidProppatch } patches = append(patches, Proppatch{Remove: remove, Props: op.Prop}) } return patches, 0, nil } ================================================ FILE: server/webdav/xml_test.go ================================================ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package webdav import ( "bytes" "encoding/xml" "fmt" "io" "net/http" "net/http/httptest" "reflect" "sort" "strings" "testing" ixml "github.com/OpenListTeam/OpenList/v4/server/webdav/internal/xml" ) func TestReadLockInfo(t *testing.T) { // The "section x.y.z" test cases come from section x.y.z of the spec at // http://www.webdav.org/specs/rfc4918.html testCases := []struct { desc string input string wantLI lockInfo wantStatus int }{{ "bad: junk", "xxx", lockInfo{}, http.StatusBadRequest, }, { "bad: invalid owner XML", "" + "\n" + " \n" + " \n" + " \n" + " no end tag \n" + " \n" + "", lockInfo{}, http.StatusBadRequest, }, { "bad: invalid UTF-8", "" + "\n" + " \n" + " \n" + " \n" + " \xff \n" + " \n" + "", lockInfo{}, http.StatusBadRequest, }, { "bad: unfinished XML #1", "" + "\n" + " \n" + " \n", lockInfo{}, http.StatusBadRequest, }, { "bad: unfinished XML #2", "" + "\n" + " \n" + " \n" + " \n", lockInfo{}, http.StatusBadRequest, }, { "good: empty", "", lockInfo{}, 0, }, { "good: plain-text owner", "" + "\n" + " \n" + " \n" + " gopher\n" + "", lockInfo{ XMLName: ixml.Name{Space: "DAV:", Local: "lockinfo"}, Exclusive: new(struct{}), Write: new(struct{}), Owner: owner{ InnerXML: "gopher", }, }, 0, }, { "section 9.10.7", "" + "\n" + " \n" + " \n" + " \n" + " http://example.org/~ejw/contact.html\n" + " \n" + "", lockInfo{ XMLName: ixml.Name{Space: "DAV:", Local: "lockinfo"}, Exclusive: new(struct{}), Write: new(struct{}), Owner: owner{ InnerXML: "\n http://example.org/~ejw/contact.html\n ", }, }, 0, }} for _, tc := range testCases { li, status, err := readLockInfo(strings.NewReader(tc.input)) if tc.wantStatus != 0 { if err == nil { t.Errorf("%s: got nil error, want non-nil", tc.desc) continue } } else if err != nil { t.Errorf("%s: %v", tc.desc, err) continue } if !reflect.DeepEqual(li, tc.wantLI) || status != tc.wantStatus { t.Errorf("%s:\ngot lockInfo=%v, status=%v\nwant lockInfo=%v, status=%v", tc.desc, li, status, tc.wantLI, tc.wantStatus) continue } } } func TestReadPropfind(t *testing.T) { testCases := []struct { desc string input string wantPF propfind wantStatus int }{{ desc: "propfind: propname", input: "" + "\n" + " \n" + "", wantPF: propfind{ XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, Propname: new(struct{}), }, }, { desc: "propfind: empty body means allprop", input: "", wantPF: propfind{ Allprop: new(struct{}), }, }, { desc: "propfind: allprop", input: "" + "\n" + " \n" + "", wantPF: propfind{ XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, Allprop: new(struct{}), }, }, { desc: "propfind: allprop followed by include", input: "" + "\n" + " \n" + " \n" + "", wantPF: propfind{ XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, Allprop: new(struct{}), Include: propfindProps{xml.Name{Space: "DAV:", Local: "displayname"}}, }, }, { desc: "propfind: include followed by allprop", input: "" + "\n" + " \n" + " \n" + "", wantPF: propfind{ XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, Allprop: new(struct{}), Include: propfindProps{xml.Name{Space: "DAV:", Local: "displayname"}}, }, }, { desc: "propfind: propfind", input: "" + "\n" + " \n" + "", wantPF: propfind{ XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, Prop: propfindProps{xml.Name{Space: "DAV:", Local: "displayname"}}, }, }, { desc: "propfind: prop with ignored comments", input: "" + "\n" + " \n" + " \n" + " \n" + " \n" + "", wantPF: propfind{ XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, Prop: propfindProps{xml.Name{Space: "DAV:", Local: "displayname"}}, }, }, { desc: "propfind: propfind with ignored whitespace", input: "" + "\n" + " \n" + "", wantPF: propfind{ XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, Prop: propfindProps{xml.Name{Space: "DAV:", Local: "displayname"}}, }, }, { desc: "propfind: propfind with ignored mixed-content", input: "" + "\n" + " foobar\n" + "", wantPF: propfind{ XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, Prop: propfindProps{xml.Name{Space: "DAV:", Local: "displayname"}}, }, }, { desc: "propfind: propname with ignored element (section A.4)", input: "" + "\n" + " \n" + " *boss*\n" + "", wantPF: propfind{ XMLName: ixml.Name{Space: "DAV:", Local: "propfind"}, Propname: new(struct{}), }, }, { desc: "propfind: bad: junk", input: "xxx", wantStatus: http.StatusBadRequest, }, { desc: "propfind: bad: propname and allprop (section A.3)", input: "" + "\n" + " " + " " + "", wantStatus: http.StatusBadRequest, }, { desc: "propfind: bad: propname and prop", input: "" + "\n" + " \n" + " \n" + "", wantStatus: http.StatusBadRequest, }, { desc: "propfind: bad: allprop and prop", input: "" + "\n" + " \n" + " \n" + "", wantStatus: http.StatusBadRequest, }, { desc: "propfind: bad: empty propfind with ignored element (section A.4)", input: "" + "\n" + " \n" + "", wantStatus: http.StatusBadRequest, }, { desc: "propfind: bad: empty prop", input: "" + "\n" + " \n" + "", wantStatus: http.StatusBadRequest, }, { desc: "propfind: bad: prop with just chardata", input: "" + "\n" + " foo\n" + "", wantStatus: http.StatusBadRequest, }, { desc: "bad: interrupted prop", input: "" + "\n" + " \n", wantStatus: http.StatusBadRequest, }, { desc: "bad: malformed end element prop", input: "" + "\n" + " \n", wantStatus: http.StatusBadRequest, }, { desc: "propfind: bad: property with chardata value", input: "" + "\n" + " bar\n" + "", wantStatus: http.StatusBadRequest, }, { desc: "propfind: bad: property with whitespace value", input: "" + "\n" + " \n" + "", wantStatus: http.StatusBadRequest, }, { desc: "propfind: bad: include without allprop", input: "" + "\n" + " \n" + "", wantStatus: http.StatusBadRequest, }} for _, tc := range testCases { pf, status, err := readPropfind(strings.NewReader(tc.input)) if tc.wantStatus != 0 { if err == nil { t.Errorf("%s: got nil error, want non-nil", tc.desc) continue } } else if err != nil { t.Errorf("%s: %v", tc.desc, err) continue } if !reflect.DeepEqual(pf, tc.wantPF) || status != tc.wantStatus { t.Errorf("%s:\ngot propfind=%v, status=%v\nwant propfind=%v, status=%v", tc.desc, pf, status, tc.wantPF, tc.wantStatus) continue } } } func TestMultistatusWriter(t *testing.T) { ///The "section x.y.z" test cases come from section x.y.z of the spec at // http://www.webdav.org/specs/rfc4918.html testCases := []struct { desc string responses []response respdesc string writeHeader bool wantXML string wantCode int wantErr error }{{ desc: "section 9.2.2 (failed dependency)", responses: []response{{ Href: []string{"http://example.com/foo"}, Propstat: []propstat{{ Prop: []Property{{ XMLName: xml.Name{ Space: "http://ns.example.com/", Local: "Authors", }, }}, Status: "HTTP/1.1 424 Failed Dependency", }, { Prop: []Property{{ XMLName: xml.Name{ Space: "http://ns.example.com/", Local: "Copyright-Owner", }, }}, Status: "HTTP/1.1 409 Conflict", }}, ResponseDescription: "Copyright Owner cannot be deleted or altered.", }}, wantXML: `` + `` + `` + ` ` + ` http://example.com/foo` + ` ` + ` ` + ` ` + ` ` + ` HTTP/1.1 424 Failed Dependency` + ` ` + ` ` + ` ` + ` ` + ` ` + ` HTTP/1.1 409 Conflict` + ` ` + ` Copyright Owner cannot be deleted or altered.` + `` + ``, wantCode: StatusMulti, }, { desc: "section 9.6.2 (lock-token-submitted)", responses: []response{{ Href: []string{"http://example.com/foo"}, Status: "HTTP/1.1 423 Locked", Error: &xmlError{ InnerXML: []byte(``), }, }}, wantXML: `` + `` + `` + ` ` + ` http://example.com/foo` + ` HTTP/1.1 423 Locked` + ` ` + ` ` + ``, wantCode: StatusMulti, }, { desc: "section 9.1.3", responses: []response{{ Href: []string{"http://example.com/foo"}, Propstat: []propstat{{ Prop: []Property{{ XMLName: xml.Name{Space: "http://ns.example.com/boxschema/", Local: "bigbox"}, InnerXML: []byte(`` + `` + `Box type A` + ``), }, { XMLName: xml.Name{Space: "http://ns.example.com/boxschema/", Local: "author"}, InnerXML: []byte(`` + `` + `J.J. Johnson` + ``), }}, Status: "HTTP/1.1 200 OK", }, { Prop: []Property{{ XMLName: xml.Name{Space: "http://ns.example.com/boxschema/", Local: "DingALing"}, }, { XMLName: xml.Name{Space: "http://ns.example.com/boxschema/", Local: "Random"}, }}, Status: "HTTP/1.1 403 Forbidden", ResponseDescription: "The user does not have access to the DingALing property.", }}, }}, respdesc: "There has been an access violation error.", wantXML: `` + `` + `` + ` ` + ` http://example.com/foo` + ` ` + ` ` + ` Box type A` + ` J.J. Johnson` + ` ` + ` HTTP/1.1 200 OK` + ` ` + ` ` + ` ` + ` ` + ` ` + ` ` + ` HTTP/1.1 403 Forbidden` + ` The user does not have access to the DingALing property.` + ` ` + ` ` + ` There has been an access violation error.` + ``, wantCode: StatusMulti, }, { desc: "no response written", // default of http.responseWriter wantCode: http.StatusOK, }, { desc: "no response written (with description)", respdesc: "too bad", // default of http.responseWriter wantCode: http.StatusOK, }, { desc: "empty multistatus with header", writeHeader: true, wantXML: ``, wantCode: StatusMulti, }, { desc: "bad: no href", responses: []response{{ Propstat: []propstat{{ Prop: []Property{{ XMLName: xml.Name{ Space: "http://example.com/", Local: "foo", }, }}, Status: "HTTP/1.1 200 OK", }}, }}, wantErr: errInvalidResponse, // default of http.responseWriter wantCode: http.StatusOK, }, { desc: "bad: multiple hrefs and no status", responses: []response{{ Href: []string{"http://example.com/foo", "http://example.com/bar"}, }}, wantErr: errInvalidResponse, // default of http.responseWriter wantCode: http.StatusOK, }, { desc: "bad: one href and no propstat", responses: []response{{ Href: []string{"http://example.com/foo"}, }}, wantErr: errInvalidResponse, // default of http.responseWriter wantCode: http.StatusOK, }, { desc: "bad: status with one href and propstat", responses: []response{{ Href: []string{"http://example.com/foo"}, Propstat: []propstat{{ Prop: []Property{{ XMLName: xml.Name{ Space: "http://example.com/", Local: "foo", }, }}, Status: "HTTP/1.1 200 OK", }}, Status: "HTTP/1.1 200 OK", }}, wantErr: errInvalidResponse, // default of http.responseWriter wantCode: http.StatusOK, }, { desc: "bad: multiple hrefs and propstat", responses: []response{{ Href: []string{ "http://example.com/foo", "http://example.com/bar", }, Propstat: []propstat{{ Prop: []Property{{ XMLName: xml.Name{ Space: "http://example.com/", Local: "foo", }, }}, Status: "HTTP/1.1 200 OK", }}, }}, wantErr: errInvalidResponse, // default of http.responseWriter wantCode: http.StatusOK, }} n := xmlNormalizer{omitWhitespace: true} loop: for _, tc := range testCases { rec := httptest.NewRecorder() w := multistatusWriter{w: rec, responseDescription: tc.respdesc} if tc.writeHeader { if err := w.writeHeader(); err != nil { t.Errorf("%s: got writeHeader error %v, want nil", tc.desc, err) continue } } for _, r := range tc.responses { if err := w.write(&r); err != nil { if err != tc.wantErr { t.Errorf("%s: got write error %v, want %v", tc.desc, err, tc.wantErr) } continue loop } } if err := w.close(); err != tc.wantErr { t.Errorf("%s: got close error %v, want %v", tc.desc, err, tc.wantErr) continue } if rec.Code != tc.wantCode { t.Errorf("%s: got HTTP status code %d, want %d\n", tc.desc, rec.Code, tc.wantCode) continue } gotXML := rec.Body.String() eq, err := n.equalXML(strings.NewReader(gotXML), strings.NewReader(tc.wantXML)) if err != nil { t.Errorf("%s: equalXML: %v", tc.desc, err) continue } if !eq { t.Errorf("%s: XML body\ngot %s\nwant %s", tc.desc, gotXML, tc.wantXML) } } } func TestReadProppatch(t *testing.T) { ppStr := func(pps []Proppatch) string { var outer []string for _, pp := range pps { var inner []string for _, p := range pp.Props { inner = append(inner, fmt.Sprintf("{XMLName: %q, Lang: %q, InnerXML: %q}", p.XMLName, p.Lang, p.InnerXML)) } outer = append(outer, fmt.Sprintf("{Remove: %t, Props: [%s]}", pp.Remove, strings.Join(inner, ", "))) } return "[" + strings.Join(outer, ", ") + "]" } testCases := []struct { desc string input string wantPP []Proppatch wantStatus int }{{ desc: "proppatch: section 9.2 (with simple property value)", input: `` + `` + `` + ` ` + ` somevalue` + ` ` + ` ` + ` ` + ` ` + ``, wantPP: []Proppatch{{ Props: []Property{{ xml.Name{Space: "http://ns.example.com/z/", Local: "Authors"}, "", []byte(`somevalue`), }}, }, { Remove: true, Props: []Property{{ xml.Name{Space: "http://ns.example.com/z/", Local: "Copyright-Owner"}, "", nil, }}, }}, }, { desc: "proppatch: lang attribute on prop", input: `` + `` + `` + ` ` + ` ` + ` ` + ` ` + ` ` + ``, wantPP: []Proppatch{{ Props: []Property{{ xml.Name{Space: "http://example.com/ns", Local: "foo"}, "en", nil, }}, }}, }, { desc: "bad: remove with value", input: `` + `` + `` + ` ` + ` ` + ` ` + ` Jim Whitehead` + ` ` + ` ` + ` ` + ``, wantStatus: http.StatusBadRequest, }, { desc: "bad: empty propertyupdate", input: `` + `` + ``, wantStatus: http.StatusBadRequest, }, { desc: "bad: empty prop", input: `` + `` + `` + ` ` + ` ` + ` ` + ``, wantStatus: http.StatusBadRequest, }} for _, tc := range testCases { pp, status, err := readProppatch(strings.NewReader(tc.input)) if tc.wantStatus != 0 { if err == nil { t.Errorf("%s: got nil error, want non-nil", tc.desc) continue } } else if err != nil { t.Errorf("%s: %v", tc.desc, err) continue } if status != tc.wantStatus { t.Errorf("%s: got status %d, want %d", tc.desc, status, tc.wantStatus) continue } if !reflect.DeepEqual(pp, tc.wantPP) || status != tc.wantStatus { t.Errorf("%s: proppatch\ngot %v\nwant %v", tc.desc, ppStr(pp), ppStr(tc.wantPP)) } } } func TestUnmarshalXMLValue(t *testing.T) { testCases := []struct { desc string input string wantVal string }{{ desc: "simple char data", input: "foo", wantVal: "foo", }, { desc: "empty element", input: "", wantVal: "", }, { desc: "preserve namespace", input: ``, wantVal: ``, }, { desc: "preserve root element namespace", input: ``, wantVal: ``, }, { desc: "preserve whitespace", input: " \t ", wantVal: " \t ", }, { desc: "preserve mixed content", input: ` a `, wantVal: ` a `, }, { desc: "section 9.2", input: `` + `` + ` Jim Whitehead` + ` Roy Fielding` + ``, wantVal: `` + ` Jim Whitehead` + ` Roy Fielding`, }, { desc: "section 4.3.1 (mixed content)", input: `` + `` + ` Jane Doe` + ` ` + ` mailto:jane.doe@example.com` + ` http://www.example.com` + ` ` + ` Jane has been working way too long on the` + ` long-awaited revision of ]]>.` + ` ` + ``, wantVal: `` + ` Jane Doe` + ` ` + ` mailto:jane.doe@example.com` + ` http://www.example.com` + ` ` + ` Jane has been working way too long on the` + ` long-awaited revision of <RFC2518>.` + ` `, }} var n xmlNormalizer for _, tc := range testCases { d := ixml.NewDecoder(strings.NewReader(tc.input)) var v xmlValue if err := d.Decode(&v); err != nil { t.Errorf("%s: got error %v, want nil", tc.desc, err) continue } eq, err := n.equalXML(bytes.NewReader(v), strings.NewReader(tc.wantVal)) if err != nil { t.Errorf("%s: equalXML: %v", tc.desc, err) continue } if !eq { t.Errorf("%s:\ngot %s\nwant %s", tc.desc, string(v), tc.wantVal) } } } // xmlNormalizer normalizes XML. type xmlNormalizer struct { // omitWhitespace instructs to ignore whitespace between element tags. omitWhitespace bool // omitComments instructs to ignore XML comments. omitComments bool } // normalize writes the normalized XML content of r to w. It applies the // following rules // // - Rename namespace prefixes according to an internal heuristic. // - Remove unnecessary namespace declarations. // - Sort attributes in XML start elements in lexical order of their // fully qualified name. // - Remove XML directives and processing instructions. // - Remove CDATA between XML tags that only contains whitespace, if // instructed to do so. // - Remove comments, if instructed to do so. func (n *xmlNormalizer) normalize(w io.Writer, r io.Reader) error { d := ixml.NewDecoder(r) e := ixml.NewEncoder(w) for { t, err := d.Token() if err != nil { if t == nil && err == io.EOF { break } return err } switch val := t.(type) { case ixml.Directive, ixml.ProcInst: continue case ixml.Comment: if n.omitComments { continue } case ixml.CharData: if n.omitWhitespace && len(bytes.TrimSpace(val)) == 0 { continue } case ixml.StartElement: start, _ := ixml.CopyToken(val).(ixml.StartElement) attr := start.Attr[:0] for _, a := range start.Attr { if a.Name.Space == "xmlns" || a.Name.Local == "xmlns" { continue } attr = append(attr, a) } sort.Sort(byName(attr)) start.Attr = attr t = start } err = e.EncodeToken(t) if err != nil { return err } } return e.Flush() } // equalXML tests for equality of the normalized XML contents of a and b. func (n *xmlNormalizer) equalXML(a, b io.Reader) (bool, error) { var buf bytes.Buffer if err := n.normalize(&buf, a); err != nil { return false, err } normA := buf.String() buf.Reset() if err := n.normalize(&buf, b); err != nil { return false, err } normB := buf.String() return normA == normB, nil } type byName []ixml.Attr func (a byName) Len() int { return len(a) } func (a byName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byName) Less(i, j int) bool { if a[i].Name.Space != a[j].Name.Space { return a[i].Name.Space < a[j].Name.Space } return a[i].Name.Local < a[j].Name.Local } ================================================ FILE: server/webdav.go ================================================ package server import ( "crypto/subtle" "net/http" "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/server/middlewares" "github.com/OpenListTeam/OpenList/v4/server/webdav" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) var handler *webdav.Handler func WebDav(dav *gin.RouterGroup) { handler = &webdav.Handler{ Prefix: path.Join(conf.URL.Path, "/dav"), LockSystem: webdav.NewMemLS(), Logger: func(request *http.Request, err error) { log.Errorf("%s %s %+v", request.Method, request.URL.Path, err) }, } dav.Use(WebDAVAuth) uploadLimiter := middlewares.UploadRateLimiter(stream.ClientUploadLimit) downloadLimiter := middlewares.DownloadRateLimiter(stream.ClientDownloadLimit) dav.Any("/*path", uploadLimiter, downloadLimiter, ServeWebDAV) dav.Any("", uploadLimiter, downloadLimiter, ServeWebDAV) dav.Handle("PROPFIND", "/*path", ServeWebDAV) dav.Handle("PROPFIND", "", ServeWebDAV) dav.Handle("MKCOL", "/*path", ServeWebDAV) dav.Handle("LOCK", "/*path", ServeWebDAV) dav.Handle("UNLOCK", "/*path", ServeWebDAV) dav.Handle("PROPPATCH", "/*path", ServeWebDAV) dav.Handle("COPY", "/*path", ServeWebDAV) dav.Handle("MOVE", "/*path", ServeWebDAV) } func ServeWebDAV(c *gin.Context) { handler.ServeHTTP(c.Writer, c.Request) } func WebDAVAuth(c *gin.Context) { // check count of login ip := c.ClientIP() guest, _ := op.GetGuest() count, cok := model.LoginCache.Get(ip) if cok && count >= model.DefaultMaxAuthRetries { if c.Request.Method == "OPTIONS" { common.GinWithValue(c, conf.UserKey, guest) c.Next() return } c.Status(http.StatusTooManyRequests) c.Abort() model.LoginCache.Expire(ip, model.DefaultLockDuration) return } username, password, ok := c.Request.BasicAuth() if !ok { bt := c.GetHeader("Authorization") log.Debugf("[webdav auth] token: %s", bt) if strings.HasPrefix(bt, "Bearer") { bt = strings.TrimPrefix(bt, "Bearer ") token := setting.GetStr(conf.Token) if token != "" && subtle.ConstantTimeCompare([]byte(bt), []byte(token)) == 1 { admin, err := op.GetAdmin() if err != nil { log.Errorf("[webdav auth] failed get admin user: %+v", err) c.Status(http.StatusInternalServerError) c.Abort() return } common.GinWithValue(c, conf.UserKey, admin) c.Next() return } } if c.Request.Method == "OPTIONS" { common.GinWithValue(c, conf.UserKey, guest) c.Next() return } c.Writer.Header()["WWW-Authenticate"] = []string{`Basic realm="openlist"`} c.Status(http.StatusUnauthorized) c.Abort() return } user, ok := tryLogin(username, password) if !ok { if c.Request.Method == "OPTIONS" { common.GinWithValue(c, conf.UserKey, guest) c.Next() return } model.LoginCache.Set(ip, count+1) c.Status(http.StatusUnauthorized) c.Abort() return } // at least auth is successful till here model.LoginCache.Del(ip) if user.Disabled || !user.CanWebdavRead() { if c.Request.Method == "OPTIONS" { common.GinWithValue(c, conf.UserKey, guest) c.Next() return } c.Status(http.StatusForbidden) c.Abort() return } if (c.Request.Method == "PUT" || c.Request.Method == "MKCOL") && (!user.CanWebdavManage() || !user.CanWrite()) { c.Status(http.StatusForbidden) c.Abort() return } if c.Request.Method == "MOVE" && (!user.CanWebdavManage() || (!user.CanMove() && !user.CanRename())) { c.Status(http.StatusForbidden) c.Abort() return } if c.Request.Method == "COPY" && (!user.CanWebdavManage() || !user.CanCopy()) { c.Status(http.StatusForbidden) c.Abort() return } if c.Request.Method == "DELETE" && (!user.CanWebdavManage() || !user.CanRemove()) { c.Status(http.StatusForbidden) c.Abort() return } if c.Request.Method == "PROPPATCH" && !user.CanWebdavManage() { c.Status(http.StatusForbidden) c.Abort() return } common.GinWithValue(c, conf.UserKey, user) c.Next() } func tryLogin(username, password string) (*model.User, bool) { user, err := op.GetUserByName(username) if err == nil { err = user.ValidateRawPassword(password) if err != nil && setting.GetBool(conf.LdapLoginEnabled) && user.AllowLdap { err = common.HandleLdapLogin(username, password) } } else if setting.GetBool(conf.LdapLoginEnabled) && model.CanWebdavRead(int32(setting.GetInt(conf.LdapDefaultPermission, 0))) { user, err = tryLdapLoginAndRegister(username, password) } return user, err == nil } ================================================ FILE: wrapper/zcc-arm64 ================================================ #!/bin/sh zig cc -target aarch64-windows-gnu $@ ================================================ FILE: wrapper/zcc-win7 ================================================ #!/bin/sh zig cc -target x86_64-windows-gnu $@ ================================================ FILE: wrapper/zcc-win7-386 ================================================ #!/bin/sh zig cc -target x86-windows-gnu $@ ================================================ FILE: wrapper/zcxx-arm64 ================================================ #!/bin/sh zig c++ -target aarch64-windows-gnu $@ ================================================ FILE: wrapper/zcxx-win7 ================================================ #!/bin/sh zig c++ -target x86_64-windows-gnu $@ ================================================ FILE: wrapper/zcxx-win7-386 ================================================ #!/bin/sh zig c++ -target x86-windows-gnu $@