Repository: ipfs/kubo Branch: master Commit: ae989328a178 Files: 921 Total size: 5.7 MB Directory structure: gitextract__oi967oe/ ├── .codeclimate.yml ├── .cspell.yml ├── .dockerignore ├── .gitattributes ├── .github/ │ ├── CODEOWNERS │ ├── FUNDING.yml │ ├── ISSUE_TEMPLATE/ │ │ ├── bug-report.yml │ │ ├── config.yml │ │ ├── doc.yml │ │ ├── enhancement.yml │ │ └── feature.yml │ ├── auto-comment.yml │ ├── build-platforms.yml │ ├── dependabot.yml │ ├── pull_request_template.md │ └── workflows/ │ ├── changelog.yml │ ├── codeql-analysis.yml │ ├── dependabot-tidy.yml │ ├── docker-check.yml │ ├── docker-image.yml │ ├── gateway-conformance.yml │ ├── generated-pr.yml │ ├── gobuild.yml │ ├── golang-analysis.yml │ ├── golint.yml │ ├── gotest.yml │ ├── interop.yml │ ├── sharness.yml │ ├── spellcheck.yml │ ├── stale.yml │ ├── sync-release-assets.yml │ └── test-migrations.yml ├── .gitignore ├── .golangci.yml ├── .hadolint.yaml ├── .mailmap ├── AGENTS.md ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── FUNDING.json ├── GNUmakefile ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT ├── Makefile ├── README.md ├── Rules.mk ├── SECURITY.md ├── assets/ │ ├── README.md │ ├── assets.go │ └── init-doc/ │ ├── about │ ├── contact │ ├── docs/ │ │ └── index │ ├── help │ ├── ping │ ├── quick-start │ ├── readme │ └── security-notes ├── bin/ │ ├── Rules.mk │ ├── archive-branches.sh │ ├── container_daemon │ ├── container_init_run │ ├── dist_get │ ├── gencmdref │ ├── get-docker-tags.sh │ ├── graphmd │ ├── ipns-republish │ ├── maketarball.sh │ ├── mkreleaselog │ ├── push-docker-tags.sh │ ├── test-go-build-platforms │ └── test-go-fmt ├── blocks/ │ └── blockstoreutil/ │ └── remove.go ├── client/ │ └── rpc/ │ ├── README.md │ ├── api.go │ ├── api_test.go │ ├── apifile.go │ ├── auth/ │ │ └── auth.go │ ├── block.go │ ├── dag.go │ ├── errors.go │ ├── errors_test.go │ ├── key.go │ ├── name.go │ ├── object.go │ ├── path.go │ ├── pin.go │ ├── pubsub.go │ ├── request.go │ ├── requestbuilder.go │ ├── response.go │ ├── routing.go │ ├── swarm.go │ └── unixfs.go ├── codecov.yml ├── commands/ │ ├── context.go │ └── reqlog.go ├── config/ │ ├── addresses.go │ ├── api.go │ ├── api_test.go │ ├── autoconf.go │ ├── autoconf_client.go │ ├── autoconf_test.go │ ├── autonat.go │ ├── autotls.go │ ├── bitswap.go │ ├── bootstrap_peers.go │ ├── bootstrap_peers_test.go │ ├── config.go │ ├── config_test.go │ ├── datastore.go │ ├── discovery.go │ ├── dns.go │ ├── experiments.go │ ├── gateway.go │ ├── http_retrieval.go │ ├── identity.go │ ├── import.go │ ├── import_test.go │ ├── init.go │ ├── init_test.go │ ├── internal.go │ ├── ipns.go │ ├── migration.go │ ├── migration_test.go │ ├── mounts.go │ ├── peering.go │ ├── plugins.go │ ├── profile.go │ ├── provide.go │ ├── provide_test.go │ ├── provider.go │ ├── pubsub.go │ ├── remotepin.go │ ├── reprovider.go │ ├── routing.go │ ├── routing_test.go │ ├── serialize/ │ │ ├── serialize.go │ │ └── serialize_test.go │ ├── swarm.go │ ├── types.go │ ├── types_test.go │ └── version.go ├── core/ │ ├── .gitignore │ ├── builder.go │ ├── commands/ │ │ ├── active.go │ │ ├── add.go │ │ ├── bitswap.go │ │ ├── block.go │ │ ├── bootstrap.go │ │ ├── cat.go │ │ ├── cid.go │ │ ├── cid_test.go │ │ ├── cmdenv/ │ │ │ ├── cidbase.go │ │ │ ├── cidbase_test.go │ │ │ ├── env.go │ │ │ ├── env_test.go │ │ │ └── file.go │ │ ├── cmdutils/ │ │ │ ├── sanitize.go │ │ │ ├── utils.go │ │ │ └── utils_test.go │ │ ├── commands.go │ │ ├── commands_test.go │ │ ├── completion.go │ │ ├── config.go │ │ ├── config_test.go │ │ ├── dag/ │ │ │ ├── dag.go │ │ │ ├── export.go │ │ │ ├── get.go │ │ │ ├── import.go │ │ │ ├── put.go │ │ │ ├── resolve.go │ │ │ └── stat.go │ │ ├── dht.go │ │ ├── dht_test.go │ │ ├── diag.go │ │ ├── e/ │ │ │ └── error.go │ │ ├── external.go │ │ ├── extra.go │ │ ├── files.go │ │ ├── files_test.go │ │ ├── filestore.go │ │ ├── get.go │ │ ├── get_test.go │ │ ├── helptext_test.go │ │ ├── id.go │ │ ├── keyencode/ │ │ │ └── keyencode.go │ │ ├── keystore.go │ │ ├── log.go │ │ ├── ls.go │ │ ├── ls_test.go │ │ ├── mount_nofuse.go │ │ ├── mount_unix.go │ │ ├── mount_windows.go │ │ ├── multibase.go │ │ ├── name/ │ │ │ ├── ipns.go │ │ │ ├── ipnsps.go │ │ │ ├── name.go │ │ │ └── publish.go │ │ ├── object/ │ │ │ ├── diff.go │ │ │ ├── object.go │ │ │ └── patch.go │ │ ├── p2p.go │ │ ├── pin/ │ │ │ ├── pin.go │ │ │ ├── remotepin.go │ │ │ └── remotepin_test.go │ │ ├── ping.go │ │ ├── profile.go │ │ ├── provide.go │ │ ├── pubsub.go │ │ ├── refs.go │ │ ├── repo.go │ │ ├── repo_verify_test.go │ │ ├── resolve.go │ │ ├── root.go │ │ ├── root_test.go │ │ ├── routing.go │ │ ├── shutdown.go │ │ ├── stat.go │ │ ├── stat_dht.go │ │ ├── stat_provide.go │ │ ├── stat_reprovide.go │ │ ├── swarm.go │ │ ├── swarm_addrs_autonat.go │ │ ├── sysdiag.go │ │ └── version.go │ ├── core.go │ ├── core_test.go │ ├── coreapi/ │ │ ├── block.go │ │ ├── coreapi.go │ │ ├── dag.go │ │ ├── key.go │ │ ├── name.go │ │ ├── object.go │ │ ├── path.go │ │ ├── pin.go │ │ ├── pubsub.go │ │ ├── routing.go │ │ ├── swarm.go │ │ ├── test/ │ │ │ ├── api_test.go │ │ │ └── path_test.go │ │ └── unixfs.go │ ├── corehttp/ │ │ ├── commands.go │ │ ├── corehttp.go │ │ ├── gateway.go │ │ ├── gateway_test.go │ │ ├── logs.go │ │ ├── metrics.go │ │ ├── metrics_test.go │ │ ├── mutex_profile.go │ │ ├── option_test.go │ │ ├── p2p_proxy.go │ │ ├── p2p_proxy_test.go │ │ ├── redirect.go │ │ ├── routing.go │ │ └── webui.go │ ├── coreiface/ │ │ ├── block.go │ │ ├── coreapi.go │ │ ├── dag.go │ │ ├── errors.go │ │ ├── idfmt.go │ │ ├── key.go │ │ ├── name.go │ │ ├── object.go │ │ ├── options/ │ │ │ ├── block.go │ │ │ ├── dht.go │ │ │ ├── global.go │ │ │ ├── key.go │ │ │ ├── name.go │ │ │ ├── object.go │ │ │ ├── pin.go │ │ │ ├── pubsub.go │ │ │ ├── routing.go │ │ │ ├── unixfs.go │ │ │ └── unixfs_test.go │ │ ├── pin.go │ │ ├── pubsub.go │ │ ├── routing.go │ │ ├── swarm.go │ │ ├── tests/ │ │ │ ├── api.go │ │ │ ├── block.go │ │ │ ├── dag.go │ │ │ ├── key.go │ │ │ ├── name.go │ │ │ ├── object.go │ │ │ ├── path.go │ │ │ ├── pin.go │ │ │ ├── pubsub.go │ │ │ ├── routing.go │ │ │ └── unixfs.go │ │ ├── unixfs.go │ │ └── util.go │ ├── corerepo/ │ │ ├── gc.go │ │ └── stat.go │ ├── coreunix/ │ │ ├── add.go │ │ ├── add_test.go │ │ ├── metadata.go │ │ ├── metadata_test.go │ │ └── test/ │ │ └── data/ │ │ ├── colors/ │ │ │ └── orange │ │ ├── corps/ │ │ │ └── apple │ │ └── fruits/ │ │ ├── apple │ │ └── orange │ ├── mock/ │ │ └── mock.go │ └── node/ │ ├── bitswap.go │ ├── builder.go │ ├── core.go │ ├── dns.go │ ├── groups.go │ ├── helpers/ │ │ └── helpers.go │ ├── helpers.go │ ├── identity.go │ ├── ipns.go │ ├── libp2p/ │ │ ├── addrs.go │ │ ├── discovery.go │ │ ├── dns.go │ │ ├── fd/ │ │ │ ├── sys_not_unix.go │ │ │ ├── sys_unix.go │ │ │ └── sys_windows.go │ │ ├── filters.go │ │ ├── host.go │ │ ├── hostopt.go │ │ ├── libp2p.go │ │ ├── libp2p_test.go │ │ ├── nat.go │ │ ├── peerstore.go │ │ ├── pnet.go │ │ ├── pubsub.go │ │ ├── pubsub_test.go │ │ ├── rcmgr.go │ │ ├── rcmgr_defaults.go │ │ ├── rcmgr_logging.go │ │ ├── rcmgr_logging_test.go │ │ ├── relay.go │ │ ├── routing.go │ │ ├── routingopt.go │ │ ├── routingopt_test.go │ │ ├── sec.go │ │ ├── smux.go │ │ ├── topicdiscovery.go │ │ └── transport.go │ ├── p2pforge_resolver.go │ ├── p2pforge_resolver_test.go │ ├── peering.go │ ├── provider.go │ └── storage.go ├── coverage/ │ ├── .gitignore │ ├── Rules.mk │ └── main/ │ └── main.go ├── doc.go ├── docker-compose.yaml ├── docs/ │ ├── EARLY_TESTERS.md │ ├── README.md │ ├── RELEASE_CHECKLIST.md │ ├── RELEASE_ISSUE_TEMPLATE.md │ ├── add-code-flow.md │ ├── changelogs/ │ │ ├── v0.10.md │ │ ├── v0.11.md │ │ ├── v0.12.md │ │ ├── v0.13.md │ │ ├── v0.14.md │ │ ├── v0.15.md │ │ ├── v0.16.md │ │ ├── v0.17.md │ │ ├── v0.18.md │ │ ├── v0.19.md │ │ ├── v0.2.md │ │ ├── v0.20.md │ │ ├── v0.21.md │ │ ├── v0.22.md │ │ ├── v0.23.md │ │ ├── v0.24.md │ │ ├── v0.25.md │ │ ├── v0.26.md │ │ ├── v0.27.md │ │ ├── v0.28.md │ │ ├── v0.29.md │ │ ├── v0.3.md │ │ ├── v0.30.md │ │ ├── v0.31.md │ │ ├── v0.32.md │ │ ├── v0.33.md │ │ ├── v0.34.md │ │ ├── v0.35.md │ │ ├── v0.36.md │ │ ├── v0.37.md │ │ ├── v0.38.md │ │ ├── v0.39.md │ │ ├── v0.4.md │ │ ├── v0.40.md │ │ ├── v0.41.md │ │ ├── v0.42.md │ │ ├── v0.5.md │ │ ├── v0.6.md │ │ ├── v0.7.md │ │ ├── v0.8.md │ │ └── v0.9.md │ ├── command-completion.md │ ├── config.md │ ├── content-blocking.md │ ├── customizing.md │ ├── datastores.md │ ├── debug-guide.md │ ├── delegated-routing.md │ ├── developer-certificate-of-origin │ ├── developer-guide.md │ ├── environment-variables.md │ ├── examples/ │ │ └── kubo-as-a-library/ │ │ ├── README.md │ │ ├── go.mod │ │ ├── go.sum │ │ ├── main.go │ │ └── main_test.go │ ├── experimental-features.md │ ├── file-transfer.md │ ├── fuse.md │ ├── gateway.md │ ├── http-rpc-clients.md │ ├── implement-api-bindings.md │ ├── libp2p-resource-management.md │ ├── metrics.md │ ├── p2p-tunnels.md │ ├── plugins.md │ ├── production/ │ │ └── reverse-proxy.md │ ├── provide-stats.md │ ├── releases.md │ ├── releases_thunderdome.md │ ├── specifications/ │ │ ├── keystore.md │ │ ├── repository.md │ │ └── repository_fs.md │ ├── telemetry.md │ ├── transports.md │ └── windows.md ├── fuse/ │ ├── ipns/ │ │ ├── common.go │ │ ├── ipns_test.go │ │ ├── ipns_unix.go │ │ ├── link_unix.go │ │ └── mount_unix.go │ ├── mfs/ │ │ ├── mfs_test.go │ │ ├── mfs_unix.go │ │ └── mount_unix.go │ ├── mount/ │ │ ├── fuse.go │ │ └── mount.go │ ├── node/ │ │ ├── mount_darwin.go │ │ ├── mount_nofuse.go │ │ ├── mount_notsupp.go │ │ ├── mount_test.go │ │ ├── mount_unix.go │ │ └── mount_windows.go │ └── readonly/ │ ├── doc.go │ ├── ipfs_test.go │ ├── mount_unix.go │ └── readonly_unix.go ├── gc/ │ ├── gc.go │ └── gc_test.go ├── go.mod ├── go.sum ├── misc/ │ ├── README.md │ ├── fsutil/ │ │ ├── fsutil.go │ │ └── fsutil_test.go │ ├── launchd/ │ │ ├── README.md │ │ ├── install.sh │ │ └── io.ipfs.ipfs-daemon.plist │ └── systemd/ │ ├── ipfs-api.socket │ ├── ipfs-gateway.socket │ ├── ipfs-hardened.service │ ├── ipfs-sysusers.conf │ └── ipfs.service ├── mk/ │ ├── footer.mk │ ├── git.mk │ ├── golang.mk │ ├── header.mk │ ├── tarball.mk │ └── util.mk ├── p2p/ │ ├── listener.go │ ├── local.go │ ├── p2p.go │ ├── remote.go │ └── stream.go ├── plugin/ │ ├── Rules.mk │ ├── daemon.go │ ├── daemoninternal.go │ ├── datastore.go │ ├── fx.go │ ├── ipld.go │ ├── loader/ │ │ ├── Rules.mk │ │ ├── load_nocgo.go │ │ ├── load_noplugin.go │ │ ├── load_unix.go │ │ ├── loader.go │ │ ├── preload.go │ │ ├── preload.sh │ │ └── preload_list │ ├── plugin.go │ ├── plugins/ │ │ ├── .gitignore │ │ ├── Rules.mk │ │ ├── badgerds/ │ │ │ └── badgerds.go │ │ ├── dagjose/ │ │ │ └── dagjose.go │ │ ├── flatfs/ │ │ │ └── flatfs.go │ │ ├── fxtest/ │ │ │ └── fxtest.go │ │ ├── gen_main.sh │ │ ├── git/ │ │ │ └── git.go │ │ ├── levelds/ │ │ │ └── levelds.go │ │ ├── nopfs/ │ │ │ └── nopfs.go │ │ ├── pebbleds/ │ │ │ └── pebbleds.go │ │ ├── peerlog/ │ │ │ ├── peerlog.go │ │ │ └── peerlog_test.go │ │ └── telemetry/ │ │ ├── telemetry.go │ │ ├── telemetry_test.go │ │ └── telemetry_uuid │ └── tracer.go ├── profile/ │ ├── goroutines.go │ ├── profile.go │ └── profile_test.go ├── repo/ │ ├── common/ │ │ ├── common.go │ │ └── common_test.go │ ├── fsrepo/ │ │ ├── config_test.go │ │ ├── datastores.go │ │ ├── doc.go │ │ ├── fsrepo.go │ │ ├── fsrepo_test.go │ │ ├── migrations/ │ │ │ ├── README.md │ │ │ ├── atomicfile/ │ │ │ │ ├── atomicfile.go │ │ │ │ └── atomicfile_test.go │ │ │ ├── common/ │ │ │ │ ├── base.go │ │ │ │ ├── config_helpers.go │ │ │ │ ├── migration.go │ │ │ │ ├── testing_helpers.go │ │ │ │ └── utils.go │ │ │ ├── embedded.go │ │ │ ├── embedded_test.go │ │ │ ├── fetch.go │ │ │ ├── fetch_test.go │ │ │ ├── fetcher.go │ │ │ ├── fs-repo-16-to-17/ │ │ │ │ ├── main.go │ │ │ │ └── migration/ │ │ │ │ ├── migration.go │ │ │ │ └── migration_test.go │ │ │ ├── fs-repo-17-to-18/ │ │ │ │ ├── main.go │ │ │ │ └── migration/ │ │ │ │ ├── migration.go │ │ │ │ └── migration_test.go │ │ │ ├── httpfetcher.go │ │ │ ├── ipfsdir.go │ │ │ ├── ipfsdir_test.go │ │ │ ├── ipfsfetcher/ │ │ │ │ ├── ipfsfetcher.go │ │ │ │ └── ipfsfetcher_test.go │ │ │ ├── migrations.go │ │ │ ├── migrations_test.go │ │ │ ├── retryfetcher.go │ │ │ ├── setup_test.go │ │ │ ├── unpack.go │ │ │ ├── unpack_test.go │ │ │ ├── versions.go │ │ │ └── versions_test.go │ │ └── misc.go │ ├── mock.go │ ├── onlyone.go │ └── repo.go ├── routing/ │ ├── composer.go │ ├── delegated.go │ ├── delegated_test.go │ ├── error.go │ └── wrapper.go ├── test/ │ ├── .gitignore │ ├── 3nodetest/ │ │ ├── GNUmakefile │ │ ├── README.md │ │ ├── bin/ │ │ │ ├── .gitignore │ │ │ ├── clean.sh │ │ │ ├── save_logs.sh │ │ │ └── save_profiling_data.sh │ │ ├── bootstrap/ │ │ │ ├── Dockerfile │ │ │ ├── README.md │ │ │ └── config │ │ ├── build/ │ │ │ ├── .gitignore │ │ │ └── .gitkeep │ │ ├── client/ │ │ │ ├── Dockerfile │ │ │ ├── config │ │ │ └── run.sh │ │ ├── data/ │ │ │ ├── .gitignore │ │ │ └── Dockerfile │ │ ├── fig.yml │ │ ├── run-test-on-img.sh │ │ └── server/ │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── config │ │ └── run.sh │ ├── README.md │ ├── Rules.mk │ ├── api-startup/ │ │ └── main.go │ ├── bench/ │ │ ├── bench_cli_ipfs_add/ │ │ │ └── main.go │ │ └── offline_add/ │ │ └── main.go │ ├── bin/ │ │ ├── .gitignore │ │ ├── Rules.mk │ │ ├── checkflags │ │ └── continueyn │ ├── cli/ │ │ ├── add_test.go │ │ ├── agent_version_unicode_test.go │ │ ├── api_file_test.go │ │ ├── autoconf/ │ │ │ ├── autoconf_test.go │ │ │ ├── dns_test.go │ │ │ ├── expand_comprehensive_test.go │ │ │ ├── expand_fallback_test.go │ │ │ ├── expand_test.go │ │ │ ├── extensibility_test.go │ │ │ ├── fuzz_test.go │ │ │ ├── ipns_test.go │ │ │ ├── routing_test.go │ │ │ ├── swarm_connect_test.go │ │ │ ├── testdata/ │ │ │ │ ├── autoconf_amino_and_ipni.json │ │ │ │ ├── autoconf_new_routing_system.json │ │ │ │ ├── autoconf_new_routing_with_filtering.json │ │ │ │ ├── autoconf_with_unsupported_paths.json │ │ │ │ ├── updated_autoconf.json │ │ │ │ └── valid_autoconf.json │ │ │ └── validation_test.go │ │ ├── backup_bootstrap_test.go │ │ ├── basic_commands_test.go │ │ ├── bitswap_config_test.go │ │ ├── block_size_test.go │ │ ├── bootstrap_auto_test.go │ │ ├── cid_profiles_test.go │ │ ├── cid_test.go │ │ ├── cli_https_test.go │ │ ├── commands_without_repo_test.go │ │ ├── completion_test.go │ │ ├── config_secrets_test.go │ │ ├── content_blocking_test.go │ │ ├── content_routing_http_test.go │ │ ├── daemon_test.go │ │ ├── dag_layout_test.go │ │ ├── dag_test.go │ │ ├── delegated_routing_v1_http_client_test.go │ │ ├── delegated_routing_v1_http_proxy_test.go │ │ ├── delegated_routing_v1_http_server_test.go │ │ ├── dht_autoclient_test.go │ │ ├── dht_opt_prov_test.go │ │ ├── diag_datastore_test.go │ │ ├── dns_resolvers_multiaddr_test.go │ │ ├── files_test.go │ │ ├── fixtures/ │ │ │ ├── README.md │ │ │ ├── TestDagStat.car │ │ │ ├── TestDagStatExpectedOutput.txt │ │ │ ├── TestGatewayHAMTDirectory.car │ │ │ ├── TestGatewayMultiRange.car │ │ │ └── TestName.car │ │ ├── fuse_test.go │ │ ├── gateway_limits_test.go │ │ ├── gateway_range_test.go │ │ ├── gateway_test.go │ │ ├── harness/ │ │ │ ├── buffer.go │ │ │ ├── harness.go │ │ │ ├── http_client.go │ │ │ ├── ipfs.go │ │ │ ├── log.go │ │ │ ├── node.go │ │ │ ├── nodes.go │ │ │ ├── pbinspect.go │ │ │ ├── peering.go │ │ │ └── run.go │ │ ├── http_gateway_over_libp2p_test.go │ │ ├── http_retrieval_client_test.go │ │ ├── identity_cid_test.go │ │ ├── init_test.go │ │ ├── ipfswatch_test.go │ │ ├── log_level_test.go │ │ ├── ls_test.go │ │ ├── migrations/ │ │ │ ├── migration_16_to_latest_test.go │ │ │ ├── migration_17_to_latest_test.go │ │ │ ├── migration_concurrent_test.go │ │ │ ├── migration_mixed_15_to_latest_test.go │ │ │ └── testdata/ │ │ │ ├── v15-repo/ │ │ │ │ ├── blocks/ │ │ │ │ │ ├── SHARDING │ │ │ │ │ ├── X3/ │ │ │ │ │ │ └── CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data │ │ │ │ │ ├── _README │ │ │ │ │ └── diskUsage.cache │ │ │ │ ├── config │ │ │ │ ├── datastore/ │ │ │ │ │ ├── 000001.log │ │ │ │ │ ├── CURRENT │ │ │ │ │ ├── LOCK │ │ │ │ │ ├── LOG │ │ │ │ │ └── MANIFEST-000000 │ │ │ │ ├── datastore_spec │ │ │ │ └── version │ │ │ └── v16-repo/ │ │ │ ├── blocks/ │ │ │ │ ├── SHARDING │ │ │ │ ├── X3/ │ │ │ │ │ └── CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data │ │ │ │ ├── _README │ │ │ │ └── diskUsage.cache │ │ │ ├── config │ │ │ ├── datastore/ │ │ │ │ ├── 000001.log │ │ │ │ ├── CURRENT │ │ │ │ ├── LOCK │ │ │ │ ├── LOG │ │ │ │ └── MANIFEST-000000 │ │ │ ├── datastore_spec │ │ │ └── version │ │ ├── must.go │ │ ├── name_test.go │ │ ├── p2p_test.go │ │ ├── peering_test.go │ │ ├── pin_ls_names_test.go │ │ ├── pin_name_validation_test.go │ │ ├── ping_test.go │ │ ├── pinning_remote_test.go │ │ ├── pins_test.go │ │ ├── provide_stats_test.go │ │ ├── provider_test.go │ │ ├── pubsub_test.go │ │ ├── rcmgr_test.go │ │ ├── repo_verify_test.go │ │ ├── routing_dht_test.go │ │ ├── rpc_auth_test.go │ │ ├── rpc_content_type_test.go │ │ ├── rpc_get_output_test.go │ │ ├── rpc_unixsocket_test.go │ │ ├── stats_test.go │ │ ├── swarm_test.go │ │ ├── telemetry_test.go │ │ ├── testutils/ │ │ │ ├── asserts.go │ │ │ ├── cids.go │ │ │ ├── files.go │ │ │ ├── floats.go │ │ │ ├── httprouting/ │ │ │ │ └── mock_http_content_router.go │ │ │ ├── json.go │ │ │ ├── pinningservice/ │ │ │ │ └── pinning.go │ │ │ ├── protobuf.go │ │ │ ├── random_deterministic.go │ │ │ ├── requires.go │ │ │ └── strings.go │ │ ├── tracing_test.go │ │ ├── transports_test.go │ │ └── webui_test.go │ ├── dependencies/ │ │ ├── GNUmakefile │ │ ├── dependencies.go │ │ ├── go.mod │ │ ├── go.sum │ │ ├── iptb/ │ │ │ └── iptb.go │ │ ├── ma-pipe-unidir/ │ │ │ ├── LICENSE │ │ │ └── main.go │ │ └── pollEndpoint/ │ │ └── main.go │ ├── integration/ │ │ ├── GNUmakefile │ │ ├── addcat_test.go │ │ ├── bench_cat_test.go │ │ ├── bench_test.go │ │ ├── bitswap_wo_routing_test.go │ │ ├── three_legged_cat_test.go │ │ └── wan_lan_dht_test.go │ ├── ipfs-test-lib.sh │ ├── sharness/ │ │ ├── .gitignore │ │ ├── GNUmakefile │ │ ├── README.md │ │ ├── Rules.mk │ │ ├── lib/ │ │ │ ├── install-sharness.sh │ │ │ ├── iptb-lib.sh │ │ │ ├── test-aggregate-junit-reports.sh │ │ │ ├── test-aggregate-results.sh │ │ │ ├── test-lib-hashes.sh │ │ │ └── test-lib.sh │ │ ├── t0001-tests-work.sh │ │ ├── t0002-docker-image.sh │ │ ├── t0003-docker-migrate.sh │ │ ├── t0012-completion-fish.sh │ │ ├── t0015-basic-sh-functions.sh │ │ ├── t0018-indent.sh │ │ ├── t0021-config.sh │ │ ├── t0022-init-default.sh │ │ ├── t0023-shutdown.sh │ │ ├── t0024-datastore-config.sh │ │ ├── t0024-files/ │ │ │ ├── spec-newshardfun │ │ │ └── spec-nosync │ │ ├── t0025-datastores.sh │ │ ├── t0026-id.sh │ │ ├── t0027-rotate.sh │ │ ├── t0030-mount.sh │ │ ├── t0031-mount-publish.sh │ │ ├── t0032-mount-sharded.sh │ │ ├── t0040-add-and-cat.sh │ │ ├── t0042-add-skip.sh │ │ ├── t0043-add-w.sh │ │ ├── t0044-add-symlink.sh │ │ ├── t0045-ls.sh │ │ ├── t0046-id-hash.sh │ │ ├── t0047-add-mode-mtime.sh │ │ ├── t0050-block-data/ │ │ │ └── testPut.pb │ │ ├── t0050-block.sh │ │ ├── t0051-object.sh │ │ ├── t0052-object-diff.sh │ │ ├── t0053-dag-data/ │ │ │ └── non-canon.cbor │ │ ├── t0053-dag.sh │ │ ├── t0054-dag-car-import-export-data/ │ │ │ ├── README.md │ │ │ └── test_dataset_car.tar.xz │ │ ├── t0054-dag-car-import-export.sh │ │ ├── t0055-dag-put-json-new-line.sh │ │ ├── t0060-daemon.sh │ │ ├── t0060-data/ │ │ │ ├── mss-noise │ │ │ ├── mss-plaintext │ │ │ └── mss-tls │ │ ├── t0061-daemon-opts.sh │ │ ├── t0062-daemon-api.sh │ │ ├── t0063-daemon-init.sh │ │ ├── t0063-external.sh │ │ ├── t0064-api-file.sh │ │ ├── t0065-active-requests.sh │ │ ├── t0066-migration.sh │ │ ├── t0067-unix-api.sh │ │ ├── t0070-user-config.sh │ │ ├── t0080-repo.sh │ │ ├── t0081-repo-pinning.sh │ │ ├── t0082-repo-gc-auto.sh │ │ ├── t0084-repo-read-rehash.sh │ │ ├── t0086-repo-verify.sh │ │ ├── t0087-repo-robust-gc.sh │ │ ├── t0088-repo-stat-symlink.sh │ │ ├── t0090-get.sh │ │ ├── t0095-refs.sh │ │ ├── t0101-iptb-name.sh │ │ ├── t0109-gateway-web-_redirects-data/ │ │ │ └── redirects.car │ │ ├── t0109-gateway-web-_redirects.sh │ │ ├── t0112-gateway-cors.sh │ │ ├── t0114-gateway-subdomains/ │ │ │ ├── 12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d.ipns-record │ │ │ ├── QmVujd5Vb7moysJj8itnGufN7MEtPRCNHkKpNuA4onsRa3.ipns-record │ │ │ ├── README.md │ │ │ └── fixtures.car │ │ ├── t0114-gateway-subdomains.sh │ │ ├── t0115-gateway-dir-listing/ │ │ │ ├── README.md │ │ │ └── fixtures.car │ │ ├── t0115-gateway-dir-listing.sh │ │ ├── t0116-gateway-cache/ │ │ │ ├── README.md │ │ │ ├── fixtures.car │ │ │ └── k51qzi5uqu5dlxdsdu5fpuu7h69wu4ohp32iwm9pdt9nq3y5rpn3ln9j12zfhe.ipns-record │ │ ├── t0116-gateway-cache.sh │ │ ├── t0119-prometheus-data/ │ │ │ ├── prometheus_metrics │ │ │ ├── prometheus_metrics_added_by_enabling_rcmgr │ │ │ └── prometheus_metrics_added_by_measure_profile │ │ ├── t0119-prometheus.sh │ │ ├── t0120-bootstrap.sh │ │ ├── t0121-bootstrap-iptb.sh │ │ ├── t0131-multinode-client-routing.sh │ │ ├── t0140-swarm.sh │ │ ├── t0141-addfilter.sh │ │ ├── t0142-testfilter.sh │ │ ├── t0150-clisuggest.sh │ │ ├── t0151-sysdiag.sh │ │ ├── t0152-profile.sh │ │ ├── t0160-resolve.sh │ │ ├── t0165-keystore-data/ │ │ │ ├── README.md │ │ │ ├── openssl_ed25519.pem │ │ │ ├── openssl_rsa.pem │ │ │ └── openssl_secp384r1.pem │ │ ├── t0165-keystore.sh │ │ ├── t0180-p2p.sh │ │ ├── t0181-private-network.sh │ │ ├── t0182-circuit-relay.sh │ │ ├── t0183-namesys-pubsub.sh │ │ ├── t0184-http-proxy-over-p2p.sh │ │ ├── t0185-autonat.sh │ │ ├── t0190-quic-ping.sh │ │ ├── t0191-webtransport-ping.sh │ │ ├── t0195-noise.sh │ │ ├── t0220-bitswap.sh │ │ ├── t0230-channel-streaming-http-content-type.sh │ │ ├── t0231-channel-streaming.sh │ │ ├── t0235-cli-request.sh │ │ ├── t0236-cli-api-dns-resolve.sh │ │ ├── t0240-republisher.sh │ │ ├── t0250-files-api.sh │ │ ├── t0251-files-flushing.sh │ │ ├── t0252-files-gc.sh │ │ ├── t0260-sharding.sh │ │ ├── t0270-filestore.sh │ │ ├── t0271-filestore-utils.sh │ │ ├── t0272-urlstore.sh │ │ ├── t0275-cid-security-data/ │ │ │ ├── CIQG6PGTD2VV34S33BE4MNCQITBRFYUPYQLDXYARR3DQW37MOT7K5XI.data │ │ │ └── EICEM7ITSI.data │ │ ├── t0275-cid-security.sh │ │ ├── t0276-cidv0v1.sh │ │ ├── t0280-plugin-dag-jose-data/ │ │ │ ├── dag-cbor/ │ │ │ │ ├── bafyreicxyzuqbx5yb7ytkgkuofwksbal3ygtswxuri25crxdxms55m5fki │ │ │ │ ├── bafyreihdfxoshbhowufyvjk7kq46dt6h7u6byejmlnifz34z7ocoq7ugk4 │ │ │ │ └── bafyreihkt4u6euddfhofkutfzxwet7w7zm5qrjpop655yhnb5dnzqw26lm │ │ │ ├── dag-jose/ │ │ │ │ ├── bagcqcera542h3xc57nudkgjcceexyzyxrkwi4ikbn773ag6dqdcyjt6z6rga │ │ │ │ ├── bagcqcera5uvz2qai6l4vmqjigwpowluilxngz3dyjnva2s3uwbfb5u4ao4fa │ │ │ │ ├── bagcqcera7azagcqlpu4ivvh4xp4iv6psmb5d7eki6ln3fnfnsnbb2hzv4nxq │ │ │ │ ├── bagcqcerakjv2mmdlbai3urym22bw5kaw7nqov73yaxf6xjnp7e56sclsrooa │ │ │ │ ├── bagcqceraqfknq7xaemcihmq2albau32ttrutxnco7xeoik6mlejismmvw5zq │ │ │ │ ├── bagcqcerauben4l6ee2wjf2fnkj7vaels4p7lnytenk35j3gl2lzcbtbgyoea │ │ │ │ ├── bagcqceravvw4bx7jgkxxjwfuqo2yoja6w4cmvmu3gkew3s7yu3vt2ce7riwa │ │ │ │ ├── bagcqceraxazmu67crshzqdeg3kwnfschs25epy5sbtqtjre2qw3d62kzplva │ │ │ │ └── bagcqceraxvt5izt4sz7kjfrm42dxrutp6ijywgsacllkznzekmfojypkvfea │ │ │ └── dag-json/ │ │ │ ├── baguqeeraloya3qpa25kl5l4y3bzgl7rhyta2p7lwaocyxx4vpvdligb7mt2q │ │ │ ├── baguqeeraovfm3rr3pvmxm27zgvxp5wycbfih35xih2uznminpnds5esm4jlq │ │ │ └── baguqeeravexfd6qijjtnzxfqq6kgknnkncztgmvhjhxm6ih352qskolt2gxa │ │ ├── t0280-plugin-dag-jose.sh │ │ ├── t0280-plugin-data/ │ │ │ └── example.go │ │ ├── t0280-plugin-fx.sh │ │ ├── t0280-plugin-git.sh │ │ ├── t0280-plugin-peerlog.sh │ │ ├── t0280-plugin.sh │ │ ├── t0290-cid.sh │ │ ├── t0295-multibase.sh │ │ ├── t0320-pubsub.sh │ │ ├── t0321-pubsub-gossipsub.sh │ │ ├── t0322-pubsub-http-rpc.sh │ │ ├── t0400-api-no-gateway/ │ │ │ ├── README.md │ │ │ └── fixtures.car │ │ ├── t0400-api-no-gateway.sh │ │ ├── t0401-api-browser-security.sh │ │ ├── t0410-api-add.sh │ │ ├── t0500-issues-and-regressions-offline.sh │ │ ├── t0600-issues-and-regressions-online.sh │ │ ├── t0701-delegated-routing-reframe/ │ │ │ ├── FindProvidersRequest │ │ │ └── FindProvidersResponse │ │ ├── t0702-delegated-routing-http/ │ │ │ └── FindProvidersResponse │ │ ├── t0800-blake3.sh │ │ └── x0601-pin-fail-test.sh │ ├── sharness_test_coverage_helper.sh │ └── unit/ │ ├── .gitignore │ └── Rules.mk ├── thirdparty/ │ ├── README.md │ ├── unit/ │ │ ├── unit.go │ │ └── unit_test.go │ └── verifbs/ │ └── verifbs.go ├── tracing/ │ ├── doc.go │ └── tracing.go ├── version.go └── version_test.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .codeclimate.yml ================================================ ratings: paths: - "**/*.go" exclude_paths: - test/ - Godeps/ - thirdparty/ - "**/*.pb.go" engines: fixme: enabled: true config: strings: - FIXME - HACK - XXX - BUG golint: enabled: true govet: enabled: true gofmt: enabled: true version: "2" checks: argument-count: enabled: false complex-logic: enabled: false file-lines: enabled: false method-complexity: enabled: false method-count: enabled: false method-lines: enabled: false nested-control-flow: enabled: false return-statements: enabled: false similar-code: enabled: false ================================================ FILE: .cspell.yml ================================================ ignoreWords: - childs # This spelling is used in the files command - NodeCreater # This spelling is used in the fuse dependency - Boddy # One of the contributors to the project - Chris Boddy - Botto # One of the contributors to the project - Santiago Botto - cose # dag-cose ================================================ FILE: .dockerignore ================================================ Dockerfile Dockerfile.fast .git/ !.git/HEAD !.git/refs/ !.git/packed-refs test/sharness/lib/sharness/ test/sharness/trash* rb-pinning-service-api/ # The Docker client might not be running on Linux # so delete any compiled binaries bin/gx bin/gx* bin/tmp ================================================ FILE: .gitattributes ================================================ # Default to text * text eol=lf # True text *.md text eol=auto LICENSE text eol=auto # Known binary types *.png binary *.tar binary *.gz binary *.xz binary *.car binary # Binary assets assets/init-doc/* binary core/coreunix/test_data/** binary test/cli/migrations/testdata/** binary # Generated test data test/cli/migrations/testdata/** linguist-generated=true test/cli/autoconf/testdata/** linguist-generated=true test/cli/fixtures/** linguist-generated=true test/sharness/t0054-dag-car-import-export-data/** linguist-generated=true test/sharness/t0109-gateway-web-_redirects-data/** linguist-generated=true test/sharness/t0114-gateway-subdomains/** linguist-generated=true test/sharness/t0115-gateway-dir-listing/** linguist-generated=true test/sharness/t0116-gateway-cache/** linguist-generated=true test/sharness/t0119-prometheus-data/** linguist-generated=true test/sharness/t0165-keystore-data/** linguist-generated=true test/sharness/t0275-cid-security-data/** linguist-generated=true test/sharness/t0280-plugin-dag-jose-data/** linguist-generated=true test/sharness/t0280-plugin-data/** linguist-generated=true test/sharness/t0280-plugin-git-data/** linguist-generated=true test/sharness/t0400-api-no-gateway/** linguist-generated=true test/sharness/t0701-delegated-routing-reframe/** linguist-generated=true test/sharness/t0702-delegated-routing-http/** linguist-generated=true ================================================ FILE: .github/CODEOWNERS ================================================ # Code owners are automatically requested for review when someone opens a pull # request that modifies code that they own. Code owners are not automatically # requested to review draft pull requests. # Default * @ipfs/kubo-maintainers # HTTP Gateway core/corehttp/ @lidel test/sharness/*gateway*.sh @lidel ================================================ FILE: .github/FUNDING.yml ================================================ custom: [ipshipyard.gitwallet.co] ================================================ FILE: .github/ISSUE_TEMPLATE/bug-report.yml ================================================ name: Bug Report description: Report a bug in Kubo. labels: - kind/bug - need/triage body: - type: markdown attributes: value: | - Make sure you are running the [latest version of Kubo][releases] before reporting an issue. - If you have an enhancement or feature request for Kubo, please select [a different option][issues]. - Please report possible security issues by email to security@ipfs.io [issues]: https://github.com/ipfs/kubo/issues/new/choose [releases]: https://github.com/ipfs/kubo/releases - type: checkboxes attributes: label: Checklist description: Please verify that you've followed these steps options: - label: This is a bug report, not a question. Ask questions on [discuss.ipfs.tech](https://discuss.ipfs.tech/c/help/13). required: true - label: I have searched on the [issue tracker](https://github.com/ipfs/kubo/issues?q=is%3Aissue) for my bug. required: true - label: I am running the latest [kubo version](https://dist.ipfs.tech/#kubo) or have an issue updating. required: true - type: dropdown id: install validations: required: true attributes: label: Installation method description: Please select your installation method options: - dist.ipfs.tech or ipfs-update - docker image - ipfs-desktop - third-party binary - built from source - type: textarea id: version attributes: label: Version render: Text description: | Enter the output of `ipfs version --all`. If you can't run that command, please include a copy of your [gateway's version page](http://localhost:8080/api/v0/version?enc=text&all=true). - type: textarea id: config attributes: label: Config render: json description: | Enter the output of `ipfs config show`. - type: textarea attributes: label: Description description: | This is where you get to tell us what went wrong. When doing so, please make sure to include *all* relevant information. Please try to include: * What you were doing when you experienced the bug. * Any error messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas). * When possible, steps to reliably produce the bug. ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false contact_links: - name: Getting Help on IPFS url: https://ipfs.tech/help about: All information about how and where to get help on IPFS. - name: Kubo configuration reference url: https://github.com/ipfs/kubo/blob/master/docs/config.md#readme about: Documentation on the different configuration settings - name: Kubo experimental features docs url: https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#readme about: Documentation on Private Networks, Filestore and other experimental features. - name: Kubo RPC API Reference url: https://docs.ipfs.tech/reference/kubo/rpc/ about: Documentation of all Kubo RPC API endpoints. - name: IPFS Official Discussion Forum url: https://discuss.ipfs.tech about: Please post general questions, support requests, and discussions here. ================================================ FILE: .github/ISSUE_TEMPLATE/doc.yml ================================================ name: Documentation Issue description: Report missing, erroneous docs, broken links or propose new Kubo docs. labels: - topic/docs-ipfs - need/triage body: - type: markdown attributes: value: | Problems with documentation on https://docs.ipfs.tech should be reported to https://github.com/ipfs/ipfs-docs - type: checkboxes attributes: label: Checklist description: Please verify the following. options: - label: I am reporting a documentation issue in this repo, not https://docs.ipfs.tech. required: true - label: I have searched on the [issue tracker](https://github.com/ipfs/kubo/issues?q=is%3Aissue) for my issue. required: true - type: input attributes: label: Location description: | If possible, please provide a link to the documentation issue. - type: textarea attributes: label: Description description: | Please describe your issue. ================================================ FILE: .github/ISSUE_TEMPLATE/enhancement.yml ================================================ name: Enhancement description: Suggest an improvement to an existing kubo feature. labels: - kind/enhancement - need/triage body: - type: markdown attributes: value: | Suggest an enhancement to Kubo (the program). If you'd like to suggest an improvement to the IPFS protocol, please discuss it on [the forum](https://discuss.ipfs.tech). Issues in this repo must be specific, actionable, and well motivated. They should be starting points for _building_ new features, not brainstorming ideas. If you have an idea you'd like to discuss, please open a new thread on [the forum](https://discuss.ipfs.tech). **Example:** > Reduce memory usage of `ipfs cat` (specific) by buffering less in ... (actionable). This would let me run Kubo on my Raspberry Pi (motivated). - type: checkboxes attributes: label: Checklist description: Please verify the following. options: - label: My issue is specific & actionable. required: true - label: I am not suggesting a protocol enhancement. required: true - label: I have searched on the [issue tracker](https://github.com/ipfs/kubo/issues?q=is%3Aissue) for my issue. required: true - type: textarea attributes: label: Description description: | Please describe your idea. When requesting an enhancement, please be sure to include your motivation and try to be as specific as possible. ================================================ FILE: .github/ISSUE_TEMPLATE/feature.yml ================================================ name: Feature description: Suggest a new feature in Kubo. labels: - kind/feature - need/triage body: - type: markdown attributes: value: | Suggest a new feature in Kubo (the program). If you'd like to suggest an improvement to the IPFS protocol, please discuss it on [the forum](https://discuss.ipfs.tech). Issues in this repo must be specific, actionable, and well motivated. They should be starting points for _building_ new features, not brainstorming ideas. If you have an idea you'd like to discuss, please open a new thread on [the forum](https://discuss.ipfs.tech). **Example:** > Add deduplication-optimized chunking of tar files in `ipfs add` (specific) by examining tar headers ... (actionable). This would let me efficiently store and update many versions of code archives (motivated). - type: checkboxes attributes: label: Checklist description: Please verify the following. options: - label: My issue is specific & actionable. required: true - label: I am not suggesting a protocol enhancement. required: true - label: I have searched on the [issue tracker](https://github.com/ipfs/kubo/issues?q=is%3Aissue) for my issue. required: true - type: textarea attributes: label: Description description: | Please describe your idea. When requesting a feature, please be sure to include your motivation and and a concrete description of how the feature should work. ================================================ FILE: .github/auto-comment.yml ================================================ # Comment to a new issue. # Disabled # issueOpened: "" # Disabled # pullRequestOpened: "" ================================================ FILE: .github/build-platforms.yml ================================================ # Build platforms configuration for Kubo # Matches https://github.com/ipfs/distributions/blob/master/dists/kubo/build_matrix # plus linux-riscv64 for emerging architecture support # # The Go compiler handles FUSE support automatically via build tags. # Platforms are simply listed - no need to specify FUSE capability. platforms: - darwin-amd64 - darwin-arm64 - freebsd-amd64 - linux-amd64 - linux-arm64 - linux-riscv64 - openbsd-amd64 - windows-amd64 - windows-arm64 ================================================ FILE: .github/dependabot.yml ================================================ # Dependabot PRs are auto-tidied by .github/workflows/dependabot-tidy.yml version: 2 updates: - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" - package-ecosystem: "gomod" directory: "/" schedule: interval: "monthly" open-pull-requests-limit: 10 labels: - "dependencies" ignore: # Updated via go-ds-* wrappers in ipfs-ecosystem group - dependency-name: "github.com/cockroachdb/pebble*" - dependency-name: "github.com/syndtr/goleveldb" - dependency-name: "github.com/dgraph-io/badger*" groups: ipfs-ecosystem: patterns: - "github.com/ipfs/*" - "github.com/ipfs-shipyard/*" - "github.com/ipshipyard/*" - "github.com/multiformats/*" - "github.com/ipld/*" libp2p-ecosystem: patterns: - "github.com/libp2p/*" golang-x: patterns: - "golang.org/x/*" opentelemetry: patterns: - "go.opentelemetry.io/*" prometheus: patterns: - "github.com/prometheus/*" - "contrib.go.opencensus.io/*" - "go.opencensus.io" uber: patterns: - "go.uber.org/*" ================================================ FILE: .github/pull_request_template.md ================================================ ================================================ FILE: .github/workflows/changelog.yml ================================================ name: Changelog on: pull_request: types: - opened - edited - synchronize - reopened - labeled - unlabeled paths: - '**.go' - '**/go.mod' - '**/go.sum' jobs: changelog: if: contains(github.event.pull_request.title, '[skip changelog]') == false && contains(github.event.pull_request.labels.*.name, 'skip/changelog') == false runs-on: ubuntu-latest name: Changelog steps: - id: changelog env: GITHUB_TOKEN: ${{ github.token }} ENDPOINT: repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files SELECTOR: 'map(select(.filename | startswith("docs/changelogs/"))) | length' run: gh api "$ENDPOINT" --jq "$SELECTOR" | xargs -I{} echo "modified={}" | tee -a $GITHUB_OUTPUT - if: steps.changelog.outputs.modified == '0' env: MESSAGE: | docs/changelogs/ was not modified in this PR. Please do one of the following: - add a changelog entry - add `[skip changelog]` to the PR title - label the PR with `skip/changelog` run: | echo "::error::${MESSAGE//$'\n'/%0A}" exit 1 ================================================ FILE: .github/workflows/codeql-analysis.yml ================================================ # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed name: CodeQL on: workflow_dispatch: push: branches: [ master ] pull_request: # The branches below must be a subset of the branches above branches: [ master ] paths-ignore: - '**/*.md' schedule: - cron: '30 12 * * 2' permissions: contents: read # to fetch code (actions/checkout) security-events: write # (github/codeql-action/autobuild) concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true jobs: codeql: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest timeout-minutes: 20 steps: - name: Checkout repository uses: actions/checkout@v6 - name: Setup Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v4 with: languages: go - name: Autobuild uses: github/codeql-action/autobuild@v4 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 ================================================ FILE: .github/workflows/dependabot-tidy.yml ================================================ # Dependabot only updates go.mod/go.sum in the root module, but this repo has # multiple Go modules (see docs/examples/). This workflow runs `make mod_tidy` # on Dependabot PRs to keep all go.sum files in sync, preventing go-check CI # failures. name: Dependabot Tidy on: pull_request_target: types: [opened, synchronize] workflow_dispatch: inputs: pr_number: description: 'PR number to run mod_tidy on' required: true type: number permissions: contents: write pull-requests: write jobs: tidy: if: github.actor == 'dependabot[bot]' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - name: Get PR info id: pr env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then pr_number="${{ inputs.pr_number }}" else pr_number="${{ github.event.pull_request.number }}" fi echo "number=$pr_number" >> $GITHUB_OUTPUT branch=$(gh pr view "$pr_number" --repo "${{ github.repository }}" --json headRefName -q '.headRefName') echo "branch=$branch" >> $GITHUB_OUTPUT - uses: actions/checkout@v6 with: ref: ${{ steps.pr.outputs.branch }} token: ${{ secrets.GITHUB_TOKEN }} - uses: actions/setup-go@v6 with: go-version-file: go.mod - name: Run make mod_tidy run: make mod_tidy - name: Check for changes id: git-check run: | if [[ -n $(git status --porcelain) ]]; then echo "modified=true" >> $GITHUB_OUTPUT fi - name: Commit changes if: steps.git-check.outputs.modified == 'true' run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add -A git commit -m "chore: run make mod_tidy" git push ================================================ FILE: .github/workflows/docker-check.yml ================================================ # This workflow performs a quick Docker build check on PRs and pushes to master. # It builds the Docker image and runs a basic smoke test to ensure the image works. # This is a lightweight check - for full multi-platform builds and publishing, see docker-image.yml name: Docker Check on: workflow_dispatch: pull_request: paths-ignore: - '**/*.md' push: branches: - 'master' concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true jobs: lint: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest timeout-minutes: 5 steps: - uses: actions/checkout@v6 - uses: hadolint/hadolint-action@v3.3.0 with: dockerfile: Dockerfile failure-threshold: warning verbose: true format: tty build: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest timeout-minutes: 10 env: IMAGE_NAME: ipfs/kubo WIP_IMAGE_TAG: wip defaults: run: shell: bash steps: - uses: actions/checkout@v6 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - name: Build Docker image with BuildKit uses: docker/build-push-action@v7 with: context: . push: false load: true tags: ${{ env.IMAGE_NAME }}:${{ env.WIP_IMAGE_TAG }} cache-from: | type=gha type=registry,ref=${{ env.IMAGE_NAME }}:buildcache cache-to: type=gha,mode=max - name: Test Docker image run: docker run --rm $IMAGE_NAME:$WIP_IMAGE_TAG --version ================================================ FILE: .github/workflows/docker-image.yml ================================================ # This workflow builds and publishes official Docker images to Docker Hub. # It handles multi-platform builds (amd64, arm/v7, arm64/v8) and pushes tagged releases. # This workflow is triggered on tags, specific branches, and can be manually dispatched. # For quick build checks during development, see docker-check.yml name: Docker Push on: workflow_dispatch: inputs: push: description: 'Push to Docker Hub' required: true default: 'false' tags: description: 'Custom tags to use for the push' required: false default: '' # # If we decide to build all images on every PR, we should make sure that # # they are NOT pushed to Docker Hub. # pull_request: # paths-ignore: # - '**/*.md' push: branches: - 'master' - 'staging' - 'bifrost-*' tags: - 'v*' permissions: contents: read # to fetch code (actions/checkout) jobs: docker-hub: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' name: Push Docker image to Docker Hub runs-on: ubuntu-latest timeout-minutes: 15 env: IMAGE_NAME: ipfs/kubo outputs: tags: ${{ steps.tags.outputs.value }} steps: - name: Check out the repo uses: actions/checkout@v6 - name: Set up QEMU uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - name: Log in to Docker Hub uses: docker/login-action@v4 with: username: ${{ vars.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Get tags id: tags if: github.event.inputs.tags == '' run: | echo "value<> $GITHUB_OUTPUT ./bin/get-docker-tags.sh "$(date -u +%F)" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT shell: bash # We have to build each platform separately because when using multi-arch # builds, only one platform is being loaded into the cache. This would # prevent us from testing the other platforms. - name: Build Docker image (linux/amd64) uses: docker/build-push-action@v7 with: platforms: linux/amd64 context: . push: false load: true file: ./Dockerfile tags: ${{ env.IMAGE_NAME }}:linux-amd64 cache-from: | type=gha type=registry,ref=${{ env.IMAGE_NAME }}:buildcache cache-to: type=gha,mode=max - name: Build Docker image (linux/arm/v7) uses: docker/build-push-action@v7 with: platforms: linux/arm/v7 context: . push: false load: true file: ./Dockerfile tags: ${{ env.IMAGE_NAME }}:linux-arm-v7 cache-from: | type=gha type=registry,ref=${{ env.IMAGE_NAME }}:buildcache cache-to: type=gha,mode=max - name: Build Docker image (linux/arm64/v8) uses: docker/build-push-action@v7 with: platforms: linux/arm64/v8 context: . push: false load: true file: ./Dockerfile tags: ${{ env.IMAGE_NAME }}:linux-arm64-v8 cache-from: | type=gha type=registry,ref=${{ env.IMAGE_NAME }}:buildcache cache-to: type=gha,mode=max # We test all the images on amd64 host here. This uses QEMU to emulate # the other platforms. # NOTE: --version should finish instantly, but sometimes # it hangs on github CI (could be qemu issue), so we retry to remove false negatives - name: Smoke-test linux-amd64 run: for i in {1..3}; do timeout 15s docker run --rm $IMAGE_NAME:linux-amd64 version --all && break || [ $i = 3 ] && exit 1; done timeout-minutes: 1 - name: Smoke-test linux-arm-v7 run: for i in {1..3}; do timeout 15s docker run --rm $IMAGE_NAME:linux-arm-v7 version --all && break || [ $i = 3 ] && exit 1; done timeout-minutes: 1 - name: Smoke-test linux-arm64-v8 run: for i in {1..3}; do timeout 15s docker run --rm $IMAGE_NAME:linux-arm64-v8 version --all && break || [ $i = 3 ] && exit 1; done timeout-minutes: 1 # This will only push the previously built images. - if: github.event_name != 'workflow_dispatch' || github.event.inputs.push == 'true' name: Publish to Docker Hub uses: docker/build-push-action@v7 with: platforms: linux/amd64,linux/arm/v7,linux/arm64/v8 context: . push: true file: ./Dockerfile tags: "${{ github.event.inputs.tags || steps.tags.outputs.value }}" cache-from: | type=gha type=registry,ref=${{ env.IMAGE_NAME }}:buildcache cache-to: | type=gha,mode=max type=registry,ref=${{ env.IMAGE_NAME }}:buildcache,mode=max ================================================ FILE: .github/workflows/gateway-conformance.yml ================================================ name: Gateway Conformance on: workflow_dispatch: push: branches: - master pull_request: paths-ignore: - '**/*.md' concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true defaults: run: shell: bash env: # hostnames expected by https://github.com/ipfs/gateway-conformance GATEWAY_PUBLIC_GATEWAYS: | { "example.com": { "UseSubdomains": true, "InlineDNSLink": true, "Paths": ["/ipfs", "/ipns"] }, "localhost": { "UseSubdomains": true, "InlineDNSLink": true, "Paths": ["/ipfs", "/ipns"] } } jobs: # Testing all gateway features via TCP port specified in Addresses.Gateway gateway-conformance: runs-on: ubuntu-latest timeout-minutes: 10 steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.12 with: output: fixtures # 2. Build the kubo-gateway - name: Checkout kubo-gateway uses: actions/checkout@v6 with: path: kubo-gateway - name: Setup Go uses: actions/setup-go@v6 with: go-version-file: 'kubo-gateway/go.mod' cache: true cache-dependency-path: kubo-gateway/go.sum - name: Build kubo-gateway run: make build working-directory: kubo-gateway # 3. Init the kubo-gateway - name: Init kubo-gateway run: | ./ipfs init -e ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS" working-directory: kubo-gateway/cmd/ipfs # 4. Populate the Kubo gateway with the gateway-conformance fixtures - name: Import fixtures run: | # Import car files find ./fixtures -name '*.car' -exec kubo-gateway/cmd/ipfs/ipfs dag import --pin-roots=false {} \; # Import ipns records records=$(find ./fixtures -name '*.ipns-record') for record in $records do key=$(basename -s .ipns-record "$record" | cut -d'_' -f1) kubo-gateway/cmd/ipfs/ipfs routing put --allow-offline "/ipns/$key" "$record" done # Import dnslink records # the IPFS_NS_MAP env will be used by the daemon echo "IPFS_NS_MAP=$(cat ./fixtures/dnslinks.IPFS_NS_MAP)" >> $GITHUB_ENV # 5. Start the kubo-gateway - name: Start kubo-gateway run: | ./ipfs daemon --offline & working-directory: kubo-gateway/cmd/ipfs # 6. Run the gateway-conformance tests - name: Run gateway-conformance tests uses: ipfs/gateway-conformance/.github/actions/test@v0.12 with: gateway-url: http://127.0.0.1:8080 subdomain-url: http://localhost:8080 args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' json: output.json xml: output.xml html: output.html markdown: output.md # 7. Upload the results - name: Upload MD summary if: failure() || success() run: cat output.md >> $GITHUB_STEP_SUMMARY - name: Upload HTML report if: failure() || success() uses: actions/upload-artifact@v7 with: name: gateway-conformance.html path: output.html - name: Upload JSON report if: failure() || success() uses: actions/upload-artifact@v7 with: name: gateway-conformance.json path: output.json # Testing trustless gateway feature subset exposed as libp2p protocol gateway-conformance-libp2p-experiment: runs-on: ubuntu-latest timeout-minutes: 10 steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.12 with: output: fixtures # 2. Build the kubo-gateway - name: Checkout kubo-gateway uses: actions/checkout@v6 with: path: kubo-gateway - name: Setup Go uses: actions/setup-go@v6 with: go-version-file: 'kubo-gateway/go.mod' cache: true cache-dependency-path: kubo-gateway/go.sum - name: Build kubo-gateway run: make build working-directory: kubo-gateway # 3. Init the kubo-gateway - name: Init kubo-gateway run: | ./ipfs init --profile=test ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS" ./ipfs config --json Experimental.GatewayOverLibp2p true ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080" ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001" working-directory: kubo-gateway/cmd/ipfs # 4. Populate the Kubo gateway with the gateway-conformance fixtures - name: Import fixtures run: | # Import car files find ./fixtures -name '*.car' -exec kubo-gateway/cmd/ipfs/ipfs dag import --pin-roots=false {} \; # 5. Start the kubo-gateway - name: Start kubo-gateway run: | ( ./ipfs daemon & ) | sed '/Daemon is ready/q' while [[ "$(./ipfs id | jq '.Addresses | length')" == '0' ]]; do sleep 1; done working-directory: kubo-gateway/cmd/ipfs # 6. Setup a kubo http-p2p-proxy to expose libp2p protocol as a regular HTTP port for gateway conformance tests - name: Init p2p-proxy kubo node env: IPFS_PATH: "~/.kubo-p2p-proxy" run: | ./ipfs init --profile=test -e ./ipfs config --json Experimental.Libp2pStreamMounting true ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8081" ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5002" working-directory: kubo-gateway/cmd/ipfs # 7. Start the kubo http-p2p-proxy - name: Start kubo http-p2p-proxy env: IPFS_PATH: "~/.kubo-p2p-proxy" run: | ( ./ipfs daemon & ) | sed '/Daemon is ready/q' while [[ "$(./ipfs id | jq '.Addresses | length')" == '0' ]]; do sleep 1; done working-directory: kubo-gateway/cmd/ipfs # 8. Start forwarding data from the http-p2p-proxy to the node serving the Gateway API over libp2p - name: Start http-over-libp2p forwarding proxy run: | gatewayNodeId=$(./ipfs --api=/ip4/127.0.0.1/tcp/5001 id -f="") ./ipfs --api=/ip4/127.0.0.1/tcp/5002 swarm connect $(./ipfs --api=/ip4/127.0.0.1/tcp/5001 swarm addrs local --id | head -n 1) ./ipfs --api=/ip4/127.0.0.1/tcp/5002 p2p forward --allow-custom-protocol /http/1.1 /ip4/127.0.0.1/tcp/8092 /p2p/$gatewayNodeId working-directory: kubo-gateway/cmd/ipfs # 9. Run the gateway-conformance tests over libp2p - name: Run gateway-conformance tests over libp2p uses: ipfs/gateway-conformance/.github/actions/test@v0.12 with: gateway-url: http://127.0.0.1:8092 args: --specs "trustless-gateway,-trustless-ipns-gateway" -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' json: output.json xml: output.xml html: output.html markdown: output.md # 10. Upload the results - name: Upload MD summary if: failure() || success() run: cat output.md >> $GITHUB_STEP_SUMMARY - name: Upload HTML report if: failure() || success() uses: actions/upload-artifact@v7 with: name: gateway-conformance-libp2p.html path: output.html - name: Upload JSON report if: failure() || success() uses: actions/upload-artifact@v7 with: name: gateway-conformance-libp2p.json path: output.json ================================================ FILE: .github/workflows/generated-pr.yml ================================================ name: Close Generated PRs on: schedule: - cron: '0 0 * * *' workflow_dispatch: permissions: issues: write pull-requests: write jobs: stale: uses: ipdxco/unified-github-workflows/.github/workflows/reusable-generated-pr.yml@v1 ================================================ FILE: .github/workflows/gobuild.yml ================================================ name: Go Build on: workflow_dispatch: pull_request: paths-ignore: - '**/*.md' push: branches: - 'master' concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true jobs: go-build: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }} timeout-minutes: 20 env: TEST_DOCKER: 0 TEST_VERBOSE: 1 GIT_PAGER: cat IPFS_CHECK_RCMGR_DEFAULTS: 1 defaults: run: shell: bash steps: - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: 'go.mod' cache: true cache-dependency-path: go.sum - name: Build all platforms run: | # Read platforms from build-platforms.yml and build each one echo "Building kubo for all platforms..." # Read and build each platform grep '^ - ' .github/build-platforms.yml | sed 's/^ - //' | while read -r platform; do if [ -z "$platform" ]; then continue fi echo "::group::Building $platform" GOOS=$(echo "$platform" | cut -d- -f1) GOARCH=$(echo "$platform" | cut -d- -f2) echo "Building $platform" echo " GOOS=$GOOS GOARCH=$GOARCH go build -o /dev/null ./cmd/ipfs" GOOS=$GOOS GOARCH=$GOARCH go build -o /dev/null ./cmd/ipfs echo "::endgroup::" done echo "All platforms built successfully" ================================================ FILE: .github/workflows/golang-analysis.yml ================================================ name: Go Check on: workflow_dispatch: pull_request: paths-ignore: - '**/*.md' push: branches: - 'master' permissions: contents: read # to fetch code (actions/checkout) concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true jobs: go-check: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@v6 with: submodules: recursive - uses: actions/setup-go@v6 with: go-version-file: 'go.mod' - name: Check that go.mod is tidy uses: protocol/multiple-go-modules@v1.4 with: run: | go mod tidy if [[ -n $(git ls-files --other --exclude-standard --directory -- go.sum) ]]; then echo "go.sum was added by go mod tidy" exit 1 fi git diff --exit-code -- go.sum go.mod - name: go fmt if: always() # run this step even if the previous one failed run: | out=$(go fmt ./...) if [[ -n "$out" ]]; then echo "Files are not go-fmt-ed:" echo "$out" exit 1 fi - name: go fix if: always() # run this step even if the previous one failed run: | go fix ./... if [[ -n $(git diff --name-only) ]]; then echo "go fix produced changes. Run 'go fix ./...' locally and commit the result." git diff exit 1 fi - name: go vet if: always() # run this step even if the previous one failed uses: protocol/multiple-go-modules@v1.4 with: run: go vet ./... ================================================ FILE: .github/workflows/golint.yml ================================================ name: Go Lint on: workflow_dispatch: pull_request: paths-ignore: - '**/*.md' push: branches: - 'master' concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true jobs: go-lint: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest timeout-minutes: 10 env: TEST_DOCKER: 0 TEST_FUSE: 0 TEST_VERBOSE: 1 GIT_PAGER: cat IPFS_CHECK_RCMGR_DEFAULTS: 1 defaults: run: shell: bash steps: - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: 'go.mod' - run: make -O test_go_lint ================================================ FILE: .github/workflows/gotest.yml ================================================ name: Go Test on: workflow_dispatch: pull_request: paths-ignore: - '**/*.md' push: branches: - 'master' concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true jobs: # Unit tests with coverage collection (uploaded to Codecov) unit-tests: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} timeout-minutes: 15 env: GOTRACEBACK: single # reduce noise on test timeout panics TEST_DOCKER: 0 TEST_FUSE: 0 TEST_VERBOSE: 1 GIT_PAGER: cat IPFS_CHECK_RCMGR_DEFAULTS: 1 defaults: run: shell: bash steps: - name: Check out Kubo uses: actions/checkout@v6 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' - name: Install missing tools run: sudo apt update && sudo apt install -y zsh - name: Run unit tests run: | make test_unit && [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]] - name: Upload coverage to Codecov uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 if: failure() || success() with: name: unittests files: coverage/unit_tests.coverprofile token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false - name: Create a proper JUnit XML report uses: ipdxco/gotest-json-to-junit-xml@v1 with: input: test/unit/gotest.json output: test/unit/gotest.junit.xml if: failure() || success() - name: Archive the JUnit XML report uses: actions/upload-artifact@v7 with: name: unit-tests-junit path: test/unit/gotest.junit.xml if: failure() || success() - name: Create a HTML report uses: ipdxco/junit-xml-to-html@v1 with: mode: no-frames input: test/unit/gotest.junit.xml output: test/unit/gotest.html if: failure() || success() - name: Archive the HTML report uses: actions/upload-artifact@v7 with: name: unit-tests-html path: test/unit/gotest.html if: failure() || success() - name: Create a Markdown report uses: ipdxco/junit-xml-to-html@v1 with: mode: summary input: test/unit/gotest.junit.xml output: test/unit/gotest.md if: failure() || success() - name: Set the summary run: cat test/unit/gotest.md >> $GITHUB_STEP_SUMMARY if: failure() || success() # End-to-end integration/regression tests from test/cli # (Go-based replacement for legacy test/sharness shell scripts) cli-tests: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} timeout-minutes: 15 env: GOTRACEBACK: single # reduce noise on test timeout panics TEST_VERBOSE: 1 GIT_PAGER: cat IPFS_CHECK_RCMGR_DEFAULTS: 1 defaults: run: shell: bash steps: - name: Check out Kubo uses: actions/checkout@v6 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' - name: Install missing tools run: sudo apt update && sudo apt install -y zsh - name: Run CLI tests env: IPFS_PATH: ${{ runner.temp }}/ipfs-test run: make test_cli - name: Create JUnit XML report uses: ipdxco/gotest-json-to-junit-xml@v1 with: input: test/cli/cli-tests.json output: test/cli/cli-tests.junit.xml if: failure() || success() - name: Archive JUnit XML report uses: actions/upload-artifact@v7 with: name: cli-tests-junit path: test/cli/cli-tests.junit.xml if: failure() || success() - name: Create HTML report uses: ipdxco/junit-xml-to-html@v1 with: mode: no-frames input: test/cli/cli-tests.junit.xml output: test/cli/cli-tests.html if: failure() || success() - name: Archive HTML report uses: actions/upload-artifact@v7 with: name: cli-tests-html path: test/cli/cli-tests.html if: failure() || success() - name: Create Markdown report uses: ipdxco/junit-xml-to-html@v1 with: mode: summary input: test/cli/cli-tests.junit.xml output: test/cli/cli-tests.md if: failure() || success() - name: Set summary run: cat test/cli/cli-tests.md >> $GITHUB_STEP_SUMMARY if: failure() || success() # Example tests (kubo-as-a-library) example-tests: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} timeout-minutes: 5 env: GOTRACEBACK: single defaults: run: shell: bash steps: - name: Check out Kubo uses: actions/checkout@v6 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' - name: Run example tests run: make test_examples ================================================ FILE: .github/workflows/interop.yml ================================================ # Interoperability Tests # # This workflow ensures Kubo remains compatible with the broader IPFS ecosystem. # It builds Kubo from source, then runs: # # 1. helia-interop: Tests compatibility with Helia (JavaScript IPFS implementation) # using Playwright-based tests from @helia/interop package. # # 2. ipfs-webui: Runs E2E tests from ipfs/ipfs-webui repository to verify # the web interface works correctly with the locally built Kubo binary. # # Both jobs use caching to speed up repeated runs (npm dependencies, Playwright # browsers, and webui build artifacts). name: Interop on: workflow_dispatch: pull_request: paths-ignore: - '**/*.md' push: branches: - 'master' concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true defaults: run: shell: bash jobs: interop-prep: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest timeout-minutes: 5 env: TEST_DOCKER: 0 TEST_FUSE: 0 TEST_VERBOSE: 1 GIT_PAGER: cat IPFS_CHECK_RCMGR_DEFAULTS: 1 defaults: run: shell: bash steps: - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: 'go.mod' - run: make build - uses: actions/upload-artifact@v7 with: name: kubo path: cmd/ipfs/ipfs helia-interop: needs: [interop-prep] runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} timeout-minutes: 20 defaults: run: shell: bash steps: - uses: actions/setup-node@v6 with: node-version: lts/* - uses: actions/download-artifact@v8 with: name: kubo path: cmd/ipfs - run: chmod +x cmd/ipfs/ipfs - run: sudo apt update - run: sudo apt install -y libxkbcommon0 libxdamage1 libgbm1 libpango-1.0-0 libcairo2 # dependencies for playwright # Cache node_modules based on latest @helia/interop version from npm registry. # This ensures we always test against the latest release while still benefiting # from caching when the version hasn't changed. - name: Get latest @helia/interop version id: helia-version run: echo "version=$(npm view @helia/interop version)" >> $GITHUB_OUTPUT - name: Cache helia-interop node_modules uses: actions/cache@v5 id: helia-cache with: path: node_modules key: ${{ runner.os }}-helia-interop-${{ steps.helia-version.outputs.version }} - name: Install @helia/interop if: steps.helia-cache.outputs.cache-hit != 'true' run: npm install @helia/interop # TODO(IPIP-499): Remove --grep --invert workaround once helia implements IPIP-499 # Tracking issue: https://github.com/ipfs/helia/issues/941 # # PROVISIONAL HACK: Skip '@helia/mfs - should have the same CID after # creating a file' test due to IPIP-499 changes in kubo. # # WHY IT FAILS: The test creates a 5-byte file in MFS on both kubo and helia, # then compares the root directory CID. With kubo PR #11148, `ipfs files write` # now produces raw CIDs for single-block files (matching `ipfs add --raw-leaves`), # while helia uses `reduceSingleLeafToSelf: false` which keeps the dag-pb wrapper. # Different file CIDs lead to different directory CIDs. # # We run aegir directly (instead of helia-interop binary) because only aegir # supports the --grep/--invert flags needed to exclude specific tests. - name: Run helia-interop tests (excluding IPIP-499 incompatible test) run: npx aegir test -t node --bail -- --grep 'should have the same CID after creating a file' --invert env: KUBO_BINARY: ${{ github.workspace }}/cmd/ipfs/ipfs working-directory: node_modules/@helia/interop ipfs-webui: needs: [interop-prep] runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} timeout-minutes: 20 env: NO_SANDBOX: true LIBP2P_TCP_REUSEPORT: false LIBP2P_ALLOW_WEAK_RSA_KEYS: 1 E2E_IPFSD_TYPE: go GIT_PAGER: cat IPFS_CHECK_RCMGR_DEFAULTS: 1 defaults: run: shell: bash steps: - uses: actions/download-artifact@v8 with: name: kubo path: cmd/ipfs - run: chmod +x cmd/ipfs/ipfs - uses: actions/checkout@v6 with: repository: ipfs/ipfs-webui path: ipfs-webui - uses: actions/setup-node@v6 with: node-version-file: 'ipfs-webui/.tool-versions' - id: webui-ref run: echo "ref=$(git rev-parse --short HEAD)" | tee -a $GITHUB_OUTPUT working-directory: ipfs-webui - id: webui-state env: GITHUB_TOKEN: ${{ github.token }} ENDPOINT: repos/ipfs/ipfs-webui/commits/${{ steps.webui-ref.outputs.ref }}/status SELECTOR: .state KEY: state run: gh api "$ENDPOINT" --jq "$SELECTOR" | xargs -I{} echo "$KEY={}" | tee -a $GITHUB_OUTPUT # Cache node_modules based on package-lock.json - name: Cache node_modules uses: actions/cache@v5 id: node-modules-cache with: path: ipfs-webui/node_modules key: ${{ runner.os }}-webui-node-modules-${{ hashFiles('ipfs-webui/package-lock.json') }} restore-keys: | ${{ runner.os }}-webui-node-modules- - name: Install dependencies if: steps.node-modules-cache.outputs.cache-hit != 'true' run: npm ci --prefer-offline --no-audit --progress=false working-directory: ipfs-webui # Cache Playwright browsers - name: Cache Playwright browsers uses: actions/cache@v5 id: playwright-cache with: path: ~/.cache/ms-playwright key: ${{ runner.os }}-playwright-${{ hashFiles('ipfs-webui/package-lock.json') }} restore-keys: | ${{ runner.os }}-playwright- # On cache miss: download browsers and install OS dependencies - name: Install Playwright with dependencies if: steps.playwright-cache.outputs.cache-hit != 'true' run: npx playwright install --with-deps working-directory: ipfs-webui # On cache hit: only ensure OS dependencies are present (fast, idempotent) - name: Install Playwright OS dependencies if: steps.playwright-cache.outputs.cache-hit == 'true' run: npx playwright install-deps working-directory: ipfs-webui # Cache test build output - name: Cache test build uses: actions/cache@v5 id: test-build-cache with: path: ipfs-webui/build key: ${{ runner.os }}-webui-build-${{ hashFiles('ipfs-webui/package-lock.json', 'ipfs-webui/src/**', 'ipfs-webui/public/**') }} restore-keys: | ${{ runner.os }}-webui-build- - name: Build ipfs-webui@${{ steps.webui-ref.outputs.ref }} (state=${{ steps.webui-state.outputs.state }}) if: steps.test-build-cache.outputs.cache-hit != 'true' run: npm run test:build working-directory: ipfs-webui - name: Test ipfs-webui@${{ steps.webui-ref.outputs.ref }} (state=${{ steps.webui-state.outputs.state }}) E2E against the locally built Kubo binary run: npm run test:e2e env: IPFS_GO_EXEC: ${{ github.workspace }}/cmd/ipfs/ipfs working-directory: ipfs-webui - name: Upload test artifacts on failure if: failure() uses: actions/upload-artifact@v7 with: name: webui-test-results path: ipfs-webui/test-results/ retention-days: 7 ================================================ FILE: .github/workflows/sharness.yml ================================================ name: Sharness on: workflow_dispatch: pull_request: paths-ignore: - "**/*.md" push: branches: - "master" concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true jobs: sharness-test: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }} timeout-minutes: ${{ github.repository == 'ipfs/kubo' && 15 || 60 }} defaults: run: shell: bash steps: - name: Checkout Kubo uses: actions/checkout@v6 with: path: kubo - name: Setup Go uses: actions/setup-go@v6 with: go-version-file: 'kubo/go.mod' - name: Install missing tools run: sudo apt update && sudo apt install -y socat net-tools fish libxml2-utils - uses: actions/cache@v5 with: path: test/sharness/lib/dependencies key: ${{ runner.os }}-test-generate-junit-html-${{ hashFiles('test/sharness/lib/test-generate-junit-html.sh') }} - name: Run Sharness tests run: | make -O -j "$PARALLEL" \ test_sharness \ coverage/sharness_tests.coverprofile \ test/sharness/test-results/sharness.xml working-directory: kubo env: TEST_DOCKER: 1 TEST_PLUGIN: 0 TEST_FUSE: 0 TEST_VERBOSE: 1 TEST_JUNIT: 1 TEST_EXPENSIVE: 1 IPFS_CHECK_RCMGR_DEFAULTS: 1 CONTINUE_ON_S_FAILURE: 1 # increasing parallelism beyond 10 doesn't speed up the tests much PARALLEL: ${{ github.repository == 'ipfs/kubo' && 10 || 3 }} - name: Upload coverage report uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 if: failure() || success() with: name: sharness files: kubo/coverage/sharness_tests.coverprofile token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false - name: Aggregate results run: find kubo/test/sharness/test-results -name 't*-*.sh.*.counts' | kubo/test/sharness/lib/sharness/aggregate-results.sh > kubo/test/sharness/test-results/summary.txt - name: 👉️ If this step failed, go to «Summary» (top left) → «HTML Report» → inspect the «Failures» column run: | cat kubo/test/sharness/test-results/summary.txt && grep 'failed\s*0' kubo/test/sharness/test-results/summary.txt - name: Add aggregate results to the summary if: failure() || success() run: | echo "# Summary" >> $GITHUB_STEP_SUMMARY echo >> $GITHUB_STEP_SUMMARY cat kubo/test/sharness/test-results/summary.txt >> $GITHUB_STEP_SUMMARY - name: Generate one-page HTML report uses: ipdxco/junit-xml-to-html@v1 if: failure() || success() with: mode: no-frames input: kubo/test/sharness/test-results/sharness.xml output: kubo/test/sharness/test-results/sharness.html - name: Upload one-page HTML report to S3 id: one-page uses: ipdxco/custom-github-runners/.github/actions/upload-artifact@main if: github.repository == 'ipfs/kubo' && (failure() || success()) with: source: kubo/test/sharness/test-results/sharness.html destination: sharness.html - name: Upload one-page HTML report if: github.repository != 'ipfs/kubo' && (failure() || success()) uses: actions/upload-artifact@v7 with: name: sharness.html path: kubo/test/sharness/test-results/sharness.html - name: Generate full HTML report uses: ipdxco/junit-xml-to-html@v1 if: failure() || success() with: mode: frames input: kubo/test/sharness/test-results/sharness.xml output: kubo/test/sharness/test-results/sharness-html - name: Upload full HTML report to S3 id: full uses: ipdxco/custom-github-runners/.github/actions/upload-artifact@main if: github.repository == 'ipfs/kubo' && (failure() || success()) with: source: kubo/test/sharness/test-results/sharness-html destination: sharness-html/ - name: Upload full HTML report if: github.repository != 'ipfs/kubo' && (failure() || success()) uses: actions/upload-artifact@v7 with: name: sharness-html path: kubo/test/sharness/test-results/sharness-html - name: Add S3 links to the summary if: github.repository == 'ipfs/kubo' && (failure() || success()) run: echo "$MD" >> $GITHUB_STEP_SUMMARY env: MD: | # HTML Reports - View the [one page HTML report](${{ steps.one-page.outputs.url }}) - View the [full HTML report](${{ steps.full.outputs.url }}index.html) ================================================ FILE: .github/workflows/spellcheck.yml ================================================ name: Spell Check on: pull_request: push: branches: ["master"] workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true jobs: spellcheck: uses: ipdxco/unified-github-workflows/.github/workflows/reusable-spellcheck.yml@v1 ================================================ FILE: .github/workflows/stale.yml ================================================ name: Close Stale Issues on: schedule: - cron: '0 0 * * *' workflow_dispatch: permissions: issues: write pull-requests: write jobs: stale: uses: ipdxco/unified-github-workflows/.github/workflows/reusable-stale-issue.yml@v1 ================================================ FILE: .github/workflows/sync-release-assets.yml ================================================ name: Sync GitHub Release Assets on: workflow_dispatch: schedule: - cron: '0 0 * * *' concurrency: group: release-assets-dist-sync cancel-in-progress: true permissions: contents: write # to upload release asset jobs: dist-ipfs-tech: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' runs-on: "ubuntu-latest" timeout-minutes: 15 steps: - uses: ipfs/download-ipfs-distribution-action@v1 - uses: ipfs/start-ipfs-daemon-action@v1 with: args: --init --init-profile=flatfs,server --enable-gc=false - uses: actions/setup-node@v6 with: node-version: 14 - name: Sync the latest 5 github releases uses: actions/github-script@v8 with: script: | const fs = require('fs').promises const max_synced = 5 // fetch github releases resp = await github.rest.repos.listReleases({ owner: context.repo.owner, repo: context.repo.repo, page: 1, per_page: max_synced }) const release_assets = []; num_synced = 0; for (const release of resp.data) { console.log("checking release tagged", release.tag_name) if (num_synced > max_synced) { console.log("done: synced", max_synced, "latest releases") break; } num_synced += 1 const github_assets = new Set() for (const asset of release.assets) { github_assets.add(asset.name) } // fetch asset info from dist.ipfs.tech p = '/ipns/dist.ipfs.tech/kubo/' + release.tag_name let stdout = '' const options = {} options.listeners = { stdout: (data) => { stdout += data.toString(); } } await exec.exec('ipfs', ['ls', p], options) const dist_assets = new Set() const missing_files = [] for (const raw_line of stdout.split("\n")) { line = raw_line.trim(); if (line.length != 0) { file = line.split(/(\s+)/).filter( function(e) { return e.trim().length > 0; } )[2] dist_assets.add(file) if (!github_assets.has(file)) { missing_files.push(file) } } } // if dist.ipfs.tech has files not found in github, copy them over for (const file of missing_files) { file_sha = file + ".sha512" file_cid = file + ".cid" // skip files that don't have .cid and .sha512 checksum files if (!dist_assets.has(file_sha) || !dist_assets.has(file_cid)) { if (!file.endsWith('.cid') && !file.endsWith('.sha512')) { // silent skip of .sha512.sha512 :) console.log(`skipping "${file}" as dist.ipfs.tech does not provide .cid and .sha512 checksum files for it`) } continue } console.log("fetching", file, "from dist.ipfs.tech") await exec.exec('ipfs', ['get', p + '/' + file]) await exec.exec('ipfs', ['get', p + '/' + file_sha]) await exec.exec('ipfs', ['get', p + '/' + file_cid]) console.log("verifying contents of", file) // compute sha512 output for file let sha_stdout = '' const sha_options = {} sha_options.listeners = { stdout: (data) => { sha_stdout += data.toString(); } } await exec.exec('sha512sum', [file], sha_options) // read expected sha512 output const sha_data = await fs.readFile(file_sha, "utf8") const digest = (s) => s.split(' ').shift() if (digest(sha_data) != digest(sha_stdout)) { console.log(`${file}.sha512: ${sha_data}`) console.log(`sha512sum ${file}: ${sha_stdout}`) throw "checksum verification failed for " + file } console.log("uploading", file, "to github release", release.tag_name) const uploadReleaseAsset = async (file) => github.rest.repos.uploadReleaseAsset({ owner: context.repo.owner, repo: context.repo.repo, release_id: release.id, headers: { "content-type": "application/octet-stream", "content-length": `${(await fs.stat(file)).size}` }, name: file, data: await fs.readFile(file) }) await uploadReleaseAsset(file) await uploadReleaseAsset(file_sha) await uploadReleaseAsset(file_cid) } // summary of assets on both sides release_assets.push({ tag: release.tag_name, github_assets, dist_assets }) } console.log(release_assets) return release_assets ================================================ FILE: .github/workflows/test-migrations.yml ================================================ name: Migrations on: workflow_dispatch: pull_request: paths: # Migration implementation files - 'repo/fsrepo/migrations/**' - 'test/cli/migrations/**' # Config and repo handling - 'repo/fsrepo/**' # This workflow file itself - '.github/workflows/test-migrations.yml' push: branches: - 'master' - 'release-*' paths: - 'repo/fsrepo/migrations/**' - 'test/cli/migrations/**' - 'repo/fsrepo/**' - '.github/workflows/test-migrations.yml' concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true jobs: test: strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} timeout-minutes: 20 env: TEST_VERBOSE: 1 IPFS_CHECK_RCMGR_DEFAULTS: 1 defaults: run: shell: bash steps: - name: Check out Kubo uses: actions/checkout@v6 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' - name: Build kubo binary run: | make build echo "Built ipfs binary at $(pwd)/cmd/ipfs/" - name: Add kubo to PATH run: | echo "$(pwd)/cmd/ipfs" >> $GITHUB_PATH - name: Verify ipfs in PATH run: | which ipfs || echo "ipfs not in PATH" ipfs version || echo "Failed to run ipfs version" - name: Run migration unit tests run: | go test ./repo/fsrepo/migrations/... - name: Run CLI migration tests env: IPFS_PATH: ${{ runner.temp }}/ipfs-test run: | export PATH="${{ github.workspace }}/cmd/ipfs:$PATH" which ipfs || echo "ipfs not found in PATH" ipfs version || echo "Failed to run ipfs version" go test ./test/cli/migrations/... - name: Upload test results if: always() uses: actions/upload-artifact@v7 with: name: ${{ matrix.os }}-test-results path: | test/**/*.log ${{ runner.temp }}/ipfs-test/ ================================================ FILE: .gitignore ================================================ # ipfs can generate profiling dump files *.cpuprof *.memprof *.swp .ipfsconfig *.out *.coverprofile *.test *.orig *~ coverage.txt gx-workspace-update.json .ipfs bin/gx bin/protoc-* bin/gx* bin/tmp bin/gocovmerge bin/cover vendor .tarball go-ipfs-source.tar.gz docs/examples/go-ipfs-as-a-library/example-folder/Qm* /test/sharness/t0054-dag-car-import-export-data/*.car # test artifacts from make test_unit / test_cli /test/unit/gotest.json /test/unit/gotest.junit.xml /test/cli/cli-tests.json # ignore build output from snapcraft /ipfs_*.snap /parts /stage /prime ================================================ FILE: .golangci.yml ================================================ linters: enable: - stylecheck linters-settings: stylecheck: checks: - all - '-ST1003' dot-import-whitelist: - github.com/ipfs/kubo/test/cli/testutils ================================================ FILE: .hadolint.yaml ================================================ # Hadolint configuration for Kubo Docker image # https://github.com/hadolint/hadolint # Ignore specific rules ignored: # DL3008: Pin versions in apt-get install # We use stable base images and prefer smaller layers over version pinning - DL3008 # Trust base images from these registries trustedRegistries: - docker.io - gcr.io ================================================ FILE: .mailmap ================================================ # This file allows re-mapping author names/emails # while maintaining the integrity of the repository # # Spec: https://www.kernel.org/pub/software/scm/git/docs/git-shortlog.html#_mapping_authors # @RubenKelevra Aaron Hill Adam Gashlin Adam Uhlir Adin Schmahmann Adrian Lanzafame Adrian Ulrich Alan Shaw Alec Brickner Alex Alfie John Alfonso Montero Ali Mirlou Andres Buritica Andrew Chin Andrew Nesbitt Andy Leap Antti Kaihola Artem Andreenko Arthur Elliott Bamvor Zhang Baptiste Jonglez Bernhard M. Wiedemann Boris Mann Brendan Mc Brendan McMillion Brendan O'Brien Brian Tiger Chow Brian Tiger Chow Caian Caio Alonso Carlos Cobo Casey Chance Cayman Nava Chas Leichner Chris Boddy Chris Chinchilla Chris Grimmett Chris P Christopher Sasarak Christian Couder Christian Kniep Christopher Buesser Cole Brown Corbin Page Cornelius Toole Dan <35669742+NukeManDan@users.noreply.github.com> Daniel Aleksandersen Daniel Grossmann-Kavanagh Daniel Mack David David Braun David Brennan David Dias David Wagner Devin Dimitris Apostolou Diogo Silva Dirk McCormick Djalil Dreamski <32184973+dreamski21@users.noreply.github.com> Dominic Della Valle Dominic Tarr Ian Preston Dylan Powers Edison Lee Elias Gabrielsson Emery Hemingway Enrique Erne Eoghan Ó Carragáin Erik Ingenito Esteban Ethan Buchman Etienne Laurin Forrest Weston Francesco Canessa Frank Sachsenheim Frederik Riedel Friedel Ziegelmayer George Antoniadis George Masgras Giulitti Salvatore Giuseppe Bertone Gowtham G Harald Nordgren Harlan T Wood Hector Sanjuan Henrique Dias Henry Herman Junge Hlib Ho-Sheng Hsiao Hucg <41573766+hcg1314@users.noreply.github.com> Iaroslav Gridin Igor Velkov Ivan JP Hastings-Spital Jack Loughran <30052269+jackloughran@users.noreply.github.com> Jakub Sztandera Jakub Kaczmarzyk James Stanley Jamie Wilkinson Jan Winkelmann Jason Carver Jeff Thompson Jeromy Johnson Jesse Weinstein Jessica Schilling Jim McDonald John Reed Johnny <9611008+johnnymatthews@users.noreply.github.com> Jon Choi Jonathan Dahan Jordan Danford Jorropo Juan Batiz-Benet Justin Drake Kacper Łukawski Karthik Bala Kejie Zhang <601172892@qq.com> Kerem Kevin Atkinson Kevin Simper Kevin Wallace Kirill Goncharov Kishan Mohanbhai Sagathiya Knut Ahlers Konstantin Koroviev Koushik Roy Kristoffer Ström Kuro1 <412681778@qq.com> Lars Gierth Leo Arias Li Zheng Lorenzo Manacorda Lorenzo Setale Louis Thibault Lucas Garron Lucas Molas Marcin Janczyk Marcin Rataj Markus Amalthea Magnuson Marten Seemann Masashi Salvador Mitsuzawa Massino Tinovan Mat Kelly Mathijs de Bruin Matouš Skála Matt Bell Matt Joiner Max Chechel Max Kerp Mib Kd743naq Michael Avila Michael Lovci Michael Muré Michael Pfister Michelle Lee Miguel Torres Mikaela Suomalainen Mildred Ki'Lya Molly Muneeb Ali Mykola Nikishov Nathan Musoke Nick Hamann Oli Evans Or Rikon Overbool Patrick Connolly Pavol Rusnak Peter Borzov Peter Rabbitson Peter Wu Philip Nelson Pierre-Alain TORET PoorPockets McNewHold Pretty Please Mark Darkly <55382229+pleasemarkdarkly@users.noreply.github.com> Péter Szilágyi Quantomic Quinn Slack Raúl Kripalani ReadmeCritic Remco Bloemen Richard Littauer RideWindX Rob Brackett Robert Carlsen Rod Vagg Roerick Sweeney Roman Khafizianov Roman Proskuryakov Ronsor RubenKelevra Ryan Carver Ryan Morey SH Sag0Sag0 Sander Pick Scott Bigelow Sean Lang Shanti Bouchez-Mongardé Shaun Bruce Sherod Taylor Simon Kirkby Simon Menke Siraj Ravel Siva Chandran Spartucus Stephan Kulla Stephan Seidt Stephen Sugden Stephen Whitmore Steve Recio Steven Allen Steven Vandevelde Sönke Hahn TUSF Tarnay Kálmán Thomas Gardner Tiger Tim Groeneveld Tim Stahel Timothy Hobbs Tom O'Donnell Tom Swindell Tommi Virtanen Tonis Tiigi Tor Arne Vestbø Travis Person Tylar John Reed Vasil Dimov Vijayee Kulkaa Vikram Vitor Baptista W. Trevor King Wes Morgan Will Scott Willi Butz Xiaoyi Wang Yuval Langer Zander Mackie ZenGround0 achingbrain adamliesko anarcat bbenshoof camelmasa chenminjian <727180553@qq.com> devedge dgrisham drathir epitron eric wu flowed forstmeier fyrchik gatesvp hannahhoward hikerpig hoenirvili hucg ivan386 klauspost kpcyrd kvm2116 mateon1 matrushka michael myself659 nmalhotra palkeo requilence rht rob-deutsch slothbag sroerick swedneck <40505480+swedneck@users.noreply.github.com> tarekbadr tcme tg theswitch verokarhu vitzli vyzo wzhd zramsay Łukasz Magiera ᴍᴀᴛᴛ ʙᴇʟʟ ᴠɪᴄᴛᴏʀ ʙᴊᴇʟᴋʜᴏʟᴍ ================================================ FILE: AGENTS.md ================================================ # AI Agent Instructions for Kubo This file provides instructions for AI coding agents working on the [Kubo](https://github.com/ipfs/kubo) codebase (the Go implementation of IPFS). Follow the [Developer Guide](docs/developer-guide.md) for full details. ## Quick Reference | Task | Command | |-------------------|----------------------------------------------------------| | Tidy deps | `make mod_tidy` (run first if `go.mod` changed) | | Build | `make build` | | Unit tests | `go test ./... -run TestName -v` | | Integration tests | `make build && go test ./test/cli/... -run TestName -v` | | Lint | `make -O test_go_lint` | | Format | `go fmt ./...` | ## Project Overview Kubo is the reference implementation of IPFS in Go. Most IPFS protocol logic lives in [boxo](https://github.com/ipfs/boxo) (the IPFS SDK); kubo wires it together and exposes it via CLI and HTTP RPC API. If a change belongs in the protocol layer, it likely belongs in boxo, not here. Key directories: | Directory | Purpose | |--------------------|----------------------------------------------------------| | `cmd/ipfs/` | CLI entry point and binary | | `core/` | core IPFS node implementation | | `core/commands/` | CLI command definitions | | `core/coreapi/` | Go API implementation | | `client/rpc/` | HTTP RPC client | | `plugin/` | plugin system | | `repo/` | repository management | | `test/cli/` | Go-based CLI integration tests (preferred for new tests) | | `test/sharness/` | legacy shell-based integration tests | | `docs/` | documentation | Other key external dependencies: [go-libp2p](https://github.com/libp2p/go-libp2p) (networking), [go-libp2p-kad-dht](https://github.com/libp2p/go-libp2p-kad-dht) (DHT). ## Go Style Follow these Go style references: - [Go Code Review Comments](https://go.dev/wiki/CodeReviewComments) - [Google Go Style Decisions](https://google.github.io/styleguide/go/decisions) Specific conventions for this project: - check the Go version in `go.mod` and use idiomatic features available at that version - readability over micro-optimization: clear code is more important than saving microseconds - prefer standard library functions and utilities over writing your own - use early returns and indent the error flow, not the happy path - use `slices.Contains`, `slices.DeleteFunc`, and the `maps` package instead of manual loops - preallocate slices and maps when the size is known: `make([]T, 0, n)` - use `map[K]struct{}` for sets, not `map[K]bool` - receiver names: single-letter abbreviations matching the type (e.g., `s *Server`, `c *Client`) - run `go fmt` after modifying Go source files, never indent manually ### Error Handling - wrap errors with `fmt.Errorf("context: %w", err)`, never discard errors silently - use `errors.Is` / `errors.As` for error checking, not string comparison - never use `panic` in library code; only in `main` or test helpers - return `nil` explicitly for the error value on success paths ### Canonical Examples When adding or modifying code, follow the patterns established in these files: - CLI command structure: `core/commands/dag/dag.go` - CLI integration test: `test/cli/dag_test.go` - Test harness usage: `test/cli/harness/` package ## Building Always run commands from the repository root. ```bash make mod_tidy # update go.mod/go.sum (use this instead of go mod tidy) make build # build the ipfs binary to cmd/ipfs/ipfs make install # install to $GOPATH/bin make -O test_go_lint # run linter (use this instead of golangci-lint directly) ``` If you modify `go.mod` (add/remove/update dependencies), you must run `make mod_tidy` first, before building or testing. Use `make mod_tidy` instead of `go mod tidy` directly, as the project has multiple `go.mod` files. If you modify any `.go` files outside of `test/`, you must run `make build` before running integration tests. ## Testing The full test suite is composed of several targets: | Make target | What it runs | |----------------------|-----------------------------------------------------------------------| | `make test` | all tests (`test_go_fmt` + `test_unit` + `test_cli` + `test_sharness`) | | `make test_short` | fast subset (`test_go_fmt` + `test_unit`) | | `make test_unit` | unit tests with coverage (excludes `test/cli`) | | `make test_cli` | CLI integration tests (requires `make build` first) | | `make test_sharness` | legacy shell-based integration tests | | `make test_go_fmt` | checks Go source formatting | | `make -O test_go_lint` | runs `golangci-lint` | During development, prefer running a specific test rather than the full suite: ```bash # run a single unit test go test ./core/... -run TestSpecificUnit -v # run a single CLI integration test (requires make build first) go test ./test/cli/... -run TestSpecificCLI -v ``` ### Environment Setup for Integration Tests Before running `test_cli` or `test_sharness`, set these environment variables from the repo root: ```bash export PATH="$PWD/cmd/ipfs:$PATH" export IPFS_PATH="$(mktemp -d)" ``` - `PATH`: integration tests use the `ipfs` binary from `PATH`, not Go source directly - `IPFS_PATH`: isolates test data from `~/.ipfs` or other running nodes If you see "version (N) is lower than repos (M)", the `ipfs` binary in `PATH` is outdated. Rebuild with `make build` and verify `PATH`. ### Running Sharness Tests Sharness tests are legacy shell-based tests. Run individual tests with a timeout: ```bash cd test/sharness && timeout 60s ./t0080-repo.sh ``` To investigate a failing test, pass `-v` for verbose output. In this mode, daemons spawned by the test are not shut down automatically and must be killed manually afterwards. ### Cleaning Up Stale Daemons Before running `test/cli` or `test/sharness`, stop any stale `ipfs daemon` processes owned by the current user. Leftover daemons hold locks and bind ports, causing test failures: ```bash pkill -f "ipfs daemon" ``` ### Writing Tests - all new integration tests go in `test/cli/`, not `test/sharness/` - if a `test/sharness` test needs significant changes, remove it and add a replacement in `test/cli/` - use [testify](https://github.com/stretchr/testify) for assertions (already a dependency) - for Go 1.25+, use `testing/synctest` when testing concurrent code (goroutines, channels, timers) - reuse existing `.car` fixtures in `test/cli/fixtures/` when possible; only add new fixtures when the test requires data not covered by existing ones - always re-run modified tests locally before submitting to confirm they pass - avoid emojis in test names and test log output ## Before Submitting Run these steps in order before considering work complete: 1. `make mod_tidy` (if `go.mod` changed) 2. `go fmt ./...` 3. `make build` (if non-test `.go` files changed) 4. `make -O test_go_lint` 5. `go test ./...` (or the relevant subset) ## Documentation and Commit Messages - after editing CLI help text in `core/commands/`, verify width: `go test ./test/cli/... -run TestCommandDocsWidth` - config options are documented in `docs/config.md` - changelogs in `docs/changelogs/`: only edit the Table of Contents and the Highlights section; the Changelog and Contributors sections are auto-generated and must not be modified - follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) - keep commit titles short and messages terse ## Writing Style When writing docs, comments, and commit messages: - avoid emojis in code, comments, and log output - keep an empty line before lists in markdown - use backticks around CLI commands, paths, environment variables, and config options ## PR Guidelines - explain what changed and why in the PR description - include test coverage for new functionality and bug fixes - run `make -O test_go_lint` and fix any lint issues before submitting - verify that `go test ./...` passes locally - when modifying `test/sharness` tests significantly, migrate them to `test/cli` instead - end the PR description with a `## References` section listing related context, one link per line - if the PR closes an issue in `ipfs/kubo`, each closing reference should be a bullet starting with `Closes`: ```markdown ## References - Closes https://github.com/ipfs/kubo/issues/1234 - Closes https://github.com/ipfs/kubo/issues/5678 - https://discuss.ipfs.tech/t/related-topic/999 ``` ## Scope and Safety Do not modify or touch: - files under `test/sharness/lib/` (third-party sharness test framework) - CI workflows in `.github/` unless explicitly asked - auto-generated sections in `docs/changelogs/` (Changelog and Contributors are generated; only TOC and Highlights are human-edited) Do not run without being asked: - `make test` or `make test_sharness` (full suite is slow; prefer targeted tests) - `ipfs daemon` without a timeout ## Running the Daemon Always run the daemon with a timeout or shut it down promptly: ```bash timeout 60s ipfs daemon # auto-kill after 60s ipfs shutdown # graceful shutdown via API ``` Kill dangling daemons before re-running tests: `pkill -f "ipfs daemon"` ================================================ FILE: CHANGELOG.md ================================================ # Kubo Changelogs - [v0.42](docs/changelogs/v0.42.md) - [v0.41](docs/changelogs/v0.41.md) - [v0.40](docs/changelogs/v0.40.md) - [v0.39](docs/changelogs/v0.39.md) - [v0.38](docs/changelogs/v0.38.md) - [v0.37](docs/changelogs/v0.37.md) - [v0.36](docs/changelogs/v0.36.md) - [v0.35](docs/changelogs/v0.35.md) - [v0.34](docs/changelogs/v0.34.md) - [v0.33](docs/changelogs/v0.33.md) - [v0.32](docs/changelogs/v0.32.md) - [v0.31](docs/changelogs/v0.31.md) - [v0.30](docs/changelogs/v0.30.md) - [v0.29](docs/changelogs/v0.29.md) - [v0.28](docs/changelogs/v0.28.md) - [v0.27](docs/changelogs/v0.27.md) - [v0.26](docs/changelogs/v0.26.md) - [v0.25](docs/changelogs/v0.25.md) - [v0.24](docs/changelogs/v0.24.md) - [v0.23](docs/changelogs/v0.23.md) - [v0.22](docs/changelogs/v0.22.md) - [v0.21](docs/changelogs/v0.21.md) - [v0.20](docs/changelogs/v0.20.md) - [v0.19](docs/changelogs/v0.19.md) - [v0.18](docs/changelogs/v0.18.md) - [v0.17](docs/changelogs/v0.17.md) - [v0.16](docs/changelogs/v0.16.md) - [v0.15](docs/changelogs/v0.15.md) - [v0.14](docs/changelogs/v0.14.md) - [v0.13](docs/changelogs/v0.13.md) - [v0.12](docs/changelogs/v0.12.md) - [v0.11](docs/changelogs/v0.11.md) - [v0.10](docs/changelogs/v0.10.md) - [v0.9](docs/changelogs/v0.9.md) - [v0.8](docs/changelogs/v0.8.md) - [v0.7](docs/changelogs/v0.7.md) - [v0.6](docs/changelogs/v0.6.md) - [v0.5](docs/changelogs/v0.5.md) - [v0.4](docs/changelogs/v0.4.md) - [v0.3](docs/changelogs/v0.3.md) - [v0.2](docs/changelogs/v0.2.md) ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing to Kubo **For development setup, building, and testing, see the [Developer Guide](docs/developer-guide.md).** IPFS as a project, including Kubo and all of its modules, follows the [standard IPFS Community contributing guidelines](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md). We also adhere to the [Go IPFS Community contributing guidelines](https://github.com/ipfs/community/blob/master/CONTRIBUTING_GO.md) which provide additional information on how to collaborate and contribute to the Go implementation of IPFS. We appreciate your time and attention for going over these. Please open an issue on ipfs/community if you have any questions. Thank you. ================================================ FILE: Dockerfile ================================================ # syntax=docker/dockerfile:1 # Enables BuildKit with cache mounts for faster builds FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.25 AS builder ARG TARGETOS TARGETARCH ENV SRC_DIR=/kubo # Cache go module downloads between builds for faster rebuilds COPY go.mod go.sum $SRC_DIR/ WORKDIR $SRC_DIR RUN --mount=type=cache,target=/go/pkg/mod \ go mod download COPY . $SRC_DIR # Preload an in-tree but disabled-by-default plugin by adding it to the IPFS_PLUGINS variable # e.g. docker build --build-arg IPFS_PLUGINS="foo bar baz" ARG IPFS_PLUGINS # Allow for other targets to be built, e.g.: docker build --build-arg MAKE_TARGET="nofuse" ARG MAKE_TARGET=build # Build ipfs binary with cached go modules and build cache. # mkdir .git/objects allows git rev-parse to read commit hash for version info RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ mkdir -p .git/objects \ && GOOS=$TARGETOS GOARCH=$TARGETARCH GOFLAGS=-buildvcs=false make ${MAKE_TARGET} IPFS_PLUGINS=$IPFS_PLUGINS # Extract required runtime tools from Debian. # We use Debian instead of Alpine because we need glibc compatibility # for the busybox base image we're using. FROM debian:bookworm-slim AS utilities RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends \ tini \ # Using gosu (~2MB) instead of su-exec (~20KB) because it's easier to # install on Debian. Useful links: # - https://github.com/ncopa/su-exec#why-reinvent-gosu # - https://github.com/tianon/gosu/issues/52#issuecomment-441946745 gosu \ # fusermount enables IPFS mount commands fuse \ ca-certificates \ ; \ apt-get clean; \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Final minimal image with shell for debugging (busybox provides sh) FROM busybox:stable-glibc # Copy ipfs binary, startup scripts, and runtime dependencies ENV SRC_DIR=/kubo COPY --from=utilities /usr/sbin/gosu /sbin/gosu COPY --from=utilities /usr/bin/tini /sbin/tini COPY --from=utilities /bin/fusermount /usr/local/bin/fusermount COPY --from=utilities /etc/ssl/certs /etc/ssl/certs COPY --from=builder $SRC_DIR/cmd/ipfs/ipfs /usr/local/bin/ipfs COPY --from=builder --chmod=755 $SRC_DIR/bin/container_daemon /usr/local/bin/start_ipfs COPY --from=builder $SRC_DIR/bin/container_init_run /usr/local/bin/container_init_run # Set SUID for fusermount to enable FUSE mounting by non-root user RUN chmod 4755 /usr/local/bin/fusermount # Swarm P2P port (TCP/UDP) - expose publicly for peer connections EXPOSE 4001 4001/udp # API port - keep private, only for trusted clients EXPOSE 5001 # Gateway port - can be exposed publicly via reverse proxy EXPOSE 8080 # Swarm WebSockets - expose publicly for browser-based peers EXPOSE 8081 # Create ipfs user (uid 1000) and required directories with proper ownership ENV IPFS_PATH=/data/ipfs RUN mkdir -p $IPFS_PATH /ipfs /ipns /mfs /container-init.d \ && adduser -D -h $IPFS_PATH -u 1000 -G users ipfs \ && chown ipfs:users $IPFS_PATH /ipfs /ipns /mfs /container-init.d # Volume for IPFS repository data persistence VOLUME $IPFS_PATH # The default logging level ENV GOLOG_LOG_LEVEL="" # Entrypoint initializes IPFS repo if needed and configures networking. # tini ensures proper signal handling and zombie process cleanup ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/start_ipfs"] # Health check verifies IPFS daemon is responsive. # Uses empty directory CID (QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn) as test HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ CMD ipfs --api=/ip4/127.0.0.1/tcp/5001 dag stat /ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn || exit 1 # Default: run IPFS daemon with auto-migration enabled CMD ["daemon", "--migrate=true", "--agent-version-suffix=docker"] ================================================ FILE: FUNDING.json ================================================ { "opRetro": { "projectId": "0x7f330267969cf845a983a9d4e7b7dbcca5c700a5191269af377836d109e0bb69" } } ================================================ FILE: GNUmakefile ================================================ # General tools SHELL=PATH='$(PATH)' /bin/sh # enable second expansion .SECONDEXPANSION: include Rules.mk ================================================ FILE: LICENSE ================================================ This project is transitioning from an MIT-only license to a dual MIT/Apache-2.0 license. Unless otherwise noted, all code contributed prior to 2019-05-06 and not contributed by a user listed in [this signoff issue](https://github.com/ipfs/go-ipfs/issues/6302) is licensed under MIT-only. All new contributions (and past contributions since 2019-05-06) are licensed under a dual MIT/Apache-2.0 license. MIT: https://www.opensource.org/licenses/mit Apache-2.0: https://www.apache.org/licenses/LICENSE-2.0 ================================================ FILE: LICENSE-APACHE ================================================ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: LICENSE-MIT ================================================ The MIT License (MIT) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ all: @gmake $@ .PHONY: all .DEFAULT: @gmake $@ ================================================ FILE: README.md ================================================


Kubo logo
Kubo: IPFS Implementation in Go

The first implementation of IPFS.

Official Part of IPFS Project Discourse Forum Matrix GitHub release

What is Kubo? | Quick Taste | Install | Documentation | Development | Getting Help

## What is Kubo? Kubo was the first [IPFS](https://docs.ipfs.tech/concepts/what-is-ipfs/) implementation and is the [most widely used one today](https://probelab.io/ipfs/topology/#chart-agent-types-avg). It takes an opinionated approach to content-addressing ([CIDs](https://docs.ipfs.tech/concepts/glossary/#cid), [DAGs](https://docs.ipfs.tech/concepts/glossary/#dag)) that maximizes interoperability: [UnixFS](https://docs.ipfs.tech/concepts/glossary/#unixfs) for files and directories, [HTTP Gateways](https://docs.ipfs.tech/concepts/glossary/#gateway) for web browsers, [Bitswap](https://docs.ipfs.tech/concepts/glossary/#bitswap) and [HTTP](https://specs.ipfs.tech/http-gateways/trustless-gateway/) for verifiable data transfer. **Features:** - Runs an IPFS node as a network service (LAN [mDNS](https://github.com/libp2p/specs/blob/master/discovery/mdns.md) and WAN [Amino DHT](https://docs.ipfs.tech/concepts/glossary/#dht)) - [Command-line interface](https://docs.ipfs.tech/reference/kubo/cli/) (`ipfs --help`) - [WebUI](https://github.com/ipfs/ipfs-webui/#readme) for node management - [HTTP Gateway](https://specs.ipfs.tech/http-gateways/) for trusted and [trustless](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval) content retrieval - [HTTP RPC API](https://docs.ipfs.tech/reference/kubo/rpc/) to control the daemon - [HTTP Routing V1](https://specs.ipfs.tech/routing/http-routing-v1/) client and server for [delegated routing](./docs/delegated-routing.md) - [Content blocking](./docs/content-blocking.md) for public node operators **Other IPFS implementations:** [Helia](https://github.com/ipfs/helia) (JavaScript), [more...](https://docs.ipfs.tech/concepts/ipfs-implementations/) ## Quick Taste After [installing Kubo](#install), verify it works: ```console $ ipfs init generating ED25519 keypair...done peer identity: 12D3KooWGcSLQdLDBi2BvoP8WnpdHvhWPbxpGcqkf93rL2XMZK7R $ ipfs daemon & Daemon is ready $ echo "hello IPFS" | ipfs add -q --cid-version 1 bafkreicouv3sksjuzxb3rbb6rziy6duakk2aikegsmtqtz5rsuppjorxsa $ ipfs cat bafkreicouv3sksjuzxb3rbb6rziy6duakk2aikegsmtqtz5rsuppjorxsa hello IPFS ``` Verify this CID is provided by your node to the IPFS network: See `ipfs add --help` for all import options. Ready for more? Follow the [command-line quick start](https://docs.ipfs.tech/how-to/command-line-quick-start/). ## Install Follow the [official installation guide](https://docs.ipfs.tech/install/command-line/), or choose: [prebuilt binary](#official-prebuilt-binaries) | [Docker](#docker) | [package manager](#package-managers) | [from source](#build-from-source). Prefer a GUI? Try [IPFS Desktop](https://docs.ipfs.tech/install/ipfs-desktop/) and/or [IPFS Companion](https://docs.ipfs.tech/install/ipfs-companion/). ### Minimal System Requirements Kubo runs on most Linux, macOS, and Windows systems. For optimal performance, we recommend at least 6 GB of RAM and 2 CPU cores (more is ideal, as Kubo is highly parallel). > [!IMPORTANT] > Larger pinsets require additional memory, with an estimated ~1 GiB of RAM per 20 million items for reproviding to the Amino DHT. > [!CAUTION] > Systems with less than the recommended memory may experience instability, frequent OOM errors or restarts, and missing data announcement (reprovider window), which can make data fully or partially inaccessible to other peers. Running Kubo on underprovisioned hardware is at your own risk. ### Official Prebuilt Binaries Download from https://dist.ipfs.tech#kubo or [GitHub Releases](https://github.com/ipfs/kubo/releases/latest). ### Docker Official images are published at https://hub.docker.com/r/ipfs/kubo/: [![Docker Image Version (latest semver)](https://img.shields.io/docker/v/ipfs/kubo?color=blue&label=kubo%20docker%20image&logo=docker&sort=semver&style=flat-square&cacheSeconds=3600)](https://hub.docker.com/r/ipfs/kubo/) #### 🟢 Release Images Use these for production deployments. - `latest` and [`release`](https://hub.docker.com/r/ipfs/kubo/tags?name=release) always point at [the latest stable release](https://github.com/ipfs/kubo/releases/latest) - [`vN.N.N`](https://hub.docker.com/r/ipfs/kubo/tags?name=v) points at a specific [release tag](https://github.com/ipfs/kubo/releases) ```console $ docker pull ipfs/kubo:latest $ docker run --rm -it --net=host ipfs/kubo:latest ``` To [customize your node](https://docs.ipfs.tech/install/run-ipfs-inside-docker/#customizing-your-node), pass config via `-e` or mount scripts in `/container-init.d`. #### 🟠 Developer Preview Images For internal testing, not intended for production. - [`master-latest`](https://hub.docker.com/r/ipfs/kubo/tags?name=master-latest) points at `HEAD` of [`master`](https://github.com/ipfs/kubo/commits/master/) - [`master-YYYY-DD-MM-GITSHA`](https://hub.docker.com/r/ipfs/kubo/tags?name=master-2) points at a specific commit #### 🔴 Internal Staging Images For testing arbitrary commits and experimental patches (force push to `staging` branch). - [`staging-latest`](https://hub.docker.com/r/ipfs/kubo/tags?name=staging-latest) points at `HEAD` of [`staging`](https://github.com/ipfs/kubo/commits/staging/) - [`staging-YYYY-DD-MM-GITSHA`](https://hub.docker.com/r/ipfs/kubo/tags?name=staging-2) points at a specific commit ### Build from Source ![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/ipfs/kubo?label=Requires%20Go&logo=go&style=flat-square&cacheSeconds=3600) ```bash git clone https://github.com/ipfs/kubo.git cd kubo make build # creates cmd/ipfs/ipfs make install # installs to $GOPATH/bin/ipfs ``` See the [Developer Guide](docs/developer-guide.md) for details, Windows instructions, and troubleshooting. ### Package Managers Kubo is available in community-maintained packages across many operating systems, Linux distributions, and package managers. See [Repology](https://repology.org/project/kubo/versions) for the full list: [![Packaging status](https://repology.org/badge/tiny-repos/kubo.svg)](https://repology.org/project/kubo/versions) > [!WARNING] > These packages are maintained by third-party volunteers. The IPFS Project and Kubo maintainers are not responsible for their contents or supply chain security. For increased security, [build from source](#build-from-source). #### Linux | Distribution | Install | Version | |--------------|---------|---------| | Ubuntu | [PPA](https://launchpad.net/~twdragon/+archive/ubuntu/ipfs): `sudo apt install ipfs-kubo` | [![PPA: twdragon](https://img.shields.io/badge/PPA-twdragon-E95420?logo=ubuntu)](https://launchpad.net/~twdragon/+archive/ubuntu/ipfs) | | Arch | `pacman -S kubo` | [![Arch package](https://repology.org/badge/version-for-repo/arch/kubo.svg)](https://archlinux.org/packages/extra/x86_64/kubo/) | | Fedora | [COPR](https://copr.fedorainfracloud.org/coprs/taw/ipfs/): `dnf install kubo` | [![COPR: taw](https://img.shields.io/badge/COPR-taw-51A2DA?logo=fedora)](https://copr.fedorainfracloud.org/coprs/taw/ipfs/) | | Nix | `nix-env -i kubo` | [![nixpkgs unstable](https://repology.org/badge/version-for-repo/nix_unstable/kubo.svg)](https://search.nixos.org/packages?query=kubo) | | Gentoo | `emerge -a net-p2p/kubo` | [![Gentoo package](https://repology.org/badge/version-for-repo/gentoo/kubo.svg)](https://packages.gentoo.org/packages/net-p2p/kubo) | | openSUSE | `zypper install kubo` | [![openSUSE Tumbleweed](https://repology.org/badge/version-for-repo/opensuse_tumbleweed/kubo.svg)](https://software.opensuse.org/package/kubo) | | Solus | `sudo eopkg install kubo` | [![Solus package](https://repology.org/badge/version-for-repo/solus/kubo.svg)](https://packages.getsol.us/shannon/k/kubo/) | | Guix | `guix install kubo` | [![Guix package](https://repology.org/badge/version-for-repo/gnuguix/kubo.svg)](https://packages.guix.gnu.org/packages/kubo/) | | _other_ | [See Repology for the full list](https://repology.org/project/kubo/versions) | | ~~Snap~~ no longer supported ([#8688](https://github.com/ipfs/kubo/issues/8688)) #### macOS | Manager | Install | Version | |---------|---------|---------| | Homebrew | `brew install ipfs` | [![Homebrew](https://repology.org/badge/version-for-repo/homebrew/kubo.svg)](https://formulae.brew.sh/formula/ipfs) | | MacPorts | `sudo port install ipfs` | [![MacPorts](https://repology.org/badge/version-for-repo/macports/kubo.svg)](https://ports.macports.org/port/ipfs/) | | Nix | `nix-env -i kubo` | [![nixpkgs unstable](https://repology.org/badge/version-for-repo/nix_unstable/kubo.svg)](https://search.nixos.org/packages?query=kubo) | | _other_ | [See Repology for the full list](https://repology.org/project/kubo/versions) | | #### Windows | Manager | Install | Version | |---------|---------|---------| | Scoop | `scoop install kubo` | [![Scoop](https://repology.org/badge/version-for-repo/scoop/kubo.svg)](https://scoop.sh/#/apps?q=kubo) | | _other_ | [See Repology for the full list](https://repology.org/project/kubo/versions) | | ~~Chocolatey~~ no longer supported ([#9341](https://github.com/ipfs/kubo/issues/9341)) ## Documentation | Topic | Description | |-------|-------------| | [Configuration](docs/config.md) | All config options reference | | [Environment variables](docs/environment-variables.md) | Runtime settings via env vars | | [Experimental features](docs/experimental-features.md) | Opt-in features in development | | [HTTP Gateway](docs/gateway.md) | Path, subdomain, and trustless gateway setup | | [HTTP RPC clients](docs/http-rpc-clients.md) | Client libraries for Go, JS | | [Delegated routing](docs/delegated-routing.md) | Multi-router and HTTP routing | | [Metrics & monitoring](docs/metrics.md) | Prometheus metrics | | [Content blocking](docs/content-blocking.md) | Denylist for public nodes | | [Customizing](docs/customizing.md) | Unsure if use Plugins, Boxo, or fork? | | [Debug guide](docs/debug-guide.md) | CPU profiles, memory analysis, tracing | | [Changelogs](docs/changelogs/) | Release notes for each version | | [All documentation](https://github.com/ipfs/kubo/tree/master/docs) | Full list of docs | ## Development See the [Developer Guide](docs/developer-guide.md) for build instructions, testing, and contribution workflow. AI coding agents should follow [AGENTS.md](AGENTS.md). ## Getting Help - [IPFS Forum](https://discuss.ipfs.tech) - community support, questions, and discussion - [Community](https://docs.ipfs.tech/community/) - chat, events, and working groups - [GitHub Issues](https://github.com/ipfs/kubo/issues) - bug reports for Kubo specifically - [IPFS Docs Issues](https://github.com/ipfs/ipfs-docs/issues) - documentation issues ## Security Issues See [`SECURITY.md`](SECURITY.md). ## Contributing [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) We welcome contributions. See [CONTRIBUTING.md](CONTRIBUTING.md) and the [Developer Guide](docs/developer-guide.md). This repository follows the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). ## Maintainer Info > [!NOTE] > Kubo is maintained by the [Shipyard](https://ipshipyard.com/) team. > > [Release Process](https://ipshipyard.notion.site/Kubo-Release-Process-6dba4f5755c9458ab5685eeb28173778) ## License Dual-licensed under Apache 2.0 and MIT: - [LICENSE-APACHE](LICENSE-APACHE) - [LICENSE-MIT](LICENSE-MIT) ================================================ FILE: Rules.mk ================================================ TGT_BIN := CLEAN := COVERAGE := DISTCLEAN := TEST := TEST_SHORT := GOCC ?= go PROTOC ?= protoc all: help # all has to be first defined target .PHONY: all include mk/git.mk # has to be before tarball.mk include mk/tarball.mk include mk/util.mk include mk/golang.mk # -------------------- # # extra properties # # -------------------- # ifeq ($(TEST_FUSE),0) GOTAGS += nofuse endif export LIBP2P_TCP_REUSEPORT=false # -------------------- # # sub-files # # -------------------- # dir := bin include $(dir)/Rules.mk # tests need access to rules from plugin dir := plugin include $(dir)/Rules.mk dir := test include $(dir)/Rules.mk dir := cmd/ipfs include $(dir)/Rules.mk # include this file only if coverage target is executed # it is quite expensive ifneq ($(filter coverage% clean distclean test/unit/gotest.junit.xml,$(MAKECMDGOALS)),) # has to be after cmd/ipfs due to PATH dir := coverage include $(dir)/Rules.mk endif # -------------------- # # universal rules # # -------------------- # %.pb.go: %.proto bin/protoc-gen-gogofaster $(PROTOC) --gogofaster_out=. --proto_path=.:$(GOPATH)/src:$(dir $@) $< # -------------------- # # core targets # # -------------------- # build: $(TGT_BIN) .PHONY: build clean: rm -rf $(CLEAN) .PHONY: clean mod_tidy: @find . -name go.mod -execdir $(GOCC) mod tidy \; .PHONY: mod_tidy coverage: $(COVERAGE) .PHONY: coverage distclean: clean rm -rf $(DISTCLEAN) git clean -ffxd .PHONY: distclean test: $(TEST) .PHONY: test test_short: $(TEST_SHORT) .PHONY: test_short deps: .PHONY: deps nofuse: GOTAGS += nofuse nofuse: build .PHONY: nofuse install: cmd/ipfs-install .PHONY: install install_unsupported: install @echo "/=======================================================================\\" @echo '| |' @echo '| `make install_unsupported` is deprecated, use `make install` instead. |' @echo '| |' @echo "\\=======================================================================/" .PHONY: install_unsupported uninstall: $(GOCC) clean -i ./cmd/ipfs .PHONY: uninstall supported: @echo "Currently supported platforms (from .github/build-platforms.yml):" @grep '^ - ' .github/build-platforms.yml | sed 's/^ - //' || (echo "Error: .github/build-platforms.yml not found"; exit 1) .PHONY: supported help: @echo 'DEPENDENCY TARGETS:' @echo '' @echo ' deps - Download dependencies using bundled gx' @echo ' test_sharness_deps - Download and build dependencies for sharness' @echo '' @echo 'BUILD TARGETS:' @echo '' @echo ' all - print this help message' @echo ' build - Build binary at ./cmd/ipfs/ipfs' @echo ' nofuse - Build binary with no fuse support' @echo ' install - Build binary and install into $$GOBIN' @echo ' mod_tidy - Remove unused dependencies from go.mod files' # @echo ' dist_install - TODO: c.f. ./cmd/ipfs/dist/README.md' @echo '' @echo 'CLEANING TARGETS:' @echo '' @echo ' clean - Remove files generated by build' @echo ' distclean - Remove files that are no part of a repository' @echo ' uninstall - Remove binary from $$GOPATH/bin' @echo '' @echo 'TESTING TARGETS:' @echo '' @echo ' test - Run all tests (test_go_fmt, test_unit, test_cli, test_sharness)' @echo ' test_short - Run fast tests (test_go_fmt, test_unit)' @echo ' test_unit - Run unit tests with coverage (excludes test/cli)' @echo ' test_cli - Run CLI integration tests (requires built binary)' @echo ' test_go_fmt - Check Go source formatting' @echo ' test_go_build - Build kubo for all platforms from .github/build-platforms.yml' @echo ' test_go_lint - Run golangci-lint' @echo ' test_sharness - Run sharness tests' @echo ' coverage - Collect coverage info from unit tests and sharness' @echo .PHONY: help ================================================ FILE: SECURITY.md ================================================ # Security Policy The IPFS protocol and its implementations are still in heavy development. This means that there may be problems in our protocols, or there may be mistakes in our implementations. We take security vulnerabilities very seriously. If you discover a security issue, please bring it to our attention right away! ## Reporting a Vulnerability If you find a vulnerability that may affect live deployments -- for example, by exposing a remote execution exploit -- please **send your report privately** to security@ipfs.io. Please **DO NOT file a public issue**. If the issue is a protocol weakness that cannot be immediately exploited or something not yet deployed, just discuss it openly. ## Reporting a non security bug For non-security bugs, please simply file a GitHub [issue](https://github.com/ipfs/go-ipfs/issues/new/choose). ================================================ FILE: assets/README.md ================================================ # Assets loaded in with IPFS This directory contains the go-ipfs assets: * Getting started documentation (`init-doc`). ================================================ FILE: assets/assets.go ================================================ package assets import ( "embed" "fmt" gopath "path" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/coreapi" "github.com/ipfs/boxo/files" cid "github.com/ipfs/go-cid" ) //go:embed init-doc var Asset embed.FS // initDocPaths lists the paths for the docs we want to seed during --init. var initDocPaths = []string{ gopath.Join("init-doc", "about"), gopath.Join("init-doc", "readme"), gopath.Join("init-doc", "help"), gopath.Join("init-doc", "contact"), gopath.Join("init-doc", "security-notes"), gopath.Join("init-doc", "quick-start"), gopath.Join("init-doc", "ping"), } // SeedInitDocs adds the list of embedded init documentation to the passed node, pins it and returns the root key. func SeedInitDocs(nd *core.IpfsNode) (cid.Cid, error) { return addAssetList(nd, initDocPaths) } func addAssetList(nd *core.IpfsNode, l []string) (cid.Cid, error) { api, err := coreapi.NewCoreAPI(nd) if err != nil { return cid.Cid{}, err } dirMap := map[string]files.Node{} for _, p := range l { d, err := Asset.ReadFile(p) if err != nil { return cid.Cid{}, fmt.Errorf("assets: could load Asset '%s': %s", p, err) } dirMap[gopath.Base(p)] = files.NewBytesFile(d) } basePath, err := api.Unixfs().Add(nd.Context(), files.NewMapDirectory(dirMap)) if err != nil { return cid.Cid{}, err } if err := api.Pin().Add(nd.Context(), basePath); err != nil { return cid.Cid{}, err } return basePath.RootCid(), nil } ================================================ FILE: assets/init-doc/about ================================================ IPFS -- Inter-Planetary File system IPFS is a global, versioned, peer-to-peer filesystem. It combines good ideas from Git, BitTorrent, Kademlia, SFS, and the Web. It is like a single bit- torrent swarm, exchanging git objects. IPFS provides an interface as simple as the HTTP web, but with permanence built-in. You can also mount the world at /ipfs. IPFS is a protocol: - defines a content-addressed file system - coordinates content delivery - combines Kademlia + BitTorrent + Git IPFS is a filesystem: - has directories and files - mountable filesystem (via FUSE) IPFS is a web: - can be used to view documents like the web - files accessible via HTTP at `http://ipfs.io/` - browsers or extensions can learn to use `ipfs://` directly - hash-addressed content guarantees the authenticity IPFS is modular: - connection layer over any network protocol - routing layer - uses a routing layer DHT (kademlia/coral) - uses a path-based naming service - uses BitTorrent-inspired block exchange IPFS uses crypto: - cryptographic-hash content addressing - block-level deduplication - file integrity + versioning - filesystem-level encryption + signing support IPFS is p2p: - worldwide peer-to-peer file transfers - completely decentralized architecture - **no** central point of failure IPFS is a CDN: - add a file to the filesystem locally, and it's now available to the world - caching-friendly (content-hash naming) - BitTorrent-based bandwidth distribution IPFS has a name service: - IPNS, an SFS inspired name system - global namespace based on PKI - serves to build trust chains - compatible with other NSes - can map DNS, .onion, .bit, etc to IPNS ================================================ FILE: assets/init-doc/contact ================================================ Come hang out in our IRC chat room if you have any questions. Contact the ipfs dev team: - Bugs: https://github.com/ipfs/go-ipfs/issues - Help: irc.freenode.org/#ipfs - Email: dev@ipfs.io ================================================ FILE: assets/init-doc/docs/index ================================================ Index ================================================ FILE: assets/init-doc/help ================================================ Some helpful resources for finding your way around ipfs: - quick-start: a quick show of various ipfs features. - ipfs commands: a list of all commands - ipfs --help: every command describes itself - https://github.com/ipfs/go-ipfs -- the src repository - #ipfs on irc.freenode.org -- the community IRC channel ================================================ FILE: assets/init-doc/ping ================================================ ipfs ================================================ FILE: assets/init-doc/quick-start ================================================ # 0.1 - Quick Start This is a set of short examples with minimal explanation. It is meant as a "quick start". Add a file to ipfs: echo "hello world" >hello ipfs add hello View it: ipfs cat Try a directory: mkdir foo mkdir foo/bar echo "baz" > foo/baz echo "baz" > foo/bar/baz ipfs add -r foo View things: ipfs ls ipfs ls /bar ipfs cat /baz ipfs cat /bar/baz ipfs cat /bar ipfs ls /baz References: ipfs refs ipfs refs -r ipfs refs --help Get: ipfs get -o foo2 diff foo foo2 Objects: ipfs object get ipfs object get /foo2 ipfs object --help Pin + GC: ipfs pin add ipfs repo gc ipfs ls ipfs pin rm ipfs repo gc Daemon: ipfs daemon (in another terminal) ipfs id Network: (must be online) ipfs swarm peers ipfs id ipfs cat Mount: (warning: fuse is finicky!) ipfs mount cd /ipfs/ ls Tool: ipfs version ipfs update ipfs commands ipfs config --help open http://localhost:5001/webui Browse: WebUI: http://localhost:5001/webui video: http://localhost:8080/ipfs/QmVc6zuAneKJzicnJpfrqCH9gSy6bz54JhcypfJYhGUFQu/play#/ipfs/QmTKZgRNwDNZwHtJSjCp6r5FYefzpULfy37JvMt9DwvXse images: http://localhost:8080/ipfs/QmZpc3HvfjEXvLWGQPWbHk3AjD5j8NEN4gmFN8Jmrd5g83/cs markdown renderer app: http://localhost:8080/ipfs/QmX7M9CiYXjVeFnkfVGf3y5ixTZ2ACeSGyL1vBJY1HvQPp/mdown ================================================ FILE: assets/init-doc/readme ================================================ Hello and Welcome to IPFS! ██╗██████╗ ███████╗███████╗ ██║██╔══██╗██╔════╝██╔════╝ ██║██████╔╝█████╗ ███████╗ ██║██╔═══╝ ██╔══╝ ╚════██║ ██║██║ ██║ ███████║ ╚═╝╚═╝ ╚═╝ ╚══════╝ If you're seeing this, you have successfully installed IPFS and are now interfacing with the ipfs merkledag! ------------------------------------------------------- | Warning: | | This is alpha software. Use at your own discretion! | | Much is missing or lacking polish. There are bugs. | | Not yet secure. Read the security notes for more. | ------------------------------------------------------- Check out some of the other files in this directory: ./about ./help ./quick-start <-- usage examples ./readme <-- this file ./security-notes ================================================ FILE: assets/init-doc/security-notes ================================================ IPFS Alpha Security Notes We try hard to ensure our system is safe and robust, but all software has bugs, especially new software. This distribution is meant to be an alpha preview, don't use it for anything mission critical. Please note the following: - This is alpha software and has not been audited. It is our goal to conduct a proper security audit once we close in on a 1.0 release. - ipfs is a networked program, and may have serious undiscovered vulnerabilities. It is written in Go, and we do not execute any user provided data. But please point any problems out to us in a github issue, or email security@ipfs.io privately. - security@ipfs.io GPG key: - 4B9665FB 92636D17 7C7A86D3 50AAE8A9 59B13AF3 - https://pgp.mit.edu/pks/lookup?op=get&search=0x50AAE8A959B13AF3 - ipfs uses encryption for all communication, but it's NOT PROVEN SECURE YET! It may be totally broken. For now, the code is included to make sure we benchmark our operations with encryption in mind. In the future, there will be an "unsafe" mode for high performance intranet apps. If this is a blocking feature for you, please contact us. ================================================ FILE: bin/Rules.mk ================================================ include mk/header.mk dist_root_$(d)="/ipfs/QmPrXH9jRVwvd7r5MC5e6nV4uauQGzLk1i2647Ye9Vbbwe" TGTS_$(d) := $(d)/protoc DISTCLEAN += $(d)/protoc $(d)/tmp PATH := $(realpath $(d)):$(PATH) $(TGTS_$(d)): rm -f $@$(?exe) ifeq ($(WINDOWS),1) cp $^$(?exe) $@$(?exe) else ln -s $(notdir $^) $@ endif bin/protoc-gen-gogofaster: $(call go-build,github.com/gogo/protobuf/protoc-gen-gogofaster) CLEAN += $(TGTS_$(d)) include mk/footer.mk ================================================ FILE: bin/archive-branches.sh ================================================ #!/usr/bin/env bash set -euo pipefail IFS=$'\n\t' auth="" #auth="-u kubuxu:$GH_TOKEN" org=ipfs repo=go-ipfs arch_repo=go-ipfs-archived api_repo="repos/$org/$repo" exclusions=( 'master' 'release' 'feat/zcash' 'feat/ai-mirror' ) gh_api_next() { links=$(grep '^Link:' | sed -e 's/Link: //' -e 's/, /\n/g') echo "$links" | grep '; rel="next"' >/dev/null || return link=$(echo "$links" | grep '; rel="next"' | sed -e 's/^.*//') curl $auth -f -sD >(gh_api_next) "$link" } gh_api() { curl $auth -f -sD >(gh_api_next) "https://api.github.com/$1" | jq -s '[.[] | .[]]' } pr_branches() { gh_api "$api_repo/pulls" | jq -r '.[].head.label | select(test("^ipfs:"))' \ | sed 's/^ipfs://' } origin_refs() { format=${1-'%(refname:short)'} git for-each-ref --format "$format" refs/remotes/origin | sed 's|^origin/||' } active_branches() { origin_refs '%(refname:short) %(committerdate:unix)' |awk \ ' BEGIN { monthAgo = systime() - 31*24*60*60 } { if ($2 > monthAgo) print $1 } ' } git remote add archived "git@github.com:$org/$arch_repo.git" || true branches_to_move="$(cat <(active_branches) <(pr_branches) <((IFS=$'\n'; echo "${exclusions[*]}")) | sort -u | comm - <(origin_refs | sort) -13)" echo "================" printf "%s\n" "$branches_to_move" echo "================" echo "Please confirm move of above branches [y/N]:" read line case $line in [Yy]|[Yy][Ee][Ss]) ;; *) exit 1 ;; esac printf "%s\n" "$branches_to_move" | \ while read -r ref; do git push archived "origin/$ref:refs/heads/$ref/$(date --rfc-3339=date)" git push origin --delete "$ref" done ================================================ FILE: bin/container_daemon ================================================ #!/bin/sh set -e user=ipfs repo="$IPFS_PATH" if [ "$(id -u)" -eq 0 ]; then echo "Changing user to $user" # ensure folder is writable gosu "$user" test -w "$repo" || chown -R -- "$user" "$repo" # restart script with new privileges exec gosu "$user" "$0" "$@" fi # 2nd invocation with regular user ipfs version if [ -e "$repo/config" ]; then echo "Found IPFS fs-repo at $repo" else ipfs init ${IPFS_PROFILE:+"--profile=$IPFS_PROFILE"} ipfs config Addresses.API /ip4/0.0.0.0/tcp/5001 ipfs config Addresses.Gateway /ip4/0.0.0.0/tcp/8080 # Set up the swarm key, if provided SWARM_KEY_FILE="$repo/swarm.key" SWARM_KEY_PERM=0400 # Create a swarm key from a given environment variable if [ -n "$IPFS_SWARM_KEY" ] ; then echo "Copying swarm key from variable..." printf "%s\n" "$IPFS_SWARM_KEY" >"$SWARM_KEY_FILE" || exit 1 chmod $SWARM_KEY_PERM "$SWARM_KEY_FILE" fi # Unset the swarm key variable unset IPFS_SWARM_KEY # Check during initialization if a swarm key was provided and # copy it to the ipfs directory with the right permissions # WARNING: This will replace the swarm key if it exists if [ -n "$IPFS_SWARM_KEY_FILE" ] ; then echo "Copying swarm key from file..." install -m $SWARM_KEY_PERM "$IPFS_SWARM_KEY_FILE" "$SWARM_KEY_FILE" || exit 1 fi # Unset the swarm key file variable unset IPFS_SWARM_KEY_FILE fi find /container-init.d -maxdepth 1 \( -type f -o -type l \) -iname '*.sh' -print0 | sort -z | xargs -n 1 -0 -r container_init_run exec ipfs "$@" ================================================ FILE: bin/container_init_run ================================================ #!/bin/sh set -e # used by the container startup script for running initialization scripts script="$1" if [ -x "$script" ] ; then printf "Executing '%s'...\n" "$script" "$script" else printf "Sourcing '%s'...\n" "$script" . "$script" fi ================================================ FILE: bin/dist_get ================================================ #!/bin/sh GOCC=${GOCC=go} die() { echo "$@" >&2 exit 1 } have_binary() { type "$1" > /dev/null 2> /dev/null } check_writable() { printf "" > "$1" && rm "$1" } try_download() { url="$1" output="$2" command="$3" util_name="$(set -- $command; echo "$1")" if ! have_binary "$util_name"; then return 1 fi printf '==> Using %s to download "%s" to "%s"\n' "$util_name" "$url" "$output" if eval "$command"; then echo "==> Download complete!" return else echo "error: couldn't download with $util_name ($?)" return 1 fi } download() { dl_url="$1" dl_output="$2" test "$#" -eq "2" || die "download requires exactly two arguments, was given $@" if ! check_writable "$dl_output"; then die "download error: cannot write to $dl_output" fi try_download "$dl_url" "$dl_output" "wget '$dl_url' -O '$dl_output'" && return try_download "$dl_url" "$dl_output" "curl --silent --fail --output '$dl_output' '$dl_url'" && return try_download "$dl_url" "$dl_output" "fetch '$dl_url' -o '$dl_output'" && return try_download "$dl_url" "$dl_output" "http '$dl_url' > '$dl_output'" && return try_download "$dl_url" "$dl_output" "ftp -o '$dl_output' '$dl_url'" && return die "Unable to download $dl_url. exiting." } unarchive() { ua_archivetype="$1" ua_infile="$2" ua_outfile="$3" ua_distname="$4" ua_binpostfix="" ua_os=$(uname -o) if [ "$ua_os" = "Msys" ] || [ "$ua_os" = "Cygwin" ] ; then ua_binpostfix=".exe" fi ua_outfile="$ua_outfile$ua_binpostfix" if ! check_writable "$ua_outfile"; then die "unarchive error: cannot write to $ua_outfile" fi case "$ua_archivetype" in tar.gz) if have_binary tar; then echo "==> using 'tar' to extract binary from archive" < "$ua_infile" tar -Ozxf - "$ua_distname/$ua_distname$ua_binpostfix" > "$ua_outfile" \ || die "tar has failed" else die "no binary on system for extracting tar files" fi ;; zip) if have_binary unzip; then echo "==> using 'unzip' to extract binary from archive" unzip -p "$ua_infile" "$ua_distname/$ua_distname$ua_binpostfix" > "$ua_outfile" \ || die "unzip has failed" else die "no installed method for extracting .zip archives" fi ;; *) die "unrecognized archive type '$ua_archivetype'" esac chmod +x "$ua_outfile" || die "chmod has failed" } get_go_vars() { if [ ! -z "$GOOS" ] && [ ! -z "$GOARCH" ]; then printf "%s-%s" "$GOOS" "$GOARCH" elif have_binary go; then printf "%s-%s" "$($GOCC env GOOS)" "$($GOCC env GOARCH)" else die "no way of determining system GOOS and GOARCH\nPlease manually set GOOS and GOARCH then retry." fi } mkurl() { m_root="$1" m_name="$2" m_vers="$3" m_archive="$4" m_govars=$(get_go_vars) || die "could not get go env vars" echo "https://ipfs.io$m_root/$m_name/$m_vers/${m_name}_${m_vers}_$m_govars.$m_archive" } distroot="$1" distname="$2" outpath="$3" version="$4" if [ -z "$distroot" ] || [ -z "$distname" ] || [ -z "$outpath" ] || [ -z "$version" ]; then die "usage: dist_get " fi case $version in v*) # correct input ;; *) echo "invalid version '$version'" >&2 die "versions must begin with 'v', for example: v0.4.0" ;; esac # TODO: don't depend on the go tool being installed to detect this goenv=$(get_go_vars) || die "could not get go env vars" case $goenv in linux-*) archive="tar.gz" ;; darwin-*) archive="tar.gz" ;; windows-*) archive="zip" ;; freebsd-*) archive="tar.gz" ;; openbsd-*) archive="tar.gz" ;; *) echo "unrecognized system environment: $goenv" >&2 die "currently only linux, darwin, windows and freebsd are supported by this script" esac mkdir -p bin/tmp url=$(mkurl "$distroot" "$distname" "$version" "$archive") tmpfi="bin/tmp/$distname.$archive" download "$url" "$tmpfi" if [ $? -ne 0 ]; then die "failed to download $url to $tmpfi" fi unarchive "$archive" "$tmpfi" "$outpath" "$distname" if [ $? -ne 0 ]; then die "failed to extract archive $tmpfi" fi ================================================ FILE: bin/gencmdref ================================================ #!/usr/bin/env python import os import sys import datetime from subprocess import check_output def run(cmd): return check_output(cmd) def main(): lines = [l.strip() for l in sys.stdin] print '# ipfs command reference' print '' print 'generated on', datetime.datetime.now() print '' for line in lines: print '- [%s](#%s)' % (line, line.replace(' ', '-')) print '' for line in lines: print '## %s' % line print '' print '```' print run((line + ' --help').split(' ')).strip() print '```' print '' if __name__ == '__main__': if '-h' in sys.argv or '--help' in sys.argv: print 'usage: ipfs commands | %s >cmdref.md' % sys.argv[0] print 'outputs all commands with --help to a markdown file' exit(0) main() ================================================ FILE: bin/get-docker-tags.sh ================================================ #!/usr/bin/env bash # get-docker-tags.sh # # Usage: # ./get-docker-tags.sh [git tag name] # # Example: # # # get tag for the master branch # ./get-docker-tags.sh $(date -u +%F) testingsha master # # # get tag for a release tag # ./get-docker-tags.sh $(date -u +%F) testingsha release v0.5.0 # set -euo pipefail if [[ $# -lt 1 ]] ; then echo 'At least 1 arg required.' echo 'Usage:' echo './get-docker-tags.sh [git commit sha1] [git branch name] [git tag name]' exit 1 fi BUILD_NUM=$1 GIT_SHA1=${2:-$(git rev-parse HEAD)} GIT_SHA1_SHORT=$(echo "$GIT_SHA1" | cut -c 1-7) GIT_BRANCH=${3:-$(git symbolic-ref -q --short HEAD || echo "unknown")} GIT_TAG=${4:-$(git describe --tags --exact-match 2> /dev/null || echo "")} IMAGE_NAME=${IMAGE_NAME:-ipfs/kubo} echoImageName () { local IMAGE_TAG=$1 echo "$IMAGE_NAME:$IMAGE_TAG" } if [[ $GIT_TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+-rc ]]; then echoImageName "$GIT_TAG" elif [[ $GIT_TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echoImageName "$GIT_TAG" echoImageName "latest" echoImageName "release" # see: https://github.com/ipfs/kubo/issues/3999#issuecomment-742228981 elif [[ $GIT_BRANCH =~ ^bifrost-.* ]]; then # sanitize the branch name since docker tags have stricter char limits than git branch names branch=$(echo "$GIT_BRANCH" | tr '/' '-' | tr --delete --complement '[:alnum:]-') echoImageName "${branch}-${BUILD_NUM}-${GIT_SHA1_SHORT}" elif [ "$GIT_BRANCH" = "master" ] || [ "$GIT_BRANCH" = "staging" ]; then echoImageName "${GIT_BRANCH}-${BUILD_NUM}-${GIT_SHA1_SHORT}" echoImageName "${GIT_BRANCH}-latest" else echo "Nothing to do. No docker tag defined for branch: $GIT_BRANCH, tag: $GIT_TAG" fi ================================================ FILE: bin/graphmd ================================================ #!/bin/sh if [ "$#" -ne 1 ]; then echo "usage: $0 ..." echo "output merkledag links in graphviz dot" echo "" echo "use it with dot:" echo " $0 QmZPAMWUfLD95GsdorXt9hH7aVrarb2SuLDMVVe6gABYmx | dot -Tsvg" echo " $0 QmZPAMWUfLD95GsdorXt9hH7aVrarb2SuLDMVVe6gABYmx | dot -Tpng" echo " $0 QmZPAMWUfLD95GsdorXt9hH7aVrarb2SuLDMVVe6gABYmx | dot -Tpdf" echo "" exit 1 fi src=' [fontsize=8 shape=box];' dst=' [fontsize=8 shape=box];' edge=' -> [label=""];' fmt="$src $dst $edge" echo "digraph {" echo " graph [rankdir=LR];" ipfs refs -r --format="$fmt" "$@" | awk '{ print "\t" $0 }' # ipfs refs -r --format="$fmt" "$@" | awk '{ print "\t" $0 }' | unflatten -l3 echo "}" ================================================ FILE: bin/ipns-republish ================================================ #!/bin/bash if [ "$#" -ne 1 ]; then echo "usage: $0 " echo "republishes an ipns name every 20 minutes" echo "(this is an icky stop-gap until ipfs nodes do it for you)" echo "" echo "example:" echo " > $0 QmSYCpuKPbPQ2iFr2swJj2hvz7wQUXfPBXPiuVsQdL5FEs" echo "" exit 1 fi # must be run online. ipfs swarm peers >/dev/null if [ $? -ne 0 ]; then echo "error: ipfs daemon must be online and connected to peers " exit 1 fi # check the object is there ipfs dag stat "$1" >/dev/null if [ $? -ne 0 ]; then echo "error: ipfs cannot find $1" exit 1 fi echo "republishing $1 every 20 minutes" while : do ipfs name publish $1 sleep 1200 done ================================================ FILE: bin/maketarball.sh ================================================ #!/usr/bin/env bash # vim: set expandtab sw=2 ts=2: # bash safe mode set -euo pipefail IFS=$'\n\t' # readlink doesn't work on macos OUTPUT="${1:-go-ipfs-source.tar.gz}" if ! [[ "$OUTPUT" = /* ]]; then OUTPUT="$PWD/$OUTPUT" fi GOCC=${GOCC=go} TEMP="$(mktemp -d)" cp -r . "$TEMP" ( cd "$TEMP" && echo $PWD && $GOCC mod vendor && (git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || true) > .tarball && chmod -R u=rwX,go=rX "$TEMP" # normalize permissions tar -czf "$OUTPUT" --exclude="./.git" . ) rm -rf "$TEMP" ================================================ FILE: bin/mkreleaselog ================================================ #!/bin/bash # # Invocation: mkreleaselog [FIRST_REF [LAST_REF]] # # Generates release notes with contributor statistics, deduplicating by GitHub handle. # GitHub handles are resolved from: # 1. GitHub noreply emails (user@users.noreply.github.com) # 2. Merge commit messages (Merge pull request #N from user/branch) # 3. GitHub API via gh CLI (for squash merges) # # Results are cached in ~/.cache/mkreleaselog/github-handles.json set -euo pipefail export GO111MODULE=on GOPATH="$(go env GOPATH)" export GOPATH # List of PCRE regular expressions to match "included" modules. INCLUDE_MODULES=( # orgs "^github.com/ipfs/" "^github.com/ipld/" "^github.com/libp2p/" "^github.com/multiformats/" "^github.com/filecoin-project/" "^github.com/ipfs-shipyard/" "^github.com/ipshipyard/" "^github.com/probe-lab/" # Authors of personal modules used by go-ipfs that should be mentioned in the # release notes. "^github.com/whyrusleeping/" "^github.com/gammazero/" "^github.com/Jorropo/" "^github.com/guillaumemichel/" "^github.com/Kubuxu/" "^github.com/jbenet/" "^github.com/Stebalien/" "^github.com/marten-seemann/" "^github.com/hsanjuan/" "^github.com/lucas-clemente/" "^github.com/warpfork/" ) # List of PCRE regular expressions to match "excluded" modules. Applied after includes. EXCLUDE_MODULES=( "^github.com/marten-seemann/qtls" ) # Ignored files as git pathspecs. These patters will match any full path component. IGNORE_FILES=( ".gx" "package.json" ".travis.yml" "go.mod" "go.sum" ".github" "*.pb.go" "cbor_gen.go" "ipldsch_*.go" "*.gen.go" ) ########################################################################################## # GitHub Handle Resolution Infrastructure ########################################################################################## # Cache location following XDG spec GITHUB_CACHE_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/mkreleaselog" GITHUB_CACHE_FILE="$GITHUB_CACHE_DIR/github-handles.json" # Timeout for gh CLI commands (seconds) GH_TIMEOUT=10 # Associative array for email -> github handle mapping (runtime cache) declare -A EMAIL_TO_GITHUB # Check if gh CLI is available and authenticated gh_available() { command -v gh >/dev/null 2>&1 && gh auth status >/dev/null 2>&1 } # Load cached email -> github handle mappings from disk load_github_cache() { EMAIL_TO_GITHUB=() if [[ ! -f "$GITHUB_CACHE_FILE" ]]; then return 0 fi # Validate JSON before loading if ! jq -e '.' "$GITHUB_CACHE_FILE" >/dev/null 2>&1; then msg "Warning: corrupted cache file, ignoring" return 0 fi local email handle while IFS=$'\t' read -r email handle; do # Validate handle format (alphanumeric, hyphens, max 39 chars) if [[ -n "$email" && -n "$handle" && "$handle" =~ ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,37}[a-zA-Z0-9])?$ ]]; then EMAIL_TO_GITHUB["$email"]="$handle" fi done < <(jq -r 'to_entries[] | "\(.key)\t\(.value)"' "$GITHUB_CACHE_FILE" 2>/dev/null) msg "Loaded ${#EMAIL_TO_GITHUB[@]} cached GitHub handle mappings" } # Save email -> github handle mappings to disk (atomic write) save_github_cache() { if [[ ${#EMAIL_TO_GITHUB[@]} -eq 0 ]]; then return 0 fi mkdir -p "$GITHUB_CACHE_DIR" local tmp_file tmp_file="$(mktemp "$GITHUB_CACHE_DIR/cache.XXXXXX")" || return 1 # Build JSON from associative array { echo "{" local first=true local key for key in "${!EMAIL_TO_GITHUB[@]}"; do if [[ "$first" == "true" ]]; then first=false else echo "," fi # Escape special characters in email for JSON printf ' %s: %s' "$(jq -n --arg e "$key" '$e')" "$(jq -n --arg h "${EMAIL_TO_GITHUB[$key]}" '$h')" done echo echo "}" } > "$tmp_file" # Validate before replacing if jq -e '.' "$tmp_file" >/dev/null 2>&1; then mv "$tmp_file" "$GITHUB_CACHE_FILE" msg "Saved ${#EMAIL_TO_GITHUB[@]} GitHub handle mappings to cache" else rm -f "$tmp_file" msg "Warning: failed to save cache (invalid JSON)" fi } # Extract GitHub handle from email if it's a GitHub noreply address # Handles: user@users.noreply.github.com and 12345678+user@users.noreply.github.com extract_handle_from_noreply() { local email="$1" if [[ "$email" =~ ^([0-9]+\+)?([a-zA-Z0-9]([a-zA-Z0-9-]{0,37}[a-zA-Z0-9])?)@users\.noreply\.github\.com$ ]]; then echo "${BASH_REMATCH[2]}" return 0 fi return 1 } # Extract GitHub handle from merge commit subject # Handles: "Merge pull request #123 from username/branch" extract_handle_from_merge_commit() { local subject="$1" if [[ "$subject" =~ ^Merge\ pull\ request\ \#[0-9]+\ from\ ([a-zA-Z0-9]([a-zA-Z0-9-]{0,37}[a-zA-Z0-9])?)/.*$ ]]; then echo "${BASH_REMATCH[1]}" return 0 fi return 1 } # Extract PR number from commit subject # Handles: "Subject (#123)" and "Merge pull request #123 from" extract_pr_number() { local subject="$1" if [[ "$subject" =~ \(#([0-9]+)\)$ ]]; then echo "${BASH_REMATCH[1]}" return 0 elif [[ "$subject" =~ ^Merge\ pull\ request\ \#([0-9]+)\ from ]]; then echo "${BASH_REMATCH[1]}" return 0 fi return 1 } # Query GitHub API for PR author (with timeout and error handling) query_pr_author() { local gh_repo="$1" # e.g., "ipfs/kubo" local pr_num="$2" if ! gh_available; then return 1 fi local handle handle="$(timeout "$GH_TIMEOUT" gh pr view "$pr_num" --repo "$gh_repo" --json author -q '.author.login' 2>/dev/null)" || return 1 # Validate handle format if [[ -n "$handle" && "$handle" =~ ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,37}[a-zA-Z0-9])?$ ]]; then echo "$handle" return 0 fi return 1 } # Query GitHub API for commit author (fallback when no PR available) query_commit_author() { local gh_repo="$1" # e.g., "ipfs/kubo" local commit_sha="$2" if ! gh_available; then return 1 fi local handle handle="$(timeout "$GH_TIMEOUT" gh api "/repos/$gh_repo/commits/$commit_sha" --jq '.author.login // empty' 2>/dev/null)" || return 1 # Validate handle format if [[ -n "$handle" && "$handle" =~ ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,37}[a-zA-Z0-9])?$ ]]; then echo "$handle" return 0 fi return 1 } # Resolve email to GitHub handle using all available methods # Args: email, commit_hash (optional), repo_dir (optional), gh_repo (optional) resolve_github_handle() { local email="$1" local commit="${2:-}" local repo_dir="${3:-}" local gh_repo="${4:-}" # Skip empty emails [[ -z "$email" ]] && return 1 # Check runtime cache first if [[ -n "${EMAIL_TO_GITHUB[$email]:-}" ]]; then echo "${EMAIL_TO_GITHUB[$email]}" return 0 fi local handle="" # Method 1: Extract from noreply email if handle="$(extract_handle_from_noreply "$email")"; then EMAIL_TO_GITHUB["$email"]="$handle" echo "$handle" return 0 fi # Method 2: Look at commit message for merge commit pattern if [[ -n "$commit" && -n "$repo_dir" ]]; then local subject subject="$(git -C "$repo_dir" log -1 --format='%s' "$commit" 2>/dev/null)" || true if [[ -n "$subject" ]]; then if handle="$(extract_handle_from_merge_commit "$subject")"; then EMAIL_TO_GITHUB["$email"]="$handle" echo "$handle" return 0 fi # Method 3: Query GitHub API for PR author if [[ -n "$gh_repo" ]]; then local pr_num if pr_num="$(extract_pr_number "$subject")"; then if handle="$(query_pr_author "$gh_repo" "$pr_num")"; then EMAIL_TO_GITHUB["$email"]="$handle" echo "$handle" return 0 fi fi fi fi fi return 1 } # Build GitHub handle mappings for all commits in a range # This does a single pass to collect PR numbers, then batch queries them build_github_mappings() { local module="$1" local start="$2" local end="${3:-HEAD}" local repo repo="$(strip_version "$module")" local dir local gh_repo="" if [[ "$module" == "github.com/ipfs/kubo" ]]; then dir="$ROOT_DIR" else dir="$GOPATH/src/$repo" fi # Extract gh_repo for API calls (e.g., "ipfs/kubo" from "github.com/ipfs/kubo") if [[ "$repo" =~ ^github\.com/(.+)$ ]]; then gh_repo="${BASH_REMATCH[1]}" fi msg "Building GitHub handle mappings for $module..." # Collect all unique emails and their commit context declare -A email_commits=() local hash email subject while IFS=$'\t' read -r hash email subject; do [[ -z "$email" ]] && continue # Skip if already resolved [[ -n "${EMAIL_TO_GITHUB[$email]:-}" ]] && continue # Try to resolve without API first local handle="" # Method 1: noreply email if handle="$(extract_handle_from_noreply "$email")"; then EMAIL_TO_GITHUB["$email"]="$handle" continue fi # Method 2: merge commit message if handle="$(extract_handle_from_merge_commit "$subject")"; then EMAIL_TO_GITHUB["$email"]="$handle" continue fi # Store for potential API lookup if [[ -z "${email_commits[$email]:-}" ]]; then email_commits["$email"]="$hash" fi done < <(git -C "$dir" log --format='tformat:%H%x09%aE%x09%s' --no-merges "$start..$end" 2>/dev/null) # API batch lookup for remaining emails (if gh is available) if gh_available && [[ -n "$gh_repo" && ${#email_commits[@]} -gt 0 ]]; then msg "Querying GitHub API for ${#email_commits[@]} unknown contributors..." local key for key in "${!email_commits[@]}"; do # Skip if already resolved [[ -n "${EMAIL_TO_GITHUB[$key]:-}" ]] && continue local commit_hash="${email_commits[$key]}" local subj handle subj="$(git -C "$dir" log -1 --format='%s' "$commit_hash" 2>/dev/null)" || true # Try PR author lookup first (cheaper API call) local pr_num if pr_num="$(extract_pr_number "$subj")"; then if handle="$(query_pr_author "$gh_repo" "$pr_num")"; then EMAIL_TO_GITHUB["$key"]="$handle" continue fi fi # Fallback: commit author API (works for any commit) if handle="$(query_commit_author "$gh_repo" "$commit_hash")"; then EMAIL_TO_GITHUB["$key"]="$handle" fi done fi } ########################################################################################## # Original infrastructure with modifications ########################################################################################## build_include_regex() { local result="" local mod for mod in "${INCLUDE_MODULES[@]}"; do if [[ -n "$result" ]]; then result="$result|$mod" else result="$mod" fi done echo "($result)" } build_exclude_regex() { local result="" local mod for mod in "${EXCLUDE_MODULES[@]}"; do if [[ -n "$result" ]]; then result="$result|$mod" else result="$mod" fi done if [[ -n "$result" ]]; then echo "($result)" else echo '$^' # match nothing fi } if [[ ${#INCLUDE_MODULES[@]} -gt 0 ]]; then INCLUDE_REGEX="$(build_include_regex)" else INCLUDE_REGEX="" # "match anything" fi if [[ ${#EXCLUDE_MODULES[@]} -gt 0 ]]; then EXCLUDE_REGEX="$(build_exclude_regex)" else EXCLUDE_REGEX='$^' # "match nothing" fi IGNORE_FILES_PATHSPEC=() for f in "${IGNORE_FILES[@]}"; do IGNORE_FILES_PATHSPEC+=(":^:**/$f" ":^:$f") # Prepend the magic "ignore this" sequence. done NL=$'\n' ROOT_DIR="$(git rev-parse --show-toplevel)" msg() { echo "$*" >&2 } statlog() { local module="$1" local rpath local gh_repo="" if [[ "$module" == "github.com/ipfs/kubo" ]]; then rpath="$ROOT_DIR" else rpath="$GOPATH/src/$(strip_version "$module")" fi # Extract gh_repo for API calls local repo repo="$(strip_version "$module")" if [[ "$repo" =~ ^github\.com/(.+)$ ]]; then gh_repo="${BASH_REMATCH[1]}" fi local start="${2:-}" local end="${3:-HEAD}" local mailmap_file="$rpath/.mailmap" if ! [[ -e "$mailmap_file" ]]; then mailmap_file="$ROOT_DIR/.mailmap" fi local stack=() local line while read -r line; do if [[ -n "$line" ]]; then stack+=("$line") continue fi local changes read -r changes local changed=0 local insertions=0 local deletions=0 local count event while read -r count event; do if [[ "$event" =~ ^file ]]; then changed=$count elif [[ "$event" =~ ^insertion ]]; then insertions=$count elif [[ "$event" =~ ^deletion ]]; then deletions=$count else echo "unknown event $event" >&2 exit 1 fi done<<<"${changes//,/$NL}" local author for author in "${stack[@]}"; do local hash name email IFS=$'\t' read -r hash name email <<<"$author" # Resolve GitHub handle local github_handle="" github_handle="$(resolve_github_handle "$email" "$hash" "$rpath" "$gh_repo")" || true jq -n \ --arg "hash" "$hash" \ --arg "name" "$name" \ --arg "email" "$email" \ --arg "github" "$github_handle" \ --argjson "changed" "$changed" \ --argjson "insertions" "$insertions" \ --argjson "deletions" "$deletions" \ '{Commit: $hash, Author: $name, Email: $email, GitHub: $github, Files: $changed, Insertions: $insertions, Deletions: $deletions}' done stack=() done < <(git -C "$rpath" -c mailmap.file="$mailmap_file" log --use-mailmap --shortstat --no-merges --pretty="tformat:%H%x09%aN%x09%aE" "$start..$end" -- . "${IGNORE_FILES_PATHSPEC[@]}") } # Returns a stream of deps changed between $1 and $2. dep_changes() { cat "$1" "$2" | jq -s 'JOIN(INDEX(.[0][]; .Path); .[1][]; .Path; {Path: .[0].Path, Old: (.[1] | del(.Path)), New: (.[0] | del(.Path))}) | select(.New.Version != .Old.Version)' } # resolve_commits resolves a git ref for each version. resolve_commits() { jq '. + {Ref: (.Version|capture("^((?.*)\\+incompatible|v.*-(0\\.)?[0-9]{14}-(?[a-f0-9]{12})|(?v.*))$") | .ref1 // .ref2 // .ref3)}' } pr_link() { local repo="$1" local prnum="$2" local ghname="${repo##github.com/}" printf -- "[%s#%s](https://%s/pull/%s)" "$ghname" "$prnum" "$repo" "$prnum" } ignored_commit() { local repo="$1" local commit="$2" local matches # Check to see if this commit includes any non-ignored files. matches=$(git -C "$repo" diff-tree --no-commit-id --name-only -r "$commit^" "$commit" \ -- "${IGNORE_FILES_PATHSPEC[@]}" | wc -l) [[ "$matches" -eq 0 ]] } # Generate a release log for a range of commits in a single repo. release_log() { local module="$1" local start="$2" local end="${3:-HEAD}" local repo repo="$(strip_version "$1")" local dir if [[ "$module" == "github.com/ipfs/kubo" ]]; then dir="$ROOT_DIR" else dir="$GOPATH/src/$repo" fi local commit subject while read -r commit subject; do # Skip commits that only touch ignored files. if ignored_commit "$dir" "$commit"; then continue fi if [[ "$subject" =~ ^Merge\ pull\ request\ \#([0-9]+)\ from ]]; then local prnum="${BASH_REMATCH[1]}" local desc desc="$(git -C "$dir" show --summary --format='tformat:%b' "$commit" | head -1)" printf -- "- %s (%s)\n" "$desc" "$(pr_link "$repo" "$prnum")" elif [[ "$subject" =~ \(#([0-9]+)\)$ ]]; then local prnum="${BASH_REMATCH[1]}" printf -- "- %s (%s)\n" "$subject" "$(pr_link "$repo" "$prnum")" else printf -- "- %s\n" "$subject" fi done < <(git -C "$dir" log --format='tformat:%H %s' --first-parent "$start..$end") } indent() { sed -e 's/^/ /' } mod_deps() { go list -mod=mod -json -m all | jq 'select(.Version != null)' } ensure() { local repo repo="$(strip_version "$1")" local commit="$2" local rpath if [[ "$1" == "github.com/ipfs/kubo" ]]; then rpath="$ROOT_DIR" else rpath="$GOPATH/src/$repo" fi if [[ "$1" != "github.com/ipfs/kubo" ]] && [[ ! -d "$rpath" ]]; then msg "Cloning $repo..." git clone "http://$repo" "$rpath" >&2 fi if ! git -C "$rpath" rev-parse --verify "$commit" >/dev/null; then msg "Fetching $repo..." git -C "$rpath" fetch --all >&2 fi git -C "$rpath" rev-parse --verify "$commit" >/dev/null || return 1 } # Summarize stats, grouping by GitHub handle (with fallback to email for dedup) statsummary() { jq -s ' # Group by GitHub handle if available, otherwise by email group_by(if .GitHub != "" then .GitHub else .Email end)[] | { # Use first non-empty GitHub handle, or fall back to Author name Author: .[0].Author, GitHub: (map(select(.GitHub != "")) | .[0].GitHub // ""), Email: .[0].Email, Commits: (. | length), Insertions: (map(.Insertions) | add), Deletions: (map(.Deletions) | add), Files: (map(.Files) | add) } ' | jq '. + {Lines: (.Deletions + .Insertions)}' } strip_version() { local repo="$1" if [[ "$repo" =~ .*/v[0-9]+$ ]]; then repo="$(dirname "$repo")" fi echo "$repo" } recursive_release_log() { local start="${1:-$(git tag -l | sort -V | grep -v -- '-rc' | grep 'v'| tail -n1)}" local end="${2:-$(git rev-parse HEAD)}" local repo_root repo_root="$(git rev-parse --show-toplevel)" local module module="$(go list -m)" local dir dir="$(go list -m -f '{{.Dir}}')" # Load cached GitHub handle mappings load_github_cache # Kubo can be run from any directory, dependencies still use GOPATH ( local result=0 local workspace workspace="$(mktemp -d)" # shellcheck disable=SC2064 trap "rm -rf '$workspace'" INT TERM EXIT cd "$workspace" echo "Computing old deps..." >&2 git -C "$repo_root" show "$start:go.mod" >go.mod mod_deps | resolve_commits | jq -s > old_deps.json echo "Computing new deps..." >&2 git -C "$repo_root" show "$end:go.mod" >go.mod mod_deps | resolve_commits | jq -s > new_deps.json rm -f go.mod go.sum printf -- "Generating Changelog for %s %s..%s\n" "$module" "$start" "$end" >&2 # Pre-build GitHub mappings for main module build_github_mappings "$module" "$start" "$end" echo "### 📝 Changelog" echo echo "
Full Changelog" echo printf -- "- %s:\n" "$module" release_log "$module" "$start" "$end" | indent statlog "$module" "$start" "$end" > statlog.json local dep_module new new_ref old old_ref while read -r dep_module new new_ref old old_ref; do if ! ensure "$dep_module" "$new_ref"; then result=1 local changelog="failed to fetch repo" else # Pre-build GitHub mappings for dependency build_github_mappings "$dep_module" "$old_ref" "$new_ref" statlog "$dep_module" "$old_ref" "$new_ref" >> statlog.json local changelog changelog="$(release_log "$dep_module" "$old_ref" "$new_ref")" fi if [[ -n "$changelog" ]]; then printf -- "- %s (%s -> %s):\n" "$dep_module" "$old" "$new" echo "$changelog" | indent fi done < <(dep_changes old_deps.json new_deps.json | jq --arg inc "$INCLUDE_REGEX" --arg exc "$EXCLUDE_REGEX" \ 'select(.Path | test($inc)) | select(.Path | test($exc) | not)' | jq -r '"\(.Path) \(.New.Version) \(.New.Ref) \(.Old.Version) \(.Old.Ref // "")"') echo echo "
" echo echo "### 👨‍👩‍👧‍👦 Contributors" echo echo "| Contributor | Commits | Lines ± | Files Changed |" echo "|-------------|---------|---------|---------------|" statsummary [git tag name] [dry run] # # Example: # # dry run. pass a 5th arg to have it print what it would do rather than do it. # ./push-docker-tags.sh $(date -u +%F) testingsha master "" dryrun # # # push tag for the master branch # ./push-docker-tags.sh $(date -u +%F) testingsha master # # # push tag for a release tag # ./push-docker-tags.sh $(date -u +%F) testingsha release v0.5.0 # set -euo pipefail if [[ $# -lt 1 ]] ; then echo 'At least 1 arg required. Pass 5 args for a dry run.' echo 'Usage:' echo './push-docker-tags.sh [git commit sha1] [git branch name] [git tag name] [dry run]' exit 1 fi BUILD_NUM=$1 GIT_SHA1=${2:-$(git rev-parse HEAD)} GIT_SHA1_SHORT=$(echo "$GIT_SHA1" | cut -c 1-7) GIT_BRANCH=${3:-$(git symbolic-ref -q --short HEAD || echo "unknown")} GIT_TAG=${4:-$(git describe --tags --exact-match || echo "")} DRY_RUN=${5:-false} WIP_IMAGE_TAG=${WIP_IMAGE_TAG:-wip} IMAGE_NAME=${IMAGE_NAME:-ipfs/kubo} pushTag () { local IMAGE_TAG=$1 if [ "$DRY_RUN" != false ]; then echo "DRY RUN! I would have tagged and pushed the following..." echo docker tag "$IMAGE_NAME:$WIP_IMAGE_TAG" "$IMAGE_NAME:$IMAGE_TAG" echo docker push "$IMAGE_NAME:$IMAGE_TAG" else echo "Tagging $IMAGE_NAME:$IMAGE_TAG and pushing to dockerhub" docker tag "$IMAGE_NAME:$WIP_IMAGE_TAG" "$IMAGE_NAME:$IMAGE_TAG" docker push "$IMAGE_NAME:$IMAGE_TAG" fi } if [[ $GIT_TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+-rc ]]; then pushTag "$GIT_TAG" elif [[ $GIT_TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then pushTag "$GIT_TAG" pushTag "latest" pushTag "release" # see: https://github.com/ipfs/kubo/issues/3999#issuecomment-742228981 elif [[ $GIT_BRANCH =~ ^bifrost-.* ]]; then # sanitize the branch name since docker tags have stricter char limits than git branch names branch=$(echo "$GIT_BRANCH" | tr '/' '-' | tr --delete --complement '[:alnum:]-') pushTag "${branch}-${BUILD_NUM}-${GIT_SHA1_SHORT}" elif [ "$GIT_BRANCH" = "master" ] || [ "$GIT_BRANCH" = "staging" ]; then pushTag "${GIT_BRANCH}-${BUILD_NUM}-${GIT_SHA1_SHORT}" pushTag "${GIT_BRANCH}-latest" else echo "Nothing to do. No docker tag defined for branch: $GIT_BRANCH, tag: $GIT_TAG" fi ================================================ FILE: bin/test-go-build-platforms ================================================ #!/bin/bash set -e echo "Building kubo for all platforms in .github/build-platforms.yml..." if [ ! -f .github/build-platforms.yml ]; then echo "Error: .github/build-platforms.yml not found" exit 1 fi grep '^ - ' .github/build-platforms.yml | sed 's/^ - //' | while read -r platform; do if [ -z "$platform" ]; then continue fi GOOS=$(echo "$platform" | cut -d- -f1) GOARCH=$(echo "$platform" | cut -d- -f2) echo "Building $platform..." echo " GOOS=$GOOS GOARCH=$GOARCH go build -o /dev/null ./cmd/ipfs" GOOS=$GOOS GOARCH=$GOARCH go build -o /dev/null ./cmd/ipfs done echo "All platforms built successfully" ================================================ FILE: bin/test-go-fmt ================================================ #!/usr/bin/env bash set -euo pipefail T="$(mktemp)" find . \ -path ./test/sharness -prune \ -o -path ./plugin/loader/preload.go -prune \ -o -name '*.go' -print0 | xargs -0 gofmt -s -l > "$T" if [ -n "$(cat $T)" ]; then echo "Following Go code is not formatted." echo "-----------------------------------" cat "$T" echo "-----------------------------------" echo "Run 'go fmt ./...' in your source directory" rm -f "$T" exit 1 fi rm -f "$T" ================================================ FILE: blocks/blockstoreutil/remove.go ================================================ // Package blockstoreutil provides utility functions for Blockstores. package blockstoreutil import ( "context" "errors" "fmt" bs "github.com/ipfs/boxo/blockstore" pin "github.com/ipfs/boxo/pinning/pinner" cid "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" ) // RemovedBlock is used to represent the result of removing a block. // If a block was removed successfully, then the Error will be empty. // If a block could not be removed, then Error will contain the // reason the block could not be removed. If the removal was aborted // due to a fatal error, Hash will be empty, Error will contain the // reason, and no more results will be sent. type RemovedBlock struct { Hash string Error error } // RmBlocksOpts is used to wrap options for RmBlocks(). type RmBlocksOpts struct { Prefix string Quiet bool Force bool } // RmBlocks removes the blocks provided in the cids slice. // It returns a channel where objects of type RemovedBlock are placed, when // not using the Quiet option. Block removal is asynchronous and will // skip any pinned blocks. func RmBlocks(ctx context.Context, blocks bs.GCBlockstore, pins pin.Pinner, cids []cid.Cid, opts RmBlocksOpts) (<-chan any, error) { // make the channel large enough to hold any result to avoid // blocking while holding the GCLock out := make(chan any, len(cids)) go func() { defer close(out) unlocker := blocks.GCLock(ctx) defer unlocker.Unlock(ctx) stillOkay := FilterPinned(ctx, pins, out, cids) for _, c := range stillOkay { // Kept for backwards compatibility. We may want to // remove this sometime in the future. has, err := blocks.Has(ctx, c) if err != nil { out <- &RemovedBlock{Hash: c.String(), Error: err} continue } if !has && !opts.Force { out <- &RemovedBlock{Hash: c.String(), Error: format.ErrNotFound{Cid: c}} continue } err = blocks.DeleteBlock(ctx, c) if err != nil { out <- &RemovedBlock{Hash: c.String(), Error: err} } else if !opts.Quiet { out <- &RemovedBlock{Hash: c.String()} } } }() return out, nil } // FilterPinned takes a slice of Cids and returns it with the pinned Cids // removed. If a Cid is pinned, it will place RemovedBlock objects in the given // out channel, with an error which indicates that the Cid is pinned. // This function is used in RmBlocks to filter out any blocks which are not // to be removed (because they are pinned). func FilterPinned(ctx context.Context, pins pin.Pinner, out chan<- any, cids []cid.Cid) []cid.Cid { stillOkay := make([]cid.Cid, 0, len(cids)) res, err := pins.CheckIfPinned(ctx, cids...) if err != nil { out <- &RemovedBlock{Error: fmt.Errorf("pin check failed: %w", err)} return nil } for _, r := range res { if !r.Pinned() { stillOkay = append(stillOkay, r.Key) } else { out <- &RemovedBlock{ Hash: r.Key.String(), Error: errors.New(r.String()), } } } return stillOkay } ================================================ FILE: client/rpc/README.md ================================================ # `coreiface.CoreAPI` over http `rpc` > IPFS CoreAPI implementation using HTTP API This package implements [`coreiface.CoreAPI`](https://pkg.go.dev/github.com/ipfs/kubo/core/coreiface#CoreAPI) over the HTTP API. ## Documentation https://pkg.go.dev/github.com/ipfs/kubo/client/rpc ### Example Pin file on your local IPFS node based on its CID: ```go package main import ( "context" "fmt" "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" "github.com/ipfs/kubo/client/rpc" ) func main() { // "Connect" to local node node, err := rpc.NewLocalApi() if err != nil { fmt.Println(err) return } // Pin a given file by its CID ctx := context.Background() c, err := cid.Decode("bafkreidtuosuw37f5xmn65b3ksdiikajy7pwjjslzj2lxxz2vc4wdy3zku") if err != nil { fmt.Println(err) return } p := path.FromCid(c) err = node.Pin().Add(ctx, p) if err != nil { fmt.Println(err) return } } ``` ================================================ FILE: client/rpc/api.go ================================================ package rpc import ( "context" "encoding/json" "errors" "fmt" "net" "net/http" "os" "path/filepath" "strings" "sync" "time" "github.com/blang/semver/v4" "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/go-cid" legacy "github.com/ipfs/go-ipld-legacy" ipfs "github.com/ipfs/kubo" iface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" "github.com/ipfs/kubo/misc/fsutil" dagpb "github.com/ipld/go-codec-dagpb" _ "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/node/basicnode" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) const ( DefaultPathName = ".ipfs" DefaultPathRoot = "~/" + DefaultPathName DefaultApiFile = "api" EnvDir = "IPFS_PATH" ) // ErrApiNotFound if we fail to find a running daemon. var ErrApiNotFound = errors.New("ipfs api address could not be found") // HttpApi implements github.com/ipfs/interface-go-ipfs-core/CoreAPI using // IPFS HTTP API. // // For interface docs see // https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI type HttpApi struct { url string httpcli http.Client Headers http.Header applyGlobal func(*requestBuilder) ipldDecoder *legacy.Decoder versionMu sync.Mutex version *semver.Version } // NewLocalApi tries to construct new HttpApi instance communicating with local // IPFS daemon // // Daemon api address is pulled from the $IPFS_PATH/api file. // If $IPFS_PATH env var is not present, it defaults to ~/.ipfs. func NewLocalApi() (*HttpApi, error) { baseDir := os.Getenv(EnvDir) if baseDir == "" { baseDir = DefaultPathRoot } return NewPathApi(baseDir) } // NewPathApi constructs new HttpApi by pulling api address from specified // ipfspath. Api file should be located at $ipfspath/api. func NewPathApi(ipfspath string) (*HttpApi, error) { a, err := ApiAddr(ipfspath) if err != nil { if os.IsNotExist(err) { err = ErrApiNotFound } return nil, err } return NewApi(a) } // ApiAddr reads api file in specified ipfs path. func ApiAddr(ipfspath string) (ma.Multiaddr, error) { baseDir, err := fsutil.ExpandHome(ipfspath) if err != nil { return nil, err } apiFile := filepath.Join(baseDir, DefaultApiFile) api, err := os.ReadFile(apiFile) if err != nil { return nil, err } return ma.NewMultiaddr(strings.TrimSpace(string(api))) } // NewApi constructs HttpApi with specified endpoint. func NewApi(a ma.Multiaddr) (*HttpApi, error) { transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, } network, address, err := manet.DialArgs(a) if err != nil { return nil, err } if network == "unix" { transport.DialContext = func(_ context.Context, _, _ string) (net.Conn, error) { return net.Dial("unix", address) } c := &http.Client{ Transport: transport, } // This will create an API client which // makes requests to `http://unix`. return NewURLApiWithClient(network, c) } c := &http.Client{ Transport: transport, } return NewApiWithClient(a, c) } // NewApiWithClient constructs HttpApi with specified endpoint and custom http client. func NewApiWithClient(a ma.Multiaddr, c *http.Client) (*HttpApi, error) { _, url, err := manet.DialArgs(a) if err != nil { return nil, err } if a, err := ma.NewMultiaddr(url); err == nil { _, host, err := manet.DialArgs(a) if err == nil { url = host } } proto := "http://" // By default, DialArgs is going to provide details suitable for connecting // a socket to, but not really suitable for making an informed choice of http // protocol. For multiaddresses specifying tls and/or https we want to make // a https request instead of a http request. protocols := a.Protocols() for _, p := range protocols { if p.Code == ma.P_HTTPS || p.Code == ma.P_TLS { proto = "https://" break } } return NewURLApiWithClient(proto+url, c) } func NewURLApiWithClient(url string, c *http.Client) (*HttpApi, error) { decoder := legacy.NewDecoder() // Add support for these codecs to match what is done in the merkledag library // Note: to match prior behavior the go-ipld-prime CBOR decoder is manually included // TODO: allow the codec registry used to be configured by the caller not through a global variable decoder.RegisterCodec(cid.DagProtobuf, dagpb.Type.PBNode, merkledag.ProtoNodeConverter) decoder.RegisterCodec(cid.Raw, basicnode.Prototype.Bytes, merkledag.RawNodeConverter) api := &HttpApi{ url: url, httpcli: *c, Headers: make(map[string][]string), applyGlobal: func(*requestBuilder) {}, ipldDecoder: decoder, } // We don't support redirects. api.httpcli.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { return fmt.Errorf("unexpected redirect") } return api, nil } func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { options, err := caopts.ApiOptions(opts...) if err != nil { return nil, err } subApi := &HttpApi{ url: api.url, httpcli: api.httpcli, Headers: api.Headers, applyGlobal: func(req *requestBuilder) { if options.Offline { req.Option("offline", options.Offline) } }, ipldDecoder: api.ipldDecoder, } return subApi, nil } func (api *HttpApi) Request(command string, args ...string) RequestBuilder { headers := make(map[string]string) if api.Headers != nil { for k := range api.Headers { headers[k] = api.Headers.Get(k) } } return &requestBuilder{ command: command, args: args, shell: api, headers: headers, } } func (api *HttpApi) Unixfs() iface.UnixfsAPI { return (*UnixfsAPI)(api) } func (api *HttpApi) Block() iface.BlockAPI { return (*BlockAPI)(api) } func (api *HttpApi) Dag() iface.APIDagService { return (*HttpDagServ)(api) } func (api *HttpApi) Name() iface.NameAPI { return (*NameAPI)(api) } func (api *HttpApi) Key() iface.KeyAPI { return (*KeyAPI)(api) } func (api *HttpApi) Pin() iface.PinAPI { return (*PinAPI)(api) } func (api *HttpApi) Object() iface.ObjectAPI { return (*ObjectAPI)(api) } func (api *HttpApi) Swarm() iface.SwarmAPI { return (*SwarmAPI)(api) } func (api *HttpApi) PubSub() iface.PubSubAPI { return (*PubsubAPI)(api) } func (api *HttpApi) Routing() iface.RoutingAPI { return (*RoutingAPI)(api) } func (api *HttpApi) loadRemoteVersion() (*semver.Version, error) { api.versionMu.Lock() defer api.versionMu.Unlock() if api.version == nil { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() resp, err := api.Request("version").Send(ctx) if err != nil { return nil, err } if resp.Error != nil { return nil, resp.Error } defer resp.Close() var out ipfs.VersionInfo dec := json.NewDecoder(resp.Output) if err := dec.Decode(&out); err != nil { return nil, err } remoteVersion, err := semver.New(out.Version) if err != nil { return nil, err } api.version = remoteVersion } return api.version, nil } ================================================ FILE: client/rpc/api_test.go ================================================ package rpc import ( "context" "errors" "net/http" "net/http/httptest" "strconv" "strings" "sync" "testing" "time" "github.com/ipfs/boxo/path" "github.com/ipfs/kubo/config" iface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/core/coreiface/tests" "github.com/ipfs/kubo/test/cli/harness" ma "github.com/multiformats/go-multiaddr" ) type NodeProvider struct{} func (np NodeProvider) MakeAPISwarm(t *testing.T, ctx context.Context, fullIdentity, online bool, n int) ([]iface.CoreAPI, error) { h := harness.NewT(t) apis := make([]iface.CoreAPI, n) nodes := h.NewNodes(n) var wg, zero sync.WaitGroup zeroNode := nodes[0] wg.Add(len(apis)) zero.Add(1) var errs []error var errsLk sync.Mutex for i, n := range nodes { go func(i int, n *harness.Node) { if err := func() error { defer wg.Done() var err error n.Init("--empty-repo") c := n.ReadConfig() c.Experimental.FilestoreEnabled = true // only provide things we pin. Allows to test // provide operations. c.Provide.Strategy = config.NewOptionalString("roots") n.WriteConfig(c) n.StartDaemon("--enable-pubsub-experiment", "--offline="+strconv.FormatBool(!online)) if online { if i > 0 { zero.Wait() n.Connect(zeroNode) } else { zero.Done() } } apiMaddr, err := n.TryAPIAddr() if err != nil { return err } api, err := NewApi(apiMaddr) if err != nil { return err } apis[i] = api // empty node is pinned even with --empty-repo, we don't want that emptyNode, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") if err != nil { return err } if err := api.Pin().Rm(ctx, emptyNode); err != nil { return err } return nil }(); err != nil { errsLk.Lock() errs = append(errs, err) errsLk.Unlock() } }(i, n) } wg.Wait() return apis, errors.Join(errs...) } func TestHttpApi(t *testing.T) { t.Parallel() tests.TestApi(NodeProvider{})(t) } func Test_NewURLApiWithClient_With_Headers(t *testing.T) { t.Parallel() var ( headerToTest = "Test-Header" expectedHeaderValue = "thisisaheadertest" ) ts := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { val := r.Header.Get(headerToTest) if val != expectedHeaderValue { w.WriteHeader(400) return } http.ServeContent(w, r, "", time.Now(), strings.NewReader("test")) }), ) defer ts.Close() api, err := NewURLApiWithClient(ts.URL, &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, }, }) if err != nil { t.Fatal(err) } api.Headers.Set(headerToTest, expectedHeaderValue) p, err := path.NewPath("/ipfs/QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv") if err != nil { t.Fatal(err) } if err := api.Pin().Rm(context.Background(), p); err != nil { t.Fatal(err) } } func Test_NewURLApiWithClient_HTTP_Variant(t *testing.T) { t.Parallel() testcases := []struct { address string expected string }{ {address: "/ip4/127.0.0.1/tcp/80", expected: "http://127.0.0.1:80"}, {address: "/ip4/127.0.0.1/tcp/443/tls", expected: "https://127.0.0.1:443"}, {address: "/ip4/127.0.0.1/tcp/443/https", expected: "https://127.0.0.1:443"}, {address: "/ip4/127.0.0.1/tcp/443/tls/http", expected: "https://127.0.0.1:443"}, } for _, tc := range testcases { address, err := ma.NewMultiaddr(tc.address) if err != nil { t.Fatal(err) } api, err := NewApiWithClient(address, &http.Client{}) if err != nil { t.Fatal(err) } if api.url != tc.expected { t.Errorf("Expected = %s; got %s", tc.expected, api.url) } } } ================================================ FILE: client/rpc/apifile.go ================================================ package rpc import ( "bytes" "context" "encoding/json" "fmt" "io" "os" "strconv" "time" "github.com/ipfs/boxo/files" unixfs "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" ) const forwardSeekLimit = 1 << 14 // 16k func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error) { if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS var err error p, _, err = api.core().ResolvePath(ctx, p) if err != nil { return nil, err } } var stat struct { Hash string Type string Size int64 // unixfs size Mode string Mtime int64 MtimeNsecs int } err := api.core().Request("files/stat", p.String()).Exec(ctx, &stat) if err != nil { return nil, err } mode, err := stringToFileMode(stat.Mode) if err != nil { return nil, err } var modTime time.Time if stat.Mtime != 0 { modTime = time.Unix(stat.Mtime, int64(stat.MtimeNsecs)).UTC() } switch stat.Type { case "file": return api.getFile(ctx, p, stat.Size, mode, modTime) case "directory": return api.getDir(ctx, p, stat.Size, mode, modTime) case "symlink": return api.getSymlink(ctx, p, modTime) default: return nil, fmt.Errorf("unsupported file type '%s'", stat.Type) } } type apiFile struct { ctx context.Context core *HttpApi size int64 path path.Path mode os.FileMode mtime time.Time r *Response at int64 } func (f *apiFile) reset() error { if f.r != nil { _ = f.r.Cancel() f.r = nil } req := f.core.Request("cat", f.path.String()) if f.at != 0 { req.Option("offset", f.at) } resp, err := req.Send(f.ctx) if err != nil { return err } if resp.Error != nil { return resp.Error } f.r = resp return nil } func (f *apiFile) Read(p []byte) (int, error) { n, err := f.r.Output.Read(p) if n > 0 { f.at += int64(n) } return n, err } func (f *apiFile) ReadAt(p []byte, off int64) (int, error) { // Always make a new request. This method should be parallel-safe. resp, err := f.core.Request("cat", f.path.String()). Option("offset", off).Option("length", len(p)).Send(f.ctx) if err != nil { return 0, err } if resp.Error != nil { return 0, resp.Error } defer resp.Output.Close() n, err := io.ReadFull(resp.Output, p) if err == io.ErrUnexpectedEOF { err = io.EOF } return n, err } func (f *apiFile) Seek(offset int64, whence int) (int64, error) { switch whence { case io.SeekEnd: offset = f.size + offset case io.SeekCurrent: offset = f.at + offset } if f.at == offset { // noop return offset, nil } if f.at < offset && offset-f.at < forwardSeekLimit { // forward skip r, err := io.CopyN(io.Discard, f.r.Output, offset-f.at) f.at += r return f.at, err } f.at = offset return f.at, f.reset() } func (f *apiFile) Close() error { if f.r != nil { return f.r.Cancel() } return nil } func (f *apiFile) Mode() os.FileMode { return f.mode } func (f *apiFile) ModTime() time.Time { return f.mtime } func (f *apiFile) Size() (int64, error) { return f.size, nil } func stringToFileMode(mode string) (os.FileMode, error) { if mode == "" { return 0, nil } mode64, err := strconv.ParseUint(mode, 8, 32) if err != nil { return 0, fmt.Errorf("cannot parse mode %s: %s", mode, err) } return os.FileMode(uint32(mode64)), nil } func (api *UnixfsAPI) getFile(ctx context.Context, p path.Path, size int64, mode os.FileMode, mtime time.Time) (files.Node, error) { f := &apiFile{ ctx: ctx, core: api.core(), size: size, path: p, mode: mode, mtime: mtime, } return f, f.reset() } type apiIter struct { ctx context.Context core *UnixfsAPI err error dec *json.Decoder curFile files.Node cur lsLink } func (it *apiIter) Err() error { return it.err } func (it *apiIter) Name() string { return it.cur.Name } func (it *apiIter) Next() bool { if it.ctx.Err() != nil { it.err = it.ctx.Err() return false } var out lsOutput if err := it.dec.Decode(&out); err != nil { if err != io.EOF { it.err = err } return false } if len(out.Objects) != 1 { it.err = fmt.Errorf("ls returned more objects than expected (%d)", len(out.Objects)) return false } if len(out.Objects[0].Links) != 1 { it.err = fmt.Errorf("ls returned more links than expected (%d)", len(out.Objects[0].Links)) return false } it.cur = out.Objects[0].Links[0] c, err := cid.Parse(it.cur.Hash) if err != nil { it.err = err return false } switch it.cur.Type { case unixfs.THAMTShard, unixfs.TMetadata, unixfs.TDirectory: it.curFile, err = it.core.getDir(it.ctx, path.FromCid(c), int64(it.cur.Size), it.cur.Mode, it.cur.ModTime) if err != nil { it.err = err return false } case unixfs.TFile: it.curFile, err = it.core.getFile(it.ctx, path.FromCid(c), int64(it.cur.Size), it.cur.Mode, it.cur.ModTime) if err != nil { it.err = err return false } case unixfs.TSymlink: it.curFile, err = it.core.getSymlink(it.ctx, path.FromCid(c), it.cur.ModTime) if err != nil { it.err = err return false } default: it.err = fmt.Errorf("file type %d not supported", it.cur.Type) return false } return true } func (it *apiIter) Node() files.Node { return it.curFile } type apiDir struct { ctx context.Context core *UnixfsAPI size int64 path path.Path mode os.FileMode mtime time.Time dec *json.Decoder } func (d *apiDir) Close() error { return nil } func (d *apiDir) Mode() os.FileMode { return d.mode } func (d *apiDir) ModTime() time.Time { return d.mtime } func (d *apiDir) Size() (int64, error) { return d.size, nil } func (d *apiDir) Entries() files.DirIterator { return &apiIter{ ctx: d.ctx, core: d.core, dec: d.dec, } } func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64, mode os.FileMode, modTime time.Time) (files.Node, error) { resp, err := api.core().Request("ls", p.String()). Option("resolve-size", true). Option("stream", true).Send(ctx) if err != nil { return nil, err } if resp.Error != nil { return nil, resp.Error } data, _ := io.ReadAll(resp.Output) rdr := bytes.NewReader(data) d := &apiDir{ ctx: ctx, core: api, size: size, path: p, mode: mode, mtime: modTime, //dec: json.NewDecoder(resp.Output), dec: json.NewDecoder(rdr), } return d, nil } func (api *UnixfsAPI) getSymlink(ctx context.Context, p path.Path, modTime time.Time) (files.Node, error) { resp, err := api.core().Request("cat", p.String()). Option("resolve-size", true). Option("stream", true).Send(ctx) if err != nil { return nil, err } if resp.Error != nil { return nil, resp.Error } target, err := io.ReadAll(resp.Output) if err != nil { return nil, err } return files.NewSymlinkFile(string(target), modTime), nil } var ( _ files.File = &apiFile{} _ files.Directory = &apiDir{} ) ================================================ FILE: client/rpc/auth/auth.go ================================================ package auth import "net/http" var _ http.RoundTripper = &AuthorizedRoundTripper{} type AuthorizedRoundTripper struct { authorization string roundTripper http.RoundTripper } // NewAuthorizedRoundTripper creates a new [http.RoundTripper] that will set the // Authorization HTTP header with the value of [authorization]. The given [roundTripper] is // the base [http.RoundTripper]. If it is nil, [http.DefaultTransport] is used. func NewAuthorizedRoundTripper(authorization string, roundTripper http.RoundTripper) http.RoundTripper { if roundTripper == nil { roundTripper = http.DefaultTransport } return &AuthorizedRoundTripper{ authorization: authorization, roundTripper: roundTripper, } } func (tp *AuthorizedRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { r.Header.Set("Authorization", tp.authorization) return tp.roundTripper.RoundTrip(r) } ================================================ FILE: client/rpc/block.go ================================================ package rpc import ( "bytes" "context" "fmt" "io" "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" iface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" mc "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" ) type BlockAPI HttpApi type blockStat struct { Key string BSize int `json:"Size"` cid cid.Cid } func (s *blockStat) Size() int { return s.BSize } func (s *blockStat) Path() path.ImmutablePath { return path.FromCid(s.cid) } func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { options, err := caopts.BlockPutOptions(opts...) px := options.CidPrefix if err != nil { return nil, err } mht, ok := mh.Codes[px.MhType] if !ok { return nil, fmt.Errorf("unknowm mhType %d", px.MhType) } var cidOptKey, cidOptVal string switch { case px.Version == 0 && px.Codec == cid.DagProtobuf: // ensure legacy --format=v0 passes as BlockPutOption still works cidOptKey = "format" cidOptVal = "v0" default: // pass codec as string cidOptKey = "cid-codec" cidOptVal = mc.Code(px.Codec).String() } req := api.core().Request("block/put"). Option("mhtype", mht). Option("mhlen", px.MhLength). Option(cidOptKey, cidOptVal). Option("pin", options.Pin). FileBody(r) var out blockStat if err := req.Exec(ctx, &out); err != nil { return nil, err } out.cid, err = cid.Parse(out.Key) if err != nil { return nil, err } return &out, nil } func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { resp, err := api.core().Request("block/get", p.String()).Send(ctx) if err != nil { return nil, err } if resp.Error != nil { return nil, parseErrNotFoundWithFallbackToError(resp.Error) } // TODO: make get return ReadCloser to avoid copying defer resp.Close() b := new(bytes.Buffer) if _, err := io.Copy(b, resp.Output); err != nil { return nil, err } return b, nil } func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRmOption) error { options, err := caopts.BlockRmOptions(opts...) if err != nil { return err } removedBlock := struct { Hash string `json:",omitempty"` Error string `json:",omitempty"` }{} req := api.core().Request("block/rm"). Option("force", options.Force). Arguments(p.String()) if err := req.Exec(ctx, &removedBlock); err != nil { return err } return parseErrNotFoundWithFallbackToMSG(removedBlock.Error) } func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { return nil, parseErrNotFoundWithFallbackToError(err) } out.cid, err = cid.Parse(out.Key) if err != nil { return nil, err } return &out, nil } func (api *BlockAPI) core() *HttpApi { return (*HttpApi)(api) } ================================================ FILE: client/rpc/dag.go ================================================ package rpc import ( "bytes" "context" "fmt" "io" "github.com/ipfs/boxo/path" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" "github.com/ipfs/kubo/core/coreiface/options" multicodec "github.com/multiformats/go-multicodec" ) type ( httpNodeAdder HttpApi HttpDagServ httpNodeAdder pinningHttpNodeAdder httpNodeAdder ) func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) { r, err := api.core().Block().Get(ctx, path.FromCid(c)) if err != nil { return nil, err } data, err := io.ReadAll(r) if err != nil { return nil, err } blk, err := blocks.NewBlockWithCid(data, c) if err != nil { return nil, err } return api.ipldDecoder.DecodeNode(ctx, blk) } func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption { out := make(chan *format.NodeOption) for _, c := range cids { // TODO: Consider limiting concurrency of this somehow go func(c cid.Cid) { n, err := api.Get(ctx, c) select { case out <- &format.NodeOption{Node: n, Err: err}: case <-ctx.Done(): } }(c) } return out } func (api *httpNodeAdder) add(ctx context.Context, nd format.Node, pin bool) error { c := nd.Cid() prefix := c.Prefix() // preserve 'cid-codec' when sent over HTTP cidCodec := multicodec.Code(prefix.Codec).String() // 'format' got replaced by 'cid-codec' in https://github.com/ipfs/interface-go-ipfs-core/pull/80 // but we still support it here for backward-compatibility with use of CIDv0 format := "" if prefix.Version == 0 { cidCodec = "" format = "v0" } stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()), options.Block.Hash(prefix.MhType, prefix.MhLength), options.Block.CidCodec(cidCodec), options.Block.Format(format), options.Block.Pin(pin)) if err != nil { return err } if !stat.Path().RootCid().Equals(c) { return fmt.Errorf("cids didn't match - local %s, remote %s", c.String(), stat.Path().RootCid().String()) } return nil } func (api *httpNodeAdder) addMany(ctx context.Context, nds []format.Node, pin bool) error { for _, nd := range nds { // TODO: optimize if err := api.add(ctx, nd, pin); err != nil { return err } } return nil } func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error { return (*httpNodeAdder)(api).addMany(ctx, nds, false) } func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { return (*httpNodeAdder)(api).add(ctx, nd, false) } func (api *pinningHttpNodeAdder) Add(ctx context.Context, nd format.Node) error { return (*httpNodeAdder)(api).add(ctx, nd, true) } func (api *pinningHttpNodeAdder) AddMany(ctx context.Context, nds []format.Node) error { return (*httpNodeAdder)(api).addMany(ctx, nds, true) } func (api *HttpDagServ) Pinning() format.NodeAdder { return (*pinningHttpNodeAdder)(api) } func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error { return api.core().Block().Rm(ctx, path.FromCid(c)) // TODO: should we force rm? } func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error { for _, c := range cids { // TODO: optimize if err := api.Remove(ctx, c); err != nil { return err } } return nil } func (api *httpNodeAdder) core() *HttpApi { return (*HttpApi)(api) } func (api *HttpDagServ) core() *HttpApi { return (*HttpApi)(api) } ================================================ FILE: client/rpc/errors.go ================================================ package rpc import ( "errors" "strings" "unicode/utf8" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" mbase "github.com/multiformats/go-multibase" ) // This file handle parsing and returning the correct ABI based errors from error messages type prePostWrappedNotFoundError struct { pre string post string wrapped ipld.ErrNotFound } func (e prePostWrappedNotFoundError) String() string { return e.Error() } func (e prePostWrappedNotFoundError) Error() string { return e.pre + e.wrapped.Error() + e.post } func (e prePostWrappedNotFoundError) Unwrap() error { return e.wrapped } func parseErrNotFoundWithFallbackToMSG(msg string) error { err, handled := parseErrNotFound(msg) if handled { return err } return errors.New(msg) } func parseErrNotFoundWithFallbackToError(msg error) error { err, handled := parseErrNotFound(msg.Error()) if handled { return err } return msg } func parseErrNotFound(msg string) (error, bool) { if msg == "" { return nil, true // Fast path } if err, handled := parseIPLDErrNotFound(msg); handled { return err, true } if err, handled := parseBlockstoreNotFound(msg); handled { return err, true } return nil, false } // Assume CIDs break on: // - Whitespaces: " \t\n\r\v\f" // - Semicolon: ";" this is to parse ipld.ErrNotFound wrapped in multierr // - Double Quotes: "\"" this is for parsing %q and %#v formatting. const cidBreakSet = " \t\n\r\v\f;\"" func parseIPLDErrNotFound(msg string) (error, bool) { // The pattern we search for is: const ipldErrNotFoundKey = "ipld: could not find " /*CID*/ // We try to parse the CID, if it's invalid we give up and return a simple text error. // We also accept "node" in place of the CID because that means it's an Undefined CID. keyIndex := strings.Index(msg, ipldErrNotFoundKey) if keyIndex < 0 { // Unknown error return nil, false } cidStart := keyIndex + len(ipldErrNotFoundKey) msgPostKey := msg[cidStart:] var c cid.Cid var postIndex int if strings.HasPrefix(msgPostKey, "node") { // Fallback case c = cid.Undef postIndex = len("node") } else { postIndex = strings.IndexFunc(msgPostKey, func(r rune) bool { return strings.ContainsAny(string(r), cidBreakSet) }) if postIndex < 0 { // no breakage meaning the string look like this something + "ipld: could not find bafy" postIndex = len(msgPostKey) } cidStr := msgPostKey[:postIndex] var err error c, err = cid.Decode(cidStr) if err != nil { // failed to decode CID give up return nil, false } // check that the CID is either a CIDv0 or a base32 multibase // because that what ipld.ErrNotFound.Error() -> cid.Cid.String() do currently if c.Version() != 0 { baseRune, _ := utf8.DecodeRuneInString(cidStr) if baseRune == utf8.RuneError || baseRune != mbase.Base32 { // not a multibase we expect, give up return nil, false } } } err := ipld.ErrNotFound{Cid: c} pre := msg[:keyIndex] post := msgPostKey[postIndex:] if len(pre) > 0 || len(post) > 0 { return prePostWrappedNotFoundError{ pre: pre, post: post, wrapped: err, }, true } return err, true } // This is a simple error type that just return msg as Error(). // But that also match ipld.ErrNotFound when called with Is(err). // That is needed to keep compatibility with code that use string.Contains(err.Error(), "blockstore: block not found") // and code using ipld.ErrNotFound. type blockstoreNotFoundMatchingIPLDErrNotFound struct { msg string } func (e blockstoreNotFoundMatchingIPLDErrNotFound) String() string { return e.Error() } func (e blockstoreNotFoundMatchingIPLDErrNotFound) Error() string { return e.msg } func (e blockstoreNotFoundMatchingIPLDErrNotFound) Is(err error) bool { _, ok := err.(ipld.ErrNotFound) return ok } func parseBlockstoreNotFound(msg string) (error, bool) { if !strings.Contains(msg, "blockstore: block not found") { return nil, false } return blockstoreNotFoundMatchingIPLDErrNotFound{msg: msg}, true } ================================================ FILE: client/rpc/errors_test.go ================================================ package rpc import ( "errors" "fmt" "testing" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" mbase "github.com/multiformats/go-multibase" mh "github.com/multiformats/go-multihash" ) var randomSha256MH = mh.Multihash{0x12, 0x20, 0x88, 0x82, 0x73, 0x37, 0x7c, 0xc1, 0xc9, 0x96, 0xad, 0xee, 0xd, 0x26, 0x84, 0x2, 0xc9, 0xc9, 0x5c, 0xf9, 0x5c, 0x4d, 0x9b, 0xc3, 0x3f, 0xfb, 0x4a, 0xd8, 0xaf, 0x28, 0x6b, 0xca, 0x1a, 0xf2} func doParseIpldNotFoundTest(t *testing.T, original error) { originalMsg := original.Error() rebuilt := parseErrNotFoundWithFallbackToMSG(originalMsg) rebuiltMsg := rebuilt.Error() if originalMsg != rebuiltMsg { t.Errorf("expected message to be %q; got %q", originalMsg, rebuiltMsg) } originalNotFound := ipld.IsNotFound(original) rebuiltNotFound := ipld.IsNotFound(rebuilt) if originalNotFound != rebuiltNotFound { t.Errorf("for %q expected Ipld.IsNotFound to be %t; got %t", originalMsg, originalNotFound, rebuiltNotFound) } } func TestParseIPLDNotFound(t *testing.T) { t.Parallel() if err := parseErrNotFoundWithFallbackToMSG(""); err != nil { t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) } cidBreaks := make([]string, len(cidBreakSet)) for i, v := range cidBreakSet { cidBreaks[i] = "%w" + string(v) } base58BTCEncoder, err := mbase.NewEncoder(mbase.Base58BTC) if err != nil { t.Fatalf("expected to find Base58BTC encoder; got error %q", err.Error()) } for _, wrap := range append(cidBreaks, "", "merkledag: %w", "testing: %w the test", "%w is wrong", ) { for _, err := range [...]error{ errors.New("ipld: could not find "), errors.New("ipld: could not find Bad_CID"), errors.New("ipld: could not find " + cid.NewCidV1(cid.Raw, randomSha256MH).Encode(base58BTCEncoder)), // Test that we only accept CIDv0 and base32 CIDs errors.New("network connection timeout"), ipld.ErrNotFound{Cid: cid.Undef}, ipld.ErrNotFound{Cid: cid.NewCidV0(randomSha256MH)}, ipld.ErrNotFound{Cid: cid.NewCidV1(cid.Raw, randomSha256MH)}, } { if wrap != "" { err = fmt.Errorf(wrap, err) } doParseIpldNotFoundTest(t, err) } } } func TestBlockstoreNotFoundMatchingIPLDErrNotFound(t *testing.T) { t.Parallel() if !ipld.IsNotFound(blockstoreNotFoundMatchingIPLDErrNotFound{}) { t.Fatalf("expected blockstoreNotFoundMatchingIPLDErrNotFound to match ipld.IsNotFound; got false") } for _, wrap := range [...]string{ "", "merkledag: %w", "testing: %w the test", "%w is wrong", } { for _, err := range [...]error{ errors.New("network connection timeout"), blockstoreNotFoundMatchingIPLDErrNotFound{"blockstore: block not found"}, } { if wrap != "" { err = fmt.Errorf(wrap, err) } doParseIpldNotFoundTest(t, err) } } } ================================================ FILE: client/rpc/key.go ================================================ package rpc import ( "bytes" "context" "errors" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" iface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multibase" ) type KeyAPI HttpApi type key struct { name string pid peer.ID path path.Path } func newKey(name, pidStr string) (*key, error) { pid, err := peer.Decode(pidStr) if err != nil { return nil, err } path, err := path.NewPath("/ipns/" + ipns.NameFromPeer(pid).String()) if err != nil { return nil, err } return &key{name: name, pid: pid, path: path}, nil } func (k *key) Name() string { return k.name } func (k *key) Path() path.Path { return k.path } func (k *key) ID() peer.ID { return k.pid } type keyOutput struct { Name string Id string } func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) { options, err := caopts.KeyGenerateOptions(opts...) if err != nil { return nil, err } var out keyOutput err = api.core().Request("key/gen", name). Option("type", options.Algorithm). Option("size", options.Size). Exec(ctx, &out) if err != nil { return nil, err } return newKey(out.Name, out.Id) } func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) { options, err := caopts.KeyRenameOptions(opts...) if err != nil { return nil, false, err } var out struct { Was string Now string Id string Overwrite bool } err = api.core().Request("key/rename", oldName, newName). Option("force", options.Force). Exec(ctx, &out) if err != nil { return nil, false, err } key, err := newKey(out.Now, out.Id) if err != nil { return nil, false, err } return key, out.Overwrite, err } func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { var out struct { Keys []keyOutput } if err := api.core().Request("key/ls").Exec(ctx, &out); err != nil { return nil, err } res := make([]iface.Key, len(out.Keys)) for i, k := range out.Keys { key, err := newKey(k.Name, k.Id) if err != nil { return nil, err } res[i] = key } return res, nil } func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { var id struct{ ID string } if err := api.core().Request("id").Exec(ctx, &id); err != nil { return nil, err } return newKey("self", id.ID) } func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { var out struct { Keys []keyOutput } if err := api.core().Request("key/rm", name).Exec(ctx, &out); err != nil { return nil, err } if len(out.Keys) != 1 { return nil, errors.New("got unexpected number of keys back") } return newKey(out.Keys[0].Name, out.Keys[0].Id) } func (api *KeyAPI) core() *HttpApi { return (*HttpApi)(api) } func (api *KeyAPI) Sign(ctx context.Context, name string, data []byte) (iface.Key, []byte, error) { var out struct { Key keyOutput Signature string } err := api.core().Request("key/sign"). Option("key", name). FileBody(bytes.NewReader(data)). Exec(ctx, &out) if err != nil { return nil, nil, err } key, err := newKey(out.Key.Name, out.Key.Id) if err != nil { return nil, nil, err } _, signature, err := multibase.Decode(out.Signature) if err != nil { return nil, nil, err } return key, signature, nil } func (api *KeyAPI) Verify(ctx context.Context, keyOrName string, signature, data []byte) (iface.Key, bool, error) { var out struct { Key keyOutput SignatureValid bool } err := api.core().Request("key/verify"). Option("key", keyOrName). Option("signature", toMultibase(signature)). FileBody(bytes.NewReader(data)). Exec(ctx, &out) if err != nil { return nil, false, err } key, err := newKey(out.Key.Name, out.Key.Id) if err != nil { return nil, false, err } return key, out.SignatureValid, nil } ================================================ FILE: client/rpc/name.go ================================================ package rpc import ( "context" "encoding/json" "fmt" "io" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/namesys" "github.com/ipfs/boxo/path" iface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" ) type NameAPI HttpApi type ipnsEntry struct { Name string `json:"Name"` Value string `json:"Value"` } func (api *NameAPI) Publish(ctx context.Context, p path.Path, opts ...caopts.NamePublishOption) (ipns.Name, error) { options, err := caopts.NamePublishOptions(opts...) if err != nil { return ipns.Name{}, err } req := api.core().Request("name/publish", p.String()). Option("key", options.Key). Option("allow-offline", options.AllowOffline). Option("lifetime", options.ValidTime). Option("resolve", false) if options.TTL != nil { req.Option("ttl", options.TTL) } var out ipnsEntry if err := req.Exec(ctx, &out); err != nil { return ipns.Name{}, err } return ipns.NameFromString(out.Name) } func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { options, err := caopts.NameResolveOptions(opts...) if err != nil { return nil, err } ropts := namesys.ProcessResolveOptions(options.ResolveOpts) if ropts.Depth != namesys.DefaultDepthLimit && ropts.Depth != 1 { return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", namesys.DefaultDepthLimit) } req := api.core().Request("name/resolve", name). Option("nocache", !options.Cache). Option("recursive", ropts.Depth != 1). Option("dht-record-count", ropts.DhtRecordCount). Option("dht-timeout", ropts.DhtTimeout). Option("stream", true) resp, err := req.Send(ctx) if err != nil { return nil, err } if resp.Error != nil { return nil, resp.Error } res := make(chan iface.IpnsResult) go func() { defer close(res) defer resp.Close() dec := json.NewDecoder(resp.Output) for { var out struct{ Path string } err := dec.Decode(&out) if err == io.EOF { return } var ires iface.IpnsResult if err == nil { p, err := path.NewPath(out.Path) if err != nil { return } ires.Path = p } select { case res <- ires: case <-ctx.Done(): } if err != nil { return } } }() return res, nil } func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (path.Path, error) { options, err := caopts.NameResolveOptions(opts...) if err != nil { return nil, err } ropts := namesys.ProcessResolveOptions(options.ResolveOpts) if ropts.Depth != namesys.DefaultDepthLimit && ropts.Depth != 1 { return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", namesys.DefaultDepthLimit) } req := api.core().Request("name/resolve", name). Option("nocache", !options.Cache). Option("recursive", ropts.Depth != 1). Option("dht-record-count", ropts.DhtRecordCount). Option("dht-timeout", ropts.DhtTimeout) var out struct{ Path string } if err := req.Exec(ctx, &out); err != nil { return nil, err } return path.NewPath(out.Path) } func (api *NameAPI) core() *HttpApi { return (*HttpApi)(api) } ================================================ FILE: client/rpc/object.go ================================================ package rpc import ( "context" "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" iface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" ) type ObjectAPI HttpApi type objectOut struct { Hash string } func (api *ObjectAPI) AddLink(ctx context.Context, base path.Path, name string, child path.Path, opts ...caopts.ObjectAddLinkOption) (path.ImmutablePath, error) { options, err := caopts.ObjectAddLinkOptions(opts...) if err != nil { return path.ImmutablePath{}, err } var out objectOut err = api.core().Request("object/patch/add-link", base.String(), name, child.String()). Option("create", options.Create). Exec(ctx, &out) if err != nil { return path.ImmutablePath{}, err } c, err := cid.Parse(out.Hash) if err != nil { return path.ImmutablePath{}, err } return path.FromCid(c), nil } func (api *ObjectAPI) RmLink(ctx context.Context, base path.Path, link string) (path.ImmutablePath, error) { var out objectOut err := api.core().Request("object/patch/rm-link", base.String(), link). Exec(ctx, &out) if err != nil { return path.ImmutablePath{}, err } c, err := cid.Parse(out.Hash) if err != nil { return path.ImmutablePath{}, err } return path.FromCid(c), nil } type change struct { Type iface.ChangeType Path string Before cid.Cid After cid.Cid } func (api *ObjectAPI) Diff(ctx context.Context, a path.Path, b path.Path) ([]iface.ObjectChange, error) { var out struct { Changes []change } if err := api.core().Request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil { return nil, err } res := make([]iface.ObjectChange, len(out.Changes)) for i, ch := range out.Changes { res[i] = iface.ObjectChange{ Type: ch.Type, Path: ch.Path, } if ch.Before != cid.Undef { res[i].Before = path.FromCid(ch.Before) } if ch.After != cid.Undef { res[i].After = path.FromCid(ch.After) } } return res, nil } func (api *ObjectAPI) core() *HttpApi { return (*HttpApi)(api) } ================================================ FILE: client/rpc/path.go ================================================ package rpc import ( "context" "github.com/ipfs/boxo/path" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" ) func (api *HttpApi) ResolvePath(ctx context.Context, p path.Path) (path.ImmutablePath, []string, error) { var out struct { Cid cid.Cid RemPath string } var err error if p.Namespace() == path.IPNSNamespace { if p, err = api.Name().Resolve(ctx, p.String()); err != nil { return path.ImmutablePath{}, nil, err } } if err := api.Request("dag/resolve", p.String()).Exec(ctx, &out); err != nil { return path.ImmutablePath{}, nil, err } p, err = path.NewPathFromSegments(p.Namespace(), out.Cid.String(), out.RemPath) if err != nil { return path.ImmutablePath{}, nil, err } imPath, err := path.NewImmutablePath(p) if err != nil { return path.ImmutablePath{}, nil, err } return imPath, path.StringToSegments(out.RemPath), nil } func (api *HttpApi) ResolveNode(ctx context.Context, p path.Path) (ipld.Node, error) { rp, _, err := api.ResolvePath(ctx, p) if err != nil { return nil, err } return api.Dag().Get(ctx, rp.RootCid()) } ================================================ FILE: client/rpc/pin.go ================================================ package rpc import ( "context" "encoding/json" "errors" "io" "strings" "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" iface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" ) type PinAPI HttpApi type pinRefKeyObject struct { Type string } type pinRefKeyList struct { Keys map[string]pinRefKeyObject } type pin struct { path path.ImmutablePath typ string name string err error } func (p pin) Err() error { return p.err } func (p pin) Path() path.ImmutablePath { return p.path } func (p pin) Name() string { return p.name } func (p pin) Type() string { return p.typ } func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOption) error { options, err := caopts.PinAddOptions(opts...) if err != nil { return err } req := api.core().Request("pin/add", p.String()). Option("recursive", options.Recursive) if options.Name != "" { req = req.Option("name", options.Name) } return req.Exec(ctx, nil) } type pinLsObject struct { Cid string Name string Type string } func (api *PinAPI) Ls(ctx context.Context, pins chan<- iface.Pin, opts ...caopts.PinLsOption) error { defer close(pins) options, err := caopts.PinLsOptions(opts...) if err != nil { return err } res, err := api.core().Request("pin/ls"). Option("type", options.Type). Option("names", options.Detailed). Option("stream", true). Send(ctx) if err != nil { return err } defer res.Output.Close() dec := json.NewDecoder(res.Output) for { var out pinLsObject err := dec.Decode(&out) if err != nil { if err != io.EOF { return err } return nil } c, err := cid.Parse(out.Cid) if err != nil { return err } select { case pins <- pin{typ: out.Type, name: out.Name, path: path.FromCid(c)}: case <-ctx.Done(): return ctx.Err() } } } // IsPinned returns whether or not the given cid is pinned // and an explanation of why its pinned. func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.PinIsPinnedOption) (string, bool, error) { options, err := caopts.PinIsPinnedOptions(opts...) if err != nil { return "", false, err } var out pinRefKeyList err = api.core().Request("pin/ls"). Option("type", options.WithType). Option("arg", p.String()). Exec(ctx, &out) if err != nil { // TODO: This error-type discrimination based on sub-string matching is brittle. // It is addressed by this open issue: https://github.com/ipfs/go-ipfs/issues/7563 if strings.Contains(err.Error(), "is not pinned") { return "", false, nil } return "", false, err } for _, obj := range out.Keys { return obj.Type, true, nil } return "", false, errors.New("http api returned no error and no results") } func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOption) error { options, err := caopts.PinRmOptions(opts...) if err != nil { return err } return api.core().Request("pin/rm", p.String()). Option("recursive", options.Recursive). Exec(ctx, nil) } func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opts ...caopts.PinUpdateOption) error { options, err := caopts.PinUpdateOptions(opts...) if err != nil { return err } return api.core().Request("pin/update", from.String(), to.String()). Option("unpin", options.Unpin).Exec(ctx, nil) } type pinVerifyRes struct { ok bool badNodes []iface.BadPinNode err error } func (r pinVerifyRes) Ok() bool { return r.ok } func (r pinVerifyRes) BadNodes() []iface.BadPinNode { return r.badNodes } func (r pinVerifyRes) Err() error { return r.err } type badNode struct { err error cid cid.Cid } func (n badNode) Path() path.ImmutablePath { return path.FromCid(n.cid) } func (n badNode) Err() error { return n.err } func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { resp, err := api.core().Request("pin/verify").Option("verbose", true).Send(ctx) if err != nil { return nil, err } if resp.Error != nil { return nil, resp.Error } res := make(chan iface.PinStatus) go func() { defer resp.Close() defer close(res) dec := json.NewDecoder(resp.Output) for { var out struct { Cid string Err string Ok bool BadNodes []struct { Cid string Err string } } if err := dec.Decode(&out); err != nil { if err == io.EOF { return } select { case res <- pinVerifyRes{err: err}: return case <-ctx.Done(): return } } if out.Err != "" { select { case res <- pinVerifyRes{err: errors.New(out.Err)}: return case <-ctx.Done(): return } } badNodes := make([]iface.BadPinNode, len(out.BadNodes)) for i, n := range out.BadNodes { c, err := cid.Decode(n.Cid) if err != nil { badNodes[i] = badNode{cid: c, err: err} continue } if n.Err != "" { err = errors.New(n.Err) } badNodes[i] = badNode{cid: c, err: err} } select { case res <- pinVerifyRes{ok: out.Ok, badNodes: badNodes}: case <-ctx.Done(): return } } }() return res, nil } func (api *PinAPI) core() *HttpApi { return (*HttpApi)(api) } ================================================ FILE: client/rpc/pubsub.go ================================================ package rpc import ( "bytes" "context" "encoding/json" "io" iface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" ) type PubsubAPI HttpApi func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) { var out struct { Strings []string } if err := api.core().Request("pubsub/ls").Exec(ctx, &out); err != nil { return nil, err } topics := make([]string, len(out.Strings)) for n, mb := range out.Strings { _, topic, err := mbase.Decode(mb) if err != nil { return nil, err } topics[n] = string(topic) } return topics, nil } func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOption) ([]peer.ID, error) { options, err := caopts.PubSubPeersOptions(opts...) if err != nil { return nil, err } var out struct { Strings []string } var optionalTopic string if len(options.Topic) > 0 { optionalTopic = toMultibase([]byte(options.Topic)) } if err := api.core().Request("pubsub/peers", optionalTopic).Exec(ctx, &out); err != nil { return nil, err } res := make([]peer.ID, len(out.Strings)) for i, sid := range out.Strings { id, err := peer.Decode(sid) if err != nil { return nil, err } res[i] = id } return res, nil } func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error { return api.core().Request("pubsub/pub", toMultibase([]byte(topic))). FileBody(bytes.NewReader(message)). Exec(ctx, nil) } type pubsubSub struct { messages chan pubsubMessage done chan struct{} rcloser func() error } type pubsubMessage struct { JFrom string `json:"from,omitempty"` JData string `json:"data,omitempty"` JSeqno string `json:"seqno,omitempty"` JTopicIDs []string `json:"topicIDs,omitempty"` // real values after unpacking from text/multibase envelopes from peer.ID data []byte seqno []byte topics []string err error } func (msg *pubsubMessage) From() peer.ID { return msg.from } func (msg *pubsubMessage) Data() []byte { return msg.data } func (msg *pubsubMessage) Seq() []byte { return msg.seqno } // TODO: do we want to keep this interface as []string, // or change to more correct [][]byte? func (msg *pubsubMessage) Topics() []string { return msg.topics } func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { select { case msg, ok := <-s.messages: if !ok { return nil, io.EOF } if msg.err != nil { return nil, msg.err } // unpack values from text/multibase envelopes var err error msg.from, err = peer.Decode(msg.JFrom) if err != nil { return nil, err } _, msg.data, err = mbase.Decode(msg.JData) if err != nil { return nil, err } _, msg.seqno, err = mbase.Decode(msg.JSeqno) if err != nil { return nil, err } for _, mbt := range msg.JTopicIDs { _, topic, err := mbase.Decode(mbt) if err != nil { return nil, err } msg.topics = append(msg.topics, string(topic)) } return &msg, nil case <-ctx.Done(): return nil, ctx.Err() } } func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { /* right now we have no options (discover got deprecated) options, err := caopts.PubSubSubscribeOptions(opts...) if err != nil { return nil, err } */ resp, err := api.core().Request("pubsub/sub", toMultibase([]byte(topic))).Send(ctx) if err != nil { return nil, err } if resp.Error != nil { return nil, resp.Error } sub := &pubsubSub{ messages: make(chan pubsubMessage), done: make(chan struct{}), rcloser: func() error { return resp.Cancel() }, } dec := json.NewDecoder(resp.Output) go func() { defer close(sub.messages) for { var msg pubsubMessage if err := dec.Decode(&msg); err != nil { if err == io.EOF { return } msg.err = err } select { case sub.messages <- msg: case <-sub.done: return case <-ctx.Done(): return } } }() return sub, nil } func (s *pubsubSub) Close() error { if s.done != nil { close(s.done) s.done = nil } return s.rcloser() } func (api *PubsubAPI) core() *HttpApi { return (*HttpApi)(api) } // Encodes bytes into URL-safe multibase that can be sent over HTTP RPC (URL or body). func toMultibase(data []byte) string { mb, _ := mbase.Encode(mbase.Base64url, data) return mb } ================================================ FILE: client/rpc/request.go ================================================ package rpc import ( "context" "io" "strings" ) type Request struct { Ctx context.Context ApiBase string Command string Args []string Opts map[string]string Body io.Reader Headers map[string]string } func NewRequest(ctx context.Context, url, command string, args ...string) *Request { if !strings.HasPrefix(url, "http") { url = "http://" + url } opts := map[string]string{ "encoding": "json", "stream-channels": "true", } return &Request{ Ctx: ctx, ApiBase: url + "/api/v0", Command: command, Args: args, Opts: opts, Headers: make(map[string]string), } } ================================================ FILE: client/rpc/requestbuilder.go ================================================ package rpc import ( "bytes" "context" "fmt" "io" "strconv" "strings" "github.com/blang/semver/v4" "github.com/ipfs/boxo/files" ) type RequestBuilder interface { Arguments(args ...string) RequestBuilder BodyString(body string) RequestBuilder BodyBytes(body []byte) RequestBuilder Body(body io.Reader) RequestBuilder FileBody(body io.Reader) RequestBuilder Option(key string, value any) RequestBuilder Header(name, value string) RequestBuilder Send(ctx context.Context) (*Response, error) Exec(ctx context.Context, res any) error } // encodedAbsolutePathVersion is the version from which the absolute path header in // multipart requests is %-encoded. Before this version, its sent raw. var encodedAbsolutePathVersion = semver.MustParse("0.23.0-dev") // requestBuilder is an IPFS commands request builder. type requestBuilder struct { command string args []string opts map[string]string headers map[string]string body io.Reader buildError error shell *HttpApi } // Arguments adds the arguments to the args. func (r *requestBuilder) Arguments(args ...string) RequestBuilder { r.args = append(r.args, args...) return r } // BodyString sets the request body to the given string. func (r *requestBuilder) BodyString(body string) RequestBuilder { return r.Body(strings.NewReader(body)) } // BodyBytes sets the request body to the given buffer. func (r *requestBuilder) BodyBytes(body []byte) RequestBuilder { return r.Body(bytes.NewReader(body)) } // Body sets the request body to the given reader. func (r *requestBuilder) Body(body io.Reader) RequestBuilder { r.body = body return r } // FileBody sets the request body to the given reader wrapped into multipartreader. func (r *requestBuilder) FileBody(body io.Reader) RequestBuilder { pr, _ := files.NewReaderPathFile("/dev/stdin", io.NopCloser(body), nil) d := files.NewMapDirectory(map[string]files.Node{"": pr}) version, err := r.shell.loadRemoteVersion() if err != nil { // Unfortunately, we cannot return an error here. Changing this API is also // not the best since we would otherwise have an inconsistent RequestBuilder. // We save the error and return it when calling [requestBuilder.Send]. r.buildError = err return r } useEncodedAbsPaths := version.LT(encodedAbsolutePathVersion) r.body = files.NewMultiFileReader(d, false, useEncodedAbsPaths) return r } // Option sets the given option. func (r *requestBuilder) Option(key string, value any) RequestBuilder { var s string switch v := value.(type) { case bool: s = strconv.FormatBool(v) case string: s = v case []byte: s = string(v) default: // slow case. s = fmt.Sprint(value) } if r.opts == nil { r.opts = make(map[string]string, 1) } r.opts[key] = s return r } // Header sets the given header. func (r *requestBuilder) Header(name, value string) RequestBuilder { if r.headers == nil { r.headers = make(map[string]string, 1) } r.headers[name] = value return r } // Send sends the request and return the response. func (r *requestBuilder) Send(ctx context.Context) (*Response, error) { if r.buildError != nil { return nil, r.buildError } r.shell.applyGlobal(r) req := NewRequest(ctx, r.shell.url, r.command, r.args...) req.Opts = r.opts req.Headers = r.headers req.Body = r.body return req.Send(&r.shell.httpcli) } // Exec sends the request a request and decodes the response. func (r *requestBuilder) Exec(ctx context.Context, res any) error { httpRes, err := r.Send(ctx) if err != nil { return err } if res == nil { lateErr := httpRes.Close() if httpRes.Error != nil { return httpRes.Error } return lateErr } return httpRes.decode(res) } var _ RequestBuilder = &requestBuilder{} ================================================ FILE: client/rpc/response.go ================================================ package rpc import ( "encoding/json" "errors" "fmt" "io" "mime" "net/http" "net/url" "os" "github.com/ipfs/boxo/files" cmds "github.com/ipfs/go-ipfs-cmds" cmdhttp "github.com/ipfs/go-ipfs-cmds/http" ) type Error = cmds.Error type trailerReader struct { resp *http.Response } func (r *trailerReader) Read(b []byte) (int, error) { n, err := r.resp.Body.Read(b) if err != nil { if e := r.resp.Trailer.Get(cmdhttp.StreamErrHeader); e != "" { err = errors.New(e) } } return n, err } func (r *trailerReader) Close() error { return r.resp.Body.Close() } type Response struct { Output io.ReadCloser Error *Error } func (r *Response) Close() error { if r.Output != nil { // drain output (response body) _, err1 := io.Copy(io.Discard, r.Output) err2 := r.Output.Close() if err1 != nil { return err1 } return err2 } return nil } // Cancel aborts running request (without draining request body). func (r *Response) Cancel() error { if r.Output != nil { return r.Output.Close() } return nil } // Decode reads request body and decodes it as json. func (r *Response) decode(dec any) error { if r.Error != nil { return r.Error } err := json.NewDecoder(r.Output).Decode(dec) err2 := r.Close() if err != nil { return err } return err2 } func (r *Request) Send(c *http.Client) (*Response, error) { url := r.getURL() req, err := http.NewRequest("POST", url, r.Body) if err != nil { return nil, err } req = req.WithContext(r.Ctx) // Add any headers that were supplied via the requestBuilder. for k, v := range r.Headers { req.Header.Add(k, v) } if fr, ok := r.Body.(*files.MultiFileReader); ok { req.Header.Set("Content-Type", "multipart/form-data; boundary="+fr.Boundary()) req.Header.Set("Content-Disposition", "form-data; name=\"files\"") } resp, err := c.Do(req) if err != nil { return nil, err } contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) if err != nil { return nil, err } nresp := new(Response) nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { e := new(Error) switch { case resp.StatusCode == http.StatusNotFound: e.Message = "command not found" case contentType == "text/plain": out, err := io.ReadAll(resp.Body) if err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err) } e.Message = string(out) // set special status codes. switch resp.StatusCode { case http.StatusNotFound, http.StatusBadRequest: e.Code = cmds.ErrClient case http.StatusTooManyRequests: e.Code = cmds.ErrRateLimited case http.StatusForbidden: e.Code = cmds.ErrForbidden } case contentType == "application/json": if err = json.NewDecoder(resp.Body).Decode(e); err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) unmarshall error: %s\n", resp.StatusCode, err) } default: // This is a server-side bug (probably). e.Code = cmds.ErrImplementation fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) out, err := io.ReadAll(resp.Body) if err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: response (%d) read error: %s\n", resp.StatusCode, err) } e.Message = fmt.Sprintf("unknown ipfs-shell error encoding: %q - %q", contentType, out) } nresp.Error = e nresp.Output = nil // drain body and close _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() } return nresp, nil } func (r *Request) getURL() string { values := make(url.Values) for _, arg := range r.Args { values.Add("arg", arg) } for k, v := range r.Opts { values.Add(k, v) } return fmt.Sprintf("%s/%s?%s", r.ApiBase, r.Command, values.Encode()) } ================================================ FILE: client/rpc/routing.go ================================================ package rpc import ( "bytes" "context" "encoding/base64" "encoding/json" "github.com/ipfs/boxo/path" "github.com/ipfs/kubo/core/coreiface/options" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" ) type RoutingAPI HttpApi func (api *RoutingAPI) Get(ctx context.Context, key string) ([]byte, error) { resp, err := api.core().Request("routing/get", key).Send(ctx) if err != nil { return nil, err } if resp.Error != nil { return nil, resp.Error } defer resp.Close() var out routing.QueryEvent dec := json.NewDecoder(resp.Output) if err := dec.Decode(&out); err != nil { return nil, err } res, err := base64.StdEncoding.DecodeString(out.Extra) if err != nil { return nil, err } return res, nil } func (api *RoutingAPI) Put(ctx context.Context, key string, value []byte, opts ...options.RoutingPutOption) error { var cfg options.RoutingPutSettings for _, o := range opts { if err := o(&cfg); err != nil { return err } } resp, err := api.core().Request("routing/put", key). Option("allow-offline", cfg.AllowOffline). FileBody(bytes.NewReader(value)). Send(ctx) if err != nil { return err } if resp.Error != nil { return resp.Error } return nil } func (api *RoutingAPI) FindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error) { var out struct { Type routing.QueryEventType Responses []peer.AddrInfo } resp, err := api.core().Request("routing/findpeer", p.String()).Send(ctx) if err != nil { return peer.AddrInfo{}, err } if resp.Error != nil { return peer.AddrInfo{}, resp.Error } defer resp.Close() dec := json.NewDecoder(resp.Output) for { if err := dec.Decode(&out); err != nil { return peer.AddrInfo{}, err } if out.Type == routing.FinalPeer { return out.Responses[0], nil } } } func (api *RoutingAPI) FindProviders(ctx context.Context, p path.Path, opts ...options.RoutingFindProvidersOption) (<-chan peer.AddrInfo, error) { options, err := options.RoutingFindProvidersOptions(opts...) if err != nil { return nil, err } rp, _, err := api.core().ResolvePath(ctx, p) if err != nil { return nil, err } resp, err := api.core().Request("routing/findprovs", rp.RootCid().String()). Option("num-providers", options.NumProviders). Send(ctx) if err != nil { return nil, err } if resp.Error != nil { return nil, resp.Error } res := make(chan peer.AddrInfo) go func() { defer resp.Close() defer close(res) dec := json.NewDecoder(resp.Output) for { var out struct { Extra string Type routing.QueryEventType Responses []peer.AddrInfo } if err := dec.Decode(&out); err != nil { return // todo: handle this somehow } if out.Type == routing.QueryError { return // usually a 'not found' error // todo: handle other errors } if out.Type == routing.Provider { for _, pi := range out.Responses { select { case res <- pi: case <-ctx.Done(): return } } } } }() return res, nil } func (api *RoutingAPI) Provide(ctx context.Context, p path.Path, opts ...options.RoutingProvideOption) error { options, err := options.RoutingProvideOptions(opts...) if err != nil { return err } rp, _, err := api.core().ResolvePath(ctx, p) if err != nil { return err } return api.core().Request("routing/provide", rp.RootCid().String()). Option("recursive", options.Recursive). Exec(ctx, nil) } func (api *RoutingAPI) core() *HttpApi { return (*HttpApi)(api) } ================================================ FILE: client/rpc/swarm.go ================================================ package rpc import ( "context" "time" iface "github.com/ipfs/kubo/core/coreiface" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" "github.com/multiformats/go-multiaddr" ) type SwarmAPI HttpApi func (api *SwarmAPI) Connect(ctx context.Context, pi peer.AddrInfo) error { pidma, err := multiaddr.NewComponent("p2p", pi.ID.String()) if err != nil { return err } saddrs := make([]string, len(pi.Addrs)) for i, addr := range pi.Addrs { saddrs[i] = addr.Encapsulate(pidma).String() } return api.core().Request("swarm/connect", saddrs...).Exec(ctx, nil) } func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) error { return api.core().Request("swarm/disconnect", addr.String()).Exec(ctx, nil) } type connInfo struct { addr multiaddr.Multiaddr peer peer.ID latency time.Duration muxer string direction network.Direction streams []protocol.ID } func (c *connInfo) ID() peer.ID { return c.peer } func (c *connInfo) Address() multiaddr.Multiaddr { return c.addr } func (c *connInfo) Direction() network.Direction { return c.direction } func (c *connInfo) Latency() (time.Duration, error) { return c.latency, nil } func (c *connInfo) Streams() ([]protocol.ID, error) { return c.streams, nil } func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) { var resp struct { Peers []struct { Addr string Peer string Latency string Muxer string Direction network.Direction Streams []struct { Protocol string } } } err := api.core().Request("swarm/peers"). Option("streams", true). Option("latency", true). Exec(ctx, &resp) if err != nil { return nil, err } res := make([]iface.ConnectionInfo, len(resp.Peers)) for i, conn := range resp.Peers { latency, _ := time.ParseDuration(conn.Latency) out := &connInfo{ latency: latency, muxer: conn.Muxer, direction: conn.Direction, } out.peer, err = peer.Decode(conn.Peer) if err != nil { return nil, err } out.addr, err = multiaddr.NewMultiaddr(conn.Addr) if err != nil { return nil, err } out.streams = make([]protocol.ID, len(conn.Streams)) for i, p := range conn.Streams { out.streams[i] = protocol.ID(p.Protocol) } res[i] = out } return res, nil } func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Multiaddr, error) { var out struct { Addrs map[string][]string } if err := api.core().Request("swarm/addrs").Exec(ctx, &out); err != nil { return nil, err } res := map[peer.ID][]multiaddr.Multiaddr{} for spid, saddrs := range out.Addrs { addrs := make([]multiaddr.Multiaddr, len(saddrs)) for i, addr := range saddrs { a, err := multiaddr.NewMultiaddr(addr) if err != nil { return nil, err } addrs[i] = a } pid, err := peer.Decode(spid) if err != nil { return nil, err } res[pid] = addrs } return res, nil } func (api *SwarmAPI) LocalAddrs(ctx context.Context) ([]multiaddr.Multiaddr, error) { var out struct { Strings []string } if err := api.core().Request("swarm/addrs/local").Exec(ctx, &out); err != nil { return nil, err } res := make([]multiaddr.Multiaddr, len(out.Strings)) for i, addr := range out.Strings { ma, err := multiaddr.NewMultiaddr(addr) if err != nil { return nil, err } res[i] = ma } return res, nil } func (api *SwarmAPI) ListenAddrs(ctx context.Context) ([]multiaddr.Multiaddr, error) { var out struct { Strings []string } if err := api.core().Request("swarm/addrs/listen").Exec(ctx, &out); err != nil { return nil, err } res := make([]multiaddr.Multiaddr, len(out.Strings)) for i, addr := range out.Strings { ma, err := multiaddr.NewMultiaddr(addr) if err != nil { return nil, err } res[i] = ma } return res, nil } func (api *SwarmAPI) core() *HttpApi { return (*HttpApi)(api) } ================================================ FILE: client/rpc/unixfs.go ================================================ package rpc import ( "context" "encoding/json" "errors" "fmt" "io" "os" "time" "github.com/ipfs/boxo/files" unixfs "github.com/ipfs/boxo/ipld/unixfs" unixfs_pb "github.com/ipfs/boxo/ipld/unixfs/pb" "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" iface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" mh "github.com/multiformats/go-multihash" ) type addEvent struct { Name string Hash string `json:",omitempty"` Bytes int64 `json:",omitempty"` Size string `json:",omitempty"` } type UnixfsAPI HttpApi func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (path.ImmutablePath, error) { options, _, err := caopts.UnixfsAddOptions(opts...) if err != nil { return path.ImmutablePath{}, err } mht, ok := mh.Codes[options.MhType] if !ok { return path.ImmutablePath{}, fmt.Errorf("unknowm mhType %d", options.MhType) } req := api.core().Request("add"). Option("hash", mht). Option("chunker", options.Chunker). Option("cid-version", options.CidVersion). Option("fscache", options.FsCache). Option("inline", options.Inline). Option("inline-limit", options.InlineLimit). Option("nocopy", options.NoCopy). Option("only-hash", options.OnlyHash). Option("pin", options.Pin). Option("silent", options.Silent). Option("progress", options.Progress) if options.RawLeavesSet { req.Option("raw-leaves", options.RawLeaves) } switch options.Layout { case caopts.BalancedLayout: // noop, default case caopts.TrickleLayout: req.Option("trickle", true) } d := files.NewMapDirectory(map[string]files.Node{"": f}) // unwrapped on the other side version, err := api.core().loadRemoteVersion() if err != nil { return path.ImmutablePath{}, err } useEncodedAbsPaths := version.LT(encodedAbsolutePathVersion) req.Body(files.NewMultiFileReader(d, false, useEncodedAbsPaths)) var out addEvent resp, err := req.Send(ctx) if err != nil { return path.ImmutablePath{}, err } if resp.Error != nil { return path.ImmutablePath{}, resp.Error } defer resp.Output.Close() dec := json.NewDecoder(resp.Output) for { var evt addEvent if err := dec.Decode(&evt); err != nil { if errors.Is(err, io.EOF) { break } return path.ImmutablePath{}, err } out = evt if options.Events != nil { ifevt := &iface.AddEvent{ Name: out.Name, Size: out.Size, Bytes: out.Bytes, } if out.Hash != "" { c, err := cid.Parse(out.Hash) if err != nil { return path.ImmutablePath{}, err } ifevt.Path = path.FromCid(c) } select { case options.Events <- ifevt: case <-ctx.Done(): return path.ImmutablePath{}, ctx.Err() } } } c, err := cid.Parse(out.Hash) if err != nil { return path.ImmutablePath{}, err } return path.FromCid(c), nil } type lsLink struct { Name, Hash string Size uint64 Type unixfs_pb.Data_DataType Target string Mode os.FileMode ModTime time.Time } type lsObject struct { Hash string Links []lsLink } type lsOutput struct { Objects []lsObject } func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, out chan<- iface.DirEntry, opts ...caopts.UnixfsLsOption) error { defer close(out) options, err := caopts.UnixfsLsOptions(opts...) if err != nil { return err } resp, err := api.core().Request("ls", p.String()). Option("resolve-type", options.ResolveChildren). Option("size", options.ResolveChildren). Option("stream", true). Send(ctx) if err != nil { return err } if resp.Error != nil { return err } defer resp.Close() dec := json.NewDecoder(resp.Output) for { var link lsOutput if err = dec.Decode(&link); err != nil { if err != io.EOF { return err } return nil } if len(link.Objects) != 1 { return errors.New("unexpected Objects len") } if len(link.Objects[0].Links) != 1 { return errors.New("unexpected Links len") } l0 := link.Objects[0].Links[0] c, err := cid.Decode(l0.Hash) if err != nil { return err } var ftype iface.FileType switch l0.Type { case unixfs.TRaw, unixfs.TFile: ftype = iface.TFile case unixfs.THAMTShard, unixfs.TDirectory, unixfs.TMetadata: ftype = iface.TDirectory case unixfs.TSymlink: ftype = iface.TSymlink } select { case out <- iface.DirEntry{ Name: l0.Name, Cid: c, Size: l0.Size, Type: ftype, Target: l0.Target, Mode: l0.Mode, ModTime: l0.ModTime, }: case <-ctx.Done(): return ctx.Err() } } } func (api *UnixfsAPI) core() *HttpApi { return (*HttpApi)(api) } ================================================ FILE: codecov.yml ================================================ codecov: ci: - "!travis-ci.org" - "!ci.ipfs.team:8111" - "!ci.ipfs.team" - "github.com" notify: require_ci_to_pass: no after_n_builds: 2 coverage: range: "50...100" status: project: default: threshold: 0.2% patch: default: threshold: 2% comment: off ================================================ FILE: commands/context.go ================================================ package commands import ( "context" "errors" "strings" "time" core "github.com/ipfs/kubo/core" coreapi "github.com/ipfs/kubo/core/coreapi" loader "github.com/ipfs/kubo/plugin/loader" cmds "github.com/ipfs/go-ipfs-cmds" logging "github.com/ipfs/go-log/v2" config "github.com/ipfs/kubo/config" coreiface "github.com/ipfs/kubo/core/coreiface" options "github.com/ipfs/kubo/core/coreiface/options" ) var log = logging.Logger("command") // Context represents request context. type Context struct { ConfigRoot string ReqLog *ReqLog Plugins *loader.PluginLoader Gateway bool api coreiface.CoreAPI node *core.IpfsNode ConstructNode func() (*core.IpfsNode, error) } func (c *Context) GetConfig() (*config.Config, error) { node, err := c.GetNode() if err != nil { return nil, err } return node.Repo.Config() } // GetNode returns the node of the current Command execution // context. It may construct it with the provided function. func (c *Context) GetNode() (*core.IpfsNode, error) { var err error if c.node == nil { if c.ConstructNode == nil { return nil, errors.New("nil ConstructNode function") } c.node, err = c.ConstructNode() } return c.node, err } // ClearCachedNode clears any cached node, forcing GetNode to construct a new one. // // This method is critical for mitigating racy FX dependency injection behavior // that can occur during daemon startup. The daemon may create multiple IpfsNode // instances during initialization - first an offline node during early init, then // the proper online daemon node. Without clearing the cache, HTTP RPC handlers may // end up using the first (offline) cached node instead of the intended online daemon node. // // This behavior was likely present forever in go-ipfs, but recent changes made it more // prominent and forced us to proactively mitigate FX shortcomings. The daemon calls // this method immediately before setting its ConstructNode function to ensure that // subsequent GetNode() calls use the correct online daemon node rather than any // stale cached offline node from initialization. func (c *Context) ClearCachedNode() { c.node = nil } // GetAPI returns CoreAPI instance backed by ipfs node. // It may construct the node with the provided function. func (c *Context) GetAPI() (coreiface.CoreAPI, error) { if c.api == nil { n, err := c.GetNode() if err != nil { return nil, err } fetchBlocks := true if c.Gateway { cfg, err := c.GetConfig() if err != nil { return nil, err } fetchBlocks = !cfg.Gateway.NoFetch } c.api, err = coreapi.NewCoreAPI(n, options.Api.FetchBlocks(fetchBlocks)) if err != nil { return nil, err } } return c.api, nil } // Context returns the node's context. func (c *Context) Context() context.Context { n, err := c.GetNode() if err != nil { log.Debug("error getting node: ", err) return context.Background() } return n.Context() } // LogRequest adds the passed request to the request log and // returns a function that should be called when the request // lifetime is over. func (c *Context) LogRequest(req *cmds.Request) func() { rle := &ReqLogEntry{ StartTime: time.Now(), Active: true, Command: strings.Join(req.Path, "/"), Options: req.Options, Args: req.Arguments, log: c.ReqLog, } c.ReqLog.AddEntry(rle) return func() { c.ReqLog.Finish(rle) } } // Close cleans up the application state. func (c *Context) Close() { // let's not forget teardown. If a node was initialized, we must close it. // Note that this means the underlying req.Context().Node variable is exposed. // this is gross, and should be changed when we extract out the exec Context. if c.node != nil { log.Info("Shutting down node...") c.node.Close() } } ================================================ FILE: commands/reqlog.go ================================================ package commands import ( "sync" "time" ) // ReqLogEntry is an entry in the request log. type ReqLogEntry struct { StartTime time.Time EndTime time.Time Active bool Command string Options map[string]any Args []string ID int log *ReqLog } // Copy returns a copy of the ReqLogEntry. func (r *ReqLogEntry) Copy() *ReqLogEntry { out := *r out.log = nil return &out } // ReqLog is a log of requests. type ReqLog struct { Requests []*ReqLogEntry nextID int lock sync.Mutex keep time.Duration } // AddEntry adds an entry to the log. func (rl *ReqLog) AddEntry(rle *ReqLogEntry) { rl.lock.Lock() defer rl.lock.Unlock() rle.ID = rl.nextID rl.nextID++ rl.Requests = append(rl.Requests, rle) if !rle.Active { rl.maybeCleanup() } } // ClearInactive removes stale entries. func (rl *ReqLog) ClearInactive() { rl.lock.Lock() defer rl.lock.Unlock() k := rl.keep rl.keep = 0 rl.cleanup() rl.keep = k } func (rl *ReqLog) maybeCleanup() { // only do it every so often or it might // become a perf issue if len(rl.Requests)%10 == 0 { rl.cleanup() } } func (rl *ReqLog) cleanup() { i := 0 now := time.Now() for j := 0; j < len(rl.Requests); j++ { rj := rl.Requests[j] if rj.Active || rl.Requests[j].EndTime.Add(rl.keep).After(now) { rl.Requests[i] = rl.Requests[j] i++ } } rl.Requests = rl.Requests[:i] } // SetKeepTime sets a duration after which an entry will be considered inactive. func (rl *ReqLog) SetKeepTime(t time.Duration) { rl.lock.Lock() defer rl.lock.Unlock() rl.keep = t } // Report generates a copy of all the entries in the requestlog. func (rl *ReqLog) Report() []*ReqLogEntry { rl.lock.Lock() defer rl.lock.Unlock() out := make([]*ReqLogEntry, len(rl.Requests)) for i, e := range rl.Requests { out[i] = e.Copy() } return out } // Finish marks an entry in the log as finished. func (rl *ReqLog) Finish(rle *ReqLogEntry) { rl.lock.Lock() defer rl.lock.Unlock() rle.Active = false rle.EndTime = time.Now() rl.maybeCleanup() } ================================================ FILE: config/addresses.go ================================================ package config // Addresses stores the (string) multiaddr addresses for the node. type Addresses struct { Swarm []string // addresses for the swarm to listen on Announce []string // swarm addresses to announce to the network, if len > 0 replaces auto detected addresses AppendAnnounce []string // similar to Announce but doesn't overwrite auto detected addresses, they are just appended NoAnnounce []string // swarm addresses not to announce to the network API Strings // address for the local API (RPC) Gateway Strings // address to listen on for IPFS HTTP object gateway } ================================================ FILE: config/api.go ================================================ package config import ( "encoding/base64" "strings" ) const ( APITag = "API" AuthorizationTag = "Authorizations" ) type RPCAuthScope struct { // AuthSecret is the secret that will be compared to the HTTP "Authorization". // header. A secret is in the format "type:value". Check the documentation for // supported types. AuthSecret string // AllowedPaths is an explicit list of RPC path prefixes to allow. // By default, none are allowed. ["/api/v0"] exposes all RPCs. AllowedPaths []string } type API struct { // HTTPHeaders are the HTTP headers to return with the API. HTTPHeaders map[string][]string // Authorization is a map of authorizations used to authenticate in the API. // If the map is empty, then the RPC API is exposed to everyone. Check the // documentation for more details. Authorizations map[string]*RPCAuthScope `json:",omitempty"` } // ConvertAuthSecret converts the given secret in the format "type:value" into an // HTTP Authorization header value. It can handle 'bearer' and 'basic' as type. // If type exists and is not known, an empty string is returned. If type does not // exist, 'bearer' type is assumed. func ConvertAuthSecret(secret string) string { if secret == "" { return secret } split := strings.SplitN(secret, ":", 2) if len(split) < 2 { // No prefix: assume bearer token. return "Bearer " + secret } if strings.HasPrefix(secret, "basic:") { if strings.Contains(split[1], ":") { // Assume basic:user:password return "Basic " + base64.StdEncoding.EncodeToString([]byte(split[1])) } else { // Assume already base64 encoded. return "Basic " + split[1] } } else if strings.HasPrefix(secret, "bearer:") { return "Bearer " + split[1] } // Unknown. Type is present, but we can't handle it. return "" } ================================================ FILE: config/api_test.go ================================================ package config import ( "testing" "github.com/stretchr/testify/assert" ) func TestConvertAuthSecret(t *testing.T) { for _, testCase := range []struct { input string output string }{ {"", ""}, {"someToken", "Bearer someToken"}, {"bearer:someToken", "Bearer someToken"}, {"basic:user:pass", "Basic dXNlcjpwYXNz"}, {"basic:dXNlcjpwYXNz", "Basic dXNlcjpwYXNz"}, } { assert.Equal(t, testCase.output, ConvertAuthSecret(testCase.input)) } } ================================================ FILE: config/autoconf.go ================================================ package config import ( "maps" "math/rand/v2" "strings" "github.com/ipfs/boxo/autoconf" logging "github.com/ipfs/go-log/v2" peer "github.com/libp2p/go-libp2p/core/peer" ) var log = logging.Logger("config") // AutoConf contains the configuration for the autoconf subsystem type AutoConf struct { // URL is the HTTP(S) URL to fetch the autoconf.json from // Default: see boxo/autoconf.MainnetAutoConfURL URL *OptionalString `json:",omitempty"` // Enabled determines whether to use autoconf // Default: true Enabled Flag `json:",omitempty"` // RefreshInterval is how often to refresh autoconf data // Default: 24h RefreshInterval *OptionalDuration `json:",omitempty"` // TLSInsecureSkipVerify allows skipping TLS verification (for testing only) // Default: false TLSInsecureSkipVerify Flag `json:",omitempty"` } const ( // AutoPlaceholder is the string used as a placeholder for autoconf values AutoPlaceholder = "auto" // DefaultAutoConfEnabled is the default value for AutoConf.Enabled DefaultAutoConfEnabled = true // DefaultAutoConfURL is the default URL for fetching autoconf DefaultAutoConfURL = autoconf.MainnetAutoConfURL // DefaultAutoConfRefreshInterval is the default interval for refreshing autoconf data DefaultAutoConfRefreshInterval = autoconf.DefaultRefreshInterval // AutoConf client configuration constants DefaultAutoConfCacheSize = autoconf.DefaultCacheSize DefaultAutoConfTimeout = autoconf.DefaultTimeout ) // getNativeSystems returns the list of systems that should be used natively based on routing type func getNativeSystems(routingType string) []string { switch routingType { case "dht", "dhtclient", "dhtserver": return []string{autoconf.SystemAminoDHT} // Only native DHT case "auto", "autoclient": return []string{autoconf.SystemAminoDHT} // Native DHT, delegated others case "delegated": return []string{} // Everything delegated case "none": return []string{} // No native systems default: return []string{} // Custom mode } } // selectRandomResolver picks a random resolver from a list for load balancing func selectRandomResolver(resolvers []string) string { if len(resolvers) == 0 { return "" } return resolvers[rand.IntN(len(resolvers))] } // DNSResolversWithAutoConf returns DNS resolvers with "auto" values replaced by autoconf values func (c *Config) DNSResolversWithAutoConf() map[string]string { if c.DNS.Resolvers == nil { return nil } resolved := make(map[string]string) autoConf := c.getAutoConf() autoExpanded := 0 // Process each configured resolver for domain, resolver := range c.DNS.Resolvers { if resolver == AutoPlaceholder { // Try to resolve from autoconf if autoConf != nil && autoConf.DNSResolvers != nil { if resolvers, exists := autoConf.DNSResolvers[domain]; exists && len(resolvers) > 0 { resolved[domain] = selectRandomResolver(resolvers) autoExpanded++ } } // If autoConf is disabled or domain not found, skip this "auto" resolver } else { // Keep custom resolver as-is resolved[domain] = resolver } } // Add default resolvers from autoconf that aren't already configured if autoConf != nil && autoConf.DNSResolvers != nil { for domain, resolvers := range autoConf.DNSResolvers { if _, exists := resolved[domain]; !exists && len(resolvers) > 0 { resolved[domain] = selectRandomResolver(resolvers) } } } // Log expansion statistics if autoExpanded > 0 { log.Debugf("expanded %d 'auto' DNS.Resolvers from autoconf", autoExpanded) } return resolved } // expandAutoConfSlice is a generic helper for expanding "auto" placeholders in string slices // It handles the common pattern of: iterate through slice, expand "auto" once, keep custom values func expandAutoConfSlice(sourceSlice []string, autoConfData []string) []string { var resolved []string autoExpanded := false for _, item := range sourceSlice { if item == AutoPlaceholder { // Replace with autoconf data (only once) if autoConfData != nil && !autoExpanded { resolved = append(resolved, autoConfData...) autoExpanded = true } // If autoConfData is nil or already expanded, skip redundant "auto" entries silently } else { // Keep custom item resolved = append(resolved, item) } } return resolved } // BootstrapWithAutoConf returns bootstrap config with "auto" values replaced by autoconf values func (c *Config) BootstrapWithAutoConf() []string { autoConf := c.getAutoConf() var autoConfData []string if autoConf != nil { routingType := c.Routing.Type.WithDefault(DefaultRoutingType) nativeSystems := getNativeSystems(routingType) autoConfData = autoConf.GetBootstrapPeers(nativeSystems...) log.Debugf("BootstrapWithAutoConf: processing with routing type: %s", routingType) } else { log.Debugf("BootstrapWithAutoConf: autoConf disabled, using original config") } result := expandAutoConfSlice(c.Bootstrap, autoConfData) log.Debugf("BootstrapWithAutoConf: final result contains %d peers", len(result)) return result } // getAutoConf is a helper to get autoconf data with fallbacks func (c *Config) getAutoConf() *autoconf.Config { if !c.AutoConf.Enabled.WithDefault(DefaultAutoConfEnabled) { log.Debugf("getAutoConf: AutoConf disabled, returning nil") return nil } // Create or get cached client with config client, err := GetAutoConfClient(c) if err != nil { log.Debugf("getAutoConf: client creation failed - %v", err) return nil } // Use GetCached to avoid network I/O during config operations // This ensures config retrieval doesn't block on network operations result := client.GetCached() log.Debugf("getAutoConf: returning autoconf data") return result } // BootstrapPeersWithAutoConf returns bootstrap peers with "auto" values replaced by autoconf values // and parsed into peer.AddrInfo structures func (c *Config) BootstrapPeersWithAutoConf() ([]peer.AddrInfo, error) { bootstrapStrings := c.BootstrapWithAutoConf() return ParseBootstrapPeers(bootstrapStrings) } // DelegatedRoutersWithAutoConf returns delegated router URLs without trailing slashes func (c *Config) DelegatedRoutersWithAutoConf() []string { autoConf := c.getAutoConf() // Use autoconf to expand the endpoints with supported paths for read operations routingType := c.Routing.Type.WithDefault(DefaultRoutingType) nativeSystems := getNativeSystems(routingType) return autoconf.ExpandDelegatedEndpoints( c.Routing.DelegatedRouters, autoConf, nativeSystems, // Kubo supports all read paths autoconf.RoutingV1ProvidersPath, autoconf.RoutingV1PeersPath, autoconf.RoutingV1IPNSPath, ) } // DelegatedPublishersWithAutoConf returns delegated publisher URLs without trailing slashes func (c *Config) DelegatedPublishersWithAutoConf() []string { autoConf := c.getAutoConf() // Use autoconf to expand the endpoints with IPNS write path routingType := c.Routing.Type.WithDefault(DefaultRoutingType) nativeSystems := getNativeSystems(routingType) return autoconf.ExpandDelegatedEndpoints( c.Ipns.DelegatedPublishers, autoConf, nativeSystems, autoconf.RoutingV1IPNSPath, // Only IPNS operations (for write) ) } // expandConfigField expands a specific config field with autoconf values // Handles both top-level fields ("Bootstrap") and nested fields ("DNS.Resolvers") func (c *Config) expandConfigField(expandedCfg map[string]any, fieldPath string) { // Check if this field supports autoconf expansion expandFunc, supported := supportedAutoConfFields[fieldPath] if !supported { return } // Handle top-level fields (no dot in path) if !strings.Contains(fieldPath, ".") { if _, exists := expandedCfg[fieldPath]; exists { expandedCfg[fieldPath] = expandFunc(c) } return } // Handle nested fields (section.field format) parts := strings.SplitN(fieldPath, ".", 2) if len(parts) != 2 { return } sectionName, fieldName := parts[0], parts[1] if section, exists := expandedCfg[sectionName]; exists { if sectionMap, ok := section.(map[string]any); ok { if _, exists := sectionMap[fieldName]; exists { sectionMap[fieldName] = expandFunc(c) expandedCfg[sectionName] = sectionMap } } } } // ExpandAutoConfValues expands "auto" placeholders in config with their actual values using the same methods as the daemon func (c *Config) ExpandAutoConfValues(cfg map[string]any) (map[string]any, error) { // Create a deep copy of the config map to avoid modifying the original expandedCfg := maps.Clone(cfg) // Use the same expansion methods that the daemon uses - ensures runtime consistency // Unified expansion for all supported autoconf fields c.expandConfigField(expandedCfg, "Bootstrap") c.expandConfigField(expandedCfg, "DNS.Resolvers") c.expandConfigField(expandedCfg, "Routing.DelegatedRouters") c.expandConfigField(expandedCfg, "Ipns.DelegatedPublishers") return expandedCfg, nil } // supportedAutoConfFields maps field keys to their expansion functions var supportedAutoConfFields = map[string]func(*Config) any{ "Bootstrap": func(c *Config) any { expanded := c.BootstrapWithAutoConf() return stringSliceToInterfaceSlice(expanded) }, "DNS.Resolvers": func(c *Config) any { expanded := c.DNSResolversWithAutoConf() return stringMapToInterfaceMap(expanded) }, "Routing.DelegatedRouters": func(c *Config) any { expanded := c.DelegatedRoutersWithAutoConf() return stringSliceToInterfaceSlice(expanded) }, "Ipns.DelegatedPublishers": func(c *Config) any { expanded := c.DelegatedPublishersWithAutoConf() return stringSliceToInterfaceSlice(expanded) }, } // ExpandConfigField expands auto values for a specific config field using the same methods as the daemon func (c *Config) ExpandConfigField(key string, value any) any { if expandFunc, supported := supportedAutoConfFields[key]; supported { return expandFunc(c) } // Return original value if no expansion needed (not a field that supports auto values) return value } // Helper functions for type conversion between string types and any types for JSON compatibility func stringSliceToInterfaceSlice(slice []string) []any { result := make([]any, len(slice)) for i, v := range slice { result[i] = v } return result } func stringMapToInterfaceMap(m map[string]string) map[string]any { result := make(map[string]any) for k, v := range m { result[k] = v } return result } ================================================ FILE: config/autoconf_client.go ================================================ package config import ( "fmt" "path/filepath" "slices" "sync" "github.com/ipfs/boxo/autoconf" logging "github.com/ipfs/go-log/v2" version "github.com/ipfs/kubo" ) var autoconfLog = logging.Logger("autoconf") // Singleton state for autoconf client var ( clientOnce sync.Once clientCache *autoconf.Client clientErr error ) // GetAutoConfClient returns a cached autoconf client or creates a new one. // This is thread-safe and uses a singleton pattern. func GetAutoConfClient(cfg *Config) (*autoconf.Client, error) { clientOnce.Do(func() { clientCache, clientErr = newAutoConfClient(cfg) }) return clientCache, clientErr } // newAutoConfClient creates a new autoconf client with the given config func newAutoConfClient(cfg *Config) (*autoconf.Client, error) { // Get repo path for cache directory repoPath, err := PathRoot() if err != nil { return nil, fmt.Errorf("failed to get repo path: %w", err) } // Prepare refresh interval with nil check refreshInterval := cfg.AutoConf.RefreshInterval if refreshInterval == nil { refreshInterval = &OptionalDuration{} } // Use default URL if not specified url := cfg.AutoConf.URL.WithDefault(DefaultAutoConfURL) // Build client options options := []autoconf.Option{ autoconf.WithCacheDir(filepath.Join(repoPath, "autoconf")), autoconf.WithUserAgent(version.GetUserAgentVersion()), autoconf.WithCacheSize(DefaultAutoConfCacheSize), autoconf.WithTimeout(DefaultAutoConfTimeout), autoconf.WithRefreshInterval(refreshInterval.WithDefault(DefaultAutoConfRefreshInterval)), autoconf.WithFallback(autoconf.GetMainnetFallbackConfig), autoconf.WithURL(url), } return autoconf.NewClient(options...) } // ValidateAutoConfWithRepo validates that autoconf setup is correct at daemon startup with repo access func ValidateAutoConfWithRepo(cfg *Config, swarmKeyExists bool) error { if !cfg.AutoConf.Enabled.WithDefault(DefaultAutoConfEnabled) { // AutoConf is disabled, check for "auto" values and warn return validateAutoConfDisabled(cfg) } // Check for private network with default mainnet URL url := cfg.AutoConf.URL.WithDefault(DefaultAutoConfURL) if swarmKeyExists && url == DefaultAutoConfURL { return fmt.Errorf("AutoConf cannot use the default mainnet URL (%s) on a private network (swarm.key or LIBP2P_FORCE_PNET detected). Either disable AutoConf by setting AutoConf.Enabled=false, or configure AutoConf.URL to point to a configuration service specific to your private swarm", DefaultAutoConfURL) } // Further validation will happen lazily when config is accessed return nil } // validateAutoConfDisabled checks for "auto" values when AutoConf is disabled and logs errors func validateAutoConfDisabled(cfg *Config) error { hasAutoValues := false var errors []string // Check Bootstrap if slices.Contains(cfg.Bootstrap, AutoPlaceholder) { hasAutoValues = true errors = append(errors, "Bootstrap contains 'auto' but AutoConf.Enabled=false") } // Check DNS.Resolvers if cfg.DNS.Resolvers != nil { for _, resolver := range cfg.DNS.Resolvers { if resolver == AutoPlaceholder { hasAutoValues = true errors = append(errors, "DNS.Resolvers contains 'auto' but AutoConf.Enabled=false") break } } } // Check Routing.DelegatedRouters if slices.Contains(cfg.Routing.DelegatedRouters, AutoPlaceholder) { hasAutoValues = true errors = append(errors, "Routing.DelegatedRouters contains 'auto' but AutoConf.Enabled=false") } // Check Ipns.DelegatedPublishers if slices.Contains(cfg.Ipns.DelegatedPublishers, AutoPlaceholder) { hasAutoValues = true errors = append(errors, "Ipns.DelegatedPublishers contains 'auto' but AutoConf.Enabled=false") } // Log all errors for _, errMsg := range errors { autoconfLog.Error(errMsg) } // If only auto values exist and no static ones, fail to start if hasAutoValues { if len(cfg.Bootstrap) == 1 && cfg.Bootstrap[0] == AutoPlaceholder { autoconfLog.Error("Kubo cannot start with only 'auto' Bootstrap values when AutoConf.Enabled=false") return fmt.Errorf("no usable bootstrap peers: AutoConf is disabled (AutoConf.Enabled=false) but 'auto' placeholder is used in Bootstrap config. Either set AutoConf.Enabled=true to enable automatic configuration, or replace 'auto' with specific Bootstrap peer addresses") } } return nil } ================================================ FILE: config/autoconf_test.go ================================================ package config import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestAutoConfDefaults(t *testing.T) { // Test that AutoConf has the correct default values cfg := &Config{ AutoConf: AutoConf{ URL: NewOptionalString(DefaultAutoConfURL), Enabled: True, }, } assert.Equal(t, DefaultAutoConfURL, cfg.AutoConf.URL.WithDefault(DefaultAutoConfURL)) assert.True(t, cfg.AutoConf.Enabled.WithDefault(DefaultAutoConfEnabled)) // Test default refresh interval if cfg.AutoConf.RefreshInterval == nil { // This is expected - nil means use default duration := (*OptionalDuration)(nil).WithDefault(DefaultAutoConfRefreshInterval) assert.Equal(t, DefaultAutoConfRefreshInterval, duration) } } func TestAutoConfProfile(t *testing.T) { cfg := &Config{ Bootstrap: []string{"some", "existing", "peers"}, DNS: DNS{ Resolvers: map[string]string{ "eth.": "https://example.com", }, }, Routing: Routing{ DelegatedRouters: []string{"https://existing.router"}, }, Ipns: Ipns{ DelegatedPublishers: []string{"https://existing.publisher"}, }, AutoConf: AutoConf{ Enabled: False, }, } // Apply autoconf profile profile, ok := Profiles["autoconf-on"] require.True(t, ok, "autoconf-on profile not found") err := profile.Transform(cfg) require.NoError(t, err) // Check that values were set to "auto" assert.Equal(t, []string{AutoPlaceholder}, cfg.Bootstrap) assert.Equal(t, AutoPlaceholder, cfg.DNS.Resolvers["."]) assert.Equal(t, []string{AutoPlaceholder}, cfg.Routing.DelegatedRouters) assert.Equal(t, []string{AutoPlaceholder}, cfg.Ipns.DelegatedPublishers) // Check that AutoConf was enabled assert.True(t, cfg.AutoConf.Enabled.WithDefault(DefaultAutoConfEnabled)) // Check that URL was set assert.Equal(t, DefaultAutoConfURL, cfg.AutoConf.URL.WithDefault(DefaultAutoConfURL)) } func TestInitWithAutoValues(t *testing.T) { identity := Identity{ PeerID: "QmTest", } cfg, err := InitWithIdentity(identity) require.NoError(t, err) // Check that Bootstrap is set to "auto" assert.Equal(t, []string{AutoPlaceholder}, cfg.Bootstrap) // Check that DNS resolver is set to "auto" assert.Equal(t, AutoPlaceholder, cfg.DNS.Resolvers["."]) // Check that DelegatedRouters is set to "auto" assert.Equal(t, []string{AutoPlaceholder}, cfg.Routing.DelegatedRouters) // Check that DelegatedPublishers is set to "auto" assert.Equal(t, []string{AutoPlaceholder}, cfg.Ipns.DelegatedPublishers) // Check that AutoConf is enabled with correct URL assert.True(t, cfg.AutoConf.Enabled.WithDefault(DefaultAutoConfEnabled)) assert.Equal(t, DefaultAutoConfURL, cfg.AutoConf.URL.WithDefault(DefaultAutoConfURL)) } ================================================ FILE: config/autonat.go ================================================ package config import ( "fmt" ) // AutoNATServiceMode configures the ipfs node's AutoNAT service. type AutoNATServiceMode int const ( // AutoNATServiceUnset indicates that the user has not set the // AutoNATService mode. // // When unset, nodes configured to be public DHT nodes will _also_ // perform limited AutoNAT dialbacks. AutoNATServiceUnset AutoNATServiceMode = iota // AutoNATServiceEnabled indicates that the user has enabled the // AutoNATService. AutoNATServiceEnabled // AutoNATServiceDisabled indicates that the user has disabled the // AutoNATService. AutoNATServiceDisabled // AutoNATServiceEnabledV1Only forces use of V1 and disables V2 // (used for testing) AutoNATServiceEnabledV1Only ) func (m *AutoNATServiceMode) UnmarshalText(text []byte) error { switch string(text) { case "": *m = AutoNATServiceUnset case "enabled": *m = AutoNATServiceEnabled case "disabled": *m = AutoNATServiceDisabled case "legacy-v1": *m = AutoNATServiceEnabledV1Only default: return fmt.Errorf("unknown autonat mode: %s", string(text)) } return nil } func (m AutoNATServiceMode) MarshalText() ([]byte, error) { switch m { case AutoNATServiceUnset: return nil, nil case AutoNATServiceEnabled: return []byte("enabled"), nil case AutoNATServiceDisabled: return []byte("disabled"), nil case AutoNATServiceEnabledV1Only: return []byte("legacy-v1"), nil default: return nil, fmt.Errorf("unknown autonat mode: %d", m) } } // AutoNATConfig configures the node's AutoNAT subsystem. type AutoNATConfig struct { // ServiceMode configures the node's AutoNAT service mode. ServiceMode AutoNATServiceMode `json:",omitempty"` // Throttle configures AutoNAT dialback throttling. // // If unset, the conservative libp2p defaults will be unset. To help the // network, please consider setting this and increasing the limits. // // By default, the limits will be a total of 30 dialbacks, with a // per-peer max of 3 peer, resetting every minute. Throttle *AutoNATThrottleConfig `json:",omitempty"` } // AutoNATThrottleConfig configures the throttle limites. type AutoNATThrottleConfig struct { // GlobalLimit and PeerLimit sets the global and per-peer dialback // limits. The AutoNAT service will only perform the specified number of // dialbacks per interval. // // Setting either to 0 will disable the appropriate limit. GlobalLimit, PeerLimit int // Interval specifies how frequently this node should reset the // global/peer dialback limits. // // When unset, this defaults to 1 minute. Interval OptionalDuration } ================================================ FILE: config/autotls.go ================================================ package config import ( "time" p2pforge "github.com/ipshipyard/p2p-forge/client" ) // AutoTLS includes optional configuration of p2p-forge client of service // for obtaining a domain and TLS certificate to improve connectivity for web // browser clients. More: https://github.com/ipshipyard/p2p-forge#readme type AutoTLS struct { // Enables the p2p-forge feature and all related features. Enabled Flag `json:",omitempty"` // Optional, controls if Kubo should add /tls/sni/.../ws listener to every /tcp port if no explicit /ws is defined in Addresses.Swarm AutoWSS Flag `json:",omitempty"` // Optional, controls whether to skip network DNS lookups for p2p-forge domains. // Applies to resolution via DNS.Resolvers, including /dns* multiaddrs in go-libp2p. // When enabled (default), A/AAAA queries for *.libp2p.direct are resolved // locally by parsing the IP directly from the hostname, avoiding network I/O. // Set to false to always use network DNS (useful for debugging). SkipDNSLookup Flag `json:",omitempty"` // Optional override of the parent domain that will be used DomainSuffix *OptionalString `json:",omitempty"` // Optional override of HTTP API that acts as ACME DNS-01 Challenge broker RegistrationEndpoint *OptionalString `json:",omitempty"` // Optional Authorization token, used with private/test instances of p2p-forge RegistrationToken *OptionalString `json:",omitempty"` // Optional registration delay used when AutoTLS.Enabled is not explicitly set to true in config RegistrationDelay *OptionalDuration `json:",omitempty"` // Optional override of CA ACME API used by p2p-forge system CAEndpoint *OptionalString `json:",omitempty"` // Optional, controls if features like AutoWSS should generate shorter /dnsX instead of /ipX/../sni/.. ShortAddrs Flag `json:",omitempty"` } const ( DefaultAutoTLSEnabled = true // with DefaultAutoTLSRegistrationDelay, unless explicitly enabled in config DefaultDomainSuffix = p2pforge.DefaultForgeDomain DefaultRegistrationEndpoint = p2pforge.DefaultForgeEndpoint DefaultCAEndpoint = p2pforge.DefaultCAEndpoint DefaultAutoWSS = true // requires AutoTLS.Enabled DefaultAutoTLSShortAddrs = true // requires AutoTLS.Enabled DefaultAutoTLSSkipDNSLookup = true // skip network DNS for p2p-forge domains DefaultAutoTLSRegistrationDelay = 1 * time.Hour ) ================================================ FILE: config/bitswap.go ================================================ package config // Bitswap holds Bitswap configuration options type Bitswap struct { // Libp2pEnabled controls if the node initializes bitswap over libp2p (enabled by default) // (This can be disabled if HTTPRetrieval.Enabled is set to true) Libp2pEnabled Flag `json:",omitempty"` // ServerEnabled controls if the node responds to WANTs (depends on Libp2pEnabled, enabled by default) ServerEnabled Flag `json:",omitempty"` } const ( DefaultBitswapLibp2pEnabled = true DefaultBitswapServerEnabled = true ) ================================================ FILE: config/bootstrap_peers.go ================================================ package config import ( "errors" peer "github.com/libp2p/go-libp2p/core/peer" ma "github.com/multiformats/go-multiaddr" ) // ErrInvalidPeerAddr signals an address is not a valid peer address. var ErrInvalidPeerAddr = errors.New("invalid peer address") func (c *Config) BootstrapPeers() ([]peer.AddrInfo, error) { return ParseBootstrapPeers(c.Bootstrap) } func (c *Config) SetBootstrapPeers(bps []peer.AddrInfo) { c.Bootstrap = BootstrapPeerStrings(bps) } // ParseBootstrapPeers parses a bootstrap list into a list of AddrInfos. func ParseBootstrapPeers(addrs []string) ([]peer.AddrInfo, error) { maddrs := make([]ma.Multiaddr, len(addrs)) for i, addr := range addrs { var err error maddrs[i], err = ma.NewMultiaddr(addr) if err != nil { return nil, err } } return peer.AddrInfosFromP2pAddrs(maddrs...) } // BootstrapPeerStrings formats a list of AddrInfos as a bootstrap peer list // suitable for serialization. func BootstrapPeerStrings(bps []peer.AddrInfo) []string { bpss := make([]string, 0, len(bps)) for _, pi := range bps { addrs, err := peer.AddrInfoToP2pAddrs(&pi) if err != nil { // programmer error. panic(err) } for _, addr := range addrs { bpss = append(bpss, addr.String()) } } return bpss } ================================================ FILE: config/bootstrap_peers_test.go ================================================ package config import ( "testing" "github.com/ipfs/boxo/autoconf" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestBootstrapPeerStrings(t *testing.T) { // Test round-trip: string -> parse -> format -> string // This ensures that parsing and formatting are inverse operations // Start with the default bootstrap peer multiaddr strings originalStrings := autoconf.FallbackBootstrapPeers // Parse multiaddr strings into structured peer data parsed, err := ParseBootstrapPeers(originalStrings) require.NoError(t, err, "parsing bootstrap peers should succeed") // Format the parsed data back into multiaddr strings formattedStrings := BootstrapPeerStrings(parsed) // Verify round-trip: we should get back exactly what we started with assert.ElementsMatch(t, originalStrings, formattedStrings, "round-trip through parse/format should preserve all bootstrap peers") } ================================================ FILE: config/config.go ================================================ // package config implements the ipfs config file datastructures and utilities. package config import ( "bytes" "encoding/json" "fmt" "os" "path/filepath" "reflect" "strings" "github.com/ipfs/kubo/misc/fsutil" ) // Config is used to load ipfs config files. type Config struct { Identity Identity // local node's peer identity Datastore Datastore // local node's storage Addresses Addresses // local node's addresses Mounts Mounts // local node's mount points Discovery Discovery // local node's discovery mechanisms Routing Routing // local node's routing settings Ipns Ipns // Ipns settings Bootstrap []string // local nodes's bootstrap peer addresses Gateway Gateway // local node's gateway server options API API // local node's API settings Swarm SwarmConfig AutoNAT AutoNATConfig AutoTLS AutoTLS Pubsub PubsubConfig Peering Peering DNS DNS Migration Migration AutoConf AutoConf Provide Provide // Merged Provider and Reprovider configuration Provider Provider // Deprecated: use Provide. Will be removed in a future release. Reprovider Reprovider // Deprecated: use Provide. Will be removed in a future release. HTTPRetrieval HTTPRetrieval Experimental Experiments Plugins Plugins Pinning Pinning Import Import Version Version Internal Internal // experimental/unstable options Bitswap Bitswap } const ( // DefaultPathName is the default config dir name. DefaultPathName = ".ipfs" // DefaultPathRoot is the path to the default config dir location. DefaultPathRoot = "~/" + DefaultPathName // DefaultConfigFile is the filename of the configuration file. DefaultConfigFile = "config" // EnvDir is the environment variable used to change the path root. EnvDir = "IPFS_PATH" ) // PathRoot returns the default configuration root directory. func PathRoot() (string, error) { dir := os.Getenv(EnvDir) var err error if len(dir) == 0 { dir, err = fsutil.ExpandHome(DefaultPathRoot) } return dir, err } // Path returns the path `extension` relative to the configuration root. If an // empty string is provided for `configroot`, the default root is used. func Path(configroot, extension string) (string, error) { if len(configroot) == 0 { dir, err := PathRoot() if err != nil { return "", err } return filepath.Join(dir, extension), nil } return filepath.Join(configroot, extension), nil } // Filename returns the configuration file path given a configuration root // directory and a user-provided configuration file path argument with the // following rules: // - If the user-provided configuration file path is empty, use the default one. // - If the configuration root directory is empty, use the default one. // - If the user-provided configuration file path is only a file name, use the // configuration root directory, otherwise use only the user-provided path // and ignore the configuration root. func Filename(configroot, userConfigFile string) (string, error) { if userConfigFile == "" { return Path(configroot, DefaultConfigFile) } if filepath.Dir(userConfigFile) == "." { return Path(configroot, userConfigFile) } return userConfigFile, nil } // HumanOutput gets a config value ready for printing. func HumanOutput(value any) ([]byte, error) { s, ok := value.(string) if ok { return []byte(strings.Trim(s, "\n")), nil } return Marshal(value) } // Marshal configuration with JSON. func Marshal(value any) ([]byte, error) { // need to prettyprint, hence MarshalIndent, instead of Encoder return json.MarshalIndent(value, "", " ") } func FromMap(v map[string]any) (*Config, error) { buf := new(bytes.Buffer) if err := json.NewEncoder(buf).Encode(v); err != nil { return nil, err } var conf Config if err := json.NewDecoder(buf).Decode(&conf); err != nil { return nil, fmt.Errorf("failure to decode config: %w", err) } return &conf, nil } func ToMap(conf *Config) (map[string]any, error) { buf := new(bytes.Buffer) if err := json.NewEncoder(buf).Encode(conf); err != nil { return nil, err } var m map[string]any if err := json.NewDecoder(buf).Decode(&m); err != nil { return nil, fmt.Errorf("failure to decode config: %w", err) } return m, nil } // Convert config to a map, without using encoding/json, since // zero/empty/'omitempty' fields are excluded by encoding/json during // marshaling. func ReflectToMap(conf any) any { v := reflect.ValueOf(conf) if !v.IsValid() { return nil } // Handle pointer type if v.Kind() == reflect.Pointer { if v.IsNil() { // Create a zero value of the pointer's element type elemType := v.Type().Elem() zero := reflect.Zero(elemType) return ReflectToMap(zero.Interface()) } v = v.Elem() } switch v.Kind() { case reflect.Struct: result := make(map[string]any) t := v.Type() for i := 0; i < v.NumField(); i++ { field := v.Field(i) // Only include exported fields if field.CanInterface() { result[t.Field(i).Name] = ReflectToMap(field.Interface()) } } return result case reflect.Map: result := make(map[string]any) iter := v.MapRange() for iter.Next() { key := iter.Key() // Convert map keys to strings for consistency keyStr := fmt.Sprint(ReflectToMap(key.Interface())) result[keyStr] = ReflectToMap(iter.Value().Interface()) } // Add a sample to differentiate between a map and a struct on validation. sample := reflect.Zero(v.Type().Elem()) if sample.CanInterface() { result["*"] = ReflectToMap(sample.Interface()) } return result case reflect.Slice, reflect.Array: result := make([]any, v.Len()) for i := 0; i < v.Len(); i++ { result[i] = ReflectToMap(v.Index(i).Interface()) } return result default: // For basic types (int, string, etc.), just return the value if v.CanInterface() { return v.Interface() } return nil } } // Clone copies the config. Use when updating. func (c *Config) Clone() (*Config, error) { var newConfig Config var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(c); err != nil { return nil, fmt.Errorf("failure to encode config: %w", err) } if err := json.NewDecoder(&buf).Decode(&newConfig); err != nil { return nil, fmt.Errorf("failure to decode config: %w", err) } return &newConfig, nil } // Check if the provided key is present in the structure. func CheckKey(key string) error { conf := Config{} // Convert an empty config to a map without JSON. cursor := ReflectToMap(&conf) // Parse the key and verify it's presence in the map. var ok bool var mapCursor map[string]any parts := strings.Split(key, ".") for i, part := range parts { mapCursor, ok = cursor.(map[string]any) if !ok { if cursor == nil { return nil } path := strings.Join(parts[:i], ".") return fmt.Errorf("%s key is not a map", path) } cursor, ok = mapCursor[part] if !ok { // If the config sections is a map, validate against the default entry. if cursor, ok = mapCursor["*"]; ok { continue } path := strings.Join(parts[:i+1], ".") return fmt.Errorf("%s not found", path) } } return nil } ================================================ FILE: config/config_test.go ================================================ package config import ( "testing" ) func TestClone(t *testing.T) { c := new(Config) c.Identity.PeerID = "faketest" c.API.HTTPHeaders = map[string][]string{"foo": {"bar"}} newCfg, err := c.Clone() if err != nil { t.Fatal(err) } if newCfg.Identity.PeerID != c.Identity.PeerID { t.Fatal("peer ID not preserved") } c.API.HTTPHeaders["foo"] = []string{"baz"} if newCfg.API.HTTPHeaders["foo"][0] != "bar" { t.Fatal("HTTP headers not preserved") } delete(c.API.HTTPHeaders, "foo") if newCfg.API.HTTPHeaders["foo"][0] != "bar" { t.Fatal("HTTP headers not preserved") } } func TestReflectToMap(t *testing.T) { // Helper function to create a test config with various field types reflectedConfig := ReflectToMap(new(Config)) mapConfig, ok := reflectedConfig.(map[string]any) if !ok { t.Fatal("Config didn't convert to map") } reflectedIdentity, ok := mapConfig["Identity"] if !ok { t.Fatal("Identity field not found") } mapIdentity, ok := reflectedIdentity.(map[string]any) if !ok { t.Fatal("Identity field didn't convert to map") } // Test string field reflection reflectedPeerID, ok := mapIdentity["PeerID"] if !ok { t.Fatal("PeerID field not found in Identity") } if _, ok := reflectedPeerID.(string); !ok { t.Fatal("PeerID field didn't convert to string") } // Test omitempty json string field reflectedPrivKey, ok := mapIdentity["PrivKey"] if !ok { t.Fatal("PrivKey omitempty field not found in Identity") } if _, ok := reflectedPrivKey.(string); !ok { t.Fatal("PrivKey omitempty field didn't convert to string") } // Test slices field reflectedBootstrap, ok := mapConfig["Bootstrap"] if !ok { t.Fatal("Bootstrap field not found in config") } bootstrap, ok := reflectedBootstrap.([]any) if !ok { t.Fatal("Bootstrap field didn't convert to []string") } if len(bootstrap) != 0 { t.Fatal("Bootstrap len is incorrect") } reflectedDatastore, ok := mapConfig["Datastore"] if !ok { t.Fatal("Datastore field not found in config") } datastore, ok := reflectedDatastore.(map[string]any) if !ok { t.Fatal("Datastore field didn't convert to map") } storageGCWatermark, ok := datastore["StorageGCWatermark"] if !ok { t.Fatal("StorageGCWatermark field not found in Datastore") } // Test int field if _, ok := storageGCWatermark.(int64); !ok { t.Fatal("StorageGCWatermark field didn't convert to int64") } noSync, ok := datastore["NoSync"] if !ok { t.Fatal("NoSync field not found in Datastore") } // Test bool field if _, ok := noSync.(bool); !ok { t.Fatal("NoSync field didn't convert to bool") } reflectedDNS, ok := mapConfig["DNS"] if !ok { t.Fatal("DNS field not found in config") } DNS, ok := reflectedDNS.(map[string]any) if !ok { t.Fatal("DNS field didn't convert to map") } reflectedResolvers, ok := DNS["Resolvers"] if !ok { t.Fatal("Resolvers field not found in DNS") } // Test map field if _, ok := reflectedResolvers.(map[string]any); !ok { t.Fatal("Resolvers field didn't convert to map") } // Test pointer field if _, ok := DNS["MaxCacheTTL"].(map[string]any); !ok { // Since OptionalDuration only field is private, we cannot test it t.Fatal("MaxCacheTTL field didn't convert to map") } } // Test validation of options set through "ipfs config" func TestCheckKey(t *testing.T) { err := CheckKey("Foo.Bar") if err == nil { t.Fatal("Foo.Bar isn't a valid key in the config") } err = CheckKey("Provide.Strategy") if err != nil { t.Fatalf("%s: %s", err, "Provide.Strategy is a valid key in the config") } err = CheckKey("Provide.DHT.MaxWorkers") if err != nil { t.Fatalf("%s: %s", err, "Provide.DHT.MaxWorkers is a valid key in the config") } err = CheckKey("Provide.DHT.Interval") if err != nil { t.Fatalf("%s: %s", err, "Provide.DHT.Interval is a valid key in the config") } err = CheckKey("Provide.Foo") if err == nil { t.Fatal("Provide.Foo isn't a valid key in the config") } err = CheckKey("Gateway.PublicGateways.Foo.Paths") if err != nil { t.Fatalf("%s: %s", err, "Gateway.PublicGateways.Foo.Paths is a valid key in the config") } err = CheckKey("Gateway.PublicGateways.Foo.Bar") if err == nil { t.Fatal("Gateway.PublicGateways.Foo.Bar isn't a valid key in the config") } err = CheckKey("Plugins.Plugins.peerlog.Config.Enabled") if err != nil { t.Fatalf("%s: %s", err, "Plugins.Plugins.peerlog.Config.Enabled is a valid key in the config") } } ================================================ FILE: config/datastore.go ================================================ package config import ( "encoding/json" ) const ( // DefaultDataStoreDirectory is the directory to store all the local IPFS data. DefaultDataStoreDirectory = "datastore" // DefaultBlockKeyCacheSize is the size for the blockstore two-queue // cache which caches block keys and sizes. DefaultBlockKeyCacheSize = 64 << 10 // DefaultWriteThrough specifies whether to use a "write-through" // Blockstore and Blockservice. This means that they will write // without performing any reads to check if the incoming blocks are // already present in the datastore. Enable for datastores with fast // writes and slower reads. DefaultWriteThrough bool = true ) // Datastore tracks the configuration of the datastore. type Datastore struct { StorageMax string // in B, kB, kiB, MB, ... StorageGCWatermark int64 // in percentage to multiply on StorageMax GCPeriod string // in ns, us, ms, s, m, h // deprecated fields, use Spec Type string `json:",omitempty"` Path string `json:",omitempty"` NoSync bool `json:",omitempty"` Params *json.RawMessage `json:",omitempty"` Spec map[string]any HashOnRead bool BloomFilterSize int BlockKeyCacheSize OptionalInteger WriteThrough Flag `json:",omitempty"` } // DataStorePath returns the default data store path given a configuration root // (set an empty string to have the default configuration root). func DataStorePath(configroot string) (string, error) { return Path(configroot, DefaultDataStoreDirectory) } ================================================ FILE: config/discovery.go ================================================ package config type Discovery struct { MDNS MDNS } type MDNS struct { Enabled bool } ================================================ FILE: config/dns.go ================================================ package config // DNS specifies DNS resolution rules using custom resolvers. type DNS struct { // Resolvers is a map of FQDNs to URLs for custom DNS resolution. // URLs starting with `https://` indicate DoH endpoints. // Support for other resolver types can be added in the future. // https://en.wikipedia.org/wiki/Fully_qualified_domain_name // https://en.wikipedia.org/wiki/DNS_over_HTTPS // // Example: // - Custom resolver for ENS: `eth.` → `https://dns.eth.limo/dns-query` // - Override the default OS resolver: `.` → `https://1.1.1.1/dns-query` Resolvers map[string]string // MaxCacheTTL is the maximum duration DNS entries are valid in the cache. MaxCacheTTL *OptionalDuration `json:",omitempty"` } ================================================ FILE: config/experiments.go ================================================ package config type Experiments struct { FilestoreEnabled bool UrlstoreEnabled bool ShardingEnabled bool `json:",omitempty"` // deprecated by autosharding: https://github.com/ipfs/kubo/pull/8527 Libp2pStreamMounting bool P2pHttpProxy bool //nolint StrategicProviding bool `json:",omitempty"` // removed, use Provider.Enabled instead OptimisticProvide bool OptimisticProvideJobsPoolSize int GatewayOverLibp2p bool `json:",omitempty"` GraphsyncEnabled graphsyncEnabled `json:",omitempty"` AcceleratedDHTClient experimentalAcceleratedDHTClient `json:",omitempty"` } ================================================ FILE: config/gateway.go ================================================ package config import ( "github.com/ipfs/boxo/gateway" ) const ( DefaultInlineDNSLink = false DefaultDeserializedResponses = true DefaultDisableHTMLErrors = false DefaultExposeRoutingAPI = true DefaultDiagnosticServiceURL = "https://check.ipfs.network" DefaultAllowCodecConversion = false // Gateway limit defaults from boxo DefaultRetrievalTimeout = gateway.DefaultRetrievalTimeout DefaultMaxRequestDuration = gateway.DefaultMaxRequestDuration DefaultMaxConcurrentRequests = gateway.DefaultMaxConcurrentRequests DefaultMaxRangeRequestFileSize = 0 // 0 means no limit ) type GatewaySpec struct { // Paths is explicit list of path prefixes that should be handled by // this gateway. Example: `["/ipfs", "/ipns"]` Paths []string // UseSubdomains indicates whether or not this gateway uses subdomains // for IPFS resources instead of paths. That is: http://CID.ipfs.GATEWAY/... // // If this flag is set, any /ipns/$id and/or /ipfs/$id paths in Paths // will be permanently redirected to http://$id.[ipns|ipfs].$gateway/. // // We do not support using both paths and subdomains for a single domain // for security reasons (Origin isolation). UseSubdomains bool // NoDNSLink configures this gateway to _not_ resolve DNSLink for the FQDN // provided in `Host` HTTP header. NoDNSLink bool // InlineDNSLink configures this gateway to always inline DNSLink names // (FQDN) into a single DNS label in order to interop with wildcard TLS certs // and Origin per CID isolation provided by rules like https://publicsuffix.org InlineDNSLink Flag // DeserializedResponses configures this gateway to respond to deserialized // responses. Disabling this option enables a Trustless Gateway, as per: // https://specs.ipfs.tech/http-gateways/trustless-gateway/. DeserializedResponses Flag } // Gateway contains options for the HTTP gateway server. type Gateway struct { // HTTPHeaders configures the headers that should be returned by this // gateway. HTTPHeaders map[string][]string // HTTP headers to return with the gateway // RootRedirect is the path to which requests to `/` on this gateway // should be redirected. RootRedirect string // NoFetch configures the gateway to _not_ fetch blocks in response to // requests. NoFetch bool // NoDNSLink configures the gateway to _not_ perform DNS TXT record // lookups in response to requests with values in `Host` HTTP header. // This flag can be overridden per FQDN in PublicGateways. NoDNSLink bool // DeserializedResponses configures this gateway to respond to deserialized // requests. Disabling this option enables a Trustless only gateway, as per: // https://specs.ipfs.tech/http-gateways/trustless-gateway/. This can // be overridden per FQDN in PublicGateways. DeserializedResponses Flag // AllowCodecConversion enables automatic conversion between codecs when // the requested format differs from the block's native codec (e.g., // converting dag-pb or dag-cbor to dag-json). When disabled, the gateway // returns 406 Not Acceptable for codec mismatches per IPIP-524. AllowCodecConversion Flag // DisableHTMLErrors disables pretty HTML pages when an error occurs. Instead, a `text/plain` // page will be sent with the raw error message. DisableHTMLErrors Flag // PublicGateways configures behavior of known public gateways. // Each key is a fully qualified domain name (FQDN). PublicGateways map[string]*GatewaySpec // ExposeRoutingAPI configures the gateway port to expose // routing system as HTTP API at /routing/v1 (https://specs.ipfs.tech/routing/http-routing-v1/). ExposeRoutingAPI Flag // RetrievalTimeout enforces a maximum duration for content retrieval: // - Time to first byte: If the gateway cannot start writing the response within // this duration (e.g., stuck searching for providers), a 504 Gateway Timeout // is returned. // - Time between writes: After the first byte, the timeout resets each time new // bytes are written to the client. If the gateway cannot write additional data // within this duration after the last successful write, the response is terminated. // This helps free resources when the gateway gets stuck looking for providers // or cannot retrieve the requested content. // A value of 0 disables this timeout. RetrievalTimeout *OptionalDuration `json:",omitempty"` // MaxRequestDuration is an absolute deadline for the entire request. // Unlike RetrievalTimeout (which resets on each data write and catches // stalled transfers), this is a hard limit on the total time a request // can take. Returns 504 Gateway Timeout when exceeded. // This protects the gateway from edge cases and slow client attacks. // A value of 0 uses the default (1 hour). MaxRequestDuration *OptionalDuration `json:",omitempty"` // MaxConcurrentRequests limits concurrent HTTP requests handled by the gateway. // Requests beyond this limit receive 429 Too Many Requests with Retry-After header. // A value of 0 disables the limit. MaxConcurrentRequests *OptionalInteger `json:",omitempty"` // MaxRangeRequestFileSize limits the maximum file size for HTTP range requests. // Range requests for files larger than this limit return 501 Not Implemented. // This protects against CDN issues with large file range requests and prevents // excessive bandwidth consumption. A value of 0 disables the limit. MaxRangeRequestFileSize *OptionalBytes `json:",omitempty"` // DiagnosticServiceURL is the URL for a service to diagnose CID retrievability issues. // When the gateway returns a 504 Gateway Timeout error, an "Inspect retrievability of CID" // button will be shown that links to this service with the CID appended as ?cid=. // Set to empty string to disable the button. DiagnosticServiceURL *OptionalString `json:",omitempty"` } ================================================ FILE: config/http_retrieval.go ================================================ package config // HTTPRetrieval is the configuration object for HTTP Retrieval settings. // Implicit defaults can be found in core/node/bitswap.go type HTTPRetrieval struct { Enabled Flag `json:",omitempty"` Allowlist []string `json:",omitempty"` Denylist []string `json:",omitempty"` NumWorkers *OptionalInteger `json:",omitempty"` MaxBlockSize *OptionalString `json:",omitempty"` TLSInsecureSkipVerify Flag `json:",omitempty"` } const ( DefaultHTTPRetrievalEnabled = true DefaultHTTPRetrievalNumWorkers = 16 DefaultHTTPRetrievalTLSInsecureSkipVerify = false // only for testing with self-signed HTTPS certs DefaultHTTPRetrievalMaxBlockSize = "2MiB" // matching bitswap: https://specs.ipfs.tech/bitswap-protocol/#block-sizes ) ================================================ FILE: config/identity.go ================================================ package config import ( "encoding/base64" ic "github.com/libp2p/go-libp2p/core/crypto" ) const ( IdentityTag = "Identity" PrivKeyTag = "PrivKey" PrivKeySelector = IdentityTag + "." + PrivKeyTag ) // Identity tracks the configuration of the local node's identity. type Identity struct { PeerID string PrivKey string `json:",omitempty"` } // DecodePrivateKey is a helper to decode the users PrivateKey. func (i *Identity) DecodePrivateKey(passphrase string) (ic.PrivKey, error) { pkb, err := base64.StdEncoding.DecodeString(i.PrivKey) if err != nil { return nil, err } // currently storing key unencrypted. in the future we need to encrypt it. // TODO(security) return ic.UnmarshalPrivateKey(pkb) } ================================================ FILE: config/import.go ================================================ package config import ( "fmt" "io" "strconv" "strings" chunk "github.com/ipfs/boxo/chunker" "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" uio "github.com/ipfs/boxo/ipld/unixfs/io" "github.com/ipfs/boxo/verifcid" mh "github.com/multiformats/go-multihash" ) const ( DefaultCidVersion = 0 DefaultUnixFSRawLeaves = false DefaultUnixFSChunker = "size-262144" DefaultHashFunction = "sha2-256" DefaultFastProvideRoot = true DefaultFastProvideWait = false DefaultUnixFSHAMTDirectorySizeThreshold = 262144 // 256KiB - https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L26 // DefaultBatchMaxNodes controls the maximum number of nodes in a // write-batch. The total size of the batch is limited by // BatchMaxnodes and BatchMaxSize. DefaultBatchMaxNodes = 128 // DefaultBatchMaxSize controls the maximum size of a single // write-batch. The total size of the batch is limited by // BatchMaxnodes and BatchMaxSize. DefaultBatchMaxSize = 100 << 20 // 20MiB // HAMTSizeEstimation values for Import.UnixFSHAMTDirectorySizeEstimation HAMTSizeEstimationLinks = "links" // legacy: estimate using link names + CID byte lengths (default) HAMTSizeEstimationBlock = "block" // full serialized dag-pb block size HAMTSizeEstimationDisabled = "disabled" // disable HAMT sharding entirely // DAGLayout values for Import.UnixFSDAGLayout DAGLayoutBalanced = "balanced" // balanced DAG layout (default) DAGLayoutTrickle = "trickle" // trickle DAG layout DefaultUnixFSHAMTDirectorySizeEstimation = HAMTSizeEstimationLinks // legacy behavior DefaultUnixFSDAGLayout = DAGLayoutBalanced // balanced DAG layout DefaultUnixFSIncludeEmptyDirs = true // include empty directories ) var ( DefaultUnixFSFileMaxLinks = int64(helpers.DefaultLinksPerBlock) DefaultUnixFSDirectoryMaxLinks = int64(0) DefaultUnixFSHAMTDirectoryMaxFanout = int64(uio.DefaultShardWidth) ) // Import configures the default options for ingesting data. This affects commands // that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'. type Import struct { CidVersion OptionalInteger UnixFSRawLeaves Flag UnixFSChunker OptionalString HashFunction OptionalString UnixFSFileMaxLinks OptionalInteger UnixFSDirectoryMaxLinks OptionalInteger UnixFSHAMTDirectoryMaxFanout OptionalInteger UnixFSHAMTDirectorySizeThreshold OptionalBytes UnixFSHAMTDirectorySizeEstimation OptionalString // "links", "block", or "disabled" UnixFSDAGLayout OptionalString // "balanced" or "trickle" BatchMaxNodes OptionalInteger BatchMaxSize OptionalInteger FastProvideRoot Flag FastProvideWait Flag } // ValidateImportConfig validates the Import configuration according to UnixFS spec requirements. // See: https://specs.ipfs.tech/unixfs/#hamt-structure-and-parameters func ValidateImportConfig(cfg *Import) error { // Validate CidVersion if !cfg.CidVersion.IsDefault() { cidVer := cfg.CidVersion.WithDefault(DefaultCidVersion) if cidVer != 0 && cidVer != 1 { return fmt.Errorf("Import.CidVersion must be 0 or 1, got %d", cidVer) } } // Validate UnixFSFileMaxLinks if !cfg.UnixFSFileMaxLinks.IsDefault() { maxLinks := cfg.UnixFSFileMaxLinks.WithDefault(DefaultUnixFSFileMaxLinks) if maxLinks <= 0 { return fmt.Errorf("Import.UnixFSFileMaxLinks must be positive, got %d", maxLinks) } } // Validate UnixFSDirectoryMaxLinks if !cfg.UnixFSDirectoryMaxLinks.IsDefault() { maxLinks := cfg.UnixFSDirectoryMaxLinks.WithDefault(DefaultUnixFSDirectoryMaxLinks) if maxLinks < 0 { return fmt.Errorf("Import.UnixFSDirectoryMaxLinks must be non-negative, got %d", maxLinks) } } // Validate UnixFSHAMTDirectoryMaxFanout if set if !cfg.UnixFSHAMTDirectoryMaxFanout.IsDefault() { fanout := cfg.UnixFSHAMTDirectoryMaxFanout.WithDefault(DefaultUnixFSHAMTDirectoryMaxFanout) // Valid values are powers of 2 between 8 and 1024: 8, 16, 32, 64, 128, 256, 512, 1024 if fanout < 8 || !isPowerOfTwo(fanout) || fanout > 1024 { return fmt.Errorf("Import.UnixFSHAMTDirectoryMaxFanout must be a power of 2, between 8 and 1024 (got %d)", fanout) } } // Validate BatchMaxNodes if !cfg.BatchMaxNodes.IsDefault() { maxNodes := cfg.BatchMaxNodes.WithDefault(DefaultBatchMaxNodes) if maxNodes <= 0 { return fmt.Errorf("Import.BatchMaxNodes must be positive, got %d", maxNodes) } } // Validate BatchMaxSize if !cfg.BatchMaxSize.IsDefault() { maxSize := cfg.BatchMaxSize.WithDefault(DefaultBatchMaxSize) if maxSize <= 0 { return fmt.Errorf("Import.BatchMaxSize must be positive, got %d", maxSize) } } // Validate UnixFSChunker format if !cfg.UnixFSChunker.IsDefault() { chunker := cfg.UnixFSChunker.WithDefault(DefaultUnixFSChunker) if !isValidChunker(chunker) { return fmt.Errorf("Import.UnixFSChunker invalid format: %q (expected \"size-\", \"rabin---\", or \"buzhash\")", chunker) } } // Validate HashFunction if !cfg.HashFunction.IsDefault() { hashFunc := cfg.HashFunction.WithDefault(DefaultHashFunction) hashCode, ok := mh.Names[strings.ToLower(hashFunc)] if !ok { return fmt.Errorf("Import.HashFunction unrecognized: %q", hashFunc) } // Check if the hash is allowed by verifcid if !verifcid.DefaultAllowlist.IsAllowed(hashCode) { return fmt.Errorf("Import.HashFunction %q is not allowed for use in IPFS", hashFunc) } } // Validate UnixFSHAMTDirectorySizeEstimation if !cfg.UnixFSHAMTDirectorySizeEstimation.IsDefault() { est := cfg.UnixFSHAMTDirectorySizeEstimation.WithDefault(DefaultUnixFSHAMTDirectorySizeEstimation) switch est { case HAMTSizeEstimationLinks, HAMTSizeEstimationBlock, HAMTSizeEstimationDisabled: // valid default: return fmt.Errorf("Import.UnixFSHAMTDirectorySizeEstimation must be %q, %q, or %q, got %q", HAMTSizeEstimationLinks, HAMTSizeEstimationBlock, HAMTSizeEstimationDisabled, est) } } // Validate UnixFSDAGLayout if !cfg.UnixFSDAGLayout.IsDefault() { layout := cfg.UnixFSDAGLayout.WithDefault(DefaultUnixFSDAGLayout) switch layout { case DAGLayoutBalanced, DAGLayoutTrickle: // valid default: return fmt.Errorf("Import.UnixFSDAGLayout must be %q or %q, got %q", DAGLayoutBalanced, DAGLayoutTrickle, layout) } } return nil } // isPowerOfTwo checks if a number is a power of 2 func isPowerOfTwo(n int64) bool { return n > 0 && (n&(n-1)) == 0 } // isValidChunker validates chunker format func isValidChunker(chunker string) bool { if chunker == "buzhash" { return true } // Check for size- format if sizeStr, ok := strings.CutPrefix(chunker, "size-"); ok { if sizeStr == "" { return false } // Check if it's a valid positive integer (no negative sign allowed) if sizeStr[0] == '-' { return false } size, err := strconv.Atoi(sizeStr) // Size must be positive (not zero) return err == nil && size > 0 } // Check for rabin--- format if strings.HasPrefix(chunker, "rabin-") { parts := strings.Split(chunker, "-") if len(parts) != 4 { return false } // Parse and validate min, avg, max values values := make([]int, 3) for i := range 3 { val, err := strconv.Atoi(parts[i+1]) if err != nil { return false } values[i] = val } // Validate ordering: min <= avg <= max min, avg, max := values[0], values[1], values[2] return min <= avg && avg <= max } return false } // HAMTSizeEstimationMode returns the boxo SizeEstimationMode based on the config value. func (i *Import) HAMTSizeEstimationMode() uio.SizeEstimationMode { switch i.UnixFSHAMTDirectorySizeEstimation.WithDefault(DefaultUnixFSHAMTDirectorySizeEstimation) { case HAMTSizeEstimationLinks: return uio.SizeEstimationLinks case HAMTSizeEstimationBlock: return uio.SizeEstimationBlock case HAMTSizeEstimationDisabled: return uio.SizeEstimationDisabled default: return uio.SizeEstimationLinks } } // UnixFSSplitterFunc returns a SplitterGen function based on Import.UnixFSChunker. // The returned function creates a Splitter for the configured chunking strategy. // The chunker string is parsed once when this method is called, not on each use. func (i *Import) UnixFSSplitterFunc() chunk.SplitterGen { chunkerStr := i.UnixFSChunker.WithDefault(DefaultUnixFSChunker) // Parse size-based chunker (most common case) and return optimized generator if sizeStr, ok := strings.CutPrefix(chunkerStr, "size-"); ok { if size, err := strconv.ParseInt(sizeStr, 10, 64); err == nil && size > 0 { return chunk.SizeSplitterGen(size) } } // For other chunker types (rabin, buzhash) or invalid config, // fall back to parsing per-use (these are rare cases) return func(r io.Reader) chunk.Splitter { s, err := chunk.FromString(r, chunkerStr) if err != nil { return chunk.DefaultSplitter(r) } return s } } ================================================ FILE: config/import_test.go ================================================ package config import ( "strings" "testing" "github.com/ipfs/boxo/ipld/unixfs/io" mh "github.com/multiformats/go-multihash" ) func TestValidateImportConfig_HAMTFanout(t *testing.T) { tests := []struct { name string fanout int64 wantErr bool errMsg string }{ // Valid values - powers of 2, multiples of 8, and <= 1024 {name: "valid 8", fanout: 8, wantErr: false}, {name: "valid 16", fanout: 16, wantErr: false}, {name: "valid 32", fanout: 32, wantErr: false}, {name: "valid 64", fanout: 64, wantErr: false}, {name: "valid 128", fanout: 128, wantErr: false}, {name: "valid 256", fanout: 256, wantErr: false}, {name: "valid 512", fanout: 512, wantErr: false}, {name: "valid 1024", fanout: 1024, wantErr: false}, // Invalid values - not powers of 2 {name: "invalid 7", fanout: 7, wantErr: true, errMsg: "must be a power of 2, between 8 and 1024"}, {name: "invalid 15", fanout: 15, wantErr: true, errMsg: "must be a power of 2, between 8 and 1024"}, {name: "invalid 100", fanout: 100, wantErr: true, errMsg: "must be a power of 2, between 8 and 1024"}, {name: "invalid 257", fanout: 257, wantErr: true, errMsg: "must be a power of 2, between 8 and 1024"}, {name: "invalid 1000", fanout: 1000, wantErr: true, errMsg: "must be a power of 2, between 8 and 1024"}, // Invalid values - powers of 2 but less than 8 {name: "invalid 1", fanout: 1, wantErr: true, errMsg: "must be a power of 2, between 8 and 1024"}, {name: "invalid 2", fanout: 2, wantErr: true, errMsg: "must be a power of 2, between 8 and 1024"}, {name: "invalid 4", fanout: 4, wantErr: true, errMsg: "must be a power of 2, between 8 and 1024"}, // Invalid values - exceeds 1024 {name: "invalid 2048", fanout: 2048, wantErr: true, errMsg: "must be a power of 2, between 8 and 1024"}, {name: "invalid 4096", fanout: 4096, wantErr: true, errMsg: "must be a power of 2, between 8 and 1024"}, // Invalid values - negative or zero {name: "invalid 0", fanout: 0, wantErr: true, errMsg: "must be a power of 2, between 8 and 1024"}, {name: "invalid -8", fanout: -8, wantErr: true, errMsg: "must be a power of 2, between 8 and 1024"}, {name: "invalid -256", fanout: -256, wantErr: true, errMsg: "must be a power of 2, between 8 and 1024"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := &Import{ UnixFSHAMTDirectoryMaxFanout: *NewOptionalInteger(tt.fanout), } err := ValidateImportConfig(cfg) if tt.wantErr { if err == nil { t.Errorf("ValidateImportConfig() expected error for fanout=%d, got nil", tt.fanout) } else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { t.Errorf("ValidateImportConfig() error = %v, want error containing %q", err, tt.errMsg) } } else { if err != nil { t.Errorf("ValidateImportConfig() unexpected error for fanout=%d: %v", tt.fanout, err) } } }) } } func TestValidateImportConfig_CidVersion(t *testing.T) { tests := []struct { name string cidVer int64 wantErr bool errMsg string }{ {name: "valid 0", cidVer: 0, wantErr: false}, {name: "valid 1", cidVer: 1, wantErr: false}, {name: "invalid 2", cidVer: 2, wantErr: true, errMsg: "must be 0 or 1"}, {name: "invalid -1", cidVer: -1, wantErr: true, errMsg: "must be 0 or 1"}, {name: "invalid 100", cidVer: 100, wantErr: true, errMsg: "must be 0 or 1"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := &Import{ CidVersion: *NewOptionalInteger(tt.cidVer), } err := ValidateImportConfig(cfg) if tt.wantErr { if err == nil { t.Errorf("ValidateImportConfig() expected error for cidVer=%d, got nil", tt.cidVer) } else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { t.Errorf("ValidateImportConfig() error = %v, want error containing %q", err, tt.errMsg) } } else { if err != nil { t.Errorf("ValidateImportConfig() unexpected error for cidVer=%d: %v", tt.cidVer, err) } } }) } } func TestValidateImportConfig_UnixFSFileMaxLinks(t *testing.T) { tests := []struct { name string maxLinks int64 wantErr bool errMsg string }{ {name: "valid 1", maxLinks: 1, wantErr: false}, {name: "valid 174", maxLinks: 174, wantErr: false}, {name: "valid 1000", maxLinks: 1000, wantErr: false}, {name: "invalid 0", maxLinks: 0, wantErr: true, errMsg: "must be positive"}, {name: "invalid -1", maxLinks: -1, wantErr: true, errMsg: "must be positive"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := &Import{ UnixFSFileMaxLinks: *NewOptionalInteger(tt.maxLinks), } err := ValidateImportConfig(cfg) if tt.wantErr { if err == nil { t.Errorf("ValidateImportConfig() expected error for maxLinks=%d, got nil", tt.maxLinks) } else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { t.Errorf("ValidateImportConfig() error = %v, want error containing %q", err, tt.errMsg) } } else { if err != nil { t.Errorf("ValidateImportConfig() unexpected error for maxLinks=%d: %v", tt.maxLinks, err) } } }) } } func TestValidateImportConfig_UnixFSDirectoryMaxLinks(t *testing.T) { tests := []struct { name string maxLinks int64 wantErr bool errMsg string }{ {name: "valid 0", maxLinks: 0, wantErr: false}, // 0 means no limit {name: "valid 1", maxLinks: 1, wantErr: false}, {name: "valid 1000", maxLinks: 1000, wantErr: false}, {name: "invalid -1", maxLinks: -1, wantErr: true, errMsg: "must be non-negative"}, {name: "invalid -100", maxLinks: -100, wantErr: true, errMsg: "must be non-negative"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := &Import{ UnixFSDirectoryMaxLinks: *NewOptionalInteger(tt.maxLinks), } err := ValidateImportConfig(cfg) if tt.wantErr { if err == nil { t.Errorf("ValidateImportConfig() expected error for maxLinks=%d, got nil", tt.maxLinks) } else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { t.Errorf("ValidateImportConfig() error = %v, want error containing %q", err, tt.errMsg) } } else { if err != nil { t.Errorf("ValidateImportConfig() unexpected error for maxLinks=%d: %v", tt.maxLinks, err) } } }) } } func TestValidateImportConfig_BatchMax(t *testing.T) { tests := []struct { name string maxNodes int64 maxSize int64 wantErr bool errMsg string }{ {name: "valid nodes 1", maxNodes: 1, maxSize: -999, wantErr: false}, {name: "valid nodes 128", maxNodes: 128, maxSize: -999, wantErr: false}, {name: "valid size 1", maxNodes: -999, maxSize: 1, wantErr: false}, {name: "valid size 20MB", maxNodes: -999, maxSize: 20 << 20, wantErr: false}, {name: "invalid nodes 0", maxNodes: 0, maxSize: -999, wantErr: true, errMsg: "BatchMaxNodes must be positive"}, {name: "invalid nodes -1", maxNodes: -1, maxSize: -999, wantErr: true, errMsg: "BatchMaxNodes must be positive"}, {name: "invalid size 0", maxNodes: -999, maxSize: 0, wantErr: true, errMsg: "BatchMaxSize must be positive"}, {name: "invalid size -1", maxNodes: -999, maxSize: -1, wantErr: true, errMsg: "BatchMaxSize must be positive"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := &Import{} if tt.maxNodes != -999 { cfg.BatchMaxNodes = *NewOptionalInteger(tt.maxNodes) } if tt.maxSize != -999 { cfg.BatchMaxSize = *NewOptionalInteger(tt.maxSize) } err := ValidateImportConfig(cfg) if tt.wantErr { if err == nil { t.Errorf("ValidateImportConfig() expected error, got nil") } else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { t.Errorf("ValidateImportConfig() error = %v, want error containing %q", err, tt.errMsg) } } else { if err != nil { t.Errorf("ValidateImportConfig() unexpected error: %v", err) } } }) } } func TestValidateImportConfig_UnixFSChunker(t *testing.T) { tests := []struct { name string chunker string wantErr bool errMsg string }{ {name: "valid size-262144", chunker: "size-262144", wantErr: false}, {name: "valid size-1", chunker: "size-1", wantErr: false}, {name: "valid size-1048576", chunker: "size-1048576", wantErr: false}, {name: "valid rabin", chunker: "rabin-128-256-512", wantErr: false}, {name: "valid rabin min", chunker: "rabin-16-32-64", wantErr: false}, {name: "valid buzhash", chunker: "buzhash", wantErr: false}, {name: "invalid size-", chunker: "size-", wantErr: true, errMsg: "invalid format"}, {name: "invalid size-abc", chunker: "size-abc", wantErr: true, errMsg: "invalid format"}, {name: "invalid rabin-", chunker: "rabin-", wantErr: true, errMsg: "invalid format"}, {name: "invalid rabin-128", chunker: "rabin-128", wantErr: true, errMsg: "invalid format"}, {name: "invalid rabin-128-256", chunker: "rabin-128-256", wantErr: true, errMsg: "invalid format"}, {name: "invalid rabin-a-b-c", chunker: "rabin-a-b-c", wantErr: true, errMsg: "invalid format"}, {name: "invalid unknown", chunker: "unknown", wantErr: true, errMsg: "invalid format"}, {name: "invalid empty", chunker: "", wantErr: true, errMsg: "invalid format"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := &Import{ UnixFSChunker: *NewOptionalString(tt.chunker), } err := ValidateImportConfig(cfg) if tt.wantErr { if err == nil { t.Errorf("ValidateImportConfig() expected error for chunker=%s, got nil", tt.chunker) } else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { t.Errorf("ValidateImportConfig() error = %v, want error containing %q", err, tt.errMsg) } } else { if err != nil { t.Errorf("ValidateImportConfig() unexpected error for chunker=%s: %v", tt.chunker, err) } } }) } } func TestValidateImportConfig_HashFunction(t *testing.T) { tests := []struct { name string hashFunc string wantErr bool errMsg string }{ {name: "valid sha2-256", hashFunc: "sha2-256", wantErr: false}, {name: "valid sha2-512", hashFunc: "sha2-512", wantErr: false}, {name: "valid sha3-256", hashFunc: "sha3-256", wantErr: false}, {name: "valid blake2b-256", hashFunc: "blake2b-256", wantErr: false}, {name: "valid blake3", hashFunc: "blake3", wantErr: false}, {name: "invalid unknown", hashFunc: "unknown-hash", wantErr: true, errMsg: "unrecognized"}, {name: "invalid empty", hashFunc: "", wantErr: true, errMsg: "unrecognized"}, } // Check for hashes that exist but are not allowed // MD5 should exist but not be allowed if code, ok := mh.Names["md5"]; ok { tests = append(tests, struct { name string hashFunc string wantErr bool errMsg string }{name: "md5 not allowed", hashFunc: "md5", wantErr: true, errMsg: "not allowed"}) _ = code // use the variable } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := &Import{ HashFunction: *NewOptionalString(tt.hashFunc), } err := ValidateImportConfig(cfg) if tt.wantErr { if err == nil { t.Errorf("ValidateImportConfig() expected error for hashFunc=%s, got nil", tt.hashFunc) } else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { t.Errorf("ValidateImportConfig() error = %v, want error containing %q", err, tt.errMsg) } } else { if err != nil { t.Errorf("ValidateImportConfig() unexpected error for hashFunc=%s: %v", tt.hashFunc, err) } } }) } } func TestValidateImportConfig_DefaultValue(t *testing.T) { // Test that default (unset) value doesn't trigger validation cfg := &Import{} err := ValidateImportConfig(cfg) if err != nil { t.Errorf("ValidateImportConfig() unexpected error for default config: %v", err) } } func TestIsValidChunker(t *testing.T) { tests := []struct { chunker string want bool }{ {"buzhash", true}, {"size-262144", true}, {"size-1", true}, {"size-0", false}, // 0 is not valid - must be positive {"size-9999999", true}, {"rabin-128-256-512", true}, {"rabin-16-32-64", true}, {"rabin-1-2-3", true}, {"rabin-512-256-128", false}, // Invalid ordering: min > avg > max {"rabin-256-128-512", false}, // Invalid ordering: min > avg {"rabin-128-512-256", false}, // Invalid ordering: avg > max {"", false}, {"size-", false}, {"size-abc", false}, {"size--1", false}, {"rabin-", false}, {"rabin-128", false}, {"rabin-128-256", false}, {"rabin-128-256-512-1024", false}, {"rabin-a-b-c", false}, {"unknown", false}, {"buzzhash", false}, // typo } for _, tt := range tests { t.Run(tt.chunker, func(t *testing.T) { if got := isValidChunker(tt.chunker); got != tt.want { t.Errorf("isValidChunker(%q) = %v, want %v", tt.chunker, got, tt.want) } }) } } func TestIsPowerOfTwo(t *testing.T) { tests := []struct { n int64 want bool }{ {0, false}, {1, true}, {2, true}, {3, false}, {4, true}, {5, false}, {6, false}, {7, false}, {8, true}, {16, true}, {32, true}, {64, true}, {100, false}, {128, true}, {256, true}, {512, true}, {1024, true}, {2048, true}, {-1, false}, {-8, false}, } for _, tt := range tests { t.Run("", func(t *testing.T) { if got := isPowerOfTwo(tt.n); got != tt.want { t.Errorf("isPowerOfTwo(%d) = %v, want %v", tt.n, got, tt.want) } }) } } func TestValidateImportConfig_HAMTSizeEstimation(t *testing.T) { tests := []struct { name string value string wantErr bool errMsg string }{ {name: "valid links", value: HAMTSizeEstimationLinks, wantErr: false}, {name: "valid block", value: HAMTSizeEstimationBlock, wantErr: false}, {name: "valid disabled", value: HAMTSizeEstimationDisabled, wantErr: false}, {name: "invalid unknown", value: "unknown", wantErr: true, errMsg: "must be"}, {name: "invalid empty", value: "", wantErr: true, errMsg: "must be"}, {name: "invalid typo", value: "link", wantErr: true, errMsg: "must be"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := &Import{ UnixFSHAMTDirectorySizeEstimation: *NewOptionalString(tt.value), } err := ValidateImportConfig(cfg) if tt.wantErr { if err == nil { t.Errorf("expected error for value=%q, got nil", tt.value) } else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { t.Errorf("error = %v, want error containing %q", err, tt.errMsg) } } else { if err != nil { t.Errorf("unexpected error for value=%q: %v", tt.value, err) } } }) } } func TestValidateImportConfig_DAGLayout(t *testing.T) { tests := []struct { name string value string wantErr bool errMsg string }{ {name: "valid balanced", value: DAGLayoutBalanced, wantErr: false}, {name: "valid trickle", value: DAGLayoutTrickle, wantErr: false}, {name: "invalid unknown", value: "unknown", wantErr: true, errMsg: "must be"}, {name: "invalid empty", value: "", wantErr: true, errMsg: "must be"}, {name: "invalid flat", value: "flat", wantErr: true, errMsg: "must be"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := &Import{ UnixFSDAGLayout: *NewOptionalString(tt.value), } err := ValidateImportConfig(cfg) if tt.wantErr { if err == nil { t.Errorf("expected error for value=%q, got nil", tt.value) } else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { t.Errorf("error = %v, want error containing %q", err, tt.errMsg) } } else { if err != nil { t.Errorf("unexpected error for value=%q: %v", tt.value, err) } } }) } } func TestImport_HAMTSizeEstimationMode(t *testing.T) { tests := []struct { cfg string want io.SizeEstimationMode }{ {HAMTSizeEstimationLinks, io.SizeEstimationLinks}, {HAMTSizeEstimationBlock, io.SizeEstimationBlock}, {HAMTSizeEstimationDisabled, io.SizeEstimationDisabled}, {"", io.SizeEstimationLinks}, // default (unset returns default) {"unknown", io.SizeEstimationLinks}, // fallback to default } for _, tt := range tests { t.Run(tt.cfg, func(t *testing.T) { var imp Import if tt.cfg != "" { imp.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(tt.cfg) } got := imp.HAMTSizeEstimationMode() if got != tt.want { t.Errorf("Import.HAMTSizeEstimationMode() with %q = %v, want %v", tt.cfg, got, tt.want) } }) } } ================================================ FILE: config/init.go ================================================ package config import ( "crypto/rand" "encoding/base64" "fmt" "io" "time" "github.com/cockroachdb/pebble/v2" "github.com/ipfs/kubo/core/coreiface/options" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" ) func Init(out io.Writer, nBitsForKeypair int) (*Config, error) { identity, err := CreateIdentity(out, []options.KeyGenerateOption{options.Key.Size(nBitsForKeypair)}) if err != nil { return nil, err } return InitWithIdentity(identity) } func InitWithIdentity(identity Identity) (*Config, error) { datastore := DefaultDatastoreConfig() conf := &Config{ API: API{ HTTPHeaders: map[string][]string{}, }, // setup the node's default addresses. // NOTE: two swarm listen addrs, one tcp, one utp. Addresses: addressesConfig(), Datastore: datastore, Bootstrap: []string{AutoPlaceholder}, Identity: identity, Discovery: Discovery{ MDNS: MDNS{ Enabled: true, }, }, // setup the node mount points. Mounts: Mounts{ IPFS: "/ipfs", IPNS: "/ipns", MFS: "/mfs", }, Ipns: Ipns{ ResolveCacheSize: 128, DelegatedPublishers: []string{AutoPlaceholder}, }, Gateway: Gateway{ RootRedirect: "", NoFetch: false, HTTPHeaders: map[string][]string{}, }, Pinning: Pinning{ RemoteServices: map[string]RemotePinningService{}, }, DNS: DNS{ Resolvers: map[string]string{ ".": AutoPlaceholder, }, }, Routing: Routing{ DelegatedRouters: []string{AutoPlaceholder}, }, } return conf, nil } // DefaultConnMgrHighWater is the default value for the connection managers // 'high water' mark. const DefaultConnMgrHighWater = 96 // DefaultConnMgrLowWater is the default value for the connection managers 'low // water' mark. const DefaultConnMgrLowWater = 32 // DefaultConnMgrGracePeriod is the default value for the connection managers // grace period. const DefaultConnMgrGracePeriod = time.Second * 20 // DefaultConnMgrSilencePeriod controls how often the connection manager enforces the limits. const DefaultConnMgrSilencePeriod = time.Second * 10 // DefaultConnMgrType is the default value for the connection managers // type. const DefaultConnMgrType = "basic" // DefaultResourceMgrMinInboundConns is a MAGIC number that probably a good // enough number of inbound conns to be a good network citizen. const DefaultResourceMgrMinInboundConns = 800 func addressesConfig() Addresses { return Addresses{ Swarm: []string{ "/ip4/0.0.0.0/tcp/4001", "/ip6/::/tcp/4001", "/ip4/0.0.0.0/udp/4001/webrtc-direct", "/ip4/0.0.0.0/udp/4001/quic-v1", "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", "/ip6/::/udp/4001/webrtc-direct", "/ip6/::/udp/4001/quic-v1", "/ip6/::/udp/4001/quic-v1/webtransport", }, Announce: []string{}, AppendAnnounce: []string{}, NoAnnounce: []string{}, API: Strings{"/ip4/127.0.0.1/tcp/5001"}, Gateway: Strings{"/ip4/127.0.0.1/tcp/8080"}, } } // DefaultDatastoreConfig is an internal function exported to aid in testing. func DefaultDatastoreConfig() Datastore { return Datastore{ StorageMax: "10GB", StorageGCWatermark: 90, // 90% GCPeriod: "1h", BloomFilterSize: 0, Spec: flatfsSpec(), } } func pebbleSpec() map[string]any { return map[string]any{ "type": "pebbleds", "prefix": "pebble.datastore", "path": "pebbleds", "formatMajorVersion": int(pebble.FormatNewest), } } func pebbleSpecMeasure() map[string]any { return map[string]any{ "type": "measure", "prefix": "pebble.datastore", "child": map[string]any{ "formatMajorVersion": int(pebble.FormatNewest), "type": "pebbleds", "path": "pebbleds", }, } } func badgerSpec() map[string]any { return map[string]any{ "type": "badgerds", "prefix": "badger.datastore", "path": "badgerds", "syncWrites": false, "truncate": true, } } func badgerSpecMeasure() map[string]any { return map[string]any{ "type": "measure", "prefix": "badger.datastore", "child": map[string]any{ "type": "badgerds", "path": "badgerds", "syncWrites": false, "truncate": true, }, } } func flatfsSpec() map[string]any { return map[string]any{ "type": "mount", "mounts": []any{ map[string]any{ "mountpoint": "/blocks", "type": "flatfs", "prefix": "flatfs.datastore", "path": "blocks", "sync": false, "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2", }, map[string]any{ "mountpoint": "/", "type": "levelds", "prefix": "leveldb.datastore", "path": "datastore", "compression": "none", }, }, } } func flatfsSpecMeasure() map[string]any { return map[string]any{ "type": "mount", "mounts": []any{ map[string]any{ "mountpoint": "/blocks", "type": "measure", "prefix": "flatfs.datastore", "child": map[string]any{ "type": "flatfs", "path": "blocks", "sync": false, "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2", }, }, map[string]any{ "mountpoint": "/", "type": "measure", "prefix": "leveldb.datastore", "child": map[string]any{ "type": "levelds", "path": "datastore", "compression": "none", }, }, }, } } // CreateIdentity initializes a new identity. func CreateIdentity(out io.Writer, opts []options.KeyGenerateOption) (Identity, error) { // TODO guard higher up ident := Identity{} settings, err := options.KeyGenerateOptions(opts...) if err != nil { return ident, err } var sk crypto.PrivKey var pk crypto.PubKey switch settings.Algorithm { case "rsa": if settings.Size == -1 { settings.Size = options.DefaultRSALen } fmt.Fprintf(out, "generating %d-bit RSA keypair...", settings.Size) priv, pub, err := crypto.GenerateKeyPair(crypto.RSA, settings.Size) if err != nil { return ident, err } sk = priv pk = pub case "ed25519": if settings.Size != -1 { return ident, fmt.Errorf("number of key bits does not apply when using ed25519 keys") } fmt.Fprintf(out, "generating ED25519 keypair...") priv, pub, err := crypto.GenerateEd25519Key(rand.Reader) if err != nil { return ident, err } sk = priv pk = pub default: return ident, fmt.Errorf("unrecognized key type: %s", settings.Algorithm) } fmt.Fprintf(out, "done\n") // currently storing key unencrypted. in the future we need to encrypt it. // TODO(security) skbytes, err := crypto.MarshalPrivateKey(sk) if err != nil { return ident, err } ident.PrivKey = base64.StdEncoding.EncodeToString(skbytes) id, err := peer.IDFromPublicKey(pk) if err != nil { return ident, err } ident.PeerID = id.String() fmt.Fprintf(out, "peer identity: %s\n", ident.PeerID) return ident, nil } ================================================ FILE: config/init_test.go ================================================ package config import ( "bytes" "testing" "github.com/ipfs/kubo/core/coreiface/options" crypto_pb "github.com/libp2p/go-libp2p/core/crypto/pb" ) func TestCreateIdentity(t *testing.T) { writer := bytes.NewBuffer(nil) id, err := CreateIdentity(writer, []options.KeyGenerateOption{options.Key.Type(options.Ed25519Key)}) if err != nil { t.Fatal(err) } pk, err := id.DecodePrivateKey("") if err != nil { t.Fatal(err) } if pk.Type() != crypto_pb.KeyType_Ed25519 { t.Fatal("unexpected type:", pk.Type()) } id, err = CreateIdentity(writer, []options.KeyGenerateOption{options.Key.Type(options.RSAKey)}) if err != nil { t.Fatal(err) } pk, err = id.DecodePrivateKey("") if err != nil { t.Fatal(err) } if pk.Type() != crypto_pb.KeyType_RSA { t.Fatal("unexpected type:", pk.Type()) } } func TestCreateIdentityOptions(t *testing.T) { var w bytes.Buffer // ed25519 keys with bit size must fail. _, err := CreateIdentity(&w, []options.KeyGenerateOption{ options.Key.Type(options.Ed25519Key), options.Key.Size(2048), }) if err == nil { t.Errorf("ed25519 keys cannot have a custom bit size") } } ================================================ FILE: config/internal.go ================================================ package config const ( // DefaultMFSNoFlushLimit is the default limit for consecutive unflushed MFS operations DefaultMFSNoFlushLimit = 256 ) type Internal struct { // All marked as omitempty since we are expecting to make changes to all subcomponents of Internal Bitswap *InternalBitswap `json:",omitempty"` UnixFSShardingSizeThreshold *OptionalString `json:",omitempty"` // moved to Import.UnixFSHAMTDirectorySizeThreshold Libp2pForceReachability *OptionalString `json:",omitempty"` BackupBootstrapInterval *OptionalDuration `json:",omitempty"` // MFSNoFlushLimit controls the maximum number of consecutive // MFS operations allowed with --flush=false before requiring a manual flush. // This prevents unbounded memory growth and ensures data consistency. // Set to 0 to disable limiting (old behavior, may cause high memory usage) // This is an EXPERIMENTAL feature and may change or be removed in future releases. // See https://github.com/ipfs/kubo/issues/10842 MFSNoFlushLimit *OptionalInteger `json:",omitempty"` } type InternalBitswap struct { TaskWorkerCount OptionalInteger EngineBlockstoreWorkerCount OptionalInteger EngineTaskWorkerCount OptionalInteger MaxOutstandingBytesPerPeer OptionalInteger ProviderSearchDelay OptionalDuration ProviderSearchMaxResults OptionalInteger WantHaveReplaceSize OptionalInteger BroadcastControl *BitswapBroadcastControl } type BitswapBroadcastControl struct { // EnableEnables or disables broadcast control functionality. Setting this // to false disables broadcast control functionality and restores the // previous broadcast behavior of sending broadcasts to all peers. When // disabled, all other BroadcastControl configuration items are ignored. // Default is [DefaultBroadcastControlEnable]. Enable Flag `json:",omitempty"` // MaxPeers sets a hard limit on the number of peers to send broadcasts to. // A value of 0 means no broadcasts are sent. A value of -1 means there is // no limit. Default is [DefaultBroadcastControlMaxPeers]. MaxPeers OptionalInteger // LocalPeers enables or disables broadcast control for peers on the local // network. If false, than always broadcast to peers on the local network. // If true, apply broadcast control to local peers. Default is // [DefaultBroadcastControlLocalPeers]. LocalPeers Flag `json:",omitempty"` // PeeredPeers enables or disables broadcast reduction for peers configured // for peering. If false, than always broadcast to peers configured for // peering. If true, apply broadcast reduction to peered peers. Default is // [DefaultBroadcastControlPeeredPeers]. PeeredPeers Flag `json:",omitempty"` // MaxRandomPeers is the number of peers to broadcast to anyway, even // though broadcast reduction logic has determined that they are not // broadcast targets. Setting this to a non-zero value ensures at least // this number of random peers receives a broadcast. This may be helpful in // cases where peers that are not receiving broadcasts my have wanted // blocks. Default is [DefaultBroadcastControlMaxRandomPeers]. MaxRandomPeers OptionalInteger // SendToPendingPeers enables or disables sending broadcasts to any peers // to which there is a pending message to send. When enabled, this sends // broadcasts to many more peers, but does so in a way that does not // increase the number of separate broadcast messages. There is still the // increased cost of the recipients having to process and respond to the // broadcasts. Default is [DefaultBroadcastControlSendToPendingPeers]. SendToPendingPeers Flag `json:",omitempty"` } const ( DefaultBroadcastControlEnable = true // Enabled DefaultBroadcastControlMaxPeers = -1 // Unlimited DefaultBroadcastControlLocalPeers = false // No control of local DefaultBroadcastControlPeeredPeers = false // No control of peered DefaultBroadcastControlMaxRandomPeers = 0 // No randoms DefaultBroadcastControlSendToPendingPeers = false // Disabled ) ================================================ FILE: config/ipns.go ================================================ package config import ( "math" "time" ) const ( DefaultIpnsMaxCacheTTL = time.Duration(math.MaxInt64) ) type Ipns struct { RepublishPeriod string RecordLifetime string ResolveCacheSize int // MaxCacheTTL is the maximum duration IPNS entries are valid in the cache. MaxCacheTTL *OptionalDuration `json:",omitempty"` // Enable namesys pubsub (--enable-namesys-pubsub) UsePubsub Flag `json:",omitempty"` // Simplified configuration for delegated IPNS publishers DelegatedPublishers []string } ================================================ FILE: config/migration.go ================================================ package config const DefaultMigrationKeep = "cache" // DefaultMigrationDownloadSources defines the default download sources for legacy migrations (repo versions <16). // Only HTTPS is supported for legacy migrations. IPFS downloads are not supported. var DefaultMigrationDownloadSources = []string{"HTTPS"} // Migration configures how legacy migrations are downloaded (repo versions <16). // // DEPRECATED: This configuration only applies to legacy external migrations for repository // versions below 16. Modern repositories (v16+) use embedded migrations that do not require // external downloads. These settings will be ignored for modern repository versions. type Migration struct { // DEPRECATED: This field is deprecated and ignored for modern repositories (repo versions ≥16). DownloadSources []string `json:",omitempty"` // DEPRECATED: This field is deprecated and ignored for modern repositories (repo versions ≥16). Keep string `json:",omitempty"` } ================================================ FILE: config/migration_test.go ================================================ package config import ( "encoding/json" "testing" ) func TestMigrationDecode(t *testing.T) { str := ` { "DownloadSources": ["IPFS", "HTTP", "127.0.0.1"], "Keep": "cache" } ` var cfg Migration if err := json.Unmarshal([]byte(str), &cfg); err != nil { t.Errorf("failed while unmarshalling migration struct: %s", err) } if len(cfg.DownloadSources) != 3 { t.Fatal("wrong number of DownloadSources") } expect := []string{"IPFS", "HTTP", "127.0.0.1"} for i := range expect { if cfg.DownloadSources[i] != expect[i] { t.Errorf("wrong DownloadSource at %d", i) } } if cfg.Keep != "cache" { t.Error("wrong value for Keep") } } ================================================ FILE: config/mounts.go ================================================ package config // Mounts stores the (string) mount points. type Mounts struct { IPFS string IPNS string MFS string FuseAllowOther bool } ================================================ FILE: config/peering.go ================================================ package config import "github.com/libp2p/go-libp2p/core/peer" // Peering configures the peering service. type Peering struct { // Peers lists the nodes to attempt to stay connected with. Peers []peer.AddrInfo } ================================================ FILE: config/plugins.go ================================================ package config type Plugins struct { Plugins map[string]Plugin // TODO: Loader Path? Leaving that out for now due to security concerns. } type Plugin struct { Disabled bool Config any `json:",omitempty"` } ================================================ FILE: config/profile.go ================================================ package config import ( "fmt" "net" "time" ) // Transformer is a function which takes configuration and applies some filter to it. type Transformer func(c *Config) error // Profile contains the profile transformer the description of the profile. type Profile struct { // Description briefly describes the functionality of the profile. Description string // Transform takes ipfs configuration and applies the profile to it. Transform Transformer // InitOnly specifies that this profile can only be applied on init. InitOnly bool } // defaultServerFilters has is a list of IPv4 and IPv6 prefixes that are private, local only, or unrouteable. // according to https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml // and https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml var defaultServerFilters = []string{ "/ip4/10.0.0.0/ipcidr/8", "/ip4/100.64.0.0/ipcidr/10", "/ip4/169.254.0.0/ipcidr/16", "/ip4/172.16.0.0/ipcidr/12", "/ip4/192.0.0.0/ipcidr/24", "/ip4/192.0.2.0/ipcidr/24", "/ip4/192.168.0.0/ipcidr/16", "/ip4/198.18.0.0/ipcidr/15", "/ip4/198.51.100.0/ipcidr/24", "/ip4/203.0.113.0/ipcidr/24", "/ip4/240.0.0.0/ipcidr/4", "/ip6/100::/ipcidr/64", "/ip6/2001:2::/ipcidr/48", "/ip6/2001:db8::/ipcidr/32", "/ip6/fc00::/ipcidr/7", "/ip6/fe80::/ipcidr/10", } // Profiles is a map holding configuration transformers. Docs are in docs/config.md. var Profiles = map[string]Profile{ "server": { Description: `Disables local host discovery, recommended when running IPFS on machines with public IPv4 addresses.`, Transform: func(c *Config) error { c.Addresses.NoAnnounce = appendSingle(c.Addresses.NoAnnounce, defaultServerFilters) c.Swarm.AddrFilters = appendSingle(c.Swarm.AddrFilters, defaultServerFilters) c.Discovery.MDNS.Enabled = false c.Swarm.DisableNatPortMap = true return nil }, }, "local-discovery": { Description: `Sets default values to fields affected by the server profile, enables discovery in local networks.`, Transform: func(c *Config) error { c.Addresses.NoAnnounce = deleteEntries(c.Addresses.NoAnnounce, defaultServerFilters) c.Swarm.AddrFilters = deleteEntries(c.Swarm.AddrFilters, defaultServerFilters) c.Discovery.MDNS.Enabled = true c.Swarm.DisableNatPortMap = false return nil }, }, "test": { Description: `Reduces external interference of IPFS daemon, this is useful when using the daemon in test environments.`, Transform: func(c *Config) error { c.Addresses.API = Strings{"/ip4/127.0.0.1/tcp/0"} c.Addresses.Gateway = Strings{"/ip4/127.0.0.1/tcp/0"} c.Addresses.Swarm = []string{ "/ip4/127.0.0.1/tcp/0", } c.Swarm.DisableNatPortMap = true c.Routing.LoopbackAddressesOnLanDHT = True c.Bootstrap = []string{} c.Discovery.MDNS.Enabled = false c.AutoTLS.Enabled = False c.AutoConf.Enabled = False // Explicitly set autoconf-controlled fields to empty when autoconf is disabled c.DNS.Resolvers = map[string]string{} c.Routing.DelegatedRouters = []string{} c.Ipns.DelegatedPublishers = []string{} return nil }, }, "default-networking": { Description: `Restores default network settings. Inverse profile of the test profile.`, Transform: func(c *Config) error { c.Addresses = addressesConfig() // Use AutoConf system for bootstrap peers c.Bootstrap = []string{AutoPlaceholder} c.AutoConf.Enabled = Default c.AutoConf.URL = nil // Clear URL to use implicit default c.Swarm.DisableNatPortMap = false c.Discovery.MDNS.Enabled = true c.AutoTLS.Enabled = Default return nil }, }, "default-datastore": { Description: `Configures the node to use the default datastore (flatfs). Read the "flatfs" profile description for more information on this datastore. This profile may only be applied when first initializing the node. `, InitOnly: true, Transform: func(c *Config) error { c.Datastore.Spec = flatfsSpec() return nil }, }, "flatfs": { Description: `Configures the node to use the flatfs datastore. This is the most battle-tested and reliable datastore. You should use this datastore if: * You need a very simple and very reliable datastore, and you trust your filesystem. This datastore stores each block as a separate file in the underlying filesystem so it's unlikely to loose data unless there's an issue with the underlying file system. * You need to run garbage collection in a way that reclaims free space as soon as possible. * You want to minimize memory usage. * You are ok with the default speed of data import, or prefer to use --nocopy. See configuration documentation at: https://github.com/ipfs/kubo/blob/master/docs/datastores.md#flatfs NOTE: This profile may only be applied when first initializing node at IPFS_PATH via 'ipfs init --profile flatfs' `, InitOnly: true, Transform: func(c *Config) error { c.Datastore.Spec = flatfsSpec() return nil }, }, "flatfs-measure": { Description: `Configures the node to use the flatfs datastore with metrics tracking wrapper. Additional '*_datastore_*' metrics will be exposed on /debug/metrics/prometheus NOTE: This profile may only be applied when first initializing node at IPFS_PATH via 'ipfs init --profile flatfs-measure' `, InitOnly: true, Transform: func(c *Config) error { c.Datastore.Spec = flatfsSpecMeasure() return nil }, }, "pebbleds": { Description: `Configures the node to use the pebble high-performance datastore. Pebble is a LevelDB/RocksDB inspired key-value store focused on performance and internal usage by CockroachDB. You should use this datastore if: - You need a datastore that is focused on performance. - You need reliability by default, but may choose to disable WAL for maximum performance when reliability is not critical. - This datastore is good for multi-terabyte data sets. - May benefit from tuning depending on read/write patterns and throughput. - Performance is helped significantly by running on a system with plenty of memory. See configuration documentation at: https://github.com/ipfs/kubo/blob/master/docs/datastores.md#pebbleds NOTE: This profile may only be applied when first initializing node at IPFS_PATH via 'ipfs init --profile pebbleds' `, InitOnly: true, Transform: func(c *Config) error { c.Datastore.Spec = pebbleSpec() return nil }, }, "pebbleds-measure": { Description: `Configures the node to use the pebble datastore with metrics tracking wrapper. Additional '*_datastore_*' metrics will be exposed on /debug/metrics/prometheus NOTE: This profile may only be applied when first initializing node at IPFS_PATH via 'ipfs init --profile pebbleds-measure' `, InitOnly: true, Transform: func(c *Config) error { c.Datastore.Spec = pebbleSpecMeasure() return nil }, }, "badgerds": { Description: `DEPRECATED: Configures the node to use the legacy badgerv1 datastore. This profile will be removed in a future Kubo release. New deployments should use 'flatfs' or 'pebbleds' instead. NOTE: this is badger 1.x, which has known bugs and is no longer supported by the upstream team. It is provided here only for pre-existing users, allowing them to migrate away to more modern datastore. Other caveats: * This datastore will not properly reclaim space when your datastore is smaller than several gigabytes. If you run IPFS with --enable-gc, you plan on storing very little data in your IPFS node, and disk usage is more critical than performance, consider using flatfs. * This datastore uses up to several gigabytes of memory. * Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte. To migrate: create a new IPFS_PATH with 'ipfs init --profile=flatfs', move pinned data via 'ipfs dag export/import' or 'ipfs pin ls -t recursive|add', and decommission the old badger-based node. When it comes to block storage, use experimental 'pebbleds' only if you are sure modern 'flatfs' does not serve your use case (most users will be perfectly fine with flatfs, it is also possible to keep flatfs for blocks and replace leveldb with pebble if preferred over leveldb). See configuration documentation at: https://github.com/ipfs/kubo/blob/master/docs/datastores.md#badgerds NOTE: This profile may only be applied when first initializing node at IPFS_PATH via 'ipfs init --profile badgerds' `, InitOnly: true, Transform: func(c *Config) error { c.Datastore.Spec = badgerSpec() return nil }, }, "badgerds-measure": { Description: `DEPRECATED: Configures the node to use the legacy badgerv1 datastore with metrics wrapper. This profile will be removed in a future Kubo release. New deployments should use 'flatfs' or 'pebbleds' instead. NOTE: This profile may only be applied when first initializing node at IPFS_PATH via 'ipfs init --profile badgerds-measure' `, InitOnly: true, Transform: func(c *Config) error { c.Datastore.Spec = badgerSpecMeasure() return nil }, }, "lowpower": { Description: `Reduces daemon overhead on the system. May affect node functionality - performance of content discovery and data fetching may be degraded. `, Transform: func(c *Config) error { // Disable "server" services (dht, autonat, limited relay) c.Routing.Type = NewOptionalString("autoclient") c.AutoNAT.ServiceMode = AutoNATServiceDisabled c.Swarm.RelayService.Enabled = False // Keep bare minimum connections around lowWater := int64(20) highWater := int64(40) gracePeriod := time.Minute c.Swarm.ConnMgr.Type = NewOptionalString("basic") c.Swarm.ConnMgr.LowWater = &OptionalInteger{value: &lowWater} c.Swarm.ConnMgr.HighWater = &OptionalInteger{value: &highWater} c.Swarm.ConnMgr.GracePeriod = &OptionalDuration{&gracePeriod} return nil }, }, "announce-off": { Description: `Disables Provide system (announcing to Amino DHT). USE WITH CAUTION: The main use case for this is setups with manual Peering.Peers config. Data from this node will not be announced on the DHT. This will make DHT-based routing and data retrieval impossible if this node is the only one hosting it, and other peers are not already connected to it. `, Transform: func(c *Config) error { c.Provide.Enabled = False c.Provide.DHT.Interval = NewOptionalDuration(0) // 0 disables periodic reprovide return nil }, }, "announce-on": { Description: `Re-enables Provide system (reverts announce-off profile).`, Transform: func(c *Config) error { c.Provide.Enabled = True c.Provide.DHT.Interval = NewOptionalDuration(DefaultProvideDHTInterval) // have to apply explicit default because nil would be ignored return nil }, }, "randomports": { Description: `Use a random port number for swarm.`, Transform: func(c *Config) error { port, err := getAvailablePort() if err != nil { return err } c.Addresses.Swarm = []string{ fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", port), fmt.Sprintf("/ip6/::/tcp/%d", port), } return nil }, }, "unixfs-v0-2015": { Description: `Legacy UnixFS import profile for backward-compatible CID generation. Produces CIDv0 with no raw leaves, sha2-256, 256 KiB chunks, and link-based HAMT size estimation. Use only when legacy CIDs are required. See https://specs.ipfs.tech/ipips/ipip-0499/. Alias: legacy-cid-v0`, Transform: applyUnixFSv02015, }, "legacy-cid-v0": { Description: `Alias for unixfs-v0-2015 profile.`, Transform: applyUnixFSv02015, }, "unixfs-v1-2025": { Description: `Recommended UnixFS import profile for cross-implementation CID determinism. Uses CIDv1, raw leaves, sha2-256, 1 MiB chunks, 1024 links per file node, 256 HAMT fanout, and block-based size estimation for HAMT threshold. See https://specs.ipfs.tech/ipips/ipip-0499/`, Transform: func(c *Config) error { c.Import.CidVersion = *NewOptionalInteger(1) c.Import.UnixFSRawLeaves = True c.Import.UnixFSChunker = *NewOptionalString("size-1048576") // 1 MiB c.Import.HashFunction = *NewOptionalString("sha2-256") c.Import.UnixFSFileMaxLinks = *NewOptionalInteger(1024) c.Import.UnixFSDirectoryMaxLinks = *NewOptionalInteger(0) c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256) c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("256KiB") c.Import.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(HAMTSizeEstimationBlock) c.Import.UnixFSDAGLayout = *NewOptionalString(DAGLayoutBalanced) return nil }, }, "autoconf-on": { Description: `Sets configuration to use implicit defaults from remote autoconf service. Bootstrap peers, DNS resolvers, delegated routers, and IPNS delegated publishers are set to "auto". This profile requires AutoConf to be enabled and configured.`, Transform: func(c *Config) error { c.Bootstrap = []string{AutoPlaceholder} c.DNS.Resolvers = map[string]string{ ".": AutoPlaceholder, } c.Routing.DelegatedRouters = []string{AutoPlaceholder} c.Ipns.DelegatedPublishers = []string{AutoPlaceholder} c.AutoConf.Enabled = True if c.AutoConf.URL == nil { c.AutoConf.URL = NewOptionalString(DefaultAutoConfURL) } return nil }, }, "autoconf-off": { Description: `Disables AutoConf and sets networking fields to empty for manual configuration. Bootstrap peers, DNS resolvers, delegated routers, and IPNS delegated publishers are set to empty. Use this when you want normal networking but prefer manual control over all endpoints.`, Transform: func(c *Config) error { c.Bootstrap = nil c.DNS.Resolvers = nil c.Routing.DelegatedRouters = nil c.Ipns.DelegatedPublishers = nil c.AutoConf.Enabled = False return nil }, }, } func getAvailablePort() (port int, err error) { ln, err := net.Listen("tcp", "[::]:0") if err != nil { return 0, err } defer ln.Close() port = ln.Addr().(*net.TCPAddr).Port return port, nil } func appendSingle(a []string, b []string) []string { out := make([]string, 0, len(a)+len(b)) m := map[string]bool{} for _, f := range a { if !m[f] { out = append(out, f) } m[f] = true } for _, f := range b { if !m[f] { out = append(out, f) } m[f] = true } return out } func deleteEntries(arr []string, del []string) []string { m := map[string]struct{}{} for _, f := range arr { m[f] = struct{}{} } for _, f := range del { delete(m, f) } return mapKeys(m) } func mapKeys(m map[string]struct{}) []string { out := make([]string, 0, len(m)) for f := range m { out = append(out, f) } return out } // applyUnixFSv02015 applies the legacy UnixFS v0 (2015) import settings. func applyUnixFSv02015(c *Config) error { c.Import.CidVersion = *NewOptionalInteger(0) c.Import.UnixFSRawLeaves = False c.Import.UnixFSChunker = *NewOptionalString("size-262144") // 256 KiB c.Import.HashFunction = *NewOptionalString("sha2-256") c.Import.UnixFSFileMaxLinks = *NewOptionalInteger(174) c.Import.UnixFSDirectoryMaxLinks = *NewOptionalInteger(0) c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256) c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("256KiB") c.Import.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(HAMTSizeEstimationLinks) c.Import.UnixFSDAGLayout = *NewOptionalString(DAGLayoutBalanced) return nil } ================================================ FILE: config/provide.go ================================================ package config import ( "fmt" "strings" "time" "github.com/libp2p/go-libp2p-kad-dht/amino" ) const ( DefaultProvideEnabled = true DefaultProvideStrategy = "all" // DHT provider defaults DefaultProvideDHTInterval = 22 * time.Hour // https://github.com/ipfs/kubo/pull/9326 DefaultProvideDHTMaxWorkers = 16 // Unified default for both sweep and legacy providers DefaultProvideDHTSweepEnabled = true DefaultProvideDHTResumeEnabled = true DefaultProvideDHTDedicatedPeriodicWorkers = 2 DefaultProvideDHTDedicatedBurstWorkers = 1 DefaultProvideDHTMaxProvideConnsPerWorker = 20 DefaultProvideDHTKeystoreBatchSize = 1 << 14 // ~544 KiB per batch (1 multihash = 34 bytes) DefaultProvideDHTOfflineDelay = 2 * time.Hour // DefaultFastProvideTimeout is the maximum time allowed for fast-provide operations. // Prevents hanging on network issues when providing root CID. // 10 seconds is sufficient for DHT operations with sweep provider or accelerated client. DefaultFastProvideTimeout = 10 * time.Second ) type ProvideStrategy int const ( ProvideStrategyAll ProvideStrategy = 1 << iota ProvideStrategyPinned ProvideStrategyRoots ProvideStrategyMFS ) // Provide configures both immediate CID announcements (provide operations) for new content // and periodic re-announcements of existing CIDs (reprovide operations). // This section combines the functionality previously split between Provider and Reprovider. type Provide struct { // Enabled controls whether both provide and reprovide systems are enabled. // When disabled, the node will not announce any content to the routing system. Enabled Flag `json:",omitempty"` // Strategy determines which CIDs are announced to the routing system. // Default: DefaultProvideStrategy Strategy *OptionalString `json:",omitempty"` // DHT configures DHT-specific provide and reprovide settings. DHT ProvideDHT } // ProvideDHT configures DHT provider settings for both immediate announcements // and periodic reprovides. type ProvideDHT struct { // Interval sets the time between rounds of reproviding local content // to the routing system. Set to "0" to disable content reproviding. // Default: DefaultProvideDHTInterval Interval *OptionalDuration `json:",omitempty"` // MaxWorkers sets the maximum number of concurrent workers for provide operations. // When SweepEnabled is false: controls NEW CID announcements only. // When SweepEnabled is true: controls total worker pool for all operations. // Default: DefaultProvideDHTMaxWorkers MaxWorkers *OptionalInteger `json:",omitempty"` // SweepEnabled activates the sweeping reprovider system which spreads // reprovide operations over time. // Default: DefaultProvideDHTSweepEnabled SweepEnabled Flag `json:",omitempty"` // DedicatedPeriodicWorkers sets workers dedicated to periodic reprovides (sweep mode only). // Default: DefaultProvideDHTDedicatedPeriodicWorkers DedicatedPeriodicWorkers *OptionalInteger `json:",omitempty"` // DedicatedBurstWorkers sets workers dedicated to burst provides (sweep mode only). // Default: DefaultProvideDHTDedicatedBurstWorkers DedicatedBurstWorkers *OptionalInteger `json:",omitempty"` // MaxProvideConnsPerWorker sets concurrent connections per worker for sending provider records (sweep mode only). // Default: DefaultProvideDHTMaxProvideConnsPerWorker MaxProvideConnsPerWorker *OptionalInteger `json:",omitempty"` // KeystoreBatchSize sets the batch size for keystore operations during reprovide refresh (sweep mode only). // Default: DefaultProvideDHTKeystoreBatchSize KeystoreBatchSize *OptionalInteger `json:",omitempty"` // OfflineDelay sets the delay after which the provider switches from Disconnected to Offline state (sweep mode only). // Default: DefaultProvideDHTOfflineDelay OfflineDelay *OptionalDuration `json:",omitempty"` // ResumeEnabled controls whether the provider resumes from its previous state on restart. // When enabled, the provider persists its reprovide cycle state and provide queue to the datastore, // and restores them on restart. When disabled, the provider starts fresh on each restart. // Default: true ResumeEnabled Flag `json:",omitempty"` } func ParseProvideStrategy(s string) ProvideStrategy { var strategy ProvideStrategy for part := range strings.SplitSeq(s, "+") { switch part { case "all", "flat", "": // special case, does not mix with others ("flat" is deprecated, maps to "all") return ProvideStrategyAll case "pinned": strategy |= ProvideStrategyPinned case "roots": strategy |= ProvideStrategyRoots case "mfs": strategy |= ProvideStrategyMFS } } return strategy } // ValidateProvideConfig validates the Provide configuration according to DHT requirements. func ValidateProvideConfig(cfg *Provide) error { // Validate Provide.DHT.Interval if !cfg.DHT.Interval.IsDefault() { interval := cfg.DHT.Interval.WithDefault(DefaultProvideDHTInterval) if interval > amino.DefaultProvideValidity { return fmt.Errorf("Provide.DHT.Interval (%v) must be less than or equal to DHT provider record validity (%v)", interval, amino.DefaultProvideValidity) } if interval < 0 { return fmt.Errorf("Provide.DHT.Interval must be non-negative, got %v", interval) } } // Validate MaxWorkers if !cfg.DHT.MaxWorkers.IsDefault() { maxWorkers := cfg.DHT.MaxWorkers.WithDefault(DefaultProvideDHTMaxWorkers) if maxWorkers <= 0 { return fmt.Errorf("Provide.DHT.MaxWorkers must be positive, got %d", maxWorkers) } } // Validate DedicatedPeriodicWorkers if !cfg.DHT.DedicatedPeriodicWorkers.IsDefault() { workers := cfg.DHT.DedicatedPeriodicWorkers.WithDefault(DefaultProvideDHTDedicatedPeriodicWorkers) if workers < 0 { return fmt.Errorf("Provide.DHT.DedicatedPeriodicWorkers must be non-negative, got %d", workers) } } // Validate DedicatedBurstWorkers if !cfg.DHT.DedicatedBurstWorkers.IsDefault() { workers := cfg.DHT.DedicatedBurstWorkers.WithDefault(DefaultProvideDHTDedicatedBurstWorkers) if workers < 0 { return fmt.Errorf("Provide.DHT.DedicatedBurstWorkers must be non-negative, got %d", workers) } } // Validate MaxProvideConnsPerWorker if !cfg.DHT.MaxProvideConnsPerWorker.IsDefault() { conns := cfg.DHT.MaxProvideConnsPerWorker.WithDefault(DefaultProvideDHTMaxProvideConnsPerWorker) if conns <= 0 { return fmt.Errorf("Provide.DHT.MaxProvideConnsPerWorker must be positive, got %d", conns) } } // Validate KeystoreBatchSize if !cfg.DHT.KeystoreBatchSize.IsDefault() { batchSize := cfg.DHT.KeystoreBatchSize.WithDefault(DefaultProvideDHTKeystoreBatchSize) if batchSize <= 0 { return fmt.Errorf("Provide.DHT.KeystoreBatchSize must be positive, got %d", batchSize) } } // Validate OfflineDelay if !cfg.DHT.OfflineDelay.IsDefault() { delay := cfg.DHT.OfflineDelay.WithDefault(DefaultProvideDHTOfflineDelay) if delay < 0 { return fmt.Errorf("Provide.DHT.OfflineDelay must be non-negative, got %v", delay) } } return nil } // ShouldProvideForStrategy determines if content should be provided based on the provide strategy // and content characteristics (pinned status, root status, MFS status). func ShouldProvideForStrategy(strategy ProvideStrategy, isPinned bool, isPinnedRoot bool, isMFS bool) bool { if strategy == ProvideStrategyAll { // 'all' strategy: always provide return true } // For combined strategies, check each component if strategy&ProvideStrategyPinned != 0 && isPinned { return true } if strategy&ProvideStrategyRoots != 0 && isPinnedRoot { return true } if strategy&ProvideStrategyMFS != 0 && isMFS { return true } return false } ================================================ FILE: config/provide_test.go ================================================ package config import ( "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestParseProvideStrategy(t *testing.T) { tests := []struct { input string expect ProvideStrategy }{ {"all", ProvideStrategyAll}, {"pinned", ProvideStrategyPinned}, {"mfs", ProvideStrategyMFS}, {"pinned+mfs", ProvideStrategyPinned | ProvideStrategyMFS}, {"invalid", 0}, {"all+invalid", ProvideStrategyAll}, {"", ProvideStrategyAll}, {"flat", ProvideStrategyAll}, // deprecated, maps to "all" {"flat+all", ProvideStrategyAll}, } for _, tt := range tests { result := ParseProvideStrategy(tt.input) if result != tt.expect { t.Errorf("ParseProvideStrategy(%q) = %d, want %d", tt.input, result, tt.expect) } } } func TestValidateProvideConfig_Interval(t *testing.T) { tests := []struct { name string interval time.Duration wantErr bool errMsg string }{ {"valid default (22h)", 22 * time.Hour, false, ""}, {"valid max (48h)", 48 * time.Hour, false, ""}, {"valid small (1h)", 1 * time.Hour, false, ""}, {"valid zero (disabled)", 0, false, ""}, {"invalid over limit (49h)", 49 * time.Hour, true, "must be less than or equal to DHT provider record validity"}, {"invalid over limit (72h)", 72 * time.Hour, true, "must be less than or equal to DHT provider record validity"}, {"invalid negative", -1 * time.Hour, true, "must be non-negative"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := &Provide{ DHT: ProvideDHT{ Interval: NewOptionalDuration(tt.interval), }, } err := ValidateProvideConfig(cfg) if tt.wantErr { require.Error(t, err, "expected error for interval=%v", tt.interval) if tt.errMsg != "" { assert.Contains(t, err.Error(), tt.errMsg, "error message mismatch") } } else { require.NoError(t, err, "unexpected error for interval=%v", tt.interval) } }) } } func TestValidateProvideConfig_MaxWorkers(t *testing.T) { tests := []struct { name string maxWorkers int64 wantErr bool errMsg string }{ {"valid default", 16, false, ""}, {"valid high", 100, false, ""}, {"valid low", 1, false, ""}, {"invalid zero", 0, true, "must be positive"}, {"invalid negative", -1, true, "must be positive"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := &Provide{ DHT: ProvideDHT{ MaxWorkers: NewOptionalInteger(tt.maxWorkers), }, } err := ValidateProvideConfig(cfg) if tt.wantErr { require.Error(t, err, "expected error for maxWorkers=%d", tt.maxWorkers) if tt.errMsg != "" { assert.Contains(t, err.Error(), tt.errMsg, "error message mismatch") } } else { require.NoError(t, err, "unexpected error for maxWorkers=%d", tt.maxWorkers) } }) } } func TestShouldProvideForStrategy(t *testing.T) { t.Run("all strategy always provides", func(t *testing.T) { // ProvideStrategyAll should return true regardless of flags testCases := []struct{ pinned, pinnedRoot, mfs bool }{ {false, false, false}, {true, true, true}, {true, false, false}, } for _, tc := range testCases { assert.True(t, ShouldProvideForStrategy( ProvideStrategyAll, tc.pinned, tc.pinnedRoot, tc.mfs)) } }) t.Run("single strategies match only their flag", func(t *testing.T) { tests := []struct { name string strategy ProvideStrategy pinned, pinnedRoot, mfs bool want bool }{ {"pinned: matches when pinned=true", ProvideStrategyPinned, true, false, false, true}, {"pinned: ignores other flags", ProvideStrategyPinned, false, true, true, false}, {"roots: matches when pinnedRoot=true", ProvideStrategyRoots, false, true, false, true}, {"roots: ignores other flags", ProvideStrategyRoots, true, false, true, false}, {"mfs: matches when mfs=true", ProvideStrategyMFS, false, false, true, true}, {"mfs: ignores other flags", ProvideStrategyMFS, true, true, false, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := ShouldProvideForStrategy(tt.strategy, tt.pinned, tt.pinnedRoot, tt.mfs) assert.Equal(t, tt.want, got) }) } }) t.Run("combined strategies use OR logic (else-if bug fix)", func(t *testing.T) { // CRITICAL: Tests the fix where bitflag combinations (pinned+mfs) didn't work // because of else-if instead of separate if statements tests := []struct { name string strategy ProvideStrategy pinned, pinnedRoot, mfs bool want bool }{ // pinned|mfs: provide if EITHER matches {"pinned|mfs when pinned", ProvideStrategyPinned | ProvideStrategyMFS, true, false, false, true}, {"pinned|mfs when mfs", ProvideStrategyPinned | ProvideStrategyMFS, false, false, true, true}, {"pinned|mfs when both", ProvideStrategyPinned | ProvideStrategyMFS, true, false, true, true}, {"pinned|mfs when neither", ProvideStrategyPinned | ProvideStrategyMFS, false, false, false, false}, // roots|mfs {"roots|mfs when root", ProvideStrategyRoots | ProvideStrategyMFS, false, true, false, true}, {"roots|mfs when mfs", ProvideStrategyRoots | ProvideStrategyMFS, false, false, true, true}, {"roots|mfs when neither", ProvideStrategyRoots | ProvideStrategyMFS, false, false, false, false}, // pinned|roots {"pinned|roots when pinned", ProvideStrategyPinned | ProvideStrategyRoots, true, false, false, true}, {"pinned|roots when root", ProvideStrategyPinned | ProvideStrategyRoots, false, true, false, true}, {"pinned|roots when neither", ProvideStrategyPinned | ProvideStrategyRoots, false, false, false, false}, // triple combination {"all-three when any matches", ProvideStrategyPinned | ProvideStrategyRoots | ProvideStrategyMFS, false, false, true, true}, {"all-three when none match", ProvideStrategyPinned | ProvideStrategyRoots | ProvideStrategyMFS, false, false, false, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := ShouldProvideForStrategy(tt.strategy, tt.pinned, tt.pinnedRoot, tt.mfs) assert.Equal(t, tt.want, got) }) } }) t.Run("zero strategy never provides", func(t *testing.T) { assert.False(t, ShouldProvideForStrategy(ProvideStrategy(0), false, false, false)) assert.False(t, ShouldProvideForStrategy(ProvideStrategy(0), true, true, true)) }) } ================================================ FILE: config/provider.go ================================================ package config // Provider configuration describes how NEW CIDs are announced the moment they are created. // For periodical reprovide configuration, see Provide.* // // Deprecated: use Provide instead. This will be removed in a future release. type Provider struct { // Deprecated: use Provide.Enabled instead. This will be removed in a future release. Enabled Flag `json:",omitempty"` // Deprecated: unused, you are likely looking for Provide.Strategy instead. This will be removed in a future release. Strategy *OptionalString `json:",omitempty"` // Deprecated: use Provide.DHT.MaxWorkers instead. This will be removed in a future release. WorkerCount *OptionalInteger `json:",omitempty"` } ================================================ FILE: config/pubsub.go ================================================ package config const ( // LastSeenMessagesStrategy is a strategy that calculates the TTL countdown // based on the last time a Pubsub message is seen. This means that if a message // is received and then seen again within the specified TTL window, it // won't be emitted until the TTL countdown expires from the last time the // message was seen. LastSeenMessagesStrategy = "last-seen" // FirstSeenMessagesStrategy is a strategy that calculates the TTL // countdown based on the first time a Pubsub message is seen. This means that if // a message is received and then seen again within the specified TTL // window, it won't be emitted. FirstSeenMessagesStrategy = "first-seen" // DefaultSeenMessagesStrategy is the strategy that is used by default if // no Pubsub.SeenMessagesStrategy is specified. DefaultSeenMessagesStrategy = LastSeenMessagesStrategy ) type PubsubConfig struct { // Router can be either floodsub (legacy) or gossipsub (new and // backwards compatible). Router string // DisableSigning disables message signing. Message signing is *enabled* // by default. DisableSigning bool // Enable pubsub (--enable-pubsub-experiment) Enabled Flag `json:",omitempty"` // SeenMessagesTTL is a value that controls the time window within which // duplicate messages will be identified and won't be emitted. SeenMessagesTTL *OptionalDuration `json:",omitempty"` // SeenMessagesStrategy is a setting that determines how the time-to-live // (TTL) countdown for deduplicating messages is calculated. SeenMessagesStrategy *OptionalString `json:",omitempty"` } ================================================ FILE: config/remotepin.go ================================================ package config var ( RemoteServicesPath = "Pinning.RemoteServices" PinningConcealSelector = []string{"Pinning", "RemoteServices", "*", "API", "Key"} ) type Pinning struct { RemoteServices map[string]RemotePinningService } type RemotePinningService struct { API RemotePinningServiceAPI Policies RemotePinningServicePolicies } type RemotePinningServiceAPI struct { Endpoint string Key string } type RemotePinningServicePolicies struct { MFS RemotePinningServiceMFSPolicy } type RemotePinningServiceMFSPolicy struct { // Enable enables watching for changes in MFS and re-pinning the MFS root cid whenever a change occurs. Enable bool // Name is the pin name for MFS. PinName string // RepinInterval determines the repin interval when the policy is enabled. In ns, us, ms, s, m, h. RepinInterval string } ================================================ FILE: config/reprovider.go ================================================ package config // Reprovider configuration describes how CID from local datastore are periodically re-announced to routing systems. // For provide behavior of ad-hoc or newly created CIDs and their first-time announcement, see Provide.* // // Deprecated: use Provide instead. This will be removed in a future release. type Reprovider struct { // Deprecated: use Provide.DHT.Interval instead. This will be removed in a future release. Interval *OptionalDuration `json:",omitempty"` // Deprecated: use Provide.Strategy instead. This will be removed in a future release. Strategy *OptionalString `json:",omitempty"` } ================================================ FILE: config/routing.go ================================================ package config import ( "encoding/json" "fmt" "os" "runtime" "slices" "strings" ) const ( DefaultAcceleratedDHTClient = false DefaultLoopbackAddressesOnLanDHT = false DefaultRoutingType = "auto" CidContactRoutingURL = "https://cid.contact" PublicGoodDelegatedRoutingURL = "https://delegated-ipfs.dev" // cid.contact + amino dht (incl. IPNS PUTs) EnvHTTPRouters = "IPFS_HTTP_ROUTERS" EnvHTTPRoutersFilterProtocols = "IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS" ) var ( // Default filter-protocols to pass along with delegated routing requests (as defined in IPIP-484) // and also filter out locally DefaultHTTPRoutersFilterProtocols = getEnvOrDefault(EnvHTTPRoutersFilterProtocols, []string{ "unknown", // allow results without protocol list, we can do libp2p identify to test them "transport-bitswap", // http is added dynamically in routing/delegated.go. // 'transport-ipfs-gateway-http' }) ) // Routing defines configuration options for libp2p routing. type Routing struct { // Type sets default daemon routing mode. // // Can be one of "auto", "autoclient", "dht", "dhtclient", "dhtserver", "none", "delegated", or "custom". // When unset or set to "auto", DHT and implicit routers are used. // When "delegated" is set, only HTTP delegated routers and IPNS publishers are used (no DHT). // When "custom" is set, user-provided Routing.Routers is used. Type *OptionalString `json:",omitempty"` AcceleratedDHTClient Flag `json:",omitempty"` LoopbackAddressesOnLanDHT Flag `json:",omitempty"` IgnoreProviders []string `json:",omitempty"` // Simplified configuration used by default when Routing.Type=auto|autoclient DelegatedRouters []string // Advanced configuration used when Routing.Type=custom Routers Routers `json:",omitempty"` Methods Methods `json:",omitempty"` } type Router struct { // Router type ID. See RouterType for more info. Type RouterType // Parameters are extra configuration that this router might need. // A common one for HTTP router is "Endpoint". Parameters any } type ( Routers map[string]RouterParser Methods map[MethodName]Method ) func (m Methods) Check() error { // Check supported methods for _, mn := range MethodNameList { _, ok := m[mn] if !ok { return fmt.Errorf("method name %q is missing from Routing.Methods config param", mn) } } // Check unsupported methods for k := range m { seen := slices.Contains(MethodNameList, k) if seen { continue } return fmt.Errorf("method name %q is not a supported method on Routing.Methods config param", k) } return nil } type RouterParser struct { Router } func (r *RouterParser) UnmarshalJSON(b []byte) error { out := Router{} out.Parameters = &json.RawMessage{} if err := json.Unmarshal(b, &out); err != nil { return err } raw := out.Parameters.(*json.RawMessage) var p any switch out.Type { case RouterTypeHTTP: p = &HTTPRouterParams{} case RouterTypeDHT: p = &DHTRouterParams{} case RouterTypeSequential: p = &ComposableRouterParams{} case RouterTypeParallel: p = &ComposableRouterParams{} } if err := json.Unmarshal(*raw, &p); err != nil { return err } r.Router.Type = out.Type r.Router.Parameters = p return nil } // Type is the routing type. // Depending of the type we need to instantiate different Routing implementations. type RouterType string const ( RouterTypeHTTP RouterType = "http" // HTTP JSON API for delegated routing systems (IPIP-337). RouterTypeDHT RouterType = "dht" // DHT router. RouterTypeSequential RouterType = "sequential" // Router helper to execute several routers sequentially. RouterTypeParallel RouterType = "parallel" // Router helper to execute several routers in parallel. ) type DHTMode string const ( DHTModeServer DHTMode = "server" DHTModeClient DHTMode = "client" DHTModeAuto DHTMode = "auto" ) type MethodName string const ( MethodNameProvide MethodName = "provide" MethodNameFindProviders MethodName = "find-providers" MethodNameFindPeers MethodName = "find-peers" MethodNameGetIPNS MethodName = "get-ipns" MethodNamePutIPNS MethodName = "put-ipns" ) var MethodNameList = []MethodName{MethodNameProvide, MethodNameFindPeers, MethodNameFindProviders, MethodNameGetIPNS, MethodNamePutIPNS} type HTTPRouterParams struct { // Endpoint is the URL where the routing implementation will point to get the information. Endpoint string // MaxProvideBatchSize determines the maximum amount of CIDs sent per batch. // Servers might not accept more than 100 elements per batch. 100 elements by default. MaxProvideBatchSize int // MaxProvideConcurrency determines the number of threads used when providing content. GOMAXPROCS by default. MaxProvideConcurrency int } func (hrp *HTTPRouterParams) FillDefaults() { if hrp.MaxProvideBatchSize == 0 { hrp.MaxProvideBatchSize = 100 } if hrp.MaxProvideConcurrency == 0 { hrp.MaxProvideConcurrency = runtime.GOMAXPROCS(0) } } type DHTRouterParams struct { Mode DHTMode AcceleratedDHTClient bool `json:",omitempty"` PublicIPNetwork bool } type ComposableRouterParams struct { Routers []ConfigRouter Timeout *OptionalDuration `json:",omitempty"` } type ConfigRouter struct { RouterName string Timeout Duration IgnoreErrors bool ExecuteAfter *OptionalDuration `json:",omitempty"` } type Method struct { RouterName string } // getEnvOrDefault reads space or comma separated strings from env if present, // and uses provided defaultValue as a fallback func getEnvOrDefault(key string, defaultValue []string) []string { if value, exists := os.LookupEnv(key); exists { splitFunc := func(r rune) bool { return r == ',' || r == ' ' } return strings.FieldsFunc(value, splitFunc) } return defaultValue } // HasHTTPProviderConfigured checks if the node is configured to use HTTP routers // for providing content announcements. This is used when determining if the node // can provide content even when not connected to libp2p peers. // // Note: Right now we only support delegated HTTP content providing if Routing.Type=custom // and Routing.Routers are configured according to: // https://github.com/ipfs/kubo/blob/master/docs/delegated-routing.md#configuration-file-example // // This uses the `ProvideBitswap` request type that is not documented anywhere, // because we hoped something like IPIP-378 (https://github.com/ipfs/specs/pull/378) // would get finalized and we'd switch to that. It never happened due to politics, // and now we are stuck with ProvideBitswap being the only API that works. // Some people have reverse engineered it (example: // https://discuss.ipfs.tech/t/only-peers-found-from-dht-seem-to-be-getting-used-as-relays-so-cant-use-http-routers/19545/9) // and use it, so what we do here is the bare minimum to ensure their use case works // using this old API until something better is available. func (c *Config) HasHTTPProviderConfigured() bool { if len(c.Routing.Routers) == 0 { // No "custom" routers return false } method, ok := c.Routing.Methods[MethodNameProvide] if !ok { // No provide method configured return false } return c.routerSupportsHTTPProviding(method.RouterName) } // routerSupportsHTTPProviding checks if the supplied custom router is or // includes an HTTP-based router. func (c *Config) routerSupportsHTTPProviding(routerName string) bool { rp, ok := c.Routing.Routers[routerName] if !ok { // Router configured for providing doesn't exist return false } switch rp.Type { case RouterTypeHTTP: return true case RouterTypeParallel, RouterTypeSequential: // Check if any child router supports HTTP if children, ok := rp.Parameters.(*ComposableRouterParams); ok { for _, childRouter := range children.Routers { if c.routerSupportsHTTPProviding(childRouter.RouterName) { return true } } } } return false } ================================================ FILE: config/routing_test.go ================================================ package config import ( "encoding/json" "testing" "time" "github.com/stretchr/testify/require" ) func TestRouterParameters(t *testing.T) { require := require.New(t) sec := time.Second min := time.Minute r := Routing{ Type: NewOptionalString("custom"), Routers: map[string]RouterParser{ "router-dht": {Router{ Type: RouterTypeDHT, Parameters: DHTRouterParams{ Mode: "auto", AcceleratedDHTClient: true, PublicIPNetwork: false, }, }}, "router-parallel": { Router{ Type: RouterTypeParallel, Parameters: ComposableRouterParams{ Routers: []ConfigRouter{ { RouterName: "router-dht", Timeout: Duration{10 * time.Second}, IgnoreErrors: true, }, { RouterName: "router-dht", Timeout: Duration{10 * time.Second}, IgnoreErrors: false, ExecuteAfter: &OptionalDuration{&sec}, }, }, Timeout: &OptionalDuration{&min}, }, }, }, "router-sequential": { Router{ Type: RouterTypeSequential, Parameters: ComposableRouterParams{ Routers: []ConfigRouter{ { RouterName: "router-dht", Timeout: Duration{10 * time.Second}, IgnoreErrors: true, }, { RouterName: "router-dht", Timeout: Duration{10 * time.Second}, IgnoreErrors: false, }, }, Timeout: &OptionalDuration{&min}, }, }, }, }, Methods: Methods{ MethodNameFindPeers: { RouterName: "router-dht", }, MethodNameFindProviders: { RouterName: "router-dht", }, MethodNameGetIPNS: { RouterName: "router-sequential", }, MethodNameProvide: { RouterName: "router-parallel", }, MethodNamePutIPNS: { RouterName: "router-parallel", }, }, } out, err := json.Marshal(r) require.NoError(err) r2 := &Routing{} err = json.Unmarshal(out, r2) require.NoError(err) require.Equal(5, len(r2.Methods)) dhtp := r2.Routers["router-dht"].Parameters require.IsType(&DHTRouterParams{}, dhtp) sp := r2.Routers["router-sequential"].Parameters require.IsType(&ComposableRouterParams{}, sp) pp := r2.Routers["router-parallel"].Parameters require.IsType(&ComposableRouterParams{}, pp) } func TestMethods(t *testing.T) { require := require.New(t) methodsOK := Methods{ MethodNameFindPeers: { RouterName: "router-wrong", }, MethodNameFindProviders: { RouterName: "router-wrong", }, MethodNameGetIPNS: { RouterName: "router-wrong", }, MethodNameProvide: { RouterName: "router-wrong", }, MethodNamePutIPNS: { RouterName: "router-wrong", }, } require.NoError(methodsOK.Check()) methodsMissing := Methods{ MethodNameFindPeers: { RouterName: "router-wrong", }, MethodNameGetIPNS: { RouterName: "router-wrong", }, MethodNameProvide: { RouterName: "router-wrong", }, MethodNamePutIPNS: { RouterName: "router-wrong", }, } require.Error(methodsMissing.Check()) } ================================================ FILE: config/serialize/serialize.go ================================================ package fsrepo import ( "encoding/json" "errors" "fmt" "io" "os" "path/filepath" "github.com/ipfs/kubo/config" "github.com/facebookgo/atomicfile" ) // ErrNotInitialized is returned when we fail to read the config because the // repo doesn't exist. var ErrNotInitialized = errors.New("ipfs not initialized, please run 'ipfs init'") // ReadConfigFile reads the config from `filename` into `cfg`. func ReadConfigFile(filename string, cfg any) error { f, err := os.Open(filename) if err != nil { if os.IsNotExist(err) { err = ErrNotInitialized } return err } defer f.Close() if err := json.NewDecoder(f).Decode(cfg); err != nil { return fmt.Errorf("failure to decode config: %w", err) } return nil } // WriteConfigFile writes the config from `cfg` into `filename`. func WriteConfigFile(filename string, cfg any) error { err := os.MkdirAll(filepath.Dir(filename), 0o755) if err != nil { return err } f, err := atomicfile.New(filename, 0o600) if err != nil { return err } defer f.Close() return encode(f, cfg) } // encode configuration with JSON. func encode(w io.Writer, value any) error { // need to prettyprint, hence MarshalIndent, instead of Encoder buf, err := config.Marshal(value) if err != nil { return err } _, err = w.Write(buf) return err } // Load reads given file and returns the read config, or error. func Load(filename string) (*config.Config, error) { var cfg config.Config err := ReadConfigFile(filename, &cfg) if err != nil { return nil, err } return &cfg, err } ================================================ FILE: config/serialize/serialize_test.go ================================================ package fsrepo import ( "os" "runtime" "testing" config "github.com/ipfs/kubo/config" ) func TestConfig(t *testing.T) { const filename = ".ipfsconfig" cfgWritten := new(config.Config) cfgWritten.Identity.PeerID = "faketest" err := WriteConfigFile(filename, cfgWritten) if err != nil { t.Fatal(err) } cfgRead, err := Load(filename) if err != nil { t.Fatal(err) } if cfgWritten.Identity.PeerID != cfgRead.Identity.PeerID { t.Fatal() } st, err := os.Stat(filename) if err != nil { t.Fatalf("cannot stat config file: %v", err) } if runtime.GOOS != "windows" { // see https://golang.org/src/os/types_windows.go if g := st.Mode().Perm(); g&0o117 != 0 { t.Fatalf("config file should not be executable or accessible to world: %v", g) } } } ================================================ FILE: config/swarm.go ================================================ package config type SwarmConfig struct { // AddrFilters specifies a set libp2p addresses that we should never // dial or receive connections from. AddrFilters []string // DisableBandwidthMetrics disables recording of bandwidth metrics for a // slight reduction in memory usage. You probably don't need to set this // flag. DisableBandwidthMetrics bool // DisableNatPortMap turns off NAT port mapping (UPnP, etc.). DisableNatPortMap bool // RelayClient controls the client side of "auto relay" feature. // When enabled, the node will use relays if it is not publicly reachable. RelayClient RelayClient // RelayService.* controls the "relay service". // When enabled, node will provide a limited relay service to other peers. RelayService RelayService // EnableHolePunching enables the hole punching service. EnableHolePunching Flag `json:",omitempty"` // Transports contains flags to enable/disable libp2p transports. Transports Transports // ConnMgr configures the connection manager. ConnMgr ConnMgr // ResourceMgr configures the libp2p Network Resource Manager ResourceMgr ResourceMgr } type RelayClient struct { // Enables the auto relay feature: will use relays if it is not publicly reachable. Enabled Flag `json:",omitempty"` // StaticRelays configures static relays to use when this node is not // publicly reachable. If set, auto relay will not try to find any // other relay servers. StaticRelays []string `json:",omitempty"` } // RelayService configures the resources of the circuit v2 relay. // For every field a reasonable default will be defined in go-ipfs. type RelayService struct { // Enables the limited relay service for other peers (circuit v2 relay). Enabled Flag `json:",omitempty"` // ConnectionDurationLimit is the time limit before resetting a relayed connection. ConnectionDurationLimit *OptionalDuration `json:",omitempty"` // ConnectionDataLimit is the limit of data relayed (on each direction) before resetting the connection. ConnectionDataLimit *OptionalInteger `json:",omitempty"` // ReservationTTL is the duration of a new (or refreshed reservation). ReservationTTL *OptionalDuration `json:",omitempty"` // MaxReservations is the maximum number of active relay slots. MaxReservations *OptionalInteger `json:",omitempty"` // MaxCircuits is the maximum number of open relay connections for each peer; defaults to 16. MaxCircuits *OptionalInteger `json:",omitempty"` // BufferSize is the size of the relayed connection buffers. BufferSize *OptionalInteger `json:",omitempty"` // MaxReservationsPerIP is the maximum number of reservations originating from the same IP address. MaxReservationsPerIP *OptionalInteger `json:",omitempty"` // MaxReservationsPerASN is the maximum number of reservations origination from the same ASN. MaxReservationsPerASN *OptionalInteger `json:",omitempty"` } type Transports struct { // Network specifies the base transports we'll use for dialing. To // listen on a transport, add the transport to your Addresses.Swarm. Network struct { // All default to on. QUIC Flag `json:",omitempty"` TCP Flag `json:",omitempty"` Websocket Flag `json:",omitempty"` Relay Flag `json:",omitempty"` WebTransport Flag `json:",omitempty"` // except WebRTCDirect which is experimental and opt-in. WebRTCDirect Flag `json:",omitempty"` } // Security specifies the transports used to encrypt insecure network // transports. Security struct { // Defaults to 100. TLS Priority `json:",omitempty"` // Defaults to 300. Noise Priority `json:",omitempty"` } // Multiplexers specifies the transports used to multiplex multiple // connections over a single duplex connection. Multiplexers struct { // Defaults to 100. Yamux Priority `json:",omitempty"` } } // ConnMgr defines configuration options for the libp2p connection manager. type ConnMgr struct { Type *OptionalString `json:",omitempty"` LowWater *OptionalInteger `json:",omitempty"` HighWater *OptionalInteger `json:",omitempty"` GracePeriod *OptionalDuration `json:",omitempty"` SilencePeriod *OptionalDuration `json:",omitempty"` } // ResourceMgr defines configuration options for the libp2p Network Resource Manager // type ResourceMgr struct { // Enables the Network Resource Manager feature, default to on. Enabled Flag `json:",omitempty"` Limits swarmLimits `json:",omitempty"` MaxMemory *OptionalBytes `json:",omitempty"` MaxFileDescriptors *OptionalInteger `json:",omitempty"` // A list of multiaddrs that can bypass normal system limits (but are still // limited by the allowlist scope). Convenience config around // https://pkg.go.dev/github.com/libp2p/go-libp2p/p2p/host/resource-manager#Allowlist.Add Allowlist []string `json:",omitempty"` } const ( ResourceMgrSystemScope = "system" ResourceMgrTransientScope = "transient" ResourceMgrServiceScopePrefix = "svc:" ResourceMgrProtocolScopePrefix = "proto:" ResourceMgrPeerScopePrefix = "peer:" ) ================================================ FILE: config/types.go ================================================ package config import ( "bytes" "encoding/json" "fmt" "io" "strings" "time" humanize "github.com/dustin/go-humanize" ) // Strings is a helper type that (un)marshals a single string to/from a single // JSON string and a slice of strings to/from a JSON array of strings. type Strings []string // UnmarshalJSON conforms to the json.Unmarshaler interface. func (o *Strings) UnmarshalJSON(data []byte) error { if data[0] == '[' { return json.Unmarshal(data, (*[]string)(o)) } var value string if err := json.Unmarshal(data, &value); err != nil { return err } if len(value) == 0 { *o = []string{} } else { *o = []string{value} } return nil } // MarshalJSON conforms to the json.Marshaler interface. func (o Strings) MarshalJSON() ([]byte, error) { switch len(o) { case 0: return json.Marshal(nil) case 1: return json.Marshal(o[0]) default: return json.Marshal([]string(o)) } } var ( _ json.Unmarshaler = (*Strings)(nil) _ json.Marshaler = (*Strings)(nil) ) // Flag represents a ternary value: false (-1), default (0), or true (+1). // // When encoded in json, False is "false", Default is "null" (or empty), and True // is "true". type Flag int8 const ( False Flag = -1 Default Flag = 0 True Flag = 1 ) // WithDefault resolves the value of the flag given the provided default value. // // Panics if Flag is an invalid value. func (f Flag) WithDefault(defaultValue bool) bool { switch f { case False: return false case Default: return defaultValue case True: return true default: panic(fmt.Sprintf("invalid flag value %d", f)) } } func (f Flag) MarshalJSON() ([]byte, error) { switch f { case Default: return json.Marshal(nil) case True: return json.Marshal(true) case False: return json.Marshal(false) default: return nil, fmt.Errorf("invalid flag value: %d", f) } } func (f *Flag) UnmarshalJSON(input []byte) error { switch string(input) { case "null": *f = Default case "false": *f = False case "true": *f = True default: return fmt.Errorf("failed to unmarshal %q into a flag: must be null/undefined, true, or false", string(input)) } return nil } func (f Flag) String() string { switch f { case Default: return "default" case True: return "true" case False: return "false" default: return fmt.Sprintf("", f) } } // ResolveBoolFromConfig returns the resolved boolean value based on: // - If userSet is true, returns userValue (user explicitly set the flag) // - Otherwise, uses configFlag.WithDefault(defaultValue) (respects config or falls back to default) func ResolveBoolFromConfig(userValue bool, userSet bool, configFlag Flag, defaultValue bool) bool { if userSet { return userValue } return configFlag.WithDefault(defaultValue) } var ( _ json.Unmarshaler = (*Flag)(nil) _ json.Marshaler = (*Flag)(nil) ) // Priority represents a value with a priority where 0 means "default" and -1 // means "disabled". // // When encoded in json, Default is encoded as "null" and Disabled is encoded as // "false". type Priority int64 const ( DefaultPriority Priority = 0 Disabled Priority = -1 ) // WithDefault resolves the priority with the given default. // // If defaultPriority is Default/0, this function will return 0. // // Panics if the priority has an invalid value (e.g., not DefaultPriority, // Disabled, or > 0). func (p Priority) WithDefault(defaultPriority Priority) (priority int64, enabled bool) { switch p { case Disabled: return 0, false case DefaultPriority: switch defaultPriority { case Disabled: return 0, false case DefaultPriority: return 0, true default: if defaultPriority <= 0 { panic(fmt.Sprintf("invalid priority %d < 0", int64(defaultPriority))) } return int64(defaultPriority), true } default: if p <= 0 { panic(fmt.Sprintf("invalid priority %d < 0", int64(p))) } return int64(p), true } } func (p Priority) MarshalJSON() ([]byte, error) { // > 0 == Priority if p > 0 { return json.Marshal(int64(p)) } // <= 0 == special switch p { case DefaultPriority: return json.Marshal(nil) case Disabled: return json.Marshal(false) default: return nil, fmt.Errorf("invalid priority value: %d", p) } } func (p *Priority) UnmarshalJSON(input []byte) error { switch string(input) { case "null", "undefined": *p = DefaultPriority case "false": *p = Disabled case "true": return fmt.Errorf("'true' is not a valid priority") default: var priority int64 err := json.Unmarshal(input, &priority) if err != nil { return err } if priority <= 0 { return fmt.Errorf("priority must be positive: %d <= 0", priority) } *p = Priority(priority) } return nil } func (p Priority) String() string { if p > 0 { return fmt.Sprintf("%d", p) } switch p { case DefaultPriority: return "default" case Disabled: return "false" default: return fmt.Sprintf("", p) } } var ( _ json.Unmarshaler = (*Priority)(nil) _ json.Marshaler = (*Priority)(nil) ) // OptionalDuration wraps time.Duration to provide json serialization and deserialization. // // NOTE: the zero value encodes to JSON nill. type OptionalDuration struct { value *time.Duration } // NewOptionalDuration returns an OptionalDuration from a string. func NewOptionalDuration(d time.Duration) *OptionalDuration { return &OptionalDuration{value: &d} } func (d *OptionalDuration) UnmarshalJSON(input []byte) error { switch string(input) { case "null", "undefined", "\"null\"", "", "default", "\"\"", "\"default\"": *d = OptionalDuration{} return nil default: text := strings.Trim(string(input), "\"") value, err := time.ParseDuration(text) if err != nil { return err } *d = OptionalDuration{value: &value} return nil } } func (d *OptionalDuration) IsDefault() bool { return d == nil || d.value == nil } func (d *OptionalDuration) WithDefault(defaultValue time.Duration) time.Duration { if d == nil || d.value == nil { return defaultValue } return *d.value } func (d OptionalDuration) MarshalJSON() ([]byte, error) { if d.value == nil { return json.Marshal(nil) } return json.Marshal(d.value.String()) } func (d OptionalDuration) String() string { if d.value == nil { return "default" } return d.value.String() } var ( _ json.Unmarshaler = (*OptionalDuration)(nil) _ json.Marshaler = (*OptionalDuration)(nil) ) type Duration struct { time.Duration } func (d Duration) MarshalJSON() ([]byte, error) { return json.Marshal(d.String()) } func (d *Duration) UnmarshalJSON(b []byte) error { var v any if err := json.Unmarshal(b, &v); err != nil { return err } switch value := v.(type) { case float64: d.Duration = time.Duration(value) return nil case string: var err error d.Duration, err = time.ParseDuration(value) if err != nil { return err } return nil default: return fmt.Errorf("unable to parse duration, expected a duration string or a float, but got %T", v) } } var ( _ json.Unmarshaler = (*Duration)(nil) _ json.Marshaler = (*Duration)(nil) ) // OptionalInteger represents an integer that has a default value // // When encoded in json, Default is encoded as "null". type OptionalInteger struct { value *int64 } // NewOptionalInteger returns an OptionalInteger from a int64. func NewOptionalInteger(v int64) *OptionalInteger { return &OptionalInteger{value: &v} } // WithDefault resolves the integer with the given default. func (p *OptionalInteger) WithDefault(defaultValue int64) (value int64) { if p == nil || p.value == nil { return defaultValue } return *p.value } // IsDefault returns if this is a default optional integer. func (p *OptionalInteger) IsDefault() bool { return p == nil || p.value == nil } func (p OptionalInteger) MarshalJSON() ([]byte, error) { if p.value != nil { return json.Marshal(p.value) } return json.Marshal(nil) } func (p *OptionalInteger) UnmarshalJSON(input []byte) error { switch string(input) { case "null", "undefined": *p = OptionalInteger{} default: var value int64 err := json.Unmarshal(input, &value) if err != nil { return err } *p = OptionalInteger{value: &value} } return nil } func (p OptionalInteger) String() string { if p.value == nil { return "default" } return fmt.Sprintf("%d", *p.value) } var ( _ json.Unmarshaler = (*OptionalInteger)(nil) _ json.Marshaler = (*OptionalInteger)(nil) ) // OptionalString represents a string that has a default value // // When encoded in json, Default is encoded as "null". type OptionalString struct { value *string } // NewOptionalString returns an OptionalString from a string. func NewOptionalString(s string) *OptionalString { return &OptionalString{value: &s} } // WithDefault resolves the integer with the given default. func (p *OptionalString) WithDefault(defaultValue string) (value string) { if p == nil || p.value == nil { return defaultValue } return *p.value } // IsDefault returns if this is a default optional integer. func (p *OptionalString) IsDefault() bool { return p == nil || p.value == nil } func (p OptionalString) MarshalJSON() ([]byte, error) { if p.value != nil { return json.Marshal(p.value) } return json.Marshal(nil) } func (p *OptionalString) UnmarshalJSON(input []byte) error { switch string(input) { case "null", "undefined": *p = OptionalString{} default: var value string err := json.Unmarshal(input, &value) if err != nil { return err } *p = OptionalString{value: &value} } return nil } func (p OptionalString) String() string { if p.value == nil { return "default" } return *p.value } var ( _ json.Unmarshaler = (*OptionalString)(nil) _ json.Marshaler = (*OptionalString)(nil) ) // OptionalBytes represents a byte size that has a default value // // When encoded in json, Default is encoded as "null". // Stores the original string representation and parses on access. // Embeds OptionalString to share common functionality. type OptionalBytes struct { OptionalString } // NewOptionalBytes returns an OptionalBytes from a string. func NewOptionalBytes(s string) *OptionalBytes { return &OptionalBytes{OptionalString{value: &s}} } // IsDefault returns if this is a default optional byte value. func (p *OptionalBytes) IsDefault() bool { if p == nil { return true } return p.OptionalString.IsDefault() } // WithDefault resolves the byte size with the given default. // Parses the stored string value using humanize.ParseBytes. func (p *OptionalBytes) WithDefault(defaultValue uint64) (value uint64) { if p.IsDefault() { return defaultValue } strValue := p.OptionalString.WithDefault("") bytes, err := humanize.ParseBytes(strValue) if err != nil { // This should never happen as values are validated during UnmarshalJSON. // If it does, it indicates either config corruption or a programming error. panic(fmt.Sprintf("invalid byte size in OptionalBytes: %q - %v", strValue, err)) } return bytes } // UnmarshalJSON validates the input is a parseable byte size. func (p *OptionalBytes) UnmarshalJSON(input []byte) error { switch string(input) { case "null", "undefined": *p = OptionalBytes{} default: var value any err := json.Unmarshal(input, &value) if err != nil { return err } switch v := value.(type) { case float64: str := fmt.Sprintf("%.0f", v) p.value = &str case string: _, err := humanize.ParseBytes(v) if err != nil { return err } p.value = &v default: return fmt.Errorf("unable to parse byte size, expected a size string (e.g., \"5GiB\") or a number, but got %T", v) } } return nil } var ( _ json.Unmarshaler = (*OptionalBytes)(nil) _ json.Marshaler = (*OptionalBytes)(nil) ) type swarmLimits doNotUse var _ json.Unmarshaler = swarmLimits(false) func (swarmLimits) UnmarshalJSON(b []byte) error { d := json.NewDecoder(bytes.NewReader(b)) for { switch tok, err := d.Token(); err { case io.EOF: return nil case nil: switch tok { case json.Delim('{'), json.Delim('}'): // accept empty objects continue } //nolint return fmt.Errorf("The Swarm.ResourceMgr.Limits configuration has been removed in Kubo 0.19 and should be empty or not present. To set custom libp2p limits, read https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md#user-supplied-override-limits") default: return err } } } type experimentalAcceleratedDHTClient doNotUse var _ json.Unmarshaler = experimentalAcceleratedDHTClient(false) func (experimentalAcceleratedDHTClient) UnmarshalJSON(b []byte) error { d := json.NewDecoder(bytes.NewReader(b)) for { switch tok, err := d.Token(); err { case io.EOF: return nil case nil: switch tok { case json.Delim('{'), json.Delim('}'): // accept empty objects continue } //nolint return fmt.Errorf("The Experimental.AcceleratedDHTClient key has been moved to Routing.AcceleratedDHTClient in Kubo 0.21, please use this new key and remove the old one.") default: return err } } } // doNotUse is a type you must not use, it should be struct{} but encoding/json // does not support omitempty on structs and I can't be bothered to write custom // marshalers on all structs that have a doNotUse field. type doNotUse bool type graphsyncEnabled doNotUse var _ json.Unmarshaler = graphsyncEnabled(false) func (graphsyncEnabled) UnmarshalJSON(b []byte) error { d := json.NewDecoder(bytes.NewReader(b)) for { switch tok, err := d.Token(); err { case io.EOF: return nil case nil: switch tok { case json.Delim('{'), json.Delim('}'), false: // accept empty objects and false continue } //nolint return fmt.Errorf("Support for Experimental.GraphsyncEnabled has been removed in Kubo 0.25.0, please remove this key. For more details see https://github.com/ipfs/kubo/pull/9747.") default: return err } } } ================================================ FILE: config/types_test.go ================================================ package config import ( "bytes" "encoding/json" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestOptionalDuration(t *testing.T) { makeDurationPointer := func(d time.Duration) *time.Duration { return &d } t.Run("marshalling and unmarshalling", func(t *testing.T) { out, err := json.Marshal(OptionalDuration{value: makeDurationPointer(time.Second)}) if err != nil { t.Fatal(err) } expected := "\"1s\"" if string(out) != expected { t.Fatalf("expected %s, got %s", expected, string(out)) } var d OptionalDuration if err := json.Unmarshal(out, &d); err != nil { t.Fatal(err) } if *d.value != time.Second { t.Fatal("expected a second") } }) t.Run("default value", func(t *testing.T) { for _, jsonStr := range []string{"null", "\"null\"", "\"\"", "\"default\""} { var d OptionalDuration if !d.IsDefault() { t.Fatal("expected value to be the default initially") } if err := json.Unmarshal([]byte(jsonStr), &d); err != nil { t.Fatalf("%s failed to unmarshall with %s", jsonStr, err) } if dur := d.WithDefault(time.Hour); dur != time.Hour { t.Fatalf("expected default value to be used, got %s", dur) } if !d.IsDefault() { t.Fatal("expected value to be the default") } } }) t.Run("omitempty with default value", func(t *testing.T) { type Foo struct { D *OptionalDuration `json:",omitempty"` } // marshall to JSON without empty field out, err := json.Marshal(new(Foo)) if err != nil { t.Fatal(err) } if string(out) != "{}" { t.Fatalf("expected omitempty to omit the duration, got %s", out) } // unmarshall missing value and get the default var foo2 Foo if err := json.Unmarshal(out, &foo2); err != nil { t.Fatalf("%s failed to unmarshall with %s", string(out), err) } if dur := foo2.D.WithDefault(time.Hour); dur != time.Hour { t.Fatalf("expected default value to be used, got %s", dur) } if !foo2.D.IsDefault() { t.Fatal("expected value to be the default") } }) t.Run("roundtrip including the default values", func(t *testing.T) { for jsonStr, goValue := range map[string]OptionalDuration{ // there are various footguns user can hit, normalize them to the canonical default "null": {}, // JSON null → default value "\"null\"": {}, // JSON string "null" sent/set by "ipfs config" cli → default value "\"default\"": {}, // explicit "default" as string "\"\"": {}, // user removed custom value, empty string should also parse as default "\"1s\"": {value: makeDurationPointer(time.Second)}, "\"42h1m3s\"": {value: makeDurationPointer(42*time.Hour + 1*time.Minute + 3*time.Second)}, } { var d OptionalDuration err := json.Unmarshal([]byte(jsonStr), &d) if err != nil { t.Fatal(err) } if goValue.value == nil && d.value == nil { } else if goValue.value == nil && d.value != nil { t.Errorf("expected nil for %s, got %s", jsonStr, d) } else if *d.value != *goValue.value { t.Fatalf("expected %s for %s, got %s", goValue, jsonStr, d) } // Test Reverse out, err := json.Marshal(goValue) if err != nil { t.Fatal(err) } if goValue.value == nil { if !bytes.Equal(out, []byte("null")) { t.Fatalf("expected JSON null for %s, got %s", jsonStr, string(out)) } continue } if string(out) != jsonStr { t.Fatalf("expected %s, got %s", jsonStr, string(out)) } } }) t.Run("invalid duration values", func(t *testing.T) { for _, invalid := range []string{ "\"s\"", "\"1ę\"", "\"-1\"", "\"1H\"", "\"day\"", } { var d OptionalDuration err := json.Unmarshal([]byte(invalid), &d) if err == nil { t.Errorf("expected to fail to decode %s as an OptionalDuration, got %s instead", invalid, d) } } }) } func TestOneStrings(t *testing.T) { out, err := json.Marshal(Strings{"one"}) if err != nil { t.Fatal(err) } expected := "\"one\"" if string(out) != expected { t.Fatalf("expected %s, got %s", expected, string(out)) } } func TestNoStrings(t *testing.T) { out, err := json.Marshal(Strings{}) if err != nil { t.Fatal(err) } expected := "null" if string(out) != expected { t.Fatalf("expected %s, got %s", expected, string(out)) } } func TestManyStrings(t *testing.T) { out, err := json.Marshal(Strings{"one", "two"}) if err != nil { t.Fatal(err) } expected := "[\"one\",\"two\"]" if string(out) != expected { t.Fatalf("expected %s, got %s", expected, string(out)) } } func TestFunkyStrings(t *testing.T) { toParse := " [ \"one\", \"two\" ] " var s Strings if err := json.Unmarshal([]byte(toParse), &s); err != nil { t.Fatal(err) } if len(s) != 2 || s[0] != "one" && s[1] != "two" { t.Fatalf("unexpected result: %v", s) } } func TestFlag(t *testing.T) { // make sure we have the right zero value. var defaultFlag Flag if defaultFlag != Default { t.Errorf("expected default flag to be %q, got %q", Default, defaultFlag) } if defaultFlag.WithDefault(true) != true { t.Error("expected default & true to be true") } if defaultFlag.WithDefault(false) != false { t.Error("expected default & false to be false") } if True.WithDefault(false) != true { t.Error("default should only apply to default") } if False.WithDefault(true) != false { t.Error("default should only apply to default") } if True.WithDefault(true) != true { t.Error("true & true is true") } if False.WithDefault(true) != false { t.Error("false & false is false") } for jsonStr, goValue := range map[string]Flag{ "null": Default, "true": True, "false": False, } { var d Flag err := json.Unmarshal([]byte(jsonStr), &d) if err != nil { t.Fatal(err) } if d != goValue { t.Fatalf("expected %s, got %s", goValue, d) } // Reverse out, err := json.Marshal(goValue) if err != nil { t.Fatal(err) } if string(out) != jsonStr { t.Fatalf("expected %s, got %s", jsonStr, string(out)) } } type Foo struct { F Flag `json:",omitempty"` } out, err := json.Marshal(new(Foo)) if err != nil { t.Fatal(err) } expected := "{}" if string(out) != expected { t.Fatal("expected omitempty to omit the flag") } } func TestPriority(t *testing.T) { // make sure we have the right zero value. var defaultPriority Priority if defaultPriority != DefaultPriority { t.Errorf("expected default priority to be %q, got %q", DefaultPriority, defaultPriority) } if _, ok := defaultPriority.WithDefault(Disabled); ok { t.Error("should have been disabled") } if p, ok := defaultPriority.WithDefault(1); !ok || p != 1 { t.Errorf("priority should have been 1, got %d", p) } if p, ok := defaultPriority.WithDefault(DefaultPriority); !ok || p != 0 { t.Errorf("priority should have been 0, got %d", p) } for jsonStr, goValue := range map[string]Priority{ "null": DefaultPriority, "false": Disabled, "1": 1, "2": 2, "100": 100, } { var d Priority err := json.Unmarshal([]byte(jsonStr), &d) if err != nil { t.Fatal(err) } if d != goValue { t.Fatalf("expected %s, got %s", goValue, d) } // Reverse out, err := json.Marshal(goValue) if err != nil { t.Fatal(err) } if string(out) != jsonStr { t.Fatalf("expected %s, got %s", jsonStr, string(out)) } } type Foo struct { P Priority `json:",omitempty"` } out, err := json.Marshal(new(Foo)) if err != nil { t.Fatal(err) } expected := "{}" if string(out) != expected { t.Fatal("expected omitempty to omit the flag") } for _, invalid := range []string{ "0", "-1", "-2", "1.1", "0.0", } { var p Priority err := json.Unmarshal([]byte(invalid), &p) if err == nil { t.Errorf("expected to fail to decode %s as a priority", invalid) } } } func TestOptionalInteger(t *testing.T) { makeInt64Pointer := func(v int64) *int64 { return &v } var defaultOptionalInt OptionalInteger if !defaultOptionalInt.IsDefault() { t.Fatal("should be the default") } if val := defaultOptionalInt.WithDefault(0); val != 0 { t.Errorf("optional integer should have been 0, got %d", val) } if val := defaultOptionalInt.WithDefault(1); val != 1 { t.Errorf("optional integer should have been 1, got %d", val) } if val := defaultOptionalInt.WithDefault(-1); val != -1 { t.Errorf("optional integer should have been -1, got %d", val) } var filledInt OptionalInteger filledInt = OptionalInteger{value: makeInt64Pointer(1)} if filledInt.IsDefault() { t.Fatal("should not be the default") } if val := filledInt.WithDefault(0); val != 1 { t.Errorf("optional integer should have been 1, got %d", val) } if val := filledInt.WithDefault(-1); val != 1 { t.Errorf("optional integer should have been 1, got %d", val) } filledInt = OptionalInteger{value: makeInt64Pointer(0)} if val := filledInt.WithDefault(1); val != 0 { t.Errorf("optional integer should have been 0, got %d", val) } for jsonStr, goValue := range map[string]OptionalInteger{ "null": {}, "0": {value: makeInt64Pointer(0)}, "1": {value: makeInt64Pointer(1)}, "-1": {value: makeInt64Pointer(-1)}, } { var d OptionalInteger err := json.Unmarshal([]byte(jsonStr), &d) if err != nil { t.Fatal(err) } if goValue.value == nil && d.value == nil { } else if goValue.value == nil && d.value != nil { t.Errorf("expected default, got %s", d) } else if *d.value != *goValue.value { t.Fatalf("expected %s, got %s", goValue, d) } // Reverse out, err := json.Marshal(goValue) if err != nil { t.Fatal(err) } if string(out) != jsonStr { t.Fatalf("expected %s, got %s", jsonStr, string(out)) } } // marshal with omitempty type Foo struct { I *OptionalInteger `json:",omitempty"` } out, err := json.Marshal(new(Foo)) if err != nil { t.Fatal(err) } expected := "{}" if string(out) != expected { t.Fatal("expected omitempty to omit the optional integer") } // unmarshal from omitempty output and get default value var foo2 Foo if err := json.Unmarshal(out, &foo2); err != nil { t.Fatalf("%s failed to unmarshall with %s", string(out), err) } if i := foo2.I.WithDefault(42); i != 42 { t.Fatalf("expected default value to be used, got %d", i) } if !foo2.I.IsDefault() { t.Fatal("expected value to be the default") } // test invalid values for _, invalid := range []string{ "foo", "-1.1", "1.1", "0.0", "[]", } { var p OptionalInteger err := json.Unmarshal([]byte(invalid), &p) if err == nil { t.Errorf("expected to fail to decode %s as a priority", invalid) } } } func TestOptionalString(t *testing.T) { makeStringPointer := func(v string) *string { return &v } var defaultOptionalString OptionalString if !defaultOptionalString.IsDefault() { t.Fatal("should be the default") } if val := defaultOptionalString.WithDefault(""); val != "" { t.Errorf("optional string should have been empty, got %s", val) } if val := defaultOptionalString.String(); val != "default" { t.Fatalf("default optional string should be the 'default' string, got %s", val) } if val := defaultOptionalString.WithDefault("foo"); val != "foo" { t.Errorf("optional string should have been foo, got %s", val) } var filledStr OptionalString filledStr = OptionalString{value: makeStringPointer("foo")} if filledStr.IsDefault() { t.Fatal("should not be the default") } if val := filledStr.WithDefault("bar"); val != "foo" { t.Errorf("optional string should have been foo, got %s", val) } if val := filledStr.String(); val != "foo" { t.Fatalf("optional string should have been foo, got %s", val) } filledStr = OptionalString{value: makeStringPointer("")} if val := filledStr.WithDefault("foo"); val != "" { t.Errorf("optional string should have been 0, got %s", val) } for jsonStr, goValue := range map[string]OptionalString{ "null": {}, "\"0\"": {value: makeStringPointer("0")}, "\"\"": {value: makeStringPointer("")}, `"1"`: {value: makeStringPointer("1")}, `"-1"`: {value: makeStringPointer("-1")}, `"qwerty"`: {value: makeStringPointer("qwerty")}, } { var d OptionalString err := json.Unmarshal([]byte(jsonStr), &d) if err != nil { t.Fatal(err) } if goValue.value == nil && d.value == nil { } else if goValue.value == nil && d.value != nil { t.Errorf("expected default, got %s", d) } else if *d.value != *goValue.value { t.Fatalf("expected %s, got %s", goValue, d) } // Reverse out, err := json.Marshal(goValue) if err != nil { t.Fatal(err) } if string(out) != jsonStr { t.Fatalf("expected %s, got %s", jsonStr, string(out)) } } // marshal with omitempty type Foo struct { S *OptionalString `json:",omitempty"` } out, err := json.Marshal(new(Foo)) if err != nil { t.Fatal(err) } expected := "{}" if string(out) != expected { t.Fatal("expected omitempty to omit the optional integer") } // unmarshal from omitempty output and get default value var foo2 Foo if err := json.Unmarshal(out, &foo2); err != nil { t.Fatalf("%s failed to unmarshall with %s", string(out), err) } if s := foo2.S.WithDefault("foo"); s != "foo" { t.Fatalf("expected default value to be used, got %s", s) } if !foo2.S.IsDefault() { t.Fatal("expected value to be the default") } for _, invalid := range []string{ "[]", "{}", "0", "a", "'b'", } { var p OptionalString err := json.Unmarshal([]byte(invalid), &p) if err == nil { t.Errorf("expected to fail to decode %s as an optional string", invalid) } } } func TestOptionalBytes(t *testing.T) { makeStringPointer := func(v string) *string { return &v } t.Run("default value", func(t *testing.T) { var b OptionalBytes assert.True(t, b.IsDefault()) assert.Equal(t, uint64(0), b.WithDefault(0)) assert.Equal(t, uint64(1024), b.WithDefault(1024)) assert.Equal(t, "default", b.String()) }) t.Run("non-default value", func(t *testing.T) { b := OptionalBytes{OptionalString{value: makeStringPointer("1MiB")}} assert.False(t, b.IsDefault()) assert.Equal(t, uint64(1048576), b.WithDefault(512)) assert.Equal(t, "1MiB", b.String()) }) t.Run("JSON roundtrip", func(t *testing.T) { testCases := []struct { jsonInput string jsonOutput string expectedValue string }{ {"null", "null", ""}, {"\"256KiB\"", "\"256KiB\"", "256KiB"}, {"\"1MiB\"", "\"1MiB\"", "1MiB"}, {"\"5GiB\"", "\"5GiB\"", "5GiB"}, {"\"256KB\"", "\"256KB\"", "256KB"}, {"1048576", "\"1048576\"", "1048576"}, } for _, tc := range testCases { t.Run(tc.jsonInput, func(t *testing.T) { var b OptionalBytes err := json.Unmarshal([]byte(tc.jsonInput), &b) require.NoError(t, err) if tc.expectedValue == "" { assert.Nil(t, b.value) } else { require.NotNil(t, b.value) assert.Equal(t, tc.expectedValue, *b.value) } out, err := json.Marshal(b) require.NoError(t, err) assert.Equal(t, tc.jsonOutput, string(out)) }) } }) t.Run("parsing byte sizes", func(t *testing.T) { testCases := []struct { input string expected uint64 }{ {"256KiB", 262144}, {"1MiB", 1048576}, {"5GiB", 5368709120}, {"256KB", 256000}, {"1048576", 1048576}, } for _, tc := range testCases { t.Run(tc.input, func(t *testing.T) { var b OptionalBytes err := json.Unmarshal([]byte("\""+tc.input+"\""), &b) require.NoError(t, err) assert.Equal(t, tc.expected, b.WithDefault(0)) }) } }) t.Run("omitempty", func(t *testing.T) { type Foo struct { B *OptionalBytes `json:",omitempty"` } out, err := json.Marshal(new(Foo)) require.NoError(t, err) assert.Equal(t, "{}", string(out)) var foo2 Foo err = json.Unmarshal(out, &foo2) require.NoError(t, err) if foo2.B != nil { assert.Equal(t, uint64(1024), foo2.B.WithDefault(1024)) assert.True(t, foo2.B.IsDefault()) } else { // When field is omitted, pointer is nil which is also considered default t.Log("B is nil, which is acceptable for omitempty") } }) t.Run("invalid values", func(t *testing.T) { invalidInputs := []string{ "\"5XiB\"", "\"invalid\"", "\"\"", "[]", "{}", } for _, invalid := range invalidInputs { t.Run(invalid, func(t *testing.T) { var b OptionalBytes err := json.Unmarshal([]byte(invalid), &b) assert.Error(t, err) }) } }) t.Run("panic on invalid stored value", func(t *testing.T) { // This tests that if somehow an invalid value gets stored // (bypassing UnmarshalJSON validation), WithDefault will panic invalidValue := "invalid-size" b := OptionalBytes{OptionalString{value: &invalidValue}} assert.Panics(t, func() { b.WithDefault(1024) }, "should panic on invalid stored value") }) } ================================================ FILE: config/version.go ================================================ package config const DefaultSwarmCheckPercentThreshold = 5 // Version allows controlling things like custom user agent and update checks. type Version struct { // Optional suffix to the AgentVersion presented by `ipfs id` and exposed // via libp2p identify protocol. AgentSuffix *OptionalString `json:",omitempty"` // Detect when to warn about new version when observed via libp2p identify SwarmCheckEnabled Flag `json:",omitempty"` SwarmCheckPercentThreshold *OptionalInteger `json:",omitempty"` } ================================================ FILE: core/.gitignore ================================================ .testdb ================================================ FILE: core/builder.go ================================================ package core import ( "context" "fmt" "reflect" "sync" "time" "github.com/ipfs/boxo/bootstrap" "github.com/ipfs/kubo/core/node" "github.com/ipfs/go-metrics-interface" "go.uber.org/dig" "go.uber.org/fx" ) // FXNodeInfo contains information useful for adding fx options. // This is the extension point for providing more info/context to fx plugins // to make decisions about what options to include. type FXNodeInfo struct { FXOptions []fx.Option } // fxOptFunc takes in some info about the IPFS node and returns the full set of fx opts to use. type fxOptFunc func(FXNodeInfo) ([]fx.Option, error) var fxOptionFuncs []fxOptFunc // RegisterFXOptionFunc registers a function that is run before the fx app is initialized. // Functions are invoked in the order they are registered, // and the resulting options are passed into the next function's FXNodeInfo. // // Note that these are applied globally, by all invocations of NewNode. // There are multiple places in Kubo that construct nodes, such as: // - Repo initialization // - Daemon initialization // - When running migrations // - etc. // // If your fx options are doing anything sophisticated, you should keep this in mind. // // For example, if you plug in a blockservice that disallows non-allowlisted CIDs, // this may break migrations that fetch migration code over IPFS. func RegisterFXOptionFunc(optFunc fxOptFunc) { fxOptionFuncs = append(fxOptionFuncs, optFunc) } // from https://stackoverflow.com/a/59348871 type valueContext struct { context.Context } func (valueContext) Deadline() (deadline time.Time, ok bool) { return } func (valueContext) Done() <-chan struct{} { return nil } func (valueContext) Err() error { return nil } type BuildCfg = node.BuildCfg // Alias for compatibility until we properly refactor the constructor interface // NewNode constructs and returns an IpfsNode using the given cfg. func NewNode(ctx context.Context, cfg *BuildCfg) (*IpfsNode, error) { // save this context as the "lifetime" ctx. lctx := ctx // derive a new context that ignores cancellations from the lifetime ctx. ctx, cancel := context.WithCancel(valueContext{ctx}) // add a metrics scope. ctx = metrics.CtxScope(ctx, "ipfs") n := &IpfsNode{ ctx: ctx, } opts := []fx.Option{ node.IPFS(ctx, cfg), fx.NopLogger, } for _, optFunc := range fxOptionFuncs { var err error opts, err = optFunc(FXNodeInfo{FXOptions: opts}) if err != nil { cancel() return nil, fmt.Errorf("building fx opts: %w", err) } } //nolint:staticcheck // https://github.com/ipfs/kubo/pull/9423#issuecomment-1341038770 opts = append(opts, fx.Extract(n)) app := fx.New(opts...) var once sync.Once var stopErr error n.stop = func() error { once.Do(func() { stopErr = app.Stop(context.Background()) if stopErr != nil { log.Error("failure on stop: ", stopErr) } // Cancel the context _after_ the app has stopped. cancel() }) return stopErr } n.IsOnline = cfg.Online go func() { // Shut down the application if the lifetime context is canceled. // NOTE: we _should_ stop the application by calling `Close()` // on the process. But we currently manage everything with contexts. select { case <-lctx.Done(): err := n.stop() if err != nil { log.Error("failure on stop: ", err) } case <-ctx.Done(): } }() if app.Err() != nil { return nil, logAndUnwrapFxError(app.Err()) } if err := app.Start(ctx); err != nil { return nil, logAndUnwrapFxError(err) } // TODO: How soon will bootstrap move to libp2p? if !cfg.Online { return n, nil } return n, n.Bootstrap(bootstrap.DefaultBootstrapConfig) } // Log the entire `app.Err()` but return only the innermost one to the user // given the full error can be very long (as it can expose the entire build // graph in a single string). // // The fx.App error exposed through `app.Err()` normally contains un-exported // errors from its low-level `dig` package: // * https://github.com/uber-go/dig/blob/5e5a20d/error.go#L82 // These usually wrap themselves in many layers to expose where in the build // chain did the error happen. Although useful for a developer that needs to // debug it, it can be very confusing for a user that just wants the IPFS error // that he can probably fix without being aware of the entire chain. // Unwrapping everything is not the best solution as there can be useful // information in the intermediate errors, mainly in the next to last error // that locates which component is the build error coming from, but it's the // best we can do at the moment given all errors in dig are private and we // just have the generic `RootCause` API. func logAndUnwrapFxError(fxAppErr error) error { if fxAppErr == nil { return nil } log.Error("constructing the node: ", fxAppErr) err := fxAppErr for { extractedErr := dig.RootCause(err) // Note that the `RootCause` name is misleading as it just unwraps only // *one* error layer at a time, so we need to continuously call it. if !reflect.TypeOf(extractedErr).Comparable() { // Some internal errors are not comparable (e.g., `dig.errMissingTypes` // which is a slice) and we can't go further. break } if extractedErr == err { // We didn't unwrap any new error in the last call, reached the innermost one. break } err = extractedErr } return fmt.Errorf("constructing the node (see log for full detail): %w", err) } ================================================ FILE: core/commands/active.go ================================================ package commands import ( "fmt" "io" "slices" "text/tabwriter" "time" oldcmds "github.com/ipfs/kubo/commands" cmds "github.com/ipfs/go-ipfs-cmds" ) const ( verboseOptionName = "verbose" ) var ActiveReqsCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List commands run on this IPFS node.", ShortDescription: ` Lists running and recently run commands. `, }, NoLocal: true, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx := env.(*oldcmds.Context) return cmds.EmitOnce(res, ctx.ReqLog.Report()) }, Options: []cmds.Option{ cmds.BoolOption(verboseOptionName, "v", "Print extra information."), }, Subcommands: map[string]*cmds.Command{ "clear": clearInactiveCmd, "set-time": setRequestClearCmd, }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *[]*cmds.ReqLogEntry) error { verbose, _ := req.Options[verboseOptionName].(bool) tw := tabwriter.NewWriter(w, 4, 4, 2, ' ', 0) if verbose { fmt.Fprint(tw, "ID\t") } fmt.Fprint(tw, "Command\t") if verbose { fmt.Fprint(tw, "Arguments\tOptions\t") } fmt.Fprintln(tw, "Active\tStartTime\tRunTime") for _, req := range *out { if verbose { fmt.Fprintf(tw, "%d\t", req.ID) } fmt.Fprintf(tw, "%s\t", req.Command) if verbose { fmt.Fprintf(tw, "%v\t[", req.Args) var keys []string for k := range req.Options { keys = append(keys, k) } slices.Sort(keys) for _, k := range keys { fmt.Fprintf(tw, "%s=%v,", k, req.Options[k]) } fmt.Fprintf(tw, "]\t") } var live time.Duration if req.Active { live = time.Since(req.StartTime) } else { live = req.EndTime.Sub(req.StartTime) } t := req.StartTime.Format(time.Stamp) fmt.Fprintf(tw, "%t\t%s\t%s\n", req.Active, t, live) } tw.Flush() return nil }), }, Type: []*cmds.ReqLogEntry{}, } var clearInactiveCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Clear inactive requests from the log.", }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx := env.(*oldcmds.Context) ctx.ReqLog.ClearInactive() return nil }, } var setRequestClearCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Set how long to keep inactive requests in the log.", }, Arguments: []cmds.Argument{ cmds.StringArg("time", true, false, "Time to keep inactive requests in log."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { tval, err := time.ParseDuration(req.Arguments[0]) if err != nil { return err } ctx := env.(*oldcmds.Context) ctx.ReqLog.SetKeepTime(tval) return nil }, } ================================================ FILE: core/commands/add.go ================================================ package commands import ( "errors" "fmt" "io" "os" gopath "path" "strconv" "strings" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" "github.com/cheggaaa/pb" "github.com/ipfs/boxo/files" uio "github.com/ipfs/boxo/ipld/unixfs/io" mfs "github.com/ipfs/boxo/mfs" "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/verifcid" cmds "github.com/ipfs/go-ipfs-cmds" ipld "github.com/ipfs/go-ipld-format" coreiface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/core/coreiface/options" mh "github.com/multiformats/go-multihash" ) // ErrDepthLimitExceeded indicates that the max depth has been exceeded. var ErrDepthLimitExceeded = errors.New("depth limit exceeded") type AddEvent struct { Name string Hash string `json:",omitempty"` Bytes int64 `json:",omitempty"` Size string `json:",omitempty"` Mode string `json:",omitempty"` Mtime int64 `json:",omitempty"` MtimeNsecs int `json:",omitempty"` } const ( pinNameOptionName = "pin-name" quietOptionName = "quiet" quieterOptionName = "quieter" silentOptionName = "silent" progressOptionName = "progress" trickleOptionName = "trickle" wrapOptionName = "wrap-with-directory" onlyHashOptionName = "only-hash" chunkerOptionName = "chunker" pinOptionName = "pin" rawLeavesOptionName = "raw-leaves" maxFileLinksOptionName = "max-file-links" maxDirectoryLinksOptionName = "max-directory-links" maxHAMTFanoutOptionName = "max-hamt-fanout" noCopyOptionName = "nocopy" fstoreCacheOptionName = "fscache" cidVersionOptionName = "cid-version" hashOptionName = "hash" inlineOptionName = "inline" inlineLimitOptionName = "inline-limit" toFilesOptionName = "to-files" preserveModeOptionName = "preserve-mode" preserveMtimeOptionName = "preserve-mtime" modeOptionName = "mode" mtimeOptionName = "mtime" mtimeNsecsOptionName = "mtime-nsecs" fastProvideRootOptionName = "fast-provide-root" fastProvideWaitOptionName = "fast-provide-wait" emptyDirsOptionName = "empty-dirs" ) const ( adderOutChanSize = 8 ) var AddCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Add a file or directory to IPFS.", ShortDescription: ` Adds the content of to IPFS. Use -r to add directories (recursively). FAST PROVIDE OPTIMIZATION: When you add content to IPFS, the sweep provider queues it for efficient DHT provides over time. While this is resource-efficient, other peers won't find your content immediately after 'ipfs add' completes. To make sharing faster, 'ipfs add' does an immediate provide of the root CID to the DHT in addition to the regular queue. This complements the sweep provider: fast-provide handles the urgent case (root CIDs that users share and reference), while the sweep provider efficiently provides all blocks according to Provide.Strategy over time. By default, this immediate provide runs in the background without blocking the command. If you need certainty that the root CID is discoverable before the command returns (e.g., sharing a link immediately), use --fast-provide-wait to wait for the provide to complete. Use --fast-provide-root=false to skip this optimization. This works best with the sweep provider and accelerated DHT client. Automatically skipped when DHT is not available. `, LongDescription: ` Adds the content of to IPFS. Use -r to add directories. Note that directories are added recursively, and big files are chunked, to form the IPFS MerkleDAG. Learn more: https://docs.ipfs.tech/concepts/merkle-dag/ If the daemon is not running, it will just add locally to the repo at $IPFS_PATH. If the daemon is started later, it will be advertised after a few seconds when the provide system runs. BASIC EXAMPLES: The wrap option, '-w', wraps the file (or files, if using the recursive option) in a directory. This directory contains only the files which have been added, and means that the file retains its filename. For example: > ipfs add example.jpg added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg > ipfs add example.jpg -w added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg added QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx You can now refer to the added file in a gateway, like so: /ipfs/QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx/example.jpg Files imported with 'ipfs add' are protected from GC (implicit '--pin=true'), but it is up to you to remember the returned CID to get the data back later. If you need to back up or transport content-addressed data using a non-IPFS medium, CID can be preserved with CAR files. See 'dag export' and 'dag import' for more information. MFS INTEGRATION: Passing '--to-files' creates a reference in Files API (MFS), making it easier to find it in the future: > ipfs files mkdir -p /myfs/dir > ipfs add example.jpg --to-files /myfs/dir/ > ipfs files ls /myfs/dir/ example.jpg See 'ipfs files --help' to learn more about using MFS for keeping track of added files and directories. SYMLINK HANDLING: By default, symbolic links are preserved as UnixFS symlink nodes that store the target path. Use --dereference-symlinks to resolve symlinks to their target content instead: > ipfs add -r --dereference-symlinks ./mydir This resolves all symlinks, including CLI arguments and those found inside directories. Symlinks to files become regular file content, symlinks to directories are traversed and their contents are added. CHUNKING EXAMPLES: The chunker option, '-s', specifies the chunking strategy that dictates how to break files into blocks. Blocks with same content can be deduplicated. Different chunking strategies will produce different hashes for the same file. The default is a fixed block size of 256 * 1024 bytes, 'size-262144'. Alternatively, you can use the Buzhash or Rabin fingerprint chunker for content defined chunking by specifying buzhash or rabin-[min]-[avg]-[max] (where min/avg/max refer to the desired chunk sizes in bytes), e.g. 'rabin-262144-524288-1048576'. The maximum accepted value for 'size-N' and rabin 'max' parameter is 2MiB minus 256 bytes (2096896 bytes). The 256-byte overhead budget is reserved for protobuf/UnixFS framing so that serialized blocks stay within the 2MiB block size limit from the bitswap spec. The buzhash chunker uses a fixed internal maximum of 512KiB and is not affected. Only the fixed-size chunker ('size-N') guarantees that the same data will always produce the same CID. The rabin and buzhash chunkers may change their internal parameters in a future release. The following examples use very small byte sizes to demonstrate the properties of the different chunkers on a small file. You'll likely want to use a 1024 times larger chunk sizes for most files. > ipfs add --chunker=size-2048 ipfs-logo.svg added QmafrLBfzRLV4XSH1XcaMMeaXEUhDJjmtDfsYU95TrWG87 ipfs-logo.svg > ipfs add --chunker=rabin-512-1024-2048 ipfs-logo.svg added Qmf1hDN65tR55Ubh2RN1FPxr69xq3giVBz1KApsresY8Gn ipfs-logo.svg You can now check what blocks have been created by: > ipfs ls QmafrLBfzRLV4XSH1XcaMMeaXEUhDJjmtDfsYU95TrWG87 QmY6yj1GsermExDXoosVE3aSPxdMNYr6aKuw3nA8LoWPRS 2059 Qmf7ZQeSxq2fJVJbCmgTrLLVN9tDR9Wy5k75DxQKuz5Gyt 1195 > ipfs ls Qmf1hDN65tR55Ubh2RN1FPxr69xq3giVBz1KApsresY8Gn QmY6yj1GsermExDXoosVE3aSPxdMNYr6aKuw3nA8LoWPRS 2059 QmerURi9k4XzKCaaPbsK6BL5pMEjF7PGphjDvkkjDtsVf3 868 QmQB28iwSriSUSMqG2nXDTLtdPHgWb4rebBrU7Q1j4vxPv 338 ADVANCED CONFIGURATION: Finally, a note on hash (CID) determinism and 'ipfs add' command. Almost all the flags provided by this command will change the final CID, and new flags may be added in the future. It is not guaranteed for the implicit defaults of 'ipfs add' to remain the same in future Kubo releases, or for other IPFS software to use the same import parameters as Kubo. Note: CIDv1 is automatically used when using non-default options like custom hash functions or when raw-leaves is explicitly enabled. Use Import.* configuration options to override global implicit defaults: https://github.com/ipfs/kubo/blob/master/docs/config.md#import `, }, Arguments: []cmds.Argument{ cmds.FileArg("path", true, true, "The path to a file to be added to IPFS.").EnableRecursive().EnableStdin(), }, Options: []cmds.Option{ // Input Processing cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive) cmds.OptionDerefArgs, // DEPRECATED: use --dereference-symlinks instead cmds.OptionStdinName, // a builtin option that optionally allows wrapping stdin into a named file cmds.OptionHidden, cmds.OptionIgnore, cmds.OptionIgnoreRules, cmds.BoolOption(emptyDirsOptionName, "E", "Include empty directories in the import.").WithDefault(config.DefaultUnixFSIncludeEmptyDirs), cmds.OptionDerefSymlinks, // resolve symlinks to their target content // Output Control cmds.BoolOption(quietOptionName, "q", "Write minimal output."), cmds.BoolOption(quieterOptionName, "Q", "Write only final hash."), cmds.BoolOption(silentOptionName, "Write no output."), cmds.BoolOption(progressOptionName, "p", "Stream progress data."), // Basic Add Behavior cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."), cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."), cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true), cmds.StringOption(pinNameOptionName, "Name to use for the pin. Requires explicit value (e.g., --pin-name=myname)."), // MFS Integration cmds.StringOption(toFilesOptionName, "Add reference to Files API (MFS) at the provided path."), // CID & Hashing cmds.IntOption(cidVersionOptionName, "CID version (0 or 1). CIDv1 automatically enables raw-leaves and is required for non-sha2-256 hashes. Default: Import.CidVersion"), cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. Default: Import.HashFunction"), cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. Note: CIDv1 automatically enables raw-leaves. Default: false for CIDv0, true for CIDv1 (Import.UnixFSRawLeaves)"), // Chunking & DAG Structure cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash. Files larger than chunk size are split into multiple blocks. Default: Import.UnixFSChunker"), cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), // Advanced UnixFS Limits cmds.IntOption(maxFileLinksOptionName, "Limit the maximum number of links in UnixFS file nodes to this value. WARNING: experimental. Default: Import.UnixFSFileMaxLinks"), cmds.IntOption(maxDirectoryLinksOptionName, "Limit the maximum number of links in UnixFS basic directory nodes to this value. WARNING: experimental, Import.UnixFSHAMTDirectorySizeThreshold is safer. Default: Import.UnixFSDirectoryMaxLinks"), cmds.IntOption(maxHAMTFanoutOptionName, "Limit the maximum number of links of a UnixFS HAMT directory node to this (power of 2, between 8 and 1024). WARNING: experimental, Import.UnixFSHAMTDirectorySizeThreshold is safer. Default: Import.UnixFSHAMTDirectoryMaxFanout"), // Experimental Features cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. WARNING: experimental"), cmds.IntOption(inlineLimitOptionName, fmt.Sprintf("Maximum block size to inline. Maximum: %d bytes. WARNING: experimental", verifcid.DefaultMaxIdentityDigestSize)).WithDefault(32), cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. WARNING: experimental"), cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. WARNING: experimental"), cmds.BoolOption(preserveModeOptionName, "Apply existing POSIX permissions to created UnixFS entries. WARNING: experimental, forces dag-pb for root block, disables raw-leaves"), cmds.BoolOption(preserveMtimeOptionName, "Apply existing POSIX modification time to created UnixFS entries. WARNING: experimental, forces dag-pb for root block, disables raw-leaves"), cmds.UintOption(modeOptionName, "Custom POSIX file mode to store in created UnixFS entries. WARNING: experimental, forces dag-pb for root block, disables raw-leaves"), cmds.Int64Option(mtimeOptionName, "Custom POSIX modification time to store in created UnixFS entries (seconds before or after the Unix Epoch). WARNING: experimental, forces dag-pb for root block, disables raw-leaves"), cmds.UintOption(mtimeNsecsOptionName, "Custom POSIX modification time (optional time fraction in nanoseconds)"), cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CID to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"), cmds.BoolOption(fastProvideWaitOptionName, "Block until the immediate provide completes before returning. Default: Import.FastProvideWait"), }, PreRun: func(req *cmds.Request, env cmds.Environment) error { quiet, _ := req.Options[quietOptionName].(bool) quieter, _ := req.Options[quieterOptionName].(bool) quiet = quiet || quieter silent, _ := req.Options[silentOptionName].(bool) if !quiet && !silent { // ipfs cli progress bar defaults to true unless quiet or silent is used _, found := req.Options[progressOptionName].(bool) if !found { req.Options[progressOptionName] = true } } return nil }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } nd, err := cmdenv.GetNode(env) if err != nil { return err } cfg, err := nd.Repo.Config() if err != nil { return err } progress, _ := req.Options[progressOptionName].(bool) trickle, trickleSet := req.Options[trickleOptionName].(bool) wrap, _ := req.Options[wrapOptionName].(bool) onlyHash, _ := req.Options[onlyHashOptionName].(bool) silent, _ := req.Options[silentOptionName].(bool) chunker, _ := req.Options[chunkerOptionName].(string) dopin, _ := req.Options[pinOptionName].(bool) pinName, pinNameSet := req.Options[pinNameOptionName].(string) rawblks, rbset := req.Options[rawLeavesOptionName].(bool) maxFileLinks, maxFileLinksSet := req.Options[maxFileLinksOptionName].(int) maxDirectoryLinks, maxDirectoryLinksSet := req.Options[maxDirectoryLinksOptionName].(int) maxHAMTFanout, maxHAMTFanoutSet := req.Options[maxHAMTFanoutOptionName].(int) var sizeEstimationMode uio.SizeEstimationMode nocopy, _ := req.Options[noCopyOptionName].(bool) fscache, _ := req.Options[fstoreCacheOptionName].(bool) cidVer, cidVerSet := req.Options[cidVersionOptionName].(int) hashFunStr, _ := req.Options[hashOptionName].(string) inline, _ := req.Options[inlineOptionName].(bool) inlineLimit, _ := req.Options[inlineLimitOptionName].(int) // Validate inline-limit doesn't exceed the maximum identity digest size if inline && inlineLimit > verifcid.DefaultMaxIdentityDigestSize { return fmt.Errorf("inline-limit %d exceeds maximum allowed size of %d bytes", inlineLimit, verifcid.DefaultMaxIdentityDigestSize) } // Validate pin name if pinNameSet { if err := cmdutils.ValidatePinName(pinName); err != nil { return err } } toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string) preserveMode, _ := req.Options[preserveModeOptionName].(bool) preserveMtime, _ := req.Options[preserveMtimeOptionName].(bool) mode, _ := req.Options[modeOptionName].(uint) mtime, _ := req.Options[mtimeOptionName].(int64) mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint) fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool) fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool) emptyDirs, _ := req.Options[emptyDirsOptionName].(bool) // Note: --dereference-args is deprecated but still works for backwards compatibility. // The help text marks it as DEPRECATED. Users should use --dereference-symlinks instead, // which is a superset (resolves both CLI arg symlinks AND nested symlinks in directories). // Wire --trickle from config if !trickleSet && !cfg.Import.UnixFSDAGLayout.IsDefault() { layout := cfg.Import.UnixFSDAGLayout.WithDefault(config.DefaultUnixFSDAGLayout) trickle = layout == config.DAGLayoutTrickle } if chunker == "" { chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker) } if hashFunStr == "" { hashFunStr = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction) } if !cidVerSet && !cfg.Import.CidVersion.IsDefault() { cidVerSet = true cidVer = int(cfg.Import.CidVersion.WithDefault(config.DefaultCidVersion)) } // Pin names are only used when explicitly provided via --pin-name=value if !rbset && cfg.Import.UnixFSRawLeaves != config.Default { rbset = true rawblks = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves) } if !maxFileLinksSet && !cfg.Import.UnixFSFileMaxLinks.IsDefault() { maxFileLinksSet = true maxFileLinks = int(cfg.Import.UnixFSFileMaxLinks.WithDefault(config.DefaultUnixFSFileMaxLinks)) } if !maxDirectoryLinksSet && !cfg.Import.UnixFSDirectoryMaxLinks.IsDefault() { maxDirectoryLinksSet = true maxDirectoryLinks = int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks)) } if !maxHAMTFanoutSet && !cfg.Import.UnixFSHAMTDirectoryMaxFanout.IsDefault() { maxHAMTFanoutSet = true maxHAMTFanout = int(cfg.Import.UnixFSHAMTDirectoryMaxFanout.WithDefault(config.DefaultUnixFSHAMTDirectoryMaxFanout)) } // SizeEstimationMode is always set from config (no CLI flag) sizeEstimationMode = cfg.Import.HAMTSizeEstimationMode() fastProvideRoot = config.ResolveBoolFromConfig(fastProvideRoot, fastProvideRootSet, cfg.Import.FastProvideRoot, config.DefaultFastProvideRoot) fastProvideWait = config.ResolveBoolFromConfig(fastProvideWait, fastProvideWaitSet, cfg.Import.FastProvideWait, config.DefaultFastProvideWait) // Storing optional mode or mtime (UnixFS 1.5) requires root block // to always be 'dag-pb' and not 'raw'. Below adjusts raw-leaves setting, if possible. if preserveMode || preserveMtime || mode != 0 || mtime != 0 { // Error if --raw-leaves flag was explicitly passed by the user. // (let user make a decision to manually disable it and retry) if rbset && rawblks { return fmt.Errorf("%s can't be used with UnixFS metadata like mode or modification time", rawLeavesOptionName) } // No explicit preference from user, disable raw-leaves and continue rbset = true rawblks = false } if onlyHash && toFilesSet { return fmt.Errorf("%s and %s options are not compatible", onlyHashOptionName, toFilesOptionName) } if !dopin && pinNameSet { return fmt.Errorf("%s option requires %s to be set", pinNameOptionName, pinOptionName) } if wrap && toFilesSet { return fmt.Errorf("%s and %s options are not compatible", wrapOptionName, toFilesOptionName) } hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)] if !ok { return fmt.Errorf("unrecognized hash function: %q", strings.ToLower(hashFunStr)) } enc, err := cmdenv.GetCidEncoder(req) if err != nil { return err } toadd := req.Files if wrap { toadd = files.NewSliceDirectory([]files.DirEntry{ files.FileEntry("", req.Files), }) } opts := []options.UnixfsAddOption{ options.Unixfs.Hash(hashFunCode), options.Unixfs.Inline(inline), options.Unixfs.InlineLimit(inlineLimit), options.Unixfs.Chunker(chunker), options.Unixfs.Pin(dopin, pinName), options.Unixfs.HashOnly(onlyHash), options.Unixfs.FsCache(fscache), options.Unixfs.Nocopy(nocopy), options.Unixfs.Progress(progress), options.Unixfs.Silent(silent), options.Unixfs.PreserveMode(preserveMode), options.Unixfs.PreserveMtime(preserveMtime), options.Unixfs.IncludeEmptyDirs(emptyDirs), } if mode != 0 { opts = append(opts, options.Unixfs.Mode(os.FileMode(mode))) } if mtime != 0 { opts = append(opts, options.Unixfs.Mtime(mtime, uint32(mtimeNsecs))) } else if mtimeNsecs != 0 { return fmt.Errorf("option %q requires %q to be provided as well", mtimeNsecsOptionName, mtimeOptionName) } if cidVerSet { opts = append(opts, options.Unixfs.CidVersion(cidVer)) } if rbset { opts = append(opts, options.Unixfs.RawLeaves(rawblks)) } if maxFileLinksSet { opts = append(opts, options.Unixfs.MaxFileLinks(maxFileLinks)) } if maxDirectoryLinksSet { opts = append(opts, options.Unixfs.MaxDirectoryLinks(maxDirectoryLinks)) } if maxHAMTFanoutSet { opts = append(opts, options.Unixfs.MaxHAMTFanout(maxHAMTFanout)) } // SizeEstimationMode is always set from config opts = append(opts, options.Unixfs.SizeEstimationMode(sizeEstimationMode)) if trickle { opts = append(opts, options.Unixfs.Layout(options.TrickleLayout)) } opts = append(opts, nil) // events option placeholder ipfsNode, err := cmdenv.GetNode(env) if err != nil { return err } var added int var fileAddedToMFS bool var lastRootCid path.ImmutablePath // Track the root CID for fast-provide addit := toadd.Entries() for addit.Next() { _, dir := addit.Node().(files.Directory) errCh := make(chan error, 1) events := make(chan any, adderOutChanSize) opts[len(opts)-1] = options.Unixfs.Events(events) go func() { var err error defer close(events) pathAdded, err := api.Unixfs().Add(req.Context, addit.Node(), opts...) if err != nil { errCh <- err return } // Store the root CID for potential fast-provide operation lastRootCid = pathAdded // creating MFS pointers when optional --to-files is set if toFilesSet { if addit.Name() == "" { errCh <- fmt.Errorf("%s: cannot add unnamed files to MFS", toFilesOptionName) return } if toFilesStr == "" { toFilesStr = "/" } toFilesDst, err := checkPath(toFilesStr) if err != nil { errCh <- fmt.Errorf("%s: %w", toFilesOptionName, err) return } dstAsDir := toFilesDst[len(toFilesDst)-1] == '/' if dstAsDir { mfsNode, err := mfs.Lookup(ipfsNode.FilesRoot, toFilesDst) // confirm dst exists if err != nil { errCh <- fmt.Errorf("%s: MFS destination directory %q does not exist: %w", toFilesOptionName, toFilesDst, err) return } // confirm dst is a dir if mfsNode.Type() != mfs.TDir { errCh <- fmt.Errorf("%s: MFS destination %q is not a directory", toFilesOptionName, toFilesDst) return } // if MFS destination is a dir, append filename to the dir path toFilesDst += gopath.Base(addit.Name()) } // error if we try to overwrite a preexisting file destination if fileAddedToMFS && !dstAsDir { errCh <- fmt.Errorf("%s: MFS destination is a file: only one entry can be copied to %q", toFilesOptionName, toFilesDst) return } _, err = mfs.Lookup(ipfsNode.FilesRoot, gopath.Dir(toFilesDst)) if err != nil { errCh <- fmt.Errorf("%s: MFS destination parent %q %q does not exist: %w", toFilesOptionName, toFilesDst, gopath.Dir(toFilesDst), err) return } var nodeAdded ipld.Node nodeAdded, err = api.Dag().Get(req.Context, pathAdded.RootCid()) if err != nil { errCh <- err return } err = mfs.PutNode(ipfsNode.FilesRoot, toFilesDst, nodeAdded) if err != nil { errCh <- fmt.Errorf("%s: cannot put node in path %q: %w", toFilesOptionName, toFilesDst, err) return } fileAddedToMFS = true } errCh <- err }() for event := range events { output, ok := event.(*coreiface.AddEvent) if !ok { return errors.New("unknown event type") } h := "" if (output.Path != path.ImmutablePath{}) { h = enc.Encode(output.Path.RootCid()) } if !dir && addit.Name() != "" { output.Name = addit.Name() } else { output.Name = gopath.Join(addit.Name(), output.Name) } output.Mode = addit.Node().Mode() if ts := addit.Node().ModTime(); !ts.IsZero() { output.Mtime = addit.Node().ModTime().Unix() output.MtimeNsecs = addit.Node().ModTime().Nanosecond() } addEvent := AddEvent{ Name: output.Name, Hash: h, Bytes: output.Bytes, Size: output.Size, Mtime: output.Mtime, MtimeNsecs: output.MtimeNsecs, } if output.Mode != 0 { addEvent.Mode = "0" + strconv.FormatUint(uint64(output.Mode), 8) } if output.Mtime > 0 { addEvent.Mtime = output.Mtime if output.MtimeNsecs > 0 { addEvent.MtimeNsecs = output.MtimeNsecs } } if err := res.Emit(&addEvent); err != nil { return err } } if err := <-errCh; err != nil { return err } added++ } if addit.Err() != nil { return addit.Err() } if added == 0 { return fmt.Errorf("expected a file argument") } // Apply fast-provide-root if the flag is enabled if fastProvideRoot && (lastRootCid != path.ImmutablePath{}) { cfg, err := ipfsNode.Repo.Config() if err != nil { return err } if err := cmdenv.ExecuteFastProvide(req.Context, ipfsNode, cfg, lastRootCid.RootCid(), fastProvideWait, dopin, dopin, toFilesSet); err != nil { return err } } else if !fastProvideRoot { if fastProvideWait { log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config", "wait-flag-ignored", true) } else { log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config") } } return nil }, PostRun: cmds.PostRunMap{ cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { sizeChan := make(chan int64, 1) outChan := make(chan any) req := res.Request() // Could be slow. go func() { size, err := req.Files.Size() if err != nil { log.Warnf("error getting files size: %s", err) // see comment above return } sizeChan <- size }() progressBar := func(wait chan struct{}) { defer close(wait) quiet, _ := req.Options[quietOptionName].(bool) quieter, _ := req.Options[quieterOptionName].(bool) quiet = quiet || quieter progress, _ := req.Options[progressOptionName].(bool) var bar *pb.ProgressBar if progress { bar = pb.New64(0).SetUnits(pb.U_BYTES) bar.ManualUpdate = true bar.ShowTimeLeft = false bar.ShowPercent = false bar.Output = os.Stderr bar.Start() } lastFile := "" lastHash := "" var totalProgress, prevFiles, lastBytes int64 LOOP: for { select { case out, ok := <-outChan: if !ok { if quieter { fmt.Fprintln(os.Stdout, lastHash) } break LOOP } output := out.(*AddEvent) if len(output.Hash) > 0 { lastHash = output.Hash if quieter { continue } if progress { // clear progress bar line before we print "added x" output fmt.Fprintf(os.Stderr, "\033[2K\r") } if quiet { fmt.Fprintf(os.Stdout, "%s\n", output.Hash) } else { fmt.Fprintf(os.Stdout, "added %s %s\n", output.Hash, cmdenv.EscNonPrint(output.Name)) } } else { if !progress { continue } if len(lastFile) == 0 { lastFile = output.Name } if output.Name != lastFile || output.Bytes < lastBytes { prevFiles += lastBytes lastFile = output.Name } lastBytes = output.Bytes delta := prevFiles + lastBytes - totalProgress totalProgress = bar.Add64(delta) } if progress { bar.Update() } case size := <-sizeChan: if progress { bar.Total = size bar.ShowPercent = true bar.ShowBar = true bar.ShowTimeLeft = true } case <-req.Context.Done(): // don't set or print error here, that happens in the goroutine below return } } if progress && bar.Total == 0 && bar.Get() != 0 { bar.Total = bar.Get() bar.ShowPercent = true bar.ShowBar = true bar.ShowTimeLeft = true bar.Update() } } if e := res.Error(); e != nil { close(outChan) return e } wait := make(chan struct{}) go progressBar(wait) defer func() { <-wait }() defer close(outChan) for { v, err := res.Next() if err != nil { if err == io.EOF { return nil } return err } select { case outChan <- v: case <-req.Context.Done(): return req.Context.Err() } } }, }, Type: AddEvent{}, } ================================================ FILE: core/commands/bitswap.go ================================================ package commands import ( "fmt" "io" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" humanize "github.com/dustin/go-humanize" bitswap "github.com/ipfs/boxo/bitswap" "github.com/ipfs/boxo/bitswap/server" cidutil "github.com/ipfs/go-cidutil" cmds "github.com/ipfs/go-ipfs-cmds" peer "github.com/libp2p/go-libp2p/core/peer" ) var BitswapCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Interact with the bitswap agent.", ShortDescription: ``, }, Subcommands: map[string]*cmds.Command{ "stat": bitswapStatCmd, "wantlist": showWantlistCmd, "ledger": ledgerCmd, "reprovide": deprecatedBitswapReprovideCmd, }, } const ( peerOptionName = "peer" ) var deprecatedBitswapReprovideCmd = &cmds.Command{ Status: cmds.Deprecated, Helptext: cmds.HelpText{ Tagline: "Deprecated command to announce to bitswap. Use 'ipfs routing reprovide' instead.", ShortDescription: ` 'ipfs bitswap reprovide' is a legacy plumbing command used to announce to DHT. Deprecated, use modern 'ipfs routing reprovide' instead.`, }, Run: reprovideRoutingCmd.Run, // alias to routing reprovide to not break existing users } var showWantlistCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Show blocks currently on the wantlist.", ShortDescription: ` Print out all blocks currently on the bitswap wantlist for the local peer.`, }, Options: []cmds.Option{ cmds.StringOption(peerOptionName, "p", "Specify which peer to show wantlist for. Default: self."), }, Type: KeyList{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } if !nd.IsOnline { return ErrNotOnline } bs := nd.Bitswap pstr, found := req.Options[peerOptionName].(string) if found { pid, err := peer.Decode(pstr) if err != nil { return err } if pid != nd.Identity { return cmds.EmitOnce(res, &KeyList{bs.WantlistForPeer(pid)}) } } return cmds.EmitOnce(res, &KeyList{bs.GetWantlist()}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *KeyList) error { enc, err := cmdenv.GetLowLevelCidEncoder(req) if err != nil { return err } // sort the keys first cidutil.Sort(out.Keys) for _, key := range out.Keys { fmt.Fprintln(w, enc.Encode(key)) } return nil }), }, } const ( bitswapVerboseOptionName = "verbose" bitswapHumanOptionName = "human" ) var bitswapStatCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Show some diagnostic information on the bitswap agent.", ShortDescription: ``, }, Options: []cmds.Option{ cmds.BoolOption(bitswapVerboseOptionName, "v", "Print extra information"), cmds.BoolOption(bitswapHumanOptionName, "Print sizes in human readable format (e.g., 1K 234M 2G)"), }, Type: bitswap.Stat{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } if !nd.IsOnline { return cmds.Errorf(cmds.ErrClient, "unable to run offline: %s", ErrNotOnline) } st, err := nd.Bitswap.Stat() if err != nil { return err } return cmds.EmitOnce(res, st) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, s *bitswap.Stat) error { enc, err := cmdenv.GetLowLevelCidEncoder(req) if err != nil { return err } verbose, _ := req.Options[bitswapVerboseOptionName].(bool) human, _ := req.Options[bitswapHumanOptionName].(bool) fmt.Fprintln(w, "bitswap status") fmt.Fprintf(w, "\tblocks received: %d\n", s.BlocksReceived) fmt.Fprintf(w, "\tblocks sent: %d\n", s.BlocksSent) if human { fmt.Fprintf(w, "\tdata received: %s\n", humanize.Bytes(s.DataReceived)) fmt.Fprintf(w, "\tdata sent: %s\n", humanize.Bytes(s.DataSent)) } else { fmt.Fprintf(w, "\tdata received: %d\n", s.DataReceived) fmt.Fprintf(w, "\tdata sent: %d\n", s.DataSent) } fmt.Fprintf(w, "\tdup blocks received: %d\n", s.DupBlksReceived) if human { fmt.Fprintf(w, "\tdup data received: %s\n", humanize.Bytes(s.DupDataReceived)) } else { fmt.Fprintf(w, "\tdup data received: %d\n", s.DupDataReceived) } fmt.Fprintf(w, "\twantlist [%d keys]\n", len(s.Wantlist)) for _, k := range s.Wantlist { fmt.Fprintf(w, "\t\t%s\n", enc.Encode(k)) } fmt.Fprintf(w, "\tpartners [%d]\n", len(s.Peers)) if verbose { for _, p := range s.Peers { fmt.Fprintf(w, "\t\t%s\n", p) } } return nil }), }, } var ledgerCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Show the current ledger for a peer.", ShortDescription: ` The Bitswap decision engine tracks the number of bytes exchanged between IPFS nodes, and stores this information as a collection of ledgers. This command prints the ledger associated with a given peer. `, }, Arguments: []cmds.Argument{ cmds.StringArg("peer", true, false, "The PeerID (B58) of the ledger to inspect."), }, Type: server.Receipt{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } if !nd.IsOnline { return ErrNotOnline } partner, err := peer.Decode(req.Arguments[0]) if err != nil { return err } return cmds.EmitOnce(res, nd.Bitswap.LedgerForPeer(partner)) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *server.Receipt) error { fmt.Fprintf(w, "Ledger for %s\n"+ "Debt ratio:\t%f\n"+ "Exchanges:\t%d\n"+ "Bytes sent:\t%d\n"+ "Bytes received:\t%d\n\n", out.Peer, out.Value, out.Exchanged, out.Sent, out.Recv) return nil }), }, } ================================================ FILE: core/commands/block.go ================================================ package commands import ( "errors" "fmt" "io" "os" "github.com/ipfs/boxo/files" "github.com/ipfs/kubo/config" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" options "github.com/ipfs/kubo/core/coreiface/options" cmds "github.com/ipfs/go-ipfs-cmds" mh "github.com/multiformats/go-multihash" ) type BlockStat struct { Key string Size int } func (bs BlockStat) String() string { return fmt.Sprintf("Key: %s\nSize: %d\n", bs.Key, bs.Size) } var BlockCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Interact with raw IPFS blocks.", ShortDescription: ` 'ipfs block' is a plumbing command used to manipulate raw IPFS blocks. Reads from stdin or writes to stdout. A block is identified by a Multihash passed with a valid CID. `, }, Subcommands: map[string]*cmds.Command{ "stat": blockStatCmd, "get": blockGetCmd, "put": blockPutCmd, "rm": blockRmCmd, }, } var blockStatCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Print information of a raw IPFS block.", ShortDescription: ` 'ipfs block stat' is a plumbing command for retrieving information on raw IPFS blocks. It outputs the following to stdout: Key - the CID of the block Size - the size of the block in bytes `, }, Arguments: []cmds.Argument{ cmds.StringArg("cid", true, false, "The CID of an existing block to stat.").EnableStdin(), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } p, err := cmdutils.PathOrCidPath(req.Arguments[0]) if err != nil { return err } b, err := api.Block().Stat(req.Context, p) if err != nil { return err } return cmds.EmitOnce(res, &BlockStat{ Key: b.Path().RootCid().String(), Size: b.Size(), }) }, Type: BlockStat{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, bs *BlockStat) error { _, err := fmt.Fprintf(w, "%s", bs) return err }), }, } var blockGetCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Get a raw IPFS block.", ShortDescription: ` 'ipfs block get' is a plumbing command for retrieving raw IPFS blocks. It takes a , and outputs the block to stdout. `, HTTP: &cmds.HTTPHelpText{ ResponseContentType: "application/vnd.ipld.raw", }, }, Arguments: []cmds.Argument{ cmds.StringArg("cid", true, false, "The CID of an existing block to get.").EnableStdin(), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } p, err := cmdutils.PathOrCidPath(req.Arguments[0]) if err != nil { return err } r, err := api.Block().Get(req.Context, p) if err != nil { return err } res.SetEncodingType(cmds.OctetStream) res.SetContentType("application/vnd.ipld.raw") return res.Emit(r) }, } const ( blockFormatOptionName = "format" blockCidCodecOptionName = "cid-codec" mhtypeOptionName = "mhtype" mhlenOptionName = "mhlen" ) var blockPutCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Store input as an IPFS block.", ShortDescription: ` 'ipfs block put' is a plumbing command for storing raw IPFS blocks. It reads data from stdin, and outputs the block's CID to stdout. Unless cid-codec is specified, this command returns raw (0x55) CIDv1 CIDs. Passing alternative --cid-codec does not modify imported data, nor run any validation. It is provided solely for convenience for users who create blocks in userland. NOTE: Do not use --format for any new code. It got superseded by --cid-codec and left only for backward compatibility when a legacy CIDv0 is required (--format=v0). `, }, Arguments: []cmds.Argument{ cmds.FileArg("data", true, true, "The data to be stored as an IPFS block.").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption(blockCidCodecOptionName, "Multicodec to use in returned CID").WithDefault("raw"), cmds.StringOption(mhtypeOptionName, "Multihash hash function"), cmds.IntOption(mhlenOptionName, "Multihash hash length").WithDefault(-1), cmds.BoolOption(pinOptionName, "Pin added blocks recursively").WithDefault(false), cmdutils.AllowBigBlockOption, cmds.StringOption(blockFormatOptionName, "f", "Use legacy format for returned CID (DEPRECATED)"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } nd, err := cmdenv.GetNode(env) if err != nil { return err } cfg, err := nd.Repo.Config() if err != nil { return err } mhtype, _ := req.Options[mhtypeOptionName].(string) if mhtype == "" { mhtype = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction) } mhtval, ok := mh.Names[mhtype] if !ok { return fmt.Errorf("unrecognized multihash function: %s", mhtype) } mhlen, ok := req.Options[mhlenOptionName].(int) if !ok { return errors.New("missing option \"mhlen\"") } cidCodec, _ := req.Options[blockCidCodecOptionName].(string) format, _ := req.Options[blockFormatOptionName].(string) // deprecated // use of legacy 'format' needs to suppress 'cid-codec' if format != "" { if cidCodec != "" && cidCodec != "raw" { return fmt.Errorf("unable to use %q (deprecated) and a custom %q at the same time", blockFormatOptionName, blockCidCodecOptionName) } cidCodec = "" // makes it no-op } pin, _ := req.Options[pinOptionName].(bool) it := req.Files.Entries() for it.Next() { file := files.FileFromEntry(it) if file == nil { return errors.New("expected a file") } p, err := api.Block().Put(req.Context, file, options.Block.Hash(mhtval, mhlen), options.Block.CidCodec(cidCodec), options.Block.Format(format), options.Block.Pin(pin)) if err != nil { return err } if err := cmdutils.CheckBlockSize(req, uint64(p.Size())); err != nil { return err } err = res.Emit(&BlockStat{ Key: p.Path().RootCid().String(), Size: p.Size(), }) if err != nil { return err } } return it.Err() }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, bs *BlockStat) error { _, err := fmt.Fprintf(w, "%s\n", bs.Key) return err }), }, Type: BlockStat{}, } const ( forceOptionName = "force" blockQuietOptionName = "quiet" ) type removedBlock struct { Hash string `json:",omitempty"` Error string `json:",omitempty"` } var blockRmCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Remove IPFS block(s) from the local datastore.", ShortDescription: ` 'ipfs block rm' is a plumbing command for removing raw ipfs blocks. It takes a list of CIDs to remove from the local datastore.. `, }, Arguments: []cmds.Argument{ cmds.StringArg("cid", true, true, "CIDs of block(s) to remove."), }, Options: []cmds.Option{ cmds.BoolOption(forceOptionName, "f", "Ignore nonexistent blocks."), cmds.BoolOption(blockQuietOptionName, "q", "Write minimal output."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } force, _ := req.Options[forceOptionName].(bool) quiet, _ := req.Options[blockQuietOptionName].(bool) // TODO: use batching coreapi when done for _, b := range req.Arguments { p, err := cmdutils.PathOrCidPath(b) if err != nil { return err } rp, _, err := api.ResolvePath(req.Context, p) if err != nil { return err } err = api.Block().Rm(req.Context, rp, options.Block.Force(force)) if err != nil { if err := res.Emit(&removedBlock{ Hash: rp.RootCid().String(), Error: err.Error(), }); err != nil { return err } continue } if !quiet { err := res.Emit(&removedBlock{ Hash: rp.RootCid().String(), }) if err != nil { return err } } } return nil }, PostRun: cmds.PostRunMap{ cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { someFailed := false for { res, err := res.Next() if err == io.EOF { break } else if err != nil { return err } r := res.(*removedBlock) if r.Hash == "" && r.Error != "" { return fmt.Errorf("aborted: %s", r.Error) } else if r.Error != "" { someFailed = true fmt.Fprintf(os.Stderr, "cannot remove %s: %s\n", r.Hash, r.Error) } else { fmt.Fprintf(os.Stdout, "removed %s\n", r.Hash) } } if someFailed { return fmt.Errorf("some blocks not removed") } return nil }, }, Type: removedBlock{}, } ================================================ FILE: core/commands/bootstrap.go ================================================ package commands import ( "errors" "fmt" "io" "slices" "strings" cmds "github.com/ipfs/go-ipfs-cmds" config "github.com/ipfs/kubo/config" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" repo "github.com/ipfs/kubo/repo" fsrepo "github.com/ipfs/kubo/repo/fsrepo" peer "github.com/libp2p/go-libp2p/core/peer" ma "github.com/multiformats/go-multiaddr" ) type BootstrapOutput struct { Peers []string } var peerOptionDesc = "A peer to add to the bootstrap list (in the format '/')" var BootstrapCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Show or edit the list of bootstrap peers.", ShortDescription: ` Running 'ipfs bootstrap' with no arguments will run 'ipfs bootstrap list'. ` + bootstrapSecurityWarning, }, Run: bootstrapListCmd.Run, Encoders: bootstrapListCmd.Encoders, Type: bootstrapListCmd.Type, Subcommands: map[string]*cmds.Command{ "list": bootstrapListCmd, "add": bootstrapAddCmd, "rm": bootstrapRemoveCmd, }, } var bootstrapAddCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Add peers to the bootstrap list.", ShortDescription: `Outputs a list of peers that were added (that weren't already in the bootstrap list). The special values 'default' and 'auto' can be used to add the default bootstrap peers. Both are equivalent and will add the 'auto' placeholder to the bootstrap list, which gets resolved using the AutoConf system. ` + bootstrapSecurityWarning, }, Arguments: []cmds.Argument{ cmds.StringArg("peer", false, true, peerOptionDesc).EnableStdin(), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { if err := req.ParseBodyArgs(); err != nil { return err } inputPeers := req.Arguments if len(inputPeers) == 0 { return errors.New("no bootstrap peers to add") } // Convert "default" to "auto" for backward compatibility for i, peer := range inputPeers { if peer == "default" { inputPeers[i] = "auto" } } cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err } r, err := fsrepo.Open(cfgRoot) if err != nil { return err } defer r.Close() cfg, err := r.Config() if err != nil { return err } // Check if trying to add "auto" when AutoConf is disabled for _, peer := range inputPeers { if peer == config.AutoPlaceholder && !cfg.AutoConf.Enabled.WithDefault(config.DefaultAutoConfEnabled) { return errors.New("cannot add default bootstrap peers: AutoConf is disabled (AutoConf.Enabled=false). Enable AutoConf by setting AutoConf.Enabled=true in your config, or add specific peer addresses instead") } } added, err := bootstrapAdd(r, cfg, inputPeers) if err != nil { return err } return cmds.EmitOnce(res, &BootstrapOutput{added}) }, Type: BootstrapOutput{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *BootstrapOutput) error { return bootstrapWritePeers(w, "added ", out.Peers) }), }, } const ( bootstrapAllOptionName = "all" ) var bootstrapRemoveCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Remove peers from the bootstrap list.", ShortDescription: `Outputs the list of peers that were removed. ` + bootstrapSecurityWarning, }, Arguments: []cmds.Argument{ cmds.StringArg("peer", false, true, peerOptionDesc).EnableStdin(), }, Options: []cmds.Option{ cmds.BoolOption(bootstrapAllOptionName, "Remove all bootstrap peers. (Deprecated, use 'all' subcommand)"), }, Subcommands: map[string]*cmds.Command{ "all": bootstrapRemoveAllCmd, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { all, _ := req.Options[bootstrapAllOptionName].(bool) cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err } r, err := fsrepo.Open(cfgRoot) if err != nil { return err } defer r.Close() cfg, err := r.Config() if err != nil { return err } var removed []string if all { removed, err = bootstrapRemoveAll(r, cfg) } else { if err := req.ParseBodyArgs(); err != nil { return err } removed, err = bootstrapRemove(r, cfg, req.Arguments) } if err != nil { return err } return cmds.EmitOnce(res, &BootstrapOutput{removed}) }, Type: BootstrapOutput{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *BootstrapOutput) error { return bootstrapWritePeers(w, "removed ", out.Peers) }), }, } var bootstrapRemoveAllCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Remove all peers from the bootstrap list.", ShortDescription: `Outputs the list of peers that were removed.`, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err } r, err := fsrepo.Open(cfgRoot) if err != nil { return err } defer r.Close() cfg, err := r.Config() if err != nil { return err } removed, err := bootstrapRemoveAll(r, cfg) if err != nil { return err } return cmds.EmitOnce(res, &BootstrapOutput{removed}) }, Type: BootstrapOutput{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *BootstrapOutput) error { return bootstrapWritePeers(w, "removed ", out.Peers) }), }, } var bootstrapListCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Show peers in the bootstrap list.", ShortDescription: "Peers are output in the format '/'.", }, Options: []cmds.Option{ cmds.BoolOption(configExpandAutoName, "Expand 'auto' placeholders from AutoConf service."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err } r, err := fsrepo.Open(cfgRoot) if err != nil { return err } defer r.Close() cfg, err := r.Config() if err != nil { return err } // Check if user wants to expand auto values expandAuto, _ := req.Options[configExpandAutoName].(bool) if expandAuto { // Use the same expansion method as the daemon expandedBootstrap := cfg.BootstrapWithAutoConf() return cmds.EmitOnce(res, &BootstrapOutput{expandedBootstrap}) } // Simply return the bootstrap config as-is, including any "auto" values return cmds.EmitOnce(res, &BootstrapOutput{cfg.Bootstrap}) }, Type: BootstrapOutput{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *BootstrapOutput) error { return bootstrapWritePeers(w, "", out.Peers) }), }, } func bootstrapWritePeers(w io.Writer, prefix string, peers []string) error { slices.SortStableFunc(peers, func(a, b string) int { return strings.Compare(a, b) }) for _, peer := range peers { _, err := w.Write([]byte(prefix + peer + "\n")) if err != nil { return err } } return nil } func bootstrapAdd(r repo.Repo, cfg *config.Config, peers []string) ([]string, error) { // Validate peers - skip validation for "auto" placeholder for _, p := range peers { if p == config.AutoPlaceholder { continue // Skip validation for "auto" placeholder } m, err := ma.NewMultiaddr(p) if err != nil { return nil, err } tpt, p2ppart := ma.SplitLast(m) if p2ppart == nil || p2ppart.Protocol().Code != ma.P_P2P { return nil, fmt.Errorf("invalid bootstrap address: %s", p) } if tpt == nil { return nil, fmt.Errorf("bootstrap address without a transport: %s", p) } } addedMap := map[string]struct{}{} addedList := make([]string, 0, len(peers)) // re-add cfg bootstrap peers to rm dupes bpeers := cfg.Bootstrap cfg.Bootstrap = nil // add new peers for _, s := range peers { if _, found := addedMap[s]; found { continue } cfg.Bootstrap = append(cfg.Bootstrap, s) addedList = append(addedList, s) addedMap[s] = struct{}{} } // add back original peers. in this order so that we output them. for _, s := range bpeers { if _, found := addedMap[s]; found { continue } cfg.Bootstrap = append(cfg.Bootstrap, s) addedMap[s] = struct{}{} } if err := r.SetConfig(cfg); err != nil { return nil, err } return addedList, nil } func bootstrapRemove(r repo.Repo, cfg *config.Config, toRemove []string) ([]string, error) { // Check if bootstrap contains "auto" hasAuto := slices.Contains(cfg.Bootstrap, config.AutoPlaceholder) if hasAuto && cfg.AutoConf.Enabled.WithDefault(config.DefaultAutoConfEnabled) { // Cannot selectively remove peers when using "auto" bootstrap // Users should either disable AutoConf or replace "auto" with specific peers return nil, fmt.Errorf("cannot remove individual bootstrap peers when using 'auto' placeholder: the 'auto' value is managed by AutoConf. Either disable AutoConf by setting AutoConf.Enabled=false and replace 'auto' with specific peer addresses, or use 'ipfs bootstrap rm --all' to remove all peers") } // Original logic for non-auto bootstrap removed := make([]peer.AddrInfo, 0, len(toRemove)) keep := make([]peer.AddrInfo, 0, len(cfg.Bootstrap)) toRemoveAddr, err := config.ParseBootstrapPeers(toRemove) if err != nil { return nil, err } toRemoveMap := make(map[peer.ID][]ma.Multiaddr, len(toRemoveAddr)) for _, addr := range toRemoveAddr { toRemoveMap[addr.ID] = addr.Addrs } peers, err := cfg.BootstrapPeers() if err != nil { return nil, err } for _, p := range peers { addrs, ok := toRemoveMap[p.ID] // not in the remove set? if !ok { keep = append(keep, p) continue } // remove the entire peer? if len(addrs) == 0 { removed = append(removed, p) continue } var keptAddrs, removedAddrs []ma.Multiaddr // remove specific addresses filter: for _, addr := range p.Addrs { for _, addr2 := range addrs { if addr.Equal(addr2) { removedAddrs = append(removedAddrs, addr) continue filter } } keptAddrs = append(keptAddrs, addr) } if len(removedAddrs) > 0 { removed = append(removed, peer.AddrInfo{ID: p.ID, Addrs: removedAddrs}) } if len(keptAddrs) > 0 { keep = append(keep, peer.AddrInfo{ID: p.ID, Addrs: keptAddrs}) } } cfg.SetBootstrapPeers(keep) if err := r.SetConfig(cfg); err != nil { return nil, err } return config.BootstrapPeerStrings(removed), nil } func bootstrapRemoveAll(r repo.Repo, cfg *config.Config) ([]string, error) { // Check if bootstrap contains "auto" - if so, we need special handling hasAuto := slices.Contains(cfg.Bootstrap, config.AutoPlaceholder) var removed []string if hasAuto { // When "auto" is present, we can't parse it as peer.AddrInfo // Just return the raw bootstrap list as strings for display removed = slices.Clone(cfg.Bootstrap) } else { // Original logic for configs without "auto" removedPeers, err := cfg.BootstrapPeers() if err != nil { return nil, err } removed = config.BootstrapPeerStrings(removedPeers) } cfg.Bootstrap = nil if err := r.SetConfig(cfg); err != nil { return nil, err } return removed, nil } const bootstrapSecurityWarning = ` SECURITY WARNING: The bootstrap command manipulates the "bootstrap list", which contains the addresses of bootstrap nodes. These are the *trusted peers* from which to learn about other peers in the network. Only edit this list if you understand the risks of adding or removing nodes from this list. ` ================================================ FILE: core/commands/cat.go ================================================ package commands import ( "context" "errors" "io" "os" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" "github.com/cheggaaa/pb" "github.com/ipfs/boxo/files" cmds "github.com/ipfs/go-ipfs-cmds" iface "github.com/ipfs/kubo/core/coreiface" ) const ( progressBarMinSize = 1024 * 1024 * 8 // show progress bar for outputs > 8MiB offsetOptionName = "offset" lengthOptionName = "length" ) var CatCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Show IPFS object data.", ShortDescription: "Displays the data contained by an IPFS or IPNS object(s) at the given path.", }, Arguments: []cmds.Argument{ cmds.StringArg("ipfs-path", true, true, "The path to the IPFS object(s) to be outputted.").EnableStdin(), }, Options: []cmds.Option{ cmds.Int64Option(offsetOptionName, "o", "Byte offset to begin reading from."), cmds.Int64Option(lengthOptionName, "l", "Maximum number of bytes to read."), cmds.BoolOption(progressOptionName, "p", "Stream progress data.").WithDefault(true), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } offset, _ := req.Options[offsetOptionName].(int64) if offset < 0 { return errors.New("cannot specify negative offset") } max, found := req.Options[lengthOptionName].(int64) if max < 0 { return errors.New("cannot specify negative length") } if !found { max = -1 } err = req.ParseBodyArgs() if err != nil { return err } readers, length, err := cat(req.Context, api, req.Arguments, int64(offset), int64(max)) if err != nil { return err } /* if err := corerepo.ConditionalGC(req.Context, node, length); err != nil { re.SetError(err, cmds.ErrNormal) return } */ res.SetLength(length) reader := io.MultiReader(readers...) // Since the reader returns the error that a block is missing, and that error is // returned from io.Copy inside Emit, we need to take Emit errors and send // them to the client. Usually we don't do that because it means the connection // is broken or we supplied an illegal argument etc. return res.Emit(reader) }, PostRun: cmds.PostRunMap{ cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { if res.Length() > 0 && res.Length() < progressBarMinSize { return cmds.Copy(re, res) } for { v, err := res.Next() if err != nil { if err == io.EOF { return nil } return err } switch val := v.(type) { case io.Reader: reader := val req := res.Request() progress, _ := req.Options[progressOptionName].(bool) if progress { var bar *pb.ProgressBar bar, reader = progressBarForReader(os.Stderr, val, int64(res.Length())) bar.Start() defer bar.Finish() } err = re.Emit(reader) if err != nil { return err } default: log.Warnf("cat postrun: received unexpected type %T", val) } } }, }, } func cat(ctx context.Context, api iface.CoreAPI, paths []string, offset int64, max int64) ([]io.Reader, uint64, error) { readers := make([]io.Reader, 0, len(paths)) length := uint64(0) if max == 0 { return nil, 0, nil } for _, pString := range paths { p, err := cmdutils.PathOrCidPath(pString) if err != nil { return nil, 0, err } f, err := api.Unixfs().Get(ctx, p) if err != nil { return nil, 0, err } var file files.File switch f := f.(type) { case files.File: file = f case files.Directory: return nil, 0, iface.ErrIsDir default: return nil, 0, iface.ErrNotSupported } fsize, err := file.Size() if err != nil { return nil, 0, err } if offset > fsize { offset = offset - fsize continue } count, err := file.Seek(offset, io.SeekStart) if err != nil { return nil, 0, err } offset = 0 fsize, err = file.Size() if err != nil { return nil, 0, err } size := uint64(fsize - count) length += size if max > 0 && length >= uint64(max) { var r io.Reader = file if overshoot := int64(length - uint64(max)); overshoot != 0 { r = io.LimitReader(file, int64(size)-overshoot) length = uint64(max) } readers = append(readers, r) break } readers = append(readers, file) } return readers, length, nil } ================================================ FILE: core/commands/cid.go ================================================ package commands import ( "cmp" "encoding/hex" "errors" "fmt" "io" "slices" "strings" "unicode" verifcid "github.com/ipfs/boxo/verifcid" cid "github.com/ipfs/go-cid" cidutil "github.com/ipfs/go-cidutil" cmds "github.com/ipfs/go-ipfs-cmds" ipldmulticodec "github.com/ipld/go-ipld-prime/multicodec" peer "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" mc "github.com/multiformats/go-multicodec" mhash "github.com/multiformats/go-multihash" ) var CidCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Convert and discover properties of CIDs", }, Subcommands: map[string]*cmds.Command{ "inspect": inspectCmd, "format": cidFmtCmd, "base32": base32Cmd, "bases": basesCmd, "codecs": codecsCmd, "hashes": hashesCmd, }, Extra: CreateCmdExtras(SetDoesNotUseRepo(true)), } const ( cidFormatOptionName = "f" cidToVersionOptionName = "v" cidCodecOptionName = "mc" cidMultibaseOptionName = "b" ) var cidFmtCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Format and convert a CID in various useful ways.", LongDescription: ` Format and converts 's in various useful ways. For a human-readable breakdown of a CID, see 'ipfs cid inspect'. The optional format string is a printf style format string: ` + cidutil.FormatRef, }, Arguments: []cmds.Argument{ cmds.StringArg("cid", true, true, "CIDs to format.").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption(cidFormatOptionName, "Printf style format string.").WithDefault("%s"), cmds.StringOption(cidToVersionOptionName, "CID version to convert to."), cmds.StringOption(cidCodecOptionName, "CID multicodec to convert to."), cmds.StringOption(cidMultibaseOptionName, "Multibase to display CID in."), }, Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { fmtStr, _ := req.Options[cidFormatOptionName].(string) verStr, _ := req.Options[cidToVersionOptionName].(string) codecStr, _ := req.Options[cidCodecOptionName].(string) baseStr, _ := req.Options[cidMultibaseOptionName].(string) opts := cidFormatOpts{} if strings.IndexByte(fmtStr, '%') == -1 { return fmt.Errorf("invalid format string: %q", fmtStr) } opts.fmtStr = fmtStr if codecStr != "" { var codec mc.Code err := codec.Set(codecStr) if err != nil { return err } opts.newCodec = uint64(codec) } // otherwise, leave it as 0 (not a valid IPLD codec) switch verStr { case "": if baseStr != "" { opts.verConv = toCidV1 } case "0": if opts.newCodec != 0 && opts.newCodec != cid.DagProtobuf { return errors.New("cannot convert to CIDv0 with any codec other than dag-pb") } if baseStr != "" && baseStr != "base58btc" { return errors.New("cannot convert to CIDv0 with any multibase other than the implicit base58btc") } opts.verConv = toCidV0 case "1": opts.verConv = toCidV1 default: return fmt.Errorf("invalid cid version: %q", verStr) } if baseStr != "" { encoder, err := mbase.EncoderByName(baseStr) if err != nil { return err } opts.newBase = encoder.Encoding() } else { opts.newBase = mbase.Encoding(-1) } return emitCids(req, resp, opts) }, PostRun: cmds.PostRunMap{ cmds.CLI: streamResult(func(v any, out io.Writer) nonFatalError { r := v.(*CidFormatRes) if r.ErrorMsg != "" { return nonFatalError(fmt.Sprintf("%s: %s", r.CidStr, r.ErrorMsg)) } fmt.Fprintf(out, "%s\n", r.Formatted) return "" }), }, Type: CidFormatRes{}, Extra: CreateCmdExtras(SetDoesNotUseRepo(true)), } type CidFormatRes struct { CidStr string // Original Cid String passed in Formatted string // Formatted Result ErrorMsg string // Error } var base32Cmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Convert CIDs to Base32 CID version 1.", ShortDescription: ` 'ipfs cid base32' normalizes passed CIDs to their canonical case-insensitive encoding. Useful when processing third-party CIDs which could come with arbitrary formats. `, }, Arguments: []cmds.Argument{ cmds.StringArg("cid", true, true, "CIDs to convert.").EnableStdin(), }, Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { opts := cidFormatOpts{ fmtStr: "%s", newBase: mbase.Encoding(mbase.Base32), verConv: toCidV1, } return emitCids(req, resp, opts) }, PostRun: cidFmtCmd.PostRun, Type: cidFmtCmd.Type, Extra: CreateCmdExtras(SetDoesNotUseRepo(true)), } type cidFormatOpts struct { fmtStr string newBase mbase.Encoding verConv func(cid cid.Cid) (cid.Cid, error) newCodec uint64 } type argumentIterator struct { args []string body cmds.StdinArguments } func (i *argumentIterator) next() (string, bool) { if len(i.args) > 0 { arg := i.args[0] i.args = i.args[1:] return arg, true } if i.body == nil || !i.body.Scan() { return "", false } return strings.TrimSpace(i.body.Argument()), true } func (i *argumentIterator) err() error { if i.body == nil { return nil } return i.body.Err() } func emitCids(req *cmds.Request, resp cmds.ResponseEmitter, opts cidFormatOpts) error { itr := argumentIterator{req.Arguments, req.BodyArgs()} var emitErr error for emitErr == nil { cidStr, ok := itr.next() if !ok { break } res := &CidFormatRes{CidStr: cidStr} c, err := cid.Decode(cidStr) if err != nil { res.ErrorMsg = err.Error() emitErr = resp.Emit(res) continue } if opts.newCodec != 0 && opts.newCodec != c.Type() { c = cid.NewCidV1(opts.newCodec, c.Hash()) } if opts.verConv != nil { c, err = opts.verConv(c) if err != nil { res.ErrorMsg = err.Error() emitErr = resp.Emit(res) continue } } base := opts.newBase if base == -1 { if c.Version() == 0 { base = mbase.Base58BTC } else { base, _ = cid.ExtractEncoding(cidStr) } } str, err := cidutil.Format(opts.fmtStr, base, c) if _, ok := err.(cidutil.FormatStringError); ok { // no point in continuing if there is a problem with the format string return err } if err != nil { res.ErrorMsg = err.Error() } else { res.Formatted = str } emitErr = resp.Emit(res) } if emitErr != nil { return emitErr } err := itr.err() if err != nil { return err } return nil } func toCidV0(c cid.Cid) (cid.Cid, error) { if c.Type() != cid.DagProtobuf { return cid.Cid{}, fmt.Errorf("can't convert non-dag-pb nodes to cidv0") } return cid.NewCidV0(c.Hash()), nil } func toCidV1(c cid.Cid) (cid.Cid, error) { return cid.NewCidV1(c.Type(), c.Hash()), nil } type CodeAndName struct { Code int Name string } const ( prefixOptionName = "prefix" numericOptionName = "numeric" ) var basesCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List available multibase encodings.", ShortDescription: ` 'ipfs cid bases' relies on https://github.com/multiformats/go-multibase `, }, Options: []cmds.Option{ cmds.BoolOption(prefixOptionName, "also include the single letter prefixes in addition to the code"), cmds.BoolOption(numericOptionName, "also include numeric codes"), }, Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { var res []CodeAndName // use EncodingToStr in case at some point there are multiple names for a given code for code, name := range mbase.EncodingToStr { res = append(res, CodeAndName{int(code), name}) } return cmds.EmitOnce(resp, res) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, val []CodeAndName) error { prefixes, _ := req.Options[prefixOptionName].(bool) numeric, _ := req.Options[numericOptionName].(bool) multibaseSorter{val}.Sort() for _, v := range val { code := v.Code if !unicode.IsPrint(rune(code)) { // don't display non-printable prefixes code = ' ' } switch { case prefixes && numeric: fmt.Fprintf(w, "%c %7d %s\n", code, v.Code, v.Name) case prefixes: fmt.Fprintf(w, "%c %s\n", code, v.Name) case numeric: fmt.Fprintf(w, "%7d %s\n", v.Code, v.Name) default: fmt.Fprintf(w, "%s\n", v.Name) } } return nil }), }, Type: []CodeAndName{}, Extra: CreateCmdExtras(SetDoesNotUseRepo(true)), } const ( codecsNumericOptionName = "numeric" codecsSupportedOptionName = "supported" ) var codecsCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List available CID multicodecs.", ShortDescription: ` 'ipfs cid codecs' relies on https://github.com/multiformats/go-multicodec `, }, Options: []cmds.Option{ cmds.BoolOption(codecsNumericOptionName, "n", "also include numeric codes"), cmds.BoolOption(codecsSupportedOptionName, "s", "list only codecs supported by go-ipfs commands"), }, Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { listSupported, _ := req.Options[codecsSupportedOptionName].(bool) supportedCodecs := make(map[uint64]struct{}) if listSupported { for _, code := range ipldmulticodec.ListEncoders() { supportedCodecs[code] = struct{}{} } for _, code := range ipldmulticodec.ListDecoders() { supportedCodecs[code] = struct{}{} } // add libp2p-key supportedCodecs[uint64(mc.Libp2pKey)] = struct{}{} } var res []CodeAndName for _, code := range mc.KnownCodes() { if code.Tag() == "ipld" { if listSupported { if _, ok := supportedCodecs[uint64(code)]; !ok { continue } } res = append(res, CodeAndName{int(code), mc.Code(code).String()}) } } return cmds.EmitOnce(resp, res) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, val []CodeAndName) error { numeric, _ := req.Options[codecsNumericOptionName].(bool) codeAndNameSorter{val}.Sort() for _, v := range val { if numeric { fmt.Fprintf(w, "%5d %s\n", v.Code, v.Name) } else { fmt.Fprintf(w, "%s\n", v.Name) } } return nil }), }, Type: []CodeAndName{}, Extra: CreateCmdExtras(SetDoesNotUseRepo(true)), } var hashesCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List available multihashes.", ShortDescription: ` 'ipfs cid hashes' relies on https://github.com/multiformats/go-multihash `, }, Options: codecsCmd.Options, Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { var res []CodeAndName // use mhash.Codes in case at some point there are multiple names for a given code for code, name := range mhash.Codes { if !verifcid.DefaultAllowlist.IsAllowed(code) { continue } res = append(res, CodeAndName{int(code), name}) } return cmds.EmitOnce(resp, res) }, Encoders: codecsCmd.Encoders, Type: codecsCmd.Type, Extra: CreateCmdExtras(SetDoesNotUseRepo(true)), } // CidInspectRes represents the response from the inspect command. type CidInspectRes struct { Cid string `json:"cid"` Version int `json:"version"` Multibase CidInspectBase `json:"multibase"` Multicodec CidInspectCodec `json:"multicodec"` Multihash CidInspectHash `json:"multihash"` CidV0 string `json:"cidV0,omitempty"` CidV1 string `json:"cidV1"` ErrorMsg string `json:"errorMsg,omitempty"` } type CidInspectBase struct { Prefix string `json:"prefix"` Name string `json:"name"` } type CidInspectCodec struct { Code uint64 `json:"code"` Name string `json:"name"` } type CidInspectHash struct { Code uint64 `json:"code"` Name string `json:"name"` Length int `json:"length"` Digest string `json:"digest"` } var inspectCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Inspect and display detailed information about a CID.", ShortDescription: ` 'ipfs cid inspect' breaks down a CID and displays its components: - CID version (0 or 1) - Multibase encoding (explicit for CIDv1, implicit for CIDv0) - Multicodec (DAG type) - Multihash (hash algorithm, length, and digest) - Equivalent CIDv0 and CIDv1 representations For CIDv0, multibase, multicodec, and multihash are marked as implicit because they are not explicitly encoded in the binary. If a PeerID string is provided instead of a CID, a helpful error with the equivalent CID representation is returned. Use --enc=json for machine-readable output same as the HTTP RPC API. `, }, Arguments: []cmds.Argument{ cmds.StringArg("cid", true, false, "CID to inspect.").EnableStdin(), }, Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { cidStr := req.Arguments[0] c, err := cid.Decode(cidStr) if err != nil { errMsg := fmt.Sprintf("invalid CID: %s", err) // PeerID fallback: try peer.Decode for legacy PeerIDs (12D3KooW..., Qm...) if pid, pidErr := peer.Decode(cidStr); pidErr == nil { pidCid := peer.ToCid(pid) cidV1, _ := pidCid.StringOfBase(mbase.Base36) errMsg += fmt.Sprintf("\nNote: the value is a PeerID; inspect its CID representation instead:\n %s", cidV1) } return cmds.EmitOnce(resp, &CidInspectRes{Cid: cidStr, ErrorMsg: errMsg}) } res := &CidInspectRes{ Cid: cidStr, Version: int(c.Version()), } // Multibase: always populated; CIDv0 uses implicit base58btc if c.Version() == 0 { res.Multibase = CidInspectBase{Prefix: "z", Name: "base58btc"} } else { baseCode, _ := cid.ExtractEncoding(cidStr) res.Multibase = CidInspectBase{ Prefix: string(rune(baseCode)), Name: mbase.EncodingToStr[baseCode], } } // Multicodec codecName := mc.Code(c.Type()).String() if codecName == "" || strings.HasPrefix(codecName, "Code(") { codecName = "unknown" } res.Multicodec = CidInspectCodec{Code: c.Type(), Name: codecName} // Multihash dmh, err := mhash.Decode(c.Hash()) if err != nil { return cmds.EmitOnce(resp, &CidInspectRes{ Cid: cidStr, ErrorMsg: fmt.Sprintf("failed to decode multihash: %s", err), }) } hashName := mhash.Codes[dmh.Code] if hashName == "" { hashName = "unknown" } res.Multihash = CidInspectHash{ Code: dmh.Code, Name: hashName, Length: dmh.Length, Digest: hex.EncodeToString(dmh.Digest), } // CIDv0: only possible with dag-pb + sha2-256-256 if c.Type() == cid.DagProtobuf && dmh.Code == mhash.SHA2_256 && dmh.Length == 32 { res.CidV0 = cid.NewCidV0(c.Hash()).String() } // CIDv1: use base36 for libp2p-key, base32 for everything else v1 := cid.NewCidV1(c.Type(), c.Hash()) v1Base := mbase.Encoding(mbase.Base32) if c.Type() == uint64(mc.Libp2pKey) { v1Base = mbase.Base36 } v1Str, err := v1.StringOfBase(v1Base) if err != nil { v1Str = v1.String() } res.CidV1 = v1Str return cmds.EmitOnce(resp, res) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, res *CidInspectRes) error { if res.ErrorMsg != "" { return fmt.Errorf("%s", res.ErrorMsg) } implicit := "" if res.Version == 0 { implicit = ", implicit" } fmt.Fprintf(w, "CID: %s\n", res.Cid) fmt.Fprintf(w, "Version: %d\n", res.Version) if res.Version == 0 { fmt.Fprintf(w, "Multibase: %s (implicit)\n", res.Multibase.Name) } else { fmt.Fprintf(w, "Multibase: %s (%s)\n", res.Multibase.Name, res.Multibase.Prefix) } fmt.Fprintf(w, "Multicodec: %s (0x%x%s)\n", res.Multicodec.Name, res.Multicodec.Code, implicit) fmt.Fprintf(w, "Multihash: %s (0x%x%s)\n", res.Multihash.Name, res.Multihash.Code, implicit) fmt.Fprintf(w, " Length: %d bytes\n", res.Multihash.Length) fmt.Fprintf(w, " Digest: %s\n", res.Multihash.Digest) if res.CidV0 != "" { fmt.Fprintf(w, "CIDv0: %s\n", res.CidV0) } else if res.Multicodec.Code != cid.DagProtobuf { fmt.Fprintf(w, "CIDv0: not possible, requires dag-pb (0x70), got %s (0x%x)\n", res.Multicodec.Name, res.Multicodec.Code) } else if res.Multihash.Code != mhash.SHA2_256 { fmt.Fprintf(w, "CIDv0: not possible, requires sha2-256 (0x12), got %s (0x%x)\n", res.Multihash.Name, res.Multihash.Code) } else if res.Multihash.Length != 32 { fmt.Fprintf(w, "CIDv0: not possible, requires 32-byte digest, got %d\n", res.Multihash.Length) } fmt.Fprintf(w, "CIDv1: %s\n", res.CidV1) return nil }), }, Type: CidInspectRes{}, Extra: CreateCmdExtras(SetDoesNotUseRepo(true)), } type multibaseSorter struct { data []CodeAndName } func (s multibaseSorter) Sort() { slices.SortFunc(s.data, func(a, b CodeAndName) int { if n := cmp.Compare(unicode.ToLower(rune(a.Code)), unicode.ToLower(rune(b.Code))); n != 0 { return n } // lowercase letters should come before uppercase return cmp.Compare(b.Code, a.Code) }) } type codeAndNameSorter struct { data []CodeAndName } func (s codeAndNameSorter) Sort() { slices.SortFunc(s.data, func(a, b CodeAndName) int { return cmp.Compare(a.Code, b.Code) }) } ================================================ FILE: core/commands/cid_test.go ================================================ package commands import ( "testing" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/multiformats/go-multibase" ) func TestCidFmtCmd(t *testing.T) { t.Parallel() // Test 'error when -v 0 is present and a custom -b is passed' t.Run("ipfs cid format -b z -v 0", func(t *testing.T) { t.Parallel() type testV0PresentAndCustomBaseCase struct { MultibaseName string ExpectedErrMsg string } var testV0PresentAndCustomBaseCases []testV0PresentAndCustomBaseCase for _, e := range multibase.EncodingToStr { var testCase testV0PresentAndCustomBaseCase if e == "base58btc" { testCase.MultibaseName = e testCase.ExpectedErrMsg = "" testV0PresentAndCustomBaseCases = append(testV0PresentAndCustomBaseCases, testCase) continue } testCase.MultibaseName = e testCase.ExpectedErrMsg = "cannot convert to CIDv0 with any multibase other than the implicit base58btc" testV0PresentAndCustomBaseCases = append(testV0PresentAndCustomBaseCases, testCase) } for _, e := range testV0PresentAndCustomBaseCases { // Mock request req := &cmds.Request{ Options: map[string]any{ cidToVersionOptionName: "0", cidMultibaseOptionName: e.MultibaseName, cidFormatOptionName: "%s", }, } // Response emitter resp := cmds.ResponseEmitter(nil) // Call the CidFmtCmd function with the mock request and response err := cidFmtCmd.Run(req, resp, nil) if err == nil && e.MultibaseName == "base58btc" { continue } errMsg := err.Error() if errMsg != e.ExpectedErrMsg { t.Errorf("Expected %s, got %s instead", e.ExpectedErrMsg, errMsg) } } }) // Test 'upgrade CID to v1 when passing a custom -b and no -v is specified' t.Run("ipfs cid format -b z", func(t *testing.T) { t.Parallel() type testImplicitVersionAndCustomMultibaseCase struct { Ver string CidV1 string CidV0 string MultibaseName string } var testCases = []testImplicitVersionAndCustomMultibaseCase{ { Ver: "", CidV1: "zdj7WWwMSWGoyxYkkT7mHgYvr6tV8CYd77aYxxqSbg9HsiMcE", CidV0: "QmPr755CxWUwt39C2Yiw4UGKrv16uZhSgeZJmoHUUS9TSJ", MultibaseName: "z", }, { Ver: "", CidV1: "CAFYBEIDI7ZABPGG3S63QW3AJG2XAZNE4NJQPN777WLWYRAIDG3TE5QFN3A======", CidV0: "QmVQVyEijmLb2cBQrowNQsaPbnUnJhfDK1sYe3wepm6ySf", MultibaseName: "base32padupper", }, } for _, e := range testCases { // Mock request req := &cmds.Request{ Options: map[string]any{ cidToVersionOptionName: e.Ver, cidMultibaseOptionName: e.MultibaseName, cidFormatOptionName: "%s", }, } // Response emitter resp := cmds.ResponseEmitter(nil) // Call the CidFmtCmd function with the mock request and response err := cidFmtCmd.Run(req, resp, nil) if err != nil { t.Error(err) } } }) } ================================================ FILE: core/commands/cmdenv/cidbase.go ================================================ package cmdenv import ( "fmt" "strings" cid "github.com/ipfs/go-cid" cidenc "github.com/ipfs/go-cidutil/cidenc" cmds "github.com/ipfs/go-ipfs-cmds" mbase "github.com/multiformats/go-multibase" ) var ( OptionCidBase = cmds.StringOption("cid-base", "Multibase encoding used for version 1 CIDs in output.") OptionUpgradeCidV0InOutput = cmds.BoolOption("upgrade-cidv0-in-output", "Upgrade version 0 to version 1 CIDs in output.") ) // GetCidEncoder processes the `cid-base` and `output-cidv1` options and // returns an encoder to use based on those parameters. func GetCidEncoder(req *cmds.Request) (cidenc.Encoder, error) { return getCidBase(req, true) } // GetLowLevelCidEncoder is like GetCidEncoder but meant to be used by lower // level commands. It differs from GetCidEncoder in that CIDv0 are not, by // default, auto-upgraded to CIDv1. func GetLowLevelCidEncoder(req *cmds.Request) (cidenc.Encoder, error) { return getCidBase(req, false) } func getCidBase(req *cmds.Request, autoUpgrade bool) (cidenc.Encoder, error) { base, _ := req.Options[OptionCidBase.Name()].(string) upgrade, upgradeDefined := req.Options[OptionUpgradeCidV0InOutput.Name()].(bool) e := cidenc.Default() if base != "" { var err error e.Base, err = mbase.EncoderByName(base) if err != nil { return e, err } if autoUpgrade { e.Upgrade = true } } if upgradeDefined { e.Upgrade = upgrade } return e, nil } // CidBaseDefined returns true if the `cid-base` option is specified on the // command line func CidBaseDefined(req *cmds.Request) bool { base, _ := req.Options["cid-base"].(string) return base != "" } // CidEncoderFromPath creates a new encoder that is influenced from the encoded // Cid in a Path. For CIDv0 the multibase from the base encoder is used and // automatic upgrades are disabled. For CIDv1 the multibase from the CID is // used and upgrades are enabled. // // This logic is intentionally fuzzy and matches anything of the form // `CidLike`, `CidLike/...`, or `/namespace/CidLike/...`. // // For example: // // * Qm... // * Qm.../... // * /ipfs/Qm... // * /ipns/bafybeiahnxfi7fpmr5wtxs2imx4abnyn7fdxeiox7xxjem6zuiioqkh6zi/... // * /bzz/bafybeiahnxfi7fpmr5wtxs2imx4abnyn7fdxeiox7xxjem6zuiioqkh6zi/... func CidEncoderFromPath(p string) (cidenc.Encoder, error) { components := strings.SplitN(p, "/", 4) var maybeCid string if components[0] != "" { // No leading slash, first component is likely CID-like. maybeCid = components[0] } else if len(components) < 3 { // Not enough components to include a CID. return cidenc.Encoder{}, fmt.Errorf("no cid in path: %s", p) } else { maybeCid = components[2] } c, err := cid.Decode(maybeCid) if err != nil { // Ok, not a CID-like thing. Keep the current encoder. return cidenc.Encoder{}, fmt.Errorf("no cid in path: %s", p) } if c.Version() == 0 { // Version 0, use the base58 non-upgrading encoder. return cidenc.Default(), nil } // Version 1+, extract multibase encoding. encoding, _, err := mbase.Decode(maybeCid) if err != nil { // This should be impossible, we've already decoded the cid. panic(fmt.Sprintf("BUG: failed to get multibase decoder for CID %s", maybeCid)) } return cidenc.Encoder{Base: mbase.MustNewEncoder(encoding), Upgrade: true}, nil } ================================================ FILE: core/commands/cmdenv/cidbase_test.go ================================================ package cmdenv import ( "testing" cidenc "github.com/ipfs/go-cidutil/cidenc" mbase "github.com/multiformats/go-multibase" ) func TestEncoderFromPath(t *testing.T) { test := func(path string, expected cidenc.Encoder) { actual, err := CidEncoderFromPath(path) if err != nil { t.Error(err) } if actual != expected { t.Errorf("CidEncoderFromPath(%s) failed: expected %#v but got %#v", path, expected, actual) } } p := "QmRqVG8VGdKZ7KARqR96MV7VNHgWvEQifk94br5HpURpfu" enc := cidenc.Default() test(p, enc) test(p+"/a", enc) test(p+"/a/b", enc) test(p+"/a/b/", enc) test(p+"/a/b/c", enc) test("/ipfs/"+p, enc) test("/ipfs/"+p+"/b", enc) p = "zb2rhfkM4FjkMLaUnygwhuqkETzbYXnUDf1P9MSmdNjW1w1Lk" enc = cidenc.Encoder{ Base: mbase.MustNewEncoder(mbase.Base58BTC), Upgrade: true, } test(p, enc) test(p+"/a", enc) test(p+"/a/b", enc) test(p+"/a/b/", enc) test(p+"/a/b/c", enc) test("/ipfs/"+p, enc) test("/ipfs/"+p+"/b", enc) test("/ipld/"+p, enc) test("/ipns/"+p, enc) // even IPNS should work. p = "bafyreifrcnyjokuw4i4ggkzg534tjlc25lqgt3ttznflmyv5fftdgu52hm" enc = cidenc.Encoder{ Base: mbase.MustNewEncoder(mbase.Base32), Upgrade: true, } test(p, enc) test("/ipfs/"+p, enc) test("/ipld/"+p, enc) for _, badPath := range []string{ "/ipld/", "/ipld", "/ipld//", "ipld//", "ipld", "", "ipns", "/ipfs/asdf", "/ipfs/...", "...", "abcdefg", "boo", } { _, err := CidEncoderFromPath(badPath) if err == nil { t.Errorf("expected error extracting encoder from bad path: %s", badPath) } } } ================================================ FILE: core/commands/cmdenv/env.go ================================================ package cmdenv import ( "context" "fmt" "strconv" "strings" "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" logging "github.com/ipfs/go-log/v2" routing "github.com/libp2p/go-libp2p/core/routing" "github.com/ipfs/kubo/commands" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core" coreiface "github.com/ipfs/kubo/core/coreiface" options "github.com/ipfs/kubo/core/coreiface/options" ) var log = logging.Logger("core/commands/cmdenv") // GetNode extracts the node from the environment. func GetNode(env any) (*core.IpfsNode, error) { ctx, ok := env.(*commands.Context) if !ok { return nil, fmt.Errorf("expected env to be of type %T, got %T", ctx, env) } return ctx.GetNode() } // GetApi extracts CoreAPI instance from the environment. func GetApi(env cmds.Environment, req *cmds.Request) (coreiface.CoreAPI, error) { //nolint ctx, ok := env.(*commands.Context) if !ok { return nil, fmt.Errorf("expected env to be of type %T, got %T", ctx, env) } offline, _ := req.Options["offline"].(bool) if !offline { offline, _ = req.Options["local"].(bool) if offline { log.Errorf("Command '%s', --local is deprecated, use --offline instead", strings.Join(req.Path, " ")) } } api, err := ctx.GetAPI() if err != nil { return nil, err } if offline { return api.WithOptions(options.Api.Offline(offline)) } return api, nil } // GetConfigRoot extracts the config root from the environment func GetConfigRoot(env cmds.Environment) (string, error) { ctx, ok := env.(*commands.Context) if !ok { return "", fmt.Errorf("expected env to be of type %T, got %T", ctx, env) } return ctx.ConfigRoot, nil } // EscNonPrint converts non-printable characters and backslash into Go escape // sequences. This is done to display all characters in a string, including // those that would otherwise not be displayed or have an undesirable effect on // the display. func EscNonPrint(s string) string { if !needEscape(s) { return s } esc := strconv.Quote(s) // Remove first and last quote, and unescape quotes. return strings.ReplaceAll(esc[1:len(esc)-1], `\"`, `"`) } func needEscape(s string) bool { if strings.ContainsRune(s, '\\') { return true } for _, r := range s { if !strconv.IsPrint(r) { return true } } return false } // provideCIDSync performs a synchronous/blocking provide operation to announce // the given CID to the DHT. // // - If the accelerated DHT client is used, a DHT lookup isn't needed, we // directly allocate provider records to closest peers. // - If Provide.DHT.SweepEnabled=true or OptimisticProvide=true, we make an // optimistic provide call. // - Else we make a standard provide call (much slower). // // IMPORTANT: The caller MUST verify DHT availability using HasActiveDHTClient() // before calling this function. Calling with a nil or invalid router will cause // a panic - this is the caller's responsibility to prevent. func provideCIDSync(ctx context.Context, router routing.Routing, c cid.Cid) error { return router.Provide(ctx, c, true) } // ExecuteFastProvide immediately provides a root CID to the DHT, bypassing the regular // provide queue for faster content discovery. This function is reusable across commands // that add or import content, such as ipfs add and ipfs dag import. // // Parameters: // - ctx: context for synchronous provides // - ipfsNode: the IPFS node instance // - cfg: node configuration // - rootCid: the CID to provide // - wait: whether to block until provide completes (sync mode) // - isPinned: whether content is pinned // - isPinnedRoot: whether this is a pinned root CID // - isMFS: whether content is in MFS // // Return value: // - Returns nil if operation succeeded or was skipped (preconditions not met) // - Returns error only in sync mode (wait=true) when provide operation fails // - In async mode (wait=false), always returns nil (errors logged in goroutine) // // The function handles all precondition checks (Provide.Enabled, DHT availability, // strategy matching) and logs appropriately. In async mode, it launches a goroutine // with a detached context and timeout. func ExecuteFastProvide( ctx context.Context, ipfsNode *core.IpfsNode, cfg *config.Config, rootCid cid.Cid, wait bool, isPinned bool, isPinnedRoot bool, isMFS bool, ) error { log.Debugw("fast-provide-root: enabled", "wait", wait) // Check preconditions for providing switch { case !cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled): log.Debugw("fast-provide-root: skipped", "reason", "Provide.Enabled is false") return nil case cfg.Provide.DHT.Interval.WithDefault(config.DefaultProvideDHTInterval) == 0: log.Debugw("fast-provide-root: skipped", "reason", "Provide.DHT.Interval is 0") return nil case !ipfsNode.HasActiveDHTClient(): log.Debugw("fast-provide-root: skipped", "reason", "DHT not available") return nil } // Check if strategy allows providing this content strategyStr := cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy) strategy := config.ParseProvideStrategy(strategyStr) shouldProvide := config.ShouldProvideForStrategy(strategy, isPinned, isPinnedRoot, isMFS) if !shouldProvide { log.Debugw("fast-provide-root: skipped", "reason", "strategy does not match content", "strategy", strategyStr, "pinned", isPinned, "pinnedRoot", isPinnedRoot, "mfs", isMFS) return nil } // Execute provide operation if wait { // Synchronous mode: block until provide completes, return error on failure log.Debugw("fast-provide-root: providing synchronously", "cid", rootCid) if err := provideCIDSync(ctx, ipfsNode.DHTClient, rootCid); err != nil { log.Warnw("fast-provide-root: sync provide failed", "cid", rootCid, "error", err) return fmt.Errorf("fast-provide: %w", err) } log.Debugw("fast-provide-root: sync provide completed", "cid", rootCid) return nil } // Asynchronous mode (default): fire-and-forget, don't block, always return nil log.Debugw("fast-provide-root: providing asynchronously", "cid", rootCid) go func() { // Use detached context with timeout to prevent hanging on network issues ctx, cancel := context.WithTimeout(context.Background(), config.DefaultFastProvideTimeout) defer cancel() if err := provideCIDSync(ctx, ipfsNode.DHTClient, rootCid); err != nil { log.Warnw("fast-provide-root: async provide failed", "cid", rootCid, "error", err) } else { log.Debugw("fast-provide-root: async provide completed", "cid", rootCid) } }() return nil } ================================================ FILE: core/commands/cmdenv/env_test.go ================================================ package cmdenv import ( "strconv" "testing" ) func TestEscNonPrint(t *testing.T) { b := []byte("hello") b[2] = 0x7f s := string(b) if !needEscape(s) { t.Fatal("string needs escaping") } if !hasNonPrintable(s) { t.Fatal("expected non-printable") } if hasNonPrintable(EscNonPrint(s)) { t.Fatal("escaped string has non-printable") } if EscNonPrint(`hel\lo`) != `hel\\lo` { t.Fatal("backslash not escaped") } s = `hello` if needEscape(s) { t.Fatal("string does not need escaping") } if EscNonPrint(s) != s { t.Fatal("string should not have changed") } s = `"hello"` if EscNonPrint(s) != s { t.Fatal("string should not have changed") } if EscNonPrint(`"hel\"lo"`) != `"hel\\"lo"` { t.Fatal("did not get expected escaped string") } } func hasNonPrintable(s string) bool { for _, r := range s { if !strconv.IsPrint(r) { return true } } return false } ================================================ FILE: core/commands/cmdenv/file.go ================================================ package cmdenv import ( "fmt" "github.com/ipfs/boxo/files" ) // GetFileArg returns the next file from the directory or an error func GetFileArg(it files.DirIterator) (files.File, error) { if !it.Next() { err := it.Err() if err == nil { err = fmt.Errorf("expected a file argument") } return nil, err } file := files.FileFromEntry(it) if file == nil { return nil, fmt.Errorf("file argument was nil") } return file, nil } ================================================ FILE: core/commands/cmdutils/sanitize.go ================================================ package cmdutils import ( "strings" "unicode" ) const maxRunes = 128 // CleanAndTrim sanitizes untrusted strings from remote peers to prevent display issues // across web UIs, terminals, and logs. It replaces control characters, format characters, // and surrogates with U+FFFD (�), then enforces a maximum length of 128 runes. // // This follows the libp2p identify specification and RFC 9839 guidance: // replacing problematic code points is preferred over deletion as deletion // is a known security risk. func CleanAndTrim(str string) string { // Build sanitized result var result []rune for _, r := range str { // Replace control characters (Cc) with U+FFFD - prevents terminal escapes, CR, LF, etc. if unicode.Is(unicode.Cc, r) { result = append(result, '\uFFFD') continue } // Replace format characters (Cf) with U+FFFD - prevents RTL/LTR overrides, zero-width chars if unicode.Is(unicode.Cf, r) { result = append(result, '\uFFFD') continue } // Replace surrogate characters (Cs) with U+FFFD - invalid in UTF-8 if unicode.Is(unicode.Cs, r) { result = append(result, '\uFFFD') continue } // Private use characters (Co) are preserved per spec result = append(result, r) } // Convert to string and trim whitespace sanitized := strings.TrimSpace(string(result)) // Enforce maximum length (128 runes, not bytes) runes := []rune(sanitized) if len(runes) > maxRunes { return string(runes[:maxRunes]) } return sanitized } ================================================ FILE: core/commands/cmdutils/utils.go ================================================ package cmdutils import ( "fmt" "slices" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" coreiface "github.com/ipfs/kubo/core/coreiface" "github.com/libp2p/go-libp2p/core/peer" ) const ( AllowBigBlockOptionName = "allow-big-block" // SoftBlockLimit is the maximum block size for bitswap transfer. // If this value changes, update the "2MiB" strings in error messages below. SoftBlockLimit = 2 * 1024 * 1024 // https://specs.ipfs.tech/bitswap-protocol/#block-sizes MaxPinNameBytes = 255 // Maximum number of bytes allowed for a pin name ) var AllowBigBlockOption cmds.Option func init() { AllowBigBlockOption = cmds.BoolOption(AllowBigBlockOptionName, "Disable block size check and allow creation of blocks bigger than 2MiB. WARNING: such blocks won't be transferable over the standard bitswap.").WithDefault(false) } func CheckCIDSize(req *cmds.Request, c cid.Cid, dagAPI coreiface.APIDagService) error { n, err := dagAPI.Get(req.Context, c) if err != nil { return fmt.Errorf("CheckCIDSize: getting dag: %w", err) } nodeSize, err := n.Size() if err != nil { return fmt.Errorf("CheckCIDSize: getting node size: %w", err) } return CheckBlockSize(req, nodeSize) } func CheckBlockSize(req *cmds.Request, size uint64) error { allowAnyBlockSize, _ := req.Options[AllowBigBlockOptionName].(bool) if allowAnyBlockSize { return nil } // Block size is limited to SoftBlockLimit (2MiB) as defined in the bitswap spec. // https://specs.ipfs.tech/bitswap-protocol/#block-sizes if size > SoftBlockLimit { return fmt.Errorf("produced block is over 2MiB: big blocks can't be exchanged with other peers. consider using UnixFS for automatic chunking of bigger files, or pass --allow-big-block to override") } return nil } // ValidatePinName validates that a pin name does not exceed the maximum allowed byte length. // Returns an error if the name exceeds MaxPinNameBytes (255 bytes). func ValidatePinName(name string) error { if name == "" { // Empty names are allowed return nil } nameBytes := len([]byte(name)) if nameBytes > MaxPinNameBytes { return fmt.Errorf("pin name is %d bytes (max %d bytes)", nameBytes, MaxPinNameBytes) } return nil } // PathOrCidPath returns a path.Path built from the argument. It keeps the old // behaviour by building a path from a CID string. func PathOrCidPath(str string) (path.Path, error) { p, err := path.NewPath(str) if err == nil { return p, nil } // Save the original error before attempting fallback originalErr := err if p, err := path.NewPath("/ipfs/" + str); err == nil { return p, nil } // Send back original err. return nil, originalErr } // CloneAddrInfo returns a copy of the AddrInfo with a cloned Addrs slice. // This prevents data races if the sender reuses the backing array. // See: https://github.com/ipfs/kubo/issues/11116 func CloneAddrInfo(ai peer.AddrInfo) peer.AddrInfo { return peer.AddrInfo{ ID: ai.ID, Addrs: slices.Clone(ai.Addrs), } } ================================================ FILE: core/commands/cmdutils/utils_test.go ================================================ package cmdutils import ( "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestPathOrCidPath(t *testing.T) { t.Run("valid path is returned as-is", func(t *testing.T) { validPath := "/ipfs/QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG" p, err := PathOrCidPath(validPath) require.NoError(t, err) assert.Equal(t, validPath, p.String()) }) t.Run("valid CID is converted to /ipfs/ path", func(t *testing.T) { cid := "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG" p, err := PathOrCidPath(cid) require.NoError(t, err) assert.Equal(t, "/ipfs/"+cid, p.String()) }) t.Run("valid ipns path is returned as-is", func(t *testing.T) { validPath := "/ipns/example.com" p, err := PathOrCidPath(validPath) require.NoError(t, err) assert.Equal(t, validPath, p.String()) }) t.Run("returns original error when both attempts fail", func(t *testing.T) { invalidInput := "invalid!@#path" _, err := PathOrCidPath(invalidInput) require.Error(t, err) // The error should reference the original input attempt. // This ensures users get meaningful error messages about their actual input. assert.Contains(t, err.Error(), invalidInput, "error should mention the original input") assert.Contains(t, err.Error(), "path does not have enough components", "error should describe the problem with the original input") }) t.Run("empty string returns error about original input", func(t *testing.T) { _, err := PathOrCidPath("") require.Error(t, err) // Verify we're not getting an error about "/ipfs/" (the fallback) errMsg := err.Error() assert.NotContains(t, errMsg, "/ipfs/", "error should be about empty input, not the fallback path") }) t.Run("invalid characters return error about original input", func(t *testing.T) { invalidInput := "not a valid path or CID with spaces and /@#$%" _, err := PathOrCidPath(invalidInput) require.Error(t, err) // The error message should help debug the original input assert.True(t, strings.Contains(err.Error(), invalidInput) || strings.Contains(err.Error(), "invalid"), "error should reference original problematic input") }) t.Run("CID with path is converted correctly", func(t *testing.T) { cidWithPath := "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG/file.txt" p, err := PathOrCidPath(cidWithPath) require.NoError(t, err) assert.Equal(t, "/ipfs/"+cidWithPath, p.String()) }) } func TestValidatePinName(t *testing.T) { t.Run("valid pin name is accepted", func(t *testing.T) { err := ValidatePinName("my-pin-name") assert.NoError(t, err) }) t.Run("empty pin name is accepted", func(t *testing.T) { err := ValidatePinName("") assert.NoError(t, err) }) t.Run("pin name at max length is accepted", func(t *testing.T) { maxName := strings.Repeat("a", MaxPinNameBytes) err := ValidatePinName(maxName) assert.NoError(t, err) }) t.Run("pin name exceeding max length is rejected", func(t *testing.T) { tooLong := strings.Repeat("a", MaxPinNameBytes+1) err := ValidatePinName(tooLong) require.Error(t, err) assert.Contains(t, err.Error(), "max") }) t.Run("pin name with unicode is counted by bytes", func(t *testing.T) { // Unicode character can be multiple bytes unicodeName := strings.Repeat("🔒", MaxPinNameBytes/4+1) // emoji is 4 bytes err := ValidatePinName(unicodeName) require.Error(t, err) assert.Contains(t, err.Error(), "bytes") }) } ================================================ FILE: core/commands/commands.go ================================================ // Package commands implements the ipfs command interface // // Using github.com/ipfs/kubo/commands to define the command line and HTTP // APIs. This is the interface available to folks using IPFS from outside of // the Go language. package commands import ( "bytes" "fmt" "io" "os" "slices" "strings" cmds "github.com/ipfs/go-ipfs-cmds" ) type commandEncoder struct { w io.Writer } func (e *commandEncoder) Encode(v any) error { var ( cmd *Command ok bool ) if cmd, ok = v.(*Command); !ok { return fmt.Errorf(`core/commands: unexpected type %T, expected *"core/commands".Command`, v) } for _, s := range cmdPathStrings(cmd, cmd.showOpts) { _, err := e.w.Write([]byte(s + "\n")) if err != nil { return err } } return nil } type Command struct { Name string Subcommands []Command Options []Option showOpts bool } type Option struct { Names []string } const ( flagsOptionName = "flags" ) // CommandsCmd takes in a root command, // and returns a command that lists the subcommands in that root func CommandsCmd(root *cmds.Command) *cmds.Command { return &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List all available commands.", ShortDescription: `Lists all available commands (and subcommands) and exits.`, }, Subcommands: map[string]*cmds.Command{ "completion": CompletionCmd(root), }, Options: []cmds.Option{ cmds.BoolOption(flagsOptionName, "f", "Show command flags"), }, Extra: CreateCmdExtras(SetDoesNotUseRepo(true)), Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { rootCmd := cmd2outputCmd("ipfs", root) rootCmd.showOpts, _ = req.Options[flagsOptionName].(bool) return cmds.EmitOnce(res, &rootCmd) }, Encoders: cmds.EncoderMap{ cmds.Text: func(req *cmds.Request) func(io.Writer) cmds.Encoder { return func(w io.Writer) cmds.Encoder { return &commandEncoder{w} } }, }, Type: Command{}, } } func cmd2outputCmd(name string, cmd *cmds.Command) Command { opts := make([]Option, len(cmd.Options)) for i, opt := range cmd.Options { opts[i] = Option{opt.Names()} } output := Command{ Name: name, Subcommands: make([]Command, 0, len(cmd.Subcommands)), Options: opts, } for name, sub := range cmd.Subcommands { output.Subcommands = append(output.Subcommands, cmd2outputCmd(name, sub)) } return output } func cmdPathStrings(cmd *Command, showOptions bool) []string { var cmds []string var recurse func(prefix string, cmd *Command) recurse = func(prefix string, cmd *Command) { newPrefix := prefix + cmd.Name cmds = append(cmds, newPrefix) if prefix != "" && showOptions { for _, options := range cmd.Options { var cmdOpts []string for _, flag := range options.Names { if len(flag) == 1 { flag = "-" + flag } else { flag = "--" + flag } cmdOpts = append(cmdOpts, newPrefix+" "+flag) } cmds = append(cmds, strings.Join(cmdOpts, " / ")) } } for _, sub := range cmd.Subcommands { recurse(newPrefix+" ", &sub) } } recurse("", cmd) slices.Sort(cmds) return cmds } func CompletionCmd(root *cmds.Command) *cmds.Command { return &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Generate shell completions.", }, NoRemote: true, Subcommands: map[string]*cmds.Command{ "bash": { Helptext: cmds.HelpText{ Tagline: "Generate bash shell completions.", ShortDescription: "Generates command completions for the bash shell.", LongDescription: ` Generates command completions for the bash shell. The simplest way to see it working is write the completions to a file and then source it: > ipfs commands completion bash > ipfs-completion.bash > source ./ipfs-completion.bash To install the completions permanently, they can be moved to /etc/bash_completion.d or sourced from your ~/.bashrc file. `, }, NoRemote: true, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { var buf bytes.Buffer if err := writeBashCompletions(root, &buf); err != nil { return err } res.SetLength(uint64(buf.Len())) return res.Emit(&buf) }, }, "zsh": { Helptext: cmds.HelpText{ Tagline: "Generate zsh shell completions.", ShortDescription: "Generates command completions for the zsh shell.", LongDescription: ` Generates command completions for the zsh shell. The simplest way to see it working is write the completions to a file and then source it: > ipfs commands completion zsh > ipfs-completion.zsh > source ./ipfs-completion.zsh To install the completions permanently, they can be moved to /etc/zsh/completions or sourced from your ~/.zshrc file. `, }, NoRemote: true, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { var buf bytes.Buffer if err := writeZshCompletions(root, &buf); err != nil { return err } res.SetLength(uint64(buf.Len())) return res.Emit(&buf) }, }, "fish": { Helptext: cmds.HelpText{ Tagline: "Generate fish shell completions.", ShortDescription: "Generates command completions for the fish shell.", LongDescription: ` Generates command completions for the fish shell. The simplest way to see it working is write the completions to a file and then source it: > ipfs commands completion fish > ipfs-completion.fish > source ./ipfs-completion.fish To install the completions permanently, they can be moved to /etc/fish/completions or ~/.config/fish/completions or sourced from your ~/.config/fish/config.fish file. `, }, NoRemote: true, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { var buf bytes.Buffer if err := writeFishCompletions(root, &buf); err != nil { return err } res.SetLength(uint64(buf.Len())) return res.Emit(&buf) }, }, }, } } type nonFatalError string // streamResult is a helper function to stream results that possibly // contain non-fatal errors. The helper function is allowed to panic // on internal errors. func streamResult(procVal func(any, io.Writer) nonFatalError) func(cmds.Response, cmds.ResponseEmitter) error { return func(res cmds.Response, re cmds.ResponseEmitter) (rerr error) { defer func() { if r := recover(); r != nil { rerr = fmt.Errorf("internal error: %v", r) } }() var errors bool for { v, err := res.Next() if err != nil { if err == io.EOF { break } rerr = err return } errorMsg := procVal(v, os.Stdout) if errorMsg != "" { errors = true fmt.Fprintf(os.Stderr, "%s\n", errorMsg) } } if errors { rerr = fmt.Errorf("errors while displaying some entries") } return } } ================================================ FILE: core/commands/commands_test.go ================================================ package commands import ( "strings" "testing" cmds "github.com/ipfs/go-ipfs-cmds" ) func collectPaths(prefix string, cmd *cmds.Command, out map[string]struct{}) { for name, sub := range cmd.Subcommands { path := prefix + "/" + name out[path] = struct{}{} collectPaths(path, sub, out) } } func TestCommands(t *testing.T) { list := []string{ "/add", "/bitswap", "/bitswap/ledger", "/bitswap/reprovide", "/bitswap/stat", "/bitswap/wantlist", "/block", "/block/get", "/block/put", "/block/rm", "/block/stat", "/bootstrap", "/bootstrap/add", "/bootstrap/list", "/bootstrap/rm", "/bootstrap/rm/all", "/cat", "/cid", "/cid/base32", "/cid/bases", "/cid/codecs", "/cid/format", "/cid/hashes", "/cid/inspect", "/commands", "/commands/completion", "/commands/completion/bash", "/commands/completion/fish", "/commands/completion/zsh", "/config", "/config/edit", "/config/profile", "/config/profile/apply", "/config/replace", "/config/show", "/dag", "/dag/export", "/dag/get", "/dag/import", "/dag/put", "/dag/resolve", "/dag/stat", "/dht", "/dht/query", "/dht/findprovs", "/dht/findpeer", "/dht/get", "/dht/provide", "/dht/put", "/routing", "/routing/put", "/routing/get", "/routing/findpeer", "/routing/findprovs", "/routing/provide", "/routing/reprovide", "/diag", "/diag/cmds", "/diag/cmds/clear", "/diag/cmds/set-time", "/diag/datastore", "/diag/datastore/count", "/diag/datastore/get", "/diag/datastore/put", "/diag/profile", "/diag/sys", "/files", "/files/chcid", "/files/cp", "/files/flush", "/files/ls", "/files/mkdir", "/files/mv", "/files/read", "/files/rm", "/files/stat", "/files/write", "/files/chmod", "/files/chroot", "/files/touch", "/filestore", "/filestore/dups", "/filestore/ls", "/filestore/verify", "/get", "/id", "/key", "/key/export", "/key/gen", "/key/import", "/key/list", "/key/ls", "/key/rename", "/key/rm", "/key/rotate", "/key/sign", "/key/verify", "/log", "/log/level", "/log/ls", "/log/tail", "/ls", "/mount", "/multibase", "/multibase/decode", "/multibase/encode", "/multibase/transcode", "/multibase/list", "/name", "/name/get", "/name/inspect", "/name/publish", "/name/pubsub", "/name/pubsub/cancel", "/name/pubsub/state", "/name/pubsub/subs", "/name/put", "/name/resolve", "/object", "/object/data", "/object/diff", "/object/get", "/object/links", "/object/new", "/object/patch", "/object/patch/add-link", "/object/patch/append-data", "/object/patch/rm-link", "/object/patch/set-data", "/object/put", "/object/stat", "/p2p", "/p2p/close", "/p2p/forward", "/p2p/listen", "/p2p/ls", "/p2p/stream", "/p2p/stream/close", "/p2p/stream/ls", "/pin", "/pin/add", "/pin/ls", "/pin/remote", "/pin/remote/add", "/pin/remote/ls", "/pin/remote/rm", "/pin/remote/service", "/pin/remote/service/add", "/pin/remote/service/ls", "/pin/remote/service/rm", "/pin/rm", "/pin/update", "/pin/verify", "/ping", "/provide", "/provide/clear", "/provide/stat", "/pubsub", "/pubsub/ls", "/pubsub/peers", "/pubsub/pub", "/pubsub/reset", "/pubsub/sub", "/refs", "/refs/local", "/repo", "/repo/gc", "/repo/migrate", "/repo/stat", "/repo/verify", "/repo/version", "/repo/ls", "/resolve", "/shutdown", "/stats", "/stats/bitswap", "/stats/bw", "/stats/dht", "/stats/provide", "/stats/reprovide", "/stats/repo", "/swarm", "/swarm/addrs", "/swarm/addrs/autonat", "/swarm/addrs/listen", "/swarm/addrs/local", "/swarm/connect", "/swarm/disconnect", "/swarm/filters", "/swarm/filters/add", "/swarm/filters/rm", "/swarm/peers", "/swarm/peering", "/swarm/peering/add", "/swarm/peering/ls", "/swarm/peering/rm", "/swarm/resources", "/update", "/version", "/version/check", "/version/deps", } cmdSet := make(map[string]struct{}) collectPaths("", Root, cmdSet) for _, path := range list { if _, ok := cmdSet[path]; !ok { t.Errorf("%q not in result", path) } else { delete(cmdSet, path) } } for path := range cmdSet { t.Errorf("%q in result but shouldn't be", path) } for _, path := range list { path = path[1:] // remove leading slash split := strings.Split(path, "/") sub, err := Root.Get(split) if err != nil { t.Errorf("error getting subcommand %q: %v", path, err) } else if sub == nil { t.Errorf("subcommand %q is nil even though there was no error", path) } } } ================================================ FILE: core/commands/completion.go ================================================ package commands import ( "io" "slices" "strings" "text/template" cmds "github.com/ipfs/go-ipfs-cmds" ) type completionCommand struct { Name string FullName string Description string Subcommands []*completionCommand Flags []*singleOption Options []*singleOption ShortFlags []string ShortOptions []string LongFlags []string LongOptions []string IsFinal bool } type singleOption struct { LongNames []string ShortNames []string Description string } func commandToCompletions(name string, fullName string, cmd *cmds.Command) *completionCommand { parsed := &completionCommand{ Name: name, FullName: fullName, Description: cmd.Helptext.Tagline, IsFinal: len(cmd.Subcommands) == 0, } for name, subCmd := range cmd.Subcommands { parsed.Subcommands = append(parsed.Subcommands, commandToCompletions(name, fullName+" "+name, subCmd)) } slices.SortFunc(parsed.Subcommands, func(a, b *completionCommand) int { return strings.Compare(a.Name, b.Name) }) for _, opt := range cmd.Options { flag := &singleOption{Description: opt.Description()} flag.LongNames = append(flag.LongNames, opt.Name()) if opt.Type() == cmds.Bool { parsed.LongFlags = append(parsed.LongFlags, opt.Name()) for _, name := range opt.Names() { if len(name) == 1 { parsed.ShortFlags = append(parsed.ShortFlags, name) flag.ShortNames = append(flag.ShortNames, name) break } } parsed.Flags = append(parsed.Flags, flag) } else { parsed.LongOptions = append(parsed.LongOptions, opt.Name()) for _, name := range opt.Names() { if len(name) == 1 { parsed.ShortOptions = append(parsed.ShortOptions, name) flag.ShortNames = append(flag.ShortNames, name) break } } parsed.Options = append(parsed.Options, flag) } } slices.Sort(parsed.LongFlags) slices.Sort(parsed.ShortFlags) slices.Sort(parsed.LongOptions) slices.Sort(parsed.ShortOptions) return parsed } var bashCompletionTemplate, fishCompletionTemplate, zshCompletionTemplate *template.Template func init() { commandTemplate := template.Must(template.New("command").Parse(` while [[ ${index} -lt ${COMP_CWORD} ]]; do case "${COMP_WORDS[index]}" in -*) let index++ continue ;; {{ range .Subcommands }} "{{ .Name }}") let index++ {{ template "command" . }} return 0 ;; {{ end }} esac break done if [[ "${word}" == -* ]]; then {{ if .ShortFlags -}} _ipfs_compgen -W $'{{ range .ShortFlags }}-{{.}} \n{{end}}' -- "${word}" {{ end -}} {{- if .ShortOptions -}} _ipfs_compgen -S = -W $'{{ range .ShortOptions }}-{{.}}\n{{end}}' -- "${word}" {{ end -}} {{- if .LongFlags -}} _ipfs_compgen -W $'{{ range .LongFlags }}--{{.}} \n{{end}}' -- "${word}" {{ end -}} {{- if .LongOptions -}} _ipfs_compgen -S = -W $'{{ range .LongOptions }}--{{.}}\n{{end}}' -- "${word}" {{ end -}} return 0 fi while [[ ${index} -lt ${COMP_CWORD} ]]; do if [[ "${COMP_WORDS[index]}" != -* ]]; then let argidx++ fi let index++ done {{- if .Subcommands }} if [[ "${argidx}" -eq 0 ]]; then _ipfs_compgen -W $'{{ range .Subcommands }}{{.Name}} \n{{end}}' -- "${word}" fi {{ end -}} `)) bashCompletionTemplate = template.Must(commandTemplate.New("root").Parse(`#!/bin/bash _ipfs_compgen() { local oldifs="$IFS" IFS=$'\n' while read -r line; do COMPREPLY+=("$line") done < <(compgen "$@") IFS="$oldifs" } _ipfs() { COMPREPLY=() local index=1 local argidx=0 local word="${COMP_WORDS[COMP_CWORD]}" {{ template "command" . }} } complete -o nosort -o nospace -o default -F _ipfs ipfs `)) zshCompletionTemplate = template.Must(commandTemplate.New("root").Parse(`#!bin/zsh autoload bashcompinit bashcompinit _ipfs_compgen() { local oldifs="$IFS" IFS=$'\n' while read -r line; do COMPREPLY+=("$line") done < <(compgen "$@") IFS="$oldifs" } _ipfs() { COMPREPLY=() local index=1 local argidx=0 local word="${COMP_WORDS[COMP_CWORD]}" {{ template "command" . }} } complete -o nosort -o nospace -o default -F _ipfs ipfs `)) fishCommandTemplate := template.Must(template.New("command").Parse(` {{- if .IsFinal -}} complete -c ipfs -n '__fish_ipfs_seen_all_subcommands_from{{ .FullName }}' -F {{ end -}} {{- range .Flags -}} complete -c ipfs -n '__fish_ipfs_seen_all_subcommands_from{{ $.FullName }}' {{ range .ShortNames }}-s {{.}} {{end}}{{ range .LongNames }}-l {{.}} {{end}}-d "{{ .Description }}" {{ end -}} {{- range .Options -}} complete -c ipfs -n '__fish_ipfs_seen_all_subcommands_from{{ $.FullName }}' -r {{ range .ShortNames }}-s {{.}} {{end}}{{ range .LongNames }}-l {{.}} {{end}}-d "{{ .Description }}" {{ end -}} {{- range .Subcommands }} #{{ .FullName }} complete -c ipfs -n '__fish_ipfs_use_subcommand{{ .FullName }}' -a {{ .Name }} -d "{{ .Description }}" {{ template "command" . }} {{ end -}} `)) fishCompletionTemplate = template.Must(fishCommandTemplate.New("root").Parse(`#!/usr/bin/env fish function __fish_ipfs_seen_all_subcommands_from set -l cmd (commandline -poc) set -e cmd[1] for c in $argv if not contains -- $c $cmd return 1 end end return 0 end function __fish_ipfs_use_subcommand set -e argv[-1] set -l cmd (commandline -poc) set -e cmd[1] for i in $cmd switch $i case '-*' continue case $argv[1] set argv $argv[2..] continue case '*' return 1 end end test -z "$argv" end complete -c ipfs -l help -d "Show the full command help text." complete -c ipfs --keep-order --no-files {{ template "command" . }} `)) } // writeBashCompletions generates a bash completion script for the given command tree. func writeBashCompletions(cmd *cmds.Command, out io.Writer) error { cmds := commandToCompletions("ipfs", "", cmd) return bashCompletionTemplate.Execute(out, cmds) } // writeFishCompletions generates a fish completion script for the given command tree. func writeFishCompletions(cmd *cmds.Command, out io.Writer) error { cmds := commandToCompletions("ipfs", "", cmd) return fishCompletionTemplate.Execute(out, cmds) } func writeZshCompletions(cmd *cmds.Command, out io.Writer) error { cmds := commandToCompletions("ipfs", "", cmd) return zshCompletionTemplate.Execute(out, cmds) } ================================================ FILE: core/commands/config.go ================================================ package commands import ( "encoding/json" "errors" "fmt" "io" "maps" "os" "os/exec" "slices" "strings" "github.com/anmitsu/go-shlex" "github.com/elgris/jsondiff" cmds "github.com/ipfs/go-ipfs-cmds" config "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/repo" "github.com/ipfs/kubo/repo/fsrepo" ) // ConfigUpdateOutput is config profile apply command's output type ConfigUpdateOutput struct { OldCfg map[string]any NewCfg map[string]any } type ConfigField struct { Key string Value any } const ( configBoolOptionName = "bool" configJSONOptionName = "json" configDryRunOptionName = "dry-run" configExpandAutoName = "expand-auto" ) var ConfigCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Get and set IPFS config values.", ShortDescription: ` 'ipfs config' controls configuration variables. It works like 'git config'. The configuration values are stored in a config file inside your IPFS_PATH.`, LongDescription: ` 'ipfs config' controls configuration variables. It works much like 'git config'. The configuration values are stored in a config file inside your IPFS repository (IPFS_PATH). Examples: Get the value of the 'Routing.Type' key: $ ipfs config Routing.Type Set the value of the 'Routing.Type' key: $ ipfs config Routing.Type auto Set multiple values in the 'Addresses.AppendAnnounce' array: $ ipfs config Addresses.AppendAnnounce --json \ '["/dns4/a.example.com/tcp/4001", "/dns4/b.example.com/tcp/4002"]' `, }, Subcommands: map[string]*cmds.Command{ "show": configShowCmd, "edit": configEditCmd, "replace": configReplaceCmd, "profile": configProfileCmd, }, Arguments: []cmds.Argument{ cmds.StringArg("key", true, false, "The key of the config entry (e.g. \"Addresses.API\")."), cmds.StringArg("value", false, false, "The value to set the config entry to."), }, Options: []cmds.Option{ cmds.BoolOption(configBoolOptionName, "Set a boolean value."), cmds.BoolOption(configJSONOptionName, "Parse stringified JSON."), cmds.BoolOption(configExpandAutoName, "Expand 'auto' placeholders to their expanded values from AutoConf service."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { args := req.Arguments key := args[0] var output *ConfigField // This is a temporary fix until we move the private key out of the config file switch strings.ToLower(key) { case "identity", "identity.privkey": return errors.New("cannot show or change private key through API") default: } // Temporary fix until we move ApiKey secrets out of the config file // (remote services are a map, so more advanced blocking is required) if blocked := matchesGlobPrefix(key, config.PinningConcealSelector); blocked { return errors.New("cannot show or change pinning services credentials") } cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err } r, err := fsrepo.Open(cfgRoot) if err != nil { return err } defer r.Close() if len(args) == 2 { // Check if user is trying to write config with expand flag if expandAuto, _ := req.Options[configExpandAutoName].(bool); expandAuto { return fmt.Errorf("--expand-auto can only be used for reading config values, not for setting them") } value := args[1] if parseJSON, _ := req.Options[configJSONOptionName].(bool); parseJSON { var jsonVal any if err := json.Unmarshal([]byte(value), &jsonVal); err != nil { err = fmt.Errorf("failed to unmarshal json. %s", err) return err } output, err = setConfig(r, key, jsonVal) } else if isbool, _ := req.Options[configBoolOptionName].(bool); isbool { output, err = setConfig(r, key, value == "true") } else { output, err = setConfig(r, key, value) } } else { // Check if user wants to expand auto values for getter expandAuto, _ := req.Options[configExpandAutoName].(bool) if expandAuto { output, err = getConfigWithAutoExpand(r, key) } else { output, err = getConfig(r, key) } } if err != nil { return err } return cmds.EmitOnce(res, output) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *ConfigField) error { if len(req.Arguments) == 2 { return nil } buf, err := config.HumanOutput(out.Value) if err != nil { return err } buf = append(buf, byte('\n')) _, err = w.Write(buf) return err }), }, Type: ConfigField{}, } // matchesGlobPrefix returns true if and only if the key matches the glob. // The key is a sequence of string "parts", separated by commas. // The glob is a sequence of string "patterns". // matchesGlobPrefix tries to match all of the first K parts to the first K patterns, respectively, // where K is the length of the shorter of key or glob. // A pattern matches a part if and only if the pattern is "*" or the lowercase pattern equals the lowercase part. // // For example: // // matchesGlobPrefix("foo.bar", []string{"*", "bar", "baz"}) returns true // matchesGlobPrefix("foo.bar.baz", []string{"*", "bar"}) returns true // matchesGlobPrefix("foo.bar", []string{"baz", "*"}) returns false func matchesGlobPrefix(key string, glob []string) bool { k := strings.Split(key, ".") for i, g := range glob { if i >= len(k) { break } if g == "*" { continue } if !strings.EqualFold(k[i], g) { return false } } return true } var configShowCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Output config file contents.", ShortDescription: ` NOTE: For security reasons, this command will omit your private key and remote services. If you would like to make a full backup of your config (private key included), you must copy the config file from your repo. `, }, Type: make(map[string]any), Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err } configFileOpt, _ := req.Options[ConfigFileOption].(string) fname, err := config.Filename(cfgRoot, configFileOpt) if err != nil { return err } data, err := os.ReadFile(fname) if err != nil { return err } var cfg map[string]any err = json.Unmarshal(data, &cfg) if err != nil { return err } // Check if user wants to expand auto values expandAuto, _ := req.Options[configExpandAutoName].(bool) if expandAuto { // Load full config to use resolution methods var fullCfg config.Config err = json.Unmarshal(data, &fullCfg) if err != nil { return err } // Expand auto values and update the map cfg, err = fullCfg.ExpandAutoConfValues(cfg) if err != nil { return err } } cfg, err = scrubValue(cfg, []string{config.IdentityTag, config.PrivKeyTag}) if err != nil { return err } cfg, err = scrubValue(cfg, []string{config.APITag, config.AuthorizationTag}) if err != nil { return err } cfg, err = scrubOptionalValue(cfg, config.PinningConcealSelector) if err != nil { return err } return cmds.EmitOnce(res, &cfg) }, Encoders: cmds.EncoderMap{ cmds.Text: HumanJSONEncoder, }, } var HumanJSONEncoder = cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *map[string]any) error { buf, err := config.HumanOutput(out) if err != nil { return err } buf = append(buf, byte('\n')) _, err = w.Write(buf) return err }) // Scrubs value and returns error if missing func scrubValue(m map[string]any, key []string) (map[string]any, error) { return scrubMapInternal(m, key, false) } // Scrubs value and returns no error if missing func scrubOptionalValue(m map[string]any, key []string) (map[string]any, error) { return scrubMapInternal(m, key, true) } func scrubEither(u any, key []string, okIfMissing bool) (any, error) { m, ok := u.(map[string]any) if ok { return scrubMapInternal(m, key, okIfMissing) } return scrubValueInternal(m, key, okIfMissing) } func scrubValueInternal(v any, key []string, okIfMissing bool) (any, error) { if v == nil && !okIfMissing { return nil, errors.New("failed to find specified key") } return nil, nil } func scrubMapInternal(m map[string]any, key []string, okIfMissing bool) (map[string]any, error) { if len(key) == 0 { return make(map[string]any), nil // delete value } n := map[string]any{} for k, v := range m { if key[0] == "*" || strings.EqualFold(key[0], k) { u, err := scrubEither(v, key[1:], okIfMissing) if err != nil { return nil, err } if u != nil { n[k] = u } } else { n[k] = v } } return n, nil } var configEditCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Open the config file for editing in $EDITOR.", ShortDescription: ` To use 'ipfs config edit', you must have the $EDITOR environment variable set to your preferred text editor. `, }, NoRemote: true, Extra: CreateCmdExtras(SetDoesNotUseRepo(true)), Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err } configFileOpt, _ := req.Options[ConfigFileOption].(string) filename, err := config.Filename(cfgRoot, configFileOpt) if err != nil { return err } return editConfig(filename) }, } var configReplaceCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Replace the config with .", ShortDescription: ` Make sure to back up the config file first if necessary, as this operation can't be undone. `, }, Arguments: []cmds.Argument{ cmds.FileArg("file", true, false, "The file to use as the new config."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err } r, err := fsrepo.Open(cfgRoot) if err != nil { return err } defer r.Close() file, err := cmdenv.GetFileArg(req.Files.Entries()) if err != nil { return err } defer file.Close() return replaceConfig(r, file) }, } var configProfileCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Apply profiles to config.", ShortDescription: fmt.Sprintf(` Available profiles: %s `, buildProfileHelp()), }, Subcommands: map[string]*cmds.Command{ "apply": configProfileApplyCmd, }, } var configProfileApplyCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Apply profile to config.", }, Options: []cmds.Option{ cmds.BoolOption(configDryRunOptionName, "print difference between the current config and the config that would be generated"), }, Arguments: []cmds.Argument{ cmds.StringArg("profile", true, false, "The profile to apply to the config."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { profile, ok := config.Profiles[req.Arguments[0]] if !ok { return fmt.Errorf("%s is not a profile", req.Arguments[0]) } dryRun, _ := req.Options[configDryRunOptionName].(bool) cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err } oldCfg, newCfg, err := transformConfig(cfgRoot, req.Arguments[0], profile.Transform, dryRun) if err != nil { return err } oldCfgMap, err := scrubPrivKey(oldCfg) if err != nil { return err } newCfgMap, err := scrubPrivKey(newCfg) if err != nil { return err } return cmds.EmitOnce(res, &ConfigUpdateOutput{ OldCfg: oldCfgMap, NewCfg: newCfgMap, }) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *ConfigUpdateOutput) error { diff := jsondiff.Compare(out.OldCfg, out.NewCfg) buf := jsondiff.Format(diff) _, err := w.Write(buf) return err }), }, Type: ConfigUpdateOutput{}, } func buildProfileHelp() string { var out string for _, name := range slices.Sorted(maps.Keys(config.Profiles)) { profile := config.Profiles[name] dlines := strings.Split(profile.Description, "\n") for i := range dlines { dlines[i] = " " + dlines[i] } out = out + fmt.Sprintf(" '%s':\n%s\n", name, strings.Join(dlines, "\n")) } return out } // scrubPrivKey scrubs private key for security reasons. func scrubPrivKey(cfg *config.Config) (map[string]any, error) { cfgMap, err := config.ToMap(cfg) if err != nil { return nil, err } cfgMap, err = scrubValue(cfgMap, []string{config.IdentityTag, config.PrivKeyTag}) if err != nil { return nil, err } return cfgMap, nil } // transformConfig returns old config and new config instead of difference between them, // because apply command can provide stable API through this way. // If dryRun is true, repo's config should not be updated and persisted // to storage. Otherwise, repo's config should be updated and persisted // to storage. func transformConfig(configRoot string, configName string, transformer config.Transformer, dryRun bool) (*config.Config, *config.Config, error) { r, err := fsrepo.Open(configRoot) if err != nil { return nil, nil, err } defer r.Close() oldCfg, err := r.Config() if err != nil { return nil, nil, err } // make a copy to avoid updating repo's config unintentionally newCfg, err := oldCfg.Clone() if err != nil { return nil, nil, err } err = transformer(newCfg) if err != nil { return nil, nil, err } if !dryRun { _, err = r.BackupConfig("pre-" + configName + "-") if err != nil { return nil, nil, err } err = r.SetConfig(newCfg) if err != nil { return nil, nil, err } } return oldCfg, newCfg, nil } func getConfig(r repo.Repo, key string) (*ConfigField, error) { value, err := r.GetConfigKey(key) if err != nil { return nil, fmt.Errorf("failed to get config value: %q", err) } return &ConfigField{ Key: key, Value: value, }, nil } func getConfigWithAutoExpand(r repo.Repo, key string) (*ConfigField, error) { // First get the current value value, err := r.GetConfigKey(key) if err != nil { return nil, fmt.Errorf("failed to get config value: %q", err) } // Load full config for resolution fullCfg, err := r.Config() if err != nil { return nil, fmt.Errorf("failed to load config: %q", err) } // Expand auto values based on the key expandedValue := fullCfg.ExpandConfigField(key, value) return &ConfigField{ Key: key, Value: expandedValue, }, nil } func setConfig(r repo.Repo, key string, value any) (*ConfigField, error) { err := r.SetConfigKey(key, value) if err != nil { return nil, fmt.Errorf("failed to set config value: %s (maybe use --json?)", err) } return getConfig(r, key) } // parseEditorCommand parses the EDITOR environment variable into command and arguments func parseEditorCommand(editor string) ([]string, error) { return shlex.Split(editor, true) } func editConfig(filename string) error { editor := os.Getenv("EDITOR") if editor == "" { return errors.New("ENV variable $EDITOR not set") } editorAndArgs, err := parseEditorCommand(editor) if err != nil { return fmt.Errorf("cannot parse $EDITOR value: %s", err) } editor = editorAndArgs[0] args := append(editorAndArgs[1:], filename) cmd := exec.Command(editor, args...) cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr return cmd.Run() } func replaceConfig(r repo.Repo, file io.Reader) error { var newCfg config.Config if err := json.NewDecoder(file).Decode(&newCfg); err != nil { return errors.New("failed to decode file as config") } // Handle Identity.PrivKey (secret) if len(newCfg.Identity.PrivKey) != 0 { return errors.New("setting private key with API is not supported") } keyF, err := getConfig(r, config.PrivKeySelector) if err != nil { return errors.New("failed to get PrivKey") } pkstr, ok := keyF.Value.(string) if !ok { return errors.New("private key in config was not a string") } newCfg.Identity.PrivKey = pkstr // Handle Pinning.RemoteServices (API.Key of each service is a secret) newServices := newCfg.Pinning.RemoteServices oldServices, err := getRemotePinningServices(r) if err != nil { return fmt.Errorf("failed to load remote pinning services info (%v)", err) } // fail fast if service lists are obviously different if len(newServices) != len(oldServices) { return errors.New("cannot add or remove remote pinning services with 'config replace'") } // re-apply API details and confirm every modified service already existed for name, oldSvc := range oldServices { if newSvc, hadSvc := newServices[name]; hadSvc { // fail if input changes any of API details // (interop with config show: allow Endpoint as long it did not change) if len(newSvc.API.Key) != 0 || (len(newSvc.API.Endpoint) != 0 && newSvc.API.Endpoint != oldSvc.API.Endpoint) { return errors.New("cannot change remote pinning services api info with `config replace`") } // re-apply API details and store service in updated config newSvc.API = oldSvc.API newCfg.Pinning.RemoteServices[name] = newSvc } else { // error on service rm attempt return errors.New("cannot add or remove remote pinning services with 'config replace'") } } return r.SetConfig(&newCfg) } func getRemotePinningServices(r repo.Repo) (map[string]config.RemotePinningService, error) { var oldServices map[string]config.RemotePinningService if remoteServicesTag, err := getConfig(r, config.RemoteServicesPath); err == nil { // seems that golang cannot type assert map[string]interface{} to map[string]config.RemotePinningService // so we have to manually copy the data :-| if val, ok := remoteServicesTag.Value.(map[string]any); ok { jsonString, err := json.Marshal(val) if err != nil { return nil, err } err = json.Unmarshal(jsonString, &oldServices) if err != nil { return nil, err } } } return oldServices, nil } ================================================ FILE: core/commands/config_test.go ================================================ package commands import "testing" func TestScrubMapInternalDelete(t *testing.T) { m, err := scrubMapInternal(nil, nil, true) if err != nil { t.Error(err) } if m == nil { t.Errorf("expecting an empty map, got nil") } if len(m) != 0 { t.Errorf("expecting an empty map, got a non-empty map") } } func TestEditorParsing(t *testing.T) { testCases := []struct { name string input string expected []string hasError bool }{ { name: "simple editor", input: "vim", expected: []string{"vim"}, hasError: false, }, { name: "editor with single flag", input: "emacs -nw", expected: []string{"emacs", "-nw"}, hasError: false, }, { name: "VS Code with wait flag (issue #9375)", input: "code --wait", expected: []string{"code", "--wait"}, hasError: false, }, { name: "VS Code with full path and wait flag (issue #9375)", input: "/opt/homebrew/bin/code --wait", expected: []string{"/opt/homebrew/bin/code", "--wait"}, hasError: false, }, { name: "editor with quoted path containing spaces", input: "\"/Applications/Visual Studio Code.app/Contents/Resources/app/bin/code\" --wait", expected: []string{"/Applications/Visual Studio Code.app/Contents/Resources/app/bin/code", "--wait"}, hasError: false, }, { name: "sublime text with wait flag", input: "subl -w", expected: []string{"subl", "-w"}, hasError: false, }, { name: "nano editor", input: "nano", expected: []string{"nano"}, hasError: false, }, { name: "gedit editor", input: "gedit", expected: []string{"gedit"}, hasError: false, }, { name: "editor with multiple flags", input: "vim -c 'set number' -c 'set hlsearch'", expected: []string{"vim", "-c", "set number", "-c", "set hlsearch"}, hasError: false, }, { name: "trailing backslash (POSIX edge case)", input: "editor\\", expected: nil, hasError: true, }, { name: "double quoted editor name with spaces", input: "\"code with spaces\" --wait", expected: []string{"code with spaces", "--wait"}, hasError: false, }, { name: "single quoted editor with flags", input: "'my editor' -flag", expected: []string{"my editor", "-flag"}, hasError: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { result, err := parseEditorCommand(tc.input) if tc.hasError { if err == nil { t.Errorf("Expected error for input '%s', but got none", tc.input) } return } if err != nil { t.Errorf("Unexpected error for input '%s': %v", tc.input, err) return } if len(result) != len(tc.expected) { t.Errorf("Expected %d args, got %d for input '%s'", len(tc.expected), len(result), tc.input) t.Errorf("Expected: %v", tc.expected) t.Errorf("Got: %v", result) return } for i, expected := range tc.expected { if result[i] != expected { t.Errorf("Expected arg %d to be '%s', got '%s' for input '%s'", i, expected, result[i], tc.input) } } }) } } ================================================ FILE: core/commands/dag/dag.go ================================================ package dagcmd import ( "encoding/csv" "encoding/json" "fmt" "io" "path" "github.com/dustin/go-humanize" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" cid "github.com/ipfs/go-cid" cidenc "github.com/ipfs/go-cidutil/cidenc" cmds "github.com/ipfs/go-ipfs-cmds" ) const ( pinRootsOptionName = "pin-roots" progressOptionName = "progress" silentOptionName = "silent" statsOptionName = "stats" fastProvideRootOptionName = "fast-provide-root" fastProvideWaitOptionName = "fast-provide-wait" ) // DagCmd provides a subset of commands for interacting with ipld dag objects var DagCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Interact with IPLD DAG objects.", ShortDescription: ` 'ipfs dag' is used for creating and manipulating DAG objects/hierarchies. This subcommand is intended to deprecate and replace the existing 'ipfs object' command moving forward. `, }, Subcommands: map[string]*cmds.Command{ "put": DagPutCmd, "get": DagGetCmd, "resolve": DagResolveCmd, "import": DagImportCmd, "export": DagExportCmd, "stat": DagStatCmd, }, } // OutputObject is the output type of 'dag put' command type OutputObject struct { Cid cid.Cid } // ResolveOutput is the output type of 'dag resolve' command type ResolveOutput struct { Cid cid.Cid RemPath string } type CarImportStats struct { BlockCount uint64 BlockBytesCount uint64 } // CarImportOutput is the output type of the 'dag import' commands type CarImportOutput struct { Root *RootMeta `json:",omitempty"` Stats *CarImportStats `json:",omitempty"` } // RootMeta is the metadata for a root pinning response type RootMeta struct { Cid cid.Cid PinErrorMsg string } // DagPutCmd is a command for adding a dag node var DagPutCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Add a DAG node to IPFS.", ShortDescription: ` 'ipfs dag put' accepts input from a file or stdin and parses it into an object of the specified format. `, }, Arguments: []cmds.Argument{ cmds.FileArg("object data", true, true, "The object to put").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption("store-codec", "Codec that the stored object will be encoded with").WithDefault("dag-cbor"), cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"), cmds.BoolOption("pin", "Pin this object when adding."), cmds.StringOption("hash", "Hash function to use"), cmdutils.AllowBigBlockOption, }, Run: dagPut, Type: OutputObject{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *OutputObject) error { enc, err := cmdenv.GetLowLevelCidEncoder(req) if err != nil { return err } fmt.Fprintln(w, enc.Encode(out.Cid)) return nil }), }, } // DagGetCmd is a command for getting a dag node from IPFS var DagGetCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Get a DAG node from IPFS.", ShortDescription: ` 'ipfs dag get' fetches a DAG node from IPFS and prints it out in the specified format. `, }, Arguments: []cmds.Argument{ cmds.StringArg("ref", true, false, "The object to get").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption("output-codec", "Format that the object will be encoded as.").WithDefault("dag-json"), }, Run: dagGet, } // DagResolveCmd returns address of highest block within a path and a path remainder var DagResolveCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Resolve IPLD block.", ShortDescription: ` 'ipfs dag resolve' fetches a DAG node from IPFS, prints its address and remaining path. `, }, Arguments: []cmds.Argument{ cmds.StringArg("ref", true, false, "The path to resolve").EnableStdin(), }, Run: dagResolve, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *ResolveOutput) error { var ( enc cidenc.Encoder err error ) switch { case !cmdenv.CidBaseDefined(req): // Not specified, check the path. enc, err = cmdenv.CidEncoderFromPath(req.Arguments[0]) if err == nil { break } // Nope, fallback on the default. fallthrough default: enc, err = cmdenv.GetLowLevelCidEncoder(req) if err != nil { return err } } p := enc.Encode(out.Cid) if out.RemPath != "" { p = path.Join(p, out.RemPath) } fmt.Fprint(w, p) return nil }), }, Type: ResolveOutput{}, } // DagImportCmd is a command for importing a car to ipfs var DagImportCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Import the contents of .car files", ShortDescription: ` 'ipfs dag import' imports all blocks present in supplied .car ( Content Address aRchive ) files, recursively pinning any roots specified in the CAR file headers, unless --pin-roots is set to false. Note: This command will import all blocks in the CAR file, not just those reachable from the specified roots. However, these other blocks will not be pinned and may be garbage collected later. The pinning of the roots happens after all car files are processed, permitting import of DAGs spanning multiple files. Pinning takes place in offline-mode exclusively, one root at a time. If the combination of blocks from the imported CAR files and what is currently present in the blockstore does not represent a complete DAG, pinning of that individual root will fail. FAST PROVIDE OPTIMIZATION: Root CIDs from CAR headers are immediately provided to the DHT in addition to the regular provide queue, allowing other peers to discover your content right away. This complements the sweep provider, which efficiently provides all blocks according to Provide.Strategy over time. By default, the provide happens in the background without blocking the command. Use --fast-provide-wait to wait for the provide to complete, or --fast-provide-root=false to skip it. Works even with --pin-roots=false. Automatically skipped when DHT is not available. Maximum supported CAR version: 2 Specification of CAR formats: https://ipld.io/specs/transport/car/ `, }, Arguments: []cmds.Argument{ cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(), }, Options: []cmds.Option{ cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true), cmds.BoolOption(silentOptionName, "No output."), cmds.BoolOption(statsOptionName, "Output stats."), cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CIDs to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"), cmds.BoolOption(fastProvideWaitOptionName, "Block until the immediate provide completes before returning. Default: Import.FastProvideWait"), cmdutils.AllowBigBlockOption, }, Type: CarImportOutput{}, Run: dagImport, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *CarImportOutput) error { silent, _ := req.Options[silentOptionName].(bool) if silent { return nil } // event should have only one of `Root` or `Stats` set, not both if event.Root == nil { if event.Stats == nil { return fmt.Errorf("unexpected message from DAG import") } stats, _ := req.Options[statsOptionName].(bool) if stats { fmt.Fprintf(w, "Imported %d blocks (%d bytes)\n", event.Stats.BlockCount, event.Stats.BlockBytesCount) } return nil } if event.Stats != nil { return fmt.Errorf("unexpected message from DAG import") } enc, err := cmdenv.GetLowLevelCidEncoder(req) if err != nil { return err } if event.Root.PinErrorMsg != "" { return fmt.Errorf("pinning root %q FAILED: %s", enc.Encode(event.Root.Cid), event.Root.PinErrorMsg) } event.Root.PinErrorMsg = "success" _, err = fmt.Fprintf( w, "Pinned root\t%s\t%s\n", enc.Encode(event.Root.Cid), event.Root.PinErrorMsg, ) return err }), }, } // DagExportCmd is a command for exporting an ipfs dag to a car var DagExportCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Streams the selected DAG as a .car stream on stdout.", ShortDescription: ` 'ipfs dag export' fetches a DAG and streams it out as a well-formed .car file. Note that at present only single root selections / .car files are supported. The output of blocks happens in strict DAG-traversal, first-seen, order. CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/ `, HTTP: &cmds.HTTPHelpText{ ResponseContentType: "application/vnd.ipld.car", }, }, Arguments: []cmds.Argument{ cmds.StringArg("root", true, false, "CID of a root to recursively export").EnableStdin(), }, Options: []cmds.Option{ cmds.BoolOption(progressOptionName, "p", "Display progress on CLI. Defaults to true when STDERR is a TTY."), }, Run: dagExport, PostRun: cmds.PostRunMap{ cmds.CLI: finishCLIExport, }, } // DagStat is a dag stat command response type DagStat struct { Cid cid.Cid Size uint64 `json:",omitempty"` NumBlocks int64 `json:",omitempty"` } func (s *DagStat) String() string { return fmt.Sprintf("%s %d %d", s.Cid.String()[:20], s.Size, s.NumBlocks) } func (s *DagStat) MarshalJSON() ([]byte, error) { type Alias DagStat /* We can't rely on cid.Cid.MarshalJSON since it uses the {"/": "..."} format. To make the output consistent and follow the Kubo API patterns we use the Cid.String method */ return json.Marshal(struct { Cid string `json:"Cid"` *Alias }{ Cid: s.Cid.String(), Alias: (*Alias)(s), }) } func (s *DagStat) UnmarshalJSON(data []byte) error { /* We can't rely on cid.Cid.UnmarshalJSON since it uses the {"/": "..."} format. To make the output consistent and follow the Kubo API patterns we use the Cid.Parse method */ type Alias DagStat aux := struct { Cid string `json:"Cid"` *Alias }{ Alias: (*Alias)(s), } if err := json.Unmarshal(data, &aux); err != nil { return err } Cid, err := cid.Parse(aux.Cid) if err != nil { return err } s.Cid = Cid return nil } type DagStatSummary struct { redundantSize uint64 `json:"-"` UniqueBlocks int `json:",omitempty"` TotalSize uint64 `json:",omitempty"` SharedSize uint64 `json:",omitempty"` Ratio float32 `json:",omitempty"` DagStatsArray []*DagStat `json:"DagStats,omitempty"` } func (s *DagStatSummary) String() string { return fmt.Sprintf("Total Size: %d (%s)\nUnique Blocks: %d\nShared Size: %d (%s)\nRatio: %f", s.TotalSize, humanize.Bytes(s.TotalSize), s.UniqueBlocks, s.SharedSize, humanize.Bytes(s.SharedSize), s.Ratio) } func (s *DagStatSummary) incrementTotalSize(size uint64) { s.TotalSize += size } func (s *DagStatSummary) incrementRedundantSize(size uint64) { s.redundantSize += size } func (s *DagStatSummary) appendStats(stats *DagStat) { s.DagStatsArray = append(s.DagStatsArray, stats) } func (s *DagStatSummary) calculateSummary() { s.Ratio = float32(s.redundantSize) / float32(s.TotalSize) s.SharedSize = s.redundantSize - s.TotalSize } // DagStatCmd is a command for getting size information about an ipfs-stored dag var DagStatCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Gets stats for a DAG.", ShortDescription: ` 'ipfs dag stat' fetches a DAG and returns various statistics about it. Statistics include size and number of blocks. Note: This command skips duplicate blocks in reporting both size and the number of blocks `, }, Arguments: []cmds.Argument{ cmds.StringArg("root", true, true, "CID of a DAG root to get statistics for").EnableStdin(), }, Options: []cmds.Option{ cmds.BoolOption(progressOptionName, "p", "Show progress on stderr. Auto-detected if stderr is a terminal."), }, Run: dagStat, Type: DagStatSummary{}, PostRun: cmds.PostRunMap{ cmds.CLI: finishCLIStat, }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStatSummary) error { fmt.Fprintln(w) csvWriter := csv.NewWriter(w) csvWriter.Comma = '\t' cidSpacing := len(event.DagStatsArray[0].Cid.String()) header := []string{fmt.Sprintf("%-*s", cidSpacing, "CID"), fmt.Sprintf("%-15s", "Blocks"), "Size"} if err := csvWriter.Write(header); err != nil { return err } for _, dagStat := range event.DagStatsArray { numBlocksStr := fmt.Sprint(dagStat.NumBlocks) err := csvWriter.Write([]string{ dagStat.Cid.String(), fmt.Sprintf("%-15s", numBlocksStr), fmt.Sprint(dagStat.Size), }) if err != nil { return err } } csvWriter.Flush() fmt.Fprint(w, "\nSummary\n") _, err := fmt.Fprintf( w, "%v\n", event, ) fmt.Fprint(w, "\n\n") return err }), cmds.JSON: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStatSummary) error { return json.NewEncoder(w).Encode(event) }, ), }, } ================================================ FILE: core/commands/dag/export.go ================================================ package dagcmd import ( "context" "errors" "fmt" "io" "os" "time" "github.com/cheggaaa/pb" cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" iface "github.com/ipfs/kubo/core/coreiface" gocar "github.com/ipld/go-car/v2" cidlink "github.com/ipld/go-ipld-prime/linking/cid" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" ) func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { // Accept CID or a content path p, err := cmdutils.PathOrCidPath(req.Arguments[0]) if err != nil { return err } api, err := cmdenv.GetApi(env, req) if err != nil { return err } // Resolve path and confirm the root block is available, fail fast if not b, err := api.Block().Stat(req.Context, p) if err != nil { return err } c := b.Path().RootCid() pipeR, pipeW := io.Pipe() errCh := make(chan error, 2) // we only report the 1st error go func() { defer func() { if err := pipeW.Close(); err != nil { errCh <- fmt.Errorf("stream flush failed: %s", err) } close(errCh) }() lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(&dagStore{dag: api.Dag(), ctx: req.Context}) // Uncomment the following to support CARv2 output. /* car, err := gocar.NewSelectiveWriter(req.Context, &lsys, c, selectorparse.CommonSelector_ExploreAllRecursively, gocar.AllowDuplicatePuts(false)) if err != nil { errCh <- err return } if _, err = car.WriteTo(pipeW); err != nil { errCh <- err return } */ _, err := gocar.TraverseV1(req.Context, &lsys, c, selectorparse.CommonSelector_ExploreAllRecursively, pipeW, gocar.AllowDuplicatePuts(false)) if err != nil { errCh <- err return } }() res.SetEncodingType(cmds.OctetStream) res.SetContentType("application/vnd.ipld.car") if err := res.Emit(pipeR); err != nil { pipeR.Close() // ignore the error if any return err } err = <-errCh // minimal user friendliness if errors.Is(err, ipld.ErrNotFound{}) { explicitOffline, _ := req.Options["offline"].(bool) if explicitOffline { err = fmt.Errorf("%s (currently offline, perhaps retry without the offline flag)", err) } else { node, envErr := cmdenv.GetNode(env) if envErr == nil && !node.IsOnline { err = fmt.Errorf("%s (currently offline, perhaps retry after attaching to the network)", err) } } } return err } func finishCLIExport(res cmds.Response, re cmds.ResponseEmitter) error { var showProgress bool val, specified := res.Request().Options[progressOptionName] if !specified { // default based on TTY availability errStat, _ := os.Stderr.Stat() if (errStat.Mode() & os.ModeCharDevice) != 0 { showProgress = true } } else if val.(bool) { showProgress = true } // simple passthrough, no progress if !showProgress { return cmds.Copy(re, res) } bar := pb.New64(0).SetUnits(pb.U_BYTES) bar.Output = os.Stderr bar.ShowSpeed = true bar.ShowElapsedTime = true bar.RefreshRate = 500 * time.Millisecond bar.Start() var processedOneResponse bool for { v, err := res.Next() if err != nil { if errors.Is(err, io.EOF) { // We only write the final bar update on success // On error it looks too weird bar.Finish() return re.Close() } return re.CloseWithError(err) } if processedOneResponse { return re.CloseWithError(errors.New("unexpected multipart response during emit, please file a bugreport")) } r, ok := v.(io.Reader) if !ok { // some sort of encoded response, this should not be happening return errors.New("unexpected non-stream passed to PostRun: please file a bugreport") } processedOneResponse = true if err = re.Emit(bar.NewProxyReader(r)); err != nil { return err } } } type dagStore struct { dag iface.APIDagService ctx context.Context } func (ds *dagStore) Get(ctx context.Context, key string) ([]byte, error) { if ctx.Err() != nil { return nil, ctx.Err() } c, err := cidFromBinString(key) if err != nil { return nil, err } block, err := ds.dag.Get(ds.ctx, c) if err != nil { return nil, err } return block.RawData(), nil } func (ds *dagStore) Has(ctx context.Context, key string) (bool, error) { _, err := ds.Get(ctx, key) if err != nil { if errors.Is(err, ipld.ErrNotFound{}) { return false, nil } return false, err } return true, nil } func cidFromBinString(key string) (cid.Cid, error) { l, k, err := cid.CidFromBytes([]byte(key)) if err != nil { return cid.Undef, fmt.Errorf("dagStore: key was not a cid: %w", err) } if l != len(key) { return cid.Undef, fmt.Errorf("dagSore: key was not a cid: had %d bytes leftover", len(key)-l) } return k, nil } ================================================ FILE: core/commands/dag/get.go ================================================ package dagcmd import ( "fmt" "io" "github.com/ipfs/boxo/path" ipldlegacy "github.com/ipfs/go-ipld-legacy" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/multicodec" "github.com/ipld/go-ipld-prime/traversal" mc "github.com/multiformats/go-multicodec" cmds "github.com/ipfs/go-ipfs-cmds" ) func dagGet(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } codecStr, _ := req.Options["output-codec"].(string) var codec mc.Code if err := codec.Set(codecStr); err != nil { return err } p, err := cmdutils.PathOrCidPath(req.Arguments[0]) if err != nil { return err } rp, remainder, err := api.ResolvePath(req.Context, p) if err != nil { return err } obj, err := api.Dag().Get(req.Context, rp.RootCid()) if err != nil { return err } universal, ok := obj.(ipldlegacy.UniversalNode) if !ok { return fmt.Errorf("%T is not a valid IPLD node", obj) } finalNode := universal.(ipld.Node) if len(remainder) > 0 { remainderPath := ipld.ParsePath(path.SegmentsToString(remainder...)) finalNode, err = traversal.Get(finalNode, remainderPath) if err != nil { return err } } encoder, err := multicodec.LookupEncoder(uint64(codec)) if err != nil { return fmt.Errorf("invalid encoding: %s - %s", codec, err) } r, w := io.Pipe() go func() { defer w.Close() if err := encoder(finalNode, w); err != nil { _ = w.CloseWithError(err) } }() return res.Emit(r) } ================================================ FILE: core/commands/dag/import.go ================================================ package dagcmd import ( "errors" "fmt" "io" "github.com/ipfs/boxo/files" blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" ipld "github.com/ipfs/go-ipld-format" ipldlegacy "github.com/ipfs/go-ipld-legacy" logging "github.com/ipfs/go-log/v2" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/coreiface/options" gocarv2 "github.com/ipld/go-car/v2" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" ) var log = logging.Logger("core/commands") func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { node, err := cmdenv.GetNode(env) if err != nil { return err } cfg, err := node.Repo.Config() if err != nil { return err } api, err := cmdenv.GetApi(env, req) if err != nil { return err } blockDecoder := ipldlegacy.NewDecoder() // on import ensure we do not reach out to the network for any reason // if a pin based on what is imported + what is in the blockstore // isn't possible: tough luck api, err = api.WithOptions(options.Api.Offline(true)) if err != nil { return err } doPinRoots, _ := req.Options[pinRootsOptionName].(bool) fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool) fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool) fastProvideRoot = config.ResolveBoolFromConfig(fastProvideRoot, fastProvideRootSet, cfg.Import.FastProvideRoot, config.DefaultFastProvideRoot) fastProvideWait = config.ResolveBoolFromConfig(fastProvideWait, fastProvideWaitSet, cfg.Import.FastProvideWait, config.DefaultFastProvideWait) // grab a pinlock ( which doubles as a GC lock ) so that regardless of the // size of the streamed-in cars nothing will disappear on us before we had // a chance to roots that may show up at the very end // This is especially important for use cases like dagger: // ipfs dag import $( ... | ipfs-dagger --stdout=carfifos ) // if doPinRoots { unlocker := node.Blockstore.PinLock(req.Context) defer unlocker.Unlock(req.Context) } // this is *not* a transaction // it is simply a way to relieve pressure on the blockstore // similar to pinner.Pin/pinner.Flush batch := ipld.NewBatch(req.Context, api.Dag(), // Default: 128. Means 128 file descriptors needed in flatfs ipld.MaxNodesBatchOption(int(cfg.Import.BatchMaxNodes.WithDefault(config.DefaultBatchMaxNodes))), // Default 100MiB. When setting block size to 1MiB, we can add // ~100 nodes maximum. With default 256KiB block-size, we will // hit the max nodes limit at 32MiB.p ipld.MaxSizeBatchOption(int(cfg.Import.BatchMaxSize.WithDefault(config.DefaultBatchMaxSize))), ) roots := cid.NewSet() var blockCount, blockBytesCount uint64 // remember last valid block and provide a meaningful error message // when a truncated/mangled CAR is being imported importError := func(previous blocks.Block, current blocks.Block, err error) error { if current != nil { return fmt.Errorf("import failed at block %q: %w", current.Cid(), err) } if previous != nil { return fmt.Errorf("import failed after block %q: %w", previous.Cid(), err) } return fmt.Errorf("import failed: %w", err) } it := req.Files.Entries() for it.Next() { file := files.FileFromEntry(it) if file == nil { return errors.New("expected a file handle") } // import blocks err = func() error { // wrap a defer-closer-scope // // every single file in it() is already open before we start // just close here sooner rather than later for neatness // and to surface potential errors writing on closed fifos // this won't/can't help with not running out of handles defer file.Close() var previous blocks.Block car, err := gocarv2.NewBlockReader(file) if err != nil { return err } for _, c := range car.Roots { roots.Add(c) } for { block, err := car.Next() if err != nil && err != io.EOF { return importError(previous, block, err) } else if block == nil { break } if err := cmdutils.CheckBlockSize(req, uint64(len(block.RawData()))); err != nil { return importError(previous, block, err) } // the double-decode is suboptimal, but we need it for batching nd, err := blockDecoder.DecodeNode(req.Context, block) if err != nil { return importError(previous, block, err) } if err := batch.Add(req.Context, nd); err != nil { return importError(previous, block, err) } blockCount++ blockBytesCount += uint64(len(block.RawData())) previous = block } return nil }() if err != nil { return err } } if err := batch.Commit(); err != nil { return err } // It is not guaranteed that a root in a header is actually present in the same ( or any ) // .car file. This is the case in version 1, and ideally in further versions too. // Accumulate any root CID seen in a header, and supplement its actual node if/when encountered // We will attempt a pin *only* at the end in case all car files were well-formed. // opportunistic pinning: try whatever sticks if doPinRoots { err = roots.ForEach(func(c cid.Cid) error { ret := RootMeta{Cid: c} // This will trigger a full read of the DAG in the pinner, to make sure we have all blocks. // Ideally we would do colloring of the pinning state while importing the blocks // and ensure the gray bucket is empty at the end (or use the network to download missing blocks). if block, err := node.Blockstore.Get(req.Context, c); err != nil { ret.PinErrorMsg = err.Error() } else if nd, err := blockDecoder.DecodeNode(req.Context, block); err != nil { ret.PinErrorMsg = err.Error() } else if err := node.Pinning.Pin(req.Context, nd, true, ""); err != nil { ret.PinErrorMsg = err.Error() } else if err := node.Pinning.Flush(req.Context); err != nil { ret.PinErrorMsg = err.Error() } return res.Emit(&CarImportOutput{Root: &ret}) }) if err != nil { return err } } stats, _ := req.Options[statsOptionName].(bool) if stats { err = res.Emit(&CarImportOutput{ Stats: &CarImportStats{ BlockCount: blockCount, BlockBytesCount: blockBytesCount, }, }) if err != nil { return err } } // Fast-provide roots for faster discovery if fastProvideRoot { err = roots.ForEach(func(c cid.Cid) error { return cmdenv.ExecuteFastProvide(req.Context, node, cfg, c, fastProvideWait, doPinRoots, doPinRoots, false) }) if err != nil { return err } } else { if fastProvideWait { log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config", "wait-flag-ignored", true) } else { log.Debugw("fast-provide-root: skipped", "reason", "disabled by flag or config") } } return nil } ================================================ FILE: core/commands/dag/put.go ================================================ package dagcmd import ( "bytes" "fmt" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ipldlegacy "github.com/ipfs/go-ipld-legacy" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" "github.com/ipld/go-ipld-prime/multicodec" basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/ipfs/boxo/files" cmds "github.com/ipfs/go-ipfs-cmds" ipld "github.com/ipfs/go-ipld-format" mc "github.com/multiformats/go-multicodec" // Expected minimal set of available format/ienc codecs. _ "github.com/ipld/go-codec-dagpb" _ "github.com/ipld/go-ipld-prime/codec/cbor" _ "github.com/ipld/go-ipld-prime/codec/dagcbor" _ "github.com/ipld/go-ipld-prime/codec/dagjson" _ "github.com/ipld/go-ipld-prime/codec/json" _ "github.com/ipld/go-ipld-prime/codec/raw" ) func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } nd, err := cmdenv.GetNode(env) if err != nil { return err } cfg, err := nd.Repo.Config() if err != nil { return err } inputCodec, _ := req.Options["input-codec"].(string) storeCodec, _ := req.Options["store-codec"].(string) hash, _ := req.Options["hash"].(string) dopin, _ := req.Options["pin"].(bool) if hash == "" { hash = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction) } var icodec mc.Code if err := icodec.Set(inputCodec); err != nil { return err } var scodec mc.Code if err := scodec.Set(storeCodec); err != nil { return err } var mhType mc.Code if err := mhType.Set(hash); err != nil { return err } cidPrefix := cid.Prefix{ Version: 1, Codec: uint64(scodec), MhType: uint64(mhType), MhLength: -1, } decoder, err := multicodec.LookupDecoder(uint64(icodec)) if err != nil { return err } encoder, err := multicodec.LookupEncoder(uint64(scodec)) if err != nil { return err } var adder ipld.NodeAdder = api.Dag() if dopin { adder = api.Dag().Pinning() } b := ipld.NewBatch(req.Context, adder) it := req.Files.Entries() for it.Next() { file := files.FileFromEntry(it) if file == nil { return fmt.Errorf("expected a regular file") } node := basicnode.Prototype.Any.NewBuilder() if err := decoder(node, file); err != nil { return err } n := node.Build() bd := bytes.NewBuffer([]byte{}) if err := encoder(n, bd); err != nil { return err } blockCid, err := cidPrefix.Sum(bd.Bytes()) if err != nil { return err } blk, err := blocks.NewBlockWithCid(bd.Bytes(), blockCid) if err != nil { return err } ln := ipldlegacy.LegacyNode{ Block: blk, Node: n, } if err := cmdutils.CheckBlockSize(req, uint64(bd.Len())); err != nil { return err } if err := b.Add(req.Context, &ln); err != nil { return err } cid := ln.Cid() if err := res.Emit(&OutputObject{Cid: cid}); err != nil { return err } } if it.Err() != nil { return it.Err() } if err := b.Commit(); err != nil { return err } return nil } ================================================ FILE: core/commands/dag/resolve.go ================================================ package dagcmd import ( "github.com/ipfs/boxo/path" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" cmds "github.com/ipfs/go-ipfs-cmds" ) func dagResolve(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } p, err := cmdutils.PathOrCidPath(req.Arguments[0]) if err != nil { return err } rp, remainder, err := api.ResolvePath(req.Context, p) if err != nil { return err } return cmds.EmitOnce(res, &ResolveOutput{ Cid: rp.RootCid(), RemPath: path.SegmentsToString(remainder...), }) } ================================================ FILE: core/commands/dag/stat.go ================================================ package dagcmd import ( "fmt" "io" "os" "github.com/dustin/go-humanize" mdag "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/boxo/ipld/merkledag/traverse" cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" "github.com/ipfs/kubo/core/commands/e" ) // TODO cache every cid traversal in a dp cache // if the cid exists in the cache, don't traverse it, and use the cached result // to compute the new state func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { // Default to true (emit intermediate states) for HTTP/RPC clients that want progress progressive := true if val, specified := req.Options[progressOptionName].(bool); specified { progressive = val } api, err := cmdenv.GetApi(env, req) if err != nil { return err } nodeGetter := mdag.NewSession(req.Context, api.Dag()) cidSet := cid.NewSet() dagStatSummary := &DagStatSummary{DagStatsArray: []*DagStat{}} for _, a := range req.Arguments { p, err := cmdutils.PathOrCidPath(a) if err != nil { return err } rp, remainder, err := api.ResolvePath(req.Context, p) if err != nil { return err } if len(remainder) > 0 { return fmt.Errorf("cannot return size for anything other than a DAG with a root CID") } obj, err := nodeGetter.Get(req.Context, rp.RootCid()) if err != nil { return err } dagstats := &DagStat{Cid: rp.RootCid()} dagStatSummary.appendStats(dagstats) err = traverse.Traverse(obj, traverse.Options{ DAG: nodeGetter, Order: traverse.DFSPre, Func: func(current traverse.State) error { currentNodeSize := uint64(len(current.Node.RawData())) dagstats.Size += currentNodeSize dagstats.NumBlocks++ if !cidSet.Has(current.Node.Cid()) { dagStatSummary.incrementTotalSize(currentNodeSize) } dagStatSummary.incrementRedundantSize(currentNodeSize) cidSet.Add(current.Node.Cid()) if progressive { if err := res.Emit(dagStatSummary); err != nil { return err } } return nil }, ErrFunc: nil, SkipDuplicates: true, }) if err != nil { return fmt.Errorf("error traversing DAG: %w", err) } } dagStatSummary.UniqueBlocks = cidSet.Len() dagStatSummary.calculateSummary() if err := res.Emit(dagStatSummary); err != nil { return err } return nil } func finishCLIStat(res cmds.Response, re cmds.ResponseEmitter) error { // Determine whether to show progress based on TTY detection or explicit flag var showProgress bool val, specified := res.Request().Options[progressOptionName] if !specified { // Auto-detect: show progress only if stderr is a TTY if errStat, err := os.Stderr.Stat(); err == nil { showProgress = (errStat.Mode() & os.ModeCharDevice) != 0 } } else { showProgress = val.(bool) } var dagStats *DagStatSummary for { v, err := res.Next() if err != nil { if err == io.EOF { break } return err } switch out := v.(type) { case *DagStatSummary: dagStats = out // Ratio == 0 means this is a progress update (not final result) if showProgress && dagStats.Ratio == 0 { // Sum up total progress across all DAGs being scanned var totalBlocks int64 var totalSize uint64 for _, stat := range dagStats.DagStatsArray { totalBlocks += stat.NumBlocks totalSize += stat.Size } fmt.Fprintf(os.Stderr, "Fetched/Processed %d blocks, %d bytes (%s)\r", totalBlocks, totalSize, humanize.Bytes(totalSize)) } default: return e.TypeErr(out, v) } } // Clear the progress line before final output if showProgress { fmt.Fprint(os.Stderr, "\033[2K\r") } return re.Emit(dagStats) } ================================================ FILE: core/commands/dht.go ================================================ package commands import ( "context" "errors" "fmt" "io" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/kubo/core/commands/cmdenv" peer "github.com/libp2p/go-libp2p/core/peer" routing "github.com/libp2p/go-libp2p/core/routing" ) var ErrNotDHT = errors.New("routing service is not a DHT") var DhtCmd = &cmds.Command{ Status: cmds.Deprecated, Helptext: cmds.HelpText{ Tagline: "Issue commands directly through the DHT.", ShortDescription: ``, }, Subcommands: map[string]*cmds.Command{ "query": queryDhtCmd, "findprovs": RemovedDHTCmd, "findpeer": RemovedDHTCmd, "get": RemovedDHTCmd, "put": RemovedDHTCmd, "provide": RemovedDHTCmd, }, } // kademlia extends the routing interface with a command to get the peers closest to the target type kademlia interface { routing.Routing GetClosestPeers(ctx context.Context, key string) ([]peer.ID, error) } var queryDhtCmd = &cmds.Command{ Status: cmds.Deprecated, Helptext: cmds.HelpText{ Tagline: "Find the closest Peer IDs to a given Peer ID by querying the DHT.", ShortDescription: "Outputs a list of newline-delimited Peer IDs.", }, Arguments: []cmds.Argument{ cmds.StringArg("peerID", true, true, "The peerID to run the query against."), }, Options: []cmds.Option{ cmds.BoolOption(dhtVerboseOptionName, "v", "Print extra information."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } if !nd.HasActiveDHTClient() { return ErrNotDHT } id, err := peer.Decode(req.Arguments[0]) if err != nil { return cmds.ClientError("invalid peer ID") } ctx, cancel := context.WithCancel(req.Context) defer cancel() ctx, events := routing.RegisterForQueryEvents(ctx) client := nd.DHTClient if nd.DHT != nil && client == nd.DHT { client = nd.DHT.WAN if !nd.DHT.WANActive() { client = nd.DHT.LAN } } if d, ok := client.(kademlia); !ok { return errors.New("dht client does not support GetClosestPeers") } else { errCh := make(chan error, 1) go func() { defer close(errCh) defer cancel() closestPeers, err := d.GetClosestPeers(ctx, string(id)) for _, p := range closestPeers { routing.PublishQueryEvent(ctx, &routing.QueryEvent{ ID: p, Type: routing.FinalPeer, }) } if err != nil { errCh <- err return } }() for e := range events { if err := res.Emit(e); err != nil { return err } } return <-errCh } }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *routing.QueryEvent) error { pfm := pfuncMap{ routing.FinalPeer: func(obj *routing.QueryEvent, out io.Writer, verbose bool) error { fmt.Fprintf(out, "%s\n", obj.ID) return nil }, } verbose, _ := req.Options[dhtVerboseOptionName].(bool) return printEvent(out, w, verbose, pfm) }), }, Type: routing.QueryEvent{}, } var RemovedDHTCmd = &cmds.Command{ Status: cmds.Removed, Helptext: cmds.HelpText{ Tagline: "Removed, use 'ipfs routing' instead.", }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { return errors.New("removed, use 'ipfs routing' instead") }, } ================================================ FILE: core/commands/dht_test.go ================================================ package commands import ( "testing" "github.com/ipfs/boxo/namesys" ipns "github.com/ipfs/boxo/ipns" "github.com/libp2p/go-libp2p/core/test" ) func TestKeyTranslation(t *testing.T) { pid := test.RandPeerIDFatal(t) pkname := namesys.PkRoutingKey(pid) ipnsname := ipns.NameFromPeer(pid).RoutingKey() pkk, err := escapeDhtKey("/pk/" + pid.String()) if err != nil { t.Fatal(err) } ipnsk, err := escapeDhtKey("/ipns/" + pid.String()) if err != nil { t.Fatal(err) } if pkk != pkname { t.Fatal("keys didn't match!") } if ipnsk != string(ipnsname) { t.Fatal("keys didn't match!") } } ================================================ FILE: core/commands/diag.go ================================================ package commands import ( "encoding/hex" "errors" "fmt" "io" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/mount" "github.com/ipfs/go-datastore/query" cmds "github.com/ipfs/go-ipfs-cmds" oldcmds "github.com/ipfs/kubo/commands" node "github.com/ipfs/kubo/core/node" fsrepo "github.com/ipfs/kubo/repo/fsrepo" ) var DiagCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Generate diagnostic reports.", }, Subcommands: map[string]*cmds.Command{ "sys": sysDiagCmd, "cmds": ActiveReqsCmd, "profile": sysProfileCmd, "datastore": diagDatastoreCmd, }, } var diagDatastoreCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Low-level datastore inspection for debugging and testing.", ShortDescription: ` 'ipfs diag datastore' provides low-level access to the datastore for debugging and testing purposes. WARNING: FOR DEBUGGING/TESTING ONLY These commands expose internal datastore details and should not be used in production workflows. The datastore format may change between versions. The daemon must not be running when calling these commands. When the provider keystore datastores exist on disk (nodes with Provide.DHT.SweepEnabled=true), they are automatically mounted into the datastore view under /provider/keystore/0/ and /provider/keystore/1/. EXAMPLES Inspecting pubsub seqno validator state: $ ipfs diag datastore count /pubsub/seqno/ 2 $ ipfs diag datastore get --hex /pubsub/seqno/12D3KooW... Key: /pubsub/seqno/12D3KooW... Hex Dump: 00000000 18 81 81 c8 91 c0 ea f6 |........| Writing a test key (debugging only): $ ipfs diag datastore put /test/mykey "hello" Inspecting provider keystore (requires SweepEnabled): $ ipfs diag datastore count /provider/keystore/0/ $ ipfs diag datastore count /provider/keystore/1/ `, }, Subcommands: map[string]*cmds.Command{ "get": diagDatastoreGetCmd, "put": diagDatastorePutCmd, "count": diagDatastoreCountCmd, }, } const diagDatastoreHexOptionName = "hex" type diagDatastoreGetResult struct { Key string `json:"key"` Value []byte `json:"value"` HexDump string `json:"hex_dump,omitempty"` } // openDiagDatastore opens the repo datastore and conditionally mounts any // provider keystore datastores that exist on disk. It returns the composite // datastore and a cleanup function that must be called when done. func openDiagDatastore(env cmds.Environment) (datastore.Datastore, func(), error) { cctx := env.(*oldcmds.Context) repo, err := fsrepo.Open(cctx.ConfigRoot) if err != nil { return nil, nil, fmt.Errorf("failed to open repo: %w", err) } extraMounts, extraCloser, err := node.MountKeystoreDatastores(repo) if err != nil { repo.Close() return nil, nil, err } closer := func() { extraCloser() repo.Close() } if len(extraMounts) == 0 { return repo.Datastore(), closer, nil } mounts := []mount.Mount{{Prefix: datastore.NewKey("/"), Datastore: repo.Datastore()}} mounts = append(mounts, extraMounts...) return mount.New(mounts), closer, nil } var diagDatastoreGetCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Read a raw key from the datastore.", ShortDescription: ` Returns the value stored at the given datastore key. Default output is raw bytes. Use --hex for human-readable hex dump. The daemon must not be running when using this command. WARNING: FOR DEBUGGING/TESTING ONLY `, }, Arguments: []cmds.Argument{ cmds.StringArg("key", true, false, "Datastore key to read (e.g., /pubsub/seqno/)"), }, Options: []cmds.Option{ cmds.BoolOption(diagDatastoreHexOptionName, "Output hex dump instead of raw bytes"), }, NoRemote: true, PreRun: DaemonNotRunning, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ds, closer, err := openDiagDatastore(env) if err != nil { return err } defer closer() keyStr := req.Arguments[0] key := datastore.NewKey(keyStr) val, err := ds.Get(req.Context, key) if err != nil { if errors.Is(err, datastore.ErrNotFound) { return fmt.Errorf("key not found: %s", keyStr) } return fmt.Errorf("failed to read key: %w", err) } result := &diagDatastoreGetResult{ Key: keyStr, Value: val, } if hexDump, _ := req.Options[diagDatastoreHexOptionName].(bool); hexDump { result.HexDump = hex.Dump(val) } return cmds.EmitOnce(res, result) }, Type: diagDatastoreGetResult{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, result *diagDatastoreGetResult) error { if result.HexDump != "" { fmt.Fprintf(w, "Key: %s\nHex Dump:\n%s", result.Key, result.HexDump) return nil } // Raw bytes output _, err := w.Write(result.Value) return err }), }, } var diagDatastorePutCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Write a raw key-value pair to the datastore.", ShortDescription: ` Stores the given value at the specified datastore key. The daemon must not be running when using this command. WARNING: FOR DEBUGGING/TESTING ONLY `, }, Arguments: []cmds.Argument{ cmds.StringArg("key", true, false, "Datastore key (e.g., /test/mykey)"), cmds.StringArg("value", true, false, "Value to store (as a string)"), }, NoRemote: true, PreRun: DaemonNotRunning, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ds, closer, err := openDiagDatastore(env) if err != nil { return err } defer closer() key := datastore.NewKey(req.Arguments[0]) if err := ds.Put(req.Context, key, []byte(req.Arguments[1])); err != nil { return fmt.Errorf("failed to put key: %w", err) } if err := ds.Sync(req.Context, key); err != nil { return fmt.Errorf("failed to sync: %w", err) } return nil }, } type diagDatastoreCountResult struct { Prefix string `json:"prefix"` Count int64 `json:"count"` } var diagDatastoreCountCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Count entries matching a datastore prefix.", ShortDescription: ` Counts the number of datastore entries whose keys start with the given prefix. The daemon must not be running when using this command. WARNING: FOR DEBUGGING/TESTING ONLY `, }, Arguments: []cmds.Argument{ cmds.StringArg("prefix", true, false, "Datastore key prefix (e.g., /pubsub/seqno/)"), }, NoRemote: true, PreRun: DaemonNotRunning, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ds, closer, err := openDiagDatastore(env) if err != nil { return err } defer closer() prefix := req.Arguments[0] q := query.Query{ Prefix: prefix, KeysOnly: true, } results, err := ds.Query(req.Context, q) if err != nil { return fmt.Errorf("failed to query datastore: %w", err) } defer results.Close() var count int64 for result := range results.Next() { if result.Error != nil { return fmt.Errorf("query error: %w", result.Error) } count++ } return cmds.EmitOnce(res, &diagDatastoreCountResult{ Prefix: prefix, Count: count, }) }, Type: diagDatastoreCountResult{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, result *diagDatastoreCountResult) error { _, err := fmt.Fprintf(w, "%d\n", result.Count) return err }), }, } ================================================ FILE: core/commands/e/error.go ================================================ package e import ( "fmt" "runtime/debug" ) // TypeErr returns an error with a string that explains what error was expected and what was received. func TypeErr(expected, actual any) error { return fmt.Errorf("expected type %T, got %T", expected, actual) } // compile time type check that HandlerError is an error var _ error = New(nil) // HandlerError adds a stack trace to an error type HandlerError struct { Err error Stack []byte } // Error makes HandlerError implement error func (err HandlerError) Error() string { return fmt.Sprintf("%s in:\n%s", err.Err.Error(), err.Stack) } // New returns a new HandlerError func New(err error) HandlerError { return HandlerError{Err: err, Stack: debug.Stack()} } ================================================ FILE: core/commands/external.go ================================================ package commands import ( "bytes" "fmt" "io" "os" "os/exec" "strings" cmds "github.com/ipfs/go-ipfs-cmds" ) func ExternalBinary(instructions string) *cmds.Command { return &cmds.Command{ Arguments: []cmds.Argument{ cmds.StringArg("args", false, true, "Arguments for subcommand."), }, External: true, NoRemote: true, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { binname := strings.Join(append([]string{"ipfs"}, req.Path...), "-") _, err := exec.LookPath(binname) if err != nil { // special case for '--help' on uninstalled binaries. for _, arg := range req.Arguments { if arg == "--help" || arg == "-h" { buf := new(bytes.Buffer) fmt.Fprintf(buf, "%s is an 'external' command.\n", binname) fmt.Fprintf(buf, "It does not currently appear to be installed.\n") fmt.Fprintf(buf, "%s\n", instructions) return res.Emit(buf) } } return fmt.Errorf("%s not installed", binname) } r, w := io.Pipe() cmd := exec.Command(binname, req.Arguments...) // TODO: make commands lib be able to pass stdin through daemon // cmd.Stdin = req.Stdin() cmd.Stdin = io.LimitReader(nil, 0) cmd.Stdout = w cmd.Stderr = w // setup env of child program osenv := os.Environ() cmd.Env = osenv err = cmd.Start() if err != nil { return fmt.Errorf("failed to start subcommand: %s", err) } errC := make(chan error) go func() { var err error defer func() { errC <- err }() err = cmd.Wait() w.Close() }() err = res.Emit(r) if err != nil { return err } return <-errC }, } } ================================================ FILE: core/commands/extra.go ================================================ package commands import ( cmds "github.com/ipfs/go-ipfs-cmds" ) func CreateCmdExtras(opts ...func(e *cmds.Extra)) *cmds.Extra { e := new(cmds.Extra) for _, o := range opts { o(e) } return e } type doesNotUseRepo struct{} func SetDoesNotUseRepo(val bool) func(e *cmds.Extra) { return func(e *cmds.Extra) { e.SetValue(doesNotUseRepo{}, val) } } func GetDoesNotUseRepo(e *cmds.Extra) (val bool, found bool) { return getBoolFlag(e, doesNotUseRepo{}) } // doesNotUseConfigAsInput describes commands that do not use the config as // input. These commands either initialize the config or perform operations // that don't require access to the config. // // pre-command hooks that require configs must not be run before these // commands. type doesNotUseConfigAsInput struct{} func SetDoesNotUseConfigAsInput(val bool) func(e *cmds.Extra) { return func(e *cmds.Extra) { e.SetValue(doesNotUseConfigAsInput{}, val) } } func GetDoesNotUseConfigAsInput(e *cmds.Extra) (val bool, found bool) { return getBoolFlag(e, doesNotUseConfigAsInput{}) } // preemptsAutoUpdate describes commands that must be executed without the // auto-update pre-command hook type preemptsAutoUpdate struct{} func SetPreemptsAutoUpdate(val bool) func(e *cmds.Extra) { return func(e *cmds.Extra) { e.SetValue(preemptsAutoUpdate{}, val) } } func GetPreemptsAutoUpdate(e *cmds.Extra) (val bool, found bool) { return getBoolFlag(e, preemptsAutoUpdate{}) } func getBoolFlag(e *cmds.Extra, key any) (val bool, found bool) { var ival any ival, found = e.GetValue(key) if !found { return false, false } val = ival.(bool) return val, found } ================================================ FILE: core/commands/files.go ================================================ package commands import ( "context" "encoding/json" "errors" "fmt" "io" "os" gopath "path" "slices" "strconv" "strings" "sync" "sync/atomic" "time" humanize "github.com/dustin/go-humanize" oldcmds "github.com/ipfs/kubo/commands" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/node" fsrepo "github.com/ipfs/kubo/repo/fsrepo" bservice "github.com/ipfs/boxo/blockservice" bstore "github.com/ipfs/boxo/blockstore" offline "github.com/ipfs/boxo/exchange/offline" dag "github.com/ipfs/boxo/ipld/merkledag" ft "github.com/ipfs/boxo/ipld/unixfs" uio "github.com/ipfs/boxo/ipld/unixfs/io" mfs "github.com/ipfs/boxo/mfs" "github.com/ipfs/boxo/path" cid "github.com/ipfs/go-cid" cidenc "github.com/ipfs/go-cidutil/cidenc" "github.com/ipfs/go-datastore" cmds "github.com/ipfs/go-ipfs-cmds" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" iface "github.com/ipfs/kubo/core/coreiface" mh "github.com/multiformats/go-multihash" ) var flog = logging.Logger("cmds/files") // Global counter for unflushed MFS operations var noFlushOperationCounter atomic.Int64 // Cached limit value (read once on first use) var ( noFlushLimit int64 noFlushLimitInit sync.Once ) // updateNoFlushCounter manages the counter for unflushed operations func updateNoFlushCounter(nd *core.IpfsNode, flush bool) error { if flush { // Reset counter when flushing noFlushOperationCounter.Store(0) return nil } // Cache the limit on first use (config doesn't change at runtime) noFlushLimitInit.Do(func() { noFlushLimit = int64(config.DefaultMFSNoFlushLimit) if cfg, err := nd.Repo.Config(); err == nil && cfg.Internal.MFSNoFlushLimit != nil { noFlushLimit = cfg.Internal.MFSNoFlushLimit.WithDefault(int64(config.DefaultMFSNoFlushLimit)) } }) // Check if limit reached if noFlushLimit > 0 && noFlushOperationCounter.Load() >= noFlushLimit { return fmt.Errorf("reached limit of %d unflushed MFS operations. "+ "To resolve: 1) run 'ipfs files flush' to persist changes, "+ "2) use --flush=true (default), or "+ "3) increase Internal.MFSNoFlushLimit in config", noFlushLimit) } noFlushOperationCounter.Add(1) return nil } // FilesCmd is the 'ipfs files' command var FilesCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Interact with unixfs files.", ShortDescription: ` Files is an API for manipulating IPFS objects as if they were a Unix filesystem. The files facility interacts with MFS (Mutable File System). MFS acts as a single, dynamic filesystem mount. MFS has a root CID that is transparently updated when a change happens (and can be checked with "ipfs files stat /"). All files and folders within MFS are respected and will not be deleted during garbage collections. However, a DAG may be referenced in MFS without being fully available locally (MFS content is lazy loaded when accessed). MFS is independent from the list of pinned items ("ipfs pin ls"). Calls to "ipfs pin add" and "ipfs pin rm" will add and remove pins independently of MFS. If MFS content that was additionally pinned is removed by calling "ipfs files rm", it will still remain pinned. Content added with "ipfs add" (which by default also becomes pinned), is not added to MFS. Any content can be lazily referenced from MFS with the command "ipfs files cp /ipfs/ /some/path/" (see ipfs files cp --help). NOTE: Most of the subcommands of 'ipfs files' accept the '--flush' flag. It defaults to true and ensures two things: 1) that the changes are reflected in the full MFS structure (updated CIDs) 2) that the parent-folder's cache is cleared. Use caution when setting this flag to false. It will improve performance for large numbers of file operations, but it does so at the cost of consistency guarantees. If the daemon is unexpectedly killed before running 'ipfs files flush' on the files in question, then data may be lost. This also applies to run 'ipfs repo gc' concurrently with '--flush=false' operations. When using '--flush=false', operations are limited to prevent unbounded memory growth. After reaching Internal.MFSNoFlushLimit operations, further operations will fail until you run 'ipfs files flush'. This explicit failure (instead of auto-flushing) ensures you maintain control over when data is persisted, preventing unexpected partial states and making batch operations predictable. We recommend flushing paths regularly, especially folders with many write operations, to clear caches, free memory, and maintain good performance.`, }, Options: []cmds.Option{ cmds.BoolOption(filesFlushOptionName, "f", "Flush target and ancestors after write.").WithDefault(true), }, Subcommands: map[string]*cmds.Command{ "read": filesReadCmd, "write": filesWriteCmd, "mv": filesMvCmd, "cp": filesCpCmd, "ls": filesLsCmd, "mkdir": filesMkdirCmd, "stat": filesStatCmd, "rm": filesRmCmd, "flush": filesFlushCmd, "chcid": filesChcidCmd, "chmod": filesChmodCmd, "chroot": filesChrootCmd, "touch": filesTouchCmd, }, } const ( filesCidVersionOptionName = "cid-version" filesHashOptionName = "hash" ) var ( cidVersionOption = cmds.IntOption(filesCidVersionOptionName, "cid-ver", "Cid version to use. (experimental)") hashOption = cmds.StringOption(filesHashOptionName, "Hash function to use. Will set Cid version to 1 if used. (experimental)") ) var errFormat = errors.New("format was set by multiple options. Only one format option is allowed") type statOutput struct { Hash string Size uint64 CumulativeSize uint64 Blocks int Type string WithLocality bool `json:",omitempty"` Local bool `json:",omitempty"` SizeLocal uint64 `json:",omitempty"` Mode uint32 `json:",omitempty"` Mtime int64 `json:",omitempty"` MtimeNsecs int `json:",omitempty"` } func (s *statOutput) MarshalJSON() ([]byte, error) { type so statOutput out := &struct { *so Mode string `json:",omitempty"` }{so: (*so)(s)} if s.Mode != 0 { out.Mode = fmt.Sprintf("%04o", s.Mode) } return json.Marshal(out) } func (s *statOutput) UnmarshalJSON(data []byte) error { var err error type so statOutput tmp := &struct { *so Mode string `json:",omitempty"` }{so: (*so)(s)} if err := json.Unmarshal(data, &tmp); err != nil { return err } if tmp.Mode != "" { mode, err := strconv.ParseUint(tmp.Mode, 8, 32) if err == nil { s.Mode = uint32(mode) } } return err } const ( defaultStatFormat = ` Size: CumulativeSize: ChildBlocks: Type: Mode: () Mtime: ` filesFormatOptionName = "format" filesSizeOptionName = "size" filesWithLocalOptionName = "with-local" filesStatUnspecified = "not set" ) var filesStatCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Display file status.", }, Arguments: []cmds.Argument{ cmds.StringArg("path", true, false, "Path to node to stat."), }, Options: []cmds.Option{ cmds.StringOption(filesFormatOptionName, "Print statistics in given format. Allowed tokens: "+ " and optional ."+ "Conflicts with other format options.").WithDefault(defaultStatFormat), cmds.BoolOption(filesHashOptionName, "Print only hash. Implies '--format='. Conflicts with other format options."), cmds.BoolOption(filesSizeOptionName, "Print only size. Implies '--format='. Conflicts with other format options."), cmds.BoolOption(filesWithLocalOptionName, "Compute the amount of the dag that is local, and if possible the total size"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { _, err := statGetFormatOptions(req) if err != nil { return cmds.Errorf(cmds.ErrClient, "invalid parameters: %s", err) } node, err := cmdenv.GetNode(env) if err != nil { return err } api, err := cmdenv.GetApi(env, req) if err != nil { return err } path, err := checkPath(req.Arguments[0]) if err != nil { return err } withLocal, _ := req.Options[filesWithLocalOptionName].(bool) enc, err := cmdenv.GetCidEncoder(req) if err != nil { return err } var dagserv ipld.DAGService if withLocal { // an offline DAGService will not fetch from the network dagserv = dag.NewDAGService(bservice.New( node.Blockstore, offline.Exchange(node.Blockstore), )) } else { dagserv = node.DAG } nd, err := getNodeFromPath(req.Context, node, api, path) if err != nil { return err } o, err := statNode(nd, enc) if err != nil { return err } if !withLocal { return cmds.EmitOnce(res, o) } local, sizeLocal, err := walkBlock(req.Context, dagserv, nd) if err != nil { return err } o.WithLocality = true o.Local = local o.SizeLocal = sizeLocal return cmds.EmitOnce(res, o) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *statOutput) error { mode, modeo := filesStatUnspecified, filesStatUnspecified if out.Mode != 0 { mode = strings.ToLower(os.FileMode(out.Mode).String()) modeo = "0" + strconv.FormatInt(int64(out.Mode&0x1FF), 8) } mtime, mtimes, mtimens := filesStatUnspecified, filesStatUnspecified, filesStatUnspecified if out.Mtime > 0 { mtime = time.Unix(out.Mtime, int64(out.MtimeNsecs)).UTC().Format("2 Jan 2006, 15:04:05 MST") mtimes = strconv.FormatInt(out.Mtime, 10) mtimens = strconv.Itoa(out.MtimeNsecs) } s, _ := statGetFormatOptions(req) s = strings.Replace(s, "", out.Hash, -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.Size), -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.CumulativeSize), -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.Blocks), -1) s = strings.Replace(s, "", out.Type, -1) s = strings.Replace(s, "", mode, -1) s = strings.Replace(s, "", modeo, -1) s = strings.Replace(s, "", mtime, -1) s = strings.Replace(s, "", mtimes, -1) s = strings.Replace(s, "", mtimens, -1) fmt.Fprintln(w, s) if out.WithLocality { fmt.Fprintf(w, "Local: %s of %s (%.2f%%)\n", humanize.Bytes(out.SizeLocal), humanize.Bytes(out.CumulativeSize), 100.0*float64(out.SizeLocal)/float64(out.CumulativeSize), ) } return nil }), }, Type: statOutput{}, } func moreThanOne(a, b, c bool) bool { return a && b || b && c || a && c } func statGetFormatOptions(req *cmds.Request) (string, error) { hash, _ := req.Options[filesHashOptionName].(bool) size, _ := req.Options[filesSizeOptionName].(bool) format, _ := req.Options[filesFormatOptionName].(string) if moreThanOne(hash, size, format != defaultStatFormat) { return "", errFormat } if hash { return "", nil } else if size { return "", nil } else { return format, nil } } func statNode(nd ipld.Node, enc cidenc.Encoder) (*statOutput, error) { c := nd.Cid() cumulsize, err := nd.Size() if err != nil { return nil, err } switch n := nd.(type) { case *dag.ProtoNode: return statProtoNode(n, enc, c, cumulsize) case *dag.RawNode: return &statOutput{ Hash: enc.Encode(c), Blocks: 0, Size: cumulsize, CumulativeSize: cumulsize, Type: "file", }, nil default: return nil, errors.New("not unixfs node (proto or raw)") } } func statProtoNode(n *dag.ProtoNode, enc cidenc.Encoder, cid cid.Cid, cumulsize uint64) (*statOutput, error) { d, err := ft.FSNodeFromBytes(n.Data()) if err != nil { return nil, err } stat := statOutput{ Hash: enc.Encode(cid), Blocks: len(n.Links()), Size: d.FileSize(), CumulativeSize: cumulsize, } switch d.Type() { case ft.TDirectory, ft.THAMTShard: stat.Type = "directory" case ft.TFile, ft.TSymlink, ft.TMetadata, ft.TRaw: stat.Type = "file" default: return nil, fmt.Errorf("unrecognized node type: %s", d.Type()) } if mode := d.Mode(); mode != 0 { stat.Mode = uint32(mode) } else if d.Type() == ft.TSymlink { stat.Mode = uint32(os.ModeSymlink | 0x1FF) } if mt := d.ModTime(); !mt.IsZero() { stat.Mtime = mt.Unix() if ns := mt.Nanosecond(); ns > 0 { stat.MtimeNsecs = ns } } return &stat, nil } func walkBlock(ctx context.Context, dagserv ipld.DAGService, nd ipld.Node) (bool, uint64, error) { // Start with the block data size sizeLocal := uint64(len(nd.RawData())) local := true for _, link := range nd.Links() { child, err := dagserv.Get(ctx, link.Cid) if ipld.IsNotFound(err) { local = false continue } if err != nil { return local, sizeLocal, err } childLocal, childLocalSize, err := walkBlock(ctx, dagserv, child) if err != nil { return local, sizeLocal, err } // Recursively add the child size local = local && childLocal sizeLocal += childLocalSize } return local, sizeLocal, nil } var errFilesCpInvalidUnixFS = errors.New("cp: source must be a valid UnixFS (dag-pb or raw codec)") var filesCpCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Add references to IPFS files and directories in MFS (or copy within MFS).", ShortDescription: ` "ipfs files cp" can be used to add references to any IPFS file or directory (usually in the form /ipfs/, but also any resolvable path) into MFS. This performs a lazy copy: the full DAG will not be fetched, only the root node being copied. It can also be used to copy files within MFS, but in the case when an IPFS-path matches an existing MFS path, the IPFS path wins. In order to add content to MFS from disk, you can use "ipfs add" to obtain the IPFS Content Identifier and then "ipfs files cp" to copy it into MFS: $ ipfs add --quieter --pin=false # ... # ... outputs the root CID at the end $ ipfs files cp /ipfs/ /your/desired/mfs/path If you wish to fully copy content from a different IPFS peer into MFS, do not forget to force IPFS to fetch the full DAG after doing a "cp" operation. i.e: $ ipfs files cp /ipfs/ /your/desired/mfs/path $ ipfs pin add The lazy-copy feature can also be used to protect partial DAG contents from garbage collection. i.e. adding the Wikipedia root to MFS would not download all the Wikipedia, but will prevent any downloaded Wikipedia-DAG content from being GC'ed. `, }, Arguments: []cmds.Argument{ cmds.StringArg("source", true, false, "Source IPFS or MFS path to copy."), cmds.StringArg("dest", true, false, "Destination within MFS."), }, Options: []cmds.Option{ cmds.BoolOption(forceOptionName, "Force overwrite of existing files."), cmds.BoolOption(filesParentsOptionName, "p", "Make parent directories as needed."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } cfg, err := nd.Repo.Config() if err != nil { return err } prefix, err := getPrefixNew(req, &cfg.Import) if err != nil { return err } api, err := cmdenv.GetApi(env, req) if err != nil { return err } src, err := checkPath(req.Arguments[0]) if err != nil { return err } src = strings.TrimRight(src, "/") dst, err := checkPath(req.Arguments[1]) if err != nil { return err } if dst[len(dst)-1] == '/' { dst += gopath.Base(src) } node, err := getNodeFromPath(req.Context, nd, api, src) if err != nil { return fmt.Errorf("cp: cannot get node from path %s: %s", src, err) } // Sanity-check: ensure root CID is a valid UnixFS (dag-pb or raw block) // Context: https://github.com/ipfs/kubo/issues/10331 srcCidType := node.Cid().Type() switch srcCidType { case cid.Raw: if _, ok := node.(*dag.RawNode); !ok { return errFilesCpInvalidUnixFS } case cid.DagProtobuf: if _, ok := node.(*dag.ProtoNode); !ok { return errFilesCpInvalidUnixFS } if _, err = ft.FSNodeFromBytes(node.(*dag.ProtoNode).Data()); err != nil { return fmt.Errorf("%w: %v", errFilesCpInvalidUnixFS, err) } default: return errFilesCpInvalidUnixFS } mkParents, _ := req.Options[filesParentsOptionName].(bool) if mkParents { maxDirLinks := int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks)) sizeEstimationMode := cfg.Import.HAMTSizeEstimationMode() err := ensureContainingDirectoryExists(nd.FilesRoot, dst, prefix, maxDirLinks, &sizeEstimationMode) if err != nil { return err } } force, _ := req.Options[forceOptionName].(bool) if force { if err = unlinkNodeIfExists(nd, dst); err != nil { return fmt.Errorf("cp: cannot unlink existing file: %s", err) } } flush, _ := req.Options[filesFlushOptionName].(bool) if err := updateNoFlushCounter(nd, flush); err != nil { return err } err = mfs.PutNode(nd.FilesRoot, dst, node) if err != nil { return fmt.Errorf("cp: cannot put node in path %s: %s", dst, err) } if flush { if _, err := mfs.FlushPath(req.Context, nd.FilesRoot, dst); err != nil { return fmt.Errorf("cp: cannot flush the created file %s: %s", dst, err) } // Flush parent to clear directory cache and free memory. parent := gopath.Dir(dst) if _, err = mfs.FlushPath(req.Context, nd.FilesRoot, parent); err != nil { return fmt.Errorf("cp: cannot flush the created file's parent folder %s: %s", dst, err) } } return nil }, } func getNodeFromPath(ctx context.Context, node *core.IpfsNode, api iface.CoreAPI, p string) (ipld.Node, error) { switch { case strings.HasPrefix(p, "/ipfs/"): pth, err := path.NewPath(p) if err != nil { return nil, err } return api.ResolveNode(ctx, pth) default: fsn, err := mfs.Lookup(node.FilesRoot, p) if err != nil { return nil, err } return fsn.GetNode() } } func unlinkNodeIfExists(node *core.IpfsNode, path string) error { dir, name := gopath.Split(path) parent, err := mfs.Lookup(node.FilesRoot, dir) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil } return err } pdir, ok := parent.(*mfs.Directory) if !ok { return fmt.Errorf("not a directory: %s", dir) } // Attempt to unlink if child is a file, ignore error since // we are only concerned with unlinking an existing file. child, err := pdir.Child(name) if err != nil { return nil // no child file, nothing to unlink } if child.Type() != mfs.TFile { return fmt.Errorf("not a file: %s", path) } return pdir.Unlink(name) } type filesLsOutput struct { Entries []mfs.NodeListing } const ( longOptionName = "long" dontSortOptionName = "U" ) var filesLsCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List directories in the local mutable namespace.", ShortDescription: ` List directories in the local mutable namespace (works on both IPFS and MFS paths). Examples: $ ipfs files ls /welcome/docs/ about contact help quick-start readme security-notes $ ipfs files ls /myfiles/a/b/c/d foo bar `, }, Arguments: []cmds.Argument{ cmds.StringArg("path", false, false, "Path to show listing for. Defaults to '/'."), }, Options: []cmds.Option{ cmds.BoolOption(longOptionName, "l", "Use long listing format."), cmds.BoolOption(dontSortOptionName, "Do not sort; list entries in directory order."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { var arg string if len(req.Arguments) == 0 { arg = "/" } else { arg = req.Arguments[0] } path, err := checkPath(arg) if err != nil { return err } nd, err := cmdenv.GetNode(env) if err != nil { return err } fsn, err := mfs.Lookup(nd.FilesRoot, path) if err != nil { return err } long, _ := req.Options[longOptionName].(bool) enc, err := cmdenv.GetCidEncoder(req) if err != nil { return err } switch fsn := fsn.(type) { case *mfs.Directory: if !long { var output []mfs.NodeListing names, err := fsn.ListNames(req.Context) if err != nil { return err } for _, name := range names { output = append(output, mfs.NodeListing{ Name: name, }) } return cmds.EmitOnce(res, &filesLsOutput{output}) } listing, err := fsn.List(req.Context) if err != nil { return err } return cmds.EmitOnce(res, &filesLsOutput{listing}) case *mfs.File: _, name := gopath.Split(path) out := &filesLsOutput{[]mfs.NodeListing{{Name: name}}} if long { out.Entries[0].Type = int(fsn.Type()) size, err := fsn.Size() if err != nil { return err } out.Entries[0].Size = size nd, err := fsn.GetNode() if err != nil { return err } out.Entries[0].Hash = enc.Encode(nd.Cid()) } return cmds.EmitOnce(res, out) default: return errors.New("unrecognized type") } }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *filesLsOutput) error { noSort, _ := req.Options[dontSortOptionName].(bool) if !noSort { slices.SortFunc(out.Entries, func(a, b mfs.NodeListing) int { return strings.Compare(a.Name, b.Name) }) } long, _ := req.Options[longOptionName].(bool) for _, o := range out.Entries { if long { if o.Type == int(mfs.TDir) { o.Name += "/" } fmt.Fprintf(w, "%s\t%s\t%d\n", o.Name, o.Hash, o.Size) } else { fmt.Fprintf(w, "%s\n", o.Name) } } return nil }), }, Type: filesLsOutput{}, } const ( filesOffsetOptionName = "offset" filesCountOptionName = "count" ) var filesReadCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Read a file from MFS.", ShortDescription: ` Read a specified number of bytes from a file at a given offset. By default, it will read the entire file similar to the Unix cat. Examples: $ ipfs files read /test/hello hello `, }, Arguments: []cmds.Argument{ cmds.StringArg("path", true, false, "Path to file to be read."), }, Options: []cmds.Option{ cmds.Int64Option(filesOffsetOptionName, "o", "Byte offset to begin reading from."), cmds.Int64Option(filesCountOptionName, "n", "Maximum number of bytes to read."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } path, err := checkPath(req.Arguments[0]) if err != nil { return err } fsn, err := mfs.Lookup(nd.FilesRoot, path) if err != nil { return fmt.Errorf("%s: %w", path, err) } fi, ok := fsn.(*mfs.File) if !ok { return fmt.Errorf("%s was not a file", path) } rfd, err := fi.Open(mfs.Flags{Read: true}) if err != nil { return err } defer rfd.Close() offset, _ := req.Options[offsetOptionName].(int64) if offset < 0 { return fmt.Errorf("cannot specify negative offset") } filen, err := rfd.Size() if err != nil { return err } if int64(offset) > filen { return fmt.Errorf("offset was past end of file (%d > %d)", offset, filen) } _, err = rfd.Seek(int64(offset), io.SeekStart) if err != nil { return err } var r io.Reader = &contextReaderWrapper{R: rfd, ctx: req.Context} count, found := req.Options[filesCountOptionName].(int64) if found { if count < 0 { return fmt.Errorf("cannot specify negative 'count'") } r = io.LimitReader(r, int64(count)) } return res.Emit(r) }, } type contextReader interface { CtxReadFull(context.Context, []byte) (int, error) } type contextReaderWrapper struct { R contextReader ctx context.Context } func (crw *contextReaderWrapper) Read(b []byte) (int, error) { return crw.R.CtxReadFull(crw.ctx, b) } var filesMvCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Move files.", ShortDescription: ` Move files around. Just like the traditional Unix mv. Example: $ ipfs files mv /myfs/a/b/c /myfs/foo/newc `, }, Arguments: []cmds.Argument{ cmds.StringArg("source", true, false, "Source file to move."), cmds.StringArg("dest", true, false, "Destination path for file to be moved to."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } flush, _ := req.Options[filesFlushOptionName].(bool) if err := updateNoFlushCounter(nd, flush); err != nil { return err } src, err := checkPath(req.Arguments[0]) if err != nil { return err } dst, err := checkPath(req.Arguments[1]) if err != nil { return err } err = mfs.Mv(nd.FilesRoot, src, dst) if err != nil { return err } if flush { parentSrc := gopath.Dir(src) parentDst := gopath.Dir(dst) // Flush parent to clear directory cache and free memory. if _, err = mfs.FlushPath(req.Context, nd.FilesRoot, parentDst); err != nil { return fmt.Errorf("cp: cannot flush the destination file's parent folder %s: %s", dst, err) } // Avoid re-flushing when moving within the same folder. if parentSrc != parentDst { if _, err = mfs.FlushPath(req.Context, nd.FilesRoot, parentSrc); err != nil { return fmt.Errorf("cp: cannot flush the source's file's parent folder %s: %s", dst, err) } } if _, err = mfs.FlushPath(req.Context, nd.FilesRoot, "/"); err != nil { return err } } return nil }, } const ( filesCreateOptionName = "create" filesParentsOptionName = "parents" filesTruncateOptionName = "truncate" filesRawLeavesOptionName = "raw-leaves" filesFlushOptionName = "flush" ) var filesWriteCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Append to (modify) a file in MFS.", ShortDescription: ` A low-level MFS command that allows you to append data to a file. If you want to add a file without modifying an existing one, use 'ipfs add --to-files' instead. `, LongDescription: ` A low-level MFS command that allows you to append data at the end of a file, or specify a beginning offset within a file to write to. The entire length of the input will be written. If the '--create' option is specified, the file will be created if it does not exist. Nonexistent intermediate directories will not be created unless the '--parents' option is specified. Newly created files will have the same CID version and hash function of the parent directory unless the '--cid-version' and '--hash' options are used. Newly created leaves will be in the legacy format (Protobuf) if the CID version is 0, or raw if the CID version is non-zero. Use of the '--raw-leaves' option will override this behavior. If the '--flush' option is set to false, changes will not be propagated to the merkledag root. This can make operations much faster when doing a large number of writes to a deeper directory structure. EXAMPLE: echo "hello world" | ipfs files write --create --parents /myfs/a/b/file echo "hello world" | ipfs files write --truncate /myfs/a/b/file WARNING: Usage of the '--flush=false' option does not guarantee data durability until the tree has been flushed. This can be accomplished by running 'ipfs files stat' on the file or any of its ancestors. WARNING: The CID produced by 'files write' will be different from 'ipfs add' because 'ipfs files write' creates a trickle-dag optimized for append-only operations. See '--trickle' in 'ipfs add --help' for more information. NOTE: The 'Import.UnixFSFileMaxLinks' config option does not apply to this command. Trickle DAG has a fixed internal structure optimized for append operations. To use configurable max-links, use 'ipfs add' with balanced DAG layout. If you want to add a file without modifying an existing one, use 'ipfs add' with '--to-files': > ipfs files mkdir -p /myfs/dir > ipfs add example.jpg --to-files /myfs/dir/ > ipfs files ls /myfs/dir/ example.jpg See '--to-files' in 'ipfs add --help' for more information. `, }, Arguments: []cmds.Argument{ cmds.StringArg("path", true, false, "Path to write to."), cmds.FileArg("data", true, false, "Data to write.").EnableStdin(), }, Options: []cmds.Option{ cmds.Int64Option(filesOffsetOptionName, "o", "Byte offset to begin writing at."), cmds.BoolOption(filesCreateOptionName, "e", "Create the file if it does not exist."), cmds.BoolOption(filesParentsOptionName, "p", "Make parent directories as needed."), cmds.BoolOption(filesTruncateOptionName, "t", "Truncate the file to size zero before writing."), cmds.Int64Option(filesCountOptionName, "n", "Maximum number of bytes to read."), cmds.BoolOption(filesRawLeavesOptionName, "Use raw blocks for newly created leaf nodes. (experimental)"), cidVersionOption, hashOption, }, Run: func(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment) (retErr error) { path, err := checkPath(req.Arguments[0]) if err != nil { return err } nd, err := cmdenv.GetNode(env) if err != nil { return err } cfg, err := nd.Repo.Config() if err != nil { return err } create, _ := req.Options[filesCreateOptionName].(bool) mkParents, _ := req.Options[filesParentsOptionName].(bool) trunc, _ := req.Options[filesTruncateOptionName].(bool) flush, _ := req.Options[filesFlushOptionName].(bool) rawLeaves, rawLeavesDef := req.Options[filesRawLeavesOptionName].(bool) if err := updateNoFlushCounter(nd, flush); err != nil { return err } if !rawLeavesDef && cfg.Import.UnixFSRawLeaves != config.Default { rawLeavesDef = true rawLeaves = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves) } prefix, err := getPrefixNew(req, &cfg.Import) if err != nil { return err } offset, _ := req.Options[filesOffsetOptionName].(int64) if offset < 0 { return fmt.Errorf("cannot have negative write offset") } if mkParents { maxDirLinks := int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks)) sizeEstimationMode := cfg.Import.HAMTSizeEstimationMode() err := ensureContainingDirectoryExists(nd.FilesRoot, path, prefix, maxDirLinks, &sizeEstimationMode) if err != nil { return err } } fi, err := getFileHandle(nd.FilesRoot, path, create, prefix) if err != nil { return err } if rawLeavesDef { fi.RawLeaves = rawLeaves } wfd, err := fi.Open(mfs.Flags{Write: true, Sync: flush}) if err != nil { return err } defer func() { err := wfd.Close() if err != nil { if retErr == nil { retErr = err } else { flog.Error("files: error closing file mfs file descriptor", err) } } if flush { // Flush parent to clear directory cache and free memory. parent := gopath.Dir(path) if _, err := mfs.FlushPath(req.Context, nd.FilesRoot, parent); err != nil { if retErr == nil { retErr = err } else { flog.Error("files: flushing the parent folder", err) } } } }() if trunc { if err := wfd.Truncate(0); err != nil { return err } } count, countfound := req.Options[filesCountOptionName].(int64) if countfound && count < 0 { return fmt.Errorf("cannot have negative byte count") } _, err = wfd.Seek(int64(offset), io.SeekStart) if err != nil { flog.Error("seekfail: ", err) return err } var r io.Reader r, err = cmdenv.GetFileArg(req.Files.Entries()) if err != nil { return err } if countfound { r = io.LimitReader(r, int64(count)) } _, err = io.Copy(wfd, r) return err }, } var filesMkdirCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Make directories.", ShortDescription: ` Create the directory if it does not already exist. The directory will have the same CID version and hash function of the parent directory unless the --cid-version and --hash options are used. NOTE: All paths must be absolute. Examples: $ ipfs files mkdir /test/newdir $ ipfs files mkdir -p /test/does/not/exist/yet `, }, Arguments: []cmds.Argument{ cmds.StringArg("path", true, false, "Path to dir to make."), }, Options: []cmds.Option{ cmds.BoolOption(filesParentsOptionName, "p", "No error if existing, make parent directories as needed."), cidVersionOption, hashOption, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := cmdenv.GetNode(env) if err != nil { return err } cfg, err := n.Repo.Config() if err != nil { return err } dashp, _ := req.Options[filesParentsOptionName].(bool) dirtomake, err := checkPath(req.Arguments[0]) if err != nil { return err } flush, _ := req.Options[filesFlushOptionName].(bool) if err := updateNoFlushCounter(n, flush); err != nil { return err } prefix, err := getPrefix(req, &cfg.Import) if err != nil { return err } root := n.FilesRoot maxDirLinks := int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks)) sizeEstimationMode := cfg.Import.HAMTSizeEstimationMode() err = mfs.Mkdir(root, dirtomake, mfs.MkdirOpts{ Mkparents: dashp, Flush: flush, CidBuilder: prefix, MaxLinks: maxDirLinks, SizeEstimationMode: &sizeEstimationMode, }) return err }, } type flushRes struct { Cid string } var filesFlushCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Flush a given path's data to disk.", ShortDescription: ` Flush a given path to the disk. This is only useful when other commands are run with the '--flush=false'. `, }, Arguments: []cmds.Argument{ cmds.StringArg("path", false, false, "Path to flush. Default: '/'."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } enc, err := cmdenv.GetCidEncoder(req) if err != nil { return err } path := "/" if len(req.Arguments) > 0 { path = req.Arguments[0] } n, err := mfs.FlushPath(req.Context, nd.FilesRoot, path) if err != nil { return err } // Reset the counter (flush always resets) noFlushOperationCounter.Store(0) return cmds.EmitOnce(res, &flushRes{enc.Encode(n.Cid())}) }, Type: flushRes{}, } var filesChcidCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Change the CID version or hash function of the root node of a given path.", ShortDescription: ` Change the CID version or hash function of the root node of a given path. `, }, Arguments: []cmds.Argument{ cmds.StringArg("path", false, false, "Path to change. Default: '/'."), }, Options: []cmds.Option{ cidVersionOption, hashOption, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } path := "/" if len(req.Arguments) > 0 { path = req.Arguments[0] } flush, _ := req.Options[filesFlushOptionName].(bool) // Note: files chcid is for explicitly changing CID format, so we don't // fall back to Import config here. If no options are provided, it does nothing. prefix, err := getPrefix(req, nil) if err != nil { return err } if err := updatePath(nd.FilesRoot, path, prefix); err != nil { return err } if flush { if _, err = mfs.FlushPath(req.Context, nd.FilesRoot, path); err != nil { return err } // Flush parent to clear directory cache and free memory. parent := gopath.Dir(path) if _, err = mfs.FlushPath(req.Context, nd.FilesRoot, parent); err != nil { return err } } return nil }, } func updatePath(rt *mfs.Root, pth string, builder cid.Builder) error { if builder == nil { return nil } nd, err := mfs.Lookup(rt, pth) if err != nil { return err } switch n := nd.(type) { case *mfs.Directory: n.SetCidBuilder(builder) default: return fmt.Errorf("can only update directories") } return nil } var filesRmCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Remove a file from MFS.", ShortDescription: ` Remove files or directories. $ ipfs files rm /foo $ ipfs files ls /bar cat dog fish $ ipfs files rm -r /bar `, }, Arguments: []cmds.Argument{ cmds.StringArg("path", true, true, "File to remove."), }, Options: []cmds.Option{ cmds.BoolOption(recursiveOptionName, "r", "Recursively remove directories."), cmds.BoolOption(forceOptionName, "Forcibly remove target at path; implies -r for directories"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { // Check if user explicitly set --flush=false if flushOpt, ok := req.Options[filesFlushOptionName]; ok { if flush, ok := flushOpt.(bool); ok && !flush { return fmt.Errorf("files rm always flushes for safety. The --flush flag cannot be set to false for this command") } } nd, err := cmdenv.GetNode(env) if err != nil { return err } // if '--force' specified, it will remove anything else, // including file, directory, corrupted node, etc force, _ := req.Options[forceOptionName].(bool) dashr, _ := req.Options[recursiveOptionName].(bool) var errs []error for _, arg := range req.Arguments { path, err := checkPath(arg) if err != nil { errs = append(errs, fmt.Errorf("%s is not a valid path: %w", arg, err)) continue } if err := removePath(nd.FilesRoot, path, force, dashr); err != nil { errs = append(errs, fmt.Errorf("%s: %w", path, err)) } } if len(errs) > 0 { for _, err = range errs { e := res.Emit(err.Error()) if e != nil { return e } } return fmt.Errorf("can't remove some files") } return nil }, } func removePath(filesRoot *mfs.Root, path string, force bool, dashr bool) error { if path == "/" { return fmt.Errorf("cannot delete root") } // 'rm a/b/c/' will fail unless we trim the slash at the end if path[len(path)-1] == '/' { path = path[:len(path)-1] } dir, name := gopath.Split(path) pdir, err := getParentDir(filesRoot, dir) if err != nil { if force && err == os.ErrNotExist { return nil } return err } if force { err := pdir.Unlink(name) if err != nil { if err == os.ErrNotExist { return nil } return err } return pdir.Flush() } // get child node by name, when the node is corrupted and nonexistent, // it will return specific error. child, err := pdir.Child(name) if err != nil { return err } switch child.(type) { case *mfs.Directory: if !dashr { return fmt.Errorf("path is a directory, use -r to remove directories") } } err = pdir.Unlink(name) if err != nil { return err } return pdir.Flush() } func getPrefixNew(req *cmds.Request, importCfg *config.Import) (cid.Builder, error) { cidVer, cidVerSet := req.Options[filesCidVersionOptionName].(int) hashFunStr, hashFunSet := req.Options[filesHashOptionName].(string) // Fall back to Import config if CLI options not set if !cidVerSet && importCfg != nil && !importCfg.CidVersion.IsDefault() { cidVer = int(importCfg.CidVersion.WithDefault(config.DefaultCidVersion)) cidVerSet = true } if !hashFunSet && importCfg != nil && !importCfg.HashFunction.IsDefault() { hashFunStr = importCfg.HashFunction.WithDefault(config.DefaultHashFunction) hashFunSet = true } if !cidVerSet && !hashFunSet { return nil, nil } if hashFunSet && cidVer == 0 { cidVer = 1 } prefix, err := dag.PrefixForCidVersion(cidVer) if err != nil { return nil, err } if hashFunSet { hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)] if !ok { return nil, fmt.Errorf("unrecognized hash function: %s", strings.ToLower(hashFunStr)) } prefix.MhType = hashFunCode prefix.MhLength = -1 } return &prefix, nil } func getPrefix(req *cmds.Request, importCfg *config.Import) (cid.Builder, error) { cidVer, cidVerSet := req.Options[filesCidVersionOptionName].(int) hashFunStr, hashFunSet := req.Options[filesHashOptionName].(string) // Fall back to Import config if CLI options not set if !cidVerSet && importCfg != nil && !importCfg.CidVersion.IsDefault() { cidVer = int(importCfg.CidVersion.WithDefault(config.DefaultCidVersion)) cidVerSet = true } if !hashFunSet && importCfg != nil && !importCfg.HashFunction.IsDefault() { hashFunStr = importCfg.HashFunction.WithDefault(config.DefaultHashFunction) hashFunSet = true } if !cidVerSet && !hashFunSet { return nil, nil } if hashFunSet && cidVer == 0 { cidVer = 1 } prefix, err := dag.PrefixForCidVersion(cidVer) if err != nil { return nil, err } if hashFunSet { hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)] if !ok { return nil, fmt.Errorf("unrecognized hash function: %s", strings.ToLower(hashFunStr)) } prefix.MhType = hashFunCode prefix.MhLength = -1 } return &prefix, nil } func ensureContainingDirectoryExists(r *mfs.Root, path string, builder cid.Builder, maxLinks int, sizeEstimationMode *uio.SizeEstimationMode) error { dirtomake := gopath.Dir(path) if dirtomake == "/" { return nil } return mfs.Mkdir(r, dirtomake, mfs.MkdirOpts{ Mkparents: true, CidBuilder: builder, MaxLinks: maxLinks, SizeEstimationMode: sizeEstimationMode, }) } func getFileHandle(r *mfs.Root, path string, create bool, builder cid.Builder) (*mfs.File, error) { target, err := mfs.Lookup(r, path) switch err { case nil: fi, ok := target.(*mfs.File) if !ok { return nil, fmt.Errorf("%s was not a file", path) } return fi, nil case os.ErrNotExist: if !create { return nil, err } // if create is specified and the file doesn't exist, we create the file dirname, fname := gopath.Split(path) pdir, err := getParentDir(r, dirname) if err != nil { return nil, err } if builder == nil { builder = pdir.GetCidBuilder() } nd := dag.NodeWithData(ft.FilePBData(nil, 0)) err = nd.SetCidBuilder(builder) if err != nil { return nil, err } err = pdir.AddChild(fname, nd) if err != nil { return nil, err } fsn, err := pdir.Child(fname) if err != nil { return nil, err } fi, ok := fsn.(*mfs.File) if !ok { return nil, errors.New("expected *mfs.File, didn't get it. This is likely a race condition") } return fi, nil default: return nil, err } } func checkPath(p string) (string, error) { if len(p) == 0 { return "", fmt.Errorf("paths must not be empty") } if p[0] != '/' { return "", fmt.Errorf("paths must start with a leading slash") } cleaned := gopath.Clean(p) if p[len(p)-1] == '/' && p != "/" { cleaned += "/" } return cleaned, nil } func getParentDir(root *mfs.Root, dir string) (*mfs.Directory, error) { parent, err := mfs.Lookup(root, dir) if err != nil { return nil, err } pdir, ok := parent.(*mfs.Directory) if !ok { return nil, errors.New("expected *mfs.Directory, didn't get it. This is likely a race condition") } return pdir, nil } var filesChmodCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Change optional POSIX mode permissions", ShortDescription: ` The mode argument must be specified in Unix numeric notation. $ ipfs files chmod 0644 /foo $ ipfs files stat /foo ... Type: file Mode: -rw-r--r-- (0644) ... `, }, Arguments: []cmds.Argument{ cmds.StringArg("mode", true, false, "Mode to apply to node (numeric notation)"), cmds.StringArg("path", true, false, "Path to apply mode"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } path, err := checkPath(req.Arguments[1]) if err != nil { return err } mode, err := strconv.ParseInt(req.Arguments[0], 8, 32) if err != nil { return err } return mfs.Chmod(nd.FilesRoot, path, os.FileMode(mode)) }, } var filesTouchCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Set or change optional POSIX modification times.", ShortDescription: ` Examples: # set modification time to now. $ ipfs files touch /foo # set a custom modification time. $ ipfs files touch --mtime=1630937926 /foo `, }, Arguments: []cmds.Argument{ cmds.StringArg("path", true, false, "Path of target to update."), }, Options: []cmds.Option{ cmds.Int64Option(mtimeOptionName, "Modification time in seconds before or since the Unix Epoch to apply to created UnixFS entries."), cmds.UintOption(mtimeNsecsOptionName, "Modification time fraction in nanoseconds"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } path, err := checkPath(req.Arguments[0]) if err != nil { return err } mtime, _ := req.Options[mtimeOptionName].(int64) nsecs, _ := req.Options[mtimeNsecsOptionName].(uint) var ts time.Time if mtime != 0 { ts = time.Unix(mtime, int64(nsecs)).UTC() } else { ts = time.Now().UTC() } return mfs.Touch(nd.FilesRoot, path, ts) }, } const chrootConfirmOptionName = "confirm" var filesChrootCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Change the MFS root CID.", ShortDescription: ` 'ipfs files chroot' changes the root CID used by MFS (Mutable File System). This is a recovery command for when MFS becomes corrupted and prevents the daemon from starting. When run without a CID argument, resets MFS to an empty directory. WARNING: The old MFS root and its unpinned children will be removed during the next garbage collection. Pin the old root first if you want to preserve. This command can only run when the daemon is not running. Examples: # Reset MFS to empty directory (recovery from corruption) $ ipfs files chroot --confirm # Restore MFS to a known good directory CID $ ipfs files chroot --confirm QmYourBackupCID `, }, Arguments: []cmds.Argument{ cmds.StringArg("cid", false, false, "New root CID (defaults to empty directory if not specified)."), }, Options: []cmds.Option{ cmds.BoolOption(chrootConfirmOptionName, "Confirm this potentially destructive operation."), }, NoRemote: true, Extra: CreateCmdExtras(SetDoesNotUseRepo(true)), Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { confirm, _ := req.Options[chrootConfirmOptionName].(bool) if !confirm { return errors.New("this is a potentially destructive operation; pass --confirm to proceed") } // Determine new root CID var newRootCid cid.Cid if len(req.Arguments) > 0 { var err error newRootCid, err = cid.Decode(req.Arguments[0]) if err != nil { return fmt.Errorf("invalid CID %q: %w", req.Arguments[0], err) } } else { // Default to empty directory newRootCid = ft.EmptyDirNode().Cid() } // Get config root to open repo directly cctx := env.(*oldcmds.Context) cfgRoot := cctx.ConfigRoot // Open repo directly (daemon must not be running) repo, err := fsrepo.Open(cfgRoot) if err != nil { return fmt.Errorf("opening repo (is the daemon running?): %w", err) } defer repo.Close() localDS := repo.Datastore() bs := bstore.NewBlockstore(localDS) // Check new root exists locally and is a directory hasBlock, err := bs.Has(req.Context, newRootCid) if err != nil { return fmt.Errorf("checking if new root exists: %w", err) } if !hasBlock { // Special case: empty dir is always available (hardcoded in boxo) emptyDirCid := ft.EmptyDirNode().Cid() if !newRootCid.Equals(emptyDirCid) { return fmt.Errorf("new root %s does not exist locally; fetch it first with 'ipfs block get'", newRootCid) } } // Validate it's a directory (not a file) if hasBlock { blk, err := bs.Get(req.Context, newRootCid) if err != nil { return fmt.Errorf("reading new root block: %w", err) } pbNode, err := dag.DecodeProtobuf(blk.RawData()) if err != nil { return fmt.Errorf("new root is not a valid dag-pb node: %w", err) } fsNode, err := ft.FSNodeFromBytes(pbNode.Data()) if err != nil { return fmt.Errorf("new root is not a valid UnixFS node: %w", err) } if fsNode.Type() != ft.TDirectory && fsNode.Type() != ft.THAMTShard { return fmt.Errorf("new root must be a directory, got %s", fsNode.Type()) } } // Get old root for display (if exists) var oldRootStr string oldRootBytes, err := localDS.Get(req.Context, node.FilesRootDatastoreKey) if err == nil { oldRootCid, err := cid.Cast(oldRootBytes) if err == nil { oldRootStr = oldRootCid.String() } } else if !errors.Is(err, datastore.ErrNotFound) { return fmt.Errorf("reading current MFS root: %w", err) } // Write new root err = localDS.Put(req.Context, node.FilesRootDatastoreKey, newRootCid.Bytes()) if err != nil { return fmt.Errorf("writing new MFS root: %w", err) } // Build output message var msg string if oldRootStr != "" { msg = fmt.Sprintf("MFS root changed from %s to %s\n", oldRootStr, newRootCid) msg += fmt.Sprintf("The old root %s will be garbage collected unless pinned.\n", oldRootStr) } else { msg = fmt.Sprintf("MFS root set to %s\n", newRootCid) } return cmds.EmitOnce(res, &MessageOutput{Message: msg}) }, Type: MessageOutput{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *MessageOutput) error { _, err := fmt.Fprint(w, out.Message) return err }), }, } ================================================ FILE: core/commands/files_test.go ================================================ package commands import ( "io" "testing" dag "github.com/ipfs/boxo/ipld/merkledag" cmds "github.com/ipfs/go-ipfs-cmds" coremock "github.com/ipfs/kubo/core/mock" "github.com/stretchr/testify/require" ) func TestFilesCp_DagCborNodeFails(t *testing.T) { ctx := t.Context() cmdCtx, err := coremock.MockCmdsCtx() require.NoError(t, err) node, err := cmdCtx.ConstructNode() require.NoError(t, err) invalidData := []byte{0x00} protoNode := dag.NodeWithData(invalidData) err = node.DAG.Add(ctx, protoNode) require.NoError(t, err) req := &cmds.Request{ Context: ctx, Arguments: []string{ "/ipfs/" + protoNode.Cid().String(), "/test-destination", }, Options: map[string]any{ "force": false, }, } _, pw := io.Pipe() res, err := cmds.NewWriterResponseEmitter(pw, req) require.NoError(t, err) err = filesCpCmd.Run(req, res, &cmdCtx) require.Error(t, err) require.ErrorContains(t, err, "cp: source must be a valid UnixFS (dag-pb or raw codec)") } ================================================ FILE: core/commands/filestore.go ================================================ package commands import ( "context" "fmt" "io" "os" filestore "github.com/ipfs/boxo/filestore" cmds "github.com/ipfs/go-ipfs-cmds" core "github.com/ipfs/kubo/core" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" e "github.com/ipfs/kubo/core/commands/e" "github.com/ipfs/go-cid" ) var FileStoreCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Interact with filestore objects.", }, Subcommands: map[string]*cmds.Command{ "ls": lsFileStore, "verify": verifyFileStore, "dups": dupsFileStore, }, } const ( fileOrderOptionName = "file-order" removeBadBlocksOptionName = "remove-bad-blocks" ) var lsFileStore = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List objects in filestore.", LongDescription: ` List objects in the filestore. If one or more is specified only list those specific objects, otherwise list all objects. The output is: `, }, Arguments: []cmds.Argument{ cmds.StringArg("obj", false, true, "Cid of objects to list."), }, Options: []cmds.Option{ cmds.BoolOption(fileOrderOptionName, "sort the results based on the path of the backing file"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { _, fs, err := getFilestore(env) if err != nil { return err } args := req.Arguments if len(args) > 0 { return listByArgs(req.Context, res, fs, args, false) } fileOrder, _ := req.Options[fileOrderOptionName].(bool) next, err := filestore.ListAll(req.Context, fs, fileOrder) if err != nil { return err } for { r := next(req.Context) if r == nil { break } if err := res.Emit(r); err != nil { return err } } return nil }, PostRun: cmds.PostRunMap{ cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { enc, err := cmdenv.GetCidEncoder(res.Request()) if err != nil { return err } return streamResult(func(v any, out io.Writer) nonFatalError { r := v.(*filestore.ListRes) if r.ErrorMsg != "" { return nonFatalError(r.ErrorMsg) } fmt.Fprintf(out, "%s\n", r.FormatLong(enc.Encode)) return "" })(res, re) }, }, Type: filestore.ListRes{}, } var verifyFileStore = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Verify objects in filestore.", LongDescription: ` Verify objects in the filestore. If one or more is specified only verify those specific objects, otherwise verify all objects. The output is: [] Where is one of: ok: the block can be reconstructed changed: the contents of the backing file have changed no-file: the backing file could not be found error: there was some other problem reading the file missing: could not be found in the filestore ERROR: internal error, most likely due to a corrupt database Where is present only when removing bad blocks and is one of: remove: link to the block will be removed from datastore keep: keep link, nothing to do For ERROR entries the error will also be printed to stderr. `, }, Arguments: []cmds.Argument{ cmds.StringArg("obj", false, true, "Cid of objects to verify."), }, Options: []cmds.Option{ cmds.BoolOption(fileOrderOptionName, "verify the objects based on the order of the backing file"), cmds.BoolOption(removeBadBlocksOptionName, "remove bad blocks. WARNING: This may remove pinned data. You should run 'ipfs pin verify' after running this command and correct any issues."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { _, fs, err := getFilestore(env) if err != nil { return err } removeBadBlocks, _ := req.Options[removeBadBlocksOptionName].(bool) args := req.Arguments if len(args) > 0 { return listByArgs(req.Context, res, fs, args, removeBadBlocks) } fileOrder, _ := req.Options[fileOrderOptionName].(bool) next, err := filestore.VerifyAll(req.Context, fs, fileOrder) if err != nil { return err } for { r := next(req.Context) if r == nil { break } if removeBadBlocks && (r.Status != filestore.StatusOk) && (r.Status != filestore.StatusOtherError) { if err = fs.FileManager().DeleteBlock(req.Context, r.Key); err != nil { return err } } if err = res.Emit(r); err != nil { return err } } return nil }, PostRun: cmds.PostRunMap{ cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { enc, err := cmdenv.GetCidEncoder(res.Request()) if err != nil { return err } req := res.Request() removeBadBlocks, _ := req.Options[removeBadBlocksOptionName].(bool) for { v, err := res.Next() if err != nil { if err == io.EOF { return nil } return err } list, ok := v.(*filestore.ListRes) if !ok { return e.TypeErr(list, v) } if list.Status == filestore.StatusOtherError { fmt.Fprintf(os.Stderr, "%s\n", list.ErrorMsg) } if removeBadBlocks { action := "keep" if removeBadBlocks && (list.Status != filestore.StatusOk) && (list.Status != filestore.StatusOtherError) { action = "remove" } fmt.Fprintf(os.Stdout, "%s %s %s\n", list.Status.Format(), list.FormatLong(enc.Encode), action) } else { fmt.Fprintf(os.Stdout, "%s %s\n", list.Status.Format(), list.FormatLong(enc.Encode)) } } }, }, Type: filestore.ListRes{}, } var dupsFileStore = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List blocks that are both in the filestore and standard block storage.", }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { _, fs, err := getFilestore(env) if err != nil { return err } enc, err := cmdenv.GetCidEncoder(req) if err != nil { return err } ch, err := fs.FileManager().AllKeysChan(req.Context) if err != nil { return err } for cid := range ch { have, err := fs.MainBlockstore().Has(req.Context, cid) if err != nil { return res.Emit(&RefWrapper{Err: err.Error()}) } if have { if err := res.Emit(&RefWrapper{Ref: enc.Encode(cid)}); err != nil { return err } } } return nil }, Encoders: refsEncoderMap, Type: RefWrapper{}, } func getFilestore(env cmds.Environment) (*core.IpfsNode, *filestore.Filestore, error) { n, err := cmdenv.GetNode(env) if err != nil { return nil, nil, err } fs := n.Filestore if fs == nil { return n, nil, filestore.ErrFilestoreNotEnabled } return n, fs, err } func listByArgs(ctx context.Context, res cmds.ResponseEmitter, fs *filestore.Filestore, args []string, removeBadBlocks bool) error { for _, arg := range args { c, err := cid.Decode(arg) if err != nil { ret := &filestore.ListRes{ Status: filestore.StatusOtherError, ErrorMsg: fmt.Sprintf("%s: %v", arg, err), } if err := res.Emit(ret); err != nil { return err } continue } r := filestore.Verify(ctx, fs, c) if removeBadBlocks && (r.Status != filestore.StatusOk) && (r.Status != filestore.StatusOtherError) { if err = fs.FileManager().DeleteBlock(ctx, r.Key); err != nil { return err } } if err = res.Emit(r); err != nil { return err } } return nil } ================================================ FILE: core/commands/get.go ================================================ package commands import ( gotar "archive/tar" "bufio" "compress/gzip" "errors" "fmt" "io" "os" gopath "path" "path/filepath" "strings" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" "github.com/ipfs/kubo/core/commands/e" "github.com/cheggaaa/pb" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/tar" cmds "github.com/ipfs/go-ipfs-cmds" ) var ErrInvalidCompressionLevel = errors.New("compression level must be between 1 and 9") const ( outputOptionName = "output" archiveOptionName = "archive" compressOptionName = "compress" compressionLevelOptionName = "compression-level" ) var GetCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Download IPFS objects.", ShortDescription: ` Stores to disk the data contained an IPFS or IPNS object(s) at the given path. By default, the output will be stored at './', but an alternate path can be specified with '--output=' or '-o='. To output a TAR archive instead of unpacked files, use '--archive' or '-a'. To compress the output with GZIP compression, use '--compress' or '-C'. You may also specify the level of compression by specifying '-l=<1-9>'. `, HTTP: &cmds.HTTPHelpText{ ResponseContentType: "application/x-tar, or application/gzip when compress=true", }, }, Arguments: []cmds.Argument{ cmds.StringArg("ipfs-path", true, false, "The path to the IPFS object(s) to be outputted.").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption(outputOptionName, "o", "The path where the output should be stored."), cmds.BoolOption(archiveOptionName, "a", "Output a TAR archive."), cmds.BoolOption(compressOptionName, "C", "Compress the output with GZIP compression."), cmds.IntOption(compressionLevelOptionName, "l", "The level of compression (1-9)."), cmds.BoolOption(progressOptionName, "p", "Stream progress data.").WithDefault(true), }, PreRun: func(req *cmds.Request, env cmds.Environment) error { _, err := getCompressOptions(req) return err }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx := req.Context cmplvl, err := getCompressOptions(req) if err != nil { return err } api, err := cmdenv.GetApi(env, req) if err != nil { return err } p, err := cmdutils.PathOrCidPath(req.Arguments[0]) if err != nil { return err } file, err := api.Unixfs().Get(ctx, p) if err != nil { return err } size, err := file.Size() if err != nil { return err } res.SetLength(uint64(size)) archive, _ := req.Options[archiveOptionName].(bool) reader, err := fileArchive(file, p.String(), archive, cmplvl) if err != nil { return err } go func() { // We cannot defer a close in the response writer (like we should) // Because the cmd framework outsmart us and doesn't call response // if the context is over. <-ctx.Done() reader.Close() }() // Set Content-Type based on output format. // When compression is enabled, output is gzip (or tar.gz for directories). // Otherwise, tar is used as the transport format. res.SetEncodingType(cmds.OctetStream) if cmplvl != gzip.NoCompression { res.SetContentType("application/gzip") } else { res.SetContentType("application/x-tar") } return res.Emit(reader) }, PostRun: cmds.PostRunMap{ cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { req := res.Request() v, err := res.Next() if err != nil { return err } outReader, ok := v.(io.Reader) if !ok { return e.New(e.TypeErr(outReader, v)) } outPath := getOutPath(req) cmplvl, err := getCompressOptions(req) if err != nil { return err } archive, _ := req.Options[archiveOptionName].(bool) progress, _ := req.Options[progressOptionName].(bool) gw := getWriter{ Out: os.Stdout, Err: os.Stderr, Archive: archive, Compression: cmplvl, Size: int64(res.Length()), Progress: progress, } return gw.Write(outReader, outPath) }, }, } type clearlineReader struct { io.Reader out io.Writer } func (r *clearlineReader) Read(p []byte) (n int, err error) { n, err = r.Reader.Read(p) if err == io.EOF { // callback fmt.Fprintf(r.out, "\033[2K\r") // clear progress bar line on EOF } return } func progressBarForReader(out io.Writer, r io.Reader, l int64) (*pb.ProgressBar, io.Reader) { bar := makeProgressBar(out, l) barR := bar.NewProxyReader(r) return bar, &clearlineReader{barR, out} } func makeProgressBar(out io.Writer, l int64) *pb.ProgressBar { // setup bar reader // TODO: get total length of files bar := pb.New64(l).SetUnits(pb.U_BYTES) bar.Output = out // the progress bar lib doesn't give us a way to get the width of the output, // so as a hack we just use a callback to measure the output, then get rid of it bar.Callback = func(line string) { terminalWidth := len(line) bar.Callback = nil log.Infof("terminal width: %v\n", terminalWidth) } return bar } func getOutPath(req *cmds.Request) string { outPath, _ := req.Options[outputOptionName].(string) if outPath == "" { trimmed := strings.TrimRight(req.Arguments[0], "/") _, outPath = filepath.Split(trimmed) outPath = filepath.Clean(outPath) } return outPath } type getWriter struct { Out io.Writer // for output to user Err io.Writer // for progress bar output Archive bool Compression int Size int64 Progress bool } func (gw *getWriter) Write(r io.Reader, fpath string) error { if gw.Archive || gw.Compression != gzip.NoCompression { return gw.writeArchive(r, fpath) } return gw.writeExtracted(r, fpath) } func (gw *getWriter) writeArchive(r io.Reader, fpath string) error { // adjust file name if tar if gw.Archive { if !strings.HasSuffix(fpath, ".tar") && !strings.HasSuffix(fpath, ".tar.gz") { fpath += ".tar" } } // adjust file name if gz if gw.Compression != gzip.NoCompression { if !strings.HasSuffix(fpath, ".gz") { fpath += ".gz" } } // create file file, err := os.Create(fpath) if err != nil { return err } defer file.Close() fmt.Fprintf(gw.Out, "Saving archive to %s\n", fpath) if gw.Progress { var bar *pb.ProgressBar bar, r = progressBarForReader(gw.Err, r, gw.Size) bar.Start() defer bar.Finish() } _, err = io.Copy(file, r) return err } func (gw *getWriter) writeExtracted(r io.Reader, fpath string) error { fmt.Fprintf(gw.Out, "Saving file(s) to %s\n", fpath) var progressCb func(int64) int64 if gw.Progress { bar := makeProgressBar(gw.Err, gw.Size) bar.Start() defer bar.Finish() defer bar.Set64(gw.Size) progressCb = bar.Add64 } extractor := &tar.Extractor{Path: fpath, Progress: progressCb} return extractor.Extract(r) } func getCompressOptions(req *cmds.Request) (int, error) { cmprs, _ := req.Options[compressOptionName].(bool) cmplvl, cmplvlFound := req.Options[compressionLevelOptionName].(int) switch { case !cmprs: return gzip.NoCompression, nil case cmprs && !cmplvlFound: return gzip.DefaultCompression, nil case cmprs && (cmplvl < 1 || cmplvl > 9): return gzip.NoCompression, ErrInvalidCompressionLevel } return cmplvl, nil } // DefaultBufSize is the buffer size for gets. for now, 1MiB, which is ~4 blocks. // TODO: does this need to be configurable? var DefaultBufSize = 1048576 type identityWriteCloser struct { w io.Writer } func (i *identityWriteCloser) Write(p []byte) (int, error) { return i.w.Write(p) } func (i *identityWriteCloser) Close() error { return nil } func fileArchive(f files.Node, name string, archive bool, compression int) (io.ReadCloser, error) { cleaned := gopath.Clean(name) _, filename := gopath.Split(cleaned) // need to connect a writer to a reader piper, pipew := io.Pipe() checkErrAndClosePipe := func(err error) bool { if err != nil { _ = pipew.CloseWithError(err) return true } return false } // use a buffered writer to parallelize task bufw := bufio.NewWriterSize(pipew, DefaultBufSize) // compression determines whether to use gzip compression. maybeGzw, err := newMaybeGzWriter(bufw, compression) if checkErrAndClosePipe(err) { return nil, err } closeGzwAndPipe := func() { if err := maybeGzw.Close(); checkErrAndClosePipe(err) { return } if err := bufw.Flush(); checkErrAndClosePipe(err) { return } pipew.Close() // everything seems to be ok. } if !archive && compression != gzip.NoCompression { // the case when the node is a file r := files.ToFile(f) if r == nil { return nil, errors.New("file is not regular") } go func() { if _, err := io.Copy(maybeGzw, r); checkErrAndClosePipe(err) { return } closeGzwAndPipe() // everything seems to be ok }() } else { // the case for 1. archive, and 2. not archived and not compressed, in // which tar is used anyway as a transport format // construct the tar writer w, err := files.NewTarWriter(maybeGzw) if checkErrAndClosePipe(err) { return nil, err } // if not creating an archive set the format to PAX in order to preserve nanoseconds if !archive { w.SetFormat(gotar.FormatPAX) } go func() { // write all the nodes recursively if err := w.WriteFile(f, filename); checkErrAndClosePipe(err) { return } w.Close() // close tar writer closeGzwAndPipe() // everything seems to be ok }() } return piper, nil } func newMaybeGzWriter(w io.Writer, compression int) (io.WriteCloser, error) { if compression != gzip.NoCompression { return gzip.NewWriterLevel(w, compression) } return &identityWriteCloser{w}, nil } ================================================ FILE: core/commands/get_test.go ================================================ package commands import ( "fmt" "testing" cmds "github.com/ipfs/go-ipfs-cmds" ) func TestGetOutputPath(t *testing.T) { cases := []struct { args []string opts cmds.OptMap outPath string }{ { args: []string{"/ipns/multiformats.io/"}, opts: map[string]any{ "output": "takes-precedence", }, outPath: "takes-precedence", }, { args: []string{"/ipns/multiformats.io/", "some-other-arg-to-be-ignored"}, opts: cmds.OptMap{ "output": "takes-precedence", }, outPath: "takes-precedence", }, { args: []string{"/ipns/multiformats.io/"}, outPath: "multiformats.io", opts: cmds.OptMap{}, }, { args: []string{"/ipns/multiformats.io/logo.svg/"}, outPath: "logo.svg", opts: cmds.OptMap{}, }, { args: []string{"/ipns/multiformats.io", "some-other-arg-to-be-ignored"}, outPath: "multiformats.io", opts: cmds.OptMap{}, }, } _, err := GetCmd.GetOptions([]string{}) if err != nil { t.Fatalf("error getting default command options: %v", err) } for i, tc := range cases { t.Run(fmt.Sprintf("%s-%d", t.Name(), i), func(t *testing.T) { ctx := t.Context() req, err := cmds.NewRequest(ctx, []string{}, tc.opts, tc.args, nil, GetCmd) if err != nil { t.Fatalf("error creating a command request: %v", err) } if outPath := getOutPath(req); outPath != tc.outPath { t.Errorf("expected outPath %s to be %s", outPath, tc.outPath) } }) } } ================================================ FILE: core/commands/helptext_test.go ================================================ package commands import ( "strings" "testing" cmds "github.com/ipfs/go-ipfs-cmds" ) func checkHelptextRecursive(t *testing.T, name []string, c *cmds.Command) { c.ProcessHelp() t.Run(strings.Join(name, "_"), func(t *testing.T) { if c.External { t.Skip("external") } t.Run("tagline", func(t *testing.T) { if c.Helptext.Tagline == "" { t.Error("no Tagline!") } }) t.Run("longDescription", func(t *testing.T) { t.Skip("not everywhere yet") if c.Helptext.LongDescription == "" { t.Error("no LongDescription!") } }) t.Run("shortDescription", func(t *testing.T) { t.Skip("not everywhere yet") if c.Helptext.ShortDescription == "" { t.Error("no ShortDescription!") } }) t.Run("synopsis", func(t *testing.T) { t.Skip("autogenerated in go-ipfs-cmds") if c.Helptext.Synopsis == "" { t.Error("no Synopsis!") } }) }) for subname, sub := range c.Subcommands { checkHelptextRecursive(t, append(name, subname), sub) } } func TestHelptexts(t *testing.T) { Root.ProcessHelp() checkHelptextRecursive(t, []string{"ipfs"}, Root) } ================================================ FILE: core/commands/id.go ================================================ package commands import ( "encoding/base64" "encoding/json" "errors" "fmt" "io" "slices" "strings" version "github.com/ipfs/kubo" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" cmds "github.com/ipfs/go-ipfs-cmds" ke "github.com/ipfs/kubo/core/commands/keyencode" kb "github.com/libp2p/go-libp2p-kbucket" ic "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" pstore "github.com/libp2p/go-libp2p/core/peerstore" "github.com/libp2p/go-libp2p/core/protocol" ) const offlineIDErrorMessage = "'ipfs id' cannot query information on remote peers without a running daemon; if you only want to convert --peerid-base, pass --offline option" type IdOutput struct { // nolint ID string PublicKey string Addresses []string AgentVersion string Protocols []protocol.ID } const ( formatOptionName = "format" idFormatOptionName = "peerid-base" ) var IDCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Show IPFS node id info.", ShortDescription: ` Prints out information about the specified peer. If no peer is specified, prints out information for local peers. 'ipfs id' supports the format option for output with the following keys: : The peers id. : Agent version. : Protocol version. : Public key. : Addresses (newline delimited). : Libp2p Protocol registrations (newline delimited). EXAMPLE: ipfs id Qmece2RkXhsKe5CRooNisBTh4SK119KrXXGmoK6V3kb8aH -f="\n" `, }, Arguments: []cmds.Argument{ cmds.StringArg("peerid", false, false, "Peer.ID of node to look up."), }, Options: []cmds.Option{ cmds.StringOption(formatOptionName, "f", "Optional output format."), cmds.StringOption(idFormatOptionName, "Encoding used for peer IDs: Can either be a multibase encoded CID or a base58btc encoded multihash. Takes {b58mh|base36|k|base32|b...}.").WithDefault("b58mh"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { keyEnc, err := ke.KeyEncoderFromString(req.Options[idFormatOptionName].(string)) if err != nil { return err } n, err := cmdenv.GetNode(env) if err != nil { return err } var id peer.ID if len(req.Arguments) > 0 { var err error id, err = peer.Decode(req.Arguments[0]) if err != nil { return errors.New("invalid peer id") } } else { id = n.Identity } if id == n.Identity { output, err := printSelf(keyEnc, n) if err != nil { return err } return cmds.EmitOnce(res, output) } offline, _ := req.Options[OfflineOption].(bool) if !offline && !n.IsOnline { return errors.New(offlineIDErrorMessage) } if !offline { // We need to actually connect to run identify. err = n.PeerHost.Connect(req.Context, peer.AddrInfo{ID: id}) switch err { case nil: case kb.ErrLookupFailure: return errors.New(offlineIDErrorMessage) default: return err } } output, err := printPeer(keyEnc, n.Peerstore, id) if err != nil { return err } return cmds.EmitOnce(res, output) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *IdOutput) error { format, found := req.Options[formatOptionName].(string) if found { output := format output = strings.Replace(output, "", out.ID, -1) output = strings.Replace(output, "", out.AgentVersion, -1) output = strings.Replace(output, "", out.PublicKey, -1) output = strings.Replace(output, "", strings.Join(out.Addresses, "\n"), -1) output = strings.Replace(output, "", strings.Join(protocol.ConvertToStrings(out.Protocols), "\n"), -1) output = strings.Replace(output, "\\n", "\n", -1) output = strings.Replace(output, "\\t", "\t", -1) fmt.Fprint(w, output) } else { marshaled, err := json.MarshalIndent(out, "", "\t") if err != nil { return err } marshaled = append(marshaled, byte('\n')) fmt.Fprintln(w, string(marshaled)) } return nil }), }, Type: IdOutput{}, } func printPeer(keyEnc ke.KeyEncoder, ps pstore.Peerstore, p peer.ID) (any, error) { if p == "" { return nil, errors.New("attempted to print nil peer") } info := new(IdOutput) info.ID = keyEnc.FormatID(p) if pk := ps.PubKey(p); pk != nil { pkb, err := ic.MarshalPublicKey(pk) if err != nil { return nil, err } info.PublicKey = base64.StdEncoding.EncodeToString(pkb) } addrInfo := ps.PeerInfo(p) addrs, err := peer.AddrInfoToP2pAddrs(&addrInfo) if err != nil { return nil, err } for _, a := range addrs { info.Addresses = append(info.Addresses, a.String()) } slices.Sort(info.Addresses) protocols, _ := ps.GetProtocols(p) // don't care about errors here. for _, proto := range protocols { info.Protocols = append(info.Protocols, protocol.ID(cmdutils.CleanAndTrim(string(proto)))) } slices.Sort(info.Protocols) if v, err := ps.Get(p, "AgentVersion"); err == nil { if vs, ok := v.(string); ok { info.AgentVersion = cmdutils.CleanAndTrim(vs) } } return info, nil } // printing self is special cased as we get values differently. func printSelf(keyEnc ke.KeyEncoder, node *core.IpfsNode) (any, error) { info := new(IdOutput) info.ID = keyEnc.FormatID(node.Identity) pk := node.PrivateKey.GetPublic() pkb, err := ic.MarshalPublicKey(pk) if err != nil { return nil, err } info.PublicKey = base64.StdEncoding.EncodeToString(pkb) if node.PeerHost != nil { addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(node.PeerHost)) if err != nil { return nil, err } for _, a := range addrs { info.Addresses = append(info.Addresses, a.String()) } slices.Sort(info.Addresses) info.Protocols = node.PeerHost.Mux().Protocols() slices.Sort(info.Protocols) } info.AgentVersion = version.GetUserAgentVersion() return info, nil } ================================================ FILE: core/commands/keyencode/keyencode.go ================================================ package keyencode import ( cmds "github.com/ipfs/go-ipfs-cmds" peer "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" ) const ipnsKeyFormatOptionName = "ipns-base" var OptionIPNSBase = cmds.StringOption(ipnsKeyFormatOptionName, "Encoding used for keys: Can either be a multibase encoded CID or a base58btc encoded multihash. Takes {b58mh|base36|k|base32|b...}.").WithDefault("base36") type KeyEncoder struct { baseEnc *mbase.Encoder } func KeyEncoderFromString(formatLabel string) (KeyEncoder, error) { switch formatLabel { case "b58mh", "v0": return KeyEncoder{}, nil default: if enc, err := mbase.EncoderByName(formatLabel); err != nil { return KeyEncoder{}, err } else { return KeyEncoder{&enc}, nil } } } func (enc KeyEncoder) FormatID(id peer.ID) string { if enc.baseEnc == nil { return id.String() } if s, err := peer.ToCid(id).StringOfBase(enc.baseEnc.Encoding()); err != nil { panic(err) } else { return s } } ================================================ FILE: core/commands/keystore.go ================================================ package commands import ( "bytes" "crypto/ed25519" "crypto/x509" "encoding/pem" "errors" "fmt" "io" "os" "path/filepath" "strings" "text/tabwriter" keystore "github.com/ipfs/boxo/keystore" cmds "github.com/ipfs/go-ipfs-cmds" oldcmds "github.com/ipfs/kubo/commands" config "github.com/ipfs/kubo/config" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/e" ke "github.com/ipfs/kubo/core/commands/keyencode" options "github.com/ipfs/kubo/core/coreiface/options" fsrepo "github.com/ipfs/kubo/repo/fsrepo" migrations "github.com/ipfs/kubo/repo/fsrepo/migrations" "github.com/libp2p/go-libp2p/core/crypto" peer "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" ) var KeyCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Create and list IPNS name keypairs", ShortDescription: ` 'ipfs key gen' generates a new keypair for usage with IPNS and 'ipfs name publish'. > ipfs key gen --type=rsa --size=2048 mykey > ipfs name publish --key=mykey QmSomeHash 'ipfs key ls' lists the available keys. > ipfs key ls self mykey `, }, Subcommands: map[string]*cmds.Command{ "gen": keyGenCmd, "export": keyExportCmd, "import": keyImportCmd, "list": keyListDeprecatedCmd, "ls": keyListCmd, "rename": keyRenameCmd, "rm": keyRmCmd, "rotate": keyRotateCmd, "sign": keySignCmd, "verify": keyVerifyCmd, }, } type KeyOutput struct { Name string Id string //nolint } type KeyOutputList struct { Keys []KeyOutput } // KeyRenameOutput define the output type of keyRenameCmd type KeyRenameOutput struct { Was string Now string Id string //nolint Overwrite bool } const ( keyStoreAlgorithmDefault = options.Ed25519Key keyStoreTypeOptionName = "type" keyStoreSizeOptionName = "size" oldKeyOptionName = "oldkey" ) var keyGenCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Create a new keypair", }, Options: []cmds.Option{ cmds.StringOption(keyStoreTypeOptionName, "t", "type of the key to create: rsa, ed25519").WithDefault(keyStoreAlgorithmDefault), cmds.IntOption(keyStoreSizeOptionName, "s", "size of the key to generate"), ke.OptionIPNSBase, }, Arguments: []cmds.Argument{ cmds.StringArg("name", true, false, "name of key to create"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } typ, f := req.Options[keyStoreTypeOptionName].(string) if !f { return errors.New("please specify a key type with --type") } name := req.Arguments[0] if name == "self" { return errors.New("cannot create key with name 'self'") } opts := []options.KeyGenerateOption{options.Key.Type(typ)} size, sizefound := req.Options[keyStoreSizeOptionName].(int) if sizefound { opts = append(opts, options.Key.Size(size)) } keyEnc, err := ke.KeyEncoderFromString(req.Options[ke.OptionIPNSBase.Name()].(string)) if err != nil { return err } key, err := api.Key().Generate(req.Context, name, opts...) if err != nil { return err } return cmds.EmitOnce(res, &KeyOutput{ Name: name, Id: keyEnc.FormatID(key.ID()), }) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, ko *KeyOutput) error { _, err := w.Write([]byte(ko.Id + "\n")) return err }), }, Type: KeyOutput{}, } const ( // Key format options used both for importing and exporting. keyFormatOptionName = "format" keyFormatPemCleartextOption = "pem-pkcs8-cleartext" keyFormatLibp2pCleartextOption = "libp2p-protobuf-cleartext" keyAllowAnyTypeOptionName = "allow-any-key-type" ) var keyExportCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Export a keypair", ShortDescription: ` Exports a named libp2p key to disk. By default, the output will be stored at './.key', but an alternate path can be specified with '--output=' or '-o='. It is possible to export a private key to interoperable PEM PKCS8 format by explicitly passing '--format=pem-pkcs8-cleartext'. The resulting PEM file can then be consumed elsewhere. For example, using openssl to get a PEM with public key: $ ipfs key export testkey --format=pem-pkcs8-cleartext -o privkey.pem $ openssl pkey -in privkey.pem -pubout > pubkey.pem `, }, Arguments: []cmds.Argument{ cmds.StringArg("name", true, false, "name of key to export").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption(outputOptionName, "o", "The path where the output should be stored."), cmds.StringOption(keyFormatOptionName, "f", "The format of the exported private key, libp2p-protobuf-cleartext or pem-pkcs8-cleartext.").WithDefault(keyFormatLibp2pCleartextOption), }, NoRemote: true, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { name := req.Arguments[0] if name == "self" { return fmt.Errorf("cannot export key with name 'self'") } cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err } // Check repo version, and error out if not matching ver, err := migrations.RepoVersion(cfgRoot) if err != nil { return err } if ver != fsrepo.RepoVersion { return fmt.Errorf("key export expects repo version (%d) but found (%d)", fsrepo.RepoVersion, ver) } // Export is read-only: safe to read it without acquiring repo lock // (this makes export work when ipfs daemon is already running) ksp := filepath.Join(cfgRoot, "keystore") ks, err := keystore.NewFSKeystore(ksp) if err != nil { return err } sk, err := ks.Get(name) if err != nil { return fmt.Errorf("key with name '%s' doesn't exist", name) } exportFormat, _ := req.Options[keyFormatOptionName].(string) var formattedKey []byte switch exportFormat { case keyFormatPemCleartextOption: stdKey, err := crypto.PrivKeyToStdKey(sk) if err != nil { return fmt.Errorf("converting libp2p private key to std Go key: %w", err) } // For some reason the ed25519.PrivateKey does not use pointer // receivers, so we need to convert it for MarshalPKCS8PrivateKey. // (We should probably change this upstream in PrivKeyToStdKey). if ed25519KeyPointer, ok := stdKey.(*ed25519.PrivateKey); ok { stdKey = *ed25519KeyPointer } // This function supports a restricted list of public key algorithms, // but we generate and use only the RSA and ed25519 types that are on that list. formattedKey, err = x509.MarshalPKCS8PrivateKey(stdKey) if err != nil { return fmt.Errorf("marshalling key to PKCS8 format: %w", err) } case keyFormatLibp2pCleartextOption: formattedKey, err = crypto.MarshalPrivateKey(sk) if err != nil { return err } default: return fmt.Errorf("unrecognized export format: %s", exportFormat) } return res.Emit(bytes.NewReader(formattedKey)) }, PostRun: cmds.PostRunMap{ cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { req := res.Request() v, err := res.Next() if err != nil { return err } outReader, ok := v.(io.Reader) if !ok { return e.New(e.TypeErr(outReader, v)) } outPath, _ := req.Options[outputOptionName].(string) exportFormat, _ := req.Options[keyFormatOptionName].(string) if outPath == "" { var fileExtension string switch exportFormat { case keyFormatPemCleartextOption: fileExtension = "pem" case keyFormatLibp2pCleartextOption: fileExtension = "key" } trimmed := strings.TrimRight(fmt.Sprintf("%s.%s", req.Arguments[0], fileExtension), "/") _, outPath = filepath.Split(trimmed) outPath = filepath.Clean(outPath) } // create file file, err := os.Create(outPath) if err != nil { return err } defer file.Close() switch exportFormat { case keyFormatPemCleartextOption: privKeyBytes, err := io.ReadAll(outReader) if err != nil { return err } err = pem.Encode(file, &pem.Block{ Type: "PRIVATE KEY", Bytes: privKeyBytes, }) if err != nil { return fmt.Errorf("encoding PEM block: %w", err) } case keyFormatLibp2pCleartextOption: _, err = io.Copy(file, outReader) if err != nil { return err } } return nil }, }, } var keyImportCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Import a key and prints imported key id", ShortDescription: ` Imports a key and stores it under the provided name. By default, the key is assumed to be in 'libp2p-protobuf-cleartext' format, however it is possible to import private keys wrapped in interoperable PEM PKCS8 by passing '--format=pem-pkcs8-cleartext'. The PEM format allows for key generation outside of the IPFS node: $ openssl genpkey -algorithm ED25519 > ed25519.pem $ ipfs key import test-openssl -f pem-pkcs8-cleartext ed25519.pem `, }, Options: []cmds.Option{ ke.OptionIPNSBase, cmds.StringOption(keyFormatOptionName, "f", "The format of the private key to import, libp2p-protobuf-cleartext or pem-pkcs8-cleartext.").WithDefault(keyFormatLibp2pCleartextOption), cmds.BoolOption(keyAllowAnyTypeOptionName, "Allow importing any key type.").WithDefault(false), }, Arguments: []cmds.Argument{ cmds.StringArg("name", true, false, "name to associate with key in keychain"), cmds.FileArg("key", true, false, "key provided by generate or export"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { name := req.Arguments[0] if name == "self" { return fmt.Errorf("cannot import key with name 'self'") } keyEnc, err := ke.KeyEncoderFromString(req.Options[ke.OptionIPNSBase.Name()].(string)) if err != nil { return err } file, err := cmdenv.GetFileArg(req.Files.Entries()) if err != nil { return err } defer file.Close() data, err := io.ReadAll(file) if err != nil { return err } importFormat, _ := req.Options[keyFormatOptionName].(string) var sk crypto.PrivKey switch importFormat { case keyFormatPemCleartextOption: pemBlock, rest := pem.Decode(data) if pemBlock == nil { return fmt.Errorf("PEM block not found in input data:\n%s", rest) } if pemBlock.Type != "PRIVATE KEY" { return fmt.Errorf("expected PRIVATE KEY type in PEM block but got: %s", pemBlock.Type) } stdKey, err := x509.ParsePKCS8PrivateKey(pemBlock.Bytes) if err != nil { return fmt.Errorf("parsing PKCS8 format: %w", err) } // In case ed25519.PrivateKey is returned we need the pointer for // conversion to libp2p (see export command for more details). if ed25519KeyPointer, ok := stdKey.(ed25519.PrivateKey); ok { stdKey = &ed25519KeyPointer } sk, _, err = crypto.KeyPairFromStdKey(stdKey) if err != nil { return fmt.Errorf("converting std Go key to libp2p key: %w", err) } case keyFormatLibp2pCleartextOption: sk, err = crypto.UnmarshalPrivateKey(data) if err != nil { // check if data is PEM, if so, provide user with hint pemBlock, _ := pem.Decode(data) if pemBlock != nil { return fmt.Errorf("unexpected PEM block for format=%s: try again with format=%s", keyFormatLibp2pCleartextOption, keyFormatPemCleartextOption) } return fmt.Errorf("unable to unmarshall format=%s: %w", keyFormatLibp2pCleartextOption, err) } default: return fmt.Errorf("unrecognized import format: %s", importFormat) } // We only allow importing keys of the same type we generate (see list in // https://github.com/ipfs/interface-go-ipfs-core/blob/1c3d8fc/options/key.go#L58-L60), // unless explicitly stated by the user. allowAnyKeyType, _ := req.Options[keyAllowAnyTypeOptionName].(bool) if !allowAnyKeyType { switch t := sk.(type) { case *crypto.RsaPrivateKey, *crypto.Ed25519PrivateKey: default: return fmt.Errorf("key type %T is not allowed to be imported, only RSA or Ed25519;"+ " use flag --%s if you are sure of what you're doing", t, keyAllowAnyTypeOptionName) } } cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err } r, err := fsrepo.Open(cfgRoot) if err != nil { return err } defer r.Close() _, err = r.Keystore().Get(name) if err == nil { return fmt.Errorf("key with name '%s' already exists", name) } err = r.Keystore().Put(name, sk) if err != nil { return err } pid, err := peer.IDFromPrivateKey(sk) if err != nil { return err } return cmds.EmitOnce(res, &KeyOutput{ Name: name, Id: keyEnc.FormatID(pid), }) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, ko *KeyOutput) error { _, err := w.Write([]byte(ko.Id + "\n")) return err }), }, Type: KeyOutput{}, } var keyListCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List all local keypairs.", }, Options: []cmds.Option{ cmds.BoolOption("l", "Show extra information about keys."), ke.OptionIPNSBase, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { keyEnc, err := ke.KeyEncoderFromString(req.Options[ke.OptionIPNSBase.Name()].(string)) if err != nil { return fmt.Errorf("cannot get key encoder: %w", err) } api, err := cmdenv.GetApi(env, req) if err != nil { return err } keys, err := api.Key().List(req.Context) if err != nil { return fmt.Errorf("listing keys failed: %w", err) } list := make([]KeyOutput, 0, len(keys)) for _, key := range keys { list = append(list, KeyOutput{ Name: key.Name(), Id: keyEnc.FormatID(key.ID()), }) } return cmds.EmitOnce(res, &KeyOutputList{list}) }, Encoders: cmds.EncoderMap{ cmds.Text: keyOutputListEncoders(), }, Type: KeyOutputList{}, } var keyListDeprecatedCmd = &cmds.Command{ Status: cmds.Deprecated, Helptext: cmds.HelpText{ Tagline: "Deprecated: use 'ipfs key ls' instead.", }, Options: keyListCmd.Options, Run: keyListCmd.Run, Encoders: keyListCmd.Encoders, Type: keyListCmd.Type, } const ( keyStoreForceOptionName = "force" ) var keyRenameCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Rename a keypair.", }, Arguments: []cmds.Argument{ cmds.StringArg("name", true, false, "name of key to rename"), cmds.StringArg("newName", true, false, "new name of the key"), }, Options: []cmds.Option{ cmds.BoolOption(keyStoreForceOptionName, "f", "Allow to overwrite an existing key."), ke.OptionIPNSBase, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } keyEnc, err := ke.KeyEncoderFromString(req.Options[ke.OptionIPNSBase.Name()].(string)) if err != nil { return err } name := req.Arguments[0] newName := req.Arguments[1] force, _ := req.Options[keyStoreForceOptionName].(bool) key, overwritten, err := api.Key().Rename(req.Context, name, newName, options.Key.Force(force)) if err != nil { return err } return cmds.EmitOnce(res, &KeyRenameOutput{ Was: name, Now: newName, Id: keyEnc.FormatID(key.ID()), Overwrite: overwritten, }) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, kro *KeyRenameOutput) error { if kro.Overwrite { fmt.Fprintf(w, "Key %s renamed to %s with overwriting\n", kro.Id, cmdenv.EscNonPrint(kro.Now)) } else { fmt.Fprintf(w, "Key %s renamed to %s\n", kro.Id, cmdenv.EscNonPrint(kro.Now)) } return nil }), }, Type: KeyRenameOutput{}, } var keyRmCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Remove a keypair.", }, Arguments: []cmds.Argument{ cmds.StringArg("name", true, true, "names of keys to remove").EnableStdin(), }, Options: []cmds.Option{ cmds.BoolOption("l", "Show extra information about keys."), ke.OptionIPNSBase, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } keyEnc, err := ke.KeyEncoderFromString(req.Options[ke.OptionIPNSBase.Name()].(string)) if err != nil { return err } names := req.Arguments list := make([]KeyOutput, 0, len(names)) for _, name := range names { key, err := api.Key().Remove(req.Context, name) if err != nil { return err } list = append(list, KeyOutput{ Name: name, Id: keyEnc.FormatID(key.ID()), }) } return cmds.EmitOnce(res, &KeyOutputList{list}) }, Encoders: cmds.EncoderMap{ cmds.Text: keyOutputListEncoders(), }, Type: KeyOutputList{}, } var keyRotateCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Rotates the IPFS identity.", ShortDescription: ` Generates a new ipfs identity and saves it to the ipfs config file. Your existing identity key will be backed up in the Keystore. The daemon must not be running when calling this command. ipfs uses a repository in the local file system. By default, the repo is located at ~/.ipfs. To change the repo location, set the $IPFS_PATH environment variable: export IPFS_PATH=/path/to/ipfsrepo `, }, Arguments: []cmds.Argument{}, Options: []cmds.Option{ cmds.StringOption(oldKeyOptionName, "o", "Keystore name to use for backing up your existing identity"), cmds.StringOption(keyStoreTypeOptionName, "t", "type of the key to create: rsa, ed25519").WithDefault(keyStoreAlgorithmDefault), cmds.IntOption(keyStoreSizeOptionName, "s", "size of the key to generate"), }, NoRemote: true, PreRun: DaemonNotRunning, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cctx := env.(*oldcmds.Context) nBitsForKeypair, nBitsGiven := req.Options[keyStoreSizeOptionName].(int) algorithm, _ := req.Options[keyStoreTypeOptionName].(string) oldKey, ok := req.Options[oldKeyOptionName].(string) if !ok { return fmt.Errorf("keystore name for backing up old key must be provided") } if oldKey == "self" { return fmt.Errorf("keystore name for back up cannot be named 'self'") } return doRotate(os.Stdout, cctx.ConfigRoot, oldKey, algorithm, nBitsForKeypair, nBitsGiven) }, } func doRotate(out io.Writer, repoRoot string, oldKey string, algorithm string, nBitsForKeypair int, nBitsGiven bool) error { // Open repo repo, err := fsrepo.Open(repoRoot) if err != nil { return fmt.Errorf("opening repo (%v)", err) } defer repo.Close() // Read config file from repo cfg, err := repo.Config() if err != nil { return fmt.Errorf("reading config from repo (%v)", err) } // Generate new identity var identity config.Identity if nBitsGiven { identity, err = config.CreateIdentity(out, []options.KeyGenerateOption{ options.Key.Size(nBitsForKeypair), options.Key.Type(algorithm), }) } else { identity, err = config.CreateIdentity(out, []options.KeyGenerateOption{ options.Key.Type(algorithm), }) } if err != nil { return fmt.Errorf("creating identity (%v)", err) } // Save old identity to keystore oldPrivKey, err := cfg.Identity.DecodePrivateKey("") if err != nil { return fmt.Errorf("decoding old private key (%v)", err) } keystore := repo.Keystore() if err := keystore.Put(oldKey, oldPrivKey); err != nil { return fmt.Errorf("saving old key in keystore (%v)", err) } // Update identity cfg.Identity = identity // Write config file to repo if err = repo.SetConfig(cfg); err != nil { return fmt.Errorf("saving new key to config (%v)", err) } return nil } func keyOutputListEncoders() cmds.EncoderFunc { return cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, list *KeyOutputList) error { withID, _ := req.Options["l"].(bool) tw := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) for _, s := range list.Keys { if withID { fmt.Fprintf(tw, "%s\t%s\t\n", s.Id, cmdenv.EscNonPrint(s.Name)) } else { fmt.Fprintf(tw, "%s\n", cmdenv.EscNonPrint(s.Name)) } } tw.Flush() return nil }) } type KeySignOutput struct { Key KeyOutput Signature string } var keySignCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Generates a signature for the given data with a specified key. Useful for proving the key ownership.", LongDescription: ` Sign arbitrary bytes, such as to prove ownership of a Peer ID or an IPNS Name. To avoid signature reuse, the signed payload is always prefixed with "libp2p-key signed message:". `, }, Options: []cmds.Option{ cmds.StringOption("key", "k", "The name of the key to use for signing."), ke.OptionIPNSBase, }, Arguments: []cmds.Argument{ cmds.FileArg("data", true, false, "The data to sign.").EnableStdin(), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } keyEnc, err := ke.KeyEncoderFromString(req.Options[ke.OptionIPNSBase.Name()].(string)) if err != nil { return err } name, _ := req.Options["key"].(string) file, err := cmdenv.GetFileArg(req.Files.Entries()) if err != nil { return err } defer file.Close() data, err := io.ReadAll(file) if err != nil { return err } key, signature, err := api.Key().Sign(req.Context, name, data) if err != nil { return err } encodedSignature, err := mbase.Encode(mbase.Base64url, signature) if err != nil { return err } return res.Emit(&KeySignOutput{ Key: KeyOutput{ Name: key.Name(), Id: keyEnc.FormatID(key.ID()), }, Signature: encodedSignature, }) }, Type: KeySignOutput{}, } type KeyVerifyOutput struct { Key KeyOutput SignatureValid bool } var keyVerifyCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Verify that the given data and signature match.", LongDescription: ` Verify if the given data and signatures match. To avoid the signature reuse, the signed payload is always prefixed with "libp2p-key signed message:". `, }, Options: []cmds.Option{ cmds.StringOption("key", "k", "The name of the key to use for verifying."), cmds.StringOption("signature", "s", "Multibase-encoded signature to verify."), ke.OptionIPNSBase, }, Arguments: []cmds.Argument{ cmds.FileArg("data", true, false, "The data to verify against the given signature.").EnableStdin(), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } keyEnc, err := ke.KeyEncoderFromString(req.Options[ke.OptionIPNSBase.Name()].(string)) if err != nil { return err } name, _ := req.Options["key"].(string) encodedSignature, _ := req.Options["signature"].(string) _, signature, err := mbase.Decode(encodedSignature) if err != nil { return err } file, err := cmdenv.GetFileArg(req.Files.Entries()) if err != nil { return err } defer file.Close() data, err := io.ReadAll(file) if err != nil { return err } key, valid, err := api.Key().Verify(req.Context, name, signature, data) if err != nil { return err } return res.Emit(&KeyVerifyOutput{ Key: KeyOutput{ Name: key.Name(), Id: keyEnc.FormatID(key.ID()), }, SignatureValid: valid, }) }, Type: KeyVerifyOutput{}, } // DaemonNotRunning checks to see if the ipfs repo is locked, indicating that // the daemon is running, and returns and error if the daemon is running. func DaemonNotRunning(req *cmds.Request, env cmds.Environment) error { cctx := env.(*oldcmds.Context) daemonLocked, err := fsrepo.LockedByOtherProcess(cctx.ConfigRoot) if err != nil { return err } log.Info("checking if daemon is running...") if daemonLocked { log.Debug("ipfs daemon is running") e := "ipfs daemon is running. please stop it to run this command" return cmds.ClientError(e) } return nil } ================================================ FILE: core/commands/log.go ================================================ package commands import ( "fmt" "io" "slices" cmds "github.com/ipfs/go-ipfs-cmds" logging "github.com/ipfs/go-log/v2" ) const ( // allLogSubsystems is used to specify all log subsystems when setting the // log level. allLogSubsystems = "*" // allLogSubsystemsAlias is a convenience alias for allLogSubsystems that // doesn't require shell escaping. allLogSubsystemsAlias = "all" // defaultLogLevel is used to request and to identify the default log // level. defaultLogLevel = "default" // defaultSubsystemKey is the subsystem name that is used to denote the // default log level. We use parentheses for UI clarity to distinguish it // from regular subsystem names. defaultSubsystemKey = "(default)" // logLevelOption is an option for the tail subcommand to select the log // level to output. logLevelOption = "log-level" // noSubsystemSpecified is used when no subsystem argument is provided noSubsystemSpecified = "" ) type logLevelOutput struct { Levels map[string]string `json:",omitempty"` Message string `json:",omitempty"` } var LogCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Interact with the daemon log output.", ShortDescription: ` 'ipfs log' contains utility commands to affect or read the logging output of a running daemon. There are also two environmental variables that direct the logging system (not just for the daemon logs, but all commands): GOLOG_LOG_LEVEL - sets the level of verbosity of the logging. One of: debug, info, warn, error, dpanic, panic, fatal GOLOG_LOG_FMT - sets formatting of the log output. One of: color, nocolor, json `, }, Subcommands: map[string]*cmds.Command{ "level": logLevelCmd, "ls": logLsCmd, "tail": logTailCmd, }, } var logLevelCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Change or get the logging level.", ShortDescription: ` Get or change the logging level of one or all logging subsystems. This command provides a runtime alternative to the GOLOG_LOG_LEVEL environment variable for debugging and troubleshooting. UNDERSTANDING DEFAULT vs '*': The "default" level is the fallback used by unconfigured subsystems. You cannot set the default level directly - it only changes when you use '*'. The '*' wildcard represents ALL subsystems including the default level. Setting '*' changes everything at once, including the default. EXAMPLES - Getting levels: ipfs log level # Show only the default fallback level ipfs log level all # Show all subsystem levels (100+ lines) ipfs log level core # Show level for 'core' subsystem only EXAMPLES - Setting levels: ipfs log level core debug # Set 'core' to 'debug' (default unchanged) ipfs log level all info # Set ALL to 'info' (including default) ipfs log level core default # Reset 'core' to use current default level WILDCARD OPTIONS: Use 'all' (convenient) or '*' (requires escaping) to affect all subsystems: ipfs log level all debug # Convenient - no shell escaping needed ipfs log level '*' debug # Equivalent but needs quotes: '*' or "*" or \* BEHAVIOR EXAMPLES: Initial state (all using default 'error'): $ ipfs log level => error $ ipfs log level core => error After setting one subsystem: $ ipfs log level core debug $ ipfs log level => error (default unchanged!) $ ipfs log level core => debug (explicitly set) $ ipfs log level dht => error (still uses default) After setting everything with 'all': $ ipfs log level all info $ ipfs log level => info (default changed!) $ ipfs log level core => info (all changed) $ ipfs log level dht => info (all changed) The 'default' keyword always refers to the current default level: $ ipfs log level => error $ ipfs log level core default # Sets core to 'error' $ ipfs log level all info # Changes default to 'info' $ ipfs log level core default # Now sets core to 'info' `, }, Arguments: []cmds.Argument{ cmds.StringArg("subsystem", false, false, fmt.Sprintf("The subsystem logging identifier. Use '%s' or '%s' to get or set the log level of all subsystems including the default. If not specified, only show the default log level.", allLogSubsystemsAlias, allLogSubsystems)), cmds.StringArg("level", false, false, fmt.Sprintf("The log level, with 'debug' as the most verbose and 'fatal' the least verbose. Use '%s' to set to the current default level. One of: debug, info, warn, error, dpanic, panic, fatal, %s", defaultLogLevel, defaultLogLevel)), }, NoLocal: true, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { var level, subsystem string if len(req.Arguments) > 0 { subsystem = req.Arguments[0] if len(req.Arguments) > 1 { level = req.Arguments[1] } // Normalize aliases to the canonical "*" form if subsystem == allLogSubsystems || subsystem == allLogSubsystemsAlias { subsystem = "*" } } // If a level is specified, then set the log level. if level != "" { if level == defaultLogLevel { level = logging.DefaultLevel().String() } if err := logging.SetLogLevel(subsystem, level); err != nil { return err } s := fmt.Sprintf("Changed log level of '%s' to '%s'\n", subsystem, level) log.Info(s) return cmds.EmitOnce(res, &logLevelOutput{Message: s}) } // Get the level for the requested subsystem. switch subsystem { case noSubsystemSpecified: // Return the default log level levelMap := map[string]string{logging.DefaultName: logging.DefaultLevel().String()} return cmds.EmitOnce(res, &logLevelOutput{Levels: levelMap}) case allLogSubsystems, allLogSubsystemsAlias: // Return levels for all subsystems (default behavior) levels := logging.SubsystemLevelNames() // Replace default subsystem key with defaultSubsystemKey. levels[defaultSubsystemKey] = levels[logging.DefaultName] delete(levels, logging.DefaultName) return cmds.EmitOnce(res, &logLevelOutput{Levels: levels}) default: // Return level for a specific subsystem. level, err := logging.SubsystemLevelName(subsystem) if err != nil { return err } levelMap := map[string]string{subsystem: level} return cmds.EmitOnce(res, &logLevelOutput{Levels: levelMap}) } }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *logLevelOutput) error { if out.Message != "" { fmt.Fprint(w, out.Message) return nil } // Check if this is an RPC call by looking for the encoding option encoding, _ := req.Options["encoding"].(string) isRPC := encoding == "json" // Determine whether to show subsystem names in output. // Show subsystem names when: // 1. It's an RPC call (needs JSON structure with named fields) // 2. Multiple subsystems are displayed (for clarity when showing many levels) showNames := isRPC || len(out.Levels) > 1 levelNames := make([]string, 0, len(out.Levels)) for subsystem, level := range out.Levels { if showNames { // Show subsystem name when it's RPC or when showing multiple subsystems levelNames = append(levelNames, fmt.Sprintf("%s: %s", subsystem, level)) } else { // For CLI calls with single subsystem, only show the level levelNames = append(levelNames, level) } } slices.Sort(levelNames) for _, ln := range levelNames { fmt.Fprintln(w, ln) } return nil }), }, Type: logLevelOutput{}, } var logLsCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List the logging subsystems.", ShortDescription: ` 'ipfs log ls' is a utility command used to list the logging subsystems of a running daemon. `, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { return cmds.EmitOnce(res, &stringList{logging.GetSubsystems()}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, list *stringList) error { for _, s := range list.Strings { fmt.Fprintln(w, s) } return nil }), }, Type: stringList{}, } var logTailCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Read and output log messages.", ShortDescription: ` Outputs log messages as they are generated. NOTE: --log-level requires the server to be logging at least at this level Example: GOLOG_LOG_LEVEL="error,bitswap=debug" ipfs daemon ipfs log tail --log-level info This will only return 'info' logs from bitswap and skip 'debug'. `, }, Options: []cmds.Option{ cmds.StringOption(logLevelOption, "Log level to listen to.").WithDefault(""), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { var pipeReader *logging.PipeReader logLevelString, _ := req.Options[logLevelOption].(string) if logLevelString != "" { logLevel, err := logging.Parse(logLevelString) if err != nil { return fmt.Errorf("setting log level %s: %w", logLevelString, err) } pipeReader = logging.NewPipeReader(logging.PipeLevel(logLevel)) } else { pipeReader = logging.NewPipeReader() } go func() { <-req.Context.Done() pipeReader.Close() }() return res.Emit(pipeReader) }, } ================================================ FILE: core/commands/ls.go ================================================ package commands import ( "context" "fmt" "io" "os" "slices" "strings" "text/tabwriter" "time" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" unixfs "github.com/ipfs/boxo/ipld/unixfs" unixfs_pb "github.com/ipfs/boxo/ipld/unixfs/pb" cmds "github.com/ipfs/go-ipfs-cmds" iface "github.com/ipfs/kubo/core/coreiface" options "github.com/ipfs/kubo/core/coreiface/options" ) // LsLink contains printable data for a single ipld link in ls output type LsLink struct { Name, Hash string Size uint64 Type unixfs_pb.Data_DataType Target string Mode os.FileMode ModTime time.Time } // LsObject is an element of LsOutput // It can represent all or part of a directory type LsObject struct { Hash string Links []LsLink } // LsOutput is a set of printable data for directories, // it can be complete or partial type LsOutput struct { Objects []LsObject } const ( lsHeadersOptionNameTime = "headers" lsResolveTypeOptionName = "resolve-type" lsSizeOptionName = "size" lsStreamOptionName = "stream" lsLongOptionName = "long" ) var LsCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List directory contents for Unix filesystem objects.", ShortDescription: ` Displays the contents of an IPFS or IPNS object(s) at the given path, with the following format: With the --long (-l) option, display optional file mode (permissions) and modification time in a format similar to Unix 'ls -l': Mode and mtime are optional UnixFS metadata. They are only present if the content was imported with 'ipfs add --preserve-mode' and '--preserve-mtime'. Without preserved metadata, both mode and mtime display '-'. Times are in UTC. Example with --long and preserved metadata: -rw-r--r-- QmZULkCELmmk5XNf... 1234 Jan 15 10:30 document.txt -rwxr-xr-x QmaRGe7bVmVaLmxb... 5678 Dec 01 2023 script.sh drwxr-xr-x QmWWEQhcLufF3qPm... - Nov 20 2023 subdir/ Example with --long without preserved metadata: - QmZULkCELmmk5XNf... 1234 - document.txt The JSON output contains type information. `, }, Arguments: []cmds.Argument{ cmds.StringArg("ipfs-path", true, true, "The path to the IPFS object(s) to list links from.").EnableStdin(), }, Options: []cmds.Option{ cmds.BoolOption(lsHeadersOptionNameTime, "v", "Print table headers (Hash, Size, Name)."), cmds.BoolOption(lsResolveTypeOptionName, "Resolve linked objects to find out their types.").WithDefault(true), cmds.BoolOption(lsSizeOptionName, "Resolve linked objects to find out their file size.").WithDefault(true), cmds.BoolOption(lsStreamOptionName, "s", "Enable experimental streaming of directory entries as they are traversed."), cmds.BoolOption(lsLongOptionName, "l", "Use a long listing format, showing file mode and modification time."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } resolveType, _ := req.Options[lsResolveTypeOptionName].(bool) resolveSize, _ := req.Options[lsSizeOptionName].(bool) stream, _ := req.Options[lsStreamOptionName].(bool) err = req.ParseBodyArgs() if err != nil { return err } paths := req.Arguments enc, err := cmdenv.GetCidEncoder(req) if err != nil { return err } var processLink func(path string, link LsLink) error var dirDone func(i int) processDir := func() (func(path string, link LsLink) error, func(i int)) { return func(path string, link LsLink) error { output := []LsObject{{ Hash: path, Links: []LsLink{link}, }} return res.Emit(&LsOutput{output}) }, func(i int) {} } done := func() error { return nil } if !stream { output := make([]LsObject, len(req.Arguments)) processDir = func() (func(path string, link LsLink) error, func(i int)) { // for each dir outputLinks := make([]LsLink, 0) return func(path string, link LsLink) error { // for each link outputLinks = append(outputLinks, link) return nil }, func(i int) { // after each dir slices.SortFunc(outputLinks, func(a, b LsLink) int { return strings.Compare(a.Name, b.Name) }) output[i] = LsObject{ Hash: paths[i], Links: outputLinks, } } } done = func() error { return cmds.EmitOnce(res, &LsOutput{output}) } } lsCtx, cancel := context.WithCancel(req.Context) defer cancel() for i, fpath := range paths { pth, err := cmdutils.PathOrCidPath(fpath) if err != nil { return err } results := make(chan iface.DirEntry) lsErr := make(chan error, 1) go func() { lsErr <- api.Unixfs().Ls(lsCtx, pth, results, options.Unixfs.ResolveChildren(resolveSize || resolveType)) }() processLink, dirDone = processDir() for link := range results { var ftype unixfs_pb.Data_DataType switch link.Type { case iface.TFile: ftype = unixfs.TFile case iface.TDirectory: ftype = unixfs.TDirectory case iface.TSymlink: ftype = unixfs.TSymlink } lsLink := LsLink{ Name: link.Name, Hash: enc.Encode(link.Cid), Size: link.Size, Type: ftype, Target: link.Target, Mode: link.Mode, ModTime: link.ModTime, } if err = processLink(paths[i], lsLink); err != nil { return err } } if err = <-lsErr; err != nil { return err } dirDone(i) } return done() }, PostRun: cmds.PostRunMap{ cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { req := res.Request() lastObjectHash := "" for { v, err := res.Next() if err != nil { if err == io.EOF { return nil } return err } out := v.(*LsOutput) lastObjectHash = tabularOutput(req, os.Stdout, out, lastObjectHash, false) } }, }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *LsOutput) error { // when streaming over HTTP using a text encoder, we cannot render breaks // between directories because we don't know the hash of the last // directory encoder ignoreBreaks, _ := req.Options[lsStreamOptionName].(bool) tabularOutput(req, w, out, "", ignoreBreaks) return nil }), }, Type: LsOutput{}, } // formatMode converts os.FileMode to a 10-character Unix ls-style string. // // Format: [type][owner rwx][group rwx][other rwx] // // Type indicators: - (regular), d (directory), l (symlink), p (named pipe), // s (socket), c (char device), b (block device). // // Special bits replace the execute position: setuid on owner (s/S), // setgid on group (s/S), sticky on other (t/T). Lowercase when the // underlying execute bit is also set, uppercase when not. func formatMode(mode os.FileMode) string { var buf [10]byte // File type - handle all special file types like ls does switch { case mode&os.ModeDir != 0: buf[0] = 'd' case mode&os.ModeSymlink != 0: buf[0] = 'l' case mode&os.ModeNamedPipe != 0: buf[0] = 'p' case mode&os.ModeSocket != 0: buf[0] = 's' case mode&os.ModeDevice != 0: if mode&os.ModeCharDevice != 0 { buf[0] = 'c' } else { buf[0] = 'b' } default: buf[0] = '-' } // Owner permissions (bits 8,7,6) buf[1] = permBit(mode, 0400, 'r') // read buf[2] = permBit(mode, 0200, 'w') // write // Handle setuid bit for owner execute if mode&os.ModeSetuid != 0 { if mode&0100 != 0 { buf[3] = 's' } else { buf[3] = 'S' } } else { buf[3] = permBit(mode, 0100, 'x') // execute } // Group permissions (bits 5,4,3) buf[4] = permBit(mode, 0040, 'r') // read buf[5] = permBit(mode, 0020, 'w') // write // Handle setgid bit for group execute if mode&os.ModeSetgid != 0 { if mode&0010 != 0 { buf[6] = 's' } else { buf[6] = 'S' } } else { buf[6] = permBit(mode, 0010, 'x') // execute } // Other permissions (bits 2,1,0) buf[7] = permBit(mode, 0004, 'r') // read buf[8] = permBit(mode, 0002, 'w') // write // Handle sticky bit for other execute if mode&os.ModeSticky != 0 { if mode&0001 != 0 { buf[9] = 't' } else { buf[9] = 'T' } } else { buf[9] = permBit(mode, 0001, 'x') // execute } return string(buf[:]) } // permBit returns the permission character if the bit is set. func permBit(mode os.FileMode, bit os.FileMode, char byte) byte { if mode&bit != 0 { return char } return '-' } // formatModTime formats time.Time for display, following Unix ls conventions. // // Returns "-" for zero time. Otherwise returns a 12-character string: // recent files (within 6 months) show "Jan 02 15:04", // older or future files show "Jan 02 2006". // // The output uses the timezone embedded in t (UTC for IPFS metadata). func formatModTime(t time.Time) string { if t.IsZero() { return "-" } // Format: "Jan 02 15:04" for times within the last 6 months // Format: "Jan 02 2006" for older times (similar to ls) now := time.Now() sixMonthsAgo := now.AddDate(0, -6, 0) if t.After(sixMonthsAgo) && t.Before(now.Add(24*time.Hour)) { return t.Format("Jan 02 15:04") } return t.Format("Jan 02 2006") } func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash string, ignoreBreaks bool) string { headers, _ := req.Options[lsHeadersOptionNameTime].(bool) stream, _ := req.Options[lsStreamOptionName].(bool) size, _ := req.Options[lsSizeOptionName].(bool) long, _ := req.Options[lsLongOptionName].(bool) // in streaming mode we can't automatically align the tabs // so we take a best guess var minTabWidth int if stream { minTabWidth = 10 } else { minTabWidth = 1 } multipleFolders := len(req.Arguments) > 1 tw := tabwriter.NewWriter(w, minTabWidth, 2, 1, ' ', 0) for _, object := range out.Objects { if !ignoreBreaks && object.Hash != lastObjectHash { if multipleFolders { if lastObjectHash != "" { fmt.Fprintln(tw) } fmt.Fprintf(tw, "%s:\n", object.Hash) } if headers { var s string if long { // Long format: Mode Hash [Size] ModTime Name if size { s = "Mode\tHash\tSize\tModTime\tName" } else { s = "Mode\tHash\tModTime\tName" } } else { // Standard format: Hash [Size] Name if size { s = "Hash\tSize\tName" } else { s = "Hash\tName" } } fmt.Fprintln(tw, s) } lastObjectHash = object.Hash } for _, link := range object.Links { var s string isDir := link.Type == unixfs.TDirectory || link.Type == unixfs.THAMTShard || link.Type == unixfs.TMetadata if long { // Long format: Mode Hash Size ModTime Name var mode string if link.Mode == 0 { // No mode metadata preserved. Show "-" to indicate // "not available" rather than "----------" (mode 0000). mode = "-" } else { mode = formatMode(link.Mode) } modTime := formatModTime(link.ModTime) if isDir { if size { s = "%s\t%s\t-\t%s\t%s/\n" } else { s = "%s\t%s\t%s\t%s/\n" } fmt.Fprintf(tw, s, mode, link.Hash, modTime, cmdenv.EscNonPrint(link.Name)) } else { if size { s = "%s\t%s\t%v\t%s\t%s\n" fmt.Fprintf(tw, s, mode, link.Hash, link.Size, modTime, cmdenv.EscNonPrint(link.Name)) } else { s = "%s\t%s\t%s\t%s\n" fmt.Fprintf(tw, s, mode, link.Hash, modTime, cmdenv.EscNonPrint(link.Name)) } } } else { // Standard format: Hash [Size] Name switch { case isDir: if size { s = "%[1]s\t-\t%[3]s/\n" } else { s = "%[1]s\t%[3]s/\n" } default: if size { s = "%s\t%v\t%s\n" } else { s = "%[1]s\t%[3]s\n" } } fmt.Fprintf(tw, s, link.Hash, link.Size, cmdenv.EscNonPrint(link.Name)) } } } tw.Flush() return lastObjectHash } ================================================ FILE: core/commands/ls_test.go ================================================ package commands import ( "os" "testing" "time" "github.com/stretchr/testify/assert" ) func TestFormatMode(t *testing.T) { t.Parallel() tests := []struct { name string mode os.FileMode expected string }{ // File types { name: "regular file with rw-r--r--", mode: 0644, expected: "-rw-r--r--", }, { name: "regular file with rwxr-xr-x", mode: 0755, expected: "-rwxr-xr-x", }, { name: "regular file with no permissions", mode: 0, expected: "----------", }, { name: "regular file with full permissions", mode: 0777, expected: "-rwxrwxrwx", }, { name: "directory with rwxr-xr-x", mode: os.ModeDir | 0755, expected: "drwxr-xr-x", }, { name: "directory with rwx------", mode: os.ModeDir | 0700, expected: "drwx------", }, { name: "symlink with rwxrwxrwx", mode: os.ModeSymlink | 0777, expected: "lrwxrwxrwx", }, { name: "named pipe with rw-r--r--", mode: os.ModeNamedPipe | 0644, expected: "prw-r--r--", }, { name: "socket with rw-rw-rw-", mode: os.ModeSocket | 0666, expected: "srw-rw-rw-", }, { name: "block device with rw-rw----", mode: os.ModeDevice | 0660, expected: "brw-rw----", }, { name: "character device with rw-rw-rw-", mode: os.ModeDevice | os.ModeCharDevice | 0666, expected: "crw-rw-rw-", }, // Special permission bits - setuid { name: "setuid with execute", mode: os.ModeSetuid | 0755, expected: "-rwsr-xr-x", }, { name: "setuid without execute", mode: os.ModeSetuid | 0644, expected: "-rwSr--r--", }, // Special permission bits - setgid { name: "setgid with execute", mode: os.ModeSetgid | 0755, expected: "-rwxr-sr-x", }, { name: "setgid without execute", mode: os.ModeSetgid | 0745, expected: "-rwxr-Sr-x", }, // Special permission bits - sticky { name: "sticky with execute", mode: os.ModeSticky | 0755, expected: "-rwxr-xr-t", }, { name: "sticky without execute", mode: os.ModeSticky | 0754, expected: "-rwxr-xr-T", }, // Combined special bits { name: "setuid + setgid + sticky all with execute", mode: os.ModeSetuid | os.ModeSetgid | os.ModeSticky | 0777, expected: "-rwsrwsrwt", }, { name: "setuid + setgid + sticky none with execute", mode: os.ModeSetuid | os.ModeSetgid | os.ModeSticky | 0666, expected: "-rwSrwSrwT", }, // Directory with special bits { name: "directory with sticky bit", mode: os.ModeDir | os.ModeSticky | 0755, expected: "drwxr-xr-t", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { t.Parallel() result := formatMode(tc.mode) assert.Equal(t, tc.expected, result) }) } } func TestFormatModTime(t *testing.T) { t.Parallel() t.Run("zero time returns dash", func(t *testing.T) { t.Parallel() result := formatModTime(time.Time{}) assert.Equal(t, "-", result) }) t.Run("old time shows year format", func(t *testing.T) { t.Parallel() // Use a time clearly in the past (more than 6 months ago) oldTime := time.Date(2020, time.March, 15, 10, 30, 0, 0, time.UTC) result := formatModTime(oldTime) // Format: "Jan 02 2006" (note: two spaces before year) assert.Equal(t, "Mar 15 2020", result) }) t.Run("very old time shows year format", func(t *testing.T) { t.Parallel() veryOldTime := time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) result := formatModTime(veryOldTime) assert.Equal(t, "Jan 01 2000", result) }) t.Run("future time shows year format", func(t *testing.T) { t.Parallel() // Times more than 24h in the future should show year format futureTime := time.Now().AddDate(1, 0, 0) result := formatModTime(futureTime) // Should contain the future year assert.Contains(t, result, " ") // two spaces before year assert.Regexp(t, `^[A-Z][a-z]{2} \d{2} \d{4}$`, result) // matches "Mon DD YYYY" assert.Contains(t, result, futureTime.Format("2006")) // contains the year }) t.Run("format lengths are consistent", func(t *testing.T) { t.Parallel() // Both formats should produce 12-character strings for alignment oldTime := time.Date(2020, time.March, 15, 10, 30, 0, 0, time.UTC) oldResult := formatModTime(oldTime) assert.Len(t, oldResult, 12, "old time format should be 12 chars") // Recent time: use 1 month ago to ensure it's always within the 6-month window recentTime := time.Now().AddDate(0, -1, 0) recentResult := formatModTime(recentTime) assert.Len(t, recentResult, 12, "recent time format should be 12 chars") }) } ================================================ FILE: core/commands/mount_nofuse.go ================================================ //go:build !windows && nofuse package commands import ( cmds "github.com/ipfs/go-ipfs-cmds" ) var MountCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Mounts ipfs to the filesystem (disabled).", ShortDescription: ` This version of ipfs is compiled without fuse support, which is required for mounting. If you'd like to be able to mount, please use a version of Kubo compiled with fuse. For the latest instructions, please check the project's repository: http://github.com/ipfs/kubo https://github.com/ipfs/kubo/blob/master/docs/fuse.md `, }, } ================================================ FILE: core/commands/mount_unix.go ================================================ //go:build !windows && !nofuse package commands import ( "fmt" "io" oldcmds "github.com/ipfs/kubo/commands" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" nodeMount "github.com/ipfs/kubo/fuse/node" cmds "github.com/ipfs/go-ipfs-cmds" config "github.com/ipfs/kubo/config" ) const ( mountIPFSPathOptionName = "ipfs-path" mountIPNSPathOptionName = "ipns-path" mountMFSPathOptionName = "mfs-path" ) var MountCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Mounts IPFS to the filesystem (read-only).", ShortDescription: ` Mount IPFS at a read-only mountpoint on the OS (default: /ipfs, /ipns, /mfs). All IPFS objects will be accessible under that directory. Note that the root will not be listable, as it is virtual. Access known paths directly. You may have to create /ipfs and /ipns before using 'ipfs mount': > sudo mkdir /ipfs /ipns /mfs > sudo chown $(whoami) /ipfs /ipns /mfs > ipfs daemon & > ipfs mount `, LongDescription: ` Mount IPFS at a read-only mountpoint on the OS. The default, /ipfs and /ipns, are set in the configuration file, but can be overridden by the options. All IPFS objects will be accessible under this directory. Note that the root will not be listable, as it is virtual. Access known paths directly. You may have to create /ipfs and /ipns before using 'ipfs mount': > sudo mkdir /ipfs /ipns /mfs > sudo chown $(whoami) /ipfs /ipns /mfs > ipfs daemon & > ipfs mount Example: # setup > mkdir foo > echo "baz" > foo/bar > ipfs add -r foo added QmWLdkp93sNxGRjnFHPaYg8tCQ35NBY3XPn6KiETd3Z4WR foo/bar added QmSh5e7S6fdcu75LAbXNZAFY2nGyZUJXyLCJDvn2zRkWyC foo > ipfs ls QmSh5e7S6fdcu75LAbXNZAFY2nGyZUJXyLCJDvn2zRkWyC QmWLdkp93sNxGRjnFHPaYg8tCQ35NBY3XPn6KiETd3Z4WR 12 bar > ipfs cat QmWLdkp93sNxGRjnFHPaYg8tCQ35NBY3XPn6KiETd3Z4WR baz # mount > ipfs daemon & > ipfs mount IPFS mounted at: /ipfs IPNS mounted at: /ipns MFS mounted at: /mfs > cd /ipfs/QmSh5e7S6fdcu75LAbXNZAFY2nGyZUJXyLCJDvn2zRkWyC > ls bar > cat bar baz > cat /ipfs/QmSh5e7S6fdcu75LAbXNZAFY2nGyZUJXyLCJDvn2zRkWyC/bar baz > cat /ipfs/QmWLdkp93sNxGRjnFHPaYg8tCQ35NBY3XPn6KiETd3Z4WR baz `, }, Options: []cmds.Option{ cmds.StringOption(mountIPFSPathOptionName, "f", "The path where IPFS should be mounted."), cmds.StringOption(mountIPNSPathOptionName, "n", "The path where IPNS should be mounted."), cmds.StringOption(mountMFSPathOptionName, "m", "The path where MFS should be mounted."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cfg, err := env.(*oldcmds.Context).GetConfig() if err != nil { return err } nd, err := cmdenv.GetNode(env) if err != nil { return err } // error if we aren't running node in online mode if !nd.IsOnline { return ErrNotOnline } fsdir, found := req.Options[mountIPFSPathOptionName].(string) if !found { fsdir = cfg.Mounts.IPFS // use default value } // get default mount points nsdir, found := req.Options[mountIPNSPathOptionName].(string) if !found { nsdir = cfg.Mounts.IPNS // NB: be sure to not redeclare! } mfsdir, found := req.Options[mountMFSPathOptionName].(string) if !found { mfsdir = cfg.Mounts.MFS } err = nodeMount.Mount(nd, fsdir, nsdir, mfsdir) if err != nil { return err } var output config.Mounts output.IPFS = fsdir output.IPNS = nsdir output.MFS = mfsdir return cmds.EmitOnce(res, &output) }, Type: config.Mounts{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, mounts *config.Mounts) error { fmt.Fprintf(w, "IPFS mounted at: %s\n", cmdenv.EscNonPrint(mounts.IPFS)) fmt.Fprintf(w, "IPNS mounted at: %s\n", cmdenv.EscNonPrint(mounts.IPNS)) fmt.Fprintf(w, "MFS mounted at: %s\n", cmdenv.EscNonPrint(mounts.MFS)) return nil }), }, } ================================================ FILE: core/commands/mount_windows.go ================================================ package commands import ( "errors" cmds "github.com/ipfs/go-ipfs-cmds" ) var MountCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Not yet implemented on Windows.", ShortDescription: "Not yet implemented on Windows. :(", }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { return errors.New("Mount isn't compatible with Windows yet") }, } ================================================ FILE: core/commands/multibase.go ================================================ package commands import ( "bytes" "fmt" "io" "strings" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/kubo/core/commands/cmdenv" mbase "github.com/multiformats/go-multibase" ) var MbaseCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Encode and decode files or stdin with multibase format", }, Subcommands: map[string]*cmds.Command{ "encode": mbaseEncodeCmd, "decode": mbaseDecodeCmd, "transcode": mbaseTranscodeCmd, "list": basesCmd, }, Extra: CreateCmdExtras(SetDoesNotUseRepo(true)), } const ( mbaseOptionName = "b" ) var mbaseEncodeCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Encode data into multibase string", LongDescription: ` This command expects a file name or data provided via stdin. By default it will use URL-safe base64url encoding, but one can customize used base with -b: > echo hello | ipfs multibase encode -b base16 > output_file > cat output_file f68656c6c6f0a > echo hello > input_file > ipfs multibase encode -b base16 input_file f68656c6c6f0a `, }, Arguments: []cmds.Argument{ cmds.FileArg("file", true, false, "data to encode").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption(mbaseOptionName, "multibase encoding").WithDefault("base64url"), }, Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { if err := req.ParseBodyArgs(); err != nil { return err } encoderName, _ := req.Options[mbaseOptionName].(string) encoder, err := mbase.EncoderByName(encoderName) if err != nil { return err } files := req.Files.Entries() file, err := cmdenv.GetFileArg(files) if err != nil { return fmt.Errorf("failed to access file: %w", err) } buf, err := io.ReadAll(file) if err != nil { return fmt.Errorf("failed to read file contents: %w", err) } encoded := encoder.Encode(buf) reader := strings.NewReader(encoded) return resp.Emit(reader) }, } var mbaseDecodeCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Decode multibase string", LongDescription: ` This command expects multibase inside of a file or via stdin: > echo -n hello | ipfs multibase encode -b base16 > file > cat file f68656c6c6f > ipfs multibase decode file hello > cat file | ipfs multibase decode hello `, }, Arguments: []cmds.Argument{ cmds.FileArg("encoded_file", true, false, "encoded data to decode").EnableStdin(), }, Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { if err := req.ParseBodyArgs(); err != nil { return err } files := req.Files.Entries() file, err := cmdenv.GetFileArg(files) if err != nil { return fmt.Errorf("failed to access file: %w", err) } encodedData, err := io.ReadAll(file) if err != nil { return fmt.Errorf("failed to read file contents: %w", err) } _, data, err := mbase.Decode(string(encodedData)) if err != nil { return fmt.Errorf("failed to decode multibase: %w", err) } reader := bytes.NewReader(data) return resp.Emit(reader) }, } var mbaseTranscodeCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Transcode multibase string between bases", LongDescription: ` This command expects multibase inside of a file or via stdin. By default it will use URL-safe base64url encoding, but one can customize used base with -b: > echo -n hello | ipfs multibase encode > file > cat file uaGVsbG8 > ipfs multibase transcode file -b base16 > transcoded_file > cat transcoded_file f68656c6c6f `, }, Arguments: []cmds.Argument{ cmds.FileArg("encoded_file", true, false, "encoded data to decode").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption(mbaseOptionName, "multibase encoding").WithDefault("base64url"), }, Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error { if err := req.ParseBodyArgs(); err != nil { return err } encoderName, _ := req.Options[mbaseOptionName].(string) encoder, err := mbase.EncoderByName(encoderName) if err != nil { return err } files := req.Files.Entries() file, err := cmdenv.GetFileArg(files) if err != nil { return fmt.Errorf("failed to access file: %w", err) } encodedData, err := io.ReadAll(file) if err != nil { return fmt.Errorf("failed to read file contents: %w", err) } _, data, err := mbase.Decode(string(encodedData)) if err != nil { return fmt.Errorf("failed to decode multibase: %w", err) } encoded := encoder.Encode(data) reader := strings.NewReader(encoded) return resp.Emit(reader) }, } ================================================ FILE: core/commands/name/ipns.go ================================================ package name import ( "errors" "fmt" "io" "strings" "time" "github.com/ipfs/boxo/namesys" "github.com/ipfs/boxo/path" cmds "github.com/ipfs/go-ipfs-cmds" logging "github.com/ipfs/go-log/v2" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" options "github.com/ipfs/kubo/core/coreiface/options" ) var log = logging.Logger("core/commands/ipns") type ResolvedPath struct { Path string } const ( recursiveOptionName = "recursive" nocacheOptionName = "nocache" dhtRecordCountOptionName = "dht-record-count" dhtTimeoutOptionName = "dht-timeout" streamOptionName = "stream" ) var IpnsCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Resolve IPNS names.", ShortDescription: ` IPNS is a PKI namespace, where names are the hashes of public keys, and the private key enables publishing new (signed) values. In both publish and resolve, the default name used is the node's own PeerID, which is the hash of its public key. `, LongDescription: ` IPNS is a PKI namespace, where names are the hashes of public keys, and the private key enables publishing new (signed) values. In both publish and resolve, the default name used is the node's own PeerID, which is the hash of its public key. You can use the 'ipfs key' commands to list and generate more names and their respective keys. Examples: Resolve the value of your name: > ipfs name resolve /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy Resolve the value of another name: > ipfs name resolve QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ /ipfs/QmSiTko9JZyabH56y2fussEt1A5oDqsFXB3CkvAqraFryz Resolve the value of a dnslink: > ipfs name resolve ipfs.io /ipfs/QmaBvfZooxWkrv7D3r8LS9moNjzD2o525XMZze69hhoxf5 `, }, Arguments: []cmds.Argument{ cmds.StringArg("name", false, false, "The IPNS name to resolve. Defaults to your node's peerID."), }, Options: []cmds.Option{ cmds.BoolOption(recursiveOptionName, "r", "Resolve until the result is not an IPNS name.").WithDefault(true), cmds.BoolOption(nocacheOptionName, "n", "Do not use cached entries."), cmds.UintOption(dhtRecordCountOptionName, "dhtrc", "Number of records to request for DHT resolution.").WithDefault(uint(namesys.DefaultResolverDhtRecordCount)), cmds.StringOption(dhtTimeoutOptionName, "dhtt", "Max time to collect values during DHT resolution e.g. \"30s\". Pass 0 for no timeout.").WithDefault(namesys.DefaultResolverDhtTimeout.String()), cmds.BoolOption(streamOptionName, "s", "Stream entries as they are found."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } nocache, _ := req.Options["nocache"].(bool) var name string if len(req.Arguments) == 0 { self, err := api.Key().Self(req.Context) if err != nil { return err } name = self.ID().String() } else { name = req.Arguments[0] } recursive, _ := req.Options[recursiveOptionName].(bool) rc, rcok := req.Options[dhtRecordCountOptionName].(uint) dhtt, dhttok := req.Options[dhtTimeoutOptionName].(string) stream, _ := req.Options[streamOptionName].(bool) opts := []options.NameResolveOption{ options.Name.Cache(!nocache), } if !recursive { opts = append(opts, options.Name.ResolveOption(namesys.ResolveWithDepth(1))) } if rcok { opts = append(opts, options.Name.ResolveOption(namesys.ResolveWithDhtRecordCount(rc))) } if dhttok { d, err := time.ParseDuration(dhtt) if err != nil { return err } if d < 0 { return errors.New("DHT timeout value must be >= 0") } opts = append(opts, options.Name.ResolveOption(namesys.ResolveWithDhtTimeout(d))) } if !strings.HasPrefix(name, "/ipns/") { name = "/ipns/" + name } if !stream { output, err := api.Name().Resolve(req.Context, name, opts...) if err != nil && (recursive || err != namesys.ErrResolveRecursion) { return err } pth, err := path.NewPath(output.String()) if err != nil { return err } return cmds.EmitOnce(res, &ResolvedPath{pth.String()}) } output, err := api.Name().Search(req.Context, name, opts...) if err != nil { return err } for v := range output { if v.Err != nil && (recursive || v.Err != namesys.ErrResolveRecursion) { return v.Err } if err := res.Emit(&ResolvedPath{v.Path.String()}); err != nil { return err } } return nil }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, rp *ResolvedPath) error { _, err := fmt.Fprintln(w, rp.Path) return err }), }, Type: ResolvedPath{}, } ================================================ FILE: core/commands/name/ipnsps.go ================================================ package name import ( "fmt" "io" "strings" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/kubo/core/commands/cmdenv" ke "github.com/ipfs/kubo/core/commands/keyencode" record "github.com/libp2p/go-libp2p-record" "github.com/libp2p/go-libp2p/core/peer" ) type ipnsPubsubState struct { Enabled bool } type ipnsPubsubCancel struct { Canceled bool } type stringList struct { Strings []string } // IpnsPubsubCmd is the subcommand that allows us to manage the IPNS pubsub system var IpnsPubsubCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "IPNS pubsub management", ShortDescription: ` Manage and inspect the state of the IPNS pubsub resolver. Note: this command is experimental and subject to change as the system is refined `, }, Subcommands: map[string]*cmds.Command{ "state": ipnspsStateCmd, "subs": ipnspsSubsCmd, "cancel": ipnspsCancelCmd, }, } var ipnspsStateCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Query the state of IPNS pubsub.", }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := cmdenv.GetNode(env) if err != nil { return err } return cmds.EmitOnce(res, &ipnsPubsubState{n.PSRouter != nil}) }, Type: ipnsPubsubState{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, ips *ipnsPubsubState) error { var state string if ips.Enabled { state = "enabled" } else { state = "disabled" } _, err := fmt.Fprintln(w, state) return err }), }, } var ipnspsSubsCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Show current name subscriptions.", }, Options: []cmds.Option{ ke.OptionIPNSBase, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { keyEnc, err := ke.KeyEncoderFromString(req.Options[ke.OptionIPNSBase.Name()].(string)) if err != nil { return err } n, err := cmdenv.GetNode(env) if err != nil { return err } if n.PSRouter == nil { return cmds.Errorf(cmds.ErrClient, "IPNS pubsub subsystem is not enabled") } var paths []string for _, key := range n.PSRouter.GetSubscriptions() { ns, k, err := record.SplitKey(key) if err != nil || ns != "ipns" { // Not necessarily an error. continue } pid, err := peer.IDFromBytes([]byte(k)) if err != nil { log.Errorf("ipns key not a valid peer ID: %s", err) continue } paths = append(paths, "/ipns/"+keyEnc.FormatID(pid)) } return cmds.EmitOnce(res, &stringList{paths}) }, Type: stringList{}, Encoders: cmds.EncoderMap{ cmds.Text: stringListEncoder(), }, } var ipnspsCancelCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Cancel a name subscription.", }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := cmdenv.GetNode(env) if err != nil { return err } if n.PSRouter == nil { return cmds.Errorf(cmds.ErrClient, "IPNS pubsub subsystem is not enabled") } name := req.Arguments[0] name = strings.TrimPrefix(name, "/ipns/") pid, err := peer.Decode(name) if err != nil { return cmds.Errorf(cmds.ErrClient, "not a valid IPNS name: %s", err) } ok, err := n.PSRouter.Cancel("/ipns/" + string(pid)) if err != nil { return err } return cmds.EmitOnce(res, &ipnsPubsubCancel{ok}) }, Arguments: []cmds.Argument{ cmds.StringArg("name", true, false, "Name to cancel the subscription for."), }, Type: ipnsPubsubCancel{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, ipc *ipnsPubsubCancel) error { var state string if ipc.Canceled { state = "canceled" } else { state = "no subscription" } _, err := fmt.Fprintln(w, state) return err }), }, } func stringListEncoder() cmds.EncoderFunc { return cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, list *stringList) error { for _, s := range list.Strings { _, err := fmt.Fprintln(w, s) if err != nil { return err } } return nil }) } ================================================ FILE: core/commands/name/name.go ================================================ package name import ( "bytes" "encoding/hex" "errors" "fmt" "io" "strings" "text/tabwriter" "time" "github.com/ipfs/boxo/ipns" ipns_pb "github.com/ipfs/boxo/ipns/pb" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/coreiface/options" "google.golang.org/protobuf/proto" ) type IpnsEntry struct { Name string Value string } var NameCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Publish and resolve IPNS names.", ShortDescription: ` IPNS is a PKI namespace, where names are the hashes of public keys, and the private key enables publishing new (signed) values. In both publish and resolve, the default name used is the node's own PeerID, which is the hash of its public key. `, LongDescription: ` IPNS is a PKI namespace, where names are the hashes of public keys, and the private key enables publishing new (signed) values. In both publish and resolve, the default name used is the node's own PeerID, which is the hash of its public key. You can use the 'ipfs key' commands to list and generate more names and their respective keys. Examples: Publish an with your default name: > ipfs name publish /ipfs/bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4 Published to k51qzi5uqu5dgklc20hksmmzhoy5lfrn5xcnryq6xp4r50b5yc0vnivpywfu9p: /ipfs/bafk... Publish an with another name, added by an 'ipfs key' command: > ipfs key gen --type=ed25519 mykey k51qzi5uqu5dlz49qkb657myg6f1buu6rauv8c6b489a9i1e4dkt7a3yo9j2wr > ipfs name publish --key=mykey /ipfs/bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4 Published to k51qzi5uqu5dlz49qkb657myg6f1buu6rauv8c6b489a9i1e4dkt7a3yo9j2wr: /ipfs/bafk... Resolve the value of your name: > ipfs name resolve /ipfs/bafk... Resolve the value of another name: > ipfs name resolve k51qzi5uqu5dlz49qkb657myg6f1buu6rauv8c6b489a9i1e4dkt7a3yo9j2wr /ipfs/bafk... Resolve the value of a dnslink: > ipfs name resolve specs.ipfs.tech /ipfs/bafy... `, }, Subcommands: map[string]*cmds.Command{ "publish": PublishCmd, "resolve": IpnsCmd, "pubsub": IpnsPubsubCmd, "inspect": IpnsInspectCmd, "get": IpnsGetCmd, "put": IpnsPutCmd, }, } type IpnsInspectValidation struct { Valid bool Reason string Name string } // IpnsInspectEntry contains the deserialized values from an IPNS Entry: // https://github.com/ipfs/specs/blob/main/ipns/IPNS.md#record-serialization-format type IpnsInspectEntry struct { Value string ValidityType *ipns.ValidityType Validity *time.Time Sequence *uint64 TTL *time.Duration } type IpnsInspectResult struct { Entry IpnsInspectEntry PbSize int SignatureType string HexDump string Validation *IpnsInspectValidation } var IpnsInspectCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Inspects an IPNS Record", ShortDescription: ` Prints values inside of IPNS Record protobuf and its DAG-CBOR Data field. Passing --verify will verify signature against provided public key. `, LongDescription: ` Prints values inside of IPNS Record protobuf and its DAG-CBOR Data field. The input can be a file or STDIN, the output can be JSON: $ ipfs routing get "/ipns/$PEERID" > ipns_record $ ipfs name inspect --enc=json < ipns_record Values in PublicKey, SignatureV1 and SignatureV2 fields are raw bytes encoded in Multibase. The Data field is DAG-CBOR represented as DAG-JSON. Passing --verify will verify signature against provided public key. `, HTTP: &cmds.HTTPHelpText{ Description: "Request body should be `multipart/form-data` with the IPNS record bytes.", }, }, Arguments: []cmds.Argument{ cmds.FileArg("record", true, false, "The IPNS record payload to be verified.").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption("verify", "CID of the public IPNS key to validate against."), cmds.BoolOption("dump", "Include a full hex dump of the raw Protobuf record.").WithDefault(true), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { file, err := cmdenv.GetFileArg(req.Files.Entries()) if err != nil { return err } defer file.Close() var b bytes.Buffer _, err = io.Copy(&b, file) if err != nil { return err } rec, err := ipns.UnmarshalRecord(b.Bytes()) if err != nil { return err } result := &IpnsInspectResult{ Entry: IpnsInspectEntry{}, } // Best effort to get the fields. Show everything we can. if v, err := rec.Value(); err == nil { result.Entry.Value = v.String() } if v, err := rec.ValidityType(); err == nil { result.Entry.ValidityType = &v } if v, err := rec.Validity(); err == nil { result.Entry.Validity = &v } if v, err := rec.Sequence(); err == nil { result.Entry.Sequence = &v } if v, err := rec.TTL(); err == nil { result.Entry.TTL = &v } // Here we need the raw protobuf just to decide the version. var pbRecord ipns_pb.IpnsRecord err = proto.Unmarshal(b.Bytes(), &pbRecord) if err != nil { return err } if len(pbRecord.SignatureV1) != 0 || len(pbRecord.Value) != 0 { result.SignatureType = "V1+V2" } else if pbRecord.Data != nil { result.SignatureType = "V2" } else { result.SignatureType = "Unknown" } result.PbSize = proto.Size(&pbRecord) if verify, ok := req.Options["verify"].(string); ok { name, err := ipns.NameFromString(verify) if err != nil { return err } result.Validation = &IpnsInspectValidation{ Name: name.String(), } err = ipns.ValidateWithName(rec, name) if err == nil { result.Validation.Valid = true } else { result.Validation.Reason = err.Error() } } if dump, ok := req.Options["dump"].(bool); ok && dump { result.HexDump = hex.Dump(b.Bytes()) } return cmds.EmitOnce(res, result) }, Type: IpnsInspectResult{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *IpnsInspectResult) error { tw := tabwriter.NewWriter(w, 0, 0, 1, ' ', 0) defer tw.Flush() if out.Entry.Value != "" { fmt.Fprintf(tw, "Value:\t%q\n", out.Entry.Value) } if out.Entry.ValidityType != nil { fmt.Fprintf(tw, "Validity Type:\t%d\n", *out.Entry.ValidityType) } if out.Entry.Validity != nil { fmt.Fprintf(tw, "Validity:\t%q\n", out.Entry.Validity.Format(time.RFC3339Nano)) } if out.Entry.Sequence != nil { fmt.Fprintf(tw, "Sequence:\t%d\n", *out.Entry.Sequence) } if out.Entry.TTL != nil { fmt.Fprintf(tw, "TTL:\t%s\n", out.Entry.TTL.String()) } fmt.Fprintf(tw, "Protobuf Size:\t%d\n", out.PbSize) fmt.Fprintf(tw, "Signature Type:\t%s\n", out.SignatureType) if out.Validation == nil { tw.Flush() fmt.Fprintf(w, "\nThis record was not verified. Pass '--verify k51...' to verify.\n") } else { tw.Flush() fmt.Fprintf(w, "\nValidation results:\n") fmt.Fprintf(tw, "\tValid:\t%v\n", out.Validation.Valid) if out.Validation.Reason != "" { fmt.Fprintf(tw, "\tReason:\t%s\n", out.Validation.Reason) } fmt.Fprintf(tw, "\tName:\t%s\n", out.Validation.Name) } if out.HexDump != "" { tw.Flush() fmt.Fprintf(w, "\nHex Dump:\n%s", out.HexDump) } return nil }), }, } var IpnsGetCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Retrieve a signed IPNS record.", ShortDescription: ` Retrieves the signed IPNS record for a given name from the routing system. The output is the raw IPNS record (protobuf) as defined in the IPNS spec: https://specs.ipfs.tech/ipns/ipns-record/ The record can be inspected with 'ipfs name inspect': ipfs name get | ipfs name inspect This is equivalent to 'ipfs routing get /ipns/' but only accepts IPNS names (not arbitrary routing keys). Note: The routing system returns the "best" IPNS record it knows about. For IPNS, "best" means the record with the highest sequence number. If multiple records exist (e.g., after using 'ipfs name put'), this command returns the one the routing system considers most current. `, HTTP: &cmds.HTTPHelpText{ ResponseContentType: "application/vnd.ipfs.ipns-record", }, }, Arguments: []cmds.Argument{ cmds.StringArg("name", true, false, "The IPNS name to look up."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } // Normalize the argument: accept both "k51..." and "/ipns/k51..." name := req.Arguments[0] if !strings.HasPrefix(name, "/ipns/") { name = "/ipns/" + name } data, err := api.Routing().Get(req.Context, name) if err != nil { return err } res.SetEncodingType(cmds.OctetStream) res.SetContentType("application/vnd.ipfs.ipns-record") return res.Emit(bytes.NewReader(data)) }, } const ( forceOptionName = "force" putAllowOfflineOption = "allow-offline" allowDelegatedOption = "allow-delegated" putQuietOptionName = "quiet" maxIPNSRecordSize = 10 << 10 // 10 KiB per IPNS spec ) var errPutAllowOffline = errors.New("can't put while offline: pass `--allow-offline` to store locally or `--allow-delegated` if Ipns.DelegatedPublishers are set up") var IpnsPutCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Store a pre-signed IPNS record in the routing system.", ShortDescription: ` Stores a pre-signed IPNS record in the routing system. This command accepts a raw IPNS record (protobuf) as defined in the IPNS spec: https://specs.ipfs.tech/ipns/ipns-record/ The record must be signed by the private key corresponding to the IPNS name. Use 'ipfs name get' to retrieve records and 'ipfs name inspect' to examine. `, LongDescription: ` Stores a pre-signed IPNS record in the routing system. This command accepts a raw IPNS record (protobuf) as defined in the IPNS spec: https://specs.ipfs.tech/ipns/ipns-record/ The record must be signed by the private key corresponding to the IPNS name. Use 'ipfs name get' to retrieve records and 'ipfs name inspect' to examine. Use Cases: - Re-publishing third-party records: store someone else's signed record - Cross-node sync: import records exported from another node - Backup/restore: export with 'name get', restore with 'name put' Validation: By default, the command validates that: - The record is a valid IPNS record (protobuf) - The record size is within 10 KiB limit - The signature matches the provided IPNS name - The record's sequence number is higher than any existing record (identical records are allowed for republishing) The --force flag skips this command's validation and passes the record directly to the routing system. Note that --force only affects this command; it does not control how the routing system handles the record. The routing system may still reject invalid records or prefer records with higher sequence numbers. Use --force primarily for testing (e.g., to observe how the routing system reacts to incorrectly signed or malformed records). Important: Even after a successful 'name put', a subsequent 'name get' may return a different record if one with a higher sequence number exists. This is expected IPNS behavior, not a bug. Publishing Modes: By default, IPNS records are published to both the DHT and any configured HTTP delegated publishers. You can control this behavior with: --allow-offline Store locally without requiring network connectivity --allow-delegated Publish via HTTP delegated publishers only (no DHT) Examples: Export and re-import a record: > ipfs name get k51... > record.bin > ipfs name put k51... record.bin Store a record received from someone else: > ipfs name put k51... third-party-record.bin Force store a record to test routing validation: > ipfs name put --force k51... possibly-invalid-record.bin `, HTTP: &cmds.HTTPHelpText{ Description: "Request body should be `multipart/form-data` with the IPNS record bytes.", }, }, Arguments: []cmds.Argument{ cmds.StringArg("name", true, false, "The IPNS name to store the record for (e.g., k51... or /ipns/k51...)."), cmds.FileArg("record", true, false, "Path to file containing the signed IPNS record.").EnableStdin(), }, Options: []cmds.Option{ cmds.BoolOption(forceOptionName, "f", "Skip validation (signature, sequence, size)."), cmds.BoolOption(putAllowOfflineOption, "Store locally without broadcasting to the network."), cmds.BoolOption(allowDelegatedOption, "Publish via HTTP delegated publishers only (no DHT)."), cmds.BoolOption(putQuietOptionName, "q", "Write no output."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } api, err := cmdenv.GetApi(env, req) if err != nil { return err } // Parse options force, _ := req.Options[forceOptionName].(bool) allowOffline, _ := req.Options[putAllowOfflineOption].(bool) allowDelegated, _ := req.Options[allowDelegatedOption].(bool) // Validate flag combinations if allowOffline && allowDelegated { return errors.New("cannot use both --allow-offline and --allow-delegated flags") } // Handle different publishing modes if allowDelegated { // AllowDelegated mode: check if delegated publishers are configured cfg, err := nd.Repo.Config() if err != nil { return fmt.Errorf("failed to read config: %w", err) } delegatedPublishers := cfg.DelegatedPublishersWithAutoConf() if len(delegatedPublishers) == 0 { return errors.New("no delegated publishers configured: add Ipns.DelegatedPublishers or use --allow-offline for local-only publishing") } // For allow-delegated mode, we proceed even if offline // since we're using HTTP publishing via delegated publishers } // Parse the IPNS name argument nameArg := req.Arguments[0] if !strings.HasPrefix(nameArg, "/ipns/") { nameArg = "/ipns/" + nameArg } // Extract the name part after /ipns/ namePart := strings.TrimPrefix(nameArg, "/ipns/") name, err := ipns.NameFromString(namePart) if err != nil { return fmt.Errorf("invalid IPNS name: %w", err) } // Read raw record bytes from file/stdin file, err := cmdenv.GetFileArg(req.Files.Entries()) if err != nil { return err } defer file.Close() // Read record data (limit to 1 MiB for memory safety) data, err := io.ReadAll(io.LimitReader(file, 1<<20)) if err != nil { return fmt.Errorf("failed to read record: %w", err) } if len(data) == 0 { return errors.New("record is empty") } // Validate unless --force if !force { // Check size limit per IPNS spec if len(data) > maxIPNSRecordSize { return fmt.Errorf("record exceeds maximum size of %d bytes, use --force to skip size check", maxIPNSRecordSize) } rec, err := ipns.UnmarshalRecord(data) if err != nil { return fmt.Errorf("invalid IPNS record: %w", err) } // Validate signature against provided name err = ipns.ValidateWithName(rec, name) if err != nil { return fmt.Errorf("record validation failed: %w", err) } // Check for sequence conflicts with existing record existingData, err := api.Routing().Get(req.Context, nameArg) if err == nil { // Allow republishing the exact same record (common use case: // get a third-party record and put it back to refresh DHT) if !bytes.Equal(existingData, data) { existingRec, parseErr := ipns.UnmarshalRecord(existingData) if parseErr == nil { existingSeq, seqErr := existingRec.Sequence() newSeq, newSeqErr := rec.Sequence() if seqErr == nil && newSeqErr == nil && existingSeq >= newSeq { return fmt.Errorf("existing IPNS record has sequence %d >= new record sequence %d, use 'ipfs name put --force' to skip this check", existingSeq, newSeq) } } } } // If Get fails (no existing record), that's fine - proceed with put } // Publish the original bytes as-is // When allowDelegated is true, we set allowOffline to allow the operation // even without DHT connectivity (delegated publishers use HTTP) opts := []options.RoutingPutOption{ options.Routing.AllowOffline(allowOffline || allowDelegated), } err = api.Routing().Put(req.Context, nameArg, data, opts...) if err != nil { if err.Error() == "can't put while offline" { return errPutAllowOffline } return err } // Extract value from the record for the response value := "" if rec, err := ipns.UnmarshalRecord(data); err == nil { if v, err := rec.Value(); err == nil { value = v.String() } } return cmds.EmitOnce(res, &IpnsEntry{ Name: name.String(), Value: value, }) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, ie *IpnsEntry) error { quiet, _ := req.Options[putQuietOptionName].(bool) if quiet { return nil } _, err := fmt.Fprintln(w, cmdenv.EscNonPrint(ie.Name)) return err }), }, Type: IpnsEntry{}, } ================================================ FILE: core/commands/name/publish.go ================================================ package name import ( "errors" "fmt" "io" "time" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" ipns "github.com/ipfs/boxo/ipns" cmds "github.com/ipfs/go-ipfs-cmds" ke "github.com/ipfs/kubo/core/commands/keyencode" iface "github.com/ipfs/kubo/core/coreiface" options "github.com/ipfs/kubo/core/coreiface/options" ) var errAllowOffline = errors.New("can't publish while offline: pass `--allow-offline` to override or `--allow-delegated` if Ipns.DelegatedPublishers are set up") const ( ipfsPathOptionName = "ipfs-path" resolveOptionName = "resolve" allowOfflineOptionName = "allow-offline" allowDelegatedOptionName = "allow-delegated" lifeTimeOptionName = "lifetime" ttlOptionName = "ttl" keyOptionName = "key" quieterOptionName = "quieter" v1compatOptionName = "v1compat" sequenceOptionName = "sequence" ) var PublishCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Publish IPNS names.", ShortDescription: ` IPNS is a PKI namespace, where names are the hashes of public keys, and the private key enables publishing new (signed) values. In both publish and resolve, the default name used is the node's own PeerID, which is the hash of its public key. `, LongDescription: ` IPNS is a PKI namespace, where names are the hashes of public keys, and the private key enables publishing new (signed) values. In both publish and resolve, the default name used is the node's own PeerID, which is the hash of its public key. You can use the 'ipfs key' commands to list and generate more names and their respective keys. Publishing Modes: By default, IPNS records are published to both the DHT and any configured HTTP delegated publishers. You can control this behavior with the following flags: --allow-offline Allow publishing when offline (publishes to local datastore, network operations are optional) --allow-delegated Allow publishing without DHT connectivity (local + HTTP delegated publishers only) Examples: Publish an with your default name: > ipfs name publish /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy Published to QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n: /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy Publish without DHT (HTTP delegated publishers only): > ipfs name publish --allow-delegated /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy Published to QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n: /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy Publish when offline (local publish, network optional): > ipfs name publish --allow-offline /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy Published to QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n: /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy Notes: The --ttl option specifies the time duration for caching IPNS records. Lower values like '1m' enable faster updates but increase network load, while the default of 1 hour reduces traffic but may delay propagation. Gateway operators may override this with Ipns.MaxCacheTTL configuration. The --sequence option sets a custom sequence number for the IPNS record. The sequence number must be monotonically increasing (greater than the current record's sequence). This is useful for manually coordinating updates across multiple writers. If not specified, the sequence number increments automatically. For faster IPNS updates, consider: - Using a lower --ttl value (e.g., '1m' for quick updates) - Enabling PubSub via Ipns.UsePubsub in the config `, }, Arguments: []cmds.Argument{ cmds.StringArg(ipfsPathOptionName, true, false, "ipfs path of the object to be published.").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption(keyOptionName, "k", "Name of the key to be used or a valid PeerID, as listed by 'ipfs key list -l'.").WithDefault("self"), cmds.BoolOption(resolveOptionName, "Check if the given path can be resolved before publishing.").WithDefault(true), cmds.StringOption(lifeTimeOptionName, "t", `Time duration the signed record will be valid for. Accepts durations such as "300s", "1.5h" or "7d2h45m"`).WithDefault(ipns.DefaultRecordLifetime.String()), cmds.StringOption(ttlOptionName, "Time duration hint, akin to --lifetime, indicating how long to cache this record before checking for updates.").WithDefault(ipns.DefaultRecordTTL.String()), cmds.BoolOption(quieterOptionName, "Q", "Write only final IPNS Name encoded as CIDv1 (for use in /ipns content paths)."), cmds.BoolOption(v1compatOptionName, "Produce a backward-compatible IPNS Record by including fields for both V1 and V2 signatures.").WithDefault(true), cmds.BoolOption(allowOfflineOptionName, "Allow publishing when offline - publishes to local datastore without requiring network connectivity."), cmds.BoolOption(allowDelegatedOptionName, "Allow publishing without DHT connectivity - uses local datastore and HTTP delegated publishers only."), cmds.Uint64Option(sequenceOptionName, "Set a custom sequence number for the IPNS record (must be higher than current)."), ke.OptionIPNSBase, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } allowOffline, _ := req.Options[allowOfflineOptionName].(bool) allowDelegated, _ := req.Options[allowDelegatedOptionName].(bool) compatibleWithV1, _ := req.Options[v1compatOptionName].(bool) kname, _ := req.Options[keyOptionName].(string) // Validate flag combinations if allowOffline && allowDelegated { return errors.New("cannot use both --allow-offline and --allow-delegated flags") } validTimeOpt, _ := req.Options[lifeTimeOptionName].(string) validTime, err := time.ParseDuration(validTimeOpt) if err != nil { return fmt.Errorf("error parsing lifetime option: %s", err) } opts := []options.NamePublishOption{ options.Name.AllowOffline(allowOffline), options.Name.AllowDelegated(allowDelegated), options.Name.Key(kname), options.Name.ValidTime(validTime), options.Name.CompatibleWithV1(compatibleWithV1), } if ttl, found := req.Options[ttlOptionName].(string); found { d, err := time.ParseDuration(ttl) if err != nil { return err } opts = append(opts, options.Name.TTL(d)) } if sequence, found := req.Options[sequenceOptionName].(uint64); found { opts = append(opts, options.Name.Sequence(sequence)) } p, err := cmdutils.PathOrCidPath(req.Arguments[0]) if err != nil { return err } if verifyExists, _ := req.Options[resolveOptionName].(bool); verifyExists { _, err := api.ResolveNode(req.Context, p) if err != nil { return err } } name, err := api.Name().Publish(req.Context, p, opts...) if err != nil { if err == iface.ErrOffline { err = errAllowOffline } return err } return cmds.EmitOnce(res, &IpnsEntry{ Name: name.String(), Value: p.String(), }) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, ie *IpnsEntry) error { var err error quieter, _ := req.Options[quieterOptionName].(bool) if quieter { _, err = fmt.Fprintln(w, cmdenv.EscNonPrint(ie.Name)) } else { _, err = fmt.Fprintf(w, "Published to %s: %s\n", cmdenv.EscNonPrint(ie.Name), cmdenv.EscNonPrint(ie.Value)) } return err }), }, Type: IpnsEntry{}, } ================================================ FILE: core/commands/object/diff.go ================================================ package objectcmd import ( "fmt" "io" "github.com/ipfs/boxo/ipld/merkledag/dagutils" "github.com/ipfs/boxo/path" cmds "github.com/ipfs/go-ipfs-cmds" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" ) const ( verboseOptionName = "verbose" ) type Changes struct { Changes []*dagutils.Change } var ObjectDiffCmd = &cmds.Command{ Status: cmds.Deprecated, // https://github.com/ipfs/kubo/issues/7936 Helptext: cmds.HelpText{ Tagline: "Display the diff between two IPFS objects.", ShortDescription: ` 'ipfs object diff' is a command used to show the differences between two IPFS objects.`, LongDescription: ` 'ipfs object diff' is a command used to show the differences between two IPFS objects. Example: > ls foo bar baz/ giraffe > ipfs add -r foo ... Added QmegHcnrPgMwC7tBiMxChD54fgQMBUecNw9nE9UUU4x1bz foo > OBJ_A=QmegHcnrPgMwC7tBiMxChD54fgQMBUecNw9nE9UUU4x1bz > echo "different content" > foo/bar > ipfs add -r foo ... Added QmcmRptkSPWhptCttgHg27QNDmnV33wAJyUkCnAvqD3eCD foo > OBJ_B=QmcmRptkSPWhptCttgHg27QNDmnV33wAJyUkCnAvqD3eCD > ipfs object diff -v $OBJ_A $OBJ_B Changed "bar" from QmNgd5cz2jNftnAHBhcRUGdtiaMzb5Rhjqd4etondHHST8 to QmRfFVsjSXkhFxrfWnLpMae2M4GBVsry6VAuYYcji5MiZb. `, }, Arguments: []cmds.Argument{ cmds.StringArg("obj_a", true, false, "Object to diff against."), cmds.StringArg("obj_b", true, false, "Object to diff."), }, Options: []cmds.Option{ cmds.BoolOption(verboseOptionName, "v", "Print extra information."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } pa, err := cmdutils.PathOrCidPath(req.Arguments[0]) if err != nil { return err } pb, err := cmdutils.PathOrCidPath(req.Arguments[1]) if err != nil { return err } changes, err := api.Object().Diff(req.Context, pa, pb) if err != nil { return err } out := make([]*dagutils.Change, len(changes)) for i, change := range changes { out[i] = &dagutils.Change{ Type: dagutils.ChangeType(change.Type), Path: change.Path, } if (change.Before != path.ImmutablePath{}) { out[i].Before = change.Before.RootCid() } if (change.After != path.ImmutablePath{}) { out[i].After = change.After.RootCid() } } return cmds.EmitOnce(res, &Changes{out}) }, Type: Changes{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *Changes) error { verbose, _ := req.Options[verboseOptionName].(bool) for _, change := range out.Changes { if verbose { switch change.Type { case dagutils.Add: fmt.Fprintf(w, "Added new link %q pointing to %s.\n", change.Path, change.After) case dagutils.Mod: fmt.Fprintf(w, "Changed %q from %s to %s.\n", change.Path, change.Before, change.After) case dagutils.Remove: fmt.Fprintf(w, "Removed link %q (was %s).\n", change.Path, change.Before) } } else { switch change.Type { case dagutils.Add: fmt.Fprintf(w, "+ %s %q\n", change.After, change.Path) case dagutils.Mod: fmt.Fprintf(w, "~ %s %s %q\n", change.Before, change.After, change.Path) case dagutils.Remove: fmt.Fprintf(w, "- %s %q\n", change.Before, change.Path) } } } return nil }), }, } ================================================ FILE: core/commands/object/object.go ================================================ package objectcmd import ( "errors" cmds "github.com/ipfs/go-ipfs-cmds" ) type Link struct { Name, Hash string Size uint64 } type Object struct { Hash string `json:"Hash,omitempty"` Links []Link `json:"Links,omitempty"` } var ErrDataEncoding = errors.New("unknown data field encoding") var ObjectCmd = &cmds.Command{ Status: cmds.Deprecated, // https://github.com/ipfs/kubo/issues/7936 Helptext: cmds.HelpText{ Tagline: "Deprecated commands to interact with dag-pb objects. Use 'dag' or 'files' instead.", ShortDescription: ` 'ipfs object' is a legacy plumbing command used to manipulate dag-pb objects directly. Deprecated, use more modern 'ipfs dag' and 'ipfs files' instead.`, }, Subcommands: map[string]*cmds.Command{ "data": RemovedObjectCmd, "diff": ObjectDiffCmd, "get": RemovedObjectCmd, "links": RemovedObjectCmd, "new": RemovedObjectCmd, "patch": ObjectPatchCmd, "put": RemovedObjectCmd, "stat": RemovedObjectCmd, }, } var RemovedObjectCmd = &cmds.Command{ Status: cmds.Removed, Helptext: cmds.HelpText{ Tagline: "Removed, use 'ipfs dag' or 'ipfs files' instead.", }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { return errors.New("removed, use 'ipfs dag' or 'ipfs files' instead") }, } ================================================ FILE: core/commands/object/patch.go ================================================ package objectcmd import ( "fmt" "io" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" "github.com/ipfs/kubo/core/coreiface/options" ) var ObjectPatchCmd = &cmds.Command{ Status: cmds.Deprecated, // https://github.com/ipfs/kubo/issues/7936 Helptext: cmds.HelpText{ Tagline: "Deprecated way to create a new merkledag object based on an existing one. Use MFS with 'files cp|rm' instead.", ShortDescription: ` 'ipfs object patch ' is a plumbing command used to build custom dag-pb objects. It mutates objects, creating new objects as a result. This is the Merkle-DAG version of modifying an object. DEPRECATED and provided for legacy reasons. For modern use cases, use MFS with 'files' commands: 'ipfs files --help'. $ ipfs files cp /ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn /some-dir $ ipfs files cp /ipfs/Qmayz4F4UzqcAMitTzU4zCSckDofvxstDuj3y7ajsLLEVs /some-dir/added-file.jpg $ ipfs files stat --hash /some-dir The above will add 'added-file.jpg' to the directory placed under /some-dir and the CID of updated directory is returned by 'files stat' 'files cp' does not download the data, only the root block, which makes it possible to build arbitrary directory trees without fetching them in full to the local node. `, }, Arguments: []cmds.Argument{}, Subcommands: map[string]*cmds.Command{ "append-data": RemovedObjectCmd, "add-link": patchAddLinkCmd, "rm-link": patchRmLinkCmd, "set-data": RemovedObjectCmd, }, Options: []cmds.Option{ cmdutils.AllowBigBlockOption, }, } var patchRmLinkCmd = &cmds.Command{ Status: cmds.Deprecated, // https://github.com/ipfs/kubo/issues/7936 Helptext: cmds.HelpText{ Tagline: "Deprecated way to remove a link from dag-pb object.", ShortDescription: ` Remove a Merkle-link from the given object and return the hash of the result. DEPRECATED and provided for legacy reasons. Use 'files rm' instead. `, }, Arguments: []cmds.Argument{ cmds.StringArg("root", true, false, "The hash of the node to modify."), cmds.StringArg("name", true, false, "Name of the link to remove."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } root, err := cmdutils.PathOrCidPath(req.Arguments[0]) if err != nil { return err } name := req.Arguments[1] p, err := api.Object().RmLink(req.Context, root, name) if err != nil { return err } if err := cmdutils.CheckCIDSize(req, p.RootCid(), api.Dag()); err != nil { return err } return cmds.EmitOnce(res, &Object{Hash: p.RootCid().String()}) }, Type: Object{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *Object) error { fmt.Fprintln(w, out.Hash) return nil }), }, } const ( createOptionName = "create" ) var patchAddLinkCmd = &cmds.Command{ Status: cmds.Deprecated, // https://github.com/ipfs/kubo/issues/7936 Helptext: cmds.HelpText{ Tagline: "Deprecated way to add a link to a given dag-pb.", ShortDescription: ` Add a Merkle-link to the given object and return the hash of the result. DEPRECATED and provided for legacy reasons. Use MFS and 'files' commands instead: $ ipfs files cp /ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn /some-dir $ ipfs files cp /ipfs/Qmayz4F4UzqcAMitTzU4zCSckDofvxstDuj3y7ajsLLEVs /some-dir/added-file.jpg $ ipfs files stat --hash /some-dir The above will add 'added-file.jpg' to the directory placed under /some-dir and the CID of updated directory is returned by 'files stat' 'files cp' does not download the data, only the root block, which makes it possible to build arbitrary directory trees without fetching them in full to the local node. `, }, Arguments: []cmds.Argument{ cmds.StringArg("root", true, false, "The hash of the node to modify."), cmds.StringArg("name", true, false, "Name of link to create."), cmds.StringArg("ref", true, false, "IPFS object to add link to."), }, Options: []cmds.Option{ cmds.BoolOption(createOptionName, "p", "Create intermediary nodes."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } root, err := cmdutils.PathOrCidPath(req.Arguments[0]) if err != nil { return err } name := req.Arguments[1] child, err := cmdutils.PathOrCidPath(req.Arguments[2]) if err != nil { return err } create, _ := req.Options[createOptionName].(bool) if err != nil { return err } p, err := api.Object().AddLink(req.Context, root, name, child, options.Object.Create(create)) if err != nil { return err } if err := cmdutils.CheckCIDSize(req, p.RootCid(), api.Dag()); err != nil { return err } return cmds.EmitOnce(res, &Object{Hash: p.RootCid().String()}) }, Type: Object{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *Object) error { fmt.Fprintln(w, out.Hash) return nil }), }, } ================================================ FILE: core/commands/p2p.go ================================================ package commands import ( "context" "errors" "fmt" "io" "strconv" "strings" "text/tabwriter" "time" core "github.com/ipfs/kubo/core" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" p2p "github.com/ipfs/kubo/p2p" cmds "github.com/ipfs/go-ipfs-cmds" peer "github.com/libp2p/go-libp2p/core/peer" pstore "github.com/libp2p/go-libp2p/core/peerstore" protocol "github.com/libp2p/go-libp2p/core/protocol" ma "github.com/multiformats/go-multiaddr" madns "github.com/multiformats/go-multiaddr-dns" ) // P2PProtoPrefix is the default required prefix for protocol names const P2PProtoPrefix = "/x/" // P2PListenerInfoOutput is output type of ls command type P2PListenerInfoOutput struct { Protocol string ListenAddress string TargetAddress string } // P2PStreamInfoOutput is output type of streams command type P2PStreamInfoOutput struct { HandlerID string Protocol string OriginAddress string TargetAddress string } // P2PLsOutput is output type of ls command type P2PLsOutput struct { Listeners []P2PListenerInfoOutput } // P2PStreamsOutput is output type of streams command type P2PStreamsOutput struct { Streams []P2PStreamInfoOutput } // P2PForegroundOutput is output type for foreground mode status messages type P2PForegroundOutput struct { Status string // "active" or "closing" Protocol string Address string } const ( allowCustomProtocolOptionName = "allow-custom-protocol" reportPeerIDOptionName = "report-peer-id" foregroundOptionName = "foreground" ) var resolveTimeout = 10 * time.Second // P2PCmd is the 'ipfs p2p' command var P2PCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Libp2p stream mounting.", ShortDescription: ` Create and use tunnels to remote peers over libp2p Note: this command is experimental and subject to change as usecases and APIs are refined`, }, Subcommands: map[string]*cmds.Command{ "stream": p2pStreamCmd, "forward": p2pForwardCmd, "listen": p2pListenCmd, "close": p2pCloseCmd, "ls": p2pLsCmd, }, } var p2pForwardCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Forward connections to libp2p service.", ShortDescription: ` Forward connections made to to via libp2p. Creates a local TCP listener that tunnels connections through libp2p to a remote peer's p2p listener. Similar to SSH port forwarding (-L flag). ARGUMENTS: Protocol name (must start with '` + P2PProtoPrefix + `') Local multiaddr (e.g., /ip4/127.0.0.1/tcp/3000) Remote peer multiaddr (e.g., /p2p/PeerID) FOREGROUND MODE (--foreground, -f): By default, the forwarder runs in the daemon and the command returns immediately. Use --foreground to block until interrupted: - Ctrl+C or SIGTERM: Removes the forwarder and exits - 'ipfs p2p close': Removes the forwarder and exits - Daemon shutdown: Forwarder is automatically removed Useful for systemd services or scripts that need cleanup on exit. EXAMPLES: # Persistent forwarder (command returns immediately) ipfs p2p forward /x/myapp /ip4/127.0.0.1/tcp/3000 /p2p/PeerID # Temporary forwarder (removed when command exits) ipfs p2p forward -f /x/myapp /ip4/127.0.0.1/tcp/3000 /p2p/PeerID Learn more: https://github.com/ipfs/kubo/blob/master/docs/p2p-tunnels.md `, }, Arguments: []cmds.Argument{ cmds.StringArg("protocol", true, false, "Protocol name."), cmds.StringArg("listen-address", true, false, "Listening endpoint."), cmds.StringArg("target-address", true, false, "Target endpoint."), }, Options: []cmds.Option{ cmds.BoolOption(allowCustomProtocolOptionName, "Don't require /x/ prefix"), cmds.BoolOption(foregroundOptionName, "f", "Run in foreground; forwarder is removed when command exits"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := p2pGetNode(env) if err != nil { return err } protoOpt := req.Arguments[0] listenOpt := req.Arguments[1] targetOpt := req.Arguments[2] proto := protocol.ID(protoOpt) listen, err := ma.NewMultiaddr(listenOpt) if err != nil { return err } targets, err := parseIpfsAddr(targetOpt) if err != nil { return err } allowCustom, _ := req.Options[allowCustomProtocolOptionName].(bool) if !allowCustom && !strings.HasPrefix(string(proto), P2PProtoPrefix) { return errors.New("protocol name must be within '" + P2PProtoPrefix + "' namespace") } listener, err := forwardLocal(n.Context(), n.P2P, n.Peerstore, proto, listen, targets) if err != nil { return err } foreground, _ := req.Options[foregroundOptionName].(bool) if foreground { if err := res.Emit(&P2PForegroundOutput{ Status: "active", Protocol: protoOpt, Address: listenOpt, }); err != nil { return err } // Wait for either context cancellation (Ctrl+C/daemon shutdown) // or listener removal (ipfs p2p close) select { case <-req.Context.Done(): // SIGTERM/Ctrl+C - cleanup silently (CLI stream already closing) n.P2P.ListenersLocal.Close(func(l p2p.Listener) bool { return l == listener }) return nil case <-listener.Done(): // Closed via "ipfs p2p close" - emit closing message return res.Emit(&P2PForegroundOutput{ Status: "closing", Protocol: protoOpt, Address: listenOpt, }) } } return nil }, Type: P2PForegroundOutput{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *P2PForegroundOutput) error { if out.Status == "active" { fmt.Fprintf(w, "Forwarding %s to %s, waiting for interrupt...\n", out.Protocol, out.Address) } else if out.Status == "closing" { fmt.Fprintf(w, "Received interrupt, removing forwarder for %s\n", out.Protocol) } return nil }), }, } // parseIpfsAddr is a function that takes in addr string and return ipfsAddrs func parseIpfsAddr(addr string) (*peer.AddrInfo, error) { multiaddr, err := ma.NewMultiaddr(addr) if err != nil { return nil, err } pi, err := peer.AddrInfoFromP2pAddr(multiaddr) if err == nil { return pi, nil } // resolve multiaddr whose protocol is not ma.P_IPFS ctx, cancel := context.WithTimeout(context.Background(), resolveTimeout) defer cancel() addrs, err := madns.Resolve(ctx, multiaddr) if err != nil { return nil, err } if len(addrs) == 0 { return nil, errors.New("fail to resolve the multiaddr:" + multiaddr.String()) } var info peer.AddrInfo for _, addr := range addrs { taddr, id := peer.SplitAddr(addr) if id == "" { // not an ipfs addr, skipping. continue } switch info.ID { case "": info.ID = id case id: default: return nil, fmt.Errorf( "ambiguous multiaddr %s could refer to %s or %s", multiaddr, info.ID, id, ) } info.Addrs = append(info.Addrs, taddr) } return &info, nil } var p2pListenCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Create libp2p service.", ShortDescription: ` Create a libp2p protocol handler that forwards incoming connections to . When a remote peer connects using 'ipfs p2p forward', the connection is forwarded to your local service. Similar to SSH port forwarding (server side). ARGUMENTS: Protocol name (must start with '` + P2PProtoPrefix + `') Local multiaddr (e.g., /ip4/127.0.0.1/tcp/3000) FOREGROUND MODE (--foreground, -f): By default, the listener runs in the daemon and the command returns immediately. Use --foreground to block until interrupted: - Ctrl+C or SIGTERM: Removes the listener and exits - 'ipfs p2p close': Removes the listener and exits - Daemon shutdown: Listener is automatically removed Useful for systemd services or scripts that need cleanup on exit. EXAMPLES: # Persistent listener (command returns immediately) ipfs p2p listen /x/myapp /ip4/127.0.0.1/tcp/3000 # Temporary listener (removed when command exits) ipfs p2p listen -f /x/myapp /ip4/127.0.0.1/tcp/3000 # Report connecting peer ID to the target application ipfs p2p listen -r /x/myapp /ip4/127.0.0.1/tcp/3000 Learn more: https://github.com/ipfs/kubo/blob/master/docs/p2p-tunnels.md `, }, Arguments: []cmds.Argument{ cmds.StringArg("protocol", true, false, "Protocol name."), cmds.StringArg("target-address", true, false, "Target endpoint."), }, Options: []cmds.Option{ cmds.BoolOption(allowCustomProtocolOptionName, "Don't require /x/ prefix"), cmds.BoolOption(reportPeerIDOptionName, "r", "Send remote base58 peerid to target when a new connection is established"), cmds.BoolOption(foregroundOptionName, "f", "Run in foreground; listener is removed when command exits"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := p2pGetNode(env) if err != nil { return err } protoOpt := req.Arguments[0] targetOpt := req.Arguments[1] proto := protocol.ID(protoOpt) target, err := ma.NewMultiaddr(targetOpt) if err != nil { return err } // port can't be 0 if err := checkPort(target); err != nil { return err } allowCustom, _ := req.Options[allowCustomProtocolOptionName].(bool) reportPeerID, _ := req.Options[reportPeerIDOptionName].(bool) if !allowCustom && !strings.HasPrefix(string(proto), P2PProtoPrefix) { return errors.New("protocol name must be within '" + P2PProtoPrefix + "' namespace") } listener, err := n.P2P.ForwardRemote(n.Context(), proto, target, reportPeerID) if err != nil { return err } foreground, _ := req.Options[foregroundOptionName].(bool) if foreground { if err := res.Emit(&P2PForegroundOutput{ Status: "active", Protocol: protoOpt, Address: targetOpt, }); err != nil { return err } // Wait for either context cancellation (Ctrl+C/daemon shutdown) // or listener removal (ipfs p2p close) select { case <-req.Context.Done(): // SIGTERM/Ctrl+C - cleanup silently (CLI stream already closing) n.P2P.ListenersP2P.Close(func(l p2p.Listener) bool { return l == listener }) return nil case <-listener.Done(): // Closed via "ipfs p2p close" - emit closing message return res.Emit(&P2PForegroundOutput{ Status: "closing", Protocol: protoOpt, Address: targetOpt, }) } } return nil }, Type: P2PForegroundOutput{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *P2PForegroundOutput) error { if out.Status == "active" { fmt.Fprintf(w, "Listening on %s, forwarding to %s, waiting for interrupt...\n", out.Protocol, out.Address) } else if out.Status == "closing" { fmt.Fprintf(w, "Received interrupt, removing listener for %s\n", out.Protocol) } return nil }), }, } // checkPort checks whether target multiaddr contains tcp or udp protocol // and whether the port is equal to 0 func checkPort(target ma.Multiaddr) error { // get tcp or udp port from multiaddr getPort := func() (string, error) { sport, _ := target.ValueForProtocol(ma.P_TCP) if sport != "" { return sport, nil } sport, _ = target.ValueForProtocol(ma.P_UDP) if sport != "" { return sport, nil } return "", errors.New("address does not contain tcp or udp protocol") } sport, err := getPort() if err != nil { return err } port, err := strconv.Atoi(sport) if err != nil { return err } if port == 0 { return errors.New("port can not be 0") } return nil } // forwardLocal forwards local connections to a libp2p service func forwardLocal(ctx context.Context, p *p2p.P2P, ps pstore.Peerstore, proto protocol.ID, bindAddr ma.Multiaddr, addr *peer.AddrInfo) (p2p.Listener, error) { ps.AddAddrs(addr.ID, addr.Addrs, pstore.TempAddrTTL) return p.ForwardLocal(ctx, addr.ID, proto, bindAddr) } const ( p2pHeadersOptionName = "headers" ) var p2pLsCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "List active p2p listeners.", }, Options: []cmds.Option{ cmds.BoolOption(p2pHeadersOptionName, "v", "Print table headers (Protocol, Listen, Target)."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := p2pGetNode(env) if err != nil { return err } output := &P2PLsOutput{} n.P2P.ListenersLocal.Lock() for _, listener := range n.P2P.ListenersLocal.Listeners { output.Listeners = append(output.Listeners, P2PListenerInfoOutput{ Protocol: string(listener.Protocol()), ListenAddress: listener.ListenAddress().String(), TargetAddress: listener.TargetAddress().String(), }) } n.P2P.ListenersLocal.Unlock() n.P2P.ListenersP2P.Lock() for _, listener := range n.P2P.ListenersP2P.Listeners { output.Listeners = append(output.Listeners, P2PListenerInfoOutput{ Protocol: string(listener.Protocol()), ListenAddress: listener.ListenAddress().String(), TargetAddress: listener.TargetAddress().String(), }) } n.P2P.ListenersP2P.Unlock() return cmds.EmitOnce(res, output) }, Type: P2PLsOutput{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *P2PLsOutput) error { headers, _ := req.Options[p2pHeadersOptionName].(bool) tw := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) for _, listener := range out.Listeners { if headers { fmt.Fprintln(tw, "Protocol\tListen Address\tTarget Address") } fmt.Fprintf(tw, "%s\t%s\t%s\n", listener.Protocol, listener.ListenAddress, listener.TargetAddress) } tw.Flush() return nil }), }, } const ( p2pAllOptionName = "all" p2pProtocolOptionName = "protocol" p2pListenAddressOptionName = "listen-address" p2pTargetAddressOptionName = "target-address" ) var p2pCloseCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Stop listening for new connections to forward.", }, Options: []cmds.Option{ cmds.BoolOption(p2pAllOptionName, "a", "Close all listeners."), cmds.StringOption(p2pProtocolOptionName, "p", "Match protocol name"), cmds.StringOption(p2pListenAddressOptionName, "l", "Match listen address"), cmds.StringOption(p2pTargetAddressOptionName, "t", "Match target address"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := p2pGetNode(env) if err != nil { return err } closeAll, _ := req.Options[p2pAllOptionName].(bool) protoOpt, p := req.Options[p2pProtocolOptionName].(string) listenOpt, l := req.Options[p2pListenAddressOptionName].(string) targetOpt, t := req.Options[p2pTargetAddressOptionName].(string) proto := protocol.ID(protoOpt) var target, listen ma.Multiaddr if l { listen, err = ma.NewMultiaddr(listenOpt) if err != nil { return err } } if t { target, err = ma.NewMultiaddr(targetOpt) if err != nil { return err } } if !(closeAll || p || l || t) { return errors.New("no matching options given") } if closeAll && (p || l || t) { return errors.New("can't combine --all with other matching options") } match := func(listener p2p.Listener) bool { if closeAll { return true } if p && proto != listener.Protocol() { return false } if l && !listen.Equal(listener.ListenAddress()) { return false } if t && !target.Equal(listener.TargetAddress()) { return false } return true } done := n.P2P.ListenersLocal.Close(match) done += n.P2P.ListenersP2P.Close(match) return cmds.EmitOnce(res, done) }, Type: int(0), Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out int) error { fmt.Fprintf(w, "Closed %d stream(s)\n", out) return nil }), }, } /////// // Stream // // p2pStreamCmd is the 'ipfs p2p stream' command var p2pStreamCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "P2P stream management.", ShortDescription: "Create and manage p2p streams", }, Subcommands: map[string]*cmds.Command{ "ls": p2pStreamLsCmd, "close": p2pStreamCloseCmd, }, } var p2pStreamLsCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "List active p2p streams.", }, Options: []cmds.Option{ cmds.BoolOption(p2pHeadersOptionName, "v", "Print table headers (ID, Protocol, Local, Remote)."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := p2pGetNode(env) if err != nil { return err } output := &P2PStreamsOutput{} n.P2P.Streams.Lock() for id, s := range n.P2P.Streams.Streams { output.Streams = append(output.Streams, P2PStreamInfoOutput{ HandlerID: strconv.FormatUint(id, 10), Protocol: string(s.Protocol), OriginAddress: s.OriginAddr.String(), TargetAddress: s.TargetAddr.String(), }) } n.P2P.Streams.Unlock() return cmds.EmitOnce(res, output) }, Type: P2PStreamsOutput{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *P2PStreamsOutput) error { headers, _ := req.Options[p2pHeadersOptionName].(bool) tw := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) for _, stream := range out.Streams { if headers { fmt.Fprintln(tw, "ID\tProtocol\tOrigin\tTarget") } fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", stream.HandlerID, stream.Protocol, stream.OriginAddress, stream.TargetAddress) } tw.Flush() return nil }), }, } var p2pStreamCloseCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Close active p2p stream.", }, Arguments: []cmds.Argument{ cmds.StringArg("id", false, false, "Stream identifier"), }, Options: []cmds.Option{ cmds.BoolOption(p2pAllOptionName, "a", "Close all streams."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := p2pGetNode(env) if err != nil { return err } closeAll, _ := req.Options[p2pAllOptionName].(bool) var handlerID uint64 if !closeAll { if len(req.Arguments) == 0 { return errors.New("no id specified") } handlerID, err = strconv.ParseUint(req.Arguments[0], 10, 64) if err != nil { return err } } toClose := make([]*p2p.Stream, 0, 1) n.P2P.Streams.Lock() for id, stream := range n.P2P.Streams.Streams { if !closeAll && handlerID != id { continue } toClose = append(toClose, stream) if !closeAll { break } } n.P2P.Streams.Unlock() for _, s := range toClose { n.P2P.Streams.Reset(s) } return nil }, } func p2pGetNode(env cmds.Environment) (*core.IpfsNode, error) { nd, err := cmdenv.GetNode(env) if err != nil { return nil, err } config, err := nd.Repo.Config() if err != nil { return nil, err } if !config.Experimental.Libp2pStreamMounting { return nil, errors.New("libp2p stream mounting not enabled") } if !nd.IsOnline { return nil, ErrNotOnline } return nd, nil } ================================================ FILE: core/commands/pin/pin.go ================================================ package pin import ( "context" "encoding/json" "fmt" "io" "os" "time" "github.com/dustin/go-humanize" bserv "github.com/ipfs/boxo/blockservice" offline "github.com/ipfs/boxo/exchange/offline" dag "github.com/ipfs/boxo/ipld/merkledag" pin "github.com/ipfs/boxo/pinning/pinner" verifcid "github.com/ipfs/boxo/verifcid" cid "github.com/ipfs/go-cid" cidenc "github.com/ipfs/go-cidutil/cidenc" cmds "github.com/ipfs/go-ipfs-cmds" coreiface "github.com/ipfs/kubo/core/coreiface" options "github.com/ipfs/kubo/core/coreiface/options" core "github.com/ipfs/kubo/core" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" e "github.com/ipfs/kubo/core/commands/e" ) var PinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Pin (and unpin) objects to local storage.", }, Subcommands: map[string]*cmds.Command{ "add": addPinCmd, "rm": rmPinCmd, "ls": listPinCmd, "verify": verifyPinCmd, "update": updatePinCmd, "remote": remotePinCmd, }, } type PinOutput struct { Pins []string } type AddPinOutput struct { Pins []string `json:",omitempty"` Progress int `json:",omitempty"` Bytes uint64 `json:",omitempty"` } const ( pinRecursiveOptionName = "recursive" pinProgressOptionName = "progress" ) var addPinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Pin objects to local storage.", ShortDescription: "Stores an IPFS object(s) from a given path locally to disk.", LongDescription: ` Create a pin for the given object, protecting resolved CID from being garbage collected. An optional name can be provided, and read back via 'ipfs pin ls --names'. Be mindful of defaults: Default pin type is 'recursive' (entire DAG). Pass -r=false to create a direct pin for a single block. Use 'pin ls -t recursive' to only list roots of recursively pinned DAGs (significantly faster when many big DAGs are pinned recursively) Default pin name is empty. Pass '--name' to 'pin add' to set one and use 'pin ls --names' to see it. Pinning a second time with a different name will update the name of the pin. If daemon is running, any missing blocks will be retrieved from the network. It may take some time. Pass '--progress' to track the progress. `, }, Arguments: []cmds.Argument{ cmds.StringArg("ipfs-path", true, true, "Path to object(s) to be pinned.").EnableStdin(), }, Options: []cmds.Option{ cmds.BoolOption(pinRecursiveOptionName, "r", "Recursively pin the object linked to by the specified object(s).").WithDefault(true), cmds.StringOption(pinNameOptionName, "n", "An optional name for created pin(s)."), cmds.BoolOption(pinProgressOptionName, "Show progress"), }, Type: AddPinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } // set recursive flag recursive, _ := req.Options[pinRecursiveOptionName].(bool) name, _ := req.Options[pinNameOptionName].(string) showProgress, _ := req.Options[pinProgressOptionName].(bool) // Validate pin name if err := cmdutils.ValidatePinName(name); err != nil { return err } if err := req.ParseBodyArgs(); err != nil { return err } enc, err := cmdenv.GetCidEncoder(req) if err != nil { return err } if !showProgress { added, err := pinAddMany(req.Context, api, enc, req.Arguments, recursive, name) if err != nil { return err } return cmds.EmitOnce(res, &AddPinOutput{Pins: added}) } v := new(dag.ProgressTracker) ctx := v.DeriveContext(req.Context) type pinResult struct { pins []string err error } ch := make(chan pinResult, 1) go func() { added, err := pinAddMany(ctx, api, enc, req.Arguments, recursive, name) ch <- pinResult{pins: added, err: err} }() ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() for { select { case val := <-ch: if val.err != nil { return val.err } if ps := v.ProgressStat(); ps.Nodes != 0 { if err := res.Emit(&AddPinOutput{Progress: ps.Nodes, Bytes: ps.Bytes}); err != nil { return err } } return res.Emit(&AddPinOutput{Pins: val.pins}) case <-ticker.C: ps := v.ProgressStat() if err := res.Emit(&AddPinOutput{Progress: ps.Nodes, Bytes: ps.Bytes}); err != nil { return err } case <-ctx.Done(): log.Error(ctx.Err()) return ctx.Err() } } }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *AddPinOutput) error { rec, found := req.Options["recursive"].(bool) var pintype string if rec || !found { pintype = "recursively" } else { pintype = "directly" } for _, k := range out.Pins { fmt.Fprintf(w, "pinned %s %s\n", k, pintype) } return nil }), }, PostRun: cmds.PostRunMap{ cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { for { v, err := res.Next() if err != nil { if err == io.EOF { return nil } return err } out, ok := v.(*AddPinOutput) if !ok { return e.TypeErr(out, v) } if out.Pins == nil { // this can only happen if the progress option is set fmt.Fprintf(os.Stderr, "Fetched/Processed %d nodes (%s)\r", out.Progress, humanize.Bytes(out.Bytes)) } else { err = re.Emit(out) if err != nil { return err } } } }, }, } func pinAddMany(ctx context.Context, api coreiface.CoreAPI, enc cidenc.Encoder, paths []string, recursive bool, name string) ([]string, error) { added := make([]string, len(paths)) for i, b := range paths { p, err := cmdutils.PathOrCidPath(b) if err != nil { return nil, err } rp, _, err := api.ResolvePath(ctx, p) if err != nil { return nil, err } if err := api.Pin().Add(ctx, rp, options.Pin.Recursive(recursive), options.Pin.Name(name)); err != nil { return nil, err } added[i] = enc.Encode(rp.RootCid()) } return added, nil } var rmPinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Remove object from pin-list.", ShortDescription: ` Removes the pin from the given object allowing it to be garbage collected if needed. (By default, recursively. Use -r=false for direct pins.) `, LongDescription: ` Removes the pin from the given object allowing it to be garbage collected if needed. (By default, recursively. Use -r=false for direct pins.) A pin may not be removed because the specified object is not pinned or pinned indirectly. To determine if the object is pinned indirectly, use the command: ipfs pin ls -t indirect `, }, Arguments: []cmds.Argument{ cmds.StringArg("ipfs-path", true, true, "Path to object(s) to be unpinned.").EnableStdin(), }, Options: []cmds.Option{ cmds.BoolOption(pinRecursiveOptionName, "r", "Recursively unpin the object linked to by the specified object(s).").WithDefault(true), }, Type: PinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } // set recursive flag recursive, _ := req.Options[pinRecursiveOptionName].(bool) if err := req.ParseBodyArgs(); err != nil { return err } enc, err := cmdenv.GetCidEncoder(req) if err != nil { return err } pins := make([]string, 0, len(req.Arguments)) for _, b := range req.Arguments { p, err := cmdutils.PathOrCidPath(b) if err != nil { return err } rp, _, err := api.ResolvePath(req.Context, p) if err != nil { return err } id := enc.Encode(rp.RootCid()) pins = append(pins, id) if err := api.Pin().Rm(req.Context, rp, options.Pin.RmRecursive(recursive)); err != nil { return err } } return cmds.EmitOnce(res, &PinOutput{pins}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinOutput) error { for _, k := range out.Pins { fmt.Fprintf(w, "unpinned %s\n", k) } return nil }), }, } const ( pinTypeOptionName = "type" pinQuietOptionName = "quiet" pinStreamOptionName = "stream" pinNamesOptionName = "names" ) var listPinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List objects pinned to local storage.", ShortDescription: ` Returns a list of objects that are pinned locally. By default, all pinned objects are returned, but the '--type' flag or arguments can restrict that to a specific pin type or to some specific objects respectively. `, LongDescription: ` Returns a list of objects that are pinned locally. By default, all pinned objects are returned, but the '--type' flag or arguments can restrict that to a specific pin type or to some specific objects respectively. Use --type= to specify the type of pinned keys to list. Valid values are: * "direct": pin that specific object. * "recursive": pin that specific object, and indirectly pin all its descendants * "indirect": pinned indirectly by an ancestor (like a refcount) * "all" By default, pin names are not included (returned as empty). Pass '--names' flag to return pin names (set with '--name' from 'pin add'). With arguments, the command fails if any of the arguments is not a pinned object. And if --type= is additionally used, the command will also fail if any of the arguments is not of the specified type. Example: $ echo "hello" | ipfs add -q QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN $ ipfs pin ls QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN recursive # now remove the pin, and repin it directly $ ipfs pin rm QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN unpinned QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN $ ipfs pin add -r=false QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN pinned QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN directly $ ipfs pin ls --type=direct QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN direct $ ipfs pin ls QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN direct `, }, Arguments: []cmds.Argument{ cmds.StringArg("ipfs-path", false, true, "Path to object(s) to be listed."), }, Options: []cmds.Option{ cmds.StringOption(pinTypeOptionName, "t", "The type of pinned keys to list. Can be \"direct\", \"indirect\", \"recursive\", or \"all\".").WithDefault("all"), cmds.BoolOption(pinQuietOptionName, "q", "Output only the CIDs of pins."), cmds.StringOption(pinNameOptionName, "n", "Limit returned pins to ones with names that contain the value provided (case-sensitive, partial match). Implies --names=true."), cmds.BoolOption(pinStreamOptionName, "s", "Enable streaming of pins as they are discovered."), cmds.BoolOption(pinNamesOptionName, "Include pin names in the output (slower, disabled by default)."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } n, err := cmdenv.GetNode(env) if err != nil { return err } if n.Pinning == nil { return fmt.Errorf("pinning service not available") } typeStr, _ := req.Options[pinTypeOptionName].(string) stream, _ := req.Options[pinStreamOptionName].(bool) displayNames, _ := req.Options[pinNamesOptionName].(bool) name, _ := req.Options[pinNameOptionName].(string) // Validate name filter if err := cmdutils.ValidatePinName(name); err != nil { return err } mode, ok := pin.StringToMode(typeStr) if !ok { return fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", typeStr) } // For backward compatibility, we accumulate the pins in the same output type as before. var emit func(PinLsOutputWrapper) error lgcList := map[string]PinLsType{} if !stream { emit = func(v PinLsOutputWrapper) error { lgcList[v.PinLsObject.Cid] = PinLsType{Type: v.PinLsObject.Type, Name: v.PinLsObject.Name} return nil } } else { emit = func(v PinLsOutputWrapper) error { return res.Emit(v) } } if len(req.Arguments) > 0 { err = pinLsKeys(req, mode, displayNames || name != "", n.Pinning, api, emit) } else { err = pinLsAll(req, typeStr, displayNames || name != "", name, api, emit) } if err != nil { return err } if !stream { return cmds.EmitOnce(res, PinLsOutputWrapper{ PinLsList: PinLsList{Keys: lgcList}, }) } return nil }, Type: PinLsOutputWrapper{}, Encoders: cmds.EncoderMap{ cmds.JSON: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out PinLsOutputWrapper) error { stream, _ := req.Options[pinStreamOptionName].(bool) enc := json.NewEncoder(w) if stream { return enc.Encode(out.PinLsObject) } return enc.Encode(out.PinLsList) }), cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out PinLsOutputWrapper) error { quiet, _ := req.Options[pinQuietOptionName].(bool) stream, _ := req.Options[pinStreamOptionName].(bool) if stream { if quiet { fmt.Fprintf(w, "%s\n", out.PinLsObject.Cid) } else if out.PinLsObject.Name == "" { fmt.Fprintf(w, "%s %s\n", out.PinLsObject.Cid, out.PinLsObject.Type) } else { fmt.Fprintf(w, "%s %s %s\n", out.PinLsObject.Cid, out.PinLsObject.Type, out.PinLsObject.Name) } return nil } for k, v := range out.PinLsList.Keys { if quiet { fmt.Fprintf(w, "%s\n", k) } else if v.Name == "" { fmt.Fprintf(w, "%s %s\n", k, v.Type) } else { fmt.Fprintf(w, "%s %s %s\n", k, v.Type, v.Name) } } return nil }), }, } // PinLsOutputWrapper is the output type of the pin ls command. // Pin ls needs to output two different type depending on if it's streamed or not. // We use this to bypass the cmds lib refusing to have interface{} type PinLsOutputWrapper struct { PinLsList PinLsObject } // PinLsList is a set of pins with their type type PinLsList struct { Keys map[string]PinLsType `json:",omitempty"` } // PinLsType contains the type of a pin type PinLsType struct { Type string Name string } // PinLsObject contains the description of a pin type PinLsObject struct { Cid string `json:",omitempty"` Name string `json:",omitempty"` Type string `json:",omitempty"` } func pinLsKeys(req *cmds.Request, mode pin.Mode, displayNames bool, pinner pin.Pinner, api coreiface.CoreAPI, emit func(value PinLsOutputWrapper) error) error { enc, err := cmdenv.GetCidEncoder(req) if err != nil { return err } // Collect CIDs to check cids := make([]cid.Cid, 0, len(req.Arguments)) for _, p := range req.Arguments { p, err := cmdutils.PathOrCidPath(p) if err != nil { return err } rp, _, err := api.ResolvePath(req.Context, p) if err != nil { return err } cids = append(cids, rp.RootCid()) } // Check pins using the new type-specific method pinned, err := pinner.CheckIfPinnedWithType(req.Context, mode, displayNames, cids...) if err != nil { return err } // Process results for i, p := range pinned { if !p.Pinned() { return fmt.Errorf("path '%s' is not pinned", req.Arguments[i]) } pinType, _ := pin.ModeToString(p.Mode) if p.Mode == pin.Indirect && p.Via.Defined() { pinType = "indirect through " + enc.Encode(p.Via) } err = emit(PinLsOutputWrapper{ PinLsObject: PinLsObject{ Type: pinType, Cid: enc.Encode(cids[i]), Name: p.Name, }, }) if err != nil { return err } } return nil } func pinLsAll(req *cmds.Request, typeStr string, detailed bool, name string, api coreiface.CoreAPI, emit func(value PinLsOutputWrapper) error) error { enc, err := cmdenv.GetCidEncoder(req) if err != nil { return err } _, ok := pin.StringToMode(typeStr) if !ok { return fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", typeStr) } opt, err := options.Pin.Ls.Type(typeStr) if err != nil { return err } pins := make(chan coreiface.Pin) lsErr := make(chan error, 1) lsCtx, cancel := context.WithCancel(req.Context) defer cancel() go func() { lsErr <- api.Pin().Ls(lsCtx, pins, opt, options.Pin.Ls.Detailed(detailed), options.Pin.Ls.Name(name)) }() for p := range pins { err = emit(PinLsOutputWrapper{ PinLsObject: PinLsObject{ Type: p.Type(), Name: p.Name(), Cid: enc.Encode(p.Path().RootCid()), }, }) if err != nil { return err } } return <-lsErr } const ( pinUnpinOptionName = "unpin" ) var updatePinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Update a recursive pin.", ShortDescription: ` Efficiently pins a new object based on differences from an existing one and, by default, removes the old pin. This command is useful when the new pin contains many similarities or is a derivative of an existing one, particularly for large objects. This allows a more efficient DAG-traversal which fully skips already-pinned branches from the old object. As a requirement, the old object needs to be an existing recursive pin. `, }, Arguments: []cmds.Argument{ cmds.StringArg("from-path", true, false, "Path to old object."), cmds.StringArg("to-path", true, false, "Path to a new object to be pinned."), }, Options: []cmds.Option{ cmds.BoolOption(pinUnpinOptionName, "Remove the old pin.").WithDefault(true), }, Type: PinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } enc, err := cmdenv.GetCidEncoder(req) if err != nil { return err } unpin, _ := req.Options[pinUnpinOptionName].(bool) fromPath, err := cmdutils.PathOrCidPath(req.Arguments[0]) if err != nil { return err } toPath, err := cmdutils.PathOrCidPath(req.Arguments[1]) if err != nil { return err } // Resolve the paths ahead of time so we can return the actual CIDs from, _, err := api.ResolvePath(req.Context, fromPath) if err != nil { return err } to, _, err := api.ResolvePath(req.Context, toPath) if err != nil { return err } err = api.Pin().Update(req.Context, from, to, options.Pin.Unpin(unpin)) if err != nil { return err } return cmds.EmitOnce(res, &PinOutput{Pins: []string{enc.Encode(from.RootCid()), enc.Encode(to.RootCid())}}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinOutput) error { fmt.Fprintf(w, "updated %s to %s\n", out.Pins[0], out.Pins[1]) return nil }), }, } const ( pinVerboseOptionName = "verbose" ) var verifyPinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Verify that recursive pins are complete.", }, Options: []cmds.Option{ cmds.BoolOption(pinVerboseOptionName, "Also write the hashes of non-broken pins."), cmds.BoolOption(pinQuietOptionName, "q", "Write just hashes of broken pins."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := cmdenv.GetNode(env) if err != nil { return err } verbose, _ := req.Options[pinVerboseOptionName].(bool) quiet, _ := req.Options[pinQuietOptionName].(bool) if verbose && quiet { return fmt.Errorf("the --verbose and --quiet options can not be used at the same time") } enc, err := cmdenv.GetCidEncoder(req) if err != nil { return err } opts := pinVerifyOpts{ explain: !quiet, includeOk: verbose, } out, err := pinVerify(req.Context, n, opts, enc) if err != nil { return err } return res.Emit(out) }, Type: PinVerifyRes{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinVerifyRes) error { quiet, _ := req.Options[pinQuietOptionName].(bool) if quiet && !out.Ok { fmt.Fprintf(w, "%s\n", out.Cid) } else if !quiet { out.Format(w) } return nil }), }, } // PinVerifyRes is the result returned for each pin checked in "pin verify" type PinVerifyRes struct { Cid string `json:",omitempty"` Err string `json:",omitempty"` PinStatus } // PinStatus is part of PinVerifyRes, do not use directly type PinStatus struct { Ok bool `json:",omitempty"` BadNodes []BadNode `json:",omitempty"` } // BadNode is used in PinVerifyRes type BadNode struct { Cid string Err string } type pinVerifyOpts struct { explain bool includeOk bool } // FIXME: this implementation is duplicated sith core/coreapi.PinAPI.Verify, remove this one and exclusively rely on CoreAPI. func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts, enc cidenc.Encoder) (<-chan any, error) { visited := make(map[cid.Cid]PinStatus) bs := n.Blocks.Blockstore() DAG := dag.NewDAGService(bserv.New(bs, offline.Exchange(bs))) getLinks := dag.GetLinksWithDAG(DAG) var checkPin func(root cid.Cid) PinStatus checkPin = func(root cid.Cid) PinStatus { key := root if status, ok := visited[key]; ok { return status } if err := verifcid.ValidateCid(verifcid.DefaultAllowlist, root); err != nil { status := PinStatus{Ok: false} if opts.explain { status.BadNodes = []BadNode{{Cid: enc.Encode(key), Err: err.Error()}} } visited[key] = status return status } links, err := getLinks(ctx, root) if err != nil { status := PinStatus{Ok: false} if opts.explain { status.BadNodes = []BadNode{{Cid: enc.Encode(key), Err: err.Error()}} } visited[key] = status return status } status := PinStatus{Ok: true} for _, lnk := range links { res := checkPin(lnk.Cid) if !res.Ok { status.Ok = false status.BadNodes = append(status.BadNodes, res.BadNodes...) } } visited[key] = status return status } out := make(chan any) go func() { defer close(out) for p := range n.Pinning.RecursiveKeys(ctx, false) { if p.Err != nil { out <- PinVerifyRes{Err: p.Err.Error()} return } pinStatus := checkPin(p.Pin.Key) if !pinStatus.Ok || opts.includeOk { select { case out <- PinVerifyRes{Cid: enc.Encode(p.Pin.Key), PinStatus: pinStatus}: case <-ctx.Done(): return } } } }() return out, nil } // Format formats PinVerifyRes func (r PinVerifyRes) Format(out io.Writer) { if r.Err != "" { fmt.Fprintf(out, "error: %s\n", r.Err) return } if r.Ok { fmt.Fprintf(out, "%s ok\n", r.Cid) return } fmt.Fprintf(out, "%s broken\n", r.Cid) for _, e := range r.BadNodes { fmt.Fprintf(out, " %s: %s\n", e.Cid, e.Err) } } ================================================ FILE: core/commands/pin/remotepin.go ================================================ package pin import ( "context" "fmt" "io" "os" "sort" "strings" "text/tabwriter" "time" neturl "net/url" gopath "path" "golang.org/x/sync/errgroup" pinclient "github.com/ipfs/boxo/pinning/remote/client" cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" logging "github.com/ipfs/go-log/v2" config "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" fsrepo "github.com/ipfs/kubo/repo/fsrepo" "github.com/libp2p/go-libp2p/core/host" peer "github.com/libp2p/go-libp2p/core/peer" ) var log = logging.Logger("core/commands/cmdenv") var remotePinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Pin (and unpin) objects to remote pinning service.", }, Subcommands: map[string]*cmds.Command{ "add": addRemotePinCmd, "ls": listRemotePinCmd, "rm": rmRemotePinCmd, "service": remotePinServiceCmd, }, } var remotePinServiceCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Configure remote pinning services.", }, Subcommands: map[string]*cmds.Command{ "add": addRemotePinServiceCmd, "ls": lsRemotePinServiceCmd, "rm": rmRemotePinServiceCmd, }, } const ( pinNameOptionName = "name" pinCIDsOptionName = "cid" pinStatusOptionName = "status" pinServiceNameOptionName = "service" pinServiceNameArgName = pinServiceNameOptionName pinServiceEndpointArgName = "endpoint" pinServiceKeyArgName = "key" pinServiceStatOptionName = "stat" pinBackgroundOptionName = "background" pinForceOptionName = "force" ) type RemotePinOutput struct { Status string Cid string Name string } func toRemotePinOutput(ps pinclient.PinStatusGetter) RemotePinOutput { return RemotePinOutput{ Name: ps.GetPin().GetName(), Status: ps.GetStatus().String(), Cid: ps.GetPin().GetCid().String(), } } func printRemotePinDetails(w io.Writer, out *RemotePinOutput) { tw := tabwriter.NewWriter(w, 0, 0, 1, ' ', 0) defer tw.Flush() fw := func(k string, v string) { fmt.Fprintf(tw, "%s:\t%s\n", k, v) } fw("CID", out.Cid) fw("Name", out.Name) fw("Status", out.Status) } // remote pin commands var pinServiceNameOption = cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use (mandatory).") var addRemotePinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Pin object to remote pinning service.", ShortDescription: "Asks remote pinning service to pin an IPFS object from a given path.", LongDescription: ` Asks remote pinning service to pin an IPFS object from a given path or a CID. To pin CID 'bafkqaaa' to service named 'mysrv' under a pin named 'mypin': $ ipfs pin remote add --service=mysrv --name=mypin bafkqaaa The above command will block until remote service returns 'pinned' status, which may take time depending on the size and available providers of the pinned data. If you prefer to not wait for pinning confirmation and return immediately after remote service confirms 'queued' status, add the '--background' flag: $ ipfs pin remote add --service=mysrv --name=mypin --background bafkqaaa Status of background pin requests can be inspected with the 'ls' command. To list all pins for the CID across all statuses: $ ipfs pin remote ls --service=mysrv --cid=bafkqaaa --status=queued \ --status=pinning --status=pinned --status=failed NOTE: a comma-separated notation is supported in CLI for convenience: $ ipfs pin remote ls --service=mysrv --cid=bafkqaaa --status=queued,pinning,pinned,failed `, }, Arguments: []cmds.Argument{ cmds.StringArg("ipfs-path", true, false, "CID or Path to be pinned."), }, Options: []cmds.Option{ pinServiceNameOption, cmds.StringOption(pinNameOptionName, "An optional name for the pin."), cmds.BoolOption(pinBackgroundOptionName, "Add to the queue on the remote service and return immediately (does not wait for pinned status).").WithDefault(false), }, Type: RemotePinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(req.Context) defer cancel() // Get remote service c, err := getRemotePinServiceFromRequest(req, env) if err != nil { return err } // Prepare value for Pin.cid if len(req.Arguments) != 1 { return fmt.Errorf("expecting one CID argument") } api, err := cmdenv.GetApi(env, req) if err != nil { return err } p, err := cmdutils.PathOrCidPath(req.Arguments[0]) if err != nil { return err } rp, _, err := api.ResolvePath(ctx, p) if err != nil { return err } // Prepare Pin.name opts := []pinclient.AddOption{} if name, nameFound := req.Options[pinNameOptionName]; nameFound { nameStr := name.(string) // Validate pin name if err := cmdutils.ValidatePinName(nameStr); err != nil { return err } opts = append(opts, pinclient.PinOpts.WithName(nameStr)) } // Prepare Pin.origins // If CID in blockstore, add own multiaddrs to the 'origins' array // so pinning service can use that as a hint and connect back to us. node, err := cmdenv.GetNode(env) if err != nil { return err } isInBlockstore, err := node.Blockstore.Has(req.Context, rp.RootCid()) if err != nil { return err } if isInBlockstore && node.PeerHost != nil { addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(node.PeerHost)) if err != nil { return err } opts = append(opts, pinclient.PinOpts.WithOrigins(addrs...)) } else if isInBlockstore && !node.IsOnline && cmds.GetEncoding(req, cmds.Text) == cmds.Text { fmt.Fprintf(os.Stdout, "WARNING: the local node is offline and remote pinning may fail if there is no other provider for this CID\n") } // Execute remote pin request // TODO: fix panic when pinning service is down ps, err := c.Add(ctx, rp.RootCid(), opts...) if err != nil { return err } // Act on PinStatus.delegates // If Pinning Service returned any delegates, proactively try to // connect to them to facilitate data exchange without waiting for DHT // lookup for _, d := range ps.GetDelegates() { // TODO: confirm this works as expected p, err := peer.AddrInfoFromP2pAddr(d) if err != nil { return err } if err := api.Swarm().Connect(ctx, *p); err != nil { log.Infof("error connecting to remote pin delegate %v : %w", d, err) } } // Block unless --background=true is passed if !req.Options[pinBackgroundOptionName].(bool) { const pinWaitTime = 500 * time.Millisecond var timer *time.Timer requestID := ps.GetRequestId() for { ps, err = c.GetStatusByID(ctx, requestID) if err != nil { return fmt.Errorf("failed to check pin status for requestid=%q due to error: %v", requestID, err) } if ps.GetRequestId() != requestID { return fmt.Errorf("failed to check pin status for requestid=%q, remote service sent unexpected requestid=%q", requestID, ps.GetRequestId()) } s := ps.GetStatus() if s == pinclient.StatusPinned { break } if s == pinclient.StatusFailed { return fmt.Errorf("remote service failed to pin requestid=%q", requestID) } if timer == nil { timer = time.NewTimer(pinWaitTime) } else { timer.Reset(pinWaitTime) } select { case <-timer.C: case <-ctx.Done(): timer.Stop() return fmt.Errorf("waiting for pin interrupted, requestid=%q remains on remote service", requestID) } } } return res.Emit(toRemotePinOutput(ps)) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RemotePinOutput) error { printRemotePinDetails(w, out) return nil }), }, } var listRemotePinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List objects pinned to remote pinning service.", ShortDescription: ` Returns a list of objects that are pinned to a remote pinning service. `, LongDescription: ` Returns a list of objects that are pinned to a remote pinning service. NOTE: By default, it will only show matching objects in 'pinned' state. Pass '--status=queued,pinning,pinned,failed' to list pins in all states. `, }, Arguments: []cmds.Argument{}, Options: []cmds.Option{ pinServiceNameOption, cmds.StringOption(pinNameOptionName, "Return pins with names that contain the value provided (case-sensitive, exact match)."), cmds.DelimitedStringsOption(",", pinCIDsOptionName, "Return pins for the specified CIDs (comma-separated)."), cmds.DelimitedStringsOption(",", pinStatusOptionName, "Return pins with the specified statuses (queued,pinning,pinned,failed).").WithDefault([]string{"pinned"}), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { c, err := getRemotePinServiceFromRequest(req, env) if err != nil { return err } ctx, cancel := context.WithCancel(req.Context) defer cancel() psCh := make(chan pinclient.PinStatusGetter) lsErr := make(chan error, 1) go func() { lsErr <- lsRemote(ctx, req, c, psCh) }() for ps := range psCh { if err := res.Emit(toRemotePinOutput(ps)); err != nil { return err } } return <-lsErr }, Type: RemotePinOutput{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RemotePinOutput) error { // pin remote ls produces a flat output similar to legacy pin ls fmt.Fprintf(w, "%s\t%s\t%s\n", out.Cid, out.Status, cmdenv.EscNonPrint(out.Name)) return nil }), }, } // Executes GET /pins/?query-with-filters func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client, out chan<- pinclient.PinStatusGetter) error { opts := []pinclient.LsOption{} if name, nameFound := req.Options[pinNameOptionName]; nameFound { nameStr := name.(string) // Validate name filter if err := cmdutils.ValidatePinName(nameStr); err != nil { close(out) return err } opts = append(opts, pinclient.PinOpts.FilterName(nameStr)) } if cidsRaw, cidsFound := req.Options[pinCIDsOptionName]; cidsFound { cidsRawArr := cidsRaw.([]string) var parsedCIDs []cid.Cid for _, rawCID := range cidsRawArr { parsedCID, err := cid.Decode(rawCID) if err != nil { close(out) return fmt.Errorf("CID %q cannot be parsed: %v", rawCID, err) } parsedCIDs = append(parsedCIDs, parsedCID) } opts = append(opts, pinclient.PinOpts.FilterCIDs(parsedCIDs...)) } if statusRaw, statusFound := req.Options[pinStatusOptionName]; statusFound { statusRawArr := statusRaw.([]string) var parsedStatuses []pinclient.Status for _, rawStatus := range statusRawArr { s := pinclient.Status(rawStatus) if s.String() == string(pinclient.StatusUnknown) { close(out) return fmt.Errorf("status %q is not valid", rawStatus) } parsedStatuses = append(parsedStatuses, s) } opts = append(opts, pinclient.PinOpts.FilterStatus(parsedStatuses...)) } return c.Ls(ctx, out, opts...) } var rmRemotePinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Remove pins from remote pinning service.", ShortDescription: "Removes the remote pin, allowing it to be garbage-collected if needed.", LongDescription: ` Removes remote pins, allowing them to be garbage-collected if needed. This command accepts the same search query parameters as 'ls', and it is good practice to execute 'ls' before 'rm' to confirm the list of pins to be removed. To remove a single pin for a specific CID: $ ipfs pin remote ls --service=mysrv --cid=bafkqaaa $ ipfs pin remote rm --service=mysrv --cid=bafkqaaa When more than one pin matches the query on the remote service, an error is returned. To confirm the removal of multiple pins, pass '--force': $ ipfs pin remote ls --service=mysrv --name=popular-name $ ipfs pin remote rm --service=mysrv --name=popular-name --force NOTE: When no '--status' is passed, implicit '--status=pinned' is used. To list and then remove all pending pin requests, pass an explicit status list: $ ipfs pin remote ls --service=mysrv --status=queued,pinning,failed $ ipfs pin remote rm --service=mysrv --status=queued,pinning,failed --force `, }, Arguments: []cmds.Argument{}, Options: []cmds.Option{ pinServiceNameOption, cmds.StringOption(pinNameOptionName, "Remove pins with names that contain provided value (case-sensitive, exact match)."), cmds.DelimitedStringsOption(",", pinCIDsOptionName, "Remove pins for the specified CIDs."), cmds.DelimitedStringsOption(",", pinStatusOptionName, "Remove pins with the specified statuses (queued,pinning,pinned,failed).").WithDefault([]string{"pinned"}), cmds.BoolOption(pinForceOptionName, "Allow removal of multiple pins matching the query without additional confirmation.").WithDefault(false), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { c, err := getRemotePinServiceFromRequest(req, env) if err != nil { return err } rmIDs := []string{} if len(req.Arguments) != 0 { return fmt.Errorf("unexpected argument %q", req.Arguments[0]) } psCh := make(chan pinclient.PinStatusGetter) errCh := make(chan error, 1) ctx, cancel := context.WithCancel(req.Context) defer cancel() go func() { errCh <- lsRemote(ctx, req, c, psCh) }() for ps := range psCh { rmIDs = append(rmIDs, ps.GetRequestId()) } if err = <-errCh; err != nil { return fmt.Errorf("error while listing remote pins: %v", err) } if len(rmIDs) > 1 && !req.Options[pinForceOptionName].(bool) { return fmt.Errorf("multiple remote pins are matching this query, add --force to confirm the bulk removal") } for _, rmID := range rmIDs { if err = c.DeleteByID(ctx, rmID); err != nil { return fmt.Errorf("removing pin identified by requestid=%q failed: %v", rmID, err) } } return nil }, } // remote service commands var addRemotePinServiceCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Add remote pinning service.", ShortDescription: "Add credentials for access to a remote pinning service.", LongDescription: ` Add credentials for access to a remote pinning service and store them in the config under Pinning.RemoteServices map. TIP: To add services and test them by fetching pin count stats: $ ipfs pin remote service add goodsrv https://pin-api.example.com secret-key $ ipfs pin remote service add badsrv https://bad-api.example.com invalid-key $ ipfs pin remote service ls --stat goodsrv https://pin-api.example.com 0/0/0/0 badsrv https://bad-api.example.com invalid `, }, Arguments: []cmds.Argument{ cmds.StringArg(pinServiceNameArgName, true, false, "Service name."), cmds.StringArg(pinServiceEndpointArgName, true, false, "Service endpoint."), cmds.StringArg(pinServiceKeyArgName, true, false, "Service key."), }, Type: nil, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err } repo, err := fsrepo.Open(cfgRoot) if err != nil { return err } defer repo.Close() if len(req.Arguments) < 3 { return fmt.Errorf("expecting three arguments: service name, endpoint and key") } name := req.Arguments[0] endpoint, err := normalizeEndpoint(req.Arguments[1]) if err != nil { return err } key := req.Arguments[2] cfg, err := repo.Config() if err != nil { return err } if cfg.Pinning.RemoteServices != nil { if _, present := cfg.Pinning.RemoteServices[name]; present { return fmt.Errorf("service already present") } } else { cfg.Pinning.RemoteServices = map[string]config.RemotePinningService{} } cfg.Pinning.RemoteServices[name] = config.RemotePinningService{ API: config.RemotePinningServiceAPI{ Endpoint: endpoint, Key: key, }, Policies: config.RemotePinningServicePolicies{}, } return repo.SetConfig(cfg) }, } var rmRemotePinServiceCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Remove remote pinning service.", ShortDescription: "Remove credentials for access to a remote pinning service.", }, Arguments: []cmds.Argument{ cmds.StringArg(pinServiceNameOptionName, true, false, "Name of remote pinning service to remove."), }, Options: []cmds.Option{}, Type: nil, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err } repo, err := fsrepo.Open(cfgRoot) if err != nil { return err } defer repo.Close() if len(req.Arguments) != 1 { return fmt.Errorf("expecting one argument: name") } name := req.Arguments[0] cfg, err := repo.Config() if err != nil { return err } if cfg.Pinning.RemoteServices != nil { delete(cfg.Pinning.RemoteServices, name) } return repo.SetConfig(cfg) }, } var lsRemotePinServiceCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List remote pinning services.", ShortDescription: "List remote pinning services.", LongDescription: ` List remote pinning services. By default, only a name and an endpoint are listed; however, one can pass '--stat' to test each endpoint by fetching pin counts for each state: $ ipfs pin remote service ls --stat goodsrv https://pin-api.example.com 0/0/0/0 badsrv https://bad-api.example.com invalid TIP: pass '--enc=json' for more useful JSON output. `, }, Arguments: []cmds.Argument{}, Options: []cmds.Option{ cmds.BoolOption(pinServiceStatOptionName, "Try to fetch and display current pin count on remote service (queued/pinning/pinned/failed).").WithDefault(false), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(req.Context) defer cancel() cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err } repo, err := fsrepo.Open(cfgRoot) if err != nil { return err } defer repo.Close() cfg, err := repo.Config() if err != nil { return err } if cfg.Pinning.RemoteServices == nil { return cmds.EmitOnce(res, &PinServicesList{make([]ServiceDetails, 0)}) } services := cfg.Pinning.RemoteServices result := PinServicesList{make([]ServiceDetails, 0, len(services))} for svcName, svcConfig := range services { svcDetails := ServiceDetails{svcName, svcConfig.API.Endpoint, nil} // if --pin-count is passed, we try to fetch pin numbers from remote service if req.Options[pinServiceStatOptionName].(bool) { lsRemotePinCount := func(ctx context.Context, env cmds.Environment, svcName string) (*PinCount, error) { c, err := getRemotePinService(env, svcName) if err != nil { return nil, err } // we only care about total count, so requesting smallest batch batch := pinclient.PinOpts.Limit(1) fs := pinclient.PinOpts.FilterStatus statuses := []pinclient.Status{ pinclient.StatusQueued, pinclient.StatusPinning, pinclient.StatusPinned, pinclient.StatusFailed, } g, ctx := errgroup.WithContext(ctx) pc := &PinCount{} for _, s := range statuses { status := s // lol https://golang.org/doc/faq#closures_and_goroutines g.Go(func() error { _, n, err := c.LsBatchSync(ctx, batch, fs(status)) if err != nil { return err } switch status { case pinclient.StatusQueued: pc.Queued = n case pinclient.StatusPinning: pc.Pinning = n case pinclient.StatusPinned: pc.Pinned = n case pinclient.StatusFailed: pc.Failed = n } return nil }) } if err := g.Wait(); err != nil { return nil, err } return pc, nil } pinCount, err := lsRemotePinCount(ctx, env, svcName) // PinCount is present only if we were able to fetch counts. // We don't want to break listing of services so this is best-effort. // (verbose err is returned by 'pin remote ls', if needed) svcDetails.Stat = &Stat{} if err == nil { svcDetails.Stat.Status = "valid" svcDetails.Stat.PinCount = pinCount } else { svcDetails.Stat.Status = "invalid" } } result.RemoteServices = append(result.RemoteServices, svcDetails) } sort.Sort(result) return cmds.EmitOnce(res, &result) }, Type: PinServicesList{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, list *PinServicesList) error { tw := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) withStat := req.Options[pinServiceStatOptionName].(bool) for _, s := range list.RemoteServices { if withStat { stat := s.Stat.Status pc := s.Stat.PinCount if s.Stat.PinCount != nil { stat = fmt.Sprintf("%d/%d/%d/%d", pc.Queued, pc.Pinning, pc.Pinned, pc.Failed) } fmt.Fprintf(tw, "%s\t%s\t%s\n", s.Service, s.ApiEndpoint, stat) } else { fmt.Fprintf(tw, "%s\t%s\n", s.Service, s.ApiEndpoint) } } tw.Flush() return nil }), }, } type ServiceDetails struct { Service string ApiEndpoint string //nolint Stat *Stat `json:",omitempty"` // present only when --stat not passed } type Stat struct { Status string PinCount *PinCount `json:",omitempty"` // missing when --stat is passed but the service is offline } type PinCount struct { Queued int Pinning int Pinned int Failed int } // Struct returned by ipfs pin remote service ls --enc=json | jq type PinServicesList struct { RemoteServices []ServiceDetails } func (l PinServicesList) Len() int { return len(l.RemoteServices) } func (l PinServicesList) Swap(i, j int) { s := l.RemoteServices s[i], s[j] = s[j], s[i] } func (l PinServicesList) Less(i, j int) bool { s := l.RemoteServices return s[i].Service < s[j].Service } func getRemotePinServiceFromRequest(req *cmds.Request, env cmds.Environment) (*pinclient.Client, error) { service, serviceFound := req.Options[pinServiceNameOptionName] if !serviceFound { return nil, fmt.Errorf("a service name must be passed") } serviceStr := service.(string) var err error c, err := getRemotePinService(env, serviceStr) if err != nil { return nil, err } return c, nil } func getRemotePinService(env cmds.Environment, name string) (*pinclient.Client, error) { if name == "" { return nil, fmt.Errorf("remote pinning service name not specified") } endpoint, key, err := getRemotePinServiceInfo(env, name) if err != nil { return nil, err } return pinclient.NewClient(endpoint, key), nil } func getRemotePinServiceInfo(env cmds.Environment, name string) (endpoint, key string, err error) { cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return "", "", err } repo, err := fsrepo.Open(cfgRoot) if err != nil { return "", "", err } defer repo.Close() cfg, err := repo.Config() if err != nil { return "", "", err } if cfg.Pinning.RemoteServices == nil { return "", "", fmt.Errorf("service not known") } service, present := cfg.Pinning.RemoteServices[name] if !present { return "", "", fmt.Errorf("service not known") } endpoint, err = normalizeEndpoint(service.API.Endpoint) if err != nil { return "", "", err } return endpoint, service.API.Key, nil } func normalizeEndpoint(endpoint string) (string, error) { uri, err := neturl.ParseRequestURI(endpoint) if err != nil || !(uri.Scheme == "http" || uri.Scheme == "https") { return "", fmt.Errorf("service endpoint must be a valid HTTP URL") } // cleanup trailing and duplicate slashes (https://github.com/ipfs/kubo/issues/7826) uri.Path = gopath.Clean(uri.Path) uri.Path = strings.TrimSuffix(uri.Path, ".") uri.Path = strings.TrimSuffix(uri.Path, "/") // remove any query params if uri.RawQuery != "" { return "", fmt.Errorf("service endpoint should be provided without any query parameters") } if strings.HasSuffix(uri.Path, "/pins") { return "", fmt.Errorf("service endpoint should be provided without the /pins suffix") } return uri.String(), nil } ================================================ FILE: core/commands/pin/remotepin_test.go ================================================ package pin import ( "testing" ) func TestNormalizeEndpoint(t *testing.T) { cases := []struct { in string err string out string }{ { in: "https://1.example.com", err: "", out: "https://1.example.com", }, { in: "https://2.example.com/", err: "", out: "https://2.example.com", }, { in: "https://3.example.com/pins/", err: "service endpoint should be provided without the /pins suffix", out: "", }, { in: "https://4.example.com/pins", err: "service endpoint should be provided without the /pins suffix", out: "", }, { in: "https://5.example.com/./some//nonsense/../path/../path/", err: "", out: "https://5.example.com/some/path", }, { in: "https://6.example.com/endpoint/?query=val", err: "service endpoint should be provided without any query parameters", out: "", }, { in: "http://192.168.0.5:45000/", err: "", out: "http://192.168.0.5:45000", }, { in: "foo://4.example.com/pins", err: "service endpoint must be a valid HTTP URL", out: "", }, } for _, tc := range cases { out, err := normalizeEndpoint(tc.in) if err != nil && tc.err != err.Error() { t.Errorf("unexpected error for %q: expected %q; got %q", tc.in, tc.err, err) continue } if out != tc.out { t.Errorf("unexpected endpoint for %q: expected %q; got %q", tc.in, tc.out, out) continue } } } ================================================ FILE: core/commands/ping.go ================================================ package commands import ( "context" "errors" "fmt" "io" "strings" "time" "github.com/ipfs/kubo/core/commands/cmdenv" cmds "github.com/ipfs/go-ipfs-cmds" peer "github.com/libp2p/go-libp2p/core/peer" pstore "github.com/libp2p/go-libp2p/core/peerstore" ping "github.com/libp2p/go-libp2p/p2p/protocol/ping" ma "github.com/multiformats/go-multiaddr" ) const kPingTimeout = 10 * time.Second type PingResult struct { Success bool Time time.Duration Text string } const ( pingCountOptionName = "count" ) // ErrPingSelf is returned when the user attempts to ping themself. var ErrPingSelf = errors.New("error: can't ping self") var PingCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Send echo request packets to IPFS hosts.", ShortDescription: ` 'ipfs ping' is a tool to test sending data to other nodes. It finds nodes via the routing system, sends pings, waits for pongs, and prints out round- trip latency information. `, }, Arguments: []cmds.Argument{ cmds.StringArg("peer ID", true, true, "ID of peer to be pinged.").EnableStdin(), }, Options: []cmds.Option{ cmds.IntOption(pingCountOptionName, "n", "Number of ping messages to send.").WithDefault(10), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := cmdenv.GetNode(env) if err != nil { return err } // Must be online! if !n.IsOnline { return ErrNotOnline } addr, pid, err := ParsePeerParam(req.Arguments[0]) if err != nil { return fmt.Errorf("failed to parse peer address '%s': %s", req.Arguments[0], err) } if pid == n.Identity { return ErrPingSelf } if addr != nil { n.Peerstore.AddAddr(pid, addr, pstore.TempAddrTTL) // temporary } numPings, _ := req.Options[pingCountOptionName].(int) if numPings <= 0 { return fmt.Errorf("ping count must be greater than 0, was %d", numPings) } if len(n.Peerstore.Addrs(pid)) == 0 { // Make sure we can find the node in question if err := res.Emit(&PingResult{ Text: fmt.Sprintf("Looking up peer %s", pid), Success: true, }); err != nil { return err } ctx, cancel := context.WithTimeout(req.Context, kPingTimeout) p, err := n.Routing.FindPeer(ctx, pid) cancel() if err != nil { return fmt.Errorf("peer lookup failed: %s", err) } n.Peerstore.AddAddrs(p.ID, p.Addrs, pstore.TempAddrTTL) } if err := res.Emit(&PingResult{ Text: fmt.Sprintf("PING %s.", pid), Success: true, }); err != nil { return err } ctx, cancel := context.WithTimeout(req.Context, kPingTimeout*time.Duration(numPings)) defer cancel() pings := ping.Ping(ctx, n.PeerHost, pid) var ( count int total time.Duration ) ticker := time.NewTicker(time.Second) defer ticker.Stop() for range numPings { r, ok := <-pings if !ok { break } if r.Error != nil { err = res.Emit(&PingResult{ Success: false, Text: fmt.Sprintf("Ping error: %s", r.Error), }) } else { count++ total += r.RTT err = res.Emit(&PingResult{ Success: true, Time: r.RTT, }) } if err != nil { return err } select { case <-ticker.C: case <-ctx.Done(): return ctx.Err() } } if count == 0 { return fmt.Errorf("ping failed") } averagems := total.Seconds() * 1000 / float64(count) return res.Emit(&PingResult{ Success: true, Text: fmt.Sprintf("Average latency: %.2fms", averagems), }) }, Type: PingResult{}, PostRun: cmds.PostRunMap{ cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { var ( total time.Duration count int ) for { event, err := res.Next() switch err { case nil: case io.EOF: return nil case context.Canceled, context.DeadlineExceeded: if count == 0 { return err } averagems := total.Seconds() * 1000 / float64(count) return re.Emit(&PingResult{ Success: true, Text: fmt.Sprintf("Average latency: %.2fms", averagems), }) default: return err } pr := event.(*PingResult) if pr.Success && pr.Text == "" { total += pr.Time count++ } err = re.Emit(event) if err != nil { return err } } }, }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PingResult) error { if len(out.Text) > 0 { fmt.Fprintln(w, out.Text) } else if out.Success { fmt.Fprintf(w, "Pong received: time=%.2f ms\n", out.Time.Seconds()*1000) } else { fmt.Fprintf(w, "Pong failed\n") } return nil }), }, } func ParsePeerParam(text string) (ma.Multiaddr, peer.ID, error) { // Multiaddr if strings.HasPrefix(text, "/") { maddr, err := ma.NewMultiaddr(text) if err != nil { return nil, "", err } transport, id := peer.SplitAddr(maddr) if id == "" { return nil, "", peer.ErrInvalidAddr } return transport, id, nil } // Raw peer ID p, err := peer.Decode(text) return nil, p, err } ================================================ FILE: core/commands/profile.go ================================================ package commands import ( "archive/zip" "fmt" "io" "os" "strings" "time" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/kubo/core/commands/e" "github.com/ipfs/kubo/profile" ) // time format that works in filenames on windows. var timeFormat = strings.ReplaceAll(time.RFC3339, ":", "_") type profileResult struct { File string } const ( collectorsOptionName = "collectors" profileTimeOption = "profile-time" mutexProfileFractionOption = "mutex-profile-fraction" blockProfileRateOption = "block-profile-rate" ) var sysProfileCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Collect a performance profile for debugging.", ShortDescription: ` Collects profiles from a running go-ipfs daemon into a single zip file. To aid in debugging, this command also attempts to include a copy of the running go-ipfs binary. `, LongDescription: ` Collects profiles from a running go-ipfs daemon into a single zipfile. To aid in debugging, this command also attempts to include a copy of the running go-ipfs binary. Profiles can be examined using 'go tool pprof', some tips can be found at https://github.com/ipfs/kubo/blob/master/docs/debug-guide.md. Privacy Notice: The output file includes: - A list of running goroutines. - A CPU profile. - A heap inuse profile. - A heap allocation profile. - A mutex profile. - A block profile. - Your copy of go-ipfs. - The output of 'ipfs version --all'. It does not include: - Any of your IPFS data or metadata. - Your config or private key. - Your IP address. - The contents of your computer's memory, filesystem, etc. However, it could reveal: - Your build path, if you built go-ipfs yourself. - If and how a command/feature is being used (inferred from running functions). - Memory offsets of various data structures. - Any modifications you've made to go-ipfs. `, HTTP: &cmds.HTTPHelpText{ ResponseContentType: "application/zip", }, }, NoLocal: true, Options: []cmds.Option{ cmds.StringOption(outputOptionName, "o", "The path where the output .zip should be stored. Default: ./ipfs-profile-[timestamp].zip"), cmds.DelimitedStringsOption(",", collectorsOptionName, "The list of collectors to use for collecting diagnostic data."). WithDefault([]string{ profile.CollectorGoroutinesStack, profile.CollectorGoroutinesPprof, profile.CollectorVersion, profile.CollectorHeap, profile.CollectorAllocs, profile.CollectorBin, profile.CollectorCPU, profile.CollectorMutex, profile.CollectorBlock, profile.CollectorTrace, }), cmds.StringOption(profileTimeOption, "The amount of time spent profiling. If this is set to 0, then sampling profiles are skipped.").WithDefault("30s"), cmds.IntOption(mutexProfileFractionOption, "The fraction 1/n of mutex contention events that are reported in the mutex profile.").WithDefault(4), cmds.StringOption(blockProfileRateOption, "The duration to wait between sampling goroutine-blocking events for the blocking profile.").WithDefault("1ms"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { collectors := req.Options[collectorsOptionName].([]string) profileTimeStr, _ := req.Options[profileTimeOption].(string) profileTime, err := time.ParseDuration(profileTimeStr) if err != nil { return fmt.Errorf("failed to parse profile duration %q: %w", profileTimeStr, err) } blockProfileRateStr, _ := req.Options[blockProfileRateOption].(string) blockProfileRate, err := time.ParseDuration(blockProfileRateStr) if err != nil { return fmt.Errorf("failed to parse block profile rate %q: %w", blockProfileRateStr, err) } mutexProfileFraction, _ := req.Options[mutexProfileFractionOption].(int) r, w := io.Pipe() go func() { archive := zip.NewWriter(w) err = profile.WriteProfiles(req.Context, archive, profile.Options{ Collectors: collectors, ProfileDuration: profileTime, MutexProfileFraction: mutexProfileFraction, BlockProfileRate: blockProfileRate, }) archive.Close() _ = w.CloseWithError(err) }() res.SetEncodingType(cmds.OctetStream) res.SetContentType("application/zip") return res.Emit(r) }, PostRun: cmds.PostRunMap{ cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { v, err := res.Next() if err != nil { return err } outReader, ok := v.(io.Reader) if !ok { return e.New(e.TypeErr(outReader, v)) } outPath, _ := res.Request().Options[outputOptionName].(string) if outPath == "" { outPath = "ipfs-profile-" + time.Now().Format(timeFormat) + ".zip" } fi, err := os.Create(outPath) if err != nil { return err } defer fi.Close() _, err = io.Copy(fi, outReader) if err != nil { return err } return re.Emit(&profileResult{File: outPath}) }, }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *profileResult) error { fmt.Fprintf(w, "Wrote profiles to: %s\n", out.File) return nil }), }, } ================================================ FILE: core/commands/provide.go ================================================ package commands import ( "context" "errors" "fmt" "io" "strings" "text/tabwriter" "time" "unicode/utf8" humanize "github.com/dustin/go-humanize" boxoprovider "github.com/ipfs/boxo/provider" cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/libp2p/go-libp2p-kad-dht/fullrt" "github.com/libp2p/go-libp2p-kad-dht/provider" "github.com/libp2p/go-libp2p-kad-dht/provider/buffered" "github.com/libp2p/go-libp2p-kad-dht/provider/dual" "github.com/libp2p/go-libp2p-kad-dht/provider/stats" routing "github.com/libp2p/go-libp2p/core/routing" "github.com/probe-lab/go-libdht/kad/key" "golang.org/x/exp/constraints" ) const ( provideQuietOptionName = "quiet" provideLanOptionName = "lan" provideStatAllOptionName = "all" provideStatCompactOptionName = "compact" provideStatNetworkOptionName = "network" provideStatConnectivityOptionName = "connectivity" provideStatOperationsOptionName = "operations" provideStatTimingsOptionName = "timings" provideStatScheduleOptionName = "schedule" provideStatQueuesOptionName = "queues" provideStatWorkersOptionName = "workers" // lowWorkerThreshold is the threshold below which worker availability warnings are shown lowWorkerThreshold = 2 ) var ProvideCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Control and monitor content providing", ShortDescription: ` Control providing operations. OVERVIEW: The provider system advertises content by publishing provider records, allowing other nodes to discover which peers have specific content. Content is reprovided periodically (every Provide.DHT.Interval) according to Provide.Strategy. CONFIGURATION: Learn more: https://github.com/ipfs/kubo/blob/master/docs/config.md#provide SEE ALSO: For ad-hoc one-time provide, see 'ipfs routing provide' `, }, Subcommands: map[string]*cmds.Command{ "clear": provideClearCmd, "stat": provideStatCmd, }, } var provideClearCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Clear all CIDs from the provide queue.", ShortDescription: ` Clear all CIDs pending to be provided for the first time. BEHAVIOR: This command removes CIDs from the provide queue that are waiting to be advertised to the DHT for the first time. It does not affect content that is already being reprovided on schedule. AUTOMATIC CLEARING: Kubo will automatically clear the queue when it detects a change of Provide.Strategy upon a restart. Learn: https://github.com/ipfs/kubo/blob/master/docs/config.md#providestrategy `, }, Options: []cmds.Option{ cmds.BoolOption(provideQuietOptionName, "q", "Do not write output."), }, Run: func(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment) error { n, err := cmdenv.GetNode(env) if err != nil { return err } quiet, _ := req.Options[provideQuietOptionName].(bool) if n.Provider == nil { return nil } cleared := n.Provider.Clear() if quiet { return nil } _ = re.Emit(cleared) return nil }, Type: int(0), Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, cleared int) error { quiet, _ := req.Options[provideQuietOptionName].(bool) if quiet { return nil } _, err := fmt.Fprintf(w, "removed %d items from provide queue\n", cleared) return err }), }, } type provideStats struct { Sweep *stats.Stats Legacy *boxoprovider.ReproviderStats FullRT bool // only used for legacy stats } // extractSweepingProvider extracts a SweepingProvider from the given provider interface. // It handles unwrapping buffered and dual providers, selecting LAN or WAN as specified. // Returns nil if the provider is not a sweeping provider type. func extractSweepingProvider(prov any, useLAN bool) *provider.SweepingProvider { switch p := prov.(type) { case *provider.SweepingProvider: return p case *dual.SweepingProvider: if useLAN { return p.LAN } return p.WAN case *buffered.SweepingProvider: // Recursively extract from the inner provider return extractSweepingProvider(p.Provider, useLAN) default: return nil } } var provideStatCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Show statistics about the provider system", ShortDescription: ` Returns statistics about the node's provider system. OVERVIEW: The provide system advertises content to the DHT by publishing provider records that map CIDs to your peer ID. These records expire after a fixed TTL to account for node churn, so content must be reprovided periodically to stay discoverable. Two provider types exist: - Sweep provider: Divides the DHT keyspace into regions and systematically sweeps through them over the reprovide interval. Batches CIDs allocated to the same DHT servers, reducing lookups from N (one per CID) to a small static number based on DHT size (~3k for 10k DHT servers). Spreads work evenly over time to prevent resource spikes and ensure announcements happen just before records expire. - Legacy provider: Processes each CID individually with separate DHT lookups. Attempts to reprovide all content as quickly as possible at the start of each cycle. Works well for small datasets but struggles with large collections. Learn more: - Config: https://github.com/ipfs/kubo/blob/master/docs/config.md#provide - Metrics: https://github.com/ipfs/kubo/blob/master/docs/provide-stats.md DEFAULT OUTPUT: Shows a brief summary including queue sizes, scheduled items, average record holders, ongoing/total provides, and worker warnings. DETAILED OUTPUT: Use --all for detailed statistics with these sections: connectivity, queues, schedule, timings, network, operations, and workers. Individual sections can be displayed with their flags (e.g., --network, --operations). Multiple flags can be combined. Use --compact for monitoring-friendly 2-column output (requires --all). EXAMPLES: Monitor provider statistics in real-time with 2-column layout: watch ipfs provide stat --all --compact Get statistics in JSON format for programmatic processing: ipfs provide stat --enc=json | jq NOTES: - This interface is experimental and may change between releases - Legacy provider shows basic stats only (no flags supported) - "Regions" are keyspace divisions for spreading reprovide work - For Dual DHT: use --lan for LAN provider stats (default is WAN) `, }, Arguments: []cmds.Argument{}, Options: []cmds.Option{ cmds.BoolOption(provideLanOptionName, "Show stats for LAN DHT only (for Sweep+Dual DHT only)"), cmds.BoolOption(provideStatAllOptionName, "a", "Display all provide sweep stats"), cmds.BoolOption(provideStatCompactOptionName, "Display stats in 2-column layout (requires --all)"), cmds.BoolOption(provideStatConnectivityOptionName, "Display DHT connectivity status"), cmds.BoolOption(provideStatNetworkOptionName, "Display network stats (peers, reachability, region size)"), cmds.BoolOption(provideStatScheduleOptionName, "Display reprovide schedule (CIDs/regions scheduled, next reprovide time)"), cmds.BoolOption(provideStatTimingsOptionName, "Display timing information (uptime, cycle start, reprovide interval)"), cmds.BoolOption(provideStatWorkersOptionName, "Display worker pool stats (active/available/queued workers)"), cmds.BoolOption(provideStatOperationsOptionName, "Display operation stats (ongoing/past provides, rates, errors)"), cmds.BoolOption(provideStatQueuesOptionName, "Display provide and reprovide queue sizes"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } if !nd.IsOnline { return ErrNotOnline } lanStats, _ := req.Options[provideLanOptionName].(bool) // Handle legacy provider if legacySys, ok := nd.Provider.(boxoprovider.System); ok { if lanStats { return errors.New("LAN stats only available for Sweep provider with Dual DHT") } stats, err := legacySys.Stat() if err != nil { return err } _, fullRT := nd.DHTClient.(*fullrt.FullRT) return res.Emit(provideStats{Legacy: &stats, FullRT: fullRT}) } // Extract sweeping provider (handles buffered and dual unwrapping) sweepingProvider := extractSweepingProvider(nd.Provider, lanStats) if sweepingProvider == nil { if lanStats { return errors.New("LAN stats only available for Sweep provider with Dual DHT") } return fmt.Errorf("stats not available with current routing system %T", nd.Provider) } s := sweepingProvider.Stats() return res.Emit(provideStats{Sweep: &s}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, s provideStats) error { wtr := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) defer wtr.Flush() all, _ := req.Options[provideStatAllOptionName].(bool) compact, _ := req.Options[provideStatCompactOptionName].(bool) connectivity, _ := req.Options[provideStatConnectivityOptionName].(bool) queues, _ := req.Options[provideStatQueuesOptionName].(bool) schedule, _ := req.Options[provideStatScheduleOptionName].(bool) network, _ := req.Options[provideStatNetworkOptionName].(bool) timings, _ := req.Options[provideStatTimingsOptionName].(bool) operations, _ := req.Options[provideStatOperationsOptionName].(bool) workers, _ := req.Options[provideStatWorkersOptionName].(bool) flagCount := 0 for _, enabled := range []bool{all, connectivity, queues, schedule, network, timings, operations, workers} { if enabled { flagCount++ } } if s.Legacy != nil { if flagCount > 0 { return errors.New("cannot use flags with legacy provide stats") } fmt.Fprintf(wtr, "TotalReprovides:\t%s\n", humanNumber(s.Legacy.TotalReprovides)) fmt.Fprintf(wtr, "AvgReprovideDuration:\t%s\n", humanDuration(s.Legacy.AvgReprovideDuration)) fmt.Fprintf(wtr, "LastReprovideDuration:\t%s\n", humanDuration(s.Legacy.LastReprovideDuration)) if !s.Legacy.LastRun.IsZero() { fmt.Fprintf(wtr, "LastReprovide:\t%s\n", humanTime(s.Legacy.LastRun)) if s.FullRT { fmt.Fprintf(wtr, "NextReprovide:\t%s\n", humanTime(s.Legacy.LastRun.Add(s.Legacy.ReprovideInterval))) } } return nil } if s.Sweep == nil { return errors.New("no provide stats available") } // Sweep provider stats if s.Sweep.Closed { fmt.Fprintf(wtr, "Provider is closed\n") return nil } if compact && !all { return errors.New("--compact requires --all flag") } brief := flagCount == 0 showHeadings := flagCount > 1 || all compactMode := all && compact var cols [2][]string col0MaxWidth := 0 // formatLine handles both normal and compact output modes: // - Normal mode: all lines go to cols[0], col parameter is ignored // - Compact mode: col 0 for left column, col 1 for right column formatLine := func(col int, format string, a ...any) { if compactMode { s := fmt.Sprintf(format, a...) cols[col] = append(cols[col], s) if col == 0 { col0MaxWidth = max(col0MaxWidth, utf8.RuneCountInString(s)) } return } format = strings.Replace(format, ": ", ":\t", 1) format = strings.Replace(format, ", ", ",\t", 1) cols[0] = append(cols[0], fmt.Sprintf(format, a...)) } addBlankLine := func(col int) { if !brief { formatLine(col, "") } } sectionTitle := func(col int, title string) { if !brief && showHeadings { formatLine(col, "%s:", title) } } indent := " " if brief || !showHeadings { indent = "" } // Connectivity if all || connectivity || brief && s.Sweep.Connectivity.Status != "online" { sectionTitle(1, "Connectivity") since := s.Sweep.Connectivity.Since if since.IsZero() { formatLine(1, "%sStatus: %s", indent, s.Sweep.Connectivity.Status) } else { formatLine(1, "%sStatus: %s (%s)", indent, s.Sweep.Connectivity.Status, humanTime(since)) } addBlankLine(1) } // Queues if all || queues || brief { sectionTitle(1, "Queues") formatLine(1, "%sProvide queue: %s CIDs, %s regions", indent, humanSI(s.Sweep.Queues.PendingKeyProvides, 1), humanSI(s.Sweep.Queues.PendingRegionProvides, 1)) formatLine(1, "%sReprovide queue: %s regions", indent, humanSI(s.Sweep.Queues.PendingRegionReprovides, 1)) addBlankLine(1) } // Schedule if all || schedule || brief { sectionTitle(0, "Schedule") formatLine(0, "%sCIDs scheduled: %s", indent, humanNumber(s.Sweep.Schedule.Keys)) formatLine(0, "%sRegions scheduled: %s", indent, humanNumberOrNA(s.Sweep.Schedule.Regions)) if !brief { formatLine(0, "%sAvg prefix length: %s", indent, humanFloatOrNA(s.Sweep.Schedule.AvgPrefixLength)) nextPrefix := key.BitString(s.Sweep.Schedule.NextReprovidePrefix) if nextPrefix == "" { nextPrefix = "N/A" } formatLine(0, "%sNext region prefix: %s", indent, nextPrefix) nextReprovideAt := s.Sweep.Schedule.NextReprovideAt.Format("15:04:05") if s.Sweep.Schedule.NextReprovideAt.IsZero() { nextReprovideAt = "N/A" } formatLine(0, "%sNext region reprovide: %s", indent, nextReprovideAt) } addBlankLine(0) } // Timings if all || timings { sectionTitle(1, "Timings") formatLine(1, "%sUptime: %s (%s)", indent, humanDuration(s.Sweep.Timing.Uptime), humanTime(time.Now().Add(-s.Sweep.Timing.Uptime))) formatLine(1, "%sCurrent time offset: %s", indent, humanDuration(s.Sweep.Timing.CurrentTimeOffset)) formatLine(1, "%sCycle started: %s", indent, humanTime(s.Sweep.Timing.CycleStart)) formatLine(1, "%sReprovide interval: %s", indent, humanDuration(s.Sweep.Timing.ReprovidesInterval)) addBlankLine(1) } // Network if all || network || brief { sectionTitle(0, "Network") formatLine(0, "%sAvg record holders: %s", indent, humanFloatOrNA(s.Sweep.Network.AvgHolders)) if !brief { formatLine(0, "%sPeers swept: %s", indent, humanInt(s.Sweep.Network.Peers)) formatLine(0, "%sFull keyspace coverage: %t", indent, s.Sweep.Network.CompleteKeyspaceCoverage) if s.Sweep.Network.Peers > 0 { formatLine(0, "%sReachable peers: %s (%s%%)", indent, humanInt(s.Sweep.Network.Reachable), humanNumber(100*s.Sweep.Network.Reachable/s.Sweep.Network.Peers)) } else { formatLine(0, "%sReachable peers: %s", indent, humanInt(s.Sweep.Network.Reachable)) } formatLine(0, "%sAvg region size: %s", indent, humanFloatOrNA(s.Sweep.Network.AvgRegionSize)) formatLine(0, "%sReplication factor: %s", indent, humanNumber(s.Sweep.Network.ReplicationFactor)) addBlankLine(0) } } // Operations if all || operations || brief { sectionTitle(1, "Operations") // Ongoing operations formatLine(1, "%sOngoing provides: %s CIDs, %s regions", indent, humanSI(s.Sweep.Operations.Ongoing.KeyProvides, 1), humanSI(s.Sweep.Operations.Ongoing.RegionProvides, 1)) formatLine(1, "%sOngoing reprovides: %s CIDs, %s regions", indent, humanSI(s.Sweep.Operations.Ongoing.KeyReprovides, 1), humanSI(s.Sweep.Operations.Ongoing.RegionReprovides, 1)) // Past operations summary formatLine(1, "%sTotal CIDs provided: %s", indent, humanNumber(s.Sweep.Operations.Past.KeysProvided)) if !brief { formatLine(1, "%sTotal records provided: %s", indent, humanNumber(s.Sweep.Operations.Past.RecordsProvided)) formatLine(1, "%sTotal provide errors: %s", indent, humanNumber(s.Sweep.Operations.Past.KeysFailed)) formatLine(1, "%sCIDs provided/min/worker: %s", indent, humanFloatOrNA(s.Sweep.Operations.Past.KeysProvidedPerMinute)) formatLine(1, "%sCIDs reprovided/min/worker: %s", indent, humanFloatOrNA(s.Sweep.Operations.Past.KeysReprovidedPerMinute)) formatLine(1, "%sRegion reprovide duration: %s", indent, humanDurationOrNA(s.Sweep.Operations.Past.RegionReprovideDuration)) formatLine(1, "%sAvg CIDs/reprovide: %s", indent, humanFloatOrNA(s.Sweep.Operations.Past.AvgKeysPerReprovide)) formatLine(1, "%sRegions reprovided (last cycle): %s", indent, humanNumber(s.Sweep.Operations.Past.RegionReprovidedLastCycle)) addBlankLine(1) } } // Workers displayWorkers := all || workers if displayWorkers || brief { availableReservedBurst := max(0, s.Sweep.Workers.DedicatedBurst-s.Sweep.Workers.ActiveBurst) availableReservedPeriodic := max(0, s.Sweep.Workers.DedicatedPeriodic-s.Sweep.Workers.ActivePeriodic) availableFreeWorkers := max(0, s.Sweep.Workers.Max-max(s.Sweep.Workers.DedicatedBurst, s.Sweep.Workers.ActiveBurst)-max(s.Sweep.Workers.DedicatedPeriodic, s.Sweep.Workers.ActivePeriodic)) availableBurst := availableFreeWorkers + availableReservedBurst availablePeriodic := availableFreeWorkers + availableReservedPeriodic if displayWorkers || availableBurst <= lowWorkerThreshold || availablePeriodic <= lowWorkerThreshold { // Either we want to display workers information, or we are low on // available workers and want to warn the user. sectionTitle(0, "Workers") specifyWorkers := " workers" if compactMode { specifyWorkers = "" } formatLine(0, "%sActive%s: %s / %s (max)", indent, specifyWorkers, humanInt(s.Sweep.Workers.Active), humanInt(s.Sweep.Workers.Max)) if brief { // Brief mode - show condensed worker info formatLine(0, "%sPeriodic%s: %s active, %s available, %s queued", indent, specifyWorkers, humanInt(s.Sweep.Workers.ActivePeriodic), humanInt(availablePeriodic), humanInt(s.Sweep.Workers.QueuedPeriodic)) formatLine(0, "%sBurst%s: %s active, %s available, %s queued\n", indent, specifyWorkers, humanInt(s.Sweep.Workers.ActiveBurst), humanInt(availableBurst), humanInt(s.Sweep.Workers.QueuedBurst)) } else { formatLine(0, "%sFree%s: %s", indent, specifyWorkers, humanInt(availableFreeWorkers)) formatLine(0, "%s %-14s %-9s %s", indent, "Workers stats:", "Periodic", "Burst") formatLine(0, "%s %-14s %-9s %s", indent, "Active:", humanInt(s.Sweep.Workers.ActivePeriodic), humanInt(s.Sweep.Workers.ActiveBurst)) formatLine(0, "%s %-14s %-9s %s", indent, "Dedicated:", humanInt(s.Sweep.Workers.DedicatedPeriodic), humanInt(s.Sweep.Workers.DedicatedBurst)) formatLine(0, "%s %-14s %-9s %s", indent, "Available:", humanInt(availablePeriodic), humanInt(availableBurst)) formatLine(0, "%s %-14s %-9s %s", indent, "Queued:", humanInt(s.Sweep.Workers.QueuedPeriodic), humanInt(s.Sweep.Workers.QueuedBurst)) formatLine(0, "%sMax connections/worker: %s", indent, humanInt(s.Sweep.Workers.MaxProvideConnsPerWorker)) addBlankLine(0) } } } if compactMode { col0Width := col0MaxWidth + 2 // Print both columns side by side maxRows := max(len(cols[0]), len(cols[1])) if maxRows == 0 { return nil } for i := range maxRows - 1 { // last line is empty var left, right string if i < len(cols[0]) { left = cols[0][i] } if i < len(cols[1]) { right = cols[1][i] } fmt.Fprintf(wtr, "%-*s %s\n", col0Width, left, right) } } else { if !brief { cols[0] = cols[0][:len(cols[0])-1] // remove last blank line } for _, line := range cols[0] { fmt.Fprintln(wtr, line) } } return nil }), }, Type: provideStats{}, } func humanDuration(val time.Duration) string { if val > time.Second { return val.Truncate(100 * time.Millisecond).String() } return val.Truncate(time.Microsecond).String() } func humanDurationOrNA(val time.Duration) string { if val <= 0 { return "N/A" } return humanDuration(val) } func humanTime(val time.Time) string { if val.IsZero() { return "N/A" } return val.Format("2006-01-02 15:04:05") } func humanNumber[T constraints.Float | constraints.Integer](n T) string { nf := float64(n) str := humanSI(nf, 0) fullStr := humanFull(nf, 0) if str != fullStr { return fmt.Sprintf("%s\t(%s)", str, fullStr) } return str } // humanNumberOrNA is like humanNumber but returns "N/A" for non-positive values. func humanNumberOrNA[T constraints.Float | constraints.Integer](n T) string { if n <= 0 { return "N/A" } return humanNumber(n) } // humanFloatOrNA formats a float with 1 decimal place, returning "N/A" for non-positive values. // This is separate from humanNumberOrNA because it provides simple decimal formatting for // continuous metrics (averages, rates) rather than SI unit formatting used for discrete counts. func humanFloatOrNA(val float64) string { if val <= 0 { return "N/A" } return humanFull(val, 1) } func humanSI[T constraints.Float | constraints.Integer](val T, decimals int) string { v, unit := humanize.ComputeSI(float64(val)) return fmt.Sprintf("%s%s", humanFull(v, decimals), unit) } func humanInt[T constraints.Integer](val T) string { return humanFull(float64(val), 0) } func humanFull(val float64, decimals int) string { return humanize.CommafWithDigits(val, decimals) } // provideCIDSync performs a synchronous/blocking provide operation to announce // the given CID to the DHT. // // - If the accelerated DHT client is used, a DHT lookup isn't needed, we // directly allocate provider records to closest peers. // - If Provide.DHT.SweepEnabled=true or OptimisticProvide=true, we make an // optimistic provide call. // - Else we make a standard provide call (much slower). // // IMPORTANT: The caller MUST verify DHT availability using HasActiveDHTClient() // before calling this function. Calling with a nil or invalid router will cause // a panic - this is the caller's responsibility to prevent. func provideCIDSync(ctx context.Context, router routing.Routing, c cid.Cid) error { return router.Provide(ctx, c, true) } ================================================ FILE: core/commands/pubsub.go ================================================ package commands import ( "context" "errors" "fmt" "io" "net/http" "slices" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" cmds "github.com/ipfs/go-ipfs-cmds" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" options "github.com/ipfs/kubo/core/coreiface/options" "github.com/ipfs/kubo/core/node/libp2p" "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" ) var PubsubCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "An experimental publish-subscribe system on ipfs.", ShortDescription: ` ipfs pubsub allows you to publish messages to a given topic, and also to subscribe to new messages on a given topic. EXPERIMENTAL FEATURE This is an opt-in feature optimized for IPNS over PubSub (https://specs.ipfs.tech/ipns/ipns-pubsub-router/). The default message validator is designed for IPNS record protocol. For custom pubsub applications requiring different validation logic, use go-libp2p-pubsub (https://github.com/libp2p/go-libp2p-pubsub) directly in a dedicated binary. To enable, set 'Pubsub.Enabled' config to true. `, }, Subcommands: map[string]*cmds.Command{ "pub": PubsubPubCmd, "sub": PubsubSubCmd, "ls": PubsubLsCmd, "peers": PubsubPeersCmd, "reset": PubsubResetCmd, }, } type pubsubMessage struct { From string `json:"from,omitempty"` Data string `json:"data,omitempty"` Seqno string `json:"seqno,omitempty"` TopicIDs []string `json:"topicIDs,omitempty"` } var PubsubSubCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Subscribe to messages on a given topic.", ShortDescription: ` ipfs pubsub sub subscribes to messages on a given topic. EXPERIMENTAL FEATURE This is an opt-in feature optimized for IPNS over PubSub (https://specs.ipfs.tech/ipns/ipns-pubsub-router/). To enable, set 'Pubsub.Enabled' config to true. PEER ENCODING Peer IDs in From fields are encoded using the default text representation from go-libp2p. This ensures the same string values as in 'ipfs pubsub peers'. TOPIC AND DATA ENCODING Topics, Data and Seqno are binary data. To ensure all bytes are transferred correctly the RPC client and server will use multibase encoding behind the scenes. You can inspect the format by passing --enc=json. The ipfs multibase commands can be used for encoding/decoding multibase strings in the userland. `, }, Arguments: []cmds.Argument{ cmds.StringArg("topic", true, false, "Name of topic to subscribe to (multibase encoded when sent over HTTP RPC)."), }, PreRun: urlArgsEncoder, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } if err := urlArgsDecoder(req, env); err != nil { return err } topic := req.Arguments[0] sub, err := api.PubSub().Subscribe(req.Context, topic) if err != nil { return err } defer sub.Close() if f, ok := res.(http.Flusher); ok { f.Flush() } for { msg, err := sub.Next(req.Context) if err == io.EOF || err == context.Canceled { return nil } else if err != nil { return err } // turn bytes into strings encoder, _ := mbase.EncoderByName("base64url") psm := pubsubMessage{ Data: encoder.Encode(msg.Data()), From: msg.From().String(), Seqno: encoder.Encode(msg.Seq()), } for _, topic := range msg.Topics() { psm.TopicIDs = append(psm.TopicIDs, encoder.Encode([]byte(topic))) } if err := res.Emit(&psm); err != nil { return err } } }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, psm *pubsubMessage) error { _, dec, err := mbase.Decode(psm.Data) if err != nil { return err } _, err = w.Write(dec) return err }), // DEPRECATED, undocumented format we used in tests, but not anymore // \n\n "ndpayload": cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, psm *pubsubMessage) error { return errors.New("--enc=ndpayload was removed, use --enc=json instead") }), // DEPRECATED, uncodumented format we used in tests, but not anymore // "lenpayload": cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, psm *pubsubMessage) error { return errors.New("--enc=lenpayload was removed, use --enc=json instead") }), }, Type: pubsubMessage{}, } var PubsubPubCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Publish data to a given pubsub topic.", ShortDescription: ` ipfs pubsub pub publishes a message to a specified topic. It reads binary data from stdin or a file. EXPERIMENTAL FEATURE This is an opt-in feature optimized for IPNS over PubSub (https://specs.ipfs.tech/ipns/ipns-pubsub-router/). To enable, set 'Pubsub.Enabled' config to true. HTTP RPC ENCODING The data to be published is sent in HTTP request body as multipart/form-data. Topic names are binary data too. To ensure all bytes are transferred correctly via URL params, the RPC client and server will use multibase encoding behind the scenes. `, }, Arguments: []cmds.Argument{ cmds.StringArg("topic", true, false, "Topic to publish to (multibase encoded when sent over HTTP RPC)."), cmds.FileArg("data", true, false, "The data to be published.").EnableStdin(), }, PreRun: urlArgsEncoder, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } if err := urlArgsDecoder(req, env); err != nil { return err } topic := req.Arguments[0] // read data passed as a file file, err := cmdenv.GetFileArg(req.Files.Entries()) if err != nil { return err } defer file.Close() data, err := io.ReadAll(file) if err != nil { return err } // publish return api.PubSub().Publish(req.Context, topic, data) }, } var PubsubLsCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "List subscribed topics by name.", ShortDescription: ` ipfs pubsub ls lists out the names of topics you are currently subscribed to. EXPERIMENTAL FEATURE This is an opt-in feature optimized for IPNS over PubSub (https://specs.ipfs.tech/ipns/ipns-pubsub-router/). To enable, set 'Pubsub.Enabled' config to true. TOPIC ENCODING Topic names are a binary data. To ensure all bytes are transferred correctly RPC client and server will use multibase encoding behind the scenes. You can inspect the format by passing --enc=json. ipfs multibase commands can be used for encoding/decoding multibase strings in the userland. `, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } l, err := api.PubSub().Ls(req.Context) if err != nil { return err } // emit topics encoded in multibase encoder, _ := mbase.EncoderByName("base64url") for n, topic := range l { l[n] = encoder.Encode([]byte(topic)) } return cmds.EmitOnce(res, stringList{l}) }, Type: stringList{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(multibaseDecodedStringListEncoder), }, } func multibaseDecodedStringListEncoder(req *cmds.Request, w io.Writer, list *stringList) error { for n, mb := range list.Strings { _, data, err := mbase.Decode(mb) if err != nil { return err } list.Strings[n] = string(data) } return safeTextListEncoder(req, w, list) } // converts list of strings to text representation where each string is placed // in separate line with non-printable/unsafe characters escaped // (this protects terminal output from being mangled by non-ascii topic names) func safeTextListEncoder(req *cmds.Request, w io.Writer, list *stringList) error { for _, str := range list.Strings { _, err := fmt.Fprintf(w, "%s\n", cmdenv.EscNonPrint(str)) if err != nil { return err } } return nil } var PubsubPeersCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "List peers we are currently pubsubbing with.", ShortDescription: ` ipfs pubsub peers with no arguments lists out the pubsub peers you are currently connected to. If given a topic, it will list connected peers who are subscribed to the named topic. EXPERIMENTAL FEATURE This is an opt-in feature optimized for IPNS over PubSub (https://specs.ipfs.tech/ipns/ipns-pubsub-router/). To enable, set 'Pubsub.Enabled' config to true. TOPIC AND DATA ENCODING Topic names are a binary data. To ensure all bytes are transferred correctly RPC client and server will use multibase encoding behind the scenes. You can inspect the format by passing --enc=json. ipfs multibase commands can be used for encoding/decoding multibase strings in the userland. `, }, Arguments: []cmds.Argument{ cmds.StringArg("topic", false, false, "Topic to list connected peers of."), }, PreRun: urlArgsEncoder, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } if err := urlArgsDecoder(req, env); err != nil { return err } var topic string if len(req.Arguments) == 1 { topic = req.Arguments[0] } peers, err := api.PubSub().Peers(req.Context, options.PubSub.Topic(topic)) if err != nil { return err } list := &stringList{make([]string, 0, len(peers))} for _, peer := range peers { list.Strings = append(list.Strings, peer.String()) } slices.Sort(list.Strings) return cmds.EmitOnce(res, list) }, Type: stringList{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder), }, } // TODO: move to cmdenv? // Encode binary data to be passed as multibase string in URL arguments. // (avoiding issues described in https://github.com/ipfs/kubo/issues/7939) func urlArgsEncoder(req *cmds.Request, env cmds.Environment) error { encoder, _ := mbase.EncoderByName("base64url") for n, arg := range req.Arguments { req.Arguments[n] = encoder.Encode([]byte(arg)) } return nil } // Decode binary data passed as multibase string in URL arguments. // (avoiding issues described in https://github.com/ipfs/kubo/issues/7939) func urlArgsDecoder(req *cmds.Request, env cmds.Environment) error { for n, arg := range req.Arguments { encoding, data, err := mbase.Decode(arg) if err != nil { return fmt.Errorf("URL arg must be multibase encoded: %w", err) } // Enforce URL-safe encoding is used for data passed via URL arguments // - without this we get data corruption similar to https://github.com/ipfs/kubo/issues/7939 // - we can't just deny base64, because there may be other bases that // are not URL-safe – better to force base64url which is known to be // safe in URL context if encoding != mbase.Base64url { return errors.New("URL arg must be base64url encoded") } req.Arguments[n] = string(data) } return nil } type pubsubResetResult struct { Deleted int64 `json:"deleted"` } var PubsubResetCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Reset pubsub validator state.", ShortDescription: ` Clears persistent sequence number state used by the pubsub validator. WARNING: FOR TESTING ONLY - DO NOT USE IN PRODUCTION Resets validator state that protects against replay attacks. After reset, previously seen messages may be accepted again until their sequence numbers are re-learned. Use cases: - Testing pubsub functionality - Recovery from a peer sending artificially high sequence numbers (which would cause subsequent messages from that peer to be rejected) The --peer flag limits the reset to a specific peer's state. Without --peer, all validator state is cleared. NOTE: This only resets the persistent seqno validator state. The in-memory seen messages cache (Pubsub.SeenMessagesTTL) auto-expires and can only be fully cleared by restarting the daemon. `, }, Options: []cmds.Option{ cmds.StringOption(peerOptionName, "p", "Only reset state for this peer ID"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := cmdenv.GetNode(env) if err != nil { return err } ds := n.Repo.Datastore() ctx := req.Context peerOpt, _ := req.Options[peerOptionName].(string) var deleted int64 if peerOpt != "" { // Reset specific peer pid, err := peer.Decode(peerOpt) if err != nil { return fmt.Errorf("invalid peer ID: %w", err) } key := datastore.NewKey(libp2p.SeqnoStorePrefix + pid.String()) exists, err := ds.Has(ctx, key) if err != nil { return fmt.Errorf("failed to check seqno state: %w", err) } if exists { if err := ds.Delete(ctx, key); err != nil { return fmt.Errorf("failed to delete seqno state: %w", err) } deleted = 1 } } else { // Reset all peers using batched delete for efficiency q := query.Query{ Prefix: libp2p.SeqnoStorePrefix, KeysOnly: true, } results, err := ds.Query(ctx, q) if err != nil { return fmt.Errorf("failed to query seqno state: %w", err) } defer results.Close() batch, err := ds.Batch(ctx) if err != nil { return fmt.Errorf("failed to create batch: %w", err) } for result := range results.Next() { if result.Error != nil { return fmt.Errorf("query error: %w", result.Error) } if err := batch.Delete(ctx, datastore.NewKey(result.Key)); err != nil { return fmt.Errorf("failed to batch delete key %s: %w", result.Key, err) } deleted++ } if err := batch.Commit(ctx); err != nil { return fmt.Errorf("failed to commit batch delete: %w", err) } } // Sync to ensure deletions are persisted if err := ds.Sync(ctx, datastore.NewKey(libp2p.SeqnoStorePrefix)); err != nil { return fmt.Errorf("failed to sync datastore: %w", err) } return cmds.EmitOnce(res, &pubsubResetResult{Deleted: deleted}) }, Type: pubsubResetResult{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, result *pubsubResetResult) error { peerOpt, _ := req.Options[peerOptionName].(string) if peerOpt != "" { if result.Deleted == 0 { _, err := fmt.Fprintf(w, "No validator state found for peer %s\n", peerOpt) return err } _, err := fmt.Fprintf(w, "Reset validator state for peer %s\n", peerOpt) return err } _, err := fmt.Fprintf(w, "Reset validator state for %d peer(s)\n", result.Deleted) return err }), }, } ================================================ FILE: core/commands/refs.go ================================================ package commands import ( "context" "errors" "fmt" "io" "strings" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" merkledag "github.com/ipfs/boxo/ipld/merkledag" cid "github.com/ipfs/go-cid" cidenc "github.com/ipfs/go-cidutil/cidenc" cmds "github.com/ipfs/go-ipfs-cmds" ipld "github.com/ipfs/go-ipld-format" iface "github.com/ipfs/kubo/core/coreiface" ) var refsEncoderMap = cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RefWrapper) error { if out.Err != "" { return errors.New(out.Err) } fmt.Fprintln(w, out.Ref) return nil }), } // KeyList is a general type for outputting lists of keys type KeyList struct { Keys []cid.Cid } const ( refsFormatOptionName = "format" refsEdgesOptionName = "edges" refsUniqueOptionName = "unique" refsRecursiveOptionName = "recursive" refsMaxDepthOptionName = "max-depth" ) // RefsCmd is the `ipfs refs` command var RefsCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List links (references) from an object.", ShortDescription: ` Lists the hashes of all the links an IPFS or IPNS object(s) contains, with the following format: List all references recursively by using the flag '-r'. NOTE: Like most other commands, Kubo will try to fetch the blocks of the passed path if they can't be found in the local store if it is running in online mode. `, }, Subcommands: map[string]*cmds.Command{ "local": RefsLocalCmd, }, Arguments: []cmds.Argument{ cmds.StringArg("ipfs-path", true, true, "Path to the object(s) to list refs from.").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption(refsFormatOptionName, "Emit edges with given format. Available tokens: .").WithDefault(""), cmds.BoolOption(refsEdgesOptionName, "e", "Emit edge format: ` -> `."), cmds.BoolOption(refsUniqueOptionName, "u", "Omit duplicate refs from output."), cmds.BoolOption(refsRecursiveOptionName, "r", "Recursively list links of child nodes."), cmds.IntOption(refsMaxDepthOptionName, "Only for recursive refs, limits fetch and listing to the given depth").WithDefault(-1), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { err := req.ParseBodyArgs() if err != nil { return err } ctx := req.Context api, err := cmdenv.GetApi(env, req) if err != nil { return err } enc, err := cmdenv.GetCidEncoder(req) if err != nil { return err } unique, _ := req.Options[refsUniqueOptionName].(bool) recursive, _ := req.Options[refsRecursiveOptionName].(bool) maxDepth, _ := req.Options[refsMaxDepthOptionName].(int) edges, _ := req.Options[refsEdgesOptionName].(bool) format, _ := req.Options[refsFormatOptionName].(string) if !recursive { maxDepth = 1 // write only direct refs } if edges { if format != "" { return errors.New("using format argument with edges is not allowed") } format = " -> " } // TODO: use session for resolving as well. objs, err := objectsForPaths(ctx, api, req.Arguments) if err != nil { return err } rw := RefWriter{ res: res, DAG: merkledag.NewSession(ctx, api.Dag()), Ctx: ctx, Unique: unique, PrintFmt: format, MaxDepth: maxDepth, } for _, o := range objs { if _, err := rw.WriteRefs(o, enc); err != nil { if err := res.Emit(&RefWrapper{Err: err.Error()}); err != nil { return err } } } return nil }, Encoders: refsEncoderMap, Type: RefWrapper{}, } var RefsLocalCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List all local references.", ShortDescription: ` Displays the hashes of all local objects. NOTE: This treats all local objects as "raw blocks" and returns CIDv1-Raw CIDs. `, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx := req.Context n, err := cmdenv.GetNode(env) if err != nil { return err } // todo: make async allKeys, err := n.Blockstore.AllKeysChan(ctx) if err != nil { return err } for k := range allKeys { err := res.Emit(&RefWrapper{Ref: k.String()}) if err != nil { return err } } return nil }, Encoders: refsEncoderMap, Type: RefWrapper{}, } func objectsForPaths(ctx context.Context, n iface.CoreAPI, paths []string) ([]cid.Cid, error) { roots := make([]cid.Cid, len(paths)) for i, sp := range paths { p, err := cmdutils.PathOrCidPath(sp) if err != nil { return nil, err } o, _, err := n.ResolvePath(ctx, p) if err != nil { return nil, err } roots[i] = o.RootCid() } return roots, nil } type RefWrapper struct { Ref string Err string } type RefWriter struct { res cmds.ResponseEmitter DAG ipld.NodeGetter Ctx context.Context Unique bool MaxDepth int PrintFmt string seen map[string]int } // WriteRefs writes refs of the given object to the underlying writer. func (rw *RefWriter) WriteRefs(c cid.Cid, enc cidenc.Encoder) (int, error) { n, err := rw.DAG.Get(rw.Ctx, c) if err != nil { return 0, err } return rw.writeRefsRecursive(n, 0, enc) } func (rw *RefWriter) writeRefsRecursive(n ipld.Node, depth int, enc cidenc.Encoder) (int, error) { nc := n.Cid() var count int for i, ng := range ipld.GetDAG(rw.Ctx, rw.DAG, n) { lc := n.Links()[i].Cid goDeeper, shouldWrite := rw.visit(lc, depth+1) // The children are at depth+1 // Avoid "Get()" on the node and continue with next Link. // We can do this if: // - We printed it before (thus it was already seen and // fetched with Get() // - AND we must not go deeper. // This is an optimization for pruned branches which have been // visited before. if !shouldWrite && !goDeeper { continue } // We must Get() the node because: // - it is new (never written) // - OR we need to go deeper. // This ensures printed refs are always fetched. nd, err := ng.Get(rw.Ctx) if err != nil { return count, err } // Write this node if not done before (or !Unique) if shouldWrite { if err := rw.WriteEdge(nc, lc, n.Links()[i].Name, enc); err != nil { return count, err } count++ } // Keep going deeper. This happens: // - On unexplored branches // - On branches not explored deep enough // Note when !Unique, branches are always considered // unexplored and only depth limits apply. if goDeeper { c, err := rw.writeRefsRecursive(nd, depth+1, enc) count += c if err != nil { return count, err } } } return count, nil } // visit returns two values: // - the first boolean is true if we should keep traversing the DAG // - the second boolean is true if we should print the CID // // visit will do branch pruning depending on rw.MaxDepth, previously visited // cids and whether rw.Unique is set. i.e. rw.Unique = false and // rw.MaxDepth = -1 disables any pruning. But setting rw.Unique to true will // prune already visited branches at the cost of keeping as set of visited // CIDs in memory. func (rw *RefWriter) visit(c cid.Cid, depth int) (bool, bool) { atMaxDepth := rw.MaxDepth >= 0 && depth == rw.MaxDepth overMaxDepth := rw.MaxDepth >= 0 && depth > rw.MaxDepth // Shortcut when we are over max depth. In practice, this // only applies when calling refs with --maxDepth=0, as root's // children are already over max depth. Otherwise nothing should // hit this. if overMaxDepth { return false, false } // We can shortcut right away if we don't need unique output: // - we keep traversing when not atMaxDepth // - always print if !rw.Unique { return !atMaxDepth, true } // Unique == true from this point. // Thus, we keep track of seen Cids, and their depth. if rw.seen == nil { rw.seen = make(map[string]int) } key := string(c.Bytes()) oldDepth, ok := rw.seen[key] // Unique == true && depth < MaxDepth (or unlimited) from this point // Branch pruning cases: // - We saw the Cid before and either: // - Depth is unlimited (MaxDepth = -1) // - We saw it higher (smaller depth) in the DAG (means we must have // explored deep enough before) // Because we saw the CID, we don't print it again. if ok && (rw.MaxDepth < 0 || oldDepth <= depth) { return false, false } // Final case, we must keep exploring the DAG from this CID // (unless we hit the depth limit). // We note down its depth because it was either not seen // or is lower than last time. // We print if it was not seen. rw.seen[key] = depth return !atMaxDepth, !ok } // Write one edge func (rw *RefWriter) WriteEdge(from, to cid.Cid, linkname string, enc cidenc.Encoder) error { if rw.Ctx != nil { select { case <-rw.Ctx.Done(): // just in case. return rw.Ctx.Err() default: } } var s string switch { case rw.PrintFmt != "": s = rw.PrintFmt s = strings.Replace(s, "", enc.Encode(from), -1) s = strings.Replace(s, "", enc.Encode(to), -1) s = strings.Replace(s, "", linkname, -1) default: s += enc.Encode(to) } return rw.res.Emit(&RefWrapper{Ref: s}) } ================================================ FILE: core/commands/repo.go ================================================ package commands import ( "context" "errors" "fmt" "io" "runtime" "strings" "sync" "text/tabwriter" "time" oldcmds "github.com/ipfs/kubo/commands" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" coreiface "github.com/ipfs/kubo/core/coreiface" corerepo "github.com/ipfs/kubo/core/corerepo" fsrepo "github.com/ipfs/kubo/repo/fsrepo" "github.com/ipfs/kubo/repo/fsrepo/migrations" humanize "github.com/dustin/go-humanize" bstore "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/path" cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" ) type RepoVersion struct { Version string } var RepoCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Manipulate the IPFS repo.", ShortDescription: ` 'ipfs repo' is a plumbing command used to manipulate the repo. `, }, Subcommands: map[string]*cmds.Command{ "stat": repoStatCmd, "gc": repoGcCmd, "version": repoVersionCmd, "verify": repoVerifyCmd, "migrate": repoMigrateCmd, "ls": RefsLocalCmd, }, } // GcResult is the result returned by "repo gc" command. type GcResult struct { Key cid.Cid Error string `json:",omitempty"` } const ( repoStreamErrorsOptionName = "stream-errors" repoQuietOptionName = "quiet" repoSilentOptionName = "silent" repoAllowDowngradeOptionName = "allow-downgrade" repoToVersionOptionName = "to" ) var repoGcCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Perform a garbage collection sweep on the repo.", ShortDescription: ` 'ipfs repo gc' is a plumbing command that will sweep the local set of stored objects and remove ones that are not pinned in order to reclaim hard disk space. `, }, Options: []cmds.Option{ cmds.BoolOption(repoStreamErrorsOptionName, "Stream errors."), cmds.BoolOption(repoQuietOptionName, "q", "Write minimal output."), cmds.BoolOption(repoSilentOptionName, "Write no output."), }, Run: func(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment) error { n, err := cmdenv.GetNode(env) if err != nil { return err } silent, _ := req.Options[repoSilentOptionName].(bool) streamErrors, _ := req.Options[repoStreamErrorsOptionName].(bool) gcOutChan := corerepo.GarbageCollectAsync(n, req.Context) if streamErrors { errs := false for res := range gcOutChan { if res.Error != nil { if err := re.Emit(&GcResult{Error: res.Error.Error()}); err != nil { return err } errs = true } else { if err := re.Emit(&GcResult{Key: res.KeyRemoved}); err != nil { return err } } } if errs { return errors.New("encountered errors during gc run") } } else { err := corerepo.CollectResult(req.Context, gcOutChan, func(k cid.Cid) { if silent { return } // Nothing to do with this error, really. This // most likely means that the client is gone but // we still need to let the GC finish. _ = re.Emit(&GcResult{Key: k}) }) if err != nil { return err } } return nil }, Type: GcResult{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, gcr *GcResult) error { quiet, _ := req.Options[repoQuietOptionName].(bool) silent, _ := req.Options[repoSilentOptionName].(bool) if silent { return nil } if gcr.Error != "" { _, err := fmt.Fprintf(w, "Error: %s\n", gcr.Error) return err } prefix := "removed " if quiet { prefix = "" } _, err := fmt.Fprintf(w, "%s%s\n", prefix, gcr.Key) return err }), }, } const ( repoSizeOnlyOptionName = "size-only" repoHumanOptionName = "human" ) var repoStatCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Get stats for the currently used repo.", ShortDescription: ` 'ipfs repo stat' provides information about the local set of stored objects. It outputs: RepoSize int Size in bytes that the repo is currently taking. StorageMax string Maximum datastore size (from configuration) NumObjects int Number of objects in the local repo. RepoPath string The path to the repo being currently used. Version string The repo version. `, }, Options: []cmds.Option{ cmds.BoolOption(repoSizeOnlyOptionName, "s", "Only report RepoSize and StorageMax."), cmds.BoolOption(repoHumanOptionName, "H", "Print sizes in human readable format (e.g., 1K 234M 2G)"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := cmdenv.GetNode(env) if err != nil { return err } sizeOnly, _ := req.Options[repoSizeOnlyOptionName].(bool) if sizeOnly { sizeStat, err := corerepo.RepoSize(req.Context, n) if err != nil { return err } return cmds.EmitOnce(res, &corerepo.Stat{ SizeStat: sizeStat, }) } stat, err := corerepo.RepoStat(req.Context, n) if err != nil { return err } return cmds.EmitOnce(res, &stat) }, Type: &corerepo.Stat{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, stat *corerepo.Stat) error { wtr := tabwriter.NewWriter(w, 0, 0, 1, ' ', 0) defer wtr.Flush() human, _ := req.Options[repoHumanOptionName].(bool) sizeOnly, _ := req.Options[repoSizeOnlyOptionName].(bool) printSize := func(name string, size uint64) { sizeStr := fmt.Sprintf("%d", size) if human { sizeStr = humanize.Bytes(size) } fmt.Fprintf(wtr, "%s:\t%s\n", name, sizeStr) } if !sizeOnly { fmt.Fprintf(wtr, "NumObjects:\t%d\n", stat.NumObjects) } printSize("RepoSize", stat.RepoSize) printSize("StorageMax", stat.StorageMax) if !sizeOnly { fmt.Fprintf(wtr, "RepoPath:\t%s\n", stat.RepoPath) fmt.Fprintf(wtr, "Version:\t%s\n", stat.Version) } return nil }), }, } // VerifyProgress reports verification progress to the user. // It contains either a message about a corrupt block or a progress counter. type VerifyProgress struct { Msg string // Message about a corrupt/healed block (empty for valid blocks) Progress int // Number of blocks processed so far } // verifyState represents the state of a block after verification. // States track both the verification result and any remediation actions taken. type verifyState int const ( verifyStateValid verifyState = iota // Block is valid and uncorrupted verifyStateCorrupt // Block is corrupt, no action taken verifyStateCorruptRemoved // Block was corrupt and successfully removed verifyStateCorruptRemoveFailed // Block was corrupt but removal failed verifyStateCorruptHealed // Block was corrupt, removed, and successfully re-fetched verifyStateCorruptHealFailed // Block was corrupt and removed, but re-fetching failed ) const ( // verifyWorkerMultiplier determines worker pool size relative to CPU count. // Since block verification is I/O-bound (disk reads + potential network fetches), // we use more workers than CPU cores to maximize throughput. verifyWorkerMultiplier = 2 ) // verifyResult contains the outcome of verifying a single block. // It includes the block's CID, its verification state, and an optional // human-readable message describing what happened. type verifyResult struct { cid cid.Cid // CID of the block that was verified state verifyState // Final state after verification and any remediation msg string // Human-readable message (empty for valid blocks) } // verifyWorkerRun processes CIDs from the keys channel, verifying their integrity. // If shouldDrop is true, corrupt blocks are removed from the blockstore. // If shouldHeal is true (implies shouldDrop), removed blocks are re-fetched from the network. // The api parameter must be non-nil when shouldHeal is true. // healTimeout specifies the maximum time to wait for each block heal (0 = no timeout). func verifyWorkerRun(ctx context.Context, wg *sync.WaitGroup, keys <-chan cid.Cid, results chan<- *verifyResult, bs bstore.Blockstore, api coreiface.CoreAPI, shouldDrop, shouldHeal bool, healTimeout time.Duration) { defer wg.Done() sendResult := func(r *verifyResult) bool { select { case results <- r: return true case <-ctx.Done(): return false } } for k := range keys { _, err := bs.Get(ctx, k) if err != nil { // Block is corrupt result := &verifyResult{cid: k, state: verifyStateCorrupt} if !shouldDrop { result.msg = fmt.Sprintf("block %s was corrupt (%s)", k, err) if !sendResult(result) { return } continue } // Try to delete if delErr := bs.DeleteBlock(ctx, k); delErr != nil { result.state = verifyStateCorruptRemoveFailed result.msg = fmt.Sprintf("block %s was corrupt (%s), failed to remove (%s)", k, err, delErr) if !sendResult(result) { return } continue } if !shouldHeal { result.state = verifyStateCorruptRemoved result.msg = fmt.Sprintf("block %s was corrupt (%s), removed", k, err) if !sendResult(result) { return } continue } // Try to heal by re-fetching from network (api is guaranteed non-nil here) healCtx := ctx var healCancel context.CancelFunc if healTimeout > 0 { healCtx, healCancel = context.WithTimeout(ctx, healTimeout) } if _, healErr := api.Block().Get(healCtx, path.FromCid(k)); healErr != nil { result.state = verifyStateCorruptHealFailed result.msg = fmt.Sprintf("block %s was corrupt (%s), removed, failed to heal (%s)", k, err, healErr) } else { result.state = verifyStateCorruptHealed result.msg = fmt.Sprintf("block %s was corrupt (%s), removed, healed", k, err) } if healCancel != nil { healCancel() } if !sendResult(result) { return } continue } // Block is valid if !sendResult(&verifyResult{cid: k, state: verifyStateValid}) { return } } } // verifyResultChan creates a channel of verification results by spawning multiple worker goroutines // to process blocks in parallel. It returns immediately with a channel that will receive results. func verifyResultChan(ctx context.Context, keys <-chan cid.Cid, bs bstore.Blockstore, api coreiface.CoreAPI, shouldDrop, shouldHeal bool, healTimeout time.Duration) <-chan *verifyResult { results := make(chan *verifyResult) go func() { defer close(results) var wg sync.WaitGroup for i := 0; i < runtime.NumCPU()*verifyWorkerMultiplier; i++ { wg.Add(1) go verifyWorkerRun(ctx, &wg, keys, results, bs, api, shouldDrop, shouldHeal, healTimeout) } wg.Wait() }() return results } var repoVerifyCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Verify all blocks in repo are not corrupted.", ShortDescription: ` 'ipfs repo verify' checks integrity of all blocks in the local datastore. Each block is read and validated against its CID to ensure data integrity. Without any flags, this is a SAFE, read-only check that only reports corrupt blocks without modifying the repository. This can be used as a "dry run" to preview what --drop or --heal would do. Use --drop to remove corrupt blocks, or --heal to remove and re-fetch from the network. Examples: ipfs repo verify # safe read-only check, reports corrupt blocks ipfs repo verify --drop # remove corrupt blocks ipfs repo verify --heal # remove and re-fetch corrupt blocks Exit Codes: 0: All blocks are valid, OR all corrupt blocks were successfully remediated (with --drop or --heal) 1: Corrupt blocks detected (without flags), OR remediation failed (block removal or healing failed with --drop or --heal) Note: --heal requires the daemon to be running in online mode with network connectivity to nodes that have the missing blocks. Make sure the daemon is online and connected to other peers. Healing will attempt to re-fetch each corrupt block from the network after removing it. If a block cannot be found on the network, it will remain deleted. WARNING: Both --drop and --heal are DESTRUCTIVE operations that permanently delete corrupt blocks from your repository. Once deleted, blocks cannot be recovered unless --heal successfully fetches them from the network. Blocks that cannot be healed will remain permanently deleted. Always backup your repository before using these options. `, }, Options: []cmds.Option{ cmds.BoolOption("drop", "Remove corrupt blocks from datastore (destructive operation)."), cmds.BoolOption("heal", "Remove corrupt blocks and re-fetch from network (destructive operation, implies --drop)."), cmds.StringOption("heal-timeout", "Maximum time to wait for each block heal (e.g., \"30s\"). Only applies with --heal.").WithDefault("30s"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } drop, _ := req.Options["drop"].(bool) heal, _ := req.Options["heal"].(bool) if heal { drop = true // heal implies drop } // Parse and validate heal-timeout timeoutStr, _ := req.Options["heal-timeout"].(string) healTimeout, err := time.ParseDuration(timeoutStr) if err != nil { return fmt.Errorf("invalid heal-timeout: %w", err) } if healTimeout < 0 { return errors.New("heal-timeout must be >= 0") } // Check online mode and API availability for healing operation var api coreiface.CoreAPI if heal { if !nd.IsOnline { return ErrNotOnline } api, err = cmdenv.GetApi(env, req) if err != nil { return err } if api == nil { return fmt.Errorf("healing requested but API is not available - make sure daemon is online and connected to other peers") } } bs := &bstore.ValidatingBlockstore{Blockstore: bstore.NewBlockstore(nd.Repo.Datastore())} keys, err := bs.AllKeysChan(req.Context) if err != nil { log.Error(err) return err } results := verifyResultChan(req.Context, keys, bs, api, drop, heal, healTimeout) // Track statistics for each type of outcome var corrupted, removed, removeFailed, healed, healFailed int var i int for result := range results { // Update counters based on the block's final state switch result.state { case verifyStateCorrupt: // Block is corrupt but no action was taken (--drop not specified) corrupted++ case verifyStateCorruptRemoved: // Block was corrupt and successfully removed (--drop specified) corrupted++ removed++ case verifyStateCorruptRemoveFailed: // Block was corrupt but couldn't be removed corrupted++ removeFailed++ case verifyStateCorruptHealed: // Block was corrupt, removed, and successfully re-fetched (--heal specified) corrupted++ removed++ healed++ case verifyStateCorruptHealFailed: // Block was corrupt and removed, but re-fetching failed corrupted++ removed++ healFailed++ default: // verifyStateValid blocks are not counted (they're the expected case) } // Emit progress message for corrupt blocks if result.state != verifyStateValid && result.msg != "" { if err := res.Emit(&VerifyProgress{Msg: result.msg}); err != nil { return err } } i++ if err := res.Emit(&VerifyProgress{Progress: i}); err != nil { return err } } if err := req.Context.Err(); err != nil { return err } if corrupted > 0 { // Build a summary of what happened with corrupt blocks summary := fmt.Sprintf("verify complete, %d blocks corrupt", corrupted) if removed > 0 { summary += fmt.Sprintf(", %d removed", removed) } if removeFailed > 0 { summary += fmt.Sprintf(", %d failed to remove", removeFailed) } if healed > 0 { summary += fmt.Sprintf(", %d healed", healed) } if healFailed > 0 { summary += fmt.Sprintf(", %d failed to heal", healFailed) } // Determine success/failure based on operation mode shouldFail := false if !drop { // Detection-only mode: always fail if corruption found shouldFail = true } else if heal { // Heal mode: fail if any removal or heal failed shouldFail = (removeFailed > 0 || healFailed > 0) } else { // Drop mode: fail if any removal failed shouldFail = (removeFailed > 0) } if shouldFail { return errors.New(summary) } // Success: emit summary as a message instead of error return res.Emit(&VerifyProgress{Msg: summary}) } return res.Emit(&VerifyProgress{Msg: "verify complete, all blocks validated."}) }, Type: &VerifyProgress{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, obj *VerifyProgress) error { if strings.Contains(obj.Msg, "was corrupt") { fmt.Fprintln(w, obj.Msg) return nil } if obj.Msg != "" { if len(obj.Msg) < 20 { obj.Msg += " " } fmt.Fprintln(w, obj.Msg) return nil } fmt.Fprintf(w, "%d blocks processed.\r", obj.Progress) return nil }), }, } var repoVersionCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Show the repo version.", ShortDescription: ` 'ipfs repo version' returns the current repo version. `, }, Options: []cmds.Option{ cmds.BoolOption(repoQuietOptionName, "q", "Write minimal output."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { return cmds.EmitOnce(res, &RepoVersion{ Version: fmt.Sprint(fsrepo.RepoVersion), }) }, Type: RepoVersion{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RepoVersion) error { quiet, _ := req.Options[repoQuietOptionName].(bool) if quiet { fmt.Fprintf(w, "fs-repo@%s\n", out.Version) } else { fmt.Fprintf(w, "ipfs repo version fs-repo@%s\n", out.Version) } return nil }), }, } var repoMigrateCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Apply repository migrations to a specific version.", ShortDescription: ` 'ipfs repo migrate' applies repository migrations to bring the repository to a specific version. By default, migrates to the latest version supported by this IPFS binary. Examples: ipfs repo migrate # Migrate to latest version ipfs repo migrate --to=17 # Migrate to version 17 ipfs repo migrate --to=16 --allow-downgrade # Downgrade to version 16 WARNING: Downgrading a repository may cause data loss and requires using an older IPFS binary that supports the target version. After downgrading, you must use an IPFS implementation compatible with that repository version. Repository versions 16+ use embedded migrations for faster, more reliable migration. Versions below 16 require external migration tools. `, }, Options: []cmds.Option{ cmds.IntOption(repoToVersionOptionName, "Target repository version").WithDefault(fsrepo.RepoVersion), cmds.BoolOption(repoAllowDowngradeOptionName, "Allow downgrading to a lower repo version"), }, NoRemote: true, // SetDoesNotUseRepo(true) might seem counter-intuitive since migrations // do access the repo, but it's correct - we need direct filesystem access // without going through the daemon. Migrations handle their own locking. Extra: CreateCmdExtras(SetDoesNotUseRepo(true)), Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cctx := env.(*oldcmds.Context) allowDowngrade, _ := req.Options[repoAllowDowngradeOptionName].(bool) targetVersion, _ := req.Options[repoToVersionOptionName].(int) // Get current repo version currentVersion, err := migrations.RepoVersion(cctx.ConfigRoot) if err != nil { return fmt.Errorf("could not get current repo version: %w", err) } // Check if migration is needed if currentVersion == targetVersion { fmt.Printf("Repository is already at version %d.\n", targetVersion) return nil } // Validate downgrade request if targetVersion < currentVersion && !allowDowngrade { return fmt.Errorf("downgrade from version %d to %d requires --allow-downgrade flag", currentVersion, targetVersion) } fmt.Printf("Migrating repository from version %d to %d...\n", currentVersion, targetVersion) // Use hybrid migration strategy that intelligently combines external and embedded migrations // Use req.Context instead of cctx.Context() to avoid opening the repo before migrations run, // which would acquire the lock that migrations need err = migrations.RunHybridMigrations(req.Context, targetVersion, cctx.ConfigRoot, allowDowngrade) if err != nil { fmt.Println("Repository migration failed:") fmt.Printf(" %s\n", err) fmt.Println("If you think this is a bug, please file an issue and include this whole log output.") fmt.Println(" https://github.com/ipfs/kubo") return err } fmt.Printf("Repository successfully migrated to version %d.\n", targetVersion) if targetVersion < fsrepo.RepoVersion { fmt.Println("WARNING: After downgrading, you must use an IPFS binary compatible with this repository version.") } return nil }, } ================================================ FILE: core/commands/repo_verify_test.go ================================================ //go:build go1.25 package commands // This file contains unit tests for the --heal-timeout flag functionality // using testing/synctest to avoid waiting for real timeouts. // // End-to-end tests for the full 'ipfs repo verify' command (including --drop // and --heal flags) are located in test/cli/repo_verify_test.go. import ( "bytes" "context" "errors" "io" "sync" "testing" "testing/synctest" "time" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" coreiface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/core/coreiface/options" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/ipfs/boxo/path" ) func TestVerifyWorkerHealTimeout(t *testing.T) { t.Run("heal succeeds before timeout", func(t *testing.T) { synctest.Test(t, func(t *testing.T) { const healTimeout = 5 * time.Second testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") // Setup channels keys := make(chan cid.Cid, 1) keys <- testCID close(keys) results := make(chan *verifyResult, 1) // Mock blockstore that returns error (simulating corruption) mockBS := &mockBlockstore{ getError: errors.New("corrupt block"), } // Mock API where Block().Get() completes before timeout mockAPI := &mockCoreAPI{ blockAPI: &mockBlockAPI{ getDelay: 2 * time.Second, // Less than healTimeout data: []byte("healed data"), }, } var wg sync.WaitGroup wg.Add(1) // Run worker go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout) // Advance time past the mock delay but before timeout time.Sleep(3 * time.Second) synctest.Wait() wg.Wait() close(results) // Verify heal succeeded result := <-results require.NotNil(t, result) assert.Equal(t, verifyStateCorruptHealed, result.state) assert.Contains(t, result.msg, "healed") }) }) t.Run("heal fails due to timeout", func(t *testing.T) { synctest.Test(t, func(t *testing.T) { const healTimeout = 2 * time.Second testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") // Setup channels keys := make(chan cid.Cid, 1) keys <- testCID close(keys) results := make(chan *verifyResult, 1) // Mock blockstore that returns error (simulating corruption) mockBS := &mockBlockstore{ getError: errors.New("corrupt block"), } // Mock API where Block().Get() takes longer than healTimeout mockAPI := &mockCoreAPI{ blockAPI: &mockBlockAPI{ getDelay: 5 * time.Second, // More than healTimeout data: []byte("healed data"), }, } var wg sync.WaitGroup wg.Add(1) // Run worker go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout) // Advance time past timeout time.Sleep(3 * time.Second) synctest.Wait() wg.Wait() close(results) // Verify heal failed due to timeout result := <-results require.NotNil(t, result) assert.Equal(t, verifyStateCorruptHealFailed, result.state) assert.Contains(t, result.msg, "failed to heal") assert.Contains(t, result.msg, "context deadline exceeded") }) }) t.Run("heal with zero timeout still attempts heal", func(t *testing.T) { synctest.Test(t, func(t *testing.T) { const healTimeout = 0 // Zero timeout means no timeout testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") // Setup channels keys := make(chan cid.Cid, 1) keys <- testCID close(keys) results := make(chan *verifyResult, 1) // Mock blockstore that returns error (simulating corruption) mockBS := &mockBlockstore{ getError: errors.New("corrupt block"), } // Mock API that succeeds quickly mockAPI := &mockCoreAPI{ blockAPI: &mockBlockAPI{ getDelay: 100 * time.Millisecond, data: []byte("healed data"), }, } var wg sync.WaitGroup wg.Add(1) // Run worker go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout) // Advance time to let heal complete time.Sleep(200 * time.Millisecond) synctest.Wait() wg.Wait() close(results) // Verify heal succeeded even with zero timeout result := <-results require.NotNil(t, result) assert.Equal(t, verifyStateCorruptHealed, result.state) assert.Contains(t, result.msg, "healed") }) }) t.Run("multiple blocks with different timeout outcomes", func(t *testing.T) { synctest.Test(t, func(t *testing.T) { const healTimeout = 3 * time.Second testCID1 := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") testCID2 := cid.MustParse("bafybeihvvulpp4evxj7x7armbqcyg6uezzuig6jp3lktpbovlqfkjtgyby") // Setup channels keys := make(chan cid.Cid, 2) keys <- testCID1 keys <- testCID2 close(keys) results := make(chan *verifyResult, 2) // Mock blockstore that always returns error (all blocks corrupt) mockBS := &mockBlockstore{ getError: errors.New("corrupt block"), } // Create two mock block APIs with different delays // We'll need to alternate which one gets used // For simplicity, use one that succeeds fast mockAPI := &mockCoreAPI{ blockAPI: &mockBlockAPI{ getDelay: 1 * time.Second, // Less than healTimeout - will succeed data: []byte("healed data"), }, } var wg sync.WaitGroup wg.Add(2) // Two workers // Run two workers go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout) go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout) // Advance time to let both complete time.Sleep(2 * time.Second) synctest.Wait() wg.Wait() close(results) // Collect results var healedCount int for result := range results { if result.state == verifyStateCorruptHealed { healedCount++ } } // Both should heal successfully (both under timeout) assert.Equal(t, 2, healedCount) }) }) t.Run("valid block is not healed", func(t *testing.T) { synctest.Test(t, func(t *testing.T) { const healTimeout = 5 * time.Second testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") // Setup channels keys := make(chan cid.Cid, 1) keys <- testCID close(keys) results := make(chan *verifyResult, 1) // Mock blockstore that returns valid block (no error) mockBS := &mockBlockstore{ block: blocks.NewBlock([]byte("valid data")), } // Mock API (won't be called since block is valid) mockAPI := &mockCoreAPI{ blockAPI: &mockBlockAPI{}, } var wg sync.WaitGroup wg.Add(1) // Run worker with heal enabled go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, false, true, healTimeout) synctest.Wait() wg.Wait() close(results) // Verify block is marked valid, not healed result := <-results require.NotNil(t, result) assert.Equal(t, verifyStateValid, result.state) assert.Empty(t, result.msg) }) }) } // mockBlockstore implements a minimal blockstore for testing type mockBlockstore struct { getError error block blocks.Block } func (m *mockBlockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) { if m.getError != nil { return nil, m.getError } return m.block, nil } func (m *mockBlockstore) DeleteBlock(ctx context.Context, c cid.Cid) error { return nil } func (m *mockBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) { return m.block != nil, nil } func (m *mockBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) { if m.block != nil { return len(m.block.RawData()), nil } return 0, errors.New("block not found") } func (m *mockBlockstore) Put(ctx context.Context, b blocks.Block) error { return nil } func (m *mockBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error { return nil } func (m *mockBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { return nil, errors.New("not implemented") } func (m *mockBlockstore) HashOnRead(enabled bool) { } // mockBlockAPI implements BlockAPI for testing type mockBlockAPI struct { getDelay time.Duration getError error data []byte } func (m *mockBlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { if m.getDelay > 0 { select { case <-time.After(m.getDelay): // Delay completed case <-ctx.Done(): return nil, ctx.Err() } } if m.getError != nil { return nil, m.getError } return bytes.NewReader(m.data), nil } func (m *mockBlockAPI) Put(ctx context.Context, r io.Reader, opts ...options.BlockPutOption) (coreiface.BlockStat, error) { return nil, errors.New("not implemented") } func (m *mockBlockAPI) Rm(ctx context.Context, p path.Path, opts ...options.BlockRmOption) error { return errors.New("not implemented") } func (m *mockBlockAPI) Stat(ctx context.Context, p path.Path) (coreiface.BlockStat, error) { return nil, errors.New("not implemented") } // mockCoreAPI implements minimal CoreAPI for testing type mockCoreAPI struct { blockAPI *mockBlockAPI } func (m *mockCoreAPI) Block() coreiface.BlockAPI { return m.blockAPI } func (m *mockCoreAPI) Unixfs() coreiface.UnixfsAPI { return nil } func (m *mockCoreAPI) Dag() coreiface.APIDagService { return nil } func (m *mockCoreAPI) Name() coreiface.NameAPI { return nil } func (m *mockCoreAPI) Key() coreiface.KeyAPI { return nil } func (m *mockCoreAPI) Pin() coreiface.PinAPI { return nil } func (m *mockCoreAPI) Object() coreiface.ObjectAPI { return nil } func (m *mockCoreAPI) Swarm() coreiface.SwarmAPI { return nil } func (m *mockCoreAPI) PubSub() coreiface.PubSubAPI { return nil } func (m *mockCoreAPI) Routing() coreiface.RoutingAPI { return nil } func (m *mockCoreAPI) ResolvePath(ctx context.Context, p path.Path) (path.ImmutablePath, []string, error) { return path.ImmutablePath{}, nil, errors.New("not implemented") } func (m *mockCoreAPI) ResolveNode(ctx context.Context, p path.Path) (ipld.Node, error) { return nil, errors.New("not implemented") } func (m *mockCoreAPI) WithOptions(...options.ApiOption) (coreiface.CoreAPI, error) { return nil, errors.New("not implemented") } ================================================ FILE: core/commands/resolve.go ================================================ package commands import ( "errors" "fmt" "io" "strings" "time" ns "github.com/ipfs/boxo/namesys" cidenc "github.com/ipfs/go-cidutil/cidenc" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" ncmd "github.com/ipfs/kubo/core/commands/name" "github.com/ipfs/boxo/path" cmds "github.com/ipfs/go-ipfs-cmds" options "github.com/ipfs/kubo/core/coreiface/options" ) const ( resolveRecursiveOptionName = "recursive" resolveDhtRecordCountOptionName = "dht-record-count" resolveDhtTimeoutOptionName = "dht-timeout" ) var ResolveCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Resolve the value of names to IPFS.", ShortDescription: ` There are a number of mutable name protocols that can link among themselves and into IPNS. This command accepts any of these identifiers and resolves them to the referenced item. `, LongDescription: ` There are a number of mutable name protocols that can link among themselves and into IPNS. For example IPNS references can (currently) point at an IPFS object, and DNS links can point at other DNS links, IPNS entries, or IPFS objects. This command accepts any of these identifiers and resolves them to the referenced item. EXAMPLES Resolve the value of your identity: $ ipfs resolve /ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj Resolve the value of another name: $ ipfs resolve /ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n /ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy Resolve the value of another name recursively: $ ipfs resolve -r /ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj Resolve the value of an IPFS DAG path: $ ipfs resolve /ipfs/QmeZy1fGbwgVSrqbfh9fKQrAWgeyRnj7h8fsHS1oy3k99x/beep/boop /ipfs/QmYRMjyvAiHKN9UTi8Bzt1HUspmSRD8T8DwxfSMzLgBon1 `, }, Arguments: []cmds.Argument{ cmds.StringArg("name", true, false, "The name to resolve.").EnableStdin(), }, Options: []cmds.Option{ cmds.BoolOption(resolveRecursiveOptionName, "r", "Resolve until the result is an IPFS name.").WithDefault(true), cmds.IntOption(resolveDhtRecordCountOptionName, "dhtrc", "Number of records to request for DHT resolution."), cmds.StringOption(resolveDhtTimeoutOptionName, "dhtt", "Max time to collect values during DHT resolution e.g. \"30s\". Pass 0 for no timeout."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } name := req.Arguments[0] recursive, _ := req.Options[resolveRecursiveOptionName].(bool) // the case when ipns is resolved step by step if strings.HasPrefix(name, "/ipns/") && !recursive { rc, rcok := req.Options[resolveDhtRecordCountOptionName].(uint) dhtt, dhttok := req.Options[resolveDhtTimeoutOptionName].(string) ropts := []options.NameResolveOption{ options.Name.ResolveOption(ns.ResolveWithDepth(1)), } if rcok { ropts = append(ropts, options.Name.ResolveOption(ns.ResolveWithDhtRecordCount(rc))) } if dhttok { d, err := time.ParseDuration(dhtt) if err != nil { return err } if d < 0 { return errors.New("DHT timeout value must be >= 0") } ropts = append(ropts, options.Name.ResolveOption(ns.ResolveWithDhtTimeout(d))) } p, err := api.Name().Resolve(req.Context, name, ropts...) // ErrResolveRecursion is fine if err != nil && err != ns.ErrResolveRecursion { return err } return cmds.EmitOnce(res, &ncmd.ResolvedPath{Path: p.String()}) } var enc cidenc.Encoder switch { case !cmdenv.CidBaseDefined(req) && !strings.HasPrefix(name, "/ipns/"): // Not specified, check the path. enc, err = cmdenv.CidEncoderFromPath(name) if err == nil { break } // Nope, fallback on the default. fallthrough default: enc, err = cmdenv.GetCidEncoder(req) if err != nil { return err } } p, err := cmdutils.PathOrCidPath(name) if err != nil { return err } // else, ipfs path or ipns with recursive flag rp, remainder, err := api.ResolvePath(req.Context, p) if err != nil { return err } // Trick to encode path with correct encoding. encodedPath := "/" + rp.Namespace() + "/" + enc.Encode(rp.RootCid()) if len(remainder) != 0 { encodedPath += path.SegmentsToString(remainder...) } // Ensure valid and sanitized. ep, err := path.NewPath(encodedPath) if err != nil { return err } return cmds.EmitOnce(res, &ncmd.ResolvedPath{Path: ep.String()}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, rp *ncmd.ResolvedPath) error { fmt.Fprintln(w, rp.Path) return nil }), }, Type: ncmd.ResolvedPath{}, } ================================================ FILE: core/commands/root.go ================================================ package commands import ( "errors" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" dag "github.com/ipfs/kubo/core/commands/dag" name "github.com/ipfs/kubo/core/commands/name" ocmd "github.com/ipfs/kubo/core/commands/object" "github.com/ipfs/kubo/core/commands/pin" cmds "github.com/ipfs/go-ipfs-cmds" logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("core/commands") var ( ErrNotOnline = errors.New("this command must be run in online mode. Try running 'ipfs daemon' first") ErrSelfUnsupported = errors.New("finding your own node in the DHT is currently not supported") ) const ( RepoDirOption = "repo-dir" ConfigFileOption = "config-file" ConfigOption = "config" DebugOption = "debug" LocalOption = "local" // DEPRECATED: use OfflineOption OfflineOption = "offline" ApiOption = "api" //nolint ApiAuthOption = "api-auth" //nolint ) var Root = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Global p2p merkle-dag filesystem.", Synopsis: "ipfs [--config= | -c] [--debug | -D] [--help] [-h] [--api=] [--offline] [--cid-base=] [--upgrade-cidv0-in-output] [--encoding= | --enc] [--timeout=] ...", Subcommands: ` BASIC COMMANDS init Initialize local IPFS configuration add Add a file to IPFS cat Show IPFS object data get Download IPFS objects ls List links from an object refs List hashes of links from an object DATA STRUCTURE COMMANDS dag Interact with IPLD DAG nodes files Interact with files as if they were a unix filesystem block Interact with raw blocks in the datastore TEXT ENCODING COMMANDS cid Convert and discover properties of CIDs multibase Encode and decode data with Multibase format ADVANCED COMMANDS daemon Start a long-running daemon process shutdown Shut down the daemon process resolve Resolve any type of content path name Publish and resolve IPNS names key Create and list IPNS name keypairs pin Pin objects to local storage repo Manipulate the IPFS repository stats Various operational stats p2p Libp2p stream mounting (experimental) filestore Manage the filestore (experimental) mount Mount an IPFS read-only mount point (experimental) provide Control providing operations NETWORK COMMANDS id Show info about IPFS peers bootstrap Add or remove bootstrap peers swarm Manage connections to the p2p network dht Query the DHT for values or peers routing Issue routing commands ping Measure the latency of a connection bitswap Inspect bitswap state pubsub Send and receive messages via pubsub TOOL COMMANDS config Manage configuration version Show IPFS version information diag Generate diagnostic reports update Download and apply go-ipfs updates commands List all available commands log Manage and show logs of running daemon Use 'ipfs --help' to learn more about each command. ipfs uses a repository in the local file system. By default, the repo is located at ~/.ipfs. To change the repo location, set the $IPFS_PATH environment variable: export IPFS_PATH=/path/to/ipfsrepo EXIT STATUS The CLI will exit with one of the following values: 0 Successful execution. 1 Failed executions. `, }, Options: []cmds.Option{ cmds.StringOption(RepoDirOption, "Path to the repository directory to use."), cmds.StringOption(ConfigFileOption, "Path to the configuration file to use."), cmds.StringOption(ConfigOption, "c", "[DEPRECATED] Path to the configuration file to use."), cmds.BoolOption(DebugOption, "D", "Operate in debug mode."), cmds.BoolOption(cmds.OptLongHelp, "Show the full command help text."), cmds.BoolOption(cmds.OptShortHelp, "Show a short version of the command help text."), cmds.BoolOption(LocalOption, "L", "Run the command locally, instead of using the daemon. DEPRECATED: use --offline."), cmds.BoolOption(OfflineOption, "Run the command offline."), cmds.StringOption(ApiOption, "Use a specific API instance (defaults to /ip4/127.0.0.1/tcp/5001)"), cmds.StringOption(ApiAuthOption, "Optional RPC API authorization secret (defined as AuthSecret in API.Authorizations config)"), // global options, added to every command cmdenv.OptionCidBase, cmdenv.OptionUpgradeCidV0InOutput, cmds.OptionEncodingType, cmds.OptionStreamChannels, cmds.OptionTimeout, }, } var CommandsDaemonCmd = CommandsCmd(Root) var rootSubcommands = map[string]*cmds.Command{ "add": AddCmd, "bitswap": BitswapCmd, "block": BlockCmd, "cat": CatCmd, "commands": CommandsDaemonCmd, "files": FilesCmd, "filestore": FileStoreCmd, "get": GetCmd, "provide": ProvideCmd, "pubsub": PubsubCmd, "repo": RepoCmd, "stats": StatsCmd, "bootstrap": BootstrapCmd, "config": ConfigCmd, "dag": dag.DagCmd, "dht": DhtCmd, "routing": RoutingCmd, "diag": DiagCmd, "id": IDCmd, "key": KeyCmd, "log": LogCmd, "ls": LsCmd, "mount": MountCmd, "name": name.NameCmd, "object": ocmd.ObjectCmd, "pin": pin.PinCmd, "ping": PingCmd, "p2p": P2PCmd, "refs": RefsCmd, "resolve": ResolveCmd, "swarm": SwarmCmd, "update": ExternalBinary("Please see https://github.com/ipfs/ipfs-update/blob/master/README.md#install for installation instructions."), "version": VersionCmd, "shutdown": daemonShutdownCmd, "cid": CidCmd, "multibase": MbaseCmd, } func init() { Root.ProcessHelp() Root.Subcommands = rootSubcommands } type MessageOutput struct { Message string } ================================================ FILE: core/commands/root_test.go ================================================ package commands import ( "testing" ) func TestCommandTree(t *testing.T) { printErrors := func(errs map[string][]error) { if errs == nil { return } t.Error("In Root command tree:") for cmd, err := range errs { t.Errorf(" In X command %s:", cmd) for _, e := range err { t.Errorf(" %s", e) } } } printErrors(Root.DebugValidate()) } ================================================ FILE: core/commands/routing.go ================================================ package commands import ( "context" "encoding/base64" "errors" "fmt" "io" "strings" "time" "github.com/ipfs/kubo/config" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" "github.com/ipfs/kubo/core/node" mh "github.com/multiformats/go-multihash" dag "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/provider" cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" ipld "github.com/ipfs/go-ipld-format" iface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/core/coreiface/options" peer "github.com/libp2p/go-libp2p/core/peer" routing "github.com/libp2p/go-libp2p/core/routing" ) var errAllowOffline = errors.New("can't put while offline: pass `--allow-offline` to override") const ( dhtVerboseOptionName = "verbose" numProvidersOptionName = "num-providers" allowOfflineOptionName = "allow-offline" ) var RoutingCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Issue routing commands.", ShortDescription: ``, }, Subcommands: map[string]*cmds.Command{ "findprovs": findProvidersRoutingCmd, "findpeer": findPeerRoutingCmd, "get": getValueRoutingCmd, "put": putValueRoutingCmd, "provide": provideRefRoutingCmd, "reprovide": reprovideRoutingCmd, }, } var findProvidersRoutingCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Find peers that can provide a specific value, given a key.", ShortDescription: "Outputs a list of newline-delimited provider Peer IDs.", }, Arguments: []cmds.Argument{ cmds.StringArg("key", true, true, "The key to find providers for."), }, Options: []cmds.Option{ cmds.BoolOption(dhtVerboseOptionName, "v", "Print extra information."), cmds.IntOption(numProvidersOptionName, "n", "The number of providers to find.").WithDefault(20), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := cmdenv.GetNode(env) if err != nil { return err } if !n.IsOnline { return ErrNotOnline } numProviders, _ := req.Options[numProvidersOptionName].(int) if numProviders < 1 { return errors.New("number of providers must be greater than 0") } c, err := cid.Parse(req.Arguments[0]) if err != nil { return err } ctx, cancel := context.WithCancel(req.Context) ctx, events := routing.RegisterForQueryEvents(ctx) go func() { defer cancel() pchan := n.Routing.FindProvidersAsync(ctx, c, numProviders) for p := range pchan { np := cmdutils.CloneAddrInfo(p) routing.PublishQueryEvent(ctx, &routing.QueryEvent{ Type: routing.Provider, Responses: []*peer.AddrInfo{&np}, }) } }() for e := range events { if err := res.Emit(e); err != nil { return err } } return nil }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *routing.QueryEvent) error { pfm := pfuncMap{ routing.FinalPeer: func(obj *routing.QueryEvent, out io.Writer, verbose bool) error { if verbose { fmt.Fprintf(out, "* closest peer %s\n", obj.ID) } return nil }, routing.Provider: func(obj *routing.QueryEvent, out io.Writer, verbose bool) error { prov := obj.Responses[0] if verbose { fmt.Fprintf(out, "provider: ") } fmt.Fprintf(out, "%s\n", prov.ID) if verbose { for _, a := range prov.Addrs { fmt.Fprintf(out, "\t%s\n", a) } } return nil }, } verbose, _ := req.Options[dhtVerboseOptionName].(bool) return printEvent(out, w, verbose, pfm) }), }, Type: routing.QueryEvent{}, } const ( recursiveOptionName = "recursive" ) var provideRefRoutingCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Announce to the network that you are providing given values.", }, Arguments: []cmds.Argument{ cmds.StringArg("key", true, true, "The key[s] to send provide records for.").EnableStdin(), }, Options: []cmds.Option{ cmds.BoolOption(dhtVerboseOptionName, "v", "Print extra information."), cmds.BoolOption(recursiveOptionName, "r", "Recursively provide entire graph."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } if !nd.IsOnline { return ErrNotOnline } // respect global config cfg, err := nd.Repo.Config() if err != nil { return err } if !cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled) { return errors.New("invalid configuration: Provide.Enabled is set to 'false'") } if len(nd.PeerHost.Network().Conns()) == 0 && !cfg.HasHTTPProviderConfigured() { // Node is depending on DHT for providing (no custom HTTP provider // configured) and currently has no connected peers. return errors.New("cannot provide, no connected peers") } // If we reach here with no connections but HTTP provider configured, // we proceed with the provide operation via HTTP // Needed to parse stdin args. // TODO: Lazy Load err = req.ParseBodyArgs() if err != nil { return err } rec, _ := req.Options[recursiveOptionName].(bool) var cids []cid.Cid for _, arg := range req.Arguments { c, err := cid.Decode(arg) if err != nil { return err } has, err := nd.Blockstore.Has(req.Context, c) if err != nil { return err } if !has { return fmt.Errorf("block %s not found locally, cannot provide", c) } cids = append(cids, c) } ctx, cancel := context.WithCancel(req.Context) ctx, events := routing.RegisterForQueryEvents(ctx) var provideErr error // TODO: not sure if necessary to call StartProviding for `ipfs routing // provide `, since either cid is already being provided, or it will // be garbage collected and not reprovided anyway. So we may simply stick // with a single (optimistic) provide, and skip StartProviding call. go func() { defer cancel() if rec { provideErr = provideCidsRec(ctx, nd.Provider, nd.DAG, cids) } else { provideErr = provideCids(nd.Provider, cids) } if provideErr != nil { routing.PublishQueryEvent(ctx, &routing.QueryEvent{ Type: routing.QueryError, Extra: provideErr.Error(), }) } }() if nd.HasActiveDHTClient() { // If node has a DHT client, provide immediately the supplied cids before // returning. for _, c := range cids { if err = provideCIDSync(req.Context, nd.DHTClient, c); err != nil { return fmt.Errorf("error providing cid: %w", err) } } } for e := range events { if err := res.Emit(e); err != nil { return err } } return provideErr }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *routing.QueryEvent) error { pfm := pfuncMap{ routing.FinalPeer: func(obj *routing.QueryEvent, out io.Writer, verbose bool) error { if verbose { fmt.Fprintf(out, "sending provider record to peer %s\n", obj.ID) } return nil }, } verbose, _ := req.Options[dhtVerboseOptionName].(bool) return printEvent(out, w, verbose, pfm) }), }, Type: routing.QueryEvent{}, } var reprovideRoutingCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Trigger reprovider.", ShortDescription: ` Trigger reprovider to announce our data to network. `, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } if !nd.IsOnline { return ErrNotOnline } // respect global config cfg, err := nd.Repo.Config() if err != nil { return err } if !cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled) { return errors.New("invalid configuration: Provide.Enabled is set to 'false'") } if cfg.Provide.DHT.Interval.WithDefault(config.DefaultProvideDHTInterval) == 0 { return errors.New("invalid configuration: Provide.DHT.Interval is set to '0'") } provideSys, ok := nd.Provider.(provider.Reprovider) if !ok { return errors.New("manual reprovide only available with legacy provider (Provide.DHT.SweepEnabled=false)") } err = provideSys.Reprovide(req.Context) if err != nil { return err } return nil }, } func provideCids(prov node.DHTProvider, cids []cid.Cid) error { mhs := make([]mh.Multihash, len(cids)) for i, c := range cids { mhs[i] = c.Hash() } // providing happens asynchronously return prov.StartProviding(true, mhs...) } func provideCidsRec(ctx context.Context, prov node.DHTProvider, dserv ipld.DAGService, cids []cid.Cid) error { for _, c := range cids { kset := cid.NewSet() err := dag.Walk(ctx, dag.GetLinksDirect(dserv), c, kset.Visit) if err != nil { return err } if err = provideCids(prov, kset.Keys()); err != nil { return err } } return nil } var findPeerRoutingCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Find the multiaddresses associated with a Peer ID.", ShortDescription: "Outputs a list of newline-delimited multiaddresses.", }, Arguments: []cmds.Argument{ cmds.StringArg("peerID", true, true, "The ID of the peer to search for."), }, Options: []cmds.Option{ cmds.BoolOption(dhtVerboseOptionName, "v", "Print extra information."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } if !nd.IsOnline { return ErrNotOnline } pid, err := peer.Decode(req.Arguments[0]) if err != nil { return err } if pid == nd.Identity { return ErrSelfUnsupported } ctx, cancel := context.WithCancel(req.Context) ctx, events := routing.RegisterForQueryEvents(ctx) var findPeerErr error go func() { defer cancel() var pi peer.AddrInfo pi, findPeerErr = nd.Routing.FindPeer(ctx, pid) if findPeerErr != nil { routing.PublishQueryEvent(ctx, &routing.QueryEvent{ Type: routing.QueryError, Extra: findPeerErr.Error(), }) return } routing.PublishQueryEvent(ctx, &routing.QueryEvent{ Type: routing.FinalPeer, Responses: []*peer.AddrInfo{&pi}, }) }() for e := range events { if err := res.Emit(e); err != nil { return err } } return findPeerErr }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *routing.QueryEvent) error { pfm := pfuncMap{ routing.FinalPeer: func(obj *routing.QueryEvent, out io.Writer, verbose bool) error { pi := obj.Responses[0] for _, a := range pi.Addrs { fmt.Fprintf(out, "%s\n", a) } return nil }, } verbose, _ := req.Options[dhtVerboseOptionName].(bool) return printEvent(out, w, verbose, pfm) }), }, Type: routing.QueryEvent{}, } var getValueRoutingCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Given a key, query the routing system for its best value.", ShortDescription: ` Outputs the best value for the given key. There may be several different values for a given key stored in the routing system; in this context 'best' means the record that is most desirable. There is no one metric for 'best': it depends entirely on the key type. For IPNS, 'best' is the record that is both valid and has the highest sequence number (freshest). Different key types can specify other 'best' rules. `, }, Arguments: []cmds.Argument{ cmds.StringArg("key", true, true, "The key to find a value for."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } r, err := api.Routing().Get(req.Context, req.Arguments[0]) if err != nil { return err } return res.Emit(routing.QueryEvent{ Extra: base64.StdEncoding.EncodeToString(r), Type: routing.Value, }) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, obj *routing.QueryEvent) error { res, err := base64.StdEncoding.DecodeString(obj.Extra) if err != nil { return err } _, err = w.Write(res) return err }), }, Type: routing.QueryEvent{}, } var putValueRoutingCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Write a key/value pair to the routing system.", ShortDescription: ` Given a key of the form /foo/bar and a valid value for that key, this will write that value to the routing system with that key. Keys have two parts: a keytype (foo) and the key name (bar). IPNS uses the /ipns keytype, and expects the key name to be a Peer ID. IPNS entries are specifically formatted (protocol buffer). You may only use keytypes that are supported in your ipfs binary: currently this is only /ipns. Unless you have a relatively deep understanding of the go-ipfs routing internals, you likely want to be using 'ipfs name publish' instead of this. The value must be a valid value for the given key type. For example, if the key is /ipns/QmFoo, the value must be IPNS record (protobuf) signed with the key identified by QmFoo. `, }, Arguments: []cmds.Argument{ cmds.StringArg("key", true, false, "The key to store the value at."), cmds.FileArg("value-file", true, false, "A path to a file containing the value to store.").EnableStdin(), }, Options: []cmds.Option{ cmds.BoolOption(allowOfflineOptionName, "When offline, save the IPNS record to the local datastore without broadcasting to the network instead of simply failing."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } file, err := cmdenv.GetFileArg(req.Files.Entries()) if err != nil { return err } defer file.Close() data, err := io.ReadAll(file) if err != nil { return err } allowOffline, _ := req.Options[allowOfflineOptionName].(bool) opts := []options.RoutingPutOption{ options.Put.AllowOffline(allowOffline), } ipnsName, err := ipns.NameFromString(req.Arguments[0]) if err != nil { return err } err = api.Routing().Put(req.Context, req.Arguments[0], data, opts...) if err != nil { if err == iface.ErrOffline { err = errAllowOffline } return err } return res.Emit(routing.QueryEvent{ Type: routing.Value, ID: ipnsName.Peer(), }) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *routing.QueryEvent) error { pfm := pfuncMap{ routing.FinalPeer: func(obj *routing.QueryEvent, out io.Writer, verbose bool) error { if verbose { fmt.Fprintf(out, "* closest peer %s\n", obj.ID) } return nil }, routing.Value: func(obj *routing.QueryEvent, out io.Writer, verbose bool) error { fmt.Fprintf(out, "%s\n", obj.ID) return nil }, } verbose, _ := req.Options[dhtVerboseOptionName].(bool) return printEvent(out, w, verbose, pfm) }), }, Type: routing.QueryEvent{}, } type ( printFunc func(obj *routing.QueryEvent, out io.Writer, verbose bool) error pfuncMap map[routing.QueryEventType]printFunc ) func printEvent(obj *routing.QueryEvent, out io.Writer, verbose bool, override pfuncMap) error { if verbose { fmt.Fprintf(out, "%s: ", time.Now().Format("15:04:05.000")) } if override != nil { if pf, ok := override[obj.Type]; ok { return pf(obj, out, verbose) } } switch obj.Type { case routing.SendingQuery: if verbose { fmt.Fprintf(out, "* querying %s\n", obj.ID) } case routing.Value: if verbose { fmt.Fprintf(out, "got value: '%s'\n", obj.Extra) } else { fmt.Fprint(out, obj.Extra) } case routing.PeerResponse: if verbose { fmt.Fprintf(out, "* %s says use ", obj.ID) for _, p := range obj.Responses { fmt.Fprintf(out, "%s ", p.ID) } fmt.Fprintln(out) } case routing.QueryError: if verbose { fmt.Fprintf(out, "error: %s\n", obj.Extra) } case routing.DialingPeer: if verbose { fmt.Fprintf(out, "dialing peer: %s\n", obj.ID) } case routing.AddingPeer: if verbose { fmt.Fprintf(out, "adding peer to query: %s\n", obj.ID) } case routing.FinalPeer: default: if verbose { fmt.Fprintf(out, "unrecognized event type: %d\n", obj.Type) } } return nil } func escapeDhtKey(s string) (string, error) { parts := strings.Split(s, "/") if len(parts) != 3 || parts[0] != "" || !(parts[1] == "ipns" || parts[1] == "pk") { return "", errors.New("invalid key") } k, err := peer.Decode(parts[2]) if err != nil { return "", err } return strings.Join(append(parts[:2], string(k)), "/"), nil } ================================================ FILE: core/commands/shutdown.go ================================================ package commands import ( cmds "github.com/ipfs/go-ipfs-cmds" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" ) var daemonShutdownCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Shut down the IPFS daemon.", }, Run: func(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } if !nd.IsDaemon { return cmds.Errorf(cmds.ErrClient, "daemon not running") } if err := nd.Close(); err != nil { log.Error("error while shutting down ipfs daemon:", err) } return nil }, } ================================================ FILE: core/commands/stat.go ================================================ package commands import ( "errors" "fmt" "io" "os" "time" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" humanize "github.com/dustin/go-humanize" cmds "github.com/ipfs/go-ipfs-cmds" metrics "github.com/libp2p/go-libp2p/core/metrics" peer "github.com/libp2p/go-libp2p/core/peer" protocol "github.com/libp2p/go-libp2p/core/protocol" ) var StatsCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Query IPFS statistics.", ShortDescription: `'ipfs stats' is a set of commands to help look at statistics for your IPFS node. `, LongDescription: `'ipfs stats' is a set of commands to help look at statistics for your IPFS node.`, }, Subcommands: map[string]*cmds.Command{ "bw": statBwCmd, "repo": repoStatCmd, "bitswap": bitswapStatCmd, "dht": statDhtCmd, "provide": statProvideCmd, "reprovide": statReprovideCmd, }, } const ( statPeerOptionName = "peer" statProtoOptionName = "proto" statPollOptionName = "poll" statIntervalOptionName = "interval" ) var statBwCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Print IPFS bandwidth information.", ShortDescription: `'ipfs stats bw' prints bandwidth information for the ipfs daemon. It displays: TotalIn, TotalOut, RateIn, RateOut. `, LongDescription: `'ipfs stats bw' prints bandwidth information for the ipfs daemon. It displays: TotalIn, TotalOut, RateIn, RateOut. By default, overall bandwidth and all protocols are shown. To limit bandwidth to a particular peer, use the 'peer' option along with that peer's multihash id. To specify a specific protocol, use the 'proto' option. The 'peer' and 'proto' options cannot be specified simultaneously. The protocols that are queried using this method are outlined in the specification: https://github.com/libp2p/specs/blob/master/_archive/7-properties.md#757-protocol-multicodecs Example protocol options: - /ipfs/id/1.0.0 - /ipfs/bitswap - /ipfs/dht Example: > ipfs stats bw -t /ipfs/bitswap Bandwidth TotalIn: 5.0MB TotalOut: 0B RateIn: 343B/s RateOut: 0B/s > ipfs stats bw -p QmepgFW7BHEtU4pZJdxaNiv75mKLLRQnPi1KaaXmQN4V1a Bandwidth TotalIn: 4.9MB TotalOut: 12MB RateIn: 0B/s RateOut: 0B/s `, }, Options: []cmds.Option{ cmds.StringOption(statPeerOptionName, "p", "Specify a peer to print bandwidth for."), cmds.StringOption(statProtoOptionName, "t", "Specify a protocol to print bandwidth for."), cmds.BoolOption(statPollOptionName, "Print bandwidth at an interval."), cmds.StringOption(statIntervalOptionName, "i", `Time interval to wait between updating output, if 'poll' is true. This accepts durations such as "300s", "1.5h" or "2h45m". Valid time units are: "ns", "us" (or "µs"), "ms", "s", "m", "h".`).WithDefault("1s"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } // Must be online! if !nd.IsOnline { return cmds.Errorf(cmds.ErrClient, "unable to run offline: %s", ErrNotOnline) } if nd.Reporter == nil { return errors.New("bandwidth reporter disabled in config") } pstr, pfound := req.Options[statPeerOptionName].(string) tstr, tfound := req.Options["proto"].(string) if pfound && tfound { return cmds.Errorf(cmds.ErrClient, "please only specify peer OR protocol") } var pid peer.ID if pfound { checkpid, err := peer.Decode(pstr) if err != nil { return err } pid = checkpid } timeS, _ := req.Options[statIntervalOptionName].(string) interval, err := time.ParseDuration(timeS) if err != nil { return err } doPoll, _ := req.Options[statPollOptionName].(bool) for { if pfound { stats := nd.Reporter.GetBandwidthForPeer(pid) if err := res.Emit(&stats); err != nil { return err } } else if tfound { protoID := protocol.ID(tstr) stats := nd.Reporter.GetBandwidthForProtocol(protoID) if err := res.Emit(&stats); err != nil { return err } } else { totals := nd.Reporter.GetBandwidthTotals() if err := res.Emit(&totals); err != nil { return err } } if !doPoll { return nil } select { case <-time.After(interval): case <-req.Context.Done(): return req.Context.Err() } } }, Type: metrics.Stats{}, PostRun: cmds.PostRunMap{ cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { polling, _ := res.Request().Options[statPollOptionName].(bool) if polling { fmt.Fprintln(os.Stdout, "Total Up Total Down Rate Up Rate Down") } for { v, err := res.Next() if err != nil { if err == io.EOF { return nil } return err } bs := v.(*metrics.Stats) if !polling { printStats(os.Stdout, bs) return nil } fmt.Fprintf(os.Stdout, "%8s ", humanize.Bytes(uint64(bs.TotalOut))) fmt.Fprintf(os.Stdout, "%8s ", humanize.Bytes(uint64(bs.TotalIn))) fmt.Fprintf(os.Stdout, "%8s/s ", humanize.Bytes(uint64(bs.RateOut))) fmt.Fprintf(os.Stdout, "%8s/s \r", humanize.Bytes(uint64(bs.RateIn))) } }, }, } func printStats(out io.Writer, bs *metrics.Stats) { fmt.Fprintln(out, "Bandwidth") fmt.Fprintf(out, "TotalIn: %s\n", humanize.Bytes(uint64(bs.TotalIn))) fmt.Fprintf(out, "TotalOut: %s\n", humanize.Bytes(uint64(bs.TotalOut))) fmt.Fprintf(out, "RateIn: %s/s\n", humanize.Bytes(uint64(bs.RateIn))) fmt.Fprintf(out, "RateOut: %s/s\n", humanize.Bytes(uint64(bs.RateOut))) } ================================================ FILE: core/commands/stat_dht.go ================================================ package commands import ( "fmt" "io" "text/tabwriter" "time" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" cmds "github.com/ipfs/go-ipfs-cmds" dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p-kad-dht/fullrt" kbucket "github.com/libp2p/go-libp2p-kbucket" "github.com/libp2p/go-libp2p/core/network" pstore "github.com/libp2p/go-libp2p/core/peerstore" ) type dhtPeerInfo struct { ID string Connected bool AgentVersion string LastUsefulAt string LastQueriedAt string } type dhtStat struct { Name string Buckets []dhtBucket } type dhtBucket struct { LastRefresh string Peers []dhtPeerInfo } var statDhtCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Returns statistics about the node's DHT(s).", ShortDescription: ` Returns statistics about the DHT(s) the node is participating in. This interface is not stable and may change from release to release. `, }, Arguments: []cmds.Argument{ cmds.StringArg("dht", false, true, "The DHT whose table should be listed (wanserver, lanserver, wan, lan). "+ "wan and lan refer to client routing tables. When using the experimental DHT client only WAN is supported. Defaults to wan and lan."), }, Options: []cmds.Option{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } if !nd.IsOnline { return ErrNotOnline } if nd.DHT == nil { return ErrNotDHT } id := kbucket.ConvertPeerID(nd.Identity) dhts := req.Arguments if len(dhts) == 0 { dhts = []string{"wan", "lan"} } dhttypeloop: for _, name := range dhts { var dht *dht.IpfsDHT var separateClient bool // Check if using separate DHT client (e.g., accelerated DHT) if nd.HasActiveDHTClient() && nd.DHTClient != nd.DHT { separateClient = true } switch name { case "wan": if separateClient { client, ok := nd.DHTClient.(*fullrt.FullRT) if !ok { return cmds.Errorf(cmds.ErrClient, "could not generate stats for the WAN DHT client type") } peerMap := client.Stat() buckets := make([]dhtBucket, 1) b := &dhtBucket{} for _, p := range peerMap { info := dhtPeerInfo{ID: p.String()} if ver, err := nd.Peerstore.Get(p, "AgentVersion"); err == nil { if vs, ok := ver.(string); ok { info.AgentVersion = cmdutils.CleanAndTrim(vs) } } else if err == pstore.ErrNotFound { // ignore } else { // this is a bug, usually. log.Errorw( "failed to get agent version from peerstore", "error", err, ) } info.Connected = nd.PeerHost.Network().Connectedness(p) == network.Connected b.Peers = append(b.Peers, info) } buckets[0] = *b if err := res.Emit(dhtStat{ Name: name, Buckets: buckets, }); err != nil { return err } continue dhttypeloop } fallthrough case "wanserver": dht = nd.DHT.WAN case "lan": if separateClient { return cmds.Errorf(cmds.ErrClient, "no LAN client found") } fallthrough case "lanserver": dht = nd.DHT.LAN default: return cmds.Errorf(cmds.ErrClient, "unknown dht type: %s", name) } rt := dht.RoutingTable() lastRefresh := rt.GetTrackedCplsForRefresh() infos := rt.GetPeerInfos() buckets := make([]dhtBucket, 0, len(lastRefresh)) for _, pi := range infos { cpl := kbucket.CommonPrefixLen(id, kbucket.ConvertPeerID(pi.Id)) if len(buckets) <= cpl { buckets = append(buckets, make([]dhtBucket, 1+cpl-len(buckets))...) } info := dhtPeerInfo{ID: pi.Id.String()} if ver, err := nd.Peerstore.Get(pi.Id, "AgentVersion"); err == nil { if vs, ok := ver.(string); ok { info.AgentVersion = cmdutils.CleanAndTrim(vs) } } else if err == pstore.ErrNotFound { // ignore } else { // this is a bug, usually. log.Errorw( "failed to get agent version from peerstore", "error", err, ) } if !pi.LastUsefulAt.IsZero() { info.LastUsefulAt = pi.LastUsefulAt.Format(time.RFC3339) } if !pi.LastSuccessfulOutboundQueryAt.IsZero() { info.LastQueriedAt = pi.LastSuccessfulOutboundQueryAt.Format(time.RFC3339) } info.Connected = nd.PeerHost.Network().Connectedness(pi.Id) == network.Connected buckets[cpl].Peers = append(buckets[cpl].Peers, info) } for i := 0; i < len(buckets) && i < len(lastRefresh); i++ { refreshTime := lastRefresh[i] if !refreshTime.IsZero() { buckets[i].LastRefresh = refreshTime.Format(time.RFC3339) } } if err := res.Emit(dhtStat{ Name: name, Buckets: buckets, }); err != nil { return err } } return nil }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out dhtStat) error { tw := tabwriter.NewWriter(w, 4, 4, 2, ' ', 0) defer tw.Flush() // Formats a time into XX ago and remove any decimal // parts. That is, change "2m3.00010101s" to "2m3s ago". now := time.Now() since := func(t time.Time) string { return now.Sub(t).Round(time.Second).String() + " ago" } count := 0 for _, bucket := range out.Buckets { count += len(bucket.Peers) } fmt.Fprintf(tw, "DHT %s (%d peers):\t\t\t\n", out.Name, count) for i, bucket := range out.Buckets { lastRefresh := "never" if bucket.LastRefresh != "" { t, err := time.Parse(time.RFC3339, bucket.LastRefresh) if err != nil { return err } lastRefresh = since(t) } fmt.Fprintf(tw, " Bucket %2d (%d peers) - refreshed %s:\t\t\t\n", i, len(bucket.Peers), lastRefresh) fmt.Fprintln(tw, " Peer\tlast useful\tlast queried\tAgent Version") for _, p := range bucket.Peers { lastUseful := "never" if p.LastUsefulAt != "" { t, err := time.Parse(time.RFC3339, p.LastUsefulAt) if err != nil { return err } lastUseful = since(t) } lastQueried := "never" if p.LastUsefulAt != "" { t, err := time.Parse(time.RFC3339, p.LastQueriedAt) if err != nil { return err } lastQueried = since(t) } state := " " if p.Connected { state = "@" } fmt.Fprintf(tw, " %s %s\t%s\t%s\t%s\n", state, p.ID, lastUseful, lastQueried, p.AgentVersion) } fmt.Fprintln(tw, "\t\t\t") } return nil }), }, Type: dhtStat{}, } ================================================ FILE: core/commands/stat_provide.go ================================================ package commands import ( cmds "github.com/ipfs/go-ipfs-cmds" ) var statProvideCmd = &cmds.Command{ Status: cmds.Deprecated, Helptext: cmds.HelpText{ Tagline: "Deprecated command, use 'ipfs provide stat' instead.", ShortDescription: ` 'ipfs stats provide' is deprecated because provide and reprovide operations are now distinct. This command may be replaced by provide only stats in the future. `, }, Arguments: provideStatCmd.Arguments, Options: provideStatCmd.Options, Run: provideStatCmd.Run, Encoders: provideStatCmd.Encoders, Type: provideStatCmd.Type, } ================================================ FILE: core/commands/stat_reprovide.go ================================================ package commands import ( cmds "github.com/ipfs/go-ipfs-cmds" ) var statReprovideCmd = &cmds.Command{ Status: cmds.Deprecated, Helptext: cmds.HelpText{ Tagline: "Deprecated command, use 'ipfs provide stat' instead.", ShortDescription: ` 'ipfs stats reprovide' is deprecated because provider stats are now available from 'ipfs provide stat'. `, }, Arguments: provideStatCmd.Arguments, Options: provideStatCmd.Options, Run: provideStatCmd.Run, Encoders: provideStatCmd.Encoders, Type: provideStatCmd.Type, } ================================================ FILE: core/commands/swarm.go ================================================ package commands import ( "context" "encoding/base64" "encoding/json" "errors" "fmt" "io" "path" "slices" "strconv" "strings" "sync" "text/tabwriter" "time" "github.com/ipfs/kubo/commands" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" "github.com/ipfs/kubo/core/node/libp2p" "github.com/ipfs/kubo/repo" "github.com/ipfs/kubo/repo/fsrepo" cmds "github.com/ipfs/go-ipfs-cmds" ic "github.com/libp2p/go-libp2p/core/crypto" inet "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" pstore "github.com/libp2p/go-libp2p/core/peerstore" "github.com/libp2p/go-libp2p/core/protocol" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" ma "github.com/multiformats/go-multiaddr" madns "github.com/multiformats/go-multiaddr-dns" mamask "github.com/whyrusleeping/multiaddr-filter" ) const ( dnsResolveTimeout = 10 * time.Second ) type stringList struct { Strings []string } type addrMap struct { Addrs map[string][]string } var SwarmCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Interact with the swarm.", ShortDescription: ` 'ipfs swarm' is a tool to manipulate the network swarm. The swarm is the component that opens, listens for, and maintains connections to other ipfs peers in the internet. `, }, Subcommands: map[string]*cmds.Command{ "addrs": swarmAddrsCmd, "connect": swarmConnectCmd, "disconnect": swarmDisconnectCmd, "filters": swarmFiltersCmd, "peers": swarmPeersCmd, "peering": swarmPeeringCmd, "resources": swarmResourcesCmd, // libp2p Network Resource Manager }, } const ( swarmVerboseOptionName = "verbose" swarmStreamsOptionName = "streams" swarmLatencyOptionName = "latency" swarmDirectionOptionName = "direction" swarmResetLimitsOptionName = "reset" swarmUsedResourcesPercentageName = "min-used-limit-perc" swarmIdentifyOptionName = "identify" ) type peeringResult struct { ID peer.ID Status string } var swarmPeeringCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Modify the peering subsystem.", ShortDescription: ` 'ipfs swarm peering' manages the peering subsystem. Peers in the peering subsystem are maintained to be connected, reconnected on disconnect with a back-off. The changes are not saved to the config. `, }, Subcommands: map[string]*cmds.Command{ "add": swarmPeeringAddCmd, "ls": swarmPeeringLsCmd, "rm": swarmPeeringRmCmd, }, } var swarmPeeringAddCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Add peers into the peering subsystem.", ShortDescription: ` 'ipfs swarm peering add' will add the new address to the peering subsystem as one that should always be connected to. `, }, Arguments: []cmds.Argument{ cmds.StringArg("address", true, true, "address of peer to add into the peering subsystem"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { addrs := make([]ma.Multiaddr, len(req.Arguments)) for i, arg := range req.Arguments { addr, err := ma.NewMultiaddr(arg) if err != nil { return err } addrs[i] = addr } addInfos, err := peer.AddrInfosFromP2pAddrs(addrs...) if err != nil { return err } node, err := cmdenv.GetNode(env) if err != nil { return err } if !node.IsOnline { return ErrNotOnline } for _, addrinfo := range addInfos { node.Peering.AddPeer(addrinfo) err = res.Emit(peeringResult{addrinfo.ID, "success"}) if err != nil { return err } } return nil }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, pr *peeringResult) error { fmt.Fprintf(w, "add %s %s\n", pr.ID.String(), pr.Status) return nil }), }, Type: peeringResult{}, } var swarmPeeringLsCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List peers registered in the peering subsystem.", ShortDescription: ` 'ipfs swarm peering ls' lists the peers that are registered in the peering subsystem and to which the daemon is always connected. `, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { node, err := cmdenv.GetNode(env) if err != nil { return err } if !node.IsOnline { return ErrNotOnline } peers := node.Peering.ListPeers() return cmds.EmitOnce(res, addrInfos{Peers: peers}) }, Type: addrInfos{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, ai *addrInfos) error { for _, info := range ai.Peers { fmt.Fprintf(w, "%s\n", info.ID) for _, addr := range info.Addrs { fmt.Fprintf(w, "\t%s\n", addr) } } return nil }), }, } type addrInfos struct { Peers []peer.AddrInfo } var swarmPeeringRmCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Remove a peer from the peering subsystem.", ShortDescription: ` 'ipfs swarm peering rm' will remove the given ID from the peering subsystem and remove it from the always-on connection. `, }, Arguments: []cmds.Argument{ cmds.StringArg("ID", true, true, "ID of peer to remove from the peering subsystem"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { node, err := cmdenv.GetNode(env) if err != nil { return err } if !node.IsOnline { return ErrNotOnline } for _, arg := range req.Arguments { id, err := peer.Decode(arg) if err != nil { return err } node.Peering.RemovePeer(id) if err = res.Emit(peeringResult{id, "success"}); err != nil { return err } } return nil }, Type: peeringResult{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, pr *peeringResult) error { fmt.Fprintf(w, "remove %s %s\n", pr.ID.String(), pr.Status) return nil }), }, } var swarmPeersCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List peers with open connections.", ShortDescription: ` 'ipfs swarm peers' lists the set of peers this node is connected to. `, }, Options: []cmds.Option{ cmds.BoolOption(swarmVerboseOptionName, "v", "display all extra information"), cmds.BoolOption(swarmStreamsOptionName, "Also list information about open streams for each peer"), cmds.BoolOption(swarmLatencyOptionName, "Also list information about latency to each peer"), cmds.BoolOption(swarmDirectionOptionName, "Also list information about the direction of connection"), cmds.BoolOption(swarmIdentifyOptionName, "Also list information about peers identify"), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } verbose, _ := req.Options[swarmVerboseOptionName].(bool) latency, _ := req.Options[swarmLatencyOptionName].(bool) streams, _ := req.Options[swarmStreamsOptionName].(bool) direction, _ := req.Options[swarmDirectionOptionName].(bool) identify, _ := req.Options[swarmIdentifyOptionName].(bool) conns, err := api.Swarm().Peers(req.Context) if err != nil { return err } var out connInfos for _, c := range conns { ci := connInfo{ Addr: c.Address().String(), Peer: c.ID().String(), } if verbose || direction { // set direction ci.Direction = c.Direction() } if verbose || latency { lat, err := c.Latency() if err != nil { return err } if lat == 0 { ci.Latency = "n/a" } else { ci.Latency = lat.String() } } if verbose || streams { strs, err := c.Streams() if err != nil { return err } for _, s := range strs { ci.Streams = append(ci.Streams, streamInfo{Protocol: cmdutils.CleanAndTrim(string(s))}) } } if verbose || identify { n, err := cmdenv.GetNode(env) if err != nil { return err } identifyResult, _ := ci.identifyPeer(n.Peerstore, c.ID()) ci.Identify = identifyResult } ci.Sort() out.Peers = append(out.Peers, ci) } out.Sort() return cmds.EmitOnce(res, &out) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, ci *connInfos) error { pipfs := ma.ProtocolWithCode(ma.P_IPFS).Name for _, info := range ci.Peers { fmt.Fprintf(w, "%s/%s/%s", info.Addr, pipfs, info.Peer) if info.Latency != "" { fmt.Fprintf(w, " %s", info.Latency) } if info.Direction != inet.DirUnknown { fmt.Fprintf(w, " %s", directionString(info.Direction)) } fmt.Fprintln(w) for _, s := range info.Streams { if s.Protocol == "" { s.Protocol = "" } fmt.Fprintf(w, " %s\n", s.Protocol) } } return nil }), }, Type: connInfos{}, } var swarmResourcesCmd = &cmds.Command{ Status: cmds.Experimental, Helptext: cmds.HelpText{ Tagline: "Get a summary of all resources accounted for by the libp2p Resource Manager.", LongDescription: ` Get a summary of all resources accounted for by the libp2p Resource Manager. This includes the limits and the usage against those limits. This can output a human readable table and JSON encoding. `, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { node, err := cmdenv.GetNode(env) if err != nil { return err } if node.ResourceManager == nil { return libp2p.ErrNoResourceMgr } cfg, err := node.Repo.Config() if err != nil { return err } userResourceOverrides, err := node.Repo.UserResourceOverrides() if err != nil { return err } // FIXME: we shouldn't recompute limits, either save them or load them from libp2p (https://github.com/libp2p/go-libp2p/issues/2166) limitConfig, _, err := libp2p.LimitConfig(cfg.Swarm, userResourceOverrides) if err != nil { return err } rapi, ok := node.ResourceManager.(rcmgr.ResourceManagerState) if !ok { // NullResourceManager return libp2p.ErrNoResourceMgr } return cmds.EmitOnce(res, libp2p.MergeLimitsAndStatsIntoLimitsConfigAndUsage(limitConfig, rapi.Stat())) }, Encoders: cmds.EncoderMap{ cmds.JSON: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, limitsAndUsage libp2p.LimitsConfigAndUsage) error { return json.NewEncoder(w).Encode(limitsAndUsage) }), cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, limitsAndUsage libp2p.LimitsConfigAndUsage) error { tw := tabwriter.NewWriter(w, 20, 8, 0, '\t', 0) defer tw.Flush() fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t\n", "Scope", "Limit Name", "Limit Value", "Limit Usage Amount", "Limit Usage Percent") for _, ri := range libp2p.LimitConfigsToInfo(limitsAndUsage) { var limit, percentage string switch ri.LimitValue { case rcmgr.Unlimited64: limit = "unlimited" percentage = "n/a" case rcmgr.BlockAllLimit64: limit = "blockAll" percentage = "n/a" default: limit = strconv.FormatInt(int64(ri.LimitValue), 10) if ri.CurrentUsage == 0 { percentage = "0%" } else { percentage = strconv.FormatFloat(float64(ri.CurrentUsage)/float64(ri.LimitValue)*100, 'f', 1, 64) + "%" } } fmt.Fprintf(tw, "%s\t%s\t%s\t%d\t%s\t\n", ri.ScopeName, ri.LimitName, limit, ri.CurrentUsage, percentage, ) } return nil }), }, Type: libp2p.LimitsConfigAndUsage{}, } type streamInfo struct { Protocol string } type connInfo struct { Addr string `json:",omitempty"` Peer string `json:",omitempty"` Latency string `json:",omitempty"` Muxer string `json:",omitempty"` Direction inet.Direction `json:",omitempty"` Streams []streamInfo `json:",omitempty"` Identify IdOutput } func (ci *connInfo) Sort() { slices.SortFunc(ci.Streams, func(a, b streamInfo) int { return strings.Compare(a.Protocol, b.Protocol) }) } type connInfos struct { Peers []connInfo } func (ci *connInfos) Sort() { slices.SortFunc(ci.Peers, func(a, b connInfo) int { return strings.Compare(a.Addr, b.Addr) }) } func (ci *connInfo) identifyPeer(ps pstore.Peerstore, p peer.ID) (IdOutput, error) { var info IdOutput info.ID = p.String() if pk := ps.PubKey(p); pk != nil { pkb, err := ic.MarshalPublicKey(pk) if err != nil { return IdOutput{}, err } info.PublicKey = base64.StdEncoding.EncodeToString(pkb) } addrInfo := ps.PeerInfo(p) addrs, err := peer.AddrInfoToP2pAddrs(&addrInfo) if err != nil { return IdOutput{}, err } for _, a := range addrs { info.Addresses = append(info.Addresses, a.String()) } slices.Sort(info.Addresses) if protocols, err := ps.GetProtocols(p); err == nil { for _, proto := range protocols { info.Protocols = append(info.Protocols, protocol.ID(cmdutils.CleanAndTrim(string(proto)))) } slices.Sort(info.Protocols) } if v, err := ps.Get(p, "AgentVersion"); err == nil { if vs, ok := v.(string); ok { info.AgentVersion = cmdutils.CleanAndTrim(vs) } } return info, nil } // directionString transfers to string func directionString(d inet.Direction) string { switch d { case inet.DirInbound: return "inbound" case inet.DirOutbound: return "outbound" default: return "" } } var swarmAddrsCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List known addresses. Useful for debugging.", ShortDescription: ` 'ipfs swarm addrs' lists all addresses this node is aware of. `, }, Subcommands: map[string]*cmds.Command{ "autonat": swarmAddrsAutoNATCmd, "local": swarmAddrsLocalCmd, "listen": swarmAddrsListenCmd, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } addrs, err := api.Swarm().KnownAddrs(req.Context) if err != nil { return err } out := make(map[string][]string) for p, paddrs := range addrs { s := p.String() for _, a := range paddrs { out[s] = append(out[s], a.String()) } } return cmds.EmitOnce(res, &addrMap{Addrs: out}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, am *addrMap) error { // sort the ids first ids := make([]string, 0, len(am.Addrs)) for p := range am.Addrs { ids = append(ids, p) } slices.Sort(ids) for _, p := range ids { paddrs := am.Addrs[p] fmt.Fprintf(w, "%s (%d)\n", p, len(paddrs)) for _, addr := range paddrs { fmt.Fprintf(w, "\t%s\n", addr) } } return nil }), }, Type: addrMap{}, } var swarmAddrsLocalCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List local addresses.", ShortDescription: ` 'ipfs swarm addrs local' lists all local listening addresses announced to the network. `, }, Options: []cmds.Option{ cmds.BoolOption("id", "Show peer ID in addresses."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } showid, _ := req.Options["id"].(bool) self, err := api.Key().Self(req.Context) if err != nil { return err } maddrs, err := api.Swarm().LocalAddrs(req.Context) if err != nil { return err } var addrs []string p2pProtocolName := ma.ProtocolWithCode(ma.P_P2P).Name for _, addr := range maddrs { saddr := addr.String() if showid { saddr = path.Join(saddr, p2pProtocolName, self.ID().String()) } addrs = append(addrs, saddr) } slices.Sort(addrs) return cmds.EmitOnce(res, &stringList{addrs}) }, Type: stringList{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder), }, } var swarmAddrsListenCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List interface listening addresses.", ShortDescription: ` 'ipfs swarm addrs listen' lists all interface addresses the node is listening on. `, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { return err } var addrs []string maddrs, err := api.Swarm().ListenAddrs(req.Context) if err != nil { return err } for _, addr := range maddrs { addrs = append(addrs, addr.String()) } slices.Sort(addrs) return cmds.EmitOnce(res, &stringList{addrs}) }, Type: stringList{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder), }, } var swarmConnectCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Open connection to a given peer.", ShortDescription: ` 'ipfs swarm connect' attempts to ensure a connection to a given peer. Multiaddresses given are advisory, for example the node may already be aware of other addresses for a given peer or may already have an established connection to the peer. The address format is a libp2p multiaddr: ipfs swarm connect /ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ `, }, Arguments: []cmds.Argument{ cmds.StringArg("address", true, true, "Address of peer to connect to.").EnableStdin(), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { node, err := cmdenv.GetNode(env) if err != nil { return err } api, err := cmdenv.GetApi(env, req) if err != nil { return err } addrs := req.Arguments pis, err := parseAddresses(req.Context, addrs, node.DNSResolver) if err != nil { return err } output := make([]string, len(pis)) for i, pi := range pis { output[i] = "connect " + pi.ID.String() err := api.Swarm().Connect(req.Context, pi) if err != nil { return fmt.Errorf("%s failure: %s", output[i], err) } output[i] += " success" } return cmds.EmitOnce(res, &stringList{output}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder), }, Type: stringList{}, } var swarmDisconnectCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Close connection to a given address.", ShortDescription: ` 'ipfs swarm disconnect' closes a connection to a peer address. The address format is an IPFS multiaddr: ipfs swarm disconnect /ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ The disconnect is not permanent; if ipfs needs to talk to that address later, it will reconnect. `, }, Arguments: []cmds.Argument{ cmds.StringArg("address", true, true, "Address of peer to disconnect from.").EnableStdin(), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { node, err := cmdenv.GetNode(env) if err != nil { return err } api, err := cmdenv.GetApi(env, req) if err != nil { return err } addrs, err := parseAddresses(req.Context, req.Arguments, node.DNSResolver) if err != nil { return err } output := make([]string, 0, len(addrs)) for _, ainfo := range addrs { maddrs, err := peer.AddrInfoToP2pAddrs(&ainfo) if err != nil { return err } // FIXME: This will print: // // disconnect QmFoo success // disconnect QmFoo success // ... // // Once per address specified. However, I'm not sure of // a good backwards compat solution. Right now, I'm just // preserving the current behavior. for _, addr := range maddrs { msg := "disconnect " + ainfo.ID.String() if err := api.Swarm().Disconnect(req.Context, addr); err != nil { msg += " failure: " + err.Error() } else { msg += " success" } output = append(output, msg) } } return cmds.EmitOnce(res, &stringList{output}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder), }, Type: stringList{}, } // parseAddresses is a function that takes in a slice of string peer addresses // (multiaddr + peerid) and returns a slice of properly constructed peers func parseAddresses(ctx context.Context, addrs []string, rslv *madns.Resolver) ([]peer.AddrInfo, error) { // resolve addresses maddrs, err := resolveAddresses(ctx, addrs, rslv) if err != nil { return nil, err } return peer.AddrInfosFromP2pAddrs(maddrs...) } // resolveAddresses resolves addresses parallelly func resolveAddresses(ctx context.Context, addrs []string, rslv *madns.Resolver) ([]ma.Multiaddr, error) { ctx, cancel := context.WithTimeout(ctx, dnsResolveTimeout) defer cancel() var maddrs []ma.Multiaddr var wg sync.WaitGroup resolveErrC := make(chan error, len(addrs)) maddrC := make(chan ma.Multiaddr) for _, addr := range addrs { maddr, err := ma.NewMultiaddr(addr) if err != nil { return nil, err } // check whether address ends in `ipfs/Qm...` if _, last := ma.SplitLast(maddr); last.Protocol().Code == ma.P_IPFS { maddrs = append(maddrs, maddr) continue } wg.Add(1) go func(maddr ma.Multiaddr) { defer wg.Done() raddrs, err := rslv.Resolve(ctx, maddr) if err != nil { resolveErrC <- err return } // filter out addresses that still doesn't end in `ipfs/Qm...` found := 0 for _, raddr := range raddrs { if _, last := ma.SplitLast(raddr); last != nil && last.Protocol().Code == ma.P_IPFS { maddrC <- raddr found++ } } if found == 0 { resolveErrC <- fmt.Errorf("found no ipfs peers at %s", maddr) } }(maddr) } go func() { wg.Wait() close(maddrC) }() for maddr := range maddrC { maddrs = append(maddrs, maddr) } select { case err := <-resolveErrC: return nil, err default: } return maddrs, nil } var swarmFiltersCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Manipulate address filters.", ShortDescription: ` 'ipfs swarm filters' will list out currently applied filters. Its subcommands can be used to add or remove said filters. Filters are specified using the multiaddr-filter format: Example: /ip4/192.168.0.0/ipcidr/16 Where the above is equivalent to the standard CIDR: 192.168.0.0/16 Filters default to those specified under the "Swarm.AddrFilters" config key. `, }, Subcommands: map[string]*cmds.Command{ "add": swarmFiltersAddCmd, "rm": swarmFiltersRmCmd, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := cmdenv.GetNode(env) if err != nil { return err } if !n.IsOnline { return ErrNotOnline } var output []string for _, f := range n.Filters.FiltersForAction(ma.ActionDeny) { s, err := mamask.ConvertIPNet(&f) if err != nil { return err } output = append(output, s) } return cmds.EmitOnce(res, &stringList{output}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder), }, Type: stringList{}, } var swarmFiltersAddCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Add an address filter.", ShortDescription: ` 'ipfs swarm filters add' will add an address filter to the daemons swarm. `, }, Arguments: []cmds.Argument{ cmds.StringArg("address", true, true, "Multiaddr to filter.").EnableStdin(), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := cmdenv.GetNode(env) if err != nil { return err } if !n.IsOnline { return ErrNotOnline } if len(req.Arguments) == 0 { return errors.New("no filters to add") } r, err := fsrepo.Open(env.(*commands.Context).ConfigRoot) if err != nil { return err } defer r.Close() cfg, err := r.Config() if err != nil { return err } for _, arg := range req.Arguments { mask, err := mamask.NewMask(arg) if err != nil { return err } n.Filters.AddFilter(*mask, ma.ActionDeny) } added, err := filtersAdd(r, cfg, req.Arguments) if err != nil { return err } return cmds.EmitOnce(res, &stringList{added}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder), }, Type: stringList{}, } var swarmFiltersRmCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Remove an address filter.", ShortDescription: ` 'ipfs swarm filters rm' will remove an address filter from the daemons swarm. `, }, Arguments: []cmds.Argument{ cmds.StringArg("address", true, true, "Multiaddr filter to remove.").EnableStdin(), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { n, err := cmdenv.GetNode(env) if err != nil { return err } if !n.IsOnline { return ErrNotOnline } r, err := fsrepo.Open(env.(*commands.Context).ConfigRoot) if err != nil { return err } defer r.Close() cfg, err := r.Config() if err != nil { return err } if req.Arguments[0] == "all" || req.Arguments[0] == "*" { fs := n.Filters.FiltersForAction(ma.ActionDeny) for _, f := range fs { n.Filters.RemoveLiteral(f) } removed, err := filtersRemoveAll(r, cfg) if err != nil { return err } return cmds.EmitOnce(res, &stringList{removed}) } for _, arg := range req.Arguments { mask, err := mamask.NewMask(arg) if err != nil { return err } n.Filters.RemoveLiteral(*mask) } removed, err := filtersRemove(r, cfg, req.Arguments) if err != nil { return err } return cmds.EmitOnce(res, &stringList{removed}) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder), }, Type: stringList{}, } func filtersAdd(r repo.Repo, cfg *config.Config, filters []string) ([]string, error) { addedMap := map[string]struct{}{} addedList := make([]string, 0, len(filters)) // re-add cfg swarm filters to rm dupes oldFilters := cfg.Swarm.AddrFilters cfg.Swarm.AddrFilters = nil // add new filters for _, filter := range filters { if _, found := addedMap[filter]; found { continue } cfg.Swarm.AddrFilters = append(cfg.Swarm.AddrFilters, filter) addedList = append(addedList, filter) addedMap[filter] = struct{}{} } // add back original filters. in this order so that we output them. for _, filter := range oldFilters { if _, found := addedMap[filter]; found { continue } cfg.Swarm.AddrFilters = append(cfg.Swarm.AddrFilters, filter) addedMap[filter] = struct{}{} } if err := r.SetConfig(cfg); err != nil { return nil, err } return addedList, nil } func filtersRemoveAll(r repo.Repo, cfg *config.Config) ([]string, error) { removed := cfg.Swarm.AddrFilters cfg.Swarm.AddrFilters = nil if err := r.SetConfig(cfg); err != nil { return nil, err } return removed, nil } func filtersRemove(r repo.Repo, cfg *config.Config, toRemoveFilters []string) ([]string, error) { removed := make([]string, 0, len(toRemoveFilters)) keep := make([]string, 0, len(cfg.Swarm.AddrFilters)) oldFilters := cfg.Swarm.AddrFilters for _, oldFilter := range oldFilters { found := false for _, toRemoveFilter := range toRemoveFilters { if oldFilter == toRemoveFilter { found = true removed = append(removed, toRemoveFilter) break } } if !found { keep = append(keep, oldFilter) } } cfg.Swarm.AddrFilters = keep if err := r.SetConfig(cfg); err != nil { return nil, err } return removed, nil } ================================================ FILE: core/commands/swarm_addrs_autonat.go ================================================ package commands import ( "fmt" "io" cmds "github.com/ipfs/go-ipfs-cmds" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/libp2p/go-libp2p/core/network" ma "github.com/multiformats/go-multiaddr" ) // reachabilityHost provides access to the AutoNAT reachability status. type reachabilityHost interface { Reachability() network.Reachability } // confirmedAddrsHost provides access to per-address reachability from AutoNAT V2. type confirmedAddrsHost interface { ConfirmedAddrs() (reachable, unreachable, unknown []ma.Multiaddr) } // autoNATResult represents the AutoNAT reachability information. type autoNATResult struct { Reachability string `json:"reachability"` Reachable []string `json:"reachable,omitempty"` Unreachable []string `json:"unreachable,omitempty"` Unknown []string `json:"unknown,omitempty"` } func multiaddrsToStrings(addrs []ma.Multiaddr) []string { out := make([]string, len(addrs)) for i, a := range addrs { out[i] = a.String() } return out } func writeAddrSection(w io.Writer, label string, addrs []string) { if len(addrs) > 0 { fmt.Fprintf(w, " %s:\n", label) for _, addr := range addrs { fmt.Fprintf(w, " %s\n", addr) } } } var swarmAddrsAutoNATCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Show address reachability as determined by AutoNAT V2.", ShortDescription: ` 'ipfs swarm addrs autonat' shows the reachability status of your node's addresses as determined by AutoNAT V2. `, LongDescription: ` 'ipfs swarm addrs autonat' shows the reachability status of your node's addresses as verified by AutoNAT V2. AutoNAT V2 probes your node's addresses to determine if they are reachable from the public internet. This helps understand whether other peers can dial your node directly. The output shows: - Reachability: Overall status (Public, Private, or Unknown) - Reachable: Addresses confirmed to be publicly reachable - Unreachable: Addresses that failed reachability checks - Unknown: Addresses that haven't been tested yet For more information on AutoNAT V2, see: https://github.com/libp2p/specs/blob/master/autonat/autonat-v2.md Example: > ipfs swarm addrs autonat AutoNAT V2 Status: Reachability: Public Per-Address Reachability: Reachable: /ip4/203.0.113.42/tcp/4001 /ip4/203.0.113.42/udp/4001/quic-v1 Unreachable: /ip6/2001:db8::1/tcp/4001 Unknown: /ip4/203.0.113.42/udp/4001/webrtc-direct `, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } if !nd.IsOnline { return ErrNotOnline } result := autoNATResult{ Reachability: network.ReachabilityUnknown.String(), } // Get per-address reachability from AutoNAT V2. // The host embeds *BasicHost (closableBasicHost, closableRoutedHost) // which implements ConfirmedAddrs. if h, ok := nd.PeerHost.(confirmedAddrsHost); ok { reachable, unreachable, unknown := h.ConfirmedAddrs() result.Reachable = multiaddrsToStrings(reachable) result.Unreachable = multiaddrsToStrings(unreachable) result.Unknown = multiaddrsToStrings(unknown) } // Get overall reachability status. if h, ok := nd.PeerHost.(reachabilityHost); ok { result.Reachability = h.Reachability().String() } return cmds.EmitOnce(res, result) }, Type: autoNATResult{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, result autoNATResult) error { fmt.Fprintln(w, "AutoNAT V2 Status:") fmt.Fprintf(w, " Reachability: %s\n", result.Reachability) fmt.Fprintln(w) fmt.Fprintln(w, "Per-Address Reachability:") writeAddrSection(w, "Reachable", result.Reachable) writeAddrSection(w, "Unreachable", result.Unreachable) writeAddrSection(w, "Unknown", result.Unknown) if len(result.Reachable) == 0 && len(result.Unreachable) == 0 && len(result.Unknown) == 0 { fmt.Fprintln(w, " (no address reachability data available)") } return nil }), }, } ================================================ FILE: core/commands/sysdiag.go ================================================ package commands import ( "os" "runtime" "github.com/ipfs/go-ipfs-cmds" version "github.com/ipfs/kubo" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" manet "github.com/multiformats/go-multiaddr/net" sysi "github.com/whyrusleeping/go-sysinfo" ) var sysDiagCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Print system diagnostic information.", ShortDescription: ` Prints out information about your computer to aid in easier debugging. `, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } info, err := getInfo(nd) if err != nil { return err } return cmds.EmitOnce(res, info) }, } func getInfo(nd *core.IpfsNode) (map[string]any, error) { info := make(map[string]any) err := runtimeInfo(info) if err != nil { return nil, err } err = envVarInfo(info) if err != nil { return nil, err } err = diskSpaceInfo(info) if err != nil { return nil, err } err = memInfo(info) if err != nil { return nil, err } err = netInfo(nd.IsOnline, info) if err != nil { return nil, err } info["ipfs_version"] = version.CurrentVersionNumber info["ipfs_commit"] = version.CurrentCommit return info, nil } func runtimeInfo(out map[string]any) error { rt := make(map[string]any) rt["os"] = runtime.GOOS rt["arch"] = runtime.GOARCH rt["compiler"] = runtime.Compiler rt["version"] = runtime.Version() rt["numcpu"] = runtime.NumCPU() rt["gomaxprocs"] = runtime.GOMAXPROCS(0) rt["numgoroutines"] = runtime.NumGoroutine() out["runtime"] = rt return nil } func envVarInfo(out map[string]any) error { ev := make(map[string]any) ev["GOPATH"] = os.Getenv("GOPATH") ev[config.EnvDir] = os.Getenv(config.EnvDir) out["environment"] = ev return nil } func diskSpaceInfo(out map[string]any) error { pathRoot, err := config.PathRoot() if err != nil { return err } dinfo, err := sysi.DiskUsage(pathRoot) if err != nil { return err } out["diskinfo"] = map[string]any{ "fstype": dinfo.FsType, "total_space": dinfo.Total, "free_space": dinfo.Free, } return nil } func memInfo(out map[string]any) error { m := make(map[string]any) meminf, err := sysi.MemoryInfo() if err != nil { return err } m["swap"] = meminf.Swap m["virt"] = meminf.Used out["memory"] = m return nil } func netInfo(online bool, out map[string]any) error { n := make(map[string]any) addrs, err := manet.InterfaceMultiaddrs() if err != nil { return err } straddrs := make([]string, len(addrs)) for i, a := range addrs { straddrs[i] = a.String() } n["interface_addresses"] = straddrs n["online"] = online out["net"] = n return nil } ================================================ FILE: core/commands/version.go ================================================ package commands import ( "errors" "fmt" "io" "runtime/debug" "strings" versioncmp "github.com/hashicorp/go-version" cmds "github.com/ipfs/go-ipfs-cmds" version "github.com/ipfs/kubo" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/libp2p/go-libp2p-kad-dht/fullrt" peer "github.com/libp2p/go-libp2p/core/peer" pstore "github.com/libp2p/go-libp2p/core/peerstore" ) const ( versionNumberOptionName = "number" versionCommitOptionName = "commit" versionRepoOptionName = "repo" versionAllOptionName = "all" versionCheckThresholdOptionName = "min-percent" ) var VersionCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Show IPFS version information.", ShortDescription: "Returns the current version of IPFS and exits.", }, Subcommands: map[string]*cmds.Command{ "deps": depsVersionCommand, "check": checkVersionCommand, }, Options: []cmds.Option{ cmds.BoolOption(versionNumberOptionName, "n", "Only show the version number."), cmds.BoolOption(versionCommitOptionName, "Show the commit hash."), cmds.BoolOption(versionRepoOptionName, "Show repo version."), cmds.BoolOption(versionAllOptionName, "Show all version information"), }, // must be permitted to run before init Extra: CreateCmdExtras(SetDoesNotUseRepo(true), SetDoesNotUseConfigAsInput(true)), Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { return cmds.EmitOnce(res, version.GetVersionInfo()) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, version *version.VersionInfo) error { all, _ := req.Options[versionAllOptionName].(bool) if all { ver := version.Version if version.Commit != "" { ver += "-" + version.Commit } out := fmt.Sprintf("Kubo version: %s\n"+ "Repo version: %s\nSystem version: %s\nGolang version: %s\n", ver, version.Repo, version.System, version.Golang) fmt.Fprint(w, out) return nil } commit, _ := req.Options[versionCommitOptionName].(bool) commitTxt := "" if commit && version.Commit != "" { commitTxt = "-" + version.Commit } repo, _ := req.Options[versionRepoOptionName].(bool) if repo { fmt.Fprintln(w, version.Repo) return nil } number, _ := req.Options[versionNumberOptionName].(bool) if number { fmt.Fprintln(w, version.Version+commitTxt) return nil } fmt.Fprintf(w, "ipfs version %s%s\n", version.Version, commitTxt) return nil }), }, Type: version.VersionInfo{}, } type Dependency struct { Path string Version string ReplacedBy string Sum string } const pkgVersionFmt = "%s@%s" var depsVersionCommand = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Shows information about dependencies used for build.", ShortDescription: ` Print out all dependencies and their versions.`, }, Type: Dependency{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { info, ok := debug.ReadBuildInfo() if !ok { return errors.New("no embedded dependency information") } toDependency := func(mod *debug.Module) (dep Dependency) { dep.Path = mod.Path dep.Version = mod.Version dep.Sum = mod.Sum if repl := mod.Replace; repl != nil { dep.ReplacedBy = fmt.Sprintf(pkgVersionFmt, repl.Path, repl.Version) } return } if err := res.Emit(toDependency(&info.Main)); err != nil { return err } for _, dep := range info.Deps { if err := res.Emit(toDependency(dep)); err != nil { return err } } return nil }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, dep Dependency) error { fmt.Fprintf(w, pkgVersionFmt, dep.Path, dep.Version) if dep.ReplacedBy != "" { fmt.Fprintf(w, " => %s", dep.ReplacedBy) } fmt.Fprintf(w, "\n") return nil }), }, } const DefaultMinimalVersionFraction = 0.05 // 5% type VersionCheckOutput struct { UpdateAvailable bool RunningVersion string GreatestVersion string PeersSampled int WithGreaterVersion int } var checkVersionCommand = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Checks Kubo version against connected peers.", ShortDescription: ` This command uses the libp2p identify protocol to check the 'AgentVersion' of connected peers and see if the Kubo version we're running is outdated. Peers with an AgentVersion that doesn't start with 'kubo/' are ignored. 'UpdateAvailable' is set to true only if the 'min-fraction' criteria are met. The 'ipfs daemon' does the same check regularly and logs when a new version is available. You can stop these regular checks by setting Version.SwarmCheckEnabled:false in the config. `, }, Options: []cmds.Option{ cmds.IntOption(versionCheckThresholdOptionName, "t", "Percentage (1-100) of sampled peers with the new Kubo version needed to trigger an update warning.").WithDefault(config.DefaultSwarmCheckPercentThreshold), }, Type: VersionCheckOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) if err != nil { return err } if !nd.IsOnline { return ErrNotOnline } minPercent, _ := req.Options[versionCheckThresholdOptionName].(int64) output, err := DetectNewKuboVersion(nd, minPercent) if err != nil { return err } if err := cmds.EmitOnce(res, output); err != nil { return err } return nil }, } // DetectNewKuboVersion observers kubo version reported by other peers via // libp2p identify protocol and notifies when threshold fraction of seen swarm // is running updated Kubo. It is used by RPC and CLI at 'ipfs version check' // and also periodically when 'ipfs daemon' is running. func DetectNewKuboVersion(nd *core.IpfsNode, minPercent int64) (VersionCheckOutput, error) { ourVersion, err := versioncmp.NewVersion(version.CurrentVersionNumber) if err != nil { return VersionCheckOutput{}, fmt.Errorf("could not parse our own version %q: %w", version.CurrentVersionNumber, err) } // MAJOR.MINOR.PATCH without any suffix ourVersion = ourVersion.Core() greatestVersionSeen := ourVersion totalPeersSampled := 1 // Us (and to avoid division-by-zero edge case) withGreaterVersion := 0 recordPeerVersion := func(agentVersion string) { // We process the version as is it assembled in GetUserAgentVersion segments := strings.Split(agentVersion, "/") if len(segments) < 2 { return } if segments[0] != "kubo" { return } versionNumber := segments[1] // As in our CurrentVersionNumber peerVersion, err := versioncmp.NewVersion(versionNumber) if err != nil { // Do not error on invalid remote versions, just ignore return } // Ignore prereleases and development releases (-dev, -rcX) if peerVersion.Metadata() != "" || peerVersion.Prerelease() != "" { return } // MAJOR.MINOR.PATCH without any suffix peerVersion = peerVersion.Core() // Valid peer version number totalPeersSampled += 1 if ourVersion.LessThan(peerVersion) { withGreaterVersion += 1 } if peerVersion.GreaterThan(greatestVersionSeen) { greatestVersionSeen = peerVersion } } processPeerstoreEntry := func(id peer.ID) { if v, err := nd.Peerstore.Get(id, "AgentVersion"); err == nil { recordPeerVersion(v.(string)) } else if errors.Is(err, pstore.ErrNotFound) { // ignore noop } else { // a bug, usually. log.Errorw("failed to get agent version from peerstore", "error", err) } } // Amino DHT client keeps information about previously seen peers if nd.HasActiveDHTClient() && nd.DHTClient != nd.DHT { client, ok := nd.DHTClient.(*fullrt.FullRT) if !ok { return VersionCheckOutput{}, errors.New("could not perform version check due to missing or incompatible DHT configuration") } for _, p := range client.Stat() { processPeerstoreEntry(p) } } else if nd.DHT != nil && nd.DHT.WAN != nil { for _, pi := range nd.DHT.WAN.RoutingTable().GetPeerInfos() { processPeerstoreEntry(pi.Id) } } else if nd.DHT != nil && nd.DHT.LAN != nil { for _, pi := range nd.DHT.LAN.RoutingTable().GetPeerInfos() { processPeerstoreEntry(pi.Id) } } else { return VersionCheckOutput{}, errors.New("could not perform version check due to missing or incompatible DHT configuration") } if minPercent < 1 || minPercent > 100 { if minPercent == 0 { minPercent = config.DefaultSwarmCheckPercentThreshold } else { return VersionCheckOutput{}, errors.New("Version.SwarmCheckPercentThreshold must be between 1 and 100") } } minFraction := float64(minPercent) / 100.0 // UpdateAvailable flag is set only if minFraction was reached greaterFraction := float64(withGreaterVersion) / float64(totalPeersSampled) // Gathered metric are returned every time return VersionCheckOutput{ UpdateAvailable: (greaterFraction >= minFraction), RunningVersion: ourVersion.String(), GreatestVersion: greatestVersionSeen.String(), PeersSampled: totalPeersSampled, WithGreaterVersion: withGreaterVersion, }, nil } ================================================ FILE: core/core.go ================================================ /* Package core implements the IpfsNode object and related methods. Packages underneath core/ provide a (relatively) stable, low-level API to carry out most IPFS-related tasks. For more details on the other interfaces and how core/... fits into the bigger IPFS picture, see: $ godoc github.com/ipfs/go-ipfs */ package core import ( "context" "encoding/json" "io" "time" "github.com/ipfs/boxo/filestore" pin "github.com/ipfs/boxo/pinning/pinner" "github.com/ipfs/go-datastore" bitswap "github.com/ipfs/boxo/bitswap" bserv "github.com/ipfs/boxo/blockservice" bstore "github.com/ipfs/boxo/blockstore" exchange "github.com/ipfs/boxo/exchange" "github.com/ipfs/boxo/fetcher" mfs "github.com/ipfs/boxo/mfs" pathresolver "github.com/ipfs/boxo/path/resolver" provider "github.com/ipfs/boxo/provider" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" ddht "github.com/libp2p/go-libp2p-kad-dht/dual" "github.com/libp2p/go-libp2p-kad-dht/fullrt" pubsub "github.com/libp2p/go-libp2p-pubsub" psrouter "github.com/libp2p/go-libp2p-pubsub-router" record "github.com/libp2p/go-libp2p-record" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" connmgr "github.com/libp2p/go-libp2p/core/connmgr" ic "github.com/libp2p/go-libp2p/core/crypto" p2phost "github.com/libp2p/go-libp2p/core/host" metrics "github.com/libp2p/go-libp2p/core/metrics" "github.com/libp2p/go-libp2p/core/network" peer "github.com/libp2p/go-libp2p/core/peer" pstore "github.com/libp2p/go-libp2p/core/peerstore" routing "github.com/libp2p/go-libp2p/core/routing" "github.com/libp2p/go-libp2p/p2p/discovery/mdns" p2pbhost "github.com/libp2p/go-libp2p/p2p/host/basic" ma "github.com/multiformats/go-multiaddr" madns "github.com/multiformats/go-multiaddr-dns" "github.com/ipfs/boxo/bootstrap" "github.com/ipfs/boxo/namesys" ipnsrp "github.com/ipfs/boxo/namesys/republisher" "github.com/ipfs/boxo/peering" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/node" "github.com/ipfs/kubo/core/node/libp2p" "github.com/ipfs/kubo/fuse/mount" "github.com/ipfs/kubo/p2p" "github.com/ipfs/kubo/repo" irouting "github.com/ipfs/kubo/routing" ) var log = logging.Logger("core") // IpfsNode is IPFS Core module. It represents an IPFS instance. type IpfsNode struct { // Self Identity peer.ID // the local node's identity Repo repo.Repo // Local node Pinning pin.Pinner // the pinning manager Mounts Mounts `optional:"true"` // current mount state, if any. PrivateKey ic.PrivKey `optional:"true"` // the local node's private Key PNetFingerprint libp2p.PNetFingerprint `optional:"true"` // fingerprint of private network // Services Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances Blockstore bstore.GCBlockstore // the block store (lower level) Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc Blocks bserv.BlockService // the block service, get/add blocks. DAG ipld.DAGService // the merkle dag service, get/add objects. IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data OfflineIPLDFetcherFactory fetcher.Factory `name:"offlineIpldFetcher"` // fetcher that paths over the IPLD data model without fetching new blocks OfflineUnixFSFetcherFactory fetcher.Factory `name:"offlineUnixfsFetcher"` // fetcher that interprets UnixFS data without fetching new blocks Reporter *metrics.BandwidthCounter `optional:"true"` Discovery mdns.Service `optional:"true"` FilesRoot *mfs.Root RecordValidator record.Validator // Online PeerHost p2phost.Host `optional:"true"` // the network host (server+client) Peering *peering.PeeringService `optional:"true"` Filters *ma.Filters `optional:"true"` Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper ContentDiscovery routing.ContentDiscovery `optional:"true"` // the discovery part of the routing system DNSResolver *madns.Resolver // the DNS resolver IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` // The IPLD path resolver that uses only locally available blocks OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks Exchange exchange.Interface // the block exchange + strategy Bitswap *bitswap.Bitswap `optional:"true"` // The Bitswap instance Namesys namesys.NameSystem // the name system, resolves paths to hashes ProvidingStrategy config.ProvideStrategy `optional:"true"` ProvidingKeyChanFunc provider.KeyChanFunc `optional:"true"` IpnsRepub *ipnsrp.Republisher `optional:"true"` ResourceManager network.ResourceManager `optional:"true"` PubSub *pubsub.PubSub `optional:"true"` PSRouter *psrouter.PubsubValueStore `optional:"true"` Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht Provider node.DHTProvider // the value provider system DHT *ddht.DHT `optional:"true"` DHTClient routing.Routing `name:"dhtc" optional:"true"` P2P *p2p.P2P `optional:"true"` ctx context.Context stop func() error // Flags IsOnline bool `optional:"true"` // Online is set when networking is enabled. IsDaemon bool `optional:"true"` // Daemon is set when running on a long-running daemon. } // Mounts defines what the node's mount state is. This should // perhaps be moved to the daemon or mount. It's here because // it needs to be accessible across daemon requests. type Mounts struct { Ipfs mount.Mount Ipns mount.Mount Mfs mount.Mount } // Close calls Close() on the App object func (n *IpfsNode) Close() error { return n.stop() } // HasActiveDHTClient checks if the node's DHT client is active and usable for DHT operations. // // Returns false for: // - nil DHTClient // - typed nil pointers (e.g., (*ddht.DHT)(nil)) // - no-op routers (routinghelpers.Null) // // Note: This method only checks for known DHT client types (ddht.DHT, fullrt.FullRT). // Custom routing.Routing implementations are not explicitly validated. // // This method prevents the "typed nil interface" bug where an interface contains // a nil pointer of a concrete type, which passes nil checks but panics when methods // are called. func (n *IpfsNode) HasActiveDHTClient() bool { if n.DHTClient == nil { return false } // Check for no-op router (Routing.Type=none) if _, ok := n.DHTClient.(routinghelpers.Null); ok { return false } // Check for typed nil *ddht.DHT (common when Routing.Type=delegated or HTTP-only) if d, ok := n.DHTClient.(*ddht.DHT); ok && d == nil { return false } // Check for typed nil *fullrt.FullRT (accelerated DHT client) if f, ok := n.DHTClient.(*fullrt.FullRT); ok && f == nil { return false } return true } // Context returns the IpfsNode context func (n *IpfsNode) Context() context.Context { if n.ctx == nil { n.ctx = context.TODO() } return n.ctx } // Bootstrap will set and call the IpfsNodes bootstrap function. func (n *IpfsNode) Bootstrap(cfg bootstrap.BootstrapConfig) error { // TODO what should return value be when in offlineMode? if n.Routing == nil { return nil } if n.Bootstrapper != nil { n.Bootstrapper.Close() // stop previous bootstrap process. } // if the caller did not specify a bootstrap peer function, get the // freshest bootstrap peers from config. this responds to live changes. if cfg.BootstrapPeers == nil { cfg.BootstrapPeers = func() []peer.AddrInfo { ps, err := n.loadBootstrapPeers() if err != nil { log.Warn("failed to parse bootstrap peers from config") return nil } return ps } } if load, _ := cfg.BackupPeers(); load == nil { save := func(ctx context.Context, peerList []peer.AddrInfo) { err := n.saveTempBootstrapPeers(ctx, peerList) if err != nil { log.Warnf("saveTempBootstrapPeers failed: %s", err) return } } load = func(ctx context.Context) []peer.AddrInfo { peerList, err := n.loadTempBootstrapPeers(ctx) if err != nil { log.Warnf("loadTempBootstrapPeers failed: %s", err) return nil } return peerList } cfg.SetBackupPeers(load, save) } repoConf, err := n.Repo.Config() if err != nil { return err } if repoConf.Internal.BackupBootstrapInterval != nil { cfg.BackupBootstrapInterval = repoConf.Internal.BackupBootstrapInterval.WithDefault(time.Hour) } n.Bootstrapper, err = bootstrap.Bootstrap(n.Identity, n.PeerHost, n.Routing, cfg) return err } var TempBootstrapPeersKey = datastore.NewKey("/local/temp_bootstrap_peers") func (n *IpfsNode) loadBootstrapPeers() ([]peer.AddrInfo, error) { cfg, err := n.Repo.Config() if err != nil { return nil, err } // Use auto-config resolution for actual bootstrap connectivity return cfg.BootstrapPeersWithAutoConf() } func (n *IpfsNode) saveTempBootstrapPeers(ctx context.Context, peerList []peer.AddrInfo) error { ds := n.Repo.Datastore() bytes, err := json.Marshal(config.BootstrapPeerStrings(peerList)) if err != nil { return err } if err := ds.Put(ctx, TempBootstrapPeersKey, bytes); err != nil { return err } return ds.Sync(ctx, TempBootstrapPeersKey) } func (n *IpfsNode) loadTempBootstrapPeers(ctx context.Context) ([]peer.AddrInfo, error) { ds := n.Repo.Datastore() bytes, err := ds.Get(ctx, TempBootstrapPeersKey) if err != nil { return nil, err } var addrs []string if err := json.Unmarshal(bytes, &addrs); err != nil { return nil, err } return config.ParseBootstrapPeers(addrs) } type ConstructPeerHostOpts struct { AddrsFactory p2pbhost.AddrsFactory DisableNatPortMap bool DisableRelay bool EnableRelayHop bool ConnectionManager connmgr.ConnManager } ================================================ FILE: core/core_test.go ================================================ package core import ( "os" "path/filepath" "testing" context "context" "github.com/ipfs/kubo/repo" "github.com/ipfs/boxo/filestore" "github.com/ipfs/boxo/keystore" datastore "github.com/ipfs/go-datastore" syncds "github.com/ipfs/go-datastore/sync" config "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/node/libp2p" golib "github.com/libp2p/go-libp2p" ddht "github.com/libp2p/go-libp2p-kad-dht/dual" "github.com/libp2p/go-libp2p-kad-dht/fullrt" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" pstore "github.com/libp2p/go-libp2p/core/peerstore" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" ) func TestInitialization(t *testing.T) { ctx := context.Background() id := testIdentity good := []*config.Config{ { Identity: id, Addresses: config.Addresses{ Swarm: []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/udp/4001/quic-v1"}, API: []string{"/ip4/127.0.0.1/tcp/8000"}, }, }, { Identity: id, Addresses: config.Addresses{ Swarm: []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/udp/4001/quic-v1"}, API: []string{"/ip4/127.0.0.1/tcp/8000"}, }, }, } bad := []*config.Config{ {}, } for i, c := range good { r := &repo.Mock{ C: *c, D: syncds.MutexWrap(datastore.NewMapDatastore()), } n, err := NewNode(ctx, &BuildCfg{Repo: r}) if n == nil || err != nil { t.Error("Should have constructed.", i, err) } } for i, c := range bad { r := &repo.Mock{ C: *c, D: syncds.MutexWrap(datastore.NewMapDatastore()), } n, err := NewNode(ctx, &BuildCfg{Repo: r}) if n != nil || err == nil { t.Error("Should have failed to construct.", i) } } } var testIdentity = config.Identity{ PeerID: "QmNgdzLieYi8tgfo2WfTUzNVH5hQK9oAYGVf6dxN12NrHt", PrivKey: "CAASrRIwggkpAgEAAoICAQCwt67GTUQ8nlJhks6CgbLKOx7F5tl1r9zF4m3TUrG3Pe8h64vi+ILDRFd7QJxaJ/n8ux9RUDoxLjzftL4uTdtv5UXl2vaufCc/C0bhCRvDhuWPhVsD75/DZPbwLsepxocwVWTyq7/ZHsCfuWdoh/KNczfy+Gn33gVQbHCnip/uhTVxT7ARTiv8Qa3d7qmmxsR+1zdL/IRO0mic/iojcb3Oc/PRnYBTiAZFbZdUEit/99tnfSjMDg02wRayZaT5ikxa6gBTMZ16Yvienq7RwSELzMQq2jFA4i/TdiGhS9uKywltiN2LrNDBcQJSN02pK12DKoiIy+wuOCRgs2NTQEhU2sXCk091v7giTTOpFX2ij9ghmiRfoSiBFPJA5RGwiH6ansCHtWKY1K8BS5UORM0o3dYk87mTnKbCsdz4bYnGtOWafujYwzueGx8r+IWiys80IPQKDeehnLW6RgoyjszKgL/2XTyP54xMLSW+Qb3BPgDcPaPO0hmop1hW9upStxKsefW2A2d46Ds4HEpJEry7PkS5M4gKL/zCKHuxuXVk14+fZQ1rstMuvKjrekpAC2aVIKMI9VRA3awtnje8HImQMdj+r+bPmv0N8rTTr3eS4J8Yl7k12i95LLfK+fWnmUh22oTNzkRlaiERQrUDyE4XNCtJc0xs1oe1yXGqazCIAQIDAQABAoICAQCk1N/ftahlRmOfAXk//8wNl7FvdJD3le6+YSKBj0uWmN1ZbUSQk64chr12iGCOM2WY180xYjy1LOS44PTXaeW5bEiTSnb3b3SH+HPHaWCNM2EiSogHltYVQjKW+3tfH39vlOdQ9uQ+l9Gh6iTLOqsCRyszpYPqIBwi1NMLY2Ej8PpVU7ftnFWouHZ9YKS7nAEiMoowhTu/7cCIVwZlAy3AySTuKxPMVj9LORqC32PVvBHZaMPJ+X1Xyijqg6aq39WyoztkXg3+Xxx5j5eOrK6vO/Lp6ZUxaQilHDXoJkKEJjgIBDZpluss08UPfOgiWAGkW+L4fgUxY0qDLDAEMhyEBAn6KOKVL1JhGTX6GjhWziI94bddSpHKYOEIDzUy4H8BXnKhtnyQV6ELS65C2hj9D0IMBTj7edCF1poJy0QfdK0cuXgMvxHLeUO5uc2YWfbNosvKxqygB9rToy4b22YvNwsZUXsTY6Jt+p9V2OgXSKfB5VPeRbjTJL6xqvvUJpQytmII/C9JmSDUtCbYceHj6X9jgigLk20VV6nWHqCTj3utXD6NPAjoycVpLKDlnWEgfVELDIk0gobxUqqSm3jTPEKRPJgxkgPxbwxYumtw++1UY2y35w3WRDc2xYPaWKBCQeZy+mL6ByXp9bWlNvxS3Knb6oZp36/ovGnf2pGvdQKCAQEAyKpipz2lIUySDyE0avVWAmQb2tWGKXALPohzj7AwkcfEg2GuwoC6GyVE2sTJD1HRazIjOKn3yQORg2uOPeG7sx7EKHxSxCKDrbPawkvLCq8JYSy9TLvhqKUVVGYPqMBzu2POSLEA81QXas+aYjKOFWA2Zrjq26zV9ey3+6Lc6WULePgRQybU8+RHJc6fdjUCCfUxgOrUO2IQOuTJ+FsDpVnrMUGlokmWn23OjL4qTL9wGDnWGUs2pjSzNbj3qA0d8iqaiMUyHX/D/VS0wpeT1osNBSm8suvSibYBn+7wbIApbwXUxZaxMv2OHGz3empae4ckvNZs7r8wsI9UwFt8mwKCAQEA4XK6gZkv9t+3YCcSPw2ensLvL/xU7i2bkC9tfTGdjnQfzZXIf5KNdVuj/SerOl2S1s45NMs3ysJbADwRb4ahElD/V71nGzV8fpFTitC20ro9fuX4J0+twmBolHqeH9pmeGTjAeL1rvt6vxs4FkeG/yNft7GdXpXTtEGaObn8Mt0tPY+aB3UnKrnCQoQAlPyGHFrVRX0UEcp6wyyNGhJCNKeNOvqCHTFObhbhO+KWpWSN0MkVHnqaIBnIn1Te8FtvP/iTwXGnKc0YXJUG6+LM6LmOguW6tg8ZqiQeYyyR+e9eCFH4csLzkrTl1GxCxwEsoSLIMm7UDcjttW6tYEghkwKCAQEAmeCO5lCPYImnN5Lu71ZTLmI2OgmjaANTnBBnDbi+hgv61gUCToUIMejSdDCTPfwv61P3TmyIZs0luPGxkiKYHTNqmOE9Vspgz8Mr7fLRMNApESuNvloVIY32XVImj/GEzh4rAfM6F15U1sN8T/EUo6+0B/Glp+9R49QzAfRSE2g48/rGwgf1JVHYfVWFUtAzUA+GdqWdOixo5cCsYJbqpNHfWVZN/bUQnBFIYwUwysnC29D+LUdQEQQ4qOm+gFAOtrWU62zMkXJ4iLt8Ify6kbrvsRXgbhQIzzGS7WH9XDarj0eZciuslr15TLMC1Azadf+cXHLR9gMHA13mT9vYIQKCAQA/DjGv8cKCkAvf7s2hqROGYAs6Jp8yhrsN1tYOwAPLRhtnCs+rLrg17M2vDptLlcRuI/vIElamdTmylRpjUQpX7yObzLO73nfVhpwRJVMdGU394iBIDncQ+JoHfUwgqJskbUM40dvZdyjbrqc/Q/4z+hbZb+oN/GXb8sVKBATPzSDMKQ/xqgisYIw+wmDPStnPsHAaIWOtni47zIgilJzD0WEk78/YjmPbUrboYvWziK5JiRRJFA1rkQqV1c0M+OXixIm+/yS8AksgCeaHr0WUieGcJtjT9uE8vyFop5ykhRiNxy9wGaq6i7IEecsrkd6DqxDHWkwhFuO1bSE83q/VAoIBAEA+RX1i/SUi08p71ggUi9WFMqXmzELp1L3hiEjOc2AklHk2rPxsaTh9+G95BvjhP7fRa/Yga+yDtYuyjO99nedStdNNSg03aPXILl9gs3r2dPiQKUEXZJ3FrH6tkils/8BlpOIRfbkszrdZIKTO9GCdLWQ30dQITDACs8zV/1GFGrHFrqnnMe/NpIFHWNZJ0/WZMi8wgWO6Ik8jHEpQtVXRiXLqy7U6hk170pa4GHOzvftfPElOZZjy9qn7KjdAQqy6spIrAE94OEL+fBgbHQZGLpuTlj6w6YGbMtPU8uo7sXKoc6WOCb68JWft3tejGLDa1946HAWqVM9B/UcneNc=", } // mockHostOption creates a HostOption that uses the provided mocknet. // Inlined to avoid import cycle with core/mock package. func mockHostOption(mn mocknet.Mocknet) libp2p.HostOption { return func(id peer.ID, ps pstore.Peerstore, opts ...golib.Option) (host.Host, error) { var cfg golib.Config if err := cfg.Apply(opts...); err != nil { return nil, err } // The mocknet does not use the provided libp2p.Option. This options include // the listening addresses we want our peer listening on. Therefore, we have // to manually parse the configuration and add them here. ps.AddAddrs(id, cfg.ListenAddrs, pstore.PermanentAddrTTL) return mn.AddPeerWithPeerstore(id, ps) } } func TestHasActiveDHTClient(t *testing.T) { // Test 1: nil DHTClient t.Run("nil DHTClient", func(t *testing.T) { node := &IpfsNode{ DHTClient: nil, } if node.HasActiveDHTClient() { t.Error("Expected false for nil DHTClient") } }) // Test 2: Typed nil *ddht.DHT (common case when Routing.Type=delegated) t.Run("typed nil ddht.DHT", func(t *testing.T) { node := &IpfsNode{ DHTClient: (*ddht.DHT)(nil), } if node.HasActiveDHTClient() { t.Error("Expected false for typed nil *ddht.DHT") } }) // Test 3: Typed nil *fullrt.FullRT (accelerated DHT client) t.Run("typed nil fullrt.FullRT", func(t *testing.T) { node := &IpfsNode{ DHTClient: (*fullrt.FullRT)(nil), } if node.HasActiveDHTClient() { t.Error("Expected false for typed nil *fullrt.FullRT") } }) // Test 4: routinghelpers.Null no-op router (Routing.Type=none) t.Run("routinghelpers.Null", func(t *testing.T) { node := &IpfsNode{ DHTClient: routinghelpers.Null{}, } if node.HasActiveDHTClient() { t.Error("Expected false for routinghelpers.Null") } }) // Test 5: Valid standard dual DHT (Routing.Type=auto/dht/dhtclient) t.Run("valid standard dual DHT", func(t *testing.T) { ctx := context.Background() mn := mocknet.New() defer mn.Close() ds := syncds.MutexWrap(datastore.NewMapDatastore()) c := config.Config{} c.Identity = testIdentity c.Addresses.Swarm = []string{"/ip4/0.0.0.0/tcp/4001"} r := &repo.Mock{ C: c, D: ds, K: keystore.NewMemKeystore(), F: filestore.NewFileManager(ds, filepath.Dir(os.TempDir())), } node, err := NewNode(ctx, &BuildCfg{ Routing: libp2p.DHTServerOption, Repo: r, Host: mockHostOption(mn), Online: true, }) if err != nil { t.Fatalf("Failed to create node with DHT: %v", err) } defer node.Close() // First verify test setup created the expected DHT type if node.DHTClient == nil { t.Fatalf("Test setup failed: DHTClient is nil") } if _, ok := node.DHTClient.(*ddht.DHT); !ok { t.Fatalf("Test setup failed: expected DHTClient to be *ddht.DHT, got %T", node.DHTClient) } // Now verify HasActiveDHTClient() correctly identifies it as active if !node.HasActiveDHTClient() { t.Error("Expected true for valid dual DHT client") } }) // Test 6: Valid accelerated DHT client (Routing.Type=autoclient) t.Run("valid accelerated DHT client", func(t *testing.T) { ctx := context.Background() mn := mocknet.New() defer mn.Close() ds := syncds.MutexWrap(datastore.NewMapDatastore()) c := config.Config{} c.Identity = testIdentity c.Addresses.Swarm = []string{"/ip4/0.0.0.0/tcp/4001"} c.Routing.AcceleratedDHTClient = config.True r := &repo.Mock{ C: c, D: ds, K: keystore.NewMemKeystore(), F: filestore.NewFileManager(ds, filepath.Dir(os.TempDir())), } node, err := NewNode(ctx, &BuildCfg{ Routing: libp2p.DHTOption, Repo: r, Host: mockHostOption(mn), Online: true, }) if err != nil { t.Fatalf("Failed to create node with accelerated DHT: %v", err) } defer node.Close() // First verify test setup created the expected accelerated DHT type if node.DHTClient == nil { t.Fatalf("Test setup failed: DHTClient is nil") } if _, ok := node.DHTClient.(*fullrt.FullRT); !ok { t.Fatalf("Test setup failed: expected DHTClient to be *fullrt.FullRT, got %T", node.DHTClient) } // Now verify HasActiveDHTClient() correctly identifies it as active if !node.HasActiveDHTClient() { t.Error("Expected true for valid accelerated DHT client") } }) } ================================================ FILE: core/coreapi/block.go ================================================ package coreapi import ( "bytes" "context" "errors" "io" "github.com/ipfs/boxo/path" pin "github.com/ipfs/boxo/pinning/pinner" blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" coreiface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" util "github.com/ipfs/kubo/blocks/blockstoreutil" "github.com/ipfs/kubo/tracing" ) type BlockAPI CoreAPI type BlockStat struct { path path.ImmutablePath size int } func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.BlockPutOption) (coreiface.BlockStat, error) { ctx, span := tracing.Span(ctx, "CoreAPI.BlockAPI", "Put") defer span.End() settings, err := caopts.BlockPutOptions(opts...) if err != nil { return nil, err } data, err := io.ReadAll(src) if err != nil { return nil, err } bcid, err := settings.CidPrefix.Sum(data) if err != nil { return nil, err } b, err := blocks.NewBlockWithCid(data, bcid) if err != nil { return nil, err } if settings.Pin { defer api.blockstore.PinLock(ctx).Unlock(ctx) } err = api.blocks.AddBlock(ctx, b) if err != nil { return nil, err } if settings.Pin { if err = api.pinning.PinWithMode(ctx, b.Cid(), pin.Recursive, ""); err != nil { return nil, err } if err := api.pinning.Flush(ctx); err != nil { return nil, err } } return &BlockStat{path: path.FromCid(b.Cid()), size: len(data)}, nil } func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { ctx, span := tracing.Span(ctx, "CoreAPI.BlockAPI", "Get", trace.WithAttributes(attribute.String("path", p.String()))) defer span.End() rp, _, err := api.core().ResolvePath(ctx, p) if err != nil { return nil, err } b, err := api.blocks.GetBlock(ctx, rp.RootCid()) if err != nil { return nil, err } return bytes.NewReader(b.RawData()), nil } func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRmOption) error { ctx, span := tracing.Span(ctx, "CoreAPI.BlockAPI", "Rm", trace.WithAttributes(attribute.String("path", p.String()))) defer span.End() rp, _, err := api.core().ResolvePath(ctx, p) if err != nil { return err } settings, err := caopts.BlockRmOptions(opts...) if err != nil { return err } cids := []cid.Cid{rp.RootCid()} o := util.RmBlocksOpts{Force: settings.Force} out, err := util.RmBlocks(ctx, api.blockstore, api.pinning, cids, o) if err != nil { return err } select { case res, ok := <-out: if !ok { return nil } remBlock, ok := res.(*util.RemovedBlock) if !ok { return errors.New("got unexpected output from util.RmBlocks") } if remBlock.Error != nil { return remBlock.Error } return nil case <-ctx.Done(): return ctx.Err() } } func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (coreiface.BlockStat, error) { ctx, span := tracing.Span(ctx, "CoreAPI.BlockAPI", "Stat", trace.WithAttributes(attribute.String("path", p.String()))) defer span.End() rp, _, err := api.core().ResolvePath(ctx, p) if err != nil { return nil, err } b, err := api.blocks.GetBlock(ctx, rp.RootCid()) if err != nil { return nil, err } return &BlockStat{ path: path.FromCid(b.Cid()), size: len(b.RawData()), }, nil } func (bs *BlockStat) Size() int { return bs.size } func (bs *BlockStat) Path() path.ImmutablePath { return bs.path } func (api *BlockAPI) core() coreiface.CoreAPI { return (*CoreAPI)(api) } ================================================ FILE: core/coreapi/coreapi.go ================================================ /* **NOTE: this package is experimental.** Package coreapi provides direct access to the core commands in IPFS. If you are embedding IPFS directly in your Go program, this package is the public interface you should use to read and write files or otherwise control IPFS. If you are running IPFS as a separate process, you should use `client/rpc` to work with it via HTTP. */ package coreapi import ( "context" "errors" "fmt" bserv "github.com/ipfs/boxo/blockservice" blockstore "github.com/ipfs/boxo/blockstore" exchange "github.com/ipfs/boxo/exchange" offlinexch "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/fetcher" dag "github.com/ipfs/boxo/ipld/merkledag" pathresolver "github.com/ipfs/boxo/path/resolver" pin "github.com/ipfs/boxo/pinning/pinner" offlineroute "github.com/ipfs/boxo/routing/offline" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/kubo/config" coreiface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/core/coreiface/options" pubsub "github.com/libp2p/go-libp2p-pubsub" record "github.com/libp2p/go-libp2p-record" ci "github.com/libp2p/go-libp2p/core/crypto" p2phost "github.com/libp2p/go-libp2p/core/host" peer "github.com/libp2p/go-libp2p/core/peer" pstore "github.com/libp2p/go-libp2p/core/peerstore" routing "github.com/libp2p/go-libp2p/core/routing" madns "github.com/multiformats/go-multiaddr-dns" "github.com/ipfs/boxo/namesys" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/node" "github.com/ipfs/kubo/repo" ) type CoreAPI struct { nctx context.Context identity peer.ID privateKey ci.PrivKey repo repo.Repo blockstore blockstore.GCBlockstore baseBlocks blockstore.Blockstore pinning pin.Pinner blocks bserv.BlockService dag ipld.DAGService ipldFetcherFactory fetcher.Factory unixFSFetcherFactory fetcher.Factory peerstore pstore.Peerstore peerHost p2phost.Host recordValidator record.Validator exchange exchange.Interface namesys namesys.NameSystem routing routing.Routing dnsResolver *madns.Resolver ipldPathResolver pathresolver.Resolver unixFSPathResolver pathresolver.Resolver provider node.DHTProvider providingStrategy config.ProvideStrategy pubSub *pubsub.PubSub checkPublishAllowed func() error checkOnline func(allowOffline bool) error // ONLY for re-applying options in WithOptions, DO NOT USE ANYWHERE ELSE nd *core.IpfsNode parentOpts options.ApiSettings } // NewCoreAPI creates new instance of IPFS CoreAPI backed by go-ipfs Node. func NewCoreAPI(n *core.IpfsNode, opts ...options.ApiOption) (coreiface.CoreAPI, error) { parentOpts, err := options.ApiOptions() if err != nil { return nil, err } return (&CoreAPI{nd: n, parentOpts: *parentOpts}).WithOptions(opts...) } // Unixfs returns the UnixfsAPI interface implementation backed by the go-ipfs node func (api *CoreAPI) Unixfs() coreiface.UnixfsAPI { return (*UnixfsAPI)(api) } // Block returns the BlockAPI interface implementation backed by the go-ipfs node func (api *CoreAPI) Block() coreiface.BlockAPI { return (*BlockAPI)(api) } // Dag returns the DagAPI interface implementation backed by the go-ipfs node func (api *CoreAPI) Dag() coreiface.APIDagService { return &dagAPI{ api.dag, api, } } // Name returns the NameAPI interface implementation backed by the go-ipfs node func (api *CoreAPI) Name() coreiface.NameAPI { return (*NameAPI)(api) } // Key returns the KeyAPI interface implementation backed by the go-ipfs node func (api *CoreAPI) Key() coreiface.KeyAPI { return (*KeyAPI)(api) } // Object returns the ObjectAPI interface implementation backed by the go-ipfs node func (api *CoreAPI) Object() coreiface.ObjectAPI { return (*ObjectAPI)(api) } // Pin returns the PinAPI interface implementation backed by the go-ipfs node func (api *CoreAPI) Pin() coreiface.PinAPI { return (*PinAPI)(api) } // Swarm returns the SwarmAPI interface implementation backed by the go-ipfs node func (api *CoreAPI) Swarm() coreiface.SwarmAPI { return (*SwarmAPI)(api) } // PubSub returns the PubSubAPI interface implementation backed by the go-ipfs node func (api *CoreAPI) PubSub() coreiface.PubSubAPI { return (*PubSubAPI)(api) } // Routing returns the RoutingAPI interface implementation backed by the kubo node func (api *CoreAPI) Routing() coreiface.RoutingAPI { return (*RoutingAPI)(api) } // WithOptions returns api with global options applied func (api *CoreAPI) WithOptions(opts ...options.ApiOption) (coreiface.CoreAPI, error) { settings := api.parentOpts // make sure to copy _, err := options.ApiOptionsTo(&settings, opts...) if err != nil { return nil, err } if api.nd == nil { return nil, errors.New("cannot apply options to api without node") } n := api.nd subAPI := &CoreAPI{ nctx: n.Context(), identity: n.Identity, privateKey: n.PrivateKey, repo: n.Repo, blockstore: n.Blockstore, baseBlocks: n.BaseBlocks, pinning: n.Pinning, blocks: n.Blocks, dag: n.DAG, ipldFetcherFactory: n.IPLDFetcherFactory, unixFSFetcherFactory: n.UnixFSFetcherFactory, peerstore: n.Peerstore, peerHost: n.PeerHost, namesys: n.Namesys, recordValidator: n.RecordValidator, exchange: n.Exchange, routing: n.Routing, dnsResolver: n.DNSResolver, ipldPathResolver: n.IPLDPathResolver, unixFSPathResolver: n.UnixFSPathResolver, provider: n.Provider, providingStrategy: n.ProvidingStrategy, pubSub: n.PubSub, nd: n, parentOpts: settings, } subAPI.checkOnline = func(allowOffline bool) error { if !n.IsOnline && !allowOffline { return coreiface.ErrOffline } return nil } subAPI.checkPublishAllowed = func() error { if n.Mounts.Ipns != nil && n.Mounts.Ipns.IsActive() { return errors.New("cannot manually publish while IPNS is mounted") } return nil } cfg, err := n.Repo.Config() if err != nil { return nil, err } if settings.Offline { cs := cfg.Ipns.ResolveCacheSize if cs == 0 { cs = node.DefaultIpnsCacheSize } if cs < 0 { return nil, errors.New("cannot specify negative resolve cache size") } nsOptions := []namesys.Option{ namesys.WithDatastore(subAPI.repo.Datastore()), namesys.WithDNSResolver(subAPI.dnsResolver), namesys.WithCache(cs), namesys.WithMaxCacheTTL(cfg.Ipns.MaxCacheTTL.WithDefault(config.DefaultIpnsMaxCacheTTL)), } subAPI.routing = offlineroute.NewOfflineRouter(subAPI.repo.Datastore(), subAPI.recordValidator) subAPI.namesys, err = namesys.NewNameSystem(subAPI.routing, nsOptions...) if err != nil { return nil, fmt.Errorf("error constructing namesys: %w", err) } subAPI.peerstore = nil subAPI.peerHost = nil subAPI.recordValidator = nil } if settings.Offline || !settings.FetchBlocks { subAPI.exchange = offlinexch.Exchange(subAPI.blockstore) subAPI.blocks = bserv.New(subAPI.blockstore, subAPI.exchange, bserv.WriteThrough(cfg.Datastore.WriteThrough.WithDefault(config.DefaultWriteThrough)), ) subAPI.dag = dag.NewDAGService(subAPI.blocks) } return subAPI, nil } // getSession returns new api backed by the same node with a read-only session DAG func (api *CoreAPI) getSession(ctx context.Context) *CoreAPI { sesAPI := *api // TODO: We could also apply this to api.blocks, and compose into writable api, // but this requires some changes in blockservice/merkledag sesAPI.dag = dag.NewReadOnlyDagService(dag.NewSession(ctx, api.dag)) return &sesAPI } ================================================ FILE: core/coreapi/dag.go ================================================ package coreapi import ( "context" dag "github.com/ipfs/boxo/ipld/merkledag" pin "github.com/ipfs/boxo/pinning/pinner" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "github.com/ipfs/kubo/tracing" ) type dagAPI struct { ipld.DAGService core *CoreAPI } type pinningAdder CoreAPI func (adder *pinningAdder) Add(ctx context.Context, nd ipld.Node) error { ctx, span := tracing.Span(ctx, "CoreAPI.PinningAdder", "Add", trace.WithAttributes(attribute.String("node", nd.String()))) defer span.End() defer adder.blockstore.PinLock(ctx).Unlock(ctx) if err := adder.dag.Add(ctx, nd); err != nil { return err } if err := adder.pinning.PinWithMode(ctx, nd.Cid(), pin.Recursive, ""); err != nil { return err } return adder.pinning.Flush(ctx) } func (adder *pinningAdder) AddMany(ctx context.Context, nds []ipld.Node) error { ctx, span := tracing.Span(ctx, "CoreAPI.PinningAdder", "AddMany", trace.WithAttributes(attribute.Int("nodes.count", len(nds)))) defer span.End() defer adder.blockstore.PinLock(ctx).Unlock(ctx) if err := adder.dag.AddMany(ctx, nds); err != nil { return err } cids := cid.NewSet() for _, nd := range nds { c := nd.Cid() if cids.Visit(c) { if err := adder.pinning.PinWithMode(ctx, c, pin.Recursive, ""); err != nil { return err } } } return adder.pinning.Flush(ctx) } func (api *dagAPI) Pinning() ipld.NodeAdder { return (*pinningAdder)(api.core) } func (api *dagAPI) Session(ctx context.Context) ipld.NodeGetter { return dag.NewSession(ctx, api.DAGService) } var ( _ ipld.DAGService = (*dagAPI)(nil) _ dag.SessionMaker = (*dagAPI)(nil) ) ================================================ FILE: core/coreapi/key.go ================================================ package coreapi import ( "context" "crypto/rand" "errors" "fmt" "sort" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" coreiface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" "github.com/ipfs/kubo/tracing" crypto "github.com/libp2p/go-libp2p/core/crypto" peer "github.com/libp2p/go-libp2p/core/peer" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) type KeyAPI CoreAPI type key struct { name string peerID peer.ID path path.Path } func newKey(name string, pid peer.ID) (*key, error) { p, err := path.NewPath("/ipns/" + ipns.NameFromPeer(pid).String()) if err != nil { return nil, fmt.Errorf("cannot create new key: %w", err) } return &key{ name: name, peerID: pid, path: p, }, nil } // Name returns the key name func (k *key) Name() string { return k.name } // Path returns the path of the key. func (k *key) Path() path.Path { return k.path } // ID returns key PeerID func (k *key) ID() peer.ID { return k.peerID } // Generate generates new key, stores it in the keystore under the specified // name and returns a base58 encoded multihash of its public key. func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (coreiface.Key, error) { _, span := tracing.Span(ctx, "CoreAPI.KeyAPI", "Generate", trace.WithAttributes(attribute.String("name", name))) defer span.End() options, err := caopts.KeyGenerateOptions(opts...) if err != nil { return nil, err } if name == "self" { return nil, errors.New("cannot create key with name 'self'") } _, err = api.repo.Keystore().Get(name) if err == nil { return nil, fmt.Errorf("key with name '%s' already exists", name) } var sk crypto.PrivKey var pk crypto.PubKey switch options.Algorithm { case "rsa": if options.Size == -1 { options.Size = caopts.DefaultRSALen } priv, pub, err := crypto.GenerateKeyPairWithReader(crypto.RSA, options.Size, rand.Reader) if err != nil { return nil, err } sk = priv pk = pub case "ed25519": priv, pub, err := crypto.GenerateEd25519Key(rand.Reader) if err != nil { return nil, err } sk = priv pk = pub default: return nil, fmt.Errorf("unrecognized key type: %s", options.Algorithm) } err = api.repo.Keystore().Put(name, sk) if err != nil { return nil, err } pid, err := peer.IDFromPublicKey(pk) if err != nil { return nil, err } return newKey(name, pid) } // List returns a list keys stored in keystore. func (api *KeyAPI) List(ctx context.Context) ([]coreiface.Key, error) { _, span := tracing.Span(ctx, "CoreAPI.KeyAPI", "List") defer span.End() keys, err := api.repo.Keystore().List() if err != nil { return nil, fmt.Errorf("cannot list keys in keystore: %w", err) } sort.Strings(keys) out := make([]coreiface.Key, 1, len(keys)+1) out[0], err = newKey("self", api.identity) if err != nil { return nil, err } for _, k := range keys { privKey, err := api.repo.Keystore().Get(k) if err != nil { log.Errorf("cannot get key from keystore: %s", err) continue } pubKey := privKey.GetPublic() pid, err := peer.IDFromPublicKey(pubKey) if err != nil { log.Errorf("cannot decode public key: %s", err) continue } k, err := newKey(k, pid) if err != nil { return nil, err } out = append(out, k) } return out, nil } // Rename renames `oldName` to `newName`. Returns the key and whether another // key was overwritten, or an error. func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (coreiface.Key, bool, error) { _, span := tracing.Span(ctx, "CoreAPI.KeyAPI", "Rename", trace.WithAttributes(attribute.String("oldname", oldName), attribute.String("newname", newName))) defer span.End() options, err := caopts.KeyRenameOptions(opts...) if err != nil { return nil, false, err } span.SetAttributes(attribute.Bool("force", options.Force)) ks := api.repo.Keystore() if oldName == "self" { return nil, false, errors.New("cannot rename key with name 'self'") } if newName == "self" { return nil, false, errors.New("cannot overwrite key with name 'self'") } oldKey, err := ks.Get(oldName) if err != nil { return nil, false, fmt.Errorf("no key named %s was found", oldName) } pubKey := oldKey.GetPublic() pid, err := peer.IDFromPublicKey(pubKey) if err != nil { return nil, false, err } // This is important, because future code will delete key `oldName` // even if it is the same as newName. if newName == oldName { k, err := newKey(oldName, pid) return k, false, err } overwrite := false if options.Force { exist, err := ks.Has(newName) if err != nil { return nil, false, err } if exist { overwrite = true err := ks.Delete(newName) if err != nil { return nil, false, err } } } err = ks.Put(newName, oldKey) if err != nil { return nil, false, err } err = ks.Delete(oldName) if err != nil { return nil, false, err } k, err := newKey(newName, pid) return k, overwrite, err } // Remove removes keys from keystore. Returns ipns path of the removed key. func (api *KeyAPI) Remove(ctx context.Context, name string) (coreiface.Key, error) { _, span := tracing.Span(ctx, "CoreAPI.KeyAPI", "Remove", trace.WithAttributes(attribute.String("name", name))) defer span.End() ks := api.repo.Keystore() if name == "self" { return nil, errors.New("cannot remove key with name 'self'") } removed, err := ks.Get(name) if err != nil { return nil, fmt.Errorf("no key named %s was found", name) } pubKey := removed.GetPublic() pid, err := peer.IDFromPublicKey(pubKey) if err != nil { return nil, err } err = ks.Delete(name) if err != nil { return nil, err } return newKey("", pid) } func (api *KeyAPI) Self(ctx context.Context) (coreiface.Key, error) { if api.identity == "" { return nil, errors.New("identity not loaded") } return newKey("self", api.identity) } const signedMessagePrefix = "libp2p-key signed message:" func (api *KeyAPI) Sign(ctx context.Context, name string, data []byte) (coreiface.Key, []byte, error) { var ( sk crypto.PrivKey err error ) if name == "" || name == "self" { name = "self" sk = api.privateKey } else { sk, err = api.repo.Keystore().Get(name) } if err != nil { return nil, nil, err } pid, err := peer.IDFromPrivateKey(sk) if err != nil { return nil, nil, err } key, err := newKey(name, pid) if err != nil { return nil, nil, err } data = append([]byte(signedMessagePrefix), data...) sig, err := sk.Sign(data) if err != nil { return nil, nil, err } return key, sig, nil } func (api *KeyAPI) Verify(ctx context.Context, keyOrName string, signature, data []byte) (coreiface.Key, bool, error) { var ( name string pk crypto.PubKey err error ) if keyOrName == "" || keyOrName == "self" { name = "self" pk = api.privateKey.GetPublic() } else if sk, err := api.repo.Keystore().Get(keyOrName); err == nil { name = keyOrName pk = sk.GetPublic() } else if ipnsName, err := ipns.NameFromString(keyOrName); err == nil { // This works for both IPNS names and Peer IDs. name = "" pk, err = ipnsName.Peer().ExtractPublicKey() if err != nil { return nil, false, err } } else { return nil, false, fmt.Errorf("'%q' is not a known key, an IPNS Name, or a valid PeerID", keyOrName) } pid, err := peer.IDFromPublicKey(pk) if err != nil { return nil, false, err } key, err := newKey(name, pid) if err != nil { return nil, false, err } data = append([]byte(signedMessagePrefix), data...) valid, err := pk.Verify(data, signature) if err != nil { return nil, false, err } return key, valid, nil } ================================================ FILE: core/coreapi/name.go ================================================ package coreapi import ( "context" "errors" "fmt" "strings" "time" "github.com/ipfs/boxo/ipns" keystore "github.com/ipfs/boxo/keystore" "github.com/ipfs/boxo/namesys" "github.com/ipfs/kubo/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "github.com/ipfs/boxo/path" coreiface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" ci "github.com/libp2p/go-libp2p/core/crypto" peer "github.com/libp2p/go-libp2p/core/peer" ) type NameAPI CoreAPI // Publish announces new IPNS name and returns the new IPNS entry. func (api *NameAPI) Publish(ctx context.Context, p path.Path, opts ...caopts.NamePublishOption) (ipns.Name, error) { ctx, span := tracing.Span(ctx, "CoreAPI.NameAPI", "Publish", trace.WithAttributes(attribute.String("path", p.String()))) defer span.End() if err := api.checkPublishAllowed(); err != nil { return ipns.Name{}, err } options, err := caopts.NamePublishOptions(opts...) if err != nil { return ipns.Name{}, err } span.SetAttributes( attribute.Bool("allowoffline", options.AllowOffline), attribute.String("key", options.Key), attribute.Float64("validtime", options.ValidTime.Seconds()), ) if options.TTL != nil { span.SetAttributes(attribute.Float64("ttl", options.TTL.Seconds())) } // Handle different publishing modes if options.AllowDelegated { // AllowDelegated mode: check if delegated publishers are configured cfg, err := api.repo.Config() if err != nil { return ipns.Name{}, fmt.Errorf("failed to read config: %w", err) } delegatedPublishers := cfg.DelegatedPublishersWithAutoConf() if len(delegatedPublishers) == 0 { return ipns.Name{}, errors.New("no delegated publishers configured: add Ipns.DelegatedPublishers or use --allow-offline for local-only publishing") } // For allow-delegated mode, we only require that we have delegated publishers configured // The node doesn't need P2P connectivity since we're using HTTP publishing } else { // Normal mode: check online status with allow-offline flag err = api.checkOnline(options.AllowOffline) if err != nil { return ipns.Name{}, err } } k, err := keylookup(api.privateKey, api.repo.Keystore(), options.Key) if err != nil { return ipns.Name{}, err } eol := time.Now().Add(options.ValidTime) publishOptions := []namesys.PublishOption{ namesys.PublishWithEOL(eol), namesys.PublishWithIPNSOption(ipns.WithV1Compatibility(options.CompatibleWithV1)), } if options.TTL != nil { publishOptions = append(publishOptions, namesys.PublishWithTTL(*options.TTL)) } if options.Sequence != nil { publishOptions = append(publishOptions, namesys.PublishWithSequence(*options.Sequence)) } err = api.namesys.Publish(ctx, k, p, publishOptions...) if err != nil { return ipns.Name{}, err } pid, err := peer.IDFromPrivateKey(k) if err != nil { return ipns.Name{}, err } return ipns.NameFromPeer(pid), nil } func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan coreiface.IpnsResult, error) { ctx, span := tracing.Span(ctx, "CoreAPI.NameAPI", "Search", trace.WithAttributes(attribute.String("name", name))) defer span.End() options, err := caopts.NameResolveOptions(opts...) if err != nil { return nil, err } span.SetAttributes(attribute.Bool("cache", options.Cache)) err = api.checkOnline(true) if err != nil { return nil, err } var resolver namesys.Resolver = api.namesys if !options.Cache { resolver, err = namesys.NewNameSystem(api.routing, namesys.WithDatastore(api.repo.Datastore()), namesys.WithDNSResolver(api.dnsResolver)) if err != nil { return nil, err } } if !strings.HasPrefix(name, "/ipns/") { name = "/ipns/" + name } p, err := path.NewPath(name) if err != nil { return nil, err } out := make(chan coreiface.IpnsResult) go func() { defer close(out) for res := range resolver.ResolveAsync(ctx, p, options.ResolveOpts...) { select { case out <- coreiface.IpnsResult{Path: res.Path, Err: res.Err}: case <-ctx.Done(): return } } }() return out, nil } // Resolve attempts to resolve the newest version of the specified name and // returns its path. func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (path.Path, error) { ctx, span := tracing.Span(ctx, "CoreAPI.NameAPI", "Resolve", trace.WithAttributes(attribute.String("name", name))) defer span.End() ctx, cancel := context.WithCancel(ctx) defer cancel() results, err := api.Search(ctx, name, opts...) if err != nil { return nil, err } err = coreiface.ErrResolveFailed var p path.Path for res := range results { p, err = res.Path, res.Err if err != nil { break } } return p, err } func keylookup(self ci.PrivKey, kstore keystore.Keystore, k string) (ci.PrivKey, error) { //////////////////// // Lookup by name // //////////////////// // First, lookup self. if k == "self" { return self, nil } // Then, look in the keystore. res, err := kstore.Get(k) if res != nil { return res, nil } if err != nil && err != keystore.ErrNoSuchKey { return nil, err } keys, err := kstore.List() if err != nil { return nil, err } ////////////////// // Lookup by ID // ////////////////// targetPid, err := peer.Decode(k) if err != nil { return nil, keystore.ErrNoSuchKey } // First, check self. pid, err := peer.IDFromPrivateKey(self) if err != nil { return nil, fmt.Errorf("failed to determine peer ID for private key: %w", err) } if pid == targetPid { return self, nil } // Then, look in the keystore. for _, key := range keys { privKey, err := kstore.Get(key) if err != nil { return nil, err } pid, err := peer.IDFromPrivateKey(privKey) if err != nil { return nil, err } if targetPid == pid { return privKey, nil } } return nil, errors.New("no key by the given name or PeerID was found") } ================================================ FILE: core/coreapi/object.go ================================================ package coreapi import ( "context" dag "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/boxo/ipld/merkledag/dagutils" ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/boxo/path" coreiface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "github.com/ipfs/kubo/tracing" ) type ObjectAPI CoreAPI type Link struct { Name, Hash string Size uint64 } type Node struct { Links []Link Data string } func (api *ObjectAPI) AddLink(ctx context.Context, base path.Path, name string, child path.Path, opts ...caopts.ObjectAddLinkOption) (path.ImmutablePath, error) { ctx, span := tracing.Span(ctx, "CoreAPI.ObjectAPI", "AddLink", trace.WithAttributes( attribute.String("base", base.String()), attribute.String("name", name), attribute.String("child", child.String()), )) defer span.End() options, err := caopts.ObjectAddLinkOptions(opts...) if err != nil { return path.ImmutablePath{}, err } span.SetAttributes(attribute.Bool("create", options.Create)) baseNd, err := api.core().ResolveNode(ctx, base) if err != nil { return path.ImmutablePath{}, err } childNd, err := api.core().ResolveNode(ctx, child) if err != nil { return path.ImmutablePath{}, err } basePb, ok := baseNd.(*dag.ProtoNode) if !ok { return path.ImmutablePath{}, dag.ErrNotProtobuf } var createfunc func() *dag.ProtoNode if options.Create { createfunc = ft.EmptyDirNode } e := dagutils.NewDagEditor(basePb, api.dag) err = e.InsertNodeAtPath(ctx, name, childNd, createfunc) if err != nil { return path.ImmutablePath{}, err } nnode, err := e.Finalize(ctx, api.dag) if err != nil { return path.ImmutablePath{}, err } return path.FromCid(nnode.Cid()), nil } func (api *ObjectAPI) RmLink(ctx context.Context, base path.Path, link string) (path.ImmutablePath, error) { ctx, span := tracing.Span(ctx, "CoreAPI.ObjectAPI", "RmLink", trace.WithAttributes( attribute.String("base", base.String()), attribute.String("link", link)), ) defer span.End() baseNd, err := api.core().ResolveNode(ctx, base) if err != nil { return path.ImmutablePath{}, err } basePb, ok := baseNd.(*dag.ProtoNode) if !ok { return path.ImmutablePath{}, dag.ErrNotProtobuf } e := dagutils.NewDagEditor(basePb, api.dag) err = e.RmLink(ctx, link) if err != nil { return path.ImmutablePath{}, err } nnode, err := e.Finalize(ctx, api.dag) if err != nil { return path.ImmutablePath{}, err } return path.FromCid(nnode.Cid()), nil } func (api *ObjectAPI) Diff(ctx context.Context, before path.Path, after path.Path) ([]coreiface.ObjectChange, error) { ctx, span := tracing.Span(ctx, "CoreAPI.ObjectAPI", "Diff", trace.WithAttributes( attribute.String("before", before.String()), attribute.String("after", after.String()), )) defer span.End() beforeNd, err := api.core().ResolveNode(ctx, before) if err != nil { return nil, err } afterNd, err := api.core().ResolveNode(ctx, after) if err != nil { return nil, err } changes, err := dagutils.Diff(ctx, api.dag, beforeNd, afterNd) if err != nil { return nil, err } out := make([]coreiface.ObjectChange, len(changes)) for i, change := range changes { out[i] = coreiface.ObjectChange{ Type: coreiface.ChangeType(change.Type), Path: change.Path, } if change.Before.Defined() { out[i].Before = path.FromCid(change.Before) } if change.After.Defined() { out[i].After = path.FromCid(change.After) } } return out, nil } func (api *ObjectAPI) core() coreiface.CoreAPI { return (*CoreAPI)(api) } ================================================ FILE: core/coreapi/path.go ================================================ package coreapi import ( "context" "errors" "fmt" "github.com/ipfs/boxo/namesys" "github.com/ipfs/kubo/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "github.com/ipfs/boxo/path" ipfspathresolver "github.com/ipfs/boxo/path/resolver" ipld "github.com/ipfs/go-ipld-format" coreiface "github.com/ipfs/kubo/core/coreiface" ) // ResolveNode resolves the path `p` using Unixfs resolver, gets and returns the // resolved Node. func (api *CoreAPI) ResolveNode(ctx context.Context, p path.Path) (ipld.Node, error) { ctx, span := tracing.Span(ctx, "CoreAPI", "ResolveNode", trace.WithAttributes(attribute.String("path", p.String()))) defer span.End() rp, _, err := api.ResolvePath(ctx, p) if err != nil { return nil, err } node, err := api.dag.Get(ctx, rp.RootCid()) if err != nil { return nil, err } return node, nil } // ResolvePath resolves the path `p` using Unixfs resolver, returns the // resolved path. func (api *CoreAPI) ResolvePath(ctx context.Context, p path.Path) (path.ImmutablePath, []string, error) { ctx, span := tracing.Span(ctx, "CoreAPI", "ResolvePath", trace.WithAttributes(attribute.String("path", p.String()))) defer span.End() res, err := namesys.Resolve(ctx, api.namesys, p) if errors.Is(err, namesys.ErrNoNamesys) { return path.ImmutablePath{}, nil, coreiface.ErrOffline } else if err != nil { return path.ImmutablePath{}, nil, err } p = res.Path var resolver ipfspathresolver.Resolver switch p.Namespace() { case path.IPLDNamespace: resolver = api.ipldPathResolver case path.IPFSNamespace: resolver = api.unixFSPathResolver default: return path.ImmutablePath{}, nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace()) } imPath, err := path.NewImmutablePath(p) if err != nil { return path.ImmutablePath{}, nil, err } node, remainder, err := resolver.ResolveToLastNode(ctx, imPath) if err != nil { return path.ImmutablePath{}, nil, err } segments := []string{p.Namespace(), node.String()} segments = append(segments, remainder...) p, err = path.NewPathFromSegments(segments...) if err != nil { return path.ImmutablePath{}, nil, err } imPath, err = path.NewImmutablePath(p) if err != nil { return path.ImmutablePath{}, nil, err } return imPath, remainder, nil } ================================================ FILE: core/coreapi/pin.go ================================================ package coreapi import ( "context" "fmt" "strings" bserv "github.com/ipfs/boxo/blockservice" offline "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/boxo/path" pin "github.com/ipfs/boxo/pinning/pinner" "github.com/ipfs/go-cid" coreiface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "github.com/ipfs/kubo/tracing" ) type PinAPI CoreAPI func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOption) error { ctx, span := tracing.Span(ctx, "CoreAPI.PinAPI", "Add", trace.WithAttributes(attribute.String("path", p.String()))) defer span.End() dagNode, err := api.core().ResolveNode(ctx, p) if err != nil { return fmt.Errorf("pin: %s", err) } settings, err := caopts.PinAddOptions(opts...) if err != nil { return err } span.SetAttributes(attribute.Bool("recursive", settings.Recursive)) defer api.blockstore.PinLock(ctx).Unlock(ctx) err = api.pinning.Pin(ctx, dagNode, settings.Recursive, settings.Name) if err != nil { return fmt.Errorf("pin: %s", err) } return api.pinning.Flush(ctx) } func (api *PinAPI) Ls(ctx context.Context, pins chan<- coreiface.Pin, opts ...caopts.PinLsOption) error { ctx, span := tracing.Span(ctx, "CoreAPI.PinAPI", "Ls") defer span.End() settings, err := caopts.PinLsOptions(opts...) if err != nil { close(pins) return err } span.SetAttributes(attribute.String("type", settings.Type)) switch settings.Type { case "all", "direct", "indirect", "recursive": default: close(pins) return fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", settings.Type) } return api.pinLsAll(ctx, settings.Type, settings.Detailed, settings.Name, pins) } func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.PinIsPinnedOption) (string, bool, error) { ctx, span := tracing.Span(ctx, "CoreAPI.PinAPI", "IsPinned", trace.WithAttributes(attribute.String("path", p.String()))) defer span.End() resolved, _, err := api.core().ResolvePath(ctx, p) if err != nil { return "", false, fmt.Errorf("error resolving path: %s", err) } settings, err := caopts.PinIsPinnedOptions(opts...) if err != nil { return "", false, err } span.SetAttributes(attribute.String("withtype", settings.WithType)) mode, ok := pin.StringToMode(settings.WithType) if !ok { return "", false, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", settings.WithType) } return api.pinning.IsPinnedWithType(ctx, resolved.RootCid(), mode) } // Rm pin rm api func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOption) error { ctx, span := tracing.Span(ctx, "CoreAPI.PinAPI", "Rm", trace.WithAttributes(attribute.String("path", p.String()))) defer span.End() rp, _, err := api.core().ResolvePath(ctx, p) if err != nil { return err } settings, err := caopts.PinRmOptions(opts...) if err != nil { return err } span.SetAttributes(attribute.Bool("recursive", settings.Recursive)) // Note: after unpin the pin sets are flushed to the blockstore, so we need // to take a lock to prevent a concurrent garbage collection defer api.blockstore.PinLock(ctx).Unlock(ctx) if err = api.pinning.Unpin(ctx, rp.RootCid(), settings.Recursive); err != nil { return err } return api.pinning.Flush(ctx) } func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opts ...caopts.PinUpdateOption) error { ctx, span := tracing.Span(ctx, "CoreAPI.PinAPI", "Update", trace.WithAttributes( attribute.String("from", from.String()), attribute.String("to", to.String()), )) defer span.End() settings, err := caopts.PinUpdateOptions(opts...) if err != nil { return err } span.SetAttributes(attribute.Bool("unpin", settings.Unpin)) fp, _, err := api.core().ResolvePath(ctx, from) if err != nil { return err } tp, _, err := api.core().ResolvePath(ctx, to) if err != nil { return err } defer api.blockstore.PinLock(ctx).Unlock(ctx) err = api.pinning.Update(ctx, fp.RootCid(), tp.RootCid(), settings.Unpin) if err != nil { return err } return api.pinning.Flush(ctx) } type pinStatus struct { err error cid cid.Cid ok bool badNodes []coreiface.BadPinNode } // BadNode is used in PinVerifyRes type badNode struct { path path.ImmutablePath err error } func (s *pinStatus) Ok() bool { return s.ok } func (s *pinStatus) BadNodes() []coreiface.BadPinNode { return s.badNodes } func (s *pinStatus) Err() error { return s.err } func (n *badNode) Path() path.ImmutablePath { return n.path } func (n *badNode) Err() error { return n.err } func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, error) { ctx, span := tracing.Span(ctx, "CoreAPI.PinAPI", "Verify") defer span.End() visited := make(map[cid.Cid]*pinStatus) bs := api.blockstore DAG := merkledag.NewDAGService(bserv.New(bs, offline.Exchange(bs))) getLinks := merkledag.GetLinksWithDAG(DAG) var checkPin func(root cid.Cid) *pinStatus checkPin = func(root cid.Cid) *pinStatus { ctx, span := tracing.Span(ctx, "CoreAPI.PinAPI", "Verify.CheckPin", trace.WithAttributes(attribute.String("cid", root.String()))) defer span.End() if status, ok := visited[root]; ok { return status } links, err := getLinks(ctx, root) if err != nil { status := &pinStatus{ok: false, cid: root} status.badNodes = []coreiface.BadPinNode{&badNode{path: path.FromCid(root), err: err}} visited[root] = status return status } status := &pinStatus{ok: true, cid: root} for _, lnk := range links { res := checkPin(lnk.Cid) if !res.ok { status.ok = false status.badNodes = append(status.badNodes, res.badNodes...) } } visited[root] = status return status } out := make(chan coreiface.PinStatus) go func() { defer close(out) for p := range api.pinning.RecursiveKeys(ctx, false) { var res *pinStatus if p.Err != nil { res = &pinStatus{err: p.Err} } else { res = checkPin(p.Pin.Key) } select { case <-ctx.Done(): return case out <- res: } } }() return out, nil } type pinInfo struct { pinType string path path.ImmutablePath name string } func (p *pinInfo) Path() path.ImmutablePath { return p.path } func (p *pinInfo) Type() string { return p.pinType } func (p *pinInfo) Name() string { return p.name } // pinLsAll is an internal function for returning a list of pins // // The caller must keep reading results until the channel is closed to prevent // leaking the goroutine that is fetching pins. func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string, detailed bool, name string, out chan<- coreiface.Pin) error { defer close(out) emittedSet := cid.NewSet() AddToResultKeys := func(c cid.Cid, pinName, typeStr string) error { if emittedSet.Visit(c) && (name == "" || strings.Contains(pinName, name)) { select { case out <- &pinInfo{ pinType: typeStr, name: pinName, path: path.FromCid(c), }: case <-ctx.Done(): return ctx.Err() } } return nil } var rkeys []cid.Cid var err error if typeStr == "recursive" || typeStr == "all" { for streamedCid := range api.pinning.RecursiveKeys(ctx, detailed) { if streamedCid.Err != nil { return streamedCid.Err } if err = AddToResultKeys(streamedCid.Pin.Key, streamedCid.Pin.Name, "recursive"); err != nil { return err } rkeys = append(rkeys, streamedCid.Pin.Key) } } if typeStr == "direct" || typeStr == "all" { for streamedCid := range api.pinning.DirectKeys(ctx, detailed) { if streamedCid.Err != nil { return streamedCid.Err } if err = AddToResultKeys(streamedCid.Pin.Key, streamedCid.Pin.Name, "direct"); err != nil { return err } } } if typeStr == "indirect" { // We need to first visit the direct pins that have priority // without emitting them for streamedCid := range api.pinning.DirectKeys(ctx, detailed) { if streamedCid.Err != nil { return streamedCid.Err } emittedSet.Add(streamedCid.Pin.Key) } for streamedCid := range api.pinning.RecursiveKeys(ctx, detailed) { if streamedCid.Err != nil { return streamedCid.Err } emittedSet.Add(streamedCid.Pin.Key) rkeys = append(rkeys, streamedCid.Pin.Key) } } if typeStr == "indirect" || typeStr == "all" { if len(rkeys) == 0 { return nil } var addErr error walkingSet := cid.NewSet() for _, k := range rkeys { err = merkledag.Walk( ctx, merkledag.GetLinksWithDAG(api.dag), k, func(c cid.Cid) bool { if !walkingSet.Visit(c) { return false } if emittedSet.Has(c) { return true // skipped } addErr = AddToResultKeys(c, "", "indirect") return addErr == nil }, merkledag.SkipRoot(), merkledag.Concurrent(), ) if err != nil { return err } if addErr != nil { return addErr } } } return nil } func (api *PinAPI) core() coreiface.CoreAPI { return (*CoreAPI)(api) } ================================================ FILE: core/coreapi/pubsub.go ================================================ package coreapi import ( "context" "errors" coreiface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" "github.com/ipfs/kubo/tracing" pubsub "github.com/libp2p/go-libp2p-pubsub" peer "github.com/libp2p/go-libp2p/core/peer" routing "github.com/libp2p/go-libp2p/core/routing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) type PubSubAPI CoreAPI type pubSubSubscription struct { subscription *pubsub.Subscription } type pubSubMessage struct { msg *pubsub.Message } func (api *PubSubAPI) Ls(ctx context.Context) ([]string, error) { _, span := tracing.Span(ctx, "CoreAPI.PubSubAPI", "Ls") defer span.End() _, err := api.checkNode() if err != nil { return nil, err } return api.pubSub.GetTopics(), nil } func (api *PubSubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOption) ([]peer.ID, error) { _, span := tracing.Span(ctx, "CoreAPI.PubSubAPI", "Peers") defer span.End() _, err := api.checkNode() if err != nil { return nil, err } settings, err := caopts.PubSubPeersOptions(opts...) if err != nil { return nil, err } span.SetAttributes(attribute.String("topic", settings.Topic)) return api.pubSub.ListPeers(settings.Topic), nil } func (api *PubSubAPI) Publish(ctx context.Context, topic string, data []byte) error { _, span := tracing.Span(ctx, "CoreAPI.PubSubAPI", "Publish", trace.WithAttributes(attribute.String("topic", topic))) defer span.End() _, err := api.checkNode() if err != nil { return err } //nolint deprecated return api.pubSub.Publish(topic, data) } func (api *PubSubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (coreiface.PubSubSubscription, error) { _, span := tracing.Span(ctx, "CoreAPI.PubSubAPI", "Subscribe", trace.WithAttributes(attribute.String("topic", topic))) defer span.End() // Parse the options to avoid introducing silent failures for invalid // options. However, we don't currently have any use for them. The only // subscription option, discovery, is now a no-op as it's handled by // pubsub itself. _, err := caopts.PubSubSubscribeOptions(opts...) if err != nil { return nil, err } _, err = api.checkNode() if err != nil { return nil, err } //nolint deprecated sub, err := api.pubSub.Subscribe(topic) if err != nil { return nil, err } return &pubSubSubscription{sub}, nil } func (api *PubSubAPI) checkNode() (routing.Routing, error) { if api.pubSub == nil { return nil, errors.New("experimental pubsub feature not enabled, run daemon with --enable-pubsub-experiment to use") } err := api.checkOnline(false) if err != nil { return nil, err } return api.routing, nil } func (sub *pubSubSubscription) Close() error { sub.subscription.Cancel() return nil } func (sub *pubSubSubscription) Next(ctx context.Context) (coreiface.PubSubMessage, error) { ctx, span := tracing.Span(ctx, "CoreAPI.PubSubSubscription", "Next") defer span.End() msg, err := sub.subscription.Next(ctx) if err != nil { return nil, err } return &pubSubMessage{msg}, nil } func (msg *pubSubMessage) From() peer.ID { return peer.ID(msg.msg.From) } func (msg *pubSubMessage) Data() []byte { return msg.msg.Data } func (msg *pubSubMessage) Seq() []byte { return msg.msg.Seqno } func (msg *pubSubMessage) Topics() []string { // TODO: handle breaking downstream changes by returning a single string. if msg.msg.Topic == nil { return nil } return []string{*msg.msg.Topic} } ================================================ FILE: core/coreapi/routing.go ================================================ package coreapi import ( "context" "errors" "fmt" "strings" blockservice "github.com/ipfs/boxo/blockservice" blockstore "github.com/ipfs/boxo/blockstore" offline "github.com/ipfs/boxo/exchange/offline" dag "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/boxo/path" cid "github.com/ipfs/go-cid" cidutil "github.com/ipfs/go-cidutil" coreiface "github.com/ipfs/kubo/core/coreiface" caopts "github.com/ipfs/kubo/core/coreiface/options" "github.com/ipfs/kubo/core/node" "github.com/ipfs/kubo/tracing" peer "github.com/libp2p/go-libp2p/core/peer" mh "github.com/multiformats/go-multihash" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) type RoutingAPI CoreAPI func (api *RoutingAPI) Get(ctx context.Context, key string) ([]byte, error) { if !api.nd.IsOnline { return nil, coreiface.ErrOffline } dhtKey, err := normalizeKey(key) if err != nil { return nil, err } return api.routing.GetValue(ctx, dhtKey) } func (api *RoutingAPI) Put(ctx context.Context, key string, value []byte, opts ...caopts.RoutingPutOption) error { options, err := caopts.RoutingPutOptions(opts...) if err != nil { return err } err = api.checkOnline(options.AllowOffline) if err != nil { return err } dhtKey, err := normalizeKey(key) if err != nil { return err } return api.routing.PutValue(ctx, dhtKey, value) } func normalizeKey(s string) (string, error) { parts := strings.Split(s, "/") if len(parts) != 3 || parts[0] != "" || !(parts[1] == "ipns" || parts[1] == "pk") { return "", errors.New("invalid key") } k, err := peer.Decode(parts[2]) if err != nil { return "", err } return strings.Join(append(parts[:2], string(k)), "/"), nil } func (api *RoutingAPI) FindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error) { ctx, span := tracing.Span(ctx, "CoreAPI.DhtAPI", "FindPeer", trace.WithAttributes(attribute.String("peer", p.String()))) defer span.End() err := api.checkOnline(false) if err != nil { return peer.AddrInfo{}, err } pi, err := api.routing.FindPeer(ctx, peer.ID(p)) if err != nil { return peer.AddrInfo{}, err } return pi, nil } func (api *RoutingAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopts.RoutingFindProvidersOption) (<-chan peer.AddrInfo, error) { ctx, span := tracing.Span(ctx, "CoreAPI.DhtAPI", "FindProviders", trace.WithAttributes(attribute.String("path", p.String()))) defer span.End() settings, err := caopts.RoutingFindProvidersOptions(opts...) if err != nil { return nil, err } span.SetAttributes(attribute.Int("numproviders", settings.NumProviders)) err = api.checkOnline(false) if err != nil { return nil, err } rp, _, err := api.core().ResolvePath(ctx, p) if err != nil { return nil, err } numProviders := settings.NumProviders if numProviders < 1 { return nil, errors.New("number of providers must be greater than 0") } pchan := api.routing.FindProvidersAsync(ctx, rp.RootCid(), numProviders) return pchan, nil } func (api *RoutingAPI) Provide(ctx context.Context, path path.Path, opts ...caopts.RoutingProvideOption) error { ctx, span := tracing.Span(ctx, "CoreAPI.DhtAPI", "Provide", trace.WithAttributes(attribute.String("path", path.String()))) defer span.End() settings, err := caopts.RoutingProvideOptions(opts...) if err != nil { return err } span.SetAttributes(attribute.Bool("recursive", settings.Recursive)) err = api.checkOnline(false) if err != nil { return err } rp, _, err := api.core().ResolvePath(ctx, path) if err != nil { return err } c := rp.RootCid() has, err := api.blockstore.Has(ctx, c) if err != nil { return err } if !has { return fmt.Errorf("block %s not found locally, cannot provide", c) } if settings.Recursive { err = provideKeysRec(ctx, api.provider, api.blockstore, []cid.Cid{c}) } else { err = api.provider.StartProviding(false, c.Hash()) } if err != nil { return err } return nil } func provideKeysRec(ctx context.Context, prov node.DHTProvider, bs blockstore.Blockstore, cids []cid.Cid) error { provided := cidutil.NewStreamingSet() // Error channel with buffer size 1 to avoid blocking the goroutine errCh := make(chan error, 1) go func() { // Always close provided.New to signal completion defer close(provided.New) // Also close error channel to distinguish between "no error" and "pending error" defer close(errCh) dserv := dag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) for _, c := range cids { if err := dag.Walk(ctx, dag.GetLinksDirect(dserv), c, provided.Visitor(ctx)); err != nil { // Send error to channel. If context is cancelled while trying to send, // exit immediately as the main loop will return ctx.Err() select { case errCh <- err: // Error sent successfully, exit goroutine case <-ctx.Done(): // Context cancelled, exit without sending error return } return } } // All CIDs walked successfully, goroutine will exit and channels will close }() keys := make([]mh.Multihash, 0) for { select { case <-ctx.Done(): // Context cancelled, return immediately return ctx.Err() case err := <-errCh: // Received error from DAG walk, return it return err case c, ok := <-provided.New: if !ok { // Channel closed means goroutine finished. // CRITICAL: Check for any error that was sent just before channel closure. // This handles the race where error is sent to errCh but main loop // sees provided.New close first. select { case err := <-errCh: if err != nil { return err } // errCh closed with nil, meaning success default: // No pending error in errCh } // All CIDs successfully processed, start providing return prov.StartProviding(true, keys...) } // Accumulate the CID for providing keys = append(keys, c.Hash()) } } } func (api *RoutingAPI) core() coreiface.CoreAPI { return (*CoreAPI)(api) } ================================================ FILE: core/coreapi/swarm.go ================================================ package coreapi import ( "context" "sort" "time" coreiface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/tracing" inet "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" pstore "github.com/libp2p/go-libp2p/core/peerstore" "github.com/libp2p/go-libp2p/core/protocol" "github.com/libp2p/go-libp2p/p2p/net/swarm" ma "github.com/multiformats/go-multiaddr" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) type SwarmAPI CoreAPI type connInfo struct { peerstore pstore.Peerstore conn inet.Conn dir inet.Direction addr ma.Multiaddr peer peer.ID } // tag used in the connection manager when explicitly connecting to a peer. const ( connectionManagerTag = "user-connect" connectionManagerWeight = 100 ) func (api *SwarmAPI) Connect(ctx context.Context, pi peer.AddrInfo) error { ctx, span := tracing.Span(ctx, "CoreAPI.SwarmAPI", "Connect", trace.WithAttributes(attribute.String("peerid", pi.ID.String()))) defer span.End() if api.peerHost == nil { return coreiface.ErrOffline } if swrm, ok := api.peerHost.Network().(*swarm.Swarm); ok { swrm.Backoff().Clear(pi.ID) } if err := api.peerHost.Connect(ctx, pi); err != nil { return err } api.peerHost.ConnManager().TagPeer(pi.ID, connectionManagerTag, connectionManagerWeight) return nil } func (api *SwarmAPI) Disconnect(ctx context.Context, addr ma.Multiaddr) error { _, span := tracing.Span(ctx, "CoreAPI.SwarmAPI", "Disconnect", trace.WithAttributes(attribute.String("addr", addr.String()))) defer span.End() if api.peerHost == nil { return coreiface.ErrOffline } taddr, id := peer.SplitAddr(addr) if id == "" { return peer.ErrInvalidAddr } span.SetAttributes(attribute.String("peerid", id.String())) net := api.peerHost.Network() if taddr == nil { if net.Connectedness(id) != inet.Connected { return coreiface.ErrNotConnected } if err := net.ClosePeer(id); err != nil { return err } return nil } for _, conn := range net.ConnsToPeer(id) { if !conn.RemoteMultiaddr().Equal(taddr) { continue } return conn.Close() } return coreiface.ErrConnNotFound } func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]ma.Multiaddr, error) { _, span := tracing.Span(ctx, "CoreAPI.SwarmAPI", "KnownAddrs") defer span.End() if api.peerHost == nil { return nil, coreiface.ErrOffline } addrs := make(map[peer.ID][]ma.Multiaddr) ps := api.peerHost.Network().Peerstore() for _, p := range ps.Peers() { addrs[p] = append(addrs[p], ps.Addrs(p)...) sort.Slice(addrs[p], func(i, j int) bool { return addrs[p][i].String() < addrs[p][j].String() }) } return addrs, nil } func (api *SwarmAPI) LocalAddrs(ctx context.Context) ([]ma.Multiaddr, error) { _, span := tracing.Span(ctx, "CoreAPI.SwarmAPI", "LocalAddrs") defer span.End() if api.peerHost == nil { return nil, coreiface.ErrOffline } return api.peerHost.Addrs(), nil } func (api *SwarmAPI) ListenAddrs(ctx context.Context) ([]ma.Multiaddr, error) { _, span := tracing.Span(ctx, "CoreAPI.SwarmAPI", "ListenAddrs") defer span.End() if api.peerHost == nil { return nil, coreiface.ErrOffline } return api.peerHost.Network().InterfaceListenAddresses() } func (api *SwarmAPI) Peers(ctx context.Context) ([]coreiface.ConnectionInfo, error) { _, span := tracing.Span(ctx, "CoreAPI.SwarmAPI", "Peers") defer span.End() if api.peerHost == nil { return nil, coreiface.ErrOffline } conns := api.peerHost.Network().Conns() out := make([]coreiface.ConnectionInfo, 0, len(conns)) for _, c := range conns { ci := &connInfo{ peerstore: api.peerstore, conn: c, dir: c.Stat().Direction, addr: c.RemoteMultiaddr(), peer: c.RemotePeer(), } /* // FIXME(steb): swcon, ok := c.(*swarm.Conn) if ok { ci.muxer = fmt.Sprintf("%T", swcon.StreamConn().Conn()) } */ out = append(out, ci) } return out, nil } func (ci *connInfo) ID() peer.ID { return ci.peer } func (ci *connInfo) Address() ma.Multiaddr { return ci.addr } func (ci *connInfo) Direction() inet.Direction { return ci.dir } func (ci *connInfo) Latency() (time.Duration, error) { return ci.peerstore.LatencyEWMA(peer.ID(ci.ID())), nil } func (ci *connInfo) Streams() ([]protocol.ID, error) { streams := ci.conn.GetStreams() out := make([]protocol.ID, len(streams)) for i, s := range streams { out[i] = s.Protocol() } return out, nil } ================================================ FILE: core/coreapi/test/api_test.go ================================================ package test import ( "context" "encoding/base64" "fmt" "os" "path/filepath" "testing" "github.com/ipfs/boxo/bootstrap" "github.com/ipfs/boxo/filestore" keystore "github.com/ipfs/boxo/keystore" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/coreapi" mock "github.com/ipfs/kubo/core/mock" "github.com/ipfs/kubo/core/node/libp2p" "github.com/ipfs/kubo/repo" "github.com/ipfs/go-datastore" syncds "github.com/ipfs/go-datastore/sync" "github.com/ipfs/kubo/config" coreiface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/core/coreiface/tests" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" ) const testPeerID = "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe" type NodeProvider struct{} func (NodeProvider) MakeAPISwarm(t *testing.T, ctx context.Context, fullIdentity bool, online bool, n int) ([]coreiface.CoreAPI, error) { mn := mocknet.New() nodes := make([]*core.IpfsNode, n) apis := make([]coreiface.CoreAPI, n) for i := range n { var ident config.Identity if fullIdentity { sk, pk, err := crypto.GenerateKeyPair(crypto.RSA, 2048) if err != nil { return nil, err } id, err := peer.IDFromPublicKey(pk) if err != nil { return nil, err } kbytes, err := crypto.MarshalPrivateKey(sk) if err != nil { return nil, err } ident = config.Identity{ PeerID: id.String(), PrivKey: base64.StdEncoding.EncodeToString(kbytes), } } else { ident = config.Identity{ PeerID: testPeerID, } } c := config.Config{} c.Addresses.Swarm = []string{fmt.Sprintf("/ip4/18.0.%d.1/tcp/4001", i)} c.Identity = ident c.Experimental.FilestoreEnabled = true c.AutoTLS.Enabled = config.False // disable so no /ws listener is added // For provider tests, avoid that content gets // auto-provided without calling "provide" (unless pinned). c.Provide.Strategy = config.NewOptionalString("roots") ds := syncds.MutexWrap(datastore.NewMapDatastore()) r := &repo.Mock{ C: c, D: ds, K: keystore.NewMemKeystore(), F: filestore.NewFileManager(ds, filepath.Dir(os.TempDir())), } node, err := core.NewNode(ctx, &core.BuildCfg{ Routing: libp2p.DHTServerOption, Repo: r, Host: mock.MockHostOption(mn), Online: online, ExtraOpts: map[string]bool{ "pubsub": true, }, }) if err != nil { return nil, err } nodes[i] = node apis[i], err = coreapi.NewCoreAPI(node) if err != nil { return nil, err } } err := mn.LinkAll() if err != nil { return nil, err } if online { bsinf := bootstrap.BootstrapConfigWithPeers( []peer.AddrInfo{ nodes[0].Peerstore.PeerInfo(nodes[0].Identity), }, ) for _, n := range nodes[1:] { if err := n.Bootstrap(bsinf); err != nil { return nil, err } } } return apis, nil } func TestIface(t *testing.T) { tests.TestApi(NodeProvider{})(t) } ================================================ FILE: core/coreapi/test/path_test.go ================================================ package test import ( "context" "strconv" "testing" "time" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/ipld/merkledag" uio "github.com/ipfs/boxo/ipld/unixfs/io" "github.com/ipfs/boxo/path" "github.com/ipfs/kubo/core/coreiface/options" "github.com/ipld/go-ipld-prime" ) func TestPathUnixFSHAMTPartial(t *testing.T) { ctx := t.Context() // Create a node apis, err := NodeProvider{}.MakeAPISwarm(t, ctx, true, true, 1) if err != nil { t.Fatal(err) } a := apis[0] // Setting this after instantiating the swarm so that it's not clobbered by loading the go-ipfs config prevVal := uio.HAMTShardingSize uio.HAMTShardingSize = 1 defer func() { uio.HAMTShardingSize = prevVal }() // Create and add a sharded directory dir := make(map[string]files.Node) // Make sure we have at least two levels of sharding for i := 0; i < uio.DefaultShardWidth+1; i++ { dir[strconv.Itoa(i)] = files.NewBytesFile([]byte(strconv.Itoa(i))) } r, err := a.Unixfs().Add(ctx, files.NewMapDirectory(dir), options.Unixfs.Pin(false, "")) if err != nil { t.Fatal(err) } // Get the root of the directory nd, err := a.Dag().Get(ctx, r.RootCid()) if err != nil { t.Fatal(err) } // Make sure the root is a DagPB node (this API might change in the future to account for ADLs) _ = nd.(ipld.Node) pbNode := nd.(*merkledag.ProtoNode) // Remove one of the sharded directory blocks if err := a.Block().Rm(ctx, path.FromCid(pbNode.Links()[0].Cid)); err != nil { t.Fatal(err) } // Try and resolve each of the entries in the sharded directory which will result in pathing over the missing block // // Note: we could just check a particular path here, but it would require either greater use of the HAMT internals // or some hard coded values in the test both of which would be a pain to follow. for k := range dir { // The node will go out to the (non-existent) network looking for the missing block. Make sure we're erroring // because we exceeded the timeout on our query timeoutCtx, timeoutCancel := context.WithTimeout(ctx, time.Second*1) newPath, err := path.Join(r, k) if err != nil { t.Fatal(err) } _, err = a.ResolveNode(timeoutCtx, newPath) if err != nil { if timeoutCtx.Err() == nil { t.Fatal(err) } } timeoutCancel() } } ================================================ FILE: core/coreapi/unixfs.go ================================================ package coreapi import ( "context" "errors" "fmt" blockservice "github.com/ipfs/boxo/blockservice" bstore "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/files" filestore "github.com/ipfs/boxo/filestore" merkledag "github.com/ipfs/boxo/ipld/merkledag" dagtest "github.com/ipfs/boxo/ipld/merkledag/test" ft "github.com/ipfs/boxo/ipld/unixfs" unixfile "github.com/ipfs/boxo/ipld/unixfs/file" uio "github.com/ipfs/boxo/ipld/unixfs/io" "github.com/ipfs/boxo/mfs" "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/provider" cid "github.com/ipfs/go-cid" cidutil "github.com/ipfs/go-cidutil" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" "github.com/ipfs/kubo/config" coreiface "github.com/ipfs/kubo/core/coreiface" options "github.com/ipfs/kubo/core/coreiface/options" "github.com/ipfs/kubo/core/coreunix" "github.com/ipfs/kubo/tracing" mh "github.com/multiformats/go-multihash" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) var log = logging.Logger("coreapi") type UnixfsAPI CoreAPI // Add builds a merkledag node from a reader, adds it to the blockstore, // and returns the key representing that node. func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options.UnixfsAddOption) (path.ImmutablePath, error) { ctx, span := tracing.Span(ctx, "CoreAPI.UnixfsAPI", "Add") defer span.End() settings, prefix, err := options.UnixfsAddOptions(opts...) if err != nil { return path.ImmutablePath{}, err } span.SetAttributes( attribute.String("chunker", settings.Chunker), attribute.Int("cidversion", settings.CidVersion), attribute.Bool("inline", settings.Inline), attribute.Int("inlinelimit", settings.InlineLimit), attribute.Bool("rawleaves", settings.RawLeaves), attribute.Bool("rawleavesset", settings.RawLeavesSet), attribute.Int("maxfilelinks", settings.MaxFileLinks), attribute.Bool("maxfilelinksset", settings.MaxFileLinksSet), attribute.Int("maxdirectorylinks", settings.MaxDirectoryLinks), attribute.Bool("maxdirectorylinksset", settings.MaxDirectoryLinksSet), attribute.Int("maxhamtfanout", settings.MaxHAMTFanout), attribute.Bool("maxhamtfanoutset", settings.MaxHAMTFanoutSet), attribute.Int("layout", int(settings.Layout)), attribute.Bool("pin", settings.Pin), attribute.String("pin-name", settings.PinName), attribute.Bool("onlyhash", settings.OnlyHash), attribute.Bool("fscache", settings.FsCache), attribute.Bool("nocopy", settings.NoCopy), attribute.Bool("silent", settings.Silent), attribute.Bool("progress", settings.Progress), ) cfg, err := api.repo.Config() if err != nil { return path.ImmutablePath{}, err } // check if repo will exceed storage limit if added // TODO: this doesn't handle the case if the hashed file is already in blocks (deduplicated) // TODO: conditional GC is disabled due to it is somehow not possible to pass the size to the daemon //if err := corerepo.ConditionalGC(req.Context(), n, uint64(size)); err != nil { // res.SetError(err, cmds.ErrNormal) // return //} if settings.NoCopy && !(cfg.Experimental.FilestoreEnabled || cfg.Experimental.UrlstoreEnabled) { return path.ImmutablePath{}, errors.New("either the filestore or the urlstore must be enabled to use nocopy, see: https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-filestore") } addblockstore := api.blockstore if !(settings.FsCache || settings.NoCopy) { addblockstore = bstore.NewGCBlockstore(api.baseBlocks, api.blockstore) } exch := api.exchange pinning := api.pinning if settings.OnlyHash { // setup a /dev/null pipeline to simulate adding the data dstore := dssync.MutexWrap(ds.NewNullDatastore()) bs := bstore.NewBlockstore(dstore, bstore.WriteThrough(true)) // we use NewNullDatastore, so ok to always WriteThrough when OnlyHash addblockstore = bstore.NewGCBlockstore(bs, nil) // gclocker will never be used exch = nil // exchange will never be used pinning = nil // pinner will never be used } bserv := blockservice.New(addblockstore, exch, blockservice.WriteThrough(cfg.Datastore.WriteThrough.WithDefault(config.DefaultWriteThrough)), ) // hash security 001 var dserv ipld.DAGService = merkledag.NewDAGService(bserv) // wrap the DAGService in a providingDAG service which provides every block written. // note about strategies: // - "all" gets handled directly at the blockstore so no need to provide // - "roots" gets handled in the pinner // - "mfs" gets handled in mfs // We need to provide the "pinned" cases only. Added blocks are not // going to be provided by the blockstore (wrong strategy for that), // nor by the pinner (the pinner doesn't traverse the pinned DAG itself, it only // handles roots). This wrapping ensures all blocks of pinned content get provided. if settings.Pin && !settings.OnlyHash && (api.providingStrategy&config.ProvideStrategyPinned) != 0 { dserv = &providingDagService{dserv, api.provider} } // add a sync call to the DagService // this ensures that data written to the DagService is persisted to the underlying datastore // TODO: propagate the Sync function from the datastore through the blockstore, blockservice and dagservice var syncDserv *syncDagService if settings.OnlyHash { syncDserv = &syncDagService{ DAGService: dserv, syncFn: func() error { return nil }, } } else { syncDserv = &syncDagService{ DAGService: dserv, syncFn: func() error { rds := api.repo.Datastore() if err := rds.Sync(ctx, bstore.BlockPrefix); err != nil { return err } return rds.Sync(ctx, filestore.FilestorePrefix) }, } } // Note: the dag service gets wrapped multiple times: // 1. providingDagService (if pinned strategy) - provides blocks as they're added // 2. syncDagService - ensures data persistence // 3. batchingDagService (in coreunix.Adder) - batches operations for efficiency fileAdder, err := coreunix.NewAdder(ctx, pinning, addblockstore, syncDserv) if err != nil { return path.ImmutablePath{}, err } fileAdder.Chunker = settings.Chunker if settings.Events != nil { fileAdder.Out = settings.Events fileAdder.Progress = settings.Progress } fileAdder.Pin = settings.Pin && !settings.OnlyHash if settings.Pin { fileAdder.PinName = settings.PinName } fileAdder.Silent = settings.Silent fileAdder.RawLeaves = settings.RawLeaves if settings.MaxFileLinksSet { fileAdder.MaxLinks = settings.MaxFileLinks } if settings.MaxDirectoryLinksSet { fileAdder.MaxDirectoryLinks = settings.MaxDirectoryLinks } if settings.MaxHAMTFanoutSet { fileAdder.MaxHAMTFanout = settings.MaxHAMTFanout } if settings.SizeEstimationModeSet { fileAdder.SizeEstimationMode = settings.SizeEstimationMode } fileAdder.NoCopy = settings.NoCopy fileAdder.CidBuilder = prefix fileAdder.PreserveMode = settings.PreserveMode fileAdder.PreserveMtime = settings.PreserveMtime fileAdder.FileMode = settings.Mode fileAdder.FileMtime = settings.Mtime if settings.IncludeEmptyDirsSet { fileAdder.IncludeEmptyDirs = settings.IncludeEmptyDirs } switch settings.Layout { case options.BalancedLayout: // Default case options.TrickleLayout: fileAdder.Trickle = true default: return path.ImmutablePath{}, fmt.Errorf("unknown layout: %d", settings.Layout) } if settings.Inline { fileAdder.CidBuilder = cidutil.InlineBuilder{ Builder: fileAdder.CidBuilder, Limit: settings.InlineLimit, } } if settings.OnlyHash { md := dagtest.Mock() emptyDirNode := ft.EmptyDirNode() // Use the same prefix for the "empty" MFS root as for the file adder. err := emptyDirNode.SetCidBuilder(fileAdder.CidBuilder) if err != nil { return path.ImmutablePath{}, err } // MFS root for OnlyHash mode: provider is nil since we're not storing/providing anything mr, err := mfs.NewRoot(ctx, md, emptyDirNode, nil, nil) if err != nil { return path.ImmutablePath{}, err } fileAdder.SetMfsRoot(mr) } nd, err := fileAdder.AddAllAndPin(ctx, files) if err != nil { return path.ImmutablePath{}, err } return path.FromCid(nd.Cid()), nil } func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error) { ctx, span := tracing.Span(ctx, "CoreAPI.UnixfsAPI", "Get", trace.WithAttributes(attribute.String("path", p.String()))) defer span.End() ses := api.core().getSession(ctx) nd, err := ses.ResolveNode(ctx, p) if err != nil { return nil, err } return unixfile.NewUnixfsFile(ctx, ses.dag, nd) } // Ls returns the contents of an IPFS or IPNS object(s) at path p, with the format: // ` ` func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, out chan<- coreiface.DirEntry, opts ...options.UnixfsLsOption) error { ctx, span := tracing.Span(ctx, "CoreAPI.UnixfsAPI", "Ls", trace.WithAttributes(attribute.String("path", p.String()))) defer span.End() defer close(out) settings, err := options.UnixfsLsOptions(opts...) if err != nil { return err } span.SetAttributes(attribute.Bool("resolvechildren", settings.ResolveChildren)) ses := api.core().getSession(ctx) uses := (*UnixfsAPI)(ses) dagnode, err := ses.ResolveNode(ctx, p) if err != nil { return err } dir, err := uio.NewDirectoryFromNode(ses.dag, dagnode) if err != nil { if errors.Is(err, uio.ErrNotADir) { return uses.lsFromLinks(ctx, dagnode.Links(), settings, out) } return err } return uses.lsFromDirLinks(ctx, dir, settings, out) } func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, settings *options.UnixfsLsSettings) (coreiface.DirEntry, error) { ctx, span := tracing.Span(ctx, "CoreAPI.UnixfsAPI", "ProcessLink") defer span.End() if linkres.Link != nil { span.SetAttributes(attribute.String("linkname", linkres.Link.Name), attribute.String("cid", linkres.Link.Cid.String())) } if linkres.Err != nil { return coreiface.DirEntry{}, linkres.Err } lnk := coreiface.DirEntry{ Name: linkres.Link.Name, Cid: linkres.Link.Cid, } switch lnk.Cid.Type() { case cid.Raw: // No need to check with raw leaves lnk.Type = coreiface.TFile lnk.Size = linkres.Link.Size case cid.DagProtobuf: if settings.ResolveChildren { linkNode, err := linkres.Link.GetNode(ctx, api.dag) if err != nil { return coreiface.DirEntry{}, err } if pn, ok := linkNode.(*merkledag.ProtoNode); ok { d, err := ft.FSNodeFromBytes(pn.Data()) if err != nil { return coreiface.DirEntry{}, err } switch d.Type() { case ft.TFile, ft.TRaw: lnk.Type = coreiface.TFile case ft.THAMTShard, ft.TDirectory, ft.TMetadata: lnk.Type = coreiface.TDirectory case ft.TSymlink: lnk.Type = coreiface.TSymlink lnk.Target = string(d.Data()) } if !settings.UseCumulativeSize { lnk.Size = d.FileSize() } lnk.Mode = d.Mode() lnk.ModTime = d.ModTime() } } if settings.UseCumulativeSize { lnk.Size = linkres.Link.Size } } return lnk, nil } func (api *UnixfsAPI) lsFromDirLinks(ctx context.Context, dir uio.Directory, settings *options.UnixfsLsSettings, out chan<- coreiface.DirEntry) error { for l := range dir.EnumLinksAsync(ctx) { dirEnt, err := api.processLink(ctx, l, settings) // TODO: perf: processing can be done in background and in parallel if err != nil { return err } select { case out <- dirEnt: case <-ctx.Done(): return nil } } return nil } func (api *UnixfsAPI) lsFromLinks(ctx context.Context, ndlinks []*ipld.Link, settings *options.UnixfsLsSettings, out chan<- coreiface.DirEntry) error { // Create links channel large enough to not block when writing to out is slower. links := make(chan coreiface.DirEntry, len(ndlinks)) errs := make(chan error, 1) go func() { defer close(links) defer close(errs) for _, l := range ndlinks { lr := ft.LinkResult{Link: &ipld.Link{Name: l.Name, Size: l.Size, Cid: l.Cid}} lnk, err := api.processLink(ctx, lr, settings) // TODO: can be parallel if settings.Async if err != nil { errs <- err return } select { case links <- lnk: case <-ctx.Done(): return } } }() for lnk := range links { out <- lnk } return <-errs } func (api *UnixfsAPI) core() *CoreAPI { return (*CoreAPI)(api) } // syncDagService is used by the Adder to ensure blocks get persisted to the underlying datastore type syncDagService struct { ipld.DAGService syncFn func() error } func (s *syncDagService) Sync() error { return s.syncFn() } type providingDagService struct { ipld.DAGService provider.MultihashProvider } func (pds *providingDagService) Add(ctx context.Context, n ipld.Node) error { if err := pds.DAGService.Add(ctx, n); err != nil { return err } // Provider errors are logged but not propagated. // We don't want DAG operations to fail due to providing issues. // The user's data is still stored successfully even if the // announcement to the routing system fails temporarily. if err := pds.StartProviding(false, n.Cid().Hash()); err != nil { log.Errorf("failed to provide new block: %s", err) } return nil } func (pds *providingDagService) AddMany(ctx context.Context, nds []ipld.Node) error { if err := pds.DAGService.AddMany(ctx, nds); err != nil { return err } keys := make([]mh.Multihash, len(nds)) for i, n := range nds { keys[i] = n.Cid().Hash() } // Same error handling philosophy as Add(): log but don't fail. if err := pds.StartProviding(false, keys...); err != nil { log.Errorf("failed to provide new blocks: %s", err) } return nil } var _ ipld.DAGService = (*providingDagService)(nil) ================================================ FILE: core/corehttp/commands.go ================================================ package corehttp import ( "errors" "fmt" "net" "net/http" "os" "strconv" "strings" cmds "github.com/ipfs/go-ipfs-cmds" cmdsHttp "github.com/ipfs/go-ipfs-cmds/http" version "github.com/ipfs/kubo" oldcmds "github.com/ipfs/kubo/commands" config "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core" corecommands "github.com/ipfs/kubo/core/commands" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) var errAPIVersionMismatch = errors.New("api version mismatch") const ( originEnvKey = "API_ORIGIN" originEnvKeyDeprecate = `You are using the ` + originEnvKey + `ENV Variable. This functionality is deprecated, and will be removed in future versions. Instead, try either adding headers to the config, or passing them via cli arguments: ipfs config API.HTTPHeaders --json '{"Access-Control-Allow-Origin": ["*"]}' ipfs daemon ` ) // APIPath is the path at which the API is mounted. const APIPath = "/api/v0" var defaultLocalhostOrigins = []string{ "http://127.0.0.1:", "https://127.0.0.1:", "http://[::1]:", "https://[::1]:", "http://localhost:", "https://localhost:", } var companionBrowserExtensionOrigins = []string{ "chrome-extension://nibjojkomfdiaoajekhjakgkdhaomnch", // ipfs-companion "chrome-extension://hjoieblefckbooibpepigmacodalfndh", // ipfs-companion-beta } func addCORSFromEnv(c *cmdsHttp.ServerConfig) { origin := os.Getenv(originEnvKey) if origin != "" { log.Warn(originEnvKeyDeprecate) c.AppendAllowedOrigins(origin) } } func addHeadersFromConfig(c *cmdsHttp.ServerConfig, nc *config.Config) { log.Info("Using API.HTTPHeaders:", nc.API.HTTPHeaders) if acao := nc.API.HTTPHeaders[cmdsHttp.ACAOrigin]; acao != nil { c.SetAllowedOrigins(acao...) } if acam := nc.API.HTTPHeaders[cmdsHttp.ACAMethods]; acam != nil { c.SetAllowedMethods(acam...) } for _, v := range nc.API.HTTPHeaders[cmdsHttp.ACACredentials] { c.SetAllowCredentials(strings.ToLower(v) == "true") } c.Headers = make(map[string][]string, len(nc.API.HTTPHeaders)+1) // Copy these because the config is shared and this function is called // in multiple places concurrently. Updating these in-place *is* racy. for h, v := range nc.API.HTTPHeaders { h = http.CanonicalHeaderKey(h) switch h { case cmdsHttp.ACAOrigin, cmdsHttp.ACAMethods, cmdsHttp.ACACredentials: // these are handled by the CORs library. default: c.Headers[h] = v } } c.Headers["Server"] = []string{"kubo/" + version.CurrentVersionNumber} } func addCORSDefaults(c *cmdsHttp.ServerConfig) { // always safelist certain origins c.AppendAllowedOrigins(defaultLocalhostOrigins...) c.AppendAllowedOrigins(companionBrowserExtensionOrigins...) // by default, use GET, PUT, POST if len(c.AllowedMethods()) == 0 { c.SetAllowedMethods(http.MethodGet, http.MethodPost, http.MethodPut) } } func patchCORSVars(c *cmdsHttp.ServerConfig, addr net.Addr) { // we have to grab the port from an addr, which may be an ip6 addr. // TODO: this should take multiaddrs and derive port from there. port := "" if tcpaddr, ok := addr.(*net.TCPAddr); ok { port = strconv.Itoa(tcpaddr.Port) } else if udpaddr, ok := addr.(*net.UDPAddr); ok { port = strconv.Itoa(udpaddr.Port) } // we're listening on tcp/udp with ports. ("udp!?" you say? yeah... it happens...) oldOrigins := c.AllowedOrigins() newOrigins := make([]string, len(oldOrigins)) for i, o := range oldOrigins { // TODO: allow replacing . tricky, ip4 and ip6 and hostnames... if port != "" { o = strings.Replace(o, "", port, -1) } newOrigins[i] = o } c.SetAllowedOrigins(newOrigins...) } func commandsOption(cctx oldcmds.Context, command *cmds.Command) ServeOption { return func(n *core.IpfsNode, l net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { cfg := cmdsHttp.NewServerConfig() cfg.AddAllowedHeaders("Origin", "Accept", "Content-Type", "X-Requested-With") cfg.SetAllowedMethods(http.MethodPost) cfg.APIPath = APIPath rcfg, err := n.Repo.Config() if err != nil { return nil, err } addHeadersFromConfig(cfg, rcfg) addCORSFromEnv(cfg) addCORSDefaults(cfg) patchCORSVars(cfg, l.Addr()) cmdHandler := cmdsHttp.NewHandler(&cctx, command, cfg) if len(rcfg.API.Authorizations) > 0 { authorizations := convertAuthorizationsMap(rcfg.API.Authorizations) cmdHandler = withAuthSecrets(authorizations, cmdHandler) } cmdHandler = otelhttp.NewHandler(cmdHandler, "corehttp.cmdsHandler", otelhttp.WithMetricAttributesFn(staticServerDomainAttrFn("api")), ) mux.Handle(APIPath+"/", cmdHandler) return mux, nil } } type rpcAuthScopeWithUser struct { config.RPCAuthScope User string } func convertAuthorizationsMap(authScopes map[string]*config.RPCAuthScope) map[string]rpcAuthScopeWithUser { // authorizations is a map where we can just check for the header value to match. authorizations := map[string]rpcAuthScopeWithUser{} for user, authScope := range authScopes { expectedHeader := config.ConvertAuthSecret(authScope.AuthSecret) if expectedHeader != "" { authorizations[expectedHeader] = rpcAuthScopeWithUser{ RPCAuthScope: *authScopes[user], User: user, } } } return authorizations } func withAuthSecrets(authorizations map[string]rpcAuthScopeWithUser, next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { authorizationHeader := r.Header.Get("Authorization") auth, ok := authorizations[authorizationHeader] if ok { // version check is implicitly allowed if r.URL.Path == "/api/v0/version" { next.ServeHTTP(w, r) return } // everything else has to be safelisted via AllowedPaths for _, prefix := range auth.AllowedPaths { if strings.HasPrefix(r.URL.Path, prefix) { next.ServeHTTP(w, r) return } } } http.Error(w, "Kubo RPC Access Denied: Please provide a valid authorization token as defined in the API.Authorizations configuration.", http.StatusForbidden) }) } // CommandsOption constructs a ServerOption for hooking the commands into the // HTTP server. It will NOT allow GET requests. func CommandsOption(cctx oldcmds.Context) ServeOption { return commandsOption(cctx, corecommands.Root) } // CheckVersionOption returns a ServeOption that checks whether the client ipfs version matches. Does nothing when the user agent string does not contain `/kubo/` or `/go-ipfs/` func CheckVersionOption() ServeOption { daemonVersion := version.ApiVersion return func(n *core.IpfsNode, l net.Listener, parent *http.ServeMux) (*http.ServeMux, error) { mux := http.NewServeMux() parent.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.URL.Path, APIPath) { cmdqry := r.URL.Path[len(APIPath):] pth := strings.Split(cmdqry, "/") // backwards compatibility to previous version check if len(pth) >= 2 && pth[1] != "version" { clientVersion := r.UserAgent() // skips check if client is not kubo (go-ipfs) if (strings.Contains(clientVersion, "/go-ipfs/") || strings.Contains(clientVersion, "/kubo/")) && daemonVersion != clientVersion { http.Error(w, fmt.Sprintf("%s (%s != %s)", errAPIVersionMismatch, daemonVersion, clientVersion), http.StatusBadRequest) return } } } mux.ServeHTTP(w, r) }) return mux, nil } } ================================================ FILE: core/corehttp/corehttp.go ================================================ /* Package corehttp provides utilities for the webui, gateways, and other high-level HTTP interfaces to IPFS. */ package corehttp import ( "context" "fmt" "net" "net/http" "time" logging "github.com/ipfs/go-log/v2" core "github.com/ipfs/kubo/core" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) var log = logging.Logger("core/server") // shutdownTimeout is the timeout after which we'll stop waiting for hung // commands to return on shutdown. const shutdownTimeout = 30 * time.Second // ServeOption registers any HTTP handlers it provides on the given mux. // It returns the mux to expose to future options, which may be a new mux if it // is interested in mediating requests to future options, or the same mux // initially passed in if not. type ServeOption func(*core.IpfsNode, net.Listener, *http.ServeMux) (*http.ServeMux, error) // MakeHandler turns a list of ServeOptions into a http.Handler that implements // all of the given options, in order. func MakeHandler(n *core.IpfsNode, l net.Listener, options ...ServeOption) (http.Handler, error) { topMux := http.NewServeMux() mux := topMux for _, option := range options { var err error mux, err = option(n, l, mux) if err != nil { return nil, err } } handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // ServeMux does not support requests with CONNECT method, // so we need to handle them separately // https://golang.org/src/net/http/request.go#L111 if r.Method == http.MethodConnect { w.WriteHeader(http.StatusOK) return } topMux.ServeHTTP(w, r) }) return handler, nil } // ListenAndServe runs an HTTP server listening at |listeningMultiAddr| with // the given serve options. The address must be provided in multiaddr format. // // TODO intelligently parse address strings in other formats so long as they // unambiguously map to a valid multiaddr. e.g. for convenience, ":8080" should // map to "/ip4/0.0.0.0/tcp/8080". func ListenAndServe(n *core.IpfsNode, listeningMultiAddr string, options ...ServeOption) error { addr, err := ma.NewMultiaddr(listeningMultiAddr) if err != nil { return err } list, err := manet.Listen(addr) if err != nil { return err } // we might have listened to /tcp/0 - let's see what we are listing on addr = list.Multiaddr() fmt.Printf("RPC API server listening on %s\n", addr) return Serve(n, manet.NetListener(list), options...) } // Serve accepts incoming HTTP connections on the listener and passes them // to ServeOption handlers. func Serve(node *core.IpfsNode, lis net.Listener, options ...ServeOption) error { return ServeWithReady(node, lis, nil, options...) } // ServeWithReady is like Serve but signals on the ready channel when the // server is about to accept connections. The channel is closed right before // server.Serve() is called. // // This is useful for callers that need to perform actions (like writing // address files) only after the server is guaranteed to be accepting // connections, avoiding race conditions where clients see the file before // the server is ready. // // Passing nil for ready is equivalent to calling Serve(). func ServeWithReady(node *core.IpfsNode, lis net.Listener, ready chan<- struct{}, options ...ServeOption) error { // make sure we close this no matter what. defer lis.Close() handler, err := MakeHandler(node, lis, options...) if err != nil { return err } addr, err := manet.FromNetAddr(lis.Addr()) if err != nil { return err } select { case <-node.Context().Done(): return fmt.Errorf("failed to start server, process closing") default: } server := &http.Server{ Handler: handler, } var serverError error serverClosed := make(chan struct{}) go func() { if ready != nil { close(ready) } serverError = server.Serve(lis) close(serverClosed) }() // wait for server to exit. select { case <-serverClosed: // if node being closed before server exits, close server case <-node.Context().Done(): log.Infof("server at %s terminating...", addr) go func() { ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() for { select { case <-ticker.C: log.Infof("waiting for server at %s to terminate...", addr) case <-serverClosed: return } } }() // This timeout shouldn't be necessary if all of our commands // are obeying their contexts but we should have *some* timeout. ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) defer cancel() err := server.Shutdown(ctx) // Should have already closed but we still need to wait for it // to set the error. <-serverClosed serverError = err } log.Infof("server at %s terminated", addr) return serverError } ================================================ FILE: core/corehttp/gateway.go ================================================ package corehttp import ( "context" "errors" "fmt" "io" "maps" "net" "net/http" "slices" "strings" "time" "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/gateway" "github.com/ipfs/boxo/namesys" "github.com/ipfs/boxo/path" offlineroute "github.com/ipfs/boxo/routing/offline" "github.com/ipfs/go-cid" version "github.com/ipfs/kubo" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core" iface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/core/node" "github.com/libp2p/go-libp2p/core/routing" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel/attribute" ) func GatewayOption(paths ...string) ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { config, headers, err := getGatewayConfig(n) if err != nil { return nil, err } backend, err := newGatewayBackend(n) if err != nil { return nil, err } handler := gateway.NewHandler(config, backend) handler = gateway.NewHeaders(headers).ApplyCors().Wrap(handler) var otelOpts []otelhttp.Option if fn := newServerDomainAttrFn(n); fn != nil { otelOpts = append(otelOpts, otelhttp.WithMetricAttributesFn(fn)) } handler = otelhttp.NewHandler(handler, "Gateway", otelOpts...) for _, p := range paths { mux.Handle(p+"/", handler) } return mux, nil } } func HostnameOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { config, headers, err := getGatewayConfig(n) if err != nil { return nil, err } backend, err := newGatewayBackend(n) if err != nil { return nil, err } childMux := http.NewServeMux() var handler http.Handler handler = gateway.NewHostnameHandler(config, backend, childMux) handler = gateway.NewHeaders(headers).ApplyCors().Wrap(handler) var otelOpts []otelhttp.Option if fn := newServerDomainAttrFn(n); fn != nil { otelOpts = append(otelOpts, otelhttp.WithMetricAttributesFn(fn)) } handler = otelhttp.NewHandler(handler, "HostnameGateway", otelOpts...) mux.Handle("/", handler) return childMux, nil } } func VersionOption() ServeOption { return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { mux.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "Commit: %s\n", version.CurrentCommit) fmt.Fprintf(w, "Client Version: %s\n", version.GetUserAgentVersion()) }) return mux, nil } } func Libp2pGatewayOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { bserv := blockservice.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore())) backend, err := gateway.NewBlocksBackend(bserv, // GatewayOverLibp2p only returns things that are in local blockstore // (same as Gateway.NoFetch=true), we have to pass offline path resolver gateway.WithResolver(n.OfflineUnixFSPathResolver), ) if err != nil { return nil, err } // Get gateway configuration from the node's config cfg, err := n.Repo.Config() if err != nil { return nil, err } gwConfig := gateway.Config{ // Keep these constraints for security DeserializedResponses: false, // Trustless-only NoDNSLink: true, // No DNS resolution DisableHTMLErrors: true, // Plain text errors only PublicGateways: nil, Menu: nil, // Apply timeout and concurrency limits from user config RetrievalTimeout: cfg.Gateway.RetrievalTimeout.WithDefault(config.DefaultRetrievalTimeout), MaxRequestDuration: cfg.Gateway.MaxRequestDuration.WithDefault(config.DefaultMaxRequestDuration), MaxConcurrentRequests: int(cfg.Gateway.MaxConcurrentRequests.WithDefault(int64(config.DefaultMaxConcurrentRequests))), MaxRangeRequestFileSize: int64(cfg.Gateway.MaxRangeRequestFileSize.WithDefault(uint64(config.DefaultMaxRangeRequestFileSize))), DiagnosticServiceURL: "", // Not used since DisableHTMLErrors=true } handler := gateway.NewHandler(gwConfig, &offlineGatewayErrWrapper{gwimpl: backend}) handler = otelhttp.NewHandler(handler, "Libp2p-Gateway", otelhttp.WithMetricAttributesFn(staticServerDomainAttrFn("libp2p")), ) mux.Handle("/ipfs/", handler) return mux, nil } } func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) { cfg, err := n.Repo.Config() if err != nil { return nil, err } bserv := n.Blocks var vsRouting routing.ValueStore = n.Routing nsys := n.Namesys pathResolver := n.UnixFSPathResolver if cfg.Gateway.NoFetch { bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore())) cs := cfg.Ipns.ResolveCacheSize if cs == 0 { cs = node.DefaultIpnsCacheSize } if cs < 0 { return nil, fmt.Errorf("cannot specify negative resolve cache size") } nsOptions := []namesys.Option{ namesys.WithDatastore(n.Repo.Datastore()), namesys.WithDNSResolver(n.DNSResolver), namesys.WithCache(cs), namesys.WithMaxCacheTTL(cfg.Ipns.MaxCacheTTL.WithDefault(config.DefaultIpnsMaxCacheTTL)), } vsRouting = offlineroute.NewOfflineRouter(n.Repo.Datastore(), n.RecordValidator) nsys, err = namesys.NewNameSystem(vsRouting, nsOptions...) if err != nil { return nil, fmt.Errorf("error constructing namesys: %w", err) } // Gateway.NoFetch=true requires offline path resolver // to avoid fetching missing blocks during path traversal pathResolver = n.OfflineUnixFSPathResolver } backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys), gateway.WithResolver(pathResolver), ) if err != nil { return nil, err } return &offlineGatewayErrWrapper{gwimpl: backend}, nil } type offlineGatewayErrWrapper struct { gwimpl gateway.IPFSBackend } func offlineErrWrap(err error) error { if errors.Is(err, iface.ErrOffline) { return fmt.Errorf("%s : %w", err.Error(), gateway.ErrServiceUnavailable) } return err } func (o *offlineGatewayErrWrapper) Get(ctx context.Context, path path.ImmutablePath, ranges ...gateway.ByteRange) (gateway.ContentPathMetadata, *gateway.GetResponse, error) { md, n, err := o.gwimpl.Get(ctx, path, ranges...) err = offlineErrWrap(err) return md, n, err } func (o *offlineGatewayErrWrapper) GetAll(ctx context.Context, path path.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { md, n, err := o.gwimpl.GetAll(ctx, path) err = offlineErrWrap(err) return md, n, err } func (o *offlineGatewayErrWrapper) GetBlock(ctx context.Context, path path.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { md, n, err := o.gwimpl.GetBlock(ctx, path) err = offlineErrWrap(err) return md, n, err } func (o *offlineGatewayErrWrapper) Head(ctx context.Context, path path.ImmutablePath) (gateway.ContentPathMetadata, *gateway.HeadResponse, error) { md, n, err := o.gwimpl.Head(ctx, path) err = offlineErrWrap(err) return md, n, err } func (o *offlineGatewayErrWrapper) ResolvePath(ctx context.Context, path path.ImmutablePath) (gateway.ContentPathMetadata, error) { md, err := o.gwimpl.ResolvePath(ctx, path) err = offlineErrWrap(err) return md, err } func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path path.ImmutablePath, params gateway.CarParams) (gateway.ContentPathMetadata, io.ReadCloser, error) { md, data, err := o.gwimpl.GetCAR(ctx, path, params) err = offlineErrWrap(err) return md, data, err } func (o *offlineGatewayErrWrapper) IsCached(ctx context.Context, path path.Path) bool { return o.gwimpl.IsCached(ctx, path) } func (o *offlineGatewayErrWrapper) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { rec, err := o.gwimpl.GetIPNSRecord(ctx, c) err = offlineErrWrap(err) return rec, err } func (o *offlineGatewayErrWrapper) ResolveMutable(ctx context.Context, path path.Path) (path.ImmutablePath, time.Duration, time.Time, error) { imPath, ttl, lastMod, err := o.gwimpl.ResolveMutable(ctx, path) err = offlineErrWrap(err) return imPath, ttl, lastMod, err } func (o *offlineGatewayErrWrapper) GetDNSLinkRecord(ctx context.Context, s string) (path.Path, error) { p, err := o.gwimpl.GetDNSLinkRecord(ctx, s) err = offlineErrWrap(err) return p, err } var _ gateway.IPFSBackend = (*offlineGatewayErrWrapper)(nil) var defaultPaths = []string{"/ipfs/", "/ipns/", "/p2p/"} // serverDomainAttrKey is the OTel attribute key for the logical server domain. // It replaces the high-cardinality server.address attribute (dropped by the // View in cmd/ipfs/kubo/daemon.go) with a bounded set of values: configured // Gateway.PublicGateways suffixes, "localhost", "loopback", "api", "libp2p", // or "other". var serverDomainAttrKey = attribute.Key("server.domain") // staticServerDomainAttrFn returns a MetricAttributesFn that always returns // a fixed server.domain value. Use for handlers where the domain is known // statically (e.g. "api", "libp2p") to keep the label set consistent across // all http_server_* metrics. func staticServerDomainAttrFn(domain string) func(*http.Request) []attribute.KeyValue { attrs := []attribute.KeyValue{serverDomainAttrKey.String(domain)} return func(*http.Request) []attribute.KeyValue { return attrs } } // newServerDomainAttrFn returns an otelhttp.WithMetricAttributesFn callback // that adds a server.domain attribute grouping requests by their matching // Gateway.PublicGateways hostname suffix (e.g. "dweb.link", "ipfs.io"). // Requests that don't match any configured gateway get "other". // // All return values are pre-allocated at setup time so the per-request // closure is zero-allocation. func newServerDomainAttrFn(n *core.IpfsNode) func(*http.Request) []attribute.KeyValue { cfg, err := n.Repo.Config() if err != nil { return nil } // Collect non-nil gateway domain suffixes, sorted longest-first // so more-specific suffixes match before shorter ones. // Strip ports from keys to match boxo's fallback behavior // (boxo tries exact match with port, then strips port and retries). seen := make(map[string]struct{}, len(cfg.Gateway.PublicGateways)) suffixes := make([]string, 0, len(cfg.Gateway.PublicGateways)) for hostname, gw := range cfg.Gateway.PublicGateways { if gw == nil { continue } if h, _, err := net.SplitHostPort(hostname); err == nil { hostname = h } if _, ok := seen[hostname]; ok { continue } seen[hostname] = struct{}{} suffixes = append(suffixes, hostname) } slices.SortFunc(suffixes, func(a, b string) int { return len(b) - len(a) }) // Pre-allocate attribute slices so the per-request closure only returns // existing slices and does not allocate. suffixAttrs := make([][]attribute.KeyValue, len(suffixes)) for i, s := range suffixes { suffixAttrs[i] = []attribute.KeyValue{serverDomainAttrKey.String(s)} } localhostAttr := []attribute.KeyValue{serverDomainAttrKey.String("localhost")} loopbackAttr := []attribute.KeyValue{serverDomainAttrKey.String("loopback")} otherAttr := []attribute.KeyValue{serverDomainAttrKey.String("other")} return func(r *http.Request) []attribute.KeyValue { host := r.Host if h, _, err := net.SplitHostPort(host); err == nil { host = h } // Check localhost/loopback before iterating suffixes. // "localhost" is an implicit default gateway (defaultKnownGateways) // not present in cfg.Gateway.PublicGateways, so it won't appear // in suffixes. if host == "localhost" || strings.HasSuffix(host, ".localhost") { return localhostAttr } if strings.HasPrefix(host, "127.") || host == "::1" { return loopbackAttr } for i, suffix := range suffixes { if strings.HasSuffix(host, suffix) { return suffixAttrs[i] } } return otherAttr } } var subdomainGatewaySpec = &gateway.PublicGateway{ Paths: defaultPaths, UseSubdomains: true, } var defaultKnownGateways = map[string]*gateway.PublicGateway{ "localhost": subdomainGatewaySpec, } func getGatewayConfig(n *core.IpfsNode) (gateway.Config, map[string][]string, error) { cfg, err := n.Repo.Config() if err != nil { return gateway.Config{}, nil, err } // Initialize gateway configuration, with empty PublicGateways, handled after. gwCfg := gateway.Config{ DeserializedResponses: cfg.Gateway.DeserializedResponses.WithDefault(config.DefaultDeserializedResponses), AllowCodecConversion: cfg.Gateway.AllowCodecConversion.WithDefault(config.DefaultAllowCodecConversion), DisableHTMLErrors: cfg.Gateway.DisableHTMLErrors.WithDefault(config.DefaultDisableHTMLErrors), NoDNSLink: cfg.Gateway.NoDNSLink, PublicGateways: map[string]*gateway.PublicGateway{}, RetrievalTimeout: cfg.Gateway.RetrievalTimeout.WithDefault(config.DefaultRetrievalTimeout), MaxRequestDuration: cfg.Gateway.MaxRequestDuration.WithDefault(config.DefaultMaxRequestDuration), MaxConcurrentRequests: int(cfg.Gateway.MaxConcurrentRequests.WithDefault(int64(config.DefaultMaxConcurrentRequests))), MaxRangeRequestFileSize: int64(cfg.Gateway.MaxRangeRequestFileSize.WithDefault(uint64(config.DefaultMaxRangeRequestFileSize))), DiagnosticServiceURL: cfg.Gateway.DiagnosticServiceURL.WithDefault(config.DefaultDiagnosticServiceURL), } // Add default implicit known gateways, such as subdomain gateway on localhost. maps.Copy(gwCfg.PublicGateways, defaultKnownGateways) // Apply values from cfg.Gateway.PublicGateways if they exist. for hostname, gw := range cfg.Gateway.PublicGateways { if gw == nil { // Remove any implicit defaults, if present. This is useful when one // wants to disable subdomain gateway on localhost, etc. delete(gwCfg.PublicGateways, hostname) continue } gwCfg.PublicGateways[hostname] = &gateway.PublicGateway{ Paths: gw.Paths, NoDNSLink: gw.NoDNSLink, UseSubdomains: gw.UseSubdomains, InlineDNSLink: gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink), DeserializedResponses: gw.DeserializedResponses.WithDefault(gwCfg.DeserializedResponses), } } return gwCfg, cfg.Gateway.HTTPHeaders, nil } ================================================ FILE: core/corehttp/gateway_test.go ================================================ package corehttp import ( "context" "errors" "io" "net/http" "net/http/httptest" "strings" "testing" "github.com/ipfs/boxo/namesys" version "github.com/ipfs/kubo" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/coreapi" "github.com/ipfs/kubo/repo" "github.com/stretchr/testify/assert" "github.com/ipfs/boxo/path" "github.com/ipfs/go-datastore" syncds "github.com/ipfs/go-datastore/sync" "github.com/ipfs/kubo/config" iface "github.com/ipfs/kubo/core/coreiface" ci "github.com/libp2p/go-libp2p/core/crypto" ) type mockNamesys map[string]path.Path func (m mockNamesys) Resolve(ctx context.Context, p path.Path, opts ...namesys.ResolveOption) (namesys.Result, error) { cfg := namesys.DefaultResolveOptions() for _, o := range opts { o(&cfg) } depth := cfg.Depth if depth == namesys.UnlimitedDepth { // max uint depth = ^uint(0) } var ( value path.Path ) name := path.SegmentsToString(p.Segments()[:2]...) for strings.HasPrefix(name, "/ipns/") { if depth == 0 { return namesys.Result{Path: value}, namesys.ErrResolveRecursion } depth-- v, ok := m[name] if !ok { return namesys.Result{}, namesys.ErrResolveFailed } value = v name = value.String() } value, err := path.Join(value, p.Segments()[2:]...) return namesys.Result{Path: value}, err } func (m mockNamesys) ResolveAsync(ctx context.Context, p path.Path, opts ...namesys.ResolveOption) <-chan namesys.AsyncResult { out := make(chan namesys.AsyncResult, 1) res, err := m.Resolve(ctx, p, opts...) out <- namesys.AsyncResult{Path: res.Path, TTL: res.TTL, LastMod: res.LastMod, Err: err} close(out) return out } func (m mockNamesys) Publish(ctx context.Context, name ci.PrivKey, value path.Path, opts ...namesys.PublishOption) error { return errors.New("not implemented for mockNamesys") } func (m mockNamesys) GetResolver(subs string) (namesys.Resolver, bool) { return nil, false } func newNodeWithMockNamesys(ns mockNamesys) (*core.IpfsNode, error) { c := config.Config{ Identity: config.Identity{ PeerID: "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe", // required by offline node }, } r := &repo.Mock{ C: c, D: syncds.MutexWrap(datastore.NewMapDatastore()), } n, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r}) if err != nil { return nil, err } n.Namesys = ns return n, nil } type delegatedHandler struct { http.Handler } func (dh *delegatedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { dh.Handler.ServeHTTP(w, r) } func doWithoutRedirect(req *http.Request) (*http.Response, error) { tag := "without-redirect" c := &http.Client{ CheckRedirect: func(req *http.Request, via []*http.Request) error { return errors.New(tag) }, } res, err := c.Do(req) if err != nil && !strings.Contains(err.Error(), tag) { return nil, err } return res, nil } func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, iface.CoreAPI, context.Context) { n, err := newNodeWithMockNamesys(ns) if err != nil { t.Fatal(err) } // need this variable here since we need to construct handler with // listener, and server with handler. yay cycles. dh := &delegatedHandler{} ts := httptest.NewServer(dh) t.Cleanup(func() { ts.Close() }) dh.Handler, err = MakeHandler(n, ts.Listener, HostnameOption(), GatewayOption("/ipfs", "/ipns"), VersionOption(), ) if err != nil { t.Fatal(err) } api, err := coreapi.NewCoreAPI(n) if err != nil { t.Fatal(err) } return ts, api, n.Context() } func TestVersion(t *testing.T) { version.CurrentCommit = "theshortcommithash" ns := mockNamesys{} ts, _, _ := newTestServerAndNode(t, ns) t.Logf("test server url: %s", ts.URL) req, err := http.NewRequest(http.MethodGet, ts.URL+"/version", nil) if err != nil { t.Fatal(err) } res, err := doWithoutRedirect(req) if err != nil { t.Fatal(err) } body, err := io.ReadAll(res.Body) if err != nil { t.Fatalf("error reading response: %s", err) } s := string(body) if !strings.Contains(s, "Commit: theshortcommithash") { t.Fatalf("response doesn't contain commit:\n%s", s) } if !strings.Contains(s, "Client Version: "+version.GetUserAgentVersion()) { t.Fatalf("response doesn't contain client version:\n%s", s) } } func TestDeserializedResponsesInheritance(t *testing.T) { for _, testCase := range []struct { globalSetting config.Flag gatewaySetting config.Flag expectedGatewaySetting bool }{ {config.True, config.Default, true}, {config.False, config.Default, false}, {config.False, config.True, true}, {config.True, config.False, false}, } { c := config.Config{ Identity: config.Identity{ PeerID: "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe", // required by offline node }, Gateway: config.Gateway{ DeserializedResponses: testCase.globalSetting, PublicGateways: map[string]*config.GatewaySpec{ "example.com": { DeserializedResponses: testCase.gatewaySetting, }, }, }, } r := &repo.Mock{ C: c, D: syncds.MutexWrap(datastore.NewMapDatastore()), } n, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r}) assert.NoError(t, err) gwCfg, _, err := getGatewayConfig(n) assert.NoError(t, err) assert.Contains(t, gwCfg.PublicGateways, "example.com") assert.Equal(t, testCase.expectedGatewaySetting, gwCfg.PublicGateways["example.com"].DeserializedResponses) } } ================================================ FILE: core/corehttp/logs.go ================================================ package corehttp import ( "bufio" "fmt" "net" "net/http" logging "github.com/ipfs/go-log/v2" core "github.com/ipfs/kubo/core" ) func LogOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { mux.HandleFunc("/logs", func(w http.ResponseWriter, r *http.Request) { // The log data comes from an io.Reader, and we need to constantly // read from it and then write to the HTTP response. pipeReader := logging.NewPipeReader() done := make(chan struct{}) // Close the pipe reader if the request context is canceled. This // is necessary to avoiding blocking on reading from the pipe // reader when the client terminates the request. go func() { select { case <-r.Context().Done(): // Client canceled request case <-n.Context().Done(): // Node shutdown case <-done: // log reader goroutine exitex } pipeReader.Close() }() errs := make(chan error, 1) go func() { defer close(errs) defer close(done) rdr := bufio.NewReader(pipeReader) for { // Read a line of log data and send it to the client. line, err := rdr.ReadString('\n') if err != nil { errs <- fmt.Errorf("error reading log message: %s", err) return } _, err = w.Write([]byte(line)) if err != nil { // Failed to write to client, probably disconnected. return } if f, ok := w.(http.Flusher); ok { f.Flush() } if r.Context().Err() != nil { return } } }() log.Info("log API client connected") err := <-errs if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } }) return mux, nil } } ================================================ FILE: core/corehttp/metrics.go ================================================ package corehttp import ( "net" "net/http" "time" core "github.com/ipfs/kubo/core" "go.opencensus.io/stats/view" "go.opencensus.io/zpages" ocprom "contrib.go.opencensus.io/exporter/prometheus" prometheus "github.com/prometheus/client_golang/prometheus" promhttp "github.com/prometheus/client_golang/prometheus/promhttp" ) // MetricsScrapingOption adds the scraping endpoint which Prometheus uses to fetch metrics. func MetricsScrapingOption(path string) ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { mux.Handle(path, promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{})) return mux, nil } } // This adds collection of OpenCensus metrics func MetricsOpenCensusCollectionOption() ServeOption { return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { log.Info("Init OpenCensus") promRegistry := prometheus.NewRegistry() pe, err := ocprom.NewExporter(ocprom.Options{ Namespace: "ipfs_oc", Registry: promRegistry, OnError: func(err error) { log.Errorw("OC ERROR", "error", err) }, }) if err != nil { return nil, err } // register prometheus with opencensus view.RegisterExporter(pe) view.SetReportingPeriod(2 * time.Second) // Construct the mux zpages.Handle(mux, "/debug/metrics/oc/debugz") mux.Handle("/debug/metrics/oc", pe) return mux, nil } } // MetricsOpenCensusDefaultPrometheusRegistry registers the default prometheus // registry as an exporter to OpenCensus metrics. This means that OpenCensus // metrics will show up in the prometheus metrics endpoint func MetricsOpenCensusDefaultPrometheusRegistry() ServeOption { return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { log.Info("Init OpenCensus with default prometheus registry") pe, err := ocprom.NewExporter(ocprom.Options{ Registry: prometheus.DefaultRegisterer.(*prometheus.Registry), OnError: func(err error) { log.Errorw("OC default registry ERROR", "error", err) }, }) if err != nil { return nil, err } // register prometheus with opencensus view.RegisterExporter(pe) return mux, nil } } // MetricsCollectionOption adds collection of net/http-related metrics. func MetricsCollectionOption(handlerName string) ServeOption { return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { // Adapted from github.com/prometheus/client_golang/prometheus/http.go // Work around https://github.com/prometheus/client_golang/pull/311 opts := prometheus.SummaryOpts{ Namespace: "ipfs", Subsystem: "http", ConstLabels: prometheus.Labels{"handler": handlerName}, Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, } // Legacy metric - new metrics are provided by boxo/gateway as gw_http_responses_total reqCnt := prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: opts.Namespace, Subsystem: opts.Subsystem, Name: "requests_total", Help: "Total number of HTTP requests made.", ConstLabels: opts.ConstLabels, }, []string{"method", "code"}, ) if err := prometheus.Register(reqCnt); err != nil { if are, ok := err.(prometheus.AlreadyRegisteredError); ok { reqCnt = are.ExistingCollector.(*prometheus.CounterVec) } else { return nil, err } } opts.Name = "request_duration_seconds" opts.Help = "The HTTP request latencies in seconds." reqDur := prometheus.NewSummaryVec(opts, nil) if err := prometheus.Register(reqDur); err != nil { if are, ok := err.(prometheus.AlreadyRegisteredError); ok { reqDur = are.ExistingCollector.(*prometheus.SummaryVec) } else { return nil, err } } opts.Name = "request_size_bytes" opts.Help = "The HTTP request sizes in bytes." reqSz := prometheus.NewSummaryVec(opts, nil) if err := prometheus.Register(reqSz); err != nil { if are, ok := err.(prometheus.AlreadyRegisteredError); ok { reqSz = are.ExistingCollector.(*prometheus.SummaryVec) } else { return nil, err } } opts.Name = "response_size_bytes" opts.Help = "The HTTP response sizes in bytes." resSz := prometheus.NewSummaryVec(opts, nil) if err := prometheus.Register(resSz); err != nil { if are, ok := err.(prometheus.AlreadyRegisteredError); ok { resSz = are.ExistingCollector.(*prometheus.SummaryVec) } else { return nil, err } } // Construct the mux childMux := http.NewServeMux() var promMux http.Handler = childMux promMux = promhttp.InstrumentHandlerResponseSize(resSz, promMux) promMux = promhttp.InstrumentHandlerRequestSize(reqSz, promMux) promMux = promhttp.InstrumentHandlerDuration(reqDur, promMux) promMux = promhttp.InstrumentHandlerCounter(reqCnt, promMux) mux.Handle("/", promMux) return childMux, nil } } var peersTotalMetric = prometheus.NewDesc( prometheus.BuildFQName("ipfs", "p2p", "peers_total"), "Number of connected peers", []string{"transport"}, nil, ) type IpfsNodeCollector struct { Node *core.IpfsNode } func (IpfsNodeCollector) Describe(ch chan<- *prometheus.Desc) { ch <- peersTotalMetric } func (c IpfsNodeCollector) Collect(ch chan<- prometheus.Metric) { for tr, val := range c.PeersTotalValues() { ch <- prometheus.MustNewConstMetric( peersTotalMetric, prometheus.GaugeValue, val, tr, ) } } func (c IpfsNodeCollector) PeersTotalValues() map[string]float64 { vals := make(map[string]float64) if c.Node.PeerHost == nil { return vals } for _, peerID := range c.Node.PeerHost.Network().Peers() { // Each peer may have more than one connection (see for an explanation // https://github.com/libp2p/go-libp2p-swarm/commit/0538806), so we grab // only one, the first (an arbitrary and non-deterministic choice), which // according to ConnsToPeer is the oldest connection in the list // (https://github.com/libp2p/go-libp2p-swarm/blob/v0.2.6/swarm.go#L362-L364). conns := c.Node.PeerHost.Network().ConnsToPeer(peerID) if len(conns) == 0 { continue } tr := "" for _, proto := range conns[0].RemoteMultiaddr().Protocols() { tr = tr + "/" + proto.Name } vals[tr] = vals[tr] + 1 } return vals } ================================================ FILE: core/corehttp/metrics_test.go ================================================ package corehttp import ( "context" "testing" "time" "github.com/ipfs/kubo/core" inet "github.com/libp2p/go-libp2p/core/network" bhost "github.com/libp2p/go-libp2p/p2p/host/basic" swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" ) // This test is based on go-libp2p/p2p/net/swarm.TestConnectednessCorrect // It builds 4 nodes and connects them, one being the sole center. // Then it checks that the center reports the correct number of peers. func TestPeersTotal(t *testing.T) { ctx := context.Background() hosts := make([]*bhost.BasicHost, 4) for i := range 4 { var err error hosts[i], err = bhost.NewHost(swarmt.GenSwarm(t), nil) if err != nil { t.Fatal(err) } } dial := func(a, b inet.Network) { swarmt.DivulgeAddresses(b, a) if _, err := a.DialPeer(ctx, b.LocalPeer()); err != nil { t.Fatalf("Failed to dial: %s", err) } } dial(hosts[0].Network(), hosts[1].Network()) dial(hosts[0].Network(), hosts[2].Network()) dial(hosts[0].Network(), hosts[3].Network()) // there's something wrong with dial, i think. it's not finishing // completely. there must be some async stuff. <-time.After(100 * time.Millisecond) node := &core.IpfsNode{PeerHost: hosts[0]} collector := IpfsNodeCollector{Node: node} peersTransport := collector.PeersTotalValues() if len(peersTransport) > 2 { t.Fatalf("expected at most 2 peers transport (tcp and upd/quic), got %d, transport map %v", len(peersTransport), peersTransport) } totalPeers := peersTransport["/ip4/tcp"] + peersTransport["/ip4/udp/quic-v1"] if totalPeers != 3 { t.Fatalf("expected 3 peers in either tcp or upd/quic transport, got %f", totalPeers) } } ================================================ FILE: core/corehttp/mutex_profile.go ================================================ package corehttp import ( "net" "net/http" "runtime" "strconv" core "github.com/ipfs/kubo/core" ) // MutexFractionOption allows to set runtime.SetMutexProfileFraction via HTTP // using POST request with parameter 'fraction'. func MutexFractionOption(path string) ServeOption { return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "only POST allowed", http.StatusMethodNotAllowed) return } if err := r.ParseForm(); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } asfr := r.Form.Get("fraction") if len(asfr) == 0 { http.Error(w, "parameter 'fraction' must be set", http.StatusBadRequest) return } fr, err := strconv.Atoi(asfr) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } log.Infof("Setting MutexProfileFraction to %d", fr) runtime.SetMutexProfileFraction(fr) }) return mux, nil } } // BlockProfileRateOption allows to set runtime.SetBlockProfileRate via HTTP // using POST request with parameter 'rate'. // The profiler tries to sample 1 event every nanoseconds. // If rate == 1, then the profiler samples every blocking event. // To disable, set rate = 0. func BlockProfileRateOption(path string) ServeOption { return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "only POST allowed", http.StatusMethodNotAllowed) return } if err := r.ParseForm(); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } rateStr := r.Form.Get("rate") if len(rateStr) == 0 { http.Error(w, "parameter 'rate' must be set", http.StatusBadRequest) return } rate, err := strconv.Atoi(rateStr) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } log.Infof("Setting BlockProfileRate to %d", rate) runtime.SetBlockProfileRate(rate) }) return mux, nil } } ================================================ FILE: core/corehttp/option_test.go ================================================ package corehttp import ( "fmt" "io" "net/http" "net/http/httptest" "testing" version "github.com/ipfs/kubo" ) type testcasecheckversion struct { userAgent string uri string shouldHandle bool responseBody string responseCode int } func (tc testcasecheckversion) body() string { if !tc.shouldHandle && tc.responseBody == "" { return fmt.Sprintf("%s (%s != %s)\n", errAPIVersionMismatch, version.ApiVersion, tc.userAgent) } return tc.responseBody } func TestCheckVersionOption(t *testing.T) { tcs := []testcasecheckversion{ {"/go-ipfs/0.1/", APIPath + "/test/", false, "", http.StatusBadRequest}, {"/go-ipfs/0.1/", APIPath + "/version", true, "check!", http.StatusOK}, {version.ApiVersion, APIPath + "/test", true, "check!", http.StatusOK}, {"Mozilla Firefox/no go-ipfs node", APIPath + "/test", true, "check!", http.StatusOK}, {"/go-ipfs/0.1/", "/webui", true, "check!", http.StatusOK}, } for _, tc := range tcs { t.Logf("%#v", tc) r := httptest.NewRequest(http.MethodPost, tc.uri, nil) r.Header.Add("User-Agent", tc.userAgent) // old version, should fail called := false root := http.NewServeMux() mux, err := CheckVersionOption()(nil, nil, root) if err != nil { t.Fatal(err) } mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { called = true if !tc.shouldHandle { t.Error("handler was called even though version didn't match") } if _, err := io.WriteString(w, "check!"); err != nil { t.Error(err) } }) w := httptest.NewRecorder() root.ServeHTTP(w, r) if tc.shouldHandle && !called { t.Error("handler wasn't called even though it should have") } if w.Code != tc.responseCode { t.Errorf("expected code %d but got %d", tc.responseCode, w.Code) } if w.Body.String() != tc.body() { t.Errorf("expected error message %q, got %q", tc.body(), w.Body.String()) } } } ================================================ FILE: core/corehttp/p2p_proxy.go ================================================ package corehttp import ( "fmt" "net" "net/http" "net/http/httputil" "net/url" "strings" core "github.com/ipfs/kubo/core" peer "github.com/libp2p/go-libp2p/core/peer" p2phttp "github.com/libp2p/go-libp2p-http" protocol "github.com/libp2p/go-libp2p/core/protocol" ) // P2PProxyOption is an endpoint for proxying a HTTP request to another ipfs peer func P2PProxyOption() ServeOption { return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { mux.HandleFunc("/p2p/", func(w http.ResponseWriter, request *http.Request) { // parse request parsedRequest, err := parseRequest(request) if err != nil { handleError(w, "failed to parse request", err, 400) return } request.Host = "" // Let URL's Host take precedence. request.URL.Path = parsedRequest.httpPath target, err := url.Parse(fmt.Sprintf("libp2p://%s", parsedRequest.target)) if err != nil { handleError(w, "failed to parse url", err, 400) return } rt := p2phttp.NewTransport(ipfsNode.PeerHost, p2phttp.ProtocolOption(parsedRequest.name)) proxy := &httputil.ReverseProxy{ Transport: rt, Rewrite: func(r *httputil.ProxyRequest) { r.SetURL(target) r.SetXForwarded() }, } proxy.ServeHTTP(w, request) }) return mux, nil } } type proxyRequest struct { target string name protocol.ID httpPath string // path to send to the proxy-host } // from the url path parse the peer-ID, name and http path // /p2p/$peer_id/http/$http_path // or // /p2p/$peer_id/x/$protocol/http/$http_path func parseRequest(request *http.Request) (*proxyRequest, error) { path := request.URL.Path split := strings.SplitN(path, "/", 5) if len(split) < 5 { return nil, fmt.Errorf("invalid request path '%s'", path) } if _, err := peer.Decode(split[2]); err != nil { return nil, fmt.Errorf("invalid request path '%s'", path) } if split[3] == "http" { return &proxyRequest{split[2], protocol.ID("/http"), split[4]}, nil } split = strings.SplitN(path, "/", 7) if len(split) < 7 || split[3] != "x" || split[5] != "http" { return nil, fmt.Errorf("invalid request path '%s'", path) } return &proxyRequest{split[2], protocol.ID("/x/" + split[4] + "/http"), split[6]}, nil } func handleError(w http.ResponseWriter, msg string, err error, code int) { http.Error(w, fmt.Sprintf("%s: %s", msg, err), code) } ================================================ FILE: core/corehttp/p2p_proxy_test.go ================================================ package corehttp import ( "net/http" "strings" "testing" protocol "github.com/libp2p/go-libp2p/core/protocol" "github.com/stretchr/testify/require" ) type TestCase struct { urlprefix string target string name string path string } var validtestCases = []TestCase{ {"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/http", "path/to/index.txt"}, {"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/x/custom/http", "path/to/index.txt"}, {"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/x/custom/http", "http/path/to/index.txt"}, } func TestParseRequest(t *testing.T) { for _, tc := range validtestCases { url := tc.urlprefix + "/p2p/" + tc.target + tc.name + "/" + tc.path req, _ := http.NewRequest(http.MethodGet, url, strings.NewReader("")) parsed, err := parseRequest(req) require.NoError(t, err) require.Equal(t, tc.path, parsed.httpPath, "proxy request path") require.Equal(t, protocol.ID(tc.name), parsed.name, "proxy request name") require.Equal(t, tc.target, parsed.target, "proxy request peer-id") } } var invalidtestCases = []string{ "http://localhost:5001/p2p/http/foobar", "http://localhost:5001/p2p/QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT/x/custom/foobar", } func TestParseRequestInvalidPath(t *testing.T) { for _, tc := range invalidtestCases { url := tc req, _ := http.NewRequest(http.MethodGet, url, strings.NewReader("")) _, err := parseRequest(req) require.Error(t, err) } } ================================================ FILE: core/corehttp/redirect.go ================================================ package corehttp import ( "net" "net/http" core "github.com/ipfs/kubo/core" ) func RedirectOption(path string, redirect string) ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { cfg, err := n.Repo.Config() if err != nil { return nil, err } handler := &redirectHandler{redirect, cfg.API.HTTPHeaders} if len(path) > 0 { mux.Handle("/"+path+"/", handler) } else { mux.Handle("/", handler) } return mux, nil } } type redirectHandler struct { path string headers map[string][]string } func (i *redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { for k, v := range i.headers { w.Header()[http.CanonicalHeaderKey(k)] = v } http.Redirect(w, r, i.path, http.StatusFound) } ================================================ FILE: core/corehttp/routing.go ================================================ package corehttp import ( "context" "errors" "fmt" "net" "net/http" "time" "github.com/ipfs/boxo/gateway" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/routing/http/server" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" cid "github.com/ipfs/go-cid" core "github.com/ipfs/kubo/core" dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p-kad-dht/dual" "github.com/libp2p/go-libp2p-kad-dht/fullrt" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" ) func RoutingOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { _, headers, err := getGatewayConfig(n) if err != nil { return nil, err } handler := server.Handler(&contentRouter{n}) handler = gateway.NewHeaders(headers).ApplyCors().Wrap(handler) mux.Handle("/routing/v1/", handler) return mux, nil } } type contentRouter struct { n *core.IpfsNode } func (r *contentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) { ctx, cancel := context.WithCancel(ctx) ch := r.n.Routing.FindProvidersAsync(ctx, key, limit) return iter.ToResultIter[types.Record](&peerChanIter{ ch: ch, cancel: cancel, }), nil } // nolint deprecated func (r *contentRouter) ProvideBitswap(ctx context.Context, req *server.BitswapWriteProvideRequest) (time.Duration, error) { return 0, routing.ErrNotSupported } func (r *contentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[*types.PeerRecord], error) { ctx, cancel := context.WithCancel(ctx) defer cancel() addr, err := r.n.Routing.FindPeer(ctx, pid) if err != nil { return nil, err } rec := &types.PeerRecord{ Schema: types.SchemaPeer, ID: &addr.ID, } for _, addr := range addr.Addrs { rec.Addrs = append(rec.Addrs, types.Multiaddr{Multiaddr: addr}) } return iter.ToResultIter[*types.PeerRecord](iter.FromSlice[*types.PeerRecord]([]*types.PeerRecord{rec})), nil } func (r *contentRouter) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() raw, err := r.n.Routing.GetValue(ctx, string(name.RoutingKey())) if err != nil { return nil, err } return ipns.UnmarshalRecord(raw) } func (r *contentRouter) PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { ctx, cancel := context.WithCancel(ctx) defer cancel() raw, err := ipns.MarshalRecord(record) if err != nil { return err } // The caller guarantees that name matches the record. This is double checked // by the internals of PutValue. return r.n.Routing.PutValue(ctx, string(name.RoutingKey()), raw) } func (r *contentRouter) GetClosestPeers(ctx context.Context, key cid.Cid) (iter.ResultIter[*types.PeerRecord], error) { // Per the spec, if the peer ID is empty, we should use self. if key == cid.Undef { return nil, errors.New("GetClosestPeers key is undefined") } keyStr := string(key.Hash()) var peers []peer.ID var err error if r.n.DHTClient == nil { return nil, fmt.Errorf("GetClosestPeers not supported: DHT is not available") } switch dhtClient := r.n.DHTClient.(type) { case *dual.DHT: // Only use WAN DHT for public HTTP Routing API. // LAN DHT contains private network peers that should not be exposed publicly. if dhtClient.WAN == nil { return nil, fmt.Errorf("GetClosestPeers not supported: WAN DHT is not available") } peers, err = dhtClient.WAN.GetClosestPeers(ctx, keyStr) case *fullrt.FullRT: peers, err = dhtClient.GetClosestPeers(ctx, keyStr) case *dht.IpfsDHT: peers, err = dhtClient.GetClosestPeers(ctx, keyStr) default: return nil, fmt.Errorf("GetClosestPeers not supported for DHT type %T", r.n.DHTClient) } if err != nil { return nil, err } // We have some DHT-closest peers. Find addresses for them. // The addresses should be in the peerstore. records := make([]*types.PeerRecord, 0, len(peers)) for _, p := range peers { addrs := r.n.Peerstore.Addrs(p) rAddrs := make([]types.Multiaddr, len(addrs)) for i, addr := range addrs { rAddrs[i] = types.Multiaddr{Multiaddr: addr} } record := types.PeerRecord{ ID: &p, Schema: types.SchemaPeer, Addrs: rAddrs, } records = append(records, &record) } return iter.ToResultIter(iter.FromSlice(records)), nil } type peerChanIter struct { ch <-chan peer.AddrInfo cancel context.CancelFunc next *peer.AddrInfo } func (it *peerChanIter) Next() bool { addr, ok := <-it.ch if ok { it.next = &addr return true } it.next = nil return false } func (it *peerChanIter) Val() types.Record { if it.next == nil { return nil } rec := &types.PeerRecord{ Schema: types.SchemaPeer, ID: &it.next.ID, } for _, addr := range it.next.Addrs { rec.Addrs = append(rec.Addrs, types.Multiaddr{Multiaddr: addr}) } return rec } func (it *peerChanIter) Close() error { it.cancel() return nil } ================================================ FILE: core/corehttp/webui.go ================================================ package corehttp import ( "fmt" "net" "net/http" "strings" "github.com/ipfs/go-cid" "github.com/ipfs/kubo/config" core "github.com/ipfs/kubo/core" ) // WebUI version confirmed to work with this Kubo version const WebUIPath = "/ipfs/bafybeihxglpcfyarpm7apn7xpezbuoqgk3l5chyk7w4gvrjwk45rqohlmm" // v4.12.0 // WebUIPaths is a list of all past webUI paths. var WebUIPaths = []string{ WebUIPath, "/ipfs/bafybeiddnr2jz65byk67sjt6jsu6g7tueddr7odhzzpzli3rgudlbnc6iq", // v4.11.1 "/ipfs/bafybeidfgbcqy435sdbhhejifdxq4o64tlsezajc272zpyxcsmz47uyc64", // v4.11.0 "/ipfs/bafybeidsjptidvb6wf6benznq2pxgnt5iyksgtecpmjoimlmswhtx2u5ua", // v4.10.0 "/ipfs/bafybeicg7e6o2eszkfdzxg5233gmuip2a7kfzoloh7voyvt2r6ivdet54u", // v4.9.1 "/ipfs/bafybeifplj2s3yegn7ko7tdnwpoxa4c5uaqnk2ajnw5geqm34slcj6b6mu", // v4.8.0 "/ipfs/bafybeibfd5kbebqqruouji6ct5qku3tay273g7mt24mmrfzrsfeewaal5y", // v4.7.0 "/ipfs/bafybeibpaa5kqrj4gkemiswbwndjqiryl65cks64ypwtyerxixu56gnvvm", // v4.6.0 "/ipfs/bafybeiata4qg7xjtwgor6r5dw63jjxyouenyromrrb4lrewxrlvav7gzgi", // v4.5.0 "/ipfs/bafybeigp3zm7cqoiciqk5anlheenqjsgovp7j7zq6hah4nu6iugdgb4nby", // v4.4.2 "/ipfs/bafybeiatztgdllxnp5p6zu7bdwhjmozsmd7jprff4bdjqjljxtylitvss4", // v4.4.1 "/ipfs/bafybeibgic2ex3fvzkinhy6k6aqyv3zy2o7bkbsmrzvzka24xetv7eeadm", // v4.4.0 "/ipfs/bafybeid4uxz7klxcu3ffsnmn64r7ihvysamlj4ohl5h2orjsffuegcpaeq", // v4.3.3 "/ipfs/bafybeif6abowqcavbkz243biyh7pde7ick5kkwwytrh7pd2hkbtuqysjxy", // v4.3.2 "/ipfs/bafybeihatzsgposbr3hrngo42yckdyqcc56yean2rynnwpzxstvdlphxf4", "/ipfs/bafybeigggyffcf6yfhx5irtwzx3cgnk6n3dwylkvcpckzhqqrigsxowjwe", "/ipfs/bafybeidf7cpkwsjkq6xs3r6fbbxghbugilx3jtezbza7gua3k5wjixpmba", "/ipfs/bafybeiamycmd52xvg6k3nzr6z3n33de6a2teyhquhj4kspdtnvetnkrfim", "/ipfs/bafybeieqdeoqkf7xf4aozd524qncgiloh33qgr25lyzrkusbcre4c3fxay", "/ipfs/bafybeicyp7ssbnj3hdzehcibmapmpuc3atrsc4ch3q6acldfh4ojjdbcxe", "/ipfs/bafybeigs6d53gpgu34553mbi5bbkb26e4ikruoaaar75jpfdywpup2r3my", "/ipfs/bafybeic4gops3d3lyrisqku37uio33nvt6fqxvkxihrwlqsuvf76yln4fm", "/ipfs/bafybeifeqt7mvxaniphyu2i3qhovjaf3sayooxbh5enfdqtiehxjv2ldte", // v2.22.0 "/ipfs/bafybeiequgo72mrvuml56j4gk7crewig5bavumrrzhkqbim6b3s2yqi7ty", "/ipfs/bafybeibjbq3tmmy7wuihhhwvbladjsd3gx3kfjepxzkq6wylik6wc3whzy", // v2.20.0 "/ipfs/bafybeiavrvt53fks6u32n5p2morgblcmck4bh4ymf4rrwu7ah5zsykmqqa", // v2.19.0 "/ipfs/bafybeiageaoxg6d7npaof6eyzqbwvbubyler7bq44hayik2hvqcggg7d2y", // v2.18.1 "/ipfs/bafybeidb5eryh72zajiokdggzo7yct2d6hhcflncji5im2y5w26uuygdsm", // v2.18.0 "/ipfs/bafybeibozpulxtpv5nhfa2ue3dcjx23ndh3gwr5vwllk7ptoyfwnfjjr4q", // v2.15.1 "/ipfs/bafybeiednzu62vskme5wpoj4bjjikeg3xovfpp4t7vxk5ty2jxdi4mv4bu", // v2.15.0 "/ipfs/bafybeihcyruaeza7uyjd6ugicbcrqumejf6uf353e5etdkhotqffwtguva", // v2.13.0 "/ipfs/bafybeiflkjt66aetfgcrgvv75izymd5kc47g6luepqmfq6zsf5w6ueth6y", "/ipfs/bafybeid26vjplsejg7t3nrh7mxmiaaxriebbm4xxrxxdunlk7o337m5sqq", "/ipfs/bafybeif4zkmu7qdhkpf3pnhwxipylqleof7rl6ojbe7mq3fzogz6m4xk3i", // v2.11.4 "/ipfs/bafybeianwe4vy7sprht5sm3hshvxjeqhwcmvbzq73u55sdhqngmohkjgs4", "/ipfs/bafybeicitin4p7ggmyjaubqpi3xwnagrwarsy6hiihraafk5rcrxqxju6m", "/ipfs/bafybeihpetclqvwb4qnmumvcn7nh4pxrtugrlpw4jgjpqicdxsv7opdm6e", "/ipfs/bafybeibnnxd4etu4tq5fuhu3z5p4rfu3buabfkeyr3o3s4h6wtesvvw6mu", "/ipfs/bafybeid6luolenf4fcsuaw5rgdwpqbyerce4x3mi3hxfdtp5pwco7h7qyq", "/ipfs/bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4", "/ipfs/bafybeicp23nbcxtt2k2twyfivcbrc6kr3l5lnaiv3ozvwbemtrb7v52r6i", "/ipfs/bafybeidatpz2hli6fgu3zul5woi27ujesdf5o5a7bu622qj6ugharciwjq", "/ipfs/QmfQkD8pBSBCBxWEwFSu4XaDVSWK6bjnNuaWZjMyQbyDub", "/ipfs/QmXc9raDM1M5G5fpBnVyQ71vR4gbnskwnB9iMEzBuLgvoZ", "/ipfs/QmenEBWcAk3tN94fSKpKFtUMwty1qNwSYw3DMDFV6cPBXA", "/ipfs/QmUnXcWZC5Ve21gUseouJsH5mLAyz5JPp8aHsg8qVUUK8e", "/ipfs/QmSDgpiHco5yXdyVTfhKxr3aiJ82ynz8V14QcGKicM3rVh", "/ipfs/QmRuvWJz1Fc8B9cTsAYANHTXqGmKR9DVfY5nvMD1uA2WQ8", "/ipfs/QmQLXHs7K98JNQdWrBB2cQLJahPhmupbDjRuH1b9ibmwVa", "/ipfs/QmXX7YRpU7nNBKfw75VG7Y1c3GwpSAGHRev67XVPgZFv9R", "/ipfs/QmXdu7HWdV6CUaUabd9q2ZeA4iHZLVyDRj3Gi4dsJsWjbr", "/ipfs/QmaaqrHyAQm7gALkRW8DcfGX3u8q9rWKnxEMmf7m9z515w", "/ipfs/QmSHDxWsMPuJQKWmVA1rB5a3NX2Eme5fPqNb63qwaqiqSp", "/ipfs/QmctngrQAt9fjpQUZr7Bx3BsXUcif52eZGTizWhvcShsjz", "/ipfs/QmS2HL9v5YeKgQkkWMvs1EMnFtUowTEdFfSSeMT4pos1e6", "/ipfs/QmR9MzChjp1MdFWik7NjEjqKQMzVmBkdK3dz14A6B5Cupm", "/ipfs/QmRyWyKWmphamkMRnJVjUTzSFSAAZowYP4rnbgnfMXC9Mr", "/ipfs/QmU3o9bvfenhTKhxUakbYrLDnZU7HezAVxPM6Ehjw9Xjqy", "/ipfs/QmPhnvn747LqwPYMJmQVorMaGbMSgA7mRRoyyZYz3DoZRQ", "/ipfs/QmQNHd1suZTktPRhP7DD4nKWG46ZRSxkwHocycHVrK3dYW", "/ipfs/QmNyMYhwJUS1cVvaWoVBhrW8KPj1qmie7rZcWo8f1Bvkhz", "/ipfs/QmVTiRTQ72qiH4usAGT4c6qVxCMv4hFMUH9fvU6mktaXdP", "/ipfs/QmYcP4sp1nraBiCYi6i9kqdaKobrK32yyMpTrM5JDA8a2C", "/ipfs/QmUtMmxgHnDvQq4bpH6Y9MaLN1hpfjJz5LZcq941BEqEXs", "/ipfs/QmPURAjo3oneGH53ovt68UZEBvsc8nNmEhQZEpsVEQUMZE", "/ipfs/QmeSXt32frzhvewLKwA1dePTSjkTfGVwTh55ZcsJxrCSnk", "/ipfs/QmcjeTciMNgEBe4xXvEaA4TQtwTRkXucx7DmKWViXSmX7m", "/ipfs/QmfNbSskgvTXYhuqP8tb9AKbCkyRcCy3WeiXwD9y5LeoqK", "/ipfs/QmPkojhjJkJ5LEGBDrAvdftrjAYmi9GU5Cq27mWvZTDieW", "/ipfs/Qmexhq2sBHnXQbvyP2GfUdbnY7HCagH2Mw5vUNSBn2nxip", } // WebUIOption provides the WebUI handler for the RPC API. func WebUIOption(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { cfg, err := n.Repo.Config() if err != nil { return nil, err } handler := &webUIHandler{ headers: cfg.API.HTTPHeaders, node: n, noFetch: cfg.Gateway.NoFetch, deserializedResponses: cfg.Gateway.DeserializedResponses.WithDefault(config.DefaultDeserializedResponses), } mux.Handle("/webui/", handler) return mux, nil } type webUIHandler struct { headers map[string][]string node *core.IpfsNode noFetch bool deserializedResponses bool } func (h *webUIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { for k, v := range h.headers { w.Header()[http.CanonicalHeaderKey(k)] = v } // Check if WebUI is incompatible with current configuration if !h.deserializedResponses { h.writeIncompatibleError(w) return } // Check if WebUI is available locally when Gateway.NoFetch is true if h.noFetch { cidStr := strings.TrimPrefix(WebUIPath, "/ipfs/") webUICID, err := cid.Parse(cidStr) if err != nil { // This should never happen with hardcoded constant log.Errorf("failed to parse WebUI CID: %v", err) } else { has, err := h.node.Blockstore.Has(r.Context(), webUICID) if err != nil { log.Debugf("error checking WebUI availability: %v", err) } else if !has { h.writeNotAvailableError(w) return } } } // Default behavior: redirect to the WebUI path http.Redirect(w, r, WebUIPath, http.StatusFound) } func (h *webUIHandler) writeIncompatibleError(w http.ResponseWriter) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusServiceUnavailable) fmt.Fprintf(w, `IPFS WebUI Incompatible WebUI is not compatible with Gateway.DeserializedResponses=false. The WebUI requires deserializing IPFS responses to render the interface. To use the WebUI, set Gateway.DeserializedResponses=true in your config. `) } func (h *webUIHandler) writeNotAvailableError(w http.ResponseWriter) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusServiceUnavailable) fmt.Fprintf(w, `IPFS WebUI Not Available WebUI at %s is not in your local node due to Gateway.NoFetch=true. To use the WebUI, either: 1. Run: ipfs pin add --progress --name ipfs-webui %s 2. Download from https://github.com/ipfs/ipfs-webui/releases and import with: ipfs dag import ipfs-webui.car `, WebUIPath, WebUIPath) } ================================================ FILE: core/coreiface/block.go ================================================ package iface import ( "context" "io" "github.com/ipfs/boxo/path" "github.com/ipfs/kubo/core/coreiface/options" ) // BlockStat contains information about a block type BlockStat interface { // Size is the size of a block Size() int // Path returns path to the block Path() path.ImmutablePath } // BlockAPI specifies the interface to the block layer type BlockAPI interface { // Put imports raw block data, hashing it using specified settings. Put(context.Context, io.Reader, ...options.BlockPutOption) (BlockStat, error) // Get attempts to resolve the path and return a reader for data in the block Get(context.Context, path.Path) (io.Reader, error) // Rm removes the block specified by the path from local blockstore. // By default an error will be returned if the block can't be found locally. // // NOTE: If the specified block is pinned it won't be removed and no error // will be returned Rm(context.Context, path.Path, ...options.BlockRmOption) error // Stat returns information on Stat(context.Context, path.Path) (BlockStat, error) } ================================================ FILE: core/coreiface/coreapi.go ================================================ // Package iface defines IPFS Core API which is a set of interfaces used to // interact with IPFS nodes. package iface import ( "context" "github.com/ipfs/boxo/path" "github.com/ipfs/kubo/core/coreiface/options" ipld "github.com/ipfs/go-ipld-format" ) // CoreAPI defines an unified interface to IPFS for Go programs type CoreAPI interface { // Unixfs returns an implementation of Unixfs API Unixfs() UnixfsAPI // Block returns an implementation of Block API Block() BlockAPI // Dag returns an implementation of Dag API Dag() APIDagService // Name returns an implementation of Name API Name() NameAPI // Key returns an implementation of Key API Key() KeyAPI // Pin returns an implementation of Pin API Pin() PinAPI // Object returns an implementation of Object API Object() ObjectAPI // Swarm returns an implementation of Swarm API Swarm() SwarmAPI // PubSub returns an implementation of PubSub API PubSub() PubSubAPI // Routing returns an implementation of Routing API Routing() RoutingAPI // ResolvePath resolves the path using UnixFS resolver, and returns the resolved // immutable path, and the remainder of the path segments that cannot be resolved // within UnixFS. ResolvePath(context.Context, path.Path) (path.ImmutablePath, []string, error) // ResolveNode resolves the path (if not resolved already) using Unixfs // resolver, gets and returns the resolved Node ResolveNode(context.Context, path.Path) (ipld.Node, error) // WithOptions creates new instance of CoreAPI based on this instance with // a set of options applied WithOptions(...options.ApiOption) (CoreAPI, error) } ================================================ FILE: core/coreiface/dag.go ================================================ package iface import ( ipld "github.com/ipfs/go-ipld-format" ) // APIDagService extends ipld.DAGService type APIDagService interface { ipld.DAGService // Pinning returns special NodeAdder which recursively pins added nodes Pinning() ipld.NodeAdder } ================================================ FILE: core/coreiface/errors.go ================================================ package iface import "errors" var ( ErrIsDir = errors.New("this dag node is a directory") ErrNotFile = errors.New("this dag node is not a regular file") ErrOffline = errors.New("this action must be run in online mode, try running 'ipfs daemon' first") ErrNotSupported = errors.New("operation not supported") ) ================================================ FILE: core/coreiface/idfmt.go ================================================ package iface import ( "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" ) func FormatKeyID(id peer.ID) string { if s, err := peer.ToCid(id).StringOfBase(mbase.Base36); err != nil { panic(err) } else { return s } } // FormatKey formats the given IPNS key in a canonical way. func FormatKey(key Key) string { return FormatKeyID(key.ID()) } ================================================ FILE: core/coreiface/key.go ================================================ package iface import ( "context" "github.com/ipfs/boxo/path" "github.com/ipfs/kubo/core/coreiface/options" "github.com/libp2p/go-libp2p/core/peer" ) // Key specifies the interface to Keys in KeyAPI Keystore type Key interface { // Key returns key name Name() string // Path returns key path Path() path.Path // ID returns key PeerID ID() peer.ID } // KeyAPI specifies the interface to Keystore type KeyAPI interface { // Generate generates new key, stores it in the keystore under the specified // name and returns a base58 encoded multihash of it's public key Generate(ctx context.Context, name string, opts ...options.KeyGenerateOption) (Key, error) // Rename renames oldName key to newName. Returns the key and whether another // key was overwritten, or an error Rename(ctx context.Context, oldName string, newName string, opts ...options.KeyRenameOption) (Key, bool, error) // List lists keys stored in keystore List(ctx context.Context) ([]Key, error) // Self returns the 'main' node key Self(ctx context.Context) (Key, error) // Remove removes keys from keystore. Returns ipns path of the removed key Remove(ctx context.Context, name string) (Key, error) // Sign signs the given data with the key named name. Returns the key used // for signing, the signature, and an error. Sign(ctx context.Context, name string, data []byte) (Key, []byte, error) // Verify verifies if the given data and signatures match. Returns the key used // for verification, whether signature and data match, and an error. Verify(ctx context.Context, keyOrName string, signature, data []byte) (Key, bool, error) } ================================================ FILE: core/coreiface/name.go ================================================ package iface import ( "context" "errors" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" "github.com/ipfs/kubo/core/coreiface/options" ) var ErrResolveFailed = errors.New("could not resolve name") type IpnsResult struct { path.Path Err error } // NameAPI specifies the interface to IPNS. // // IPNS is a PKI namespace, where names are the hashes of public keys, and the // private key enables publishing new (signed) values. In both publish and // resolve, the default name used is the node's own PeerID, which is the hash of // its public key. // // You can use .Key API to list and generate more names and their respective keys. type NameAPI interface { // Publish announces new IPNS name Publish(ctx context.Context, path path.Path, opts ...options.NamePublishOption) (ipns.Name, error) // Resolve attempts to resolve the newest version of the specified name Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (path.Path, error) // Search is a version of Resolve which outputs paths as they are discovered, // reducing the time to first entry // // Note: by default, all paths read from the channel are considered unsafe, // except the latest (last path in channel read buffer). Search(ctx context.Context, name string, opts ...options.NameResolveOption) (<-chan IpnsResult, error) } ================================================ FILE: core/coreiface/object.go ================================================ package iface import ( "context" "github.com/ipfs/boxo/path" "github.com/ipfs/kubo/core/coreiface/options" ) // ChangeType denotes type of change in ObjectChange type ChangeType int const ( // DiffAdd is set when a link was added to the graph DiffAdd ChangeType = iota // DiffRemove is set when a link was removed from the graph DiffRemove // DiffMod is set when a link was changed in the graph DiffMod ) // ObjectChange represents a change ia a graph type ObjectChange struct { // Type of the change, either: // * DiffAdd - Added a link // * DiffRemove - Removed a link // * DiffMod - Modified a link Type ChangeType // Path to the changed link Path string // Before holds the link path before the change. Note that when a link is // added, this will be nil. Before path.ImmutablePath // After holds the link path after the change. Note that when a link is // removed, this will be nil. After path.ImmutablePath } // ObjectAPI specifies the interface to MerkleDAG and contains useful utilities // for manipulating MerkleDAG data structures. type ObjectAPI interface { // AddLink adds a link under the specified path. child path can point to a // subdirectory within the patent which must be present (can be overridden // with WithCreate option). AddLink(ctx context.Context, base path.Path, name string, child path.Path, opts ...options.ObjectAddLinkOption) (path.ImmutablePath, error) // RmLink removes a link from the node RmLink(ctx context.Context, base path.Path, link string) (path.ImmutablePath, error) // Diff returns a set of changes needed to transform the first object into the // second. Diff(context.Context, path.Path, path.Path) ([]ObjectChange, error) } ================================================ FILE: core/coreiface/options/block.go ================================================ package options import ( "fmt" cid "github.com/ipfs/go-cid" mc "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" ) type BlockPutSettings struct { CidPrefix cid.Prefix Pin bool } type BlockRmSettings struct { Force bool } type ( BlockPutOption func(*BlockPutSettings) error BlockRmOption func(*BlockRmSettings) error ) func BlockPutOptions(opts ...BlockPutOption) (*BlockPutSettings, error) { var cidPrefix cid.Prefix // Baseline is CIDv1 raw sha2-255-32 (can be tweaked later via opts) cidPrefix.Version = 1 cidPrefix.Codec = uint64(mc.Raw) cidPrefix.MhType = mh.SHA2_256 cidPrefix.MhLength = -1 // -1 means len is to be calculated during mh.Sum() options := &BlockPutSettings{ CidPrefix: cidPrefix, Pin: false, } // Apply any overrides for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } func BlockRmOptions(opts ...BlockRmOption) (*BlockRmSettings, error) { options := &BlockRmSettings{ Force: false, } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } type blockOpts struct{} var Block blockOpts // CidCodec is the modern option for Block.Put which specifies the multicodec to use // in the CID returned by the Block.Put operation. // It uses correct codes from go-multicodec and replaces the old Format now with CIDv1 as the default. func (blockOpts) CidCodec(codecName string) BlockPutOption { return func(settings *BlockPutSettings) error { if codecName == "" { return nil } code, err := codeFromName(codecName) if err != nil { return err } settings.CidPrefix.Codec = uint64(code) return nil } } // Map string to code from go-multicodec func codeFromName(codecName string) (mc.Code, error) { var cidCodec mc.Code err := cidCodec.Set(codecName) return cidCodec, err } // Format is a legacy option for Block.Put which specifies the multicodec to // use to serialize the object. // Provided for backward-compatibility only. Use CidCodec instead. func (blockOpts) Format(format string) BlockPutOption { return func(settings *BlockPutSettings) error { if format == "" { return nil } // Opt-in CIDv0 support for backward-compatibility if format == "v0" { settings.CidPrefix.Version = 0 } // Fixup a legacy (invalid) names for dag-pb (0x70) if format == "v0" || format == "protobuf" { format = "dag-pb" } // Fixup invalid name for dag-cbor (0x71) if format == "cbor" { format = "dag-cbor" } // Set code based on name passed as "format" code, err := codeFromName(format) if err != nil { return err } settings.CidPrefix.Codec = uint64(code) // If CIDv0, ensure all parameters are compatible // (in theory go-cid would validate this anyway, but we want to provide better errors) pref := settings.CidPrefix if pref.Version == 0 { if pref.Codec != uint64(mc.DagPb) { return fmt.Errorf("only dag-pb is allowed with CIDv0") } if pref.MhType != mh.SHA2_256 || (pref.MhLength != -1 && pref.MhLength != 32) { return fmt.Errorf("only sha2-255-32 is allowed with CIDv0") } } return nil } } // Hash is an option for Block.Put which specifies the multihash settings to use // when hashing the object. Default is mh.SHA2_256 (0x12). // If mhLen is set to -1, default length for the hash will be used func (blockOpts) Hash(mhType uint64, mhLen int) BlockPutOption { return func(settings *BlockPutSettings) error { settings.CidPrefix.MhType = mhType settings.CidPrefix.MhLength = mhLen return nil } } // Pin is an option for Block.Put which specifies whether to (recursively) pin // added blocks func (blockOpts) Pin(pin bool) BlockPutOption { return func(settings *BlockPutSettings) error { settings.Pin = pin return nil } } // Force is an option for Block.Rm which, when set to true, will ignore // non-existing blocks func (blockOpts) Force(force bool) BlockRmOption { return func(settings *BlockRmSettings) error { settings.Force = force return nil } } ================================================ FILE: core/coreiface/options/dht.go ================================================ package options // nolint deprecated // Deprecated: use [RoutingProvideSettings] instead. type DhtProvideSettings = RoutingProvideSettings // nolint deprecated // Deprecated: use [RoutingFindProvidersSettings] instead. type DhtFindProvidersSettings = RoutingFindProvidersSettings // nolint deprecated // Deprecated: use [RoutingProvideOption] instead. type DhtProvideOption = RoutingProvideOption // nolint deprecated // Deprecated: use [RoutingFindProvidersOption] instead. type DhtFindProvidersOption = RoutingFindProvidersOption // nolint deprecated // Deprecated: use [RoutingProvideOptions] instead. var DhtProvideOptions = RoutingProvideOptions // nolint deprecated // Deprecated: use [RoutingFindProvidersOptions] instead. var DhtFindProvidersOptions = RoutingFindProvidersOptions // nolint deprecated // Deprecated: use [Routing] instead. var Dht = Routing ================================================ FILE: core/coreiface/options/global.go ================================================ package options type ApiSettings struct { Offline bool FetchBlocks bool } type ApiOption func(*ApiSettings) error func ApiOptions(opts ...ApiOption) (*ApiSettings, error) { options := &ApiSettings{ Offline: false, FetchBlocks: true, } return ApiOptionsTo(options, opts...) } func ApiOptionsTo(options *ApiSettings, opts ...ApiOption) (*ApiSettings, error) { for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } type apiOpts struct{} var Api apiOpts func (apiOpts) Offline(offline bool) ApiOption { return func(settings *ApiSettings) error { settings.Offline = offline return nil } } // FetchBlocks when set to false prevents api from fetching blocks from the // network while allowing other services such as IPNS to still be online func (apiOpts) FetchBlocks(fetch bool) ApiOption { return func(settings *ApiSettings) error { settings.FetchBlocks = fetch return nil } } ================================================ FILE: core/coreiface/options/key.go ================================================ package options const ( RSAKey = "rsa" Ed25519Key = "ed25519" DefaultRSALen = 2048 ) type KeyGenerateSettings struct { Algorithm string Size int } type KeyRenameSettings struct { Force bool } type ( KeyGenerateOption func(*KeyGenerateSettings) error KeyRenameOption func(*KeyRenameSettings) error ) func KeyGenerateOptions(opts ...KeyGenerateOption) (*KeyGenerateSettings, error) { options := &KeyGenerateSettings{ Algorithm: RSAKey, Size: -1, } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } func KeyRenameOptions(opts ...KeyRenameOption) (*KeyRenameSettings, error) { options := &KeyRenameSettings{ Force: false, } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } type keyOpts struct{} var Key keyOpts // Type is an option for Key.Generate which specifies which algorithm // should be used for the key. Default is options.RSAKey // // Supported key types: // * options.RSAKey // * options.Ed25519Key func (keyOpts) Type(algorithm string) KeyGenerateOption { return func(settings *KeyGenerateSettings) error { settings.Algorithm = algorithm return nil } } // Size is an option for Key.Generate which specifies the size of the key to // generated. Default is -1 // // value of -1 means 'use default size for key type': // - 2048 for RSA func (keyOpts) Size(size int) KeyGenerateOption { return func(settings *KeyGenerateSettings) error { settings.Size = size return nil } } // Force is an option for Key.Rename which specifies whether to allow to // replace existing keys. func (keyOpts) Force(force bool) KeyRenameOption { return func(settings *KeyRenameSettings) error { settings.Force = force return nil } } ================================================ FILE: core/coreiface/options/name.go ================================================ package options import ( "time" "github.com/ipfs/boxo/namesys" ) const ( DefaultNameValidTime = 24 * time.Hour ) type NamePublishSettings struct { ValidTime time.Duration Key string TTL *time.Duration CompatibleWithV1 bool AllowOffline bool AllowDelegated bool Sequence *uint64 } type NameResolveSettings struct { Cache bool ResolveOpts []namesys.ResolveOption } type ( NamePublishOption func(*NamePublishSettings) error NameResolveOption func(*NameResolveSettings) error ) func NamePublishOptions(opts ...NamePublishOption) (*NamePublishSettings, error) { options := &NamePublishSettings{ ValidTime: DefaultNameValidTime, Key: "self", AllowOffline: false, AllowDelegated: false, } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } func NameResolveOptions(opts ...NameResolveOption) (*NameResolveSettings, error) { options := &NameResolveSettings{ Cache: true, } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } type nameOpts struct{} var Name nameOpts // ValidTime is an option for Name.Publish which specifies for how long the // entry will remain valid. Default value is 24h func (nameOpts) ValidTime(validTime time.Duration) NamePublishOption { return func(settings *NamePublishSettings) error { settings.ValidTime = validTime return nil } } // Key is an option for Name.Publish which specifies the key to use for // publishing. Default value is "self" which is the node's own PeerID. // The key parameter must be either PeerID or keystore key alias. // // You can use KeyAPI to list and generate more names and their respective keys. func (nameOpts) Key(key string) NamePublishOption { return func(settings *NamePublishSettings) error { settings.Key = key return nil } } // AllowOffline is an option for Name.Publish which specifies whether to allow // publishing when the node is offline. Default value is false func (nameOpts) AllowOffline(allow bool) NamePublishOption { return func(settings *NamePublishSettings) error { settings.AllowOffline = allow return nil } } // AllowDelegated is an option for Name.Publish which allows publishing without // DHT connectivity, using local datastore and HTTP delegated publishers only. // Default value is false func (nameOpts) AllowDelegated(allowDelegated bool) NamePublishOption { return func(settings *NamePublishSettings) error { settings.AllowDelegated = allowDelegated return nil } } // TTL is an option for Name.Publish which specifies the time duration the // published record should be cached for (caution: experimental). func (nameOpts) TTL(ttl time.Duration) NamePublishOption { return func(settings *NamePublishSettings) error { settings.TTL = &ttl return nil } } // Sequence is an option for Name.Publish which specifies the sequence number of // a namesys record. func (nameOpts) Sequence(seq uint64) NamePublishOption { return func(settings *NamePublishSettings) error { settings.Sequence = &seq return nil } } // CompatibleWithV1 is an option for [Name.Publish] which specifies if the // created record should be backwards compatible with V1 IPNS Records. func (nameOpts) CompatibleWithV1(compatible bool) NamePublishOption { return func(settings *NamePublishSettings) error { settings.CompatibleWithV1 = compatible return nil } } // Cache is an option for Name.Resolve which specifies if cache should be used. // Default value is true func (nameOpts) Cache(cache bool) NameResolveOption { return func(settings *NameResolveSettings) error { settings.Cache = cache return nil } } func (nameOpts) ResolveOption(opt namesys.ResolveOption) NameResolveOption { return func(settings *NameResolveSettings) error { settings.ResolveOpts = append(settings.ResolveOpts, opt) return nil } } ================================================ FILE: core/coreiface/options/object.go ================================================ package options type ObjectAddLinkSettings struct { Create bool } type ( ObjectAddLinkOption func(*ObjectAddLinkSettings) error ) func ObjectAddLinkOptions(opts ...ObjectAddLinkOption) (*ObjectAddLinkSettings, error) { options := &ObjectAddLinkSettings{ Create: false, } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } type objectOpts struct{} var Object objectOpts // Create is an option for Object.AddLink which specifies whether create required // directories for the child func (objectOpts) Create(create bool) ObjectAddLinkOption { return func(settings *ObjectAddLinkSettings) error { settings.Create = create return nil } } ================================================ FILE: core/coreiface/options/pin.go ================================================ package options import "fmt" // PinAddSettings represent the settings for PinAPI.Add type PinAddSettings struct { Recursive bool Name string } // PinLsSettings represent the settings for PinAPI.Ls type PinLsSettings struct { Type string Detailed bool Name string } // PinIsPinnedSettings represent the settings for PinAPI.IsPinned type PinIsPinnedSettings struct { WithType string } // PinRmSettings represents the settings for PinAPI.Rm type PinRmSettings struct { Recursive bool } // PinUpdateSettings represent the settings for PinAPI.Update type PinUpdateSettings struct { Unpin bool } // PinAddOption is the signature of an option for PinAPI.Add type PinAddOption func(*PinAddSettings) error // PinLsOption is the signature of an option for PinAPI.Ls type PinLsOption func(*PinLsSettings) error // PinIsPinnedOption is the signature of an option for PinAPI.IsPinned type PinIsPinnedOption func(*PinIsPinnedSettings) error // PinRmOption is the signature of an option for PinAPI.Rm type PinRmOption func(*PinRmSettings) error // PinUpdateOption is the signature of an option for PinAPI.Update type PinUpdateOption func(*PinUpdateSettings) error // PinAddOptions compile a series of PinAddOption into a ready to use // PinAddSettings and set the default values. func PinAddOptions(opts ...PinAddOption) (*PinAddSettings, error) { options := &PinAddSettings{ Recursive: true, } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } // PinLsOptions compile a series of PinLsOption into a ready to use // PinLsSettings and set the default values. func PinLsOptions(opts ...PinLsOption) (*PinLsSettings, error) { options := &PinLsSettings{ Type: "all", } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } // PinIsPinnedOptions compile a series of PinIsPinnedOption into a ready to use // PinIsPinnedSettings and set the default values. func PinIsPinnedOptions(opts ...PinIsPinnedOption) (*PinIsPinnedSettings, error) { options := &PinIsPinnedSettings{ WithType: "all", } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } // PinRmOptions compile a series of PinRmOption into a ready to use // PinRmSettings and set the default values. func PinRmOptions(opts ...PinRmOption) (*PinRmSettings, error) { options := &PinRmSettings{ Recursive: true, } for _, opt := range opts { if err := opt(options); err != nil { return nil, err } } return options, nil } // PinUpdateOptions compile a series of PinUpdateOption into a ready to use // PinUpdateSettings and set the default values. func PinUpdateOptions(opts ...PinUpdateOption) (*PinUpdateSettings, error) { options := &PinUpdateSettings{ Unpin: true, } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } type pinOpts struct { Ls pinLsOpts IsPinned pinIsPinnedOpts } // Pin provide an access to all the options for the Pin API. var Pin pinOpts type pinLsOpts struct{} // All is an option for Pin.Ls which will make it return all pins. It is // the default func (pinLsOpts) All() PinLsOption { return Pin.Ls.pinType("all") } // Recursive is an option for Pin.Ls which will make it only return recursive // pins func (pinLsOpts) Recursive() PinLsOption { return Pin.Ls.pinType("recursive") } // Direct is an option for Pin.Ls which will make it only return direct (non // recursive) pins func (pinLsOpts) Direct() PinLsOption { return Pin.Ls.pinType("direct") } // Indirect is an option for Pin.Ls which will make it only return indirect pins // (objects referenced by other recursively pinned objects) func (pinLsOpts) Indirect() PinLsOption { return Pin.Ls.pinType("indirect") } // Type is an option for Pin.Ls which will make it only return pins of the given // type. // // Supported values: // - "direct" - directly pinned objects // - "recursive" - roots of recursive pins // - "indirect" - indirectly pinned objects (referenced by recursively pinned // objects) // - "all" - all pinned objects (default) func (pinLsOpts) Type(typeStr string) (PinLsOption, error) { switch typeStr { case "all", "direct", "indirect", "recursive": return Pin.Ls.pinType(typeStr), nil default: return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", typeStr) } } // pinType is an option for Pin.Ls which allows to specify which pin types should // be returned // // Supported values: // - "direct" - directly pinned objects // - "recursive" - roots of recursive pins // - "indirect" - indirectly pinned objects (referenced by recursively pinned // objects) // - "all" - all pinned objects (default) func (pinLsOpts) pinType(t string) PinLsOption { return func(settings *PinLsSettings) error { settings.Type = t return nil } } // Detailed is an option for [Pin.Ls] which sets whether or not to return // detailed information, such as pin names and modes. func (pinLsOpts) Detailed(detailed bool) PinLsOption { return func(settings *PinLsSettings) error { settings.Detailed = detailed return nil } } func (pinLsOpts) Name(name string) PinLsOption { return func(settings *PinLsSettings) error { settings.Name = name return nil } } type pinIsPinnedOpts struct{} // All is an option for Pin.IsPinned which will make it search in all type of pins. // It is the default func (pinIsPinnedOpts) All() PinIsPinnedOption { return Pin.IsPinned.pinType("all") } // Recursive is an option for Pin.IsPinned which will make it only search in // recursive pins func (pinIsPinnedOpts) Recursive() PinIsPinnedOption { return Pin.IsPinned.pinType("recursive") } // Direct is an option for Pin.IsPinned which will make it only search in direct // (non recursive) pins func (pinIsPinnedOpts) Direct() PinIsPinnedOption { return Pin.IsPinned.pinType("direct") } // Indirect is an option for Pin.IsPinned which will make it only search indirect // pins (objects referenced by other recursively pinned objects) func (pinIsPinnedOpts) Indirect() PinIsPinnedOption { return Pin.IsPinned.pinType("indirect") } // Type is an option for Pin.IsPinned which will make it only search pins of the given // type. // // Supported values: // - "direct" - directly pinned objects // - "recursive" - roots of recursive pins // - "indirect" - indirectly pinned objects (referenced by recursively pinned // objects) // - "all" - all pinned objects (default) func (pinIsPinnedOpts) Type(typeStr string) (PinIsPinnedOption, error) { switch typeStr { case "all", "direct", "indirect", "recursive": return Pin.IsPinned.pinType(typeStr), nil default: return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", typeStr) } } // pinType is an option for Pin.IsPinned which allows to specify which pin type the given // pin is expected to be, speeding up the research. // // Supported values: // - "direct" - directly pinned objects // - "recursive" - roots of recursive pins // - "indirect" - indirectly pinned objects (referenced by recursively pinned // objects) // - "all" - all pinned objects (default) func (pinIsPinnedOpts) pinType(t string) PinIsPinnedOption { return func(settings *PinIsPinnedSettings) error { settings.WithType = t return nil } } // Recursive is an option for Pin.Add which specifies whether to pin an entire // object tree or just one object. Default: true func (pinOpts) Recursive(recursive bool) PinAddOption { return func(settings *PinAddSettings) error { settings.Recursive = recursive return nil } } // Name is an option for Pin.Add which specifies an optional name to add to the pin. func (pinOpts) Name(name string) PinAddOption { return func(settings *PinAddSettings) error { settings.Name = name return nil } } // RmRecursive is an option for Pin.Rm which specifies whether to recursively // unpin the object linked to by the specified object(s). This does not remove // indirect pins referenced by other recursive pins. func (pinOpts) RmRecursive(recursive bool) PinRmOption { return func(settings *PinRmSettings) error { settings.Recursive = recursive return nil } } // Unpin is an option for Pin.Update which specifies whether to remove the old pin. // Default is true. func (pinOpts) Unpin(unpin bool) PinUpdateOption { return func(settings *PinUpdateSettings) error { settings.Unpin = unpin return nil } } ================================================ FILE: core/coreiface/options/pubsub.go ================================================ package options type PubSubPeersSettings struct { Topic string } type PubSubSubscribeSettings struct { Discover bool } type ( PubSubPeersOption func(*PubSubPeersSettings) error PubSubSubscribeOption func(*PubSubSubscribeSettings) error ) func PubSubPeersOptions(opts ...PubSubPeersOption) (*PubSubPeersSettings, error) { options := &PubSubPeersSettings{ Topic: "", } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } func PubSubSubscribeOptions(opts ...PubSubSubscribeOption) (*PubSubSubscribeSettings, error) { options := &PubSubSubscribeSettings{ Discover: false, } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } type pubsubOpts struct{} var PubSub pubsubOpts func (pubsubOpts) Topic(topic string) PubSubPeersOption { return func(settings *PubSubPeersSettings) error { settings.Topic = topic return nil } } func (pubsubOpts) Discover(discover bool) PubSubSubscribeOption { return func(settings *PubSubSubscribeSettings) error { settings.Discover = discover return nil } } ================================================ FILE: core/coreiface/options/routing.go ================================================ package options type RoutingPutSettings struct { AllowOffline bool } type RoutingPutOption func(*RoutingPutSettings) error func RoutingPutOptions(opts ...RoutingPutOption) (*RoutingPutSettings, error) { options := &RoutingPutSettings{ AllowOffline: false, } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } // nolint deprecated // Deprecated: use [Routing] instead. var Put = Routing type RoutingProvideSettings struct { Recursive bool } type RoutingFindProvidersSettings struct { NumProviders int } type ( RoutingProvideOption func(*DhtProvideSettings) error RoutingFindProvidersOption func(*DhtFindProvidersSettings) error ) func RoutingProvideOptions(opts ...RoutingProvideOption) (*RoutingProvideSettings, error) { options := &RoutingProvideSettings{ Recursive: false, } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } func RoutingFindProvidersOptions(opts ...RoutingFindProvidersOption) (*RoutingFindProvidersSettings, error) { options := &RoutingFindProvidersSettings{ NumProviders: 20, } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } type routingOpts struct{} var Routing routingOpts // Recursive is an option for [Routing.Provide] which specifies whether to provide // the given path recursively. func (routingOpts) Recursive(recursive bool) RoutingProvideOption { return func(settings *DhtProvideSettings) error { settings.Recursive = recursive return nil } } // NumProviders is an option for [Routing.FindProviders] which specifies the // number of peers to look for. Default is 20. func (routingOpts) NumProviders(numProviders int) RoutingFindProvidersOption { return func(settings *DhtFindProvidersSettings) error { settings.NumProviders = numProviders return nil } } // AllowOffline is an option for [Routing.Put] which specifies whether to allow // publishing when the node is offline. Default value is false func (routingOpts) AllowOffline(allow bool) RoutingPutOption { return func(settings *RoutingPutSettings) error { settings.AllowOffline = allow return nil } } ================================================ FILE: core/coreiface/options/unixfs.go ================================================ package options import ( "errors" "fmt" "os" "time" dag "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" "github.com/ipfs/boxo/ipld/unixfs/io" cid "github.com/ipfs/go-cid" mh "github.com/multiformats/go-multihash" ) type Layout int const ( BalancedLayout Layout = iota TrickleLayout ) type UnixfsAddSettings struct { CidVersion int MhType uint64 Inline bool InlineLimit int RawLeaves bool RawLeavesSet bool MaxFileLinks int MaxFileLinksSet bool MaxDirectoryLinks int MaxDirectoryLinksSet bool MaxHAMTFanout int MaxHAMTFanoutSet bool SizeEstimationMode *io.SizeEstimationMode SizeEstimationModeSet bool Chunker string Layout Layout Pin bool PinName string OnlyHash bool FsCache bool NoCopy bool Events chan<- any Silent bool Progress bool PreserveMode bool PreserveMtime bool Mode os.FileMode Mtime time.Time IncludeEmptyDirs bool IncludeEmptyDirsSet bool } type UnixfsLsSettings struct { ResolveChildren bool UseCumulativeSize bool } type ( UnixfsAddOption func(*UnixfsAddSettings) error UnixfsLsOption func(*UnixfsLsSettings) error ) func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, error) { options := &UnixfsAddSettings{ CidVersion: -1, MhType: mh.SHA2_256, Inline: false, InlineLimit: 32, RawLeaves: false, RawLeavesSet: false, MaxFileLinks: helpers.DefaultLinksPerBlock, MaxFileLinksSet: false, MaxDirectoryLinks: 0, MaxDirectoryLinksSet: false, MaxHAMTFanout: io.DefaultShardWidth, MaxHAMTFanoutSet: false, Chunker: "size-262144", Layout: BalancedLayout, Pin: false, PinName: "", OnlyHash: false, FsCache: false, NoCopy: false, Events: nil, Silent: false, Progress: false, PreserveMode: false, PreserveMtime: false, Mode: 0, Mtime: time.Time{}, IncludeEmptyDirs: true, // default: include empty directories IncludeEmptyDirsSet: false, } for _, opt := range opts { err := opt(options) if err != nil { return nil, cid.Prefix{}, err } } // nocopy -> rawblocks if options.NoCopy && !options.RawLeaves { // fixed? if options.RawLeavesSet { return nil, cid.Prefix{}, fmt.Errorf("nocopy option requires '--raw-leaves' to be enabled as well") } // No, satisfy mandatory constraint. options.RawLeaves = true } // (hash != "sha2-256") -> CIDv1 if options.MhType != mh.SHA2_256 { switch options.CidVersion { case 0: return nil, cid.Prefix{}, errors.New("CIDv0 only supports sha2-256") case 1, -1: options.CidVersion = 1 default: return nil, cid.Prefix{}, fmt.Errorf("unknown CID version: %d", options.CidVersion) } } else { if options.CidVersion < 0 { // Default to CIDv0 options.CidVersion = 0 } } if !options.Mtime.IsZero() && options.PreserveMtime { options.PreserveMtime = false } if options.Mode != 0 && options.PreserveMode { options.PreserveMode = false } // cidV1 -> raw blocks (by default) if options.CidVersion > 0 && !options.RawLeavesSet { options.RawLeaves = true } prefix, err := dag.PrefixForCidVersion(options.CidVersion) if err != nil { return nil, cid.Prefix{}, err } prefix.MhType = options.MhType prefix.MhLength = -1 return options, prefix, nil } func UnixfsLsOptions(opts ...UnixfsLsOption) (*UnixfsLsSettings, error) { options := &UnixfsLsSettings{ ResolveChildren: true, } for _, opt := range opts { err := opt(options) if err != nil { return nil, err } } return options, nil } type unixfsOpts struct{} var Unixfs unixfsOpts // CidVersion specifies which CID version to use. Defaults to 0 unless an option // that depends on CIDv1 is passed. func (unixfsOpts) CidVersion(version int) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.CidVersion = version return nil } } // Hash function to use. Implies CIDv1 if not set to sha2-256 (default). // // Table of functions is declared in https://github.com/multiformats/go-multihash/blob/master/multihash.go func (unixfsOpts) Hash(mhtype uint64) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.MhType = mhtype return nil } } // RawLeaves specifies whether to use raw blocks for leaves (data nodes with no // links) instead of wrapping them with unixfs structures. func (unixfsOpts) RawLeaves(enable bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.RawLeaves = enable settings.RawLeavesSet = true return nil } } // MaxFileLinks specifies the maximum number of children for UnixFS file // nodes. func (unixfsOpts) MaxFileLinks(n int) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.MaxFileLinks = n settings.MaxFileLinksSet = true return nil } } // MaxDirectoryLinks specifies the maximum number of children for UnixFS basic // directory nodes. func (unixfsOpts) MaxDirectoryLinks(n int) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.MaxDirectoryLinks = n settings.MaxDirectoryLinksSet = true return nil } } // MaxHAMTFanout specifies the maximum width of the HAMT directory shards. // Per the UnixFS spec, the value must be a power of 2, minimum 8 // (for byte-aligned bitfields), and maximum 1024. func (unixfsOpts) MaxHAMTFanout(n int) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { if n < 8 || n&(n-1) != 0 || n > 1024 { return fmt.Errorf("HAMT fanout must be a power of 2, between 8 and 1024 (got %d)", n) } settings.MaxHAMTFanout = n settings.MaxHAMTFanoutSet = true return nil } } // SizeEstimationMode specifies how directory size is estimated for HAMT sharding decisions. func (unixfsOpts) SizeEstimationMode(mode io.SizeEstimationMode) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.SizeEstimationMode = &mode settings.SizeEstimationModeSet = true return nil } } // Inline tells the adder to inline small blocks into CIDs func (unixfsOpts) Inline(enable bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.Inline = enable return nil } } // InlineLimit sets the amount of bytes below which blocks will be encoded // directly into CID instead of being stored and addressed by it's hash. // Specifying this option won't enable block inlining. For that use `Inline` // option. Default: 32 bytes // // Note that while there is no hard limit on the number of bytes, it should be // kept at a reasonably low value, such as 64; implementations may choose to // reject anything larger. func (unixfsOpts) InlineLimit(limit int) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.InlineLimit = limit return nil } } // Chunker specifies settings for the chunking algorithm to use. // // Default: size-262144, formats: // size-[bytes] - Simple chunker splitting data into blocks of n bytes // rabin-[min]-[avg]-[max] - Rabin chunker func (unixfsOpts) Chunker(chunker string) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.Chunker = chunker return nil } } // Layout tells the adder how to balance data between leaves. // options.BalancedLayout is the default, it's optimized for static seekable // files. // options.TrickleLayout is optimized for streaming data, func (unixfsOpts) Layout(layout Layout) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.Layout = layout return nil } } // Pin tells the adder to pin the file root recursively after adding func (unixfsOpts) Pin(pin bool, pinName string) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.Pin = pin if pin { settings.PinName = pinName } return nil } } // HashOnly will make the adder calculate data hash without storing it in the // blockstore or announcing it to the network func (unixfsOpts) HashOnly(hashOnly bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.OnlyHash = hashOnly return nil } } // Events specifies channel which will be used to report events about ongoing // Add operation. // // Note that if this channel blocks it may slowdown the adder func (unixfsOpts) Events(sink chan<- any) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.Events = sink return nil } } // Silent reduces event output func (unixfsOpts) Silent(silent bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.Silent = silent return nil } } // Progress tells the adder whether to enable progress events func (unixfsOpts) Progress(enable bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.Progress = enable return nil } } // FsCache tells the adder to check the filestore for pre-existing blocks // // Experimental func (unixfsOpts) FsCache(enable bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.FsCache = enable return nil } } // NoCopy tells the adder to add the files using filestore. Implies RawLeaves. // // Experimental func (unixfsOpts) Nocopy(enable bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.NoCopy = enable return nil } } func (unixfsOpts) ResolveChildren(resolve bool) UnixfsLsOption { return func(settings *UnixfsLsSettings) error { settings.ResolveChildren = resolve return nil } } func (unixfsOpts) UseCumulativeSize(use bool) UnixfsLsOption { return func(settings *UnixfsLsSettings) error { settings.UseCumulativeSize = use return nil } } // PreserveMode tells the adder to store the file permissions func (unixfsOpts) PreserveMode(enable bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.PreserveMode = enable return nil } } // PreserveMtime tells the adder to store the file modification time func (unixfsOpts) PreserveMtime(enable bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.PreserveMtime = enable return nil } } // Mode represents a unix file mode func (unixfsOpts) Mode(mode os.FileMode) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.Mode = mode return nil } } // Mtime represents a unix file mtime func (unixfsOpts) Mtime(seconds int64, nsecs uint32) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { if nsecs > 999999999 { return errors.New("mtime nanoseconds must be in range [1, 999999999]") } settings.Mtime = time.Unix(seconds, int64(nsecs)) return nil } } // IncludeEmptyDirs tells the adder to include empty directories in the DAG func (unixfsOpts) IncludeEmptyDirs(include bool) UnixfsAddOption { return func(settings *UnixfsAddSettings) error { settings.IncludeEmptyDirs = include settings.IncludeEmptyDirsSet = true return nil } } ================================================ FILE: core/coreiface/options/unixfs_test.go ================================================ package options import ( "testing" "github.com/stretchr/testify/require" ) func TestMaxHAMTFanoutValidation(t *testing.T) { valid := []int{8, 16, 32, 64, 128, 256, 512, 1024} for _, v := range valid { _, _, err := UnixfsAddOptions(Unixfs.MaxHAMTFanout(v)) require.NoError(t, err, "fanout %d should be valid", v) } invalid := []int{-1, 0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 12, 24, 48, 100, 2048, 4096, 999999} for _, v := range invalid { _, _, err := UnixfsAddOptions(Unixfs.MaxHAMTFanout(v)) require.Error(t, err, "fanout %d should be invalid", v) require.Contains(t, err.Error(), "HAMT fanout must be") } } ================================================ FILE: core/coreiface/pin.go ================================================ package iface import ( "context" "github.com/ipfs/boxo/path" "github.com/ipfs/kubo/core/coreiface/options" ) // Pin holds information about pinned resource type Pin interface { // Path to the pinned object Path() path.ImmutablePath // Name is the name of the pin. Name() string // Type of the pin Type() string } // PinStatus holds information about pin health type PinStatus interface { // Ok indicates whether the pin has been verified to be correct Ok() bool // BadNodes returns any bad (usually missing) nodes from the pin BadNodes() []BadPinNode // if not nil, an error happened. Everything else should be ignored. Err() error } // BadPinNode is a node that has been marked as bad by Pin.Verify type BadPinNode interface { // Path is the path of the node Path() path.ImmutablePath // Err is the reason why the node has been marked as bad Err() error } // PinAPI specifies the interface to pining type PinAPI interface { // Add creates new pin, be default recursive - pinning the whole referenced // tree Add(context.Context, path.Path, ...options.PinAddOption) error // Ls returns this node's pinned objects on the provided channel. The // channel is closed when there are no more pins and an error is returned. Ls(context.Context, chan<- Pin, ...options.PinLsOption) error // IsPinned returns whether or not the given cid is pinned // and an explanation of why its pinned IsPinned(context.Context, path.Path, ...options.PinIsPinnedOption) (string, bool, error) // Rm removes pin for object specified by the path Rm(context.Context, path.Path, ...options.PinRmOption) error // Update changes one pin to another, skipping checks for matching paths in // the old tree Update(ctx context.Context, from path.Path, to path.Path, opts ...options.PinUpdateOption) error // Verify verifies the integrity of pinned objects Verify(context.Context) (<-chan PinStatus, error) } ================================================ FILE: core/coreiface/pubsub.go ================================================ package iface import ( "context" "io" "github.com/ipfs/kubo/core/coreiface/options" "github.com/libp2p/go-libp2p/core/peer" ) // PubSubSubscription is an active PubSub subscription type PubSubSubscription interface { io.Closer // Next return the next incoming message Next(context.Context) (PubSubMessage, error) } // PubSubMessage is a single PubSub message type PubSubMessage interface { // From returns id of a peer from which the message has arrived From() peer.ID // Data returns the message body Data() []byte // Seq returns message identifier Seq() []byte // Topics returns list of topics this message was set to Topics() []string } // PubSubAPI specifies the interface to PubSub type PubSubAPI interface { // Ls lists subscribed topics by name Ls(context.Context) ([]string, error) // Peers list peers we are currently pubsubbing with Peers(context.Context, ...options.PubSubPeersOption) ([]peer.ID, error) // Publish a message to a given pubsub topic Publish(context.Context, string, []byte) error // Subscribe to messages on a given topic Subscribe(context.Context, string, ...options.PubSubSubscribeOption) (PubSubSubscription, error) } ================================================ FILE: core/coreiface/routing.go ================================================ package iface import ( "context" "github.com/ipfs/boxo/path" "github.com/ipfs/kubo/core/coreiface/options" "github.com/libp2p/go-libp2p/core/peer" ) // RoutingAPI specifies the interface to the routing layer. type RoutingAPI interface { // Get retrieves the best value for a given key Get(context.Context, string) ([]byte, error) // Put sets a value for a given key Put(ctx context.Context, key string, value []byte, opts ...options.RoutingPutOption) error // FindPeer queries the routing system for all the multiaddresses associated // with the given [peer.ID]. FindPeer(context.Context, peer.ID) (peer.AddrInfo, error) // FindProviders finds the peers in the routing system who can provide a specific // value given a key. FindProviders(context.Context, path.Path, ...options.RoutingFindProvidersOption) (<-chan peer.AddrInfo, error) // Provide announces to the network that you are providing given values Provide(context.Context, path.Path, ...options.RoutingProvideOption) error } ================================================ FILE: core/coreiface/swarm.go ================================================ package iface import ( "context" "errors" "time" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" ma "github.com/multiformats/go-multiaddr" ) var ( ErrNotConnected = errors.New("not connected") ErrConnNotFound = errors.New("conn not found") ) // ConnectionInfo contains information about a peer type ConnectionInfo interface { // ID returns PeerID ID() peer.ID // Address returns the multiaddress via which we are connected with the peer Address() ma.Multiaddr // Direction returns which way the connection was established Direction() network.Direction // Latency returns last known round trip time to the peer Latency() (time.Duration, error) // Streams returns list of streams established with the peer Streams() ([]protocol.ID, error) } // SwarmAPI specifies the interface to libp2p swarm type SwarmAPI interface { // Connect to a given peer Connect(context.Context, peer.AddrInfo) error // Disconnect from a given address Disconnect(context.Context, ma.Multiaddr) error // Peers returns the list of peers we are connected to Peers(context.Context) ([]ConnectionInfo, error) // KnownAddrs returns the list of all addresses this node is aware of KnownAddrs(context.Context) (map[peer.ID][]ma.Multiaddr, error) // LocalAddrs returns the list of announced listening addresses LocalAddrs(context.Context) ([]ma.Multiaddr, error) // ListenAddrs returns the list of all listening addresses ListenAddrs(context.Context) ([]ma.Multiaddr, error) } ================================================ FILE: core/coreiface/tests/api.go ================================================ package tests import ( "context" "errors" "testing" "time" coreiface "github.com/ipfs/kubo/core/coreiface" ) var errAPINotImplemented = errors.New("api not implemented") type Provider interface { // Make creates n nodes. fullIdentity set to false can be ignored MakeAPISwarm(t *testing.T, ctx context.Context, fullIdentity bool, online bool, n int) ([]coreiface.CoreAPI, error) } func (tp *TestSuite) makeAPISwarm(t *testing.T, ctx context.Context, fullIdentity bool, online bool, n int) ([]coreiface.CoreAPI, error) { if tp.apis != nil { tp.apis <- 1 go func() { <-ctx.Done() tp.apis <- -1 }() } return tp.Provider.MakeAPISwarm(t, ctx, fullIdentity, online, n) } func (tp *TestSuite) makeAPI(t *testing.T, ctx context.Context) (coreiface.CoreAPI, error) { api, err := tp.makeAPISwarm(t, ctx, false, false, 1) if err != nil { return nil, err } return api[0], nil } func (tp *TestSuite) makeAPIWithIdentityAndOffline(t *testing.T, ctx context.Context) (coreiface.CoreAPI, error) { api, err := tp.makeAPISwarm(t, ctx, true, false, 1) if err != nil { return nil, err } return api[0], nil } func (tp *TestSuite) MakeAPISwarm(t *testing.T, ctx context.Context, n int) ([]coreiface.CoreAPI, error) { return tp.makeAPISwarm(t, ctx, true, true, n) } type TestSuite struct { Provider apis chan int } func TestApi(p Provider) func(t *testing.T) { running := 1 apis := make(chan int) zeroRunning := make(chan struct{}) go func() { for i := range apis { running += i if running < 1 { close(zeroRunning) return } } }() tp := &TestSuite{Provider: p, apis: apis} return func(t *testing.T) { t.Run("Block", tp.TestBlock) t.Run("Dag", tp.TestDag) t.Run("Key", tp.TestKey) t.Run("Name", tp.TestName) t.Run("Object", tp.TestObject) t.Run("Path", tp.TestPath) t.Run("Pin", tp.TestPin) t.Run("PubSub", tp.TestPubSub) t.Run("Routing", tp.TestRouting) t.Run("Unixfs", tp.TestUnixfs) apis <- -1 t.Run("TestsCancelCtx", func(t *testing.T) { select { case <-zeroRunning: case <-time.After(time.Second): t.Errorf("%d test swarms(s) not closed", running) } }) } } func (tp *TestSuite) hasApi(t *testing.T, tf func(coreiface.CoreAPI) error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } if err := tf(api); err != nil { t.Fatal(api) } } ================================================ FILE: core/coreiface/tests/block.go ================================================ package tests import ( "bytes" "io" "strings" "testing" "github.com/ipfs/boxo/path" ipld "github.com/ipfs/go-ipld-format" coreiface "github.com/ipfs/kubo/core/coreiface" opt "github.com/ipfs/kubo/core/coreiface/options" mh "github.com/multiformats/go-multihash" ) var ( pbCidV0 = "QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN" // dag-pb pbCid = "bafybeiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4" // dag-pb rawCid = "bafkreiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4" // raw bytes cborCid = "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm" // dag-cbor cborKCid = "bafyr2qgsohbwdlk7ajmmbb4lhoytmest4wdbe5xnexfvtxeatuyqqmwv3fgxp3pmhpc27gwey2cct56gloqefoqwcf3yqiqzsaqb7p4jefhcw" // dag-cbor keccak-512 ) // dag-pb func pbBlock() io.Reader { return bytes.NewReader([]byte{10, 12, 8, 2, 18, 6, 104, 101, 108, 108, 111, 10, 24, 6}) } // dag-cbor func cborBlock() io.Reader { return bytes.NewReader([]byte{101, 72, 101, 108, 108, 111}) } func (tp *TestSuite) TestBlock(t *testing.T) { tp.hasApi(t, func(api coreiface.CoreAPI) error { if api.Block() == nil { return errAPINotImplemented } return nil }) t.Run("TestBlockPut (get raw CIDv1)", tp.TestBlockPut) t.Run("TestBlockPutCidCodec: dag-pb", tp.TestBlockPutCidCodecDagPb) t.Run("TestBlockPutCidCodec: dag-cbor", tp.TestBlockPutCidCodecDagCbor) t.Run("TestBlockPutFormat (legacy): cbor → dag-cbor", tp.TestBlockPutFormatDagCbor) t.Run("TestBlockPutFormat (legacy): protobuf → dag-pb", tp.TestBlockPutFormatDagPb) t.Run("TestBlockPutFormat (legacy): v0 → CIDv0", tp.TestBlockPutFormatV0) t.Run("TestBlockPutHash", tp.TestBlockPutHash) t.Run("TestBlockGet", tp.TestBlockGet) t.Run("TestBlockRm", tp.TestBlockRm) t.Run("TestBlockStat", tp.TestBlockStat) t.Run("TestBlockPin", tp.TestBlockPin) } // when no opts are passed, produced CID has 'raw' codec func (tp *TestSuite) TestBlockPut(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } res, err := api.Block().Put(ctx, pbBlock()) if err != nil { t.Fatal(err) } if res.Path().RootCid().String() != rawCid { t.Errorf("got wrong cid: %s", res.Path().RootCid().String()) } } // Format is deprecated, it used invalid codec names. // Confirm 'cbor' gets fixed to 'dag-cbor' func (tp *TestSuite) TestBlockPutFormatDagCbor(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } res, err := api.Block().Put(ctx, cborBlock(), opt.Block.Format("cbor")) if err != nil { t.Fatal(err) } if res.Path().RootCid().String() != cborCid { t.Errorf("got wrong cid: %s", res.Path().RootCid().String()) } } // Format is deprecated, it used invalid codec names. // Confirm 'protobuf' got fixed to 'dag-pb' func (tp *TestSuite) TestBlockPutFormatDagPb(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } res, err := api.Block().Put(ctx, pbBlock(), opt.Block.Format("protobuf")) if err != nil { t.Fatal(err) } if res.Path().RootCid().String() != pbCid { t.Errorf("got wrong cid: %s", res.Path().RootCid().String()) } } // Format is deprecated, it used invalid codec names. // Confirm fake codec 'v0' got fixed to CIDv0 (with implicit dag-pb codec) func (tp *TestSuite) TestBlockPutFormatV0(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } res, err := api.Block().Put(ctx, pbBlock(), opt.Block.Format("v0")) if err != nil { t.Fatal(err) } if res.Path().RootCid().String() != pbCidV0 { t.Errorf("got wrong cid: %s", res.Path().RootCid().String()) } } func (tp *TestSuite) TestBlockPutCidCodecDagCbor(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } res, err := api.Block().Put(ctx, cborBlock(), opt.Block.CidCodec("dag-cbor")) if err != nil { t.Fatal(err) } if res.Path().RootCid().String() != cborCid { t.Errorf("got wrong cid: %s", res.Path().RootCid().String()) } } func (tp *TestSuite) TestBlockPutCidCodecDagPb(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } res, err := api.Block().Put(ctx, pbBlock(), opt.Block.CidCodec("dag-pb")) if err != nil { t.Fatal(err) } if res.Path().RootCid().String() != pbCid { t.Errorf("got wrong cid: %s", res.Path().RootCid().String()) } } func (tp *TestSuite) TestBlockPutHash(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } res, err := api.Block().Put( ctx, cborBlock(), opt.Block.Hash(mh.KECCAK_512, -1), opt.Block.CidCodec("dag-cbor"), ) if err != nil { t.Fatal(err) } if res.Path().RootCid().String() != cborKCid { t.Errorf("got wrong cid: %s", res.Path().RootCid().String()) } } func (tp *TestSuite) TestBlockGet(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } res, err := api.Block().Put(ctx, strings.NewReader(`Hello`), opt.Block.Format("raw")) if err != nil { t.Fatal(err) } r, err := api.Block().Get(ctx, res.Path()) if err != nil { t.Fatal(err) } d, err := io.ReadAll(r) if err != nil { t.Fatal(err) } if string(d) != "Hello" { t.Error("didn't get correct data back") } p := path.FromCid(res.Path().RootCid()) rp, _, err := api.ResolvePath(ctx, p) if err != nil { t.Fatal(err) } if rp.RootCid().String() != res.Path().RootCid().String() { t.Error("paths didn't match") } } func (tp *TestSuite) TestBlockRm(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } res, err := api.Block().Put(ctx, strings.NewReader(`Hello`), opt.Block.Format("raw")) if err != nil { t.Fatal(err) } r, err := api.Block().Get(ctx, res.Path()) if err != nil { t.Fatal(err) } d, err := io.ReadAll(r) if err != nil { t.Fatal(err) } if string(d) != "Hello" { t.Error("didn't get correct data back") } err = api.Block().Rm(ctx, res.Path()) if err != nil { t.Fatal(err) } _, err = api.Block().Get(ctx, res.Path()) if err == nil { t.Fatal("expected err to exist") } if !ipld.IsNotFound(err) { t.Errorf("unexpected error; %s", err.Error()) } err = api.Block().Rm(ctx, res.Path()) if err == nil { t.Fatal("expected err to exist") } if !ipld.IsNotFound(err) { t.Errorf("unexpected error; %s", err.Error()) } err = api.Block().Rm(ctx, res.Path(), opt.Block.Force(true)) if err != nil { t.Fatal(err) } } func (tp *TestSuite) TestBlockStat(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } res, err := api.Block().Put(ctx, strings.NewReader(`Hello`), opt.Block.Format("raw")) if err != nil { t.Fatal(err) } stat, err := api.Block().Stat(ctx, res.Path()) if err != nil { t.Fatal(err) } if stat.Path().String() != res.Path().String() { t.Error("paths don't match") } if stat.Size() != len("Hello") { t.Error("length doesn't match") } } func (tp *TestSuite) TestBlockPin(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } _, err = api.Block().Put(ctx, strings.NewReader(`Hello`), opt.Block.Format("raw")) if err != nil { t.Fatal(err) } pinCh := make(chan coreiface.Pin) go func() { err = api.Pin().Ls(ctx, pinCh) }() for range pinCh { t.Fatal("expected 0 pins") } if err != nil { t.Fatal(err) } res, err := api.Block().Put( ctx, strings.NewReader(`Hello`), opt.Block.Pin(true), opt.Block.Format("raw"), ) if err != nil { t.Fatal(err) } pins, err := accPins(ctx, api) if err != nil { t.Fatal(err) } if len(pins) != 1 { t.Fatal("expected 1 pin") } if pins[0].Type() != "recursive" { t.Error("expected a recursive pin") } if pins[0].Path().String() != res.Path().String() { t.Error("pin path didn't match") } } ================================================ FILE: core/coreiface/tests/dag.go ================================================ package tests import ( "math" "strings" "testing" "github.com/ipfs/boxo/path" coreiface "github.com/ipfs/kubo/core/coreiface" ipldcbor "github.com/ipfs/go-ipld-cbor" ipld "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" ) func (tp *TestSuite) TestDag(t *testing.T) { tp.hasApi(t, func(api coreiface.CoreAPI) error { if api.Dag() == nil { return errAPINotImplemented } return nil }) t.Run("TestPut", tp.TestPut) t.Run("TestPutWithHash", tp.TestPutWithHash) t.Run("TestPath", tp.TestDagPath) t.Run("TestTree", tp.TestTree) t.Run("TestBatch", tp.TestBatch) } var treeExpected = map[string]struct{}{ "a": {}, "b": {}, "c": {}, "c/d": {}, "c/e": {}, } func (tp *TestSuite) TestPut(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } nd, err := ipldcbor.FromJSON(strings.NewReader(`"Hello"`), math.MaxUint64, -1) if err != nil { t.Fatal(err) } err = api.Dag().Add(ctx, nd) if err != nil { t.Fatal(err) } if nd.Cid().String() != "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm" { t.Errorf("got wrong cid: %s", nd.Cid().String()) } } func (tp *TestSuite) TestPutWithHash(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } nd, err := ipldcbor.FromJSON(strings.NewReader(`"Hello"`), mh.SHA3_256, -1) if err != nil { t.Fatal(err) } err = api.Dag().Add(ctx, nd) if err != nil { t.Fatal(err) } if nd.Cid().String() != "bafyrmifu7haikttpqqgc5ewvmp76z3z4ebp7h2ph4memw7dq4nt6btmxny" { t.Errorf("got wrong cid: %s", nd.Cid().String()) } } func (tp *TestSuite) TestDagPath(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } snd, err := ipldcbor.FromJSON(strings.NewReader(`"foo"`), math.MaxUint64, -1) if err != nil { t.Fatal(err) } err = api.Dag().Add(ctx, snd) if err != nil { t.Fatal(err) } nd, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+snd.Cid().String()+`"}}`), math.MaxUint64, -1) if err != nil { t.Fatal(err) } err = api.Dag().Add(ctx, nd) if err != nil { t.Fatal(err) } p, err := path.Join(path.FromCid(nd.Cid()), "lnk") if err != nil { t.Fatal(err) } rp, _, err := api.ResolvePath(ctx, p) if err != nil { t.Fatal(err) } ndd, err := api.Dag().Get(ctx, rp.RootCid()) if err != nil { t.Fatal(err) } if ndd.Cid().String() != snd.Cid().String() { t.Errorf("got unexpected cid %s, expected %s", ndd.Cid().String(), snd.Cid().String()) } } func (tp *TestSuite) TestTree(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } nd, err := ipldcbor.FromJSON(strings.NewReader(`{"a": 123, "b": "foo", "c": {"d": 321, "e": 111}}`), math.MaxUint64, -1) if err != nil { t.Fatal(err) } err = api.Dag().Add(ctx, nd) if err != nil { t.Fatal(err) } res, err := api.Dag().Get(ctx, nd.Cid()) if err != nil { t.Fatal(err) } lst := res.Tree("", -1) if len(lst) != len(treeExpected) { t.Errorf("tree length of %d doesn't match expected %d", len(lst), len(treeExpected)) } for _, ent := range lst { if _, ok := treeExpected[ent]; !ok { t.Errorf("unexpected tree entry %s", ent) } } } func (tp *TestSuite) TestBatch(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } nd, err := ipldcbor.FromJSON(strings.NewReader(`"Hello"`), math.MaxUint64, -1) if err != nil { t.Fatal(err) } if nd.Cid().String() != "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm" { t.Errorf("got wrong cid: %s", nd.Cid().String()) } _, err = api.Dag().Get(ctx, nd.Cid()) if err == nil || !strings.Contains(err.Error(), "not found") { t.Fatal(err) } if err := api.Dag().AddMany(ctx, []ipld.Node{nd}); err != nil { t.Fatal(err) } _, err = api.Dag().Get(ctx, nd.Cid()) if err != nil { t.Fatal(err) } } ================================================ FILE: core/coreiface/tests/key.go ================================================ package tests import ( "strings" "testing" "github.com/ipfs/boxo/ipns" "github.com/ipfs/go-cid" iface "github.com/ipfs/kubo/core/coreiface" opt "github.com/ipfs/kubo/core/coreiface/options" "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func (tp *TestSuite) TestKey(t *testing.T) { tp.hasApi(t, func(api iface.CoreAPI) error { if api.Key() == nil { return errAPINotImplemented } return nil }) t.Run("TestListSelf", tp.TestListSelf) t.Run("TestRenameSelf", tp.TestRenameSelf) t.Run("TestRemoveSelf", tp.TestRemoveSelf) t.Run("TestGenerate", tp.TestGenerate) t.Run("TestGenerateSize", tp.TestGenerateSize) t.Run("TestGenerateType", tp.TestGenerateType) t.Run("TestGenerateExisting", tp.TestGenerateExisting) t.Run("TestList", tp.TestList) t.Run("TestRename", tp.TestRename) t.Run("TestRenameToSelf", tp.TestRenameToSelf) t.Run("TestRenameToSelfForce", tp.TestRenameToSelfForce) t.Run("TestRenameOverwriteNoForce", tp.TestRenameOverwriteNoForce) t.Run("TestRenameOverwrite", tp.TestRenameOverwrite) t.Run("TestRenameSameNameNoForce", tp.TestRenameSameNameNoForce) t.Run("TestRenameSameName", tp.TestRenameSameName) t.Run("TestSign", tp.TestSign) t.Run("TestVerify", tp.TestVerify) } func (tp *TestSuite) TestListSelf(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) self, err := api.Key().Self(ctx) require.NoError(t, err) keys, err := api.Key().List(ctx) require.NoError(t, err) require.Len(t, keys, 1) assert.Equal(t, "self", keys[0].Name()) assert.Equal(t, "/ipns/"+iface.FormatKeyID(self.ID()), keys[0].Path().String()) } func (tp *TestSuite) TestRenameSelf(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) _, _, err = api.Key().Rename(ctx, "self", "foo") require.ErrorContains(t, err, "cannot rename key with name 'self'") _, _, err = api.Key().Rename(ctx, "self", "foo", opt.Key.Force(true)) require.ErrorContains(t, err, "cannot rename key with name 'self'") } func (tp *TestSuite) TestRemoveSelf(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) _, err = api.Key().Remove(ctx, "self") require.ErrorContains(t, err, "cannot remove key with name 'self'") } func (tp *TestSuite) TestGenerate(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) k, err := api.Key().Generate(ctx, "foo") require.NoError(t, err) require.Equal(t, "foo", k.Name()) verifyIPNSPath(t, k.Path().String()) } func verifyIPNSPath(t *testing.T, p string) { t.Helper() require.True(t, strings.HasPrefix(p, "/ipns/")) k := p[len("/ipns/"):] c, err := cid.Decode(k) require.NoError(t, err) b36, err := c.StringOfBase(mbase.Base36) require.NoError(t, err) require.Equal(t, k, b36) } func (tp *TestSuite) TestGenerateSize(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) k, err := api.Key().Generate(ctx, "foo", opt.Key.Size(2048)) require.NoError(t, err) require.Equal(t, "foo", k.Name()) verifyIPNSPath(t, k.Path().String()) } func (tp *TestSuite) TestGenerateType(t *testing.T) { t.Skip("disabled until libp2p/specs#111 is fixed") ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) k, err := api.Key().Generate(ctx, "bar", opt.Key.Type(opt.Ed25519Key)) require.NoError(t, err) require.Equal(t, "bar", k.Name()) // Expected to be an inlined identity hash. require.True(t, strings.HasPrefix(k.Path().String(), "/ipns/12")) } func (tp *TestSuite) TestGenerateExisting(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) _, err = api.Key().Generate(ctx, "foo") require.NoError(t, err) _, err = api.Key().Generate(ctx, "foo") require.ErrorContains(t, err, "key with name 'foo' already exists") _, err = api.Key().Generate(ctx, "self") require.ErrorContains(t, err, "cannot create key with name 'self'") } func (tp *TestSuite) TestList(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) _, err = api.Key().Generate(ctx, "foo") require.NoError(t, err) l, err := api.Key().List(ctx) require.NoError(t, err) require.Len(t, l, 2) require.Equal(t, "self", l[0].Name()) require.Equal(t, "foo", l[1].Name()) verifyIPNSPath(t, l[0].Path().String()) verifyIPNSPath(t, l[1].Path().String()) } func (tp *TestSuite) TestRename(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) _, err = api.Key().Generate(ctx, "foo") require.NoError(t, err) k, overwrote, err := api.Key().Rename(ctx, "foo", "bar") require.NoError(t, err) assert.False(t, overwrote) assert.Equal(t, "bar", k.Name()) } func (tp *TestSuite) TestRenameToSelf(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) _, err = api.Key().Generate(ctx, "foo") require.NoError(t, err) _, _, err = api.Key().Rename(ctx, "foo", "self") require.ErrorContains(t, err, "cannot overwrite key with name 'self'") } func (tp *TestSuite) TestRenameToSelfForce(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) _, err = api.Key().Generate(ctx, "foo") require.NoError(t, err) _, _, err = api.Key().Rename(ctx, "foo", "self", opt.Key.Force(true)) require.ErrorContains(t, err, "cannot overwrite key with name 'self'") } func (tp *TestSuite) TestRenameOverwriteNoForce(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) _, err = api.Key().Generate(ctx, "foo") require.NoError(t, err) _, err = api.Key().Generate(ctx, "bar") require.NoError(t, err) _, _, err = api.Key().Rename(ctx, "foo", "bar") require.ErrorContains(t, err, "key by that name already exists, refusing to overwrite") } func (tp *TestSuite) TestRenameOverwrite(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) kfoo, err := api.Key().Generate(ctx, "foo") require.NoError(t, err) _, err = api.Key().Generate(ctx, "bar") require.NoError(t, err) k, overwrote, err := api.Key().Rename(ctx, "foo", "bar", opt.Key.Force(true)) require.NoError(t, err) require.True(t, overwrote) assert.Equal(t, "bar", k.Name()) assert.Equal(t, kfoo.Path().String(), k.Path().String()) } func (tp *TestSuite) TestRenameSameNameNoForce(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) _, err = api.Key().Generate(ctx, "foo") require.NoError(t, err) k, overwrote, err := api.Key().Rename(ctx, "foo", "foo") require.NoError(t, err) assert.False(t, overwrote) assert.Equal(t, "foo", k.Name()) } func (tp *TestSuite) TestRenameSameName(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) _, err = api.Key().Generate(ctx, "foo") require.NoError(t, err) k, overwrote, err := api.Key().Rename(ctx, "foo", "foo", opt.Key.Force(true)) require.NoError(t, err) assert.False(t, overwrote) assert.Equal(t, "foo", k.Name()) } func (tp *TestSuite) TestRemove(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) k, err := api.Key().Generate(ctx, "foo") require.NoError(t, err) l, err := api.Key().List(ctx) require.NoError(t, err) require.Len(t, l, 2) p, err := api.Key().Remove(ctx, "foo") require.NoError(t, err) assert.Equal(t, p.Path().String(), k.Path().String()) l, err = api.Key().List(ctx) require.NoError(t, err) require.Len(t, l, 1) assert.Equal(t, "self", l[0].Name()) } func (tp *TestSuite) TestSign(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) key1, err := api.Key().Generate(ctx, "foo", opt.Key.Type(opt.Ed25519Key)) require.NoError(t, err) data := []byte("hello world") key2, signature, err := api.Key().Sign(ctx, "foo", data) require.NoError(t, err) require.Equal(t, key1.Name(), key2.Name()) require.Equal(t, key1.ID(), key2.ID()) pk, err := key1.ID().ExtractPublicKey() require.NoError(t, err) valid, err := pk.Verify(append([]byte("libp2p-key signed message:"), data...), signature) require.NoError(t, err) require.True(t, valid) } func (tp *TestSuite) TestVerify(t *testing.T) { t.Parallel() t.Run("Verify Own Key", func(t *testing.T) { t.Parallel() ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) _, err = api.Key().Generate(ctx, "foo", opt.Key.Type(opt.Ed25519Key)) require.NoError(t, err) data := []byte("hello world") _, signature, err := api.Key().Sign(ctx, "foo", data) require.NoError(t, err) _, valid, err := api.Key().Verify(ctx, "foo", signature, data) require.NoError(t, err) require.True(t, valid) }) t.Run("Verify Self", func(t *testing.T) { t.Parallel() ctx := t.Context() api, err := tp.makeAPIWithIdentityAndOffline(t, ctx) require.NoError(t, err) data := []byte("hello world") _, signature, err := api.Key().Sign(ctx, "", data) require.NoError(t, err) _, valid, err := api.Key().Verify(ctx, "", signature, data) require.NoError(t, err) require.True(t, valid) }) t.Run("Verify With Key In Different Formats", func(t *testing.T) { t.Parallel() // Spin some node and get signature out. ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) key, err := api.Key().Generate(ctx, "foo", opt.Key.Type(opt.Ed25519Key)) require.NoError(t, err) data := []byte("hello world") _, signature, err := api.Key().Sign(ctx, "foo", data) require.NoError(t, err) for _, testCase := range [][]string{ {"Base58 Encoded Peer ID", key.ID().String()}, {"CIDv1 Encoded Peer ID", peer.ToCid(key.ID()).String()}, {"CIDv1 Encoded IPNS Name", ipns.NameFromPeer(key.ID()).String()}, {"Prefixed IPNS Path", ipns.NameFromPeer(key.ID()).AsPath().String()}, } { t.Run(testCase[0], func(t *testing.T) { ctx := t.Context() // Spin new node. api, err := tp.makeAPI(t, ctx) require.NoError(t, err) _, valid, err := api.Key().Verify(ctx, testCase[1], signature, data) require.NoError(t, err) require.True(t, valid) }) } }) } ================================================ FILE: core/coreiface/tests/name.go ================================================ package tests import ( "context" "io" "math/rand" "testing" "time" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" coreiface "github.com/ipfs/kubo/core/coreiface" opt "github.com/ipfs/kubo/core/coreiface/options" "github.com/stretchr/testify/require" ) func (tp *TestSuite) TestName(t *testing.T) { tp.hasApi(t, func(api coreiface.CoreAPI) error { if api.Name() == nil { return errAPINotImplemented } return nil }) t.Run("TestPublishResolve", tp.TestPublishResolve) t.Run("TestBasicPublishResolveKey", tp.TestBasicPublishResolveKey) t.Run("TestBasicPublishResolveTimeout", tp.TestBasicPublishResolveTimeout) } var rnd = rand.New(rand.NewSource(0x62796532303137)) func addTestObject(ctx context.Context, api coreiface.CoreAPI) (path.Path, error) { return api.Unixfs().Add(ctx, files.NewReaderFile(&io.LimitedReader{R: rnd, N: 4092})) } func (tp *TestSuite) TestPublishResolve(t *testing.T) { ctx := t.Context() init := func() (coreiface.CoreAPI, path.Path) { apis, err := tp.MakeAPISwarm(t, ctx, 5) require.NoError(t, err) api := apis[0] p, err := addTestObject(ctx, api) require.NoError(t, err) return api, p } run := func(t *testing.T, ropts []opt.NameResolveOption) { t.Run("basic", func(t *testing.T) { api, p := init() name, err := api.Name().Publish(ctx, p) require.NoError(t, err) self, err := api.Key().Self(ctx) require.NoError(t, err) require.Equal(t, name.String(), ipns.NameFromPeer(self.ID()).String()) resPath, err := api.Name().Resolve(ctx, name.String(), ropts...) require.NoError(t, err) require.Equal(t, p.String(), resPath.String()) }) t.Run("publishPath", func(t *testing.T) { api, p := init() p, err := path.Join(p, "/test") require.NoError(t, err) name, err := api.Name().Publish(ctx, p) require.NoError(t, err) self, err := api.Key().Self(ctx) require.NoError(t, err) require.Equal(t, name.String(), ipns.NameFromPeer(self.ID()).String()) resPath, err := api.Name().Resolve(ctx, name.String(), ropts...) require.NoError(t, err) require.Equal(t, p.String(), resPath.String()) }) t.Run("revolvePath", func(t *testing.T) { api, p := init() name, err := api.Name().Publish(ctx, p) require.NoError(t, err) self, err := api.Key().Self(ctx) require.NoError(t, err) require.Equal(t, name.String(), ipns.NameFromPeer(self.ID()).String()) resPath, err := api.Name().Resolve(ctx, name.String()+"/test", ropts...) require.NoError(t, err) require.Equal(t, p.String()+"/test", resPath.String()) }) t.Run("publishRevolvePath", func(t *testing.T) { api, p := init() p, err := path.Join(p, "/a") require.NoError(t, err) name, err := api.Name().Publish(ctx, p) require.NoError(t, err) self, err := api.Key().Self(ctx) require.NoError(t, err) require.Equal(t, name.String(), ipns.NameFromPeer(self.ID()).String()) resPath, err := api.Name().Resolve(ctx, name.String()+"/b", ropts...) require.NoError(t, err) require.Equal(t, p.String()+"/b", resPath.String()) }) } t.Run("default", func(t *testing.T) { run(t, []opt.NameResolveOption{}) }) t.Run("nocache", func(t *testing.T) { run(t, []opt.NameResolveOption{opt.Name.Cache(false)}) }) } func (tp *TestSuite) TestBasicPublishResolveKey(t *testing.T) { ctx := t.Context() apis, err := tp.MakeAPISwarm(t, ctx, 5) require.NoError(t, err) api := apis[0] k, err := api.Key().Generate(ctx, "foo") require.NoError(t, err) p, err := addTestObject(ctx, api) require.NoError(t, err) name, err := api.Name().Publish(ctx, p, opt.Name.Key(k.Name())) require.NoError(t, err) require.Equal(t, name.String(), ipns.NameFromPeer(k.ID()).String()) resPath, err := api.Name().Resolve(ctx, name.String()) require.NoError(t, err) require.Equal(t, p.String(), resPath.String()) } func (tp *TestSuite) TestBasicPublishResolveTimeout(t *testing.T) { ctx := t.Context() apis, err := tp.MakeAPISwarm(t, ctx, 5) require.NoError(t, err) api := apis[0] p, err := addTestObject(ctx, api) require.NoError(t, err) self, err := api.Key().Self(ctx) require.NoError(t, err) name, err := api.Name().Publish(ctx, p, opt.Name.ValidTime(time.Second*1)) require.NoError(t, err) require.Equal(t, name.String(), ipns.NameFromPeer(self.ID()).String()) // First resolve should succeed (before expiration) resPath, err := api.Name().Resolve(ctx, name.String()) require.NoError(t, err) require.Equal(t, p.String(), resPath.String()) // Wait for record to expire (1 second ValidTime + buffer) time.Sleep(time.Second * 2) // Second resolve should now fail after ValidTime expiration (cached) _, err = api.Name().Resolve(ctx, name.String()) require.Error(t, err, "IPNS resolution should fail after ValidTime expires (cached)") // Third resolve should also fail after ValidTime expiration (non-cached) _, err = api.Name().Resolve(ctx, name.String(), opt.Name.Cache(false)) require.Error(t, err, "IPNS resolution should fail after ValidTime expires (non-cached)") } // TODO: When swarm api is created, add multinode tests ================================================ FILE: core/coreiface/tests/object.go ================================================ package tests import ( "context" "testing" dag "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/boxo/path" ipld "github.com/ipfs/go-ipld-format" iface "github.com/ipfs/kubo/core/coreiface" opt "github.com/ipfs/kubo/core/coreiface/options" "github.com/stretchr/testify/require" ) func (tp *TestSuite) TestObject(t *testing.T) { tp.hasApi(t, func(api iface.CoreAPI) error { if api.Object() == nil { return errAPINotImplemented } return nil }) t.Run("TestObjectAddLink", tp.TestObjectAddLink) t.Run("TestObjectAddLinkCreate", tp.TestObjectAddLinkCreate) t.Run("TestObjectRmLink", tp.TestObjectRmLink) t.Run("TestDiffTest", tp.TestDiffTest) } func putDagPbNode(t *testing.T, ctx context.Context, api iface.CoreAPI, data string, links []*ipld.Link) path.ImmutablePath { dagnode := new(dag.ProtoNode) if data != "" { dagnode.SetData([]byte(data)) } if links != nil { err := dagnode.SetLinks(links) require.NoError(t, err) } err := api.Dag().Add(ctx, dagnode) require.NoError(t, err) return path.FromCid(dagnode.Cid()) } func (tp *TestSuite) TestObjectAddLink(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) p1 := putDagPbNode(t, ctx, api, "foo", nil) p2 := putDagPbNode(t, ctx, api, "bazz", []*ipld.Link{ { Name: "bar", Cid: p1.RootCid(), Size: 3, }, }) p3, err := api.Object().AddLink(ctx, p2, "abc", p2) require.NoError(t, err) nd, err := api.Dag().Get(ctx, p3.RootCid()) require.NoError(t, err) links := nd.Links() require.Len(t, links, 2) require.Equal(t, "abc", links[0].Name) require.Equal(t, "bar", links[1].Name) } func (tp *TestSuite) TestObjectAddLinkCreate(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) p1 := putDagPbNode(t, ctx, api, "foo", nil) p2 := putDagPbNode(t, ctx, api, "bazz", []*ipld.Link{ { Name: "bar", Cid: p1.RootCid(), Size: 3, }, }) _, err = api.Object().AddLink(ctx, p2, "abc/d", p2) require.ErrorContains(t, err, "no link by that name") p3, err := api.Object().AddLink(ctx, p2, "abc/d", p2, opt.Object.Create(true)) require.NoError(t, err) nd, err := api.Dag().Get(ctx, p3.RootCid()) require.NoError(t, err) links := nd.Links() require.Len(t, links, 2) require.Equal(t, "abc", links[0].Name) require.Equal(t, "bar", links[1].Name) } func (tp *TestSuite) TestObjectRmLink(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) p1 := putDagPbNode(t, ctx, api, "foo", nil) p2 := putDagPbNode(t, ctx, api, "bazz", []*ipld.Link{ { Name: "bar", Cid: p1.RootCid(), Size: 3, }, }) p3, err := api.Object().RmLink(ctx, p2, "bar") require.NoError(t, err) nd, err := api.Dag().Get(ctx, p3.RootCid()) require.NoError(t, err) links := nd.Links() require.Len(t, links, 0) } func (tp *TestSuite) TestDiffTest(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) p1 := putDagPbNode(t, ctx, api, "foo", nil) p2 := putDagPbNode(t, ctx, api, "bar", nil) changes, err := api.Object().Diff(ctx, p1, p2) require.NoError(t, err) require.Len(t, changes, 1) require.Equal(t, iface.DiffMod, changes[0].Type) require.Equal(t, p1.String(), changes[0].Before.String()) require.Equal(t, p2.String(), changes[0].After.String()) } ================================================ FILE: core/coreiface/tests/path.go ================================================ package tests import ( "fmt" "math" "strings" "testing" "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" ipldcbor "github.com/ipfs/go-ipld-cbor" "github.com/ipfs/kubo/core/coreiface/options" "github.com/stretchr/testify/require" ) func newIPLDPath(t *testing.T, cid cid.Cid) path.ImmutablePath { p, err := path.NewPath(fmt.Sprintf("/%s/%s", path.IPLDNamespace, cid.String())) require.NoError(t, err) im, err := path.NewImmutablePath(p) require.NoError(t, err) return im } func (tp *TestSuite) TestPath(t *testing.T) { t.Run("TestMutablePath", tp.TestMutablePath) t.Run("TestPathRemainder", tp.TestPathRemainder) t.Run("TestEmptyPathRemainder", tp.TestEmptyPathRemainder) t.Run("TestInvalidPathRemainder", tp.TestInvalidPathRemainder) t.Run("TestPathRoot", tp.TestPathRoot) t.Run("TestPathJoin", tp.TestPathJoin) } func (tp *TestSuite) TestMutablePath(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) blk, err := api.Block().Put(ctx, strings.NewReader(`foo`)) require.NoError(t, err) require.False(t, blk.Path().Mutable()) require.NotNil(t, api.Key()) keys, err := api.Key().List(ctx) require.NoError(t, err) require.True(t, keys[0].Path().Mutable()) } func (tp *TestSuite) TestPathRemainder(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) require.NotNil(t, api.Dag()) nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"bar": "baz"}}`), math.MaxUint64, -1) require.NoError(t, err) err = api.Dag().Add(ctx, nd) require.NoError(t, err) p, err := path.Join(path.FromCid(nd.Cid()), "foo", "bar") require.NoError(t, err) _, remainder, err := api.ResolvePath(ctx, p) require.NoError(t, err) require.Equal(t, "/foo/bar", path.SegmentsToString(remainder...)) } func (tp *TestSuite) TestEmptyPathRemainder(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) require.NotNil(t, api.Dag()) nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"bar": "baz"}}`), math.MaxUint64, -1) require.NoError(t, err) err = api.Dag().Add(ctx, nd) require.NoError(t, err) _, remainder, err := api.ResolvePath(ctx, path.FromCid(nd.Cid())) require.NoError(t, err) require.Empty(t, remainder) } func (tp *TestSuite) TestInvalidPathRemainder(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) require.NotNil(t, api.Dag()) nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"bar": "baz"}}`), math.MaxUint64, -1) require.NoError(t, err) err = api.Dag().Add(ctx, nd) require.NoError(t, err) p, err := path.Join(newIPLDPath(t, nd.Cid()), "/bar/baz") require.NoError(t, err) _, _, err = api.ResolvePath(ctx, p) require.NotNil(t, err) require.ErrorContains(t, err, `no link named "bar"`) } func (tp *TestSuite) TestPathRoot(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) require.NotNil(t, api.Block()) blk, err := api.Block().Put(ctx, strings.NewReader(`foo`), options.Block.Format("raw")) require.NoError(t, err) require.NotNil(t, api.Dag()) nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"/": "`+blk.Path().RootCid().String()+`"}}`), math.MaxUint64, -1) require.NoError(t, err) err = api.Dag().Add(ctx, nd) require.NoError(t, err) p, err := path.Join(newIPLDPath(t, nd.Cid()), "/foo") require.NoError(t, err) rp, _, err := api.ResolvePath(ctx, p) require.NoError(t, err) require.Equal(t, rp.RootCid().String(), blk.Path().RootCid().String()) } func (tp *TestSuite) TestPathJoin(t *testing.T) { p1, err := path.NewPath("/ipfs/QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6/bar/baz") require.NoError(t, err) p2, err := path.Join(p1, "foo") require.NoError(t, err) require.Equal(t, "/ipfs/QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6/bar/baz/foo", p2.String()) } ================================================ FILE: core/coreiface/tests/pin.go ================================================ package tests import ( "context" "math" "strings" "testing" "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" ipldcbor "github.com/ipfs/go-ipld-cbor" ipld "github.com/ipfs/go-ipld-format" iface "github.com/ipfs/kubo/core/coreiface" opt "github.com/ipfs/kubo/core/coreiface/options" "github.com/stretchr/testify/require" ) func (tp *TestSuite) TestPin(t *testing.T) { tp.hasApi(t, func(api iface.CoreAPI) error { if api.Pin() == nil { return errAPINotImplemented } return nil }) t.Run("TestPinAdd", tp.TestPinAdd) t.Run("TestPinSimple", tp.TestPinSimple) t.Run("TestPinRecursive", tp.TestPinRecursive) t.Run("TestPinLsIndirect", tp.TestPinLsIndirect) t.Run("TestPinLsPrecedence", tp.TestPinLsPrecedence) t.Run("TestPinIsPinned", tp.TestPinIsPinned) t.Run("TestPinNames", tp.TestPinNames) } func (tp *TestSuite) TestPinAdd(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } p, err := api.Unixfs().Add(ctx, strFile("foo")()) if err != nil { t.Fatal(err) } err = api.Pin().Add(ctx, p) if err != nil { t.Fatal(err) } } func (tp *TestSuite) TestPinSimple(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } p, err := api.Unixfs().Add(ctx, strFile("foo")()) if err != nil { t.Fatal(err) } err = api.Pin().Add(ctx, p) if err != nil { t.Fatal(err) } list, err := accPins(ctx, api) if err != nil { t.Fatal(err) } if len(list) != 1 { t.Errorf("unexpected pin list len: %d", len(list)) } if list[0].Path().RootCid().String() != p.RootCid().String() { t.Error("paths don't match") } if list[0].Type() != "recursive" { t.Error("unexpected pin type") } assertIsPinned(t, ctx, api, p, "recursive") err = api.Pin().Rm(ctx, p) if err != nil { t.Fatal(err) } list, err = accPins(ctx, api) if err != nil { t.Fatal(err) } if len(list) != 0 { t.Errorf("unexpected pin list len: %d", len(list)) } } func (tp *TestSuite) TestPinRecursive(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } p0, err := api.Unixfs().Add(ctx, strFile("foo")()) if err != nil { t.Fatal(err) } p1, err := api.Unixfs().Add(ctx, strFile("bar")()) if err != nil { t.Fatal(err) } nd2, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+p0.RootCid().String()+`"}}`), math.MaxUint64, -1) if err != nil { t.Fatal(err) } nd3, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+p1.RootCid().String()+`"}}`), math.MaxUint64, -1) if err != nil { t.Fatal(err) } if err := api.Dag().AddMany(ctx, []ipld.Node{nd2, nd3}); err != nil { t.Fatal(err) } err = api.Pin().Add(ctx, path.FromCid(nd2.Cid())) if err != nil { t.Fatal(err) } err = api.Pin().Add(ctx, path.FromCid(nd3.Cid()), opt.Pin.Recursive(false)) if err != nil { t.Fatal(err) } list, err := accPins(ctx, api) if err != nil { t.Fatal(err) } if len(list) != 3 { t.Errorf("unexpected pin list len: %d", len(list)) } list, err = accPins(ctx, api, opt.Pin.Ls.Direct()) if err != nil { t.Fatal(err) } if len(list) != 1 { t.Errorf("unexpected pin list len: %d", len(list)) } if list[0].Path().String() != path.FromCid(nd3.Cid()).String() { t.Errorf("unexpected path, %s != %s", list[0].Path().String(), path.FromCid(nd3.Cid()).String()) } list, err = accPins(ctx, api, opt.Pin.Ls.Recursive()) if err != nil { t.Fatal(err) } if len(list) != 1 { t.Errorf("unexpected pin list len: %d", len(list)) } if list[0].Path().String() != path.FromCid(nd2.Cid()).String() { t.Errorf("unexpected path, %s != %s", list[0].Path().String(), path.FromCid(nd2.Cid()).String()) } list, err = accPins(ctx, api, opt.Pin.Ls.Indirect()) if err != nil { t.Fatal(err) } if len(list) != 1 { t.Errorf("unexpected pin list len: %d", len(list)) } if list[0].Path().RootCid().String() != p0.RootCid().String() { t.Errorf("unexpected path, %s != %s", list[0].Path().RootCid().String(), p0.RootCid().String()) } res, err := api.Pin().Verify(ctx) if err != nil { t.Fatal(err) } n := 0 for r := range res { if err := r.Err(); err != nil { t.Error(err) } if !r.Ok() { t.Error("expected pin to be ok") } n++ } if n != 1 { t.Errorf("unexpected verify result count: %d", n) } // TODO: figure out a way to test verify without touching IpfsNode /* err = api.Block().Rm(ctx, p0, opt.Block.Force(true)) if err != nil { t.Fatal(err) } res, err = api.Pin().Verify(ctx) if err != nil { t.Fatal(err) } n = 0 for r := range res { if r.Ok() { t.Error("expected pin to not be ok") } if len(r.BadNodes()) != 1 { t.Fatalf("unexpected badNodes len") } if r.BadNodes()[0].Path().Cid().String() != p0.Cid().String() { t.Error("unexpected badNode path") } if r.BadNodes()[0].Err().Error() != "merkledag: not found" { t.Errorf("unexpected badNode error: %s", r.BadNodes()[0].Err().Error()) } n++ } if n != 1 { t.Errorf("unexpected verify result count: %d", n) } */ } // TestPinLsIndirect verifies that indirect nodes are listed by pin ls even if a parent node is directly pinned func (tp *TestSuite) TestPinLsIndirect(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "foo") err = api.Pin().Add(ctx, path.FromCid(grandparent.Cid())) if err != nil { t.Fatal(err) } err = api.Pin().Add(ctx, path.FromCid(parent.Cid()), opt.Pin.Recursive(false)) if err != nil { t.Fatal(err) } assertPinTypes(t, ctx, api, []cidContainer{grandparent}, []cidContainer{parent}, []cidContainer{leaf}) } // TestPinLsPrecedence verifies the precedence of pins (recursive > direct > indirect) func (tp *TestSuite) TestPinLsPrecedence(t *testing.T) { // Testing precedence of recursive, direct and indirect pins // Results should be recursive > indirect, direct > indirect, and recursive > direct t.Run("TestPinLsPredenceRecursiveIndirect", tp.TestPinLsPredenceRecursiveIndirect) t.Run("TestPinLsPrecedenceDirectIndirect", tp.TestPinLsPrecedenceDirectIndirect) t.Run("TestPinLsPrecedenceRecursiveDirect", tp.TestPinLsPrecedenceRecursiveDirect) } func (tp *TestSuite) TestPinLsPredenceRecursiveIndirect(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } // Test recursive > indirect leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "recursive > indirect") err = api.Pin().Add(ctx, path.FromCid(grandparent.Cid())) if err != nil { t.Fatal(err) } err = api.Pin().Add(ctx, path.FromCid(parent.Cid())) if err != nil { t.Fatal(err) } assertPinTypes(t, ctx, api, []cidContainer{grandparent, parent}, []cidContainer{}, []cidContainer{leaf}) } func (tp *TestSuite) TestPinLsPrecedenceDirectIndirect(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } // Test direct > indirect leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "direct > indirect") err = api.Pin().Add(ctx, path.FromCid(grandparent.Cid())) if err != nil { t.Fatal(err) } err = api.Pin().Add(ctx, path.FromCid(parent.Cid()), opt.Pin.Recursive(false)) if err != nil { t.Fatal(err) } assertPinTypes(t, ctx, api, []cidContainer{grandparent}, []cidContainer{parent}, []cidContainer{leaf}) } func (tp *TestSuite) TestPinLsPrecedenceRecursiveDirect(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } // Test recursive > direct leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "recursive + direct = error") err = api.Pin().Add(ctx, path.FromCid(parent.Cid())) if err != nil { t.Fatal(err) } err = api.Pin().Add(ctx, path.FromCid(parent.Cid()), opt.Pin.Recursive(false)) if err == nil { t.Fatal("expected error directly pinning a recursively pinned node") } assertPinTypes(t, ctx, api, []cidContainer{parent}, []cidContainer{}, []cidContainer{leaf}) err = api.Pin().Add(ctx, path.FromCid(grandparent.Cid()), opt.Pin.Recursive(false)) if err != nil { t.Fatal(err) } err = api.Pin().Add(ctx, path.FromCid(grandparent.Cid())) if err != nil { t.Fatal(err) } assertPinTypes(t, ctx, api, []cidContainer{grandparent, parent}, []cidContainer{}, []cidContainer{leaf}) } func (tp *TestSuite) TestPinIsPinned(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "foofoo") assertNotPinned(t, ctx, api, newIPLDPath(t, grandparent.Cid())) assertNotPinned(t, ctx, api, newIPLDPath(t, parent.Cid())) assertNotPinned(t, ctx, api, newIPLDPath(t, leaf.Cid())) err = api.Pin().Add(ctx, newIPLDPath(t, parent.Cid()), opt.Pin.Recursive(true)) if err != nil { t.Fatal(err) } assertNotPinned(t, ctx, api, newIPLDPath(t, grandparent.Cid())) assertIsPinned(t, ctx, api, newIPLDPath(t, parent.Cid()), "recursive") assertIsPinned(t, ctx, api, newIPLDPath(t, leaf.Cid()), "indirect") err = api.Pin().Add(ctx, newIPLDPath(t, grandparent.Cid()), opt.Pin.Recursive(false)) if err != nil { t.Fatal(err) } assertIsPinned(t, ctx, api, newIPLDPath(t, grandparent.Cid()), "direct") assertIsPinned(t, ctx, api, newIPLDPath(t, parent.Cid()), "recursive") assertIsPinned(t, ctx, api, newIPLDPath(t, leaf.Cid()), "indirect") } type cidContainer interface { Cid() cid.Cid } type immutablePathCidContainer struct { path.ImmutablePath } func (i immutablePathCidContainer) Cid() cid.Cid { return i.RootCid() } func getThreeChainedNodes(t *testing.T, ctx context.Context, api iface.CoreAPI, leafData string) (cidContainer, cidContainer, cidContainer) { leaf, err := api.Unixfs().Add(ctx, strFile(leafData)()) if err != nil { t.Fatal(err) } parent, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+leaf.RootCid().String()+`"}}`), math.MaxUint64, -1) if err != nil { t.Fatal(err) } grandparent, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+parent.Cid().String()+`"}}`), math.MaxUint64, -1) if err != nil { t.Fatal(err) } if err := api.Dag().AddMany(ctx, []ipld.Node{parent, grandparent}); err != nil { t.Fatal(err) } return immutablePathCidContainer{leaf}, parent, grandparent } func assertPinTypes(t *testing.T, ctx context.Context, api iface.CoreAPI, recursive, direct, indirect []cidContainer) { assertPinLsAllConsistency(t, ctx, api) list, err := accPins(ctx, api, opt.Pin.Ls.Recursive()) if err != nil { t.Fatal(err) } assertPinCids(t, list, recursive...) list, err = accPins(ctx, api, opt.Pin.Ls.Direct()) if err != nil { t.Fatal(err) } assertPinCids(t, list, direct...) list, err = accPins(ctx, api, opt.Pin.Ls.Indirect()) if err != nil { t.Fatal(err) } assertPinCids(t, list, indirect...) } // assertPinCids verifies that the pins match the expected cids func assertPinCids(t *testing.T, pins []iface.Pin, cids ...cidContainer) { t.Helper() if expected, actual := len(cids), len(pins); expected != actual { t.Fatalf("expected pin list to have len %d, was %d", expected, actual) } cSet := cid.NewSet() for _, c := range cids { cSet.Add(c.Cid()) } valid := true for _, p := range pins { c := p.Path().RootCid() if cSet.Has(c) { cSet.Remove(c) } else { valid = false break } } valid = valid && cSet.Len() == 0 if !valid { pinStrs := make([]string, len(pins)) for i, p := range pins { pinStrs[i] = p.Path().RootCid().String() } pathStrs := make([]string, len(cids)) for i, c := range cids { pathStrs[i] = c.Cid().String() } t.Fatalf("expected: %s \nactual: %s", strings.Join(pathStrs, ", "), strings.Join(pinStrs, ", ")) } } // assertPinLsAllConsistency verifies that listing all pins gives the same result as listing the pin types individually func assertPinLsAllConsistency(t *testing.T, ctx context.Context, api iface.CoreAPI) { t.Helper() allPins, err := accPins(ctx, api) if err != nil { t.Fatal(err) } type pinTypeProps struct { *cid.Set opt.PinLsOption } all, recursive, direct, indirect := cid.NewSet(), cid.NewSet(), cid.NewSet(), cid.NewSet() typeMap := map[string]*pinTypeProps{ "recursive": {recursive, opt.Pin.Ls.Recursive()}, "direct": {direct, opt.Pin.Ls.Direct()}, "indirect": {indirect, opt.Pin.Ls.Indirect()}, } for _, p := range allPins { if !all.Visit(p.Path().RootCid()) { t.Fatalf("pin ls returned the same cid multiple times") } typeStr := p.Type() if typeSet, ok := typeMap[p.Type()]; ok { typeSet.Add(p.Path().RootCid()) } else { t.Fatalf("unknown pin type: %s", typeStr) } } for typeStr, pinProps := range typeMap { pins, err := accPins(ctx, api, pinProps.PinLsOption) if err != nil { t.Fatal(err) } if expected, actual := len(pins), pinProps.Set.Len(); expected != actual { t.Fatalf("pin ls all has %d pins of type %s, but pin ls for the type has %d", expected, typeStr, actual) } for _, p := range pins { if pinType := p.Type(); pinType != typeStr { t.Fatalf("returned wrong pin type: expected %s, got %s", typeStr, pinType) } if c := p.Path().RootCid(); !pinProps.Has(c) { t.Fatalf("%s expected to be in pin ls all as type %s", c.String(), typeStr) } } } } func assertIsPinned(t *testing.T, ctx context.Context, api iface.CoreAPI, p path.Path, typeStr string) { t.Helper() withType, err := opt.Pin.IsPinned.Type(typeStr) if err != nil { t.Fatal("unhandled pin type") } whyPinned, pinned, err := api.Pin().IsPinned(ctx, p, withType) if err != nil { t.Fatal(err) } if !pinned { t.Fatalf("%s expected to be pinned with type %s", p, typeStr) } switch typeStr { case "recursive", "direct": if typeStr != whyPinned { t.Fatalf("reason for pinning expected to be %s for %s, got %s", typeStr, p, whyPinned) } case "indirect": if whyPinned == "" { t.Fatalf("expected to have a pin reason for %s", p) } } } func (tp *TestSuite) TestPinNames(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) require.NoError(t, err) // Create test content p1, err := api.Unixfs().Add(ctx, strFile("content1")()) require.NoError(t, err) p2, err := api.Unixfs().Add(ctx, strFile("content2")()) require.NoError(t, err) p3, err := api.Unixfs().Add(ctx, strFile("content3")()) require.NoError(t, err) p4, err := api.Unixfs().Add(ctx, strFile("content4")()) require.NoError(t, err) // Test 1: Pin with name err = api.Pin().Add(ctx, p1, opt.Pin.Name("test-pin-1")) require.NoError(t, err, "failed to add pin with name") // Test 2: Pin without name err = api.Pin().Add(ctx, p2) require.NoError(t, err, "failed to add pin without name") // Test 3: List pins with detailed option to get names pins := make(chan iface.Pin) go func() { err = api.Pin().Ls(ctx, pins, opt.Pin.Ls.Detailed(true)) }() pinMap := make(map[string]string) for pin := range pins { pinMap[pin.Path().String()] = pin.Name() } require.NoError(t, err, "failed to list pins with names") // Verify pin names name1, ok := pinMap[p1.String()] require.True(t, ok, "pin for %s not found", p1) require.Equal(t, "test-pin-1", name1, "unexpected pin name for %s", p1) name2, ok := pinMap[p2.String()] require.True(t, ok, "pin for %s not found", p2) require.Empty(t, name2, "expected empty pin name for %s, got '%s'", p2, name2) // Test 4: Pin update preserves name err = api.Pin().Add(ctx, p3, opt.Pin.Name("updatable-pin")) require.NoError(t, err, "failed to add pin with name for update test") err = api.Pin().Update(ctx, p3, p4) require.NoError(t, err, "failed to update pin") // Verify name was preserved after update pins2 := make(chan iface.Pin) go func() { err = api.Pin().Ls(ctx, pins2, opt.Pin.Ls.Detailed(true)) }() updatedPinMap := make(map[string]string) for pin := range pins2 { updatedPinMap[pin.Path().String()] = pin.Name() } require.NoError(t, err, "failed to list pins after update") // Old pin should not exist _, oldExists := updatedPinMap[p3.String()] require.False(t, oldExists, "old pin %s should not exist after update", p3) // New pin should have the preserved name name4, ok := updatedPinMap[p4.String()] require.True(t, ok, "updated pin for %s not found", p4) require.Equal(t, "updatable-pin", name4, "pin name not preserved after update from %s to %s", p3, p4) // Test 5: Re-pinning with different name updates the name err = api.Pin().Add(ctx, p1, opt.Pin.Name("new-name-for-p1")) require.NoError(t, err, "failed to re-pin with new name") // Verify name was updated pins3 := make(chan iface.Pin) go func() { err = api.Pin().Ls(ctx, pins3, opt.Pin.Ls.Detailed(true)) }() repinMap := make(map[string]string) for pin := range pins3 { repinMap[pin.Path().String()] = pin.Name() } require.NoError(t, err, "failed to list pins after re-pin") rePinnedName, ok := repinMap[p1.String()] require.True(t, ok, "re-pinned content %s not found", p1) require.Equal(t, "new-name-for-p1", rePinnedName, "pin name not updated after re-pinning %s", p1) // Test 6: Direct pin with name p5, err := api.Unixfs().Add(ctx, strFile("direct-content")()) require.NoError(t, err) err = api.Pin().Add(ctx, p5, opt.Pin.Recursive(false), opt.Pin.Name("direct-pin-name")) require.NoError(t, err, "failed to add direct pin with name") // Verify direct pin has name directPins := make(chan iface.Pin) typeOpt, err := opt.Pin.Ls.Type("direct") require.NoError(t, err, "failed to create type option") go func() { err = api.Pin().Ls(ctx, directPins, typeOpt, opt.Pin.Ls.Detailed(true)) }() directPinMap := make(map[string]string) for pin := range directPins { directPinMap[pin.Path().String()] = pin.Name() } require.NoError(t, err, "failed to list direct pins") directName, ok := directPinMap[p5.String()] require.True(t, ok, "direct pin %s not found", p5) require.Equal(t, "direct-pin-name", directName, "unexpected name for direct pin %s", p5) // Test 7: List without detailed option doesn't return names pinsNoDetails := make(chan iface.Pin) go func() { err = api.Pin().Ls(ctx, pinsNoDetails) }() noDetailsMap := make(map[string]string) for pin := range pinsNoDetails { noDetailsMap[pin.Path().String()] = pin.Name() } require.NoError(t, err, "failed to list pins without detailed option") // All names should be empty without detailed option for path, name := range noDetailsMap { require.Empty(t, name, "expected empty name for %s without detailed option, got '%s'", path, name) } } func assertNotPinned(t *testing.T, ctx context.Context, api iface.CoreAPI, p path.Path) { t.Helper() _, pinned, err := api.Pin().IsPinned(ctx, p) if err != nil { t.Fatal(err) } if pinned { t.Fatalf("%s expected to not be pinned", p) } } func accPins(ctx context.Context, api iface.CoreAPI, opts ...opt.PinLsOption) ([]iface.Pin, error) { var err error pins := make(chan iface.Pin) go func() { err = api.Pin().Ls(ctx, pins, opts...) }() var results []iface.Pin for pin := range pins { results = append(results, pin) } if err != nil { return nil, err } return results, nil } ================================================ FILE: core/coreiface/tests/pubsub.go ================================================ package tests import ( "context" "testing" "time" iface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/core/coreiface/options" ) func (tp *TestSuite) TestPubSub(t *testing.T) { tp.hasApi(t, func(api iface.CoreAPI) error { if api.PubSub() == nil { return errAPINotImplemented } return nil }) t.Run("TestBasicPubSub", tp.TestBasicPubSub) } func (tp *TestSuite) TestBasicPubSub(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() apis, err := tp.MakeAPISwarm(t, ctx, 2) if err != nil { t.Fatal(err) } sub, err := apis[0].PubSub().Subscribe(ctx, "testch") if err != nil { t.Fatal(err) } done := make(chan struct{}) go func() { defer close(done) ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { err := apis[1].PubSub().Publish(ctx, "testch", []byte("hello world")) switch err { case nil: case context.Canceled: return default: t.Error(err) cancel() return } select { case <-ticker.C: case <-ctx.Done(): return } } }() // Wait for the sender to finish before we return. // Otherwise, we can get random errors as publish fails. defer func() { cancel() <-done }() m, err := sub.Next(ctx) if err != nil { t.Fatal(err) } if string(m.Data()) != "hello world" { t.Errorf("got invalid data: %s", string(m.Data())) } self1, err := apis[1].Key().Self(ctx) if err != nil { t.Fatal(err) } if m.From() != self1.ID() { t.Errorf("m.From didn't match") } peers, err := apis[1].PubSub().Peers(ctx, options.PubSub.Topic("testch")) if err != nil { t.Fatal(err) } if len(peers) != 1 { t.Fatalf("got incorrect number of peers: %d", len(peers)) } self0, err := apis[0].Key().Self(ctx) if err != nil { t.Fatal(err) } if peers[0] != self0.ID() { t.Errorf("peer didn't match") } peers, err = apis[1].PubSub().Peers(ctx, options.PubSub.Topic("nottestch")) if err != nil { t.Fatal(err) } if len(peers) != 0 { t.Fatalf("got incorrect number of peers: %d", len(peers)) } topics, err := apis[0].PubSub().Ls(ctx) if err != nil { t.Fatal(err) } if len(topics) != 1 { t.Fatalf("got incorrect number of topics: %d", len(peers)) } if topics[0] != "testch" { t.Errorf("topic didn't match") } topics, err = apis[1].PubSub().Ls(ctx) if err != nil { t.Fatal(err) } if len(topics) != 0 { t.Fatalf("got incorrect number of topics: %d", len(peers)) } } ================================================ FILE: core/coreiface/tests/routing.go ================================================ package tests import ( "context" "io" "testing" "time" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" iface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/core/coreiface/options" "github.com/stretchr/testify/require" ) func (tp *TestSuite) TestRouting(t *testing.T) { tp.hasApi(t, func(api iface.CoreAPI) error { if api.Routing() == nil { return errAPINotImplemented } return nil }) t.Run("TestRoutingGet", tp.TestRoutingGet) t.Run("TestRoutingPut", tp.TestRoutingPut) t.Run("TestRoutingPutOffline", tp.TestRoutingPutOffline) t.Run("TestRoutingFindPeer", tp.TestRoutingFindPeer) t.Run("TestRoutingFindProviders", tp.TestRoutingFindProviders) t.Run("TestRoutingProvide", tp.TestRoutingProvide) } func (tp *TestSuite) testRoutingPublishKey(t *testing.T, ctx context.Context, api iface.CoreAPI, opts ...options.NamePublishOption) (path.Path, ipns.Name) { p, err := addTestObject(ctx, api) require.NoError(t, err) name, err := api.Name().Publish(ctx, p, opts...) require.NoError(t, err) time.Sleep(3 * time.Second) return p, name } func (tp *TestSuite) TestRoutingGet(t *testing.T) { ctx := t.Context() apis, err := tp.MakeAPISwarm(t, ctx, 2) require.NoError(t, err) // Node 1: publishes an IPNS name p, name := tp.testRoutingPublishKey(t, ctx, apis[0]) // Node 2: retrieves the best value for the IPNS name. data, err := apis[1].Routing().Get(ctx, ipns.NamespacePrefix+name.String()) require.NoError(t, err) rec, err := ipns.UnmarshalRecord(data) require.NoError(t, err) val, err := rec.Value() require.NoError(t, err) require.Equal(t, p.String(), val.String()) } func (tp *TestSuite) TestRoutingPut(t *testing.T) { ctx := t.Context() apis, err := tp.MakeAPISwarm(t, ctx, 2) require.NoError(t, err) // Create and publish IPNS entry. _, name := tp.testRoutingPublishKey(t, ctx, apis[0]) // Get valid routing value. data, err := apis[0].Routing().Get(ctx, ipns.NamespacePrefix+name.String()) require.NoError(t, err) // Put routing value. err = apis[1].Routing().Put(ctx, ipns.NamespacePrefix+name.String(), data) require.NoError(t, err) } func (tp *TestSuite) TestRoutingPutOffline(t *testing.T) { ctx := t.Context() // init a swarm & publish an IPNS entry to get a valid payload apis, err := tp.MakeAPISwarm(t, ctx, 2) require.NoError(t, err) _, name := tp.testRoutingPublishKey(t, ctx, apis[0], options.Name.AllowOffline(true)) data, err := apis[0].Routing().Get(ctx, ipns.NamespacePrefix+name.String()) require.NoError(t, err) // init our offline node and try to put the payload api, err := tp.makeAPIWithIdentityAndOffline(t, ctx) require.NoError(t, err) err = api.Routing().Put(ctx, ipns.NamespacePrefix+name.String(), data) require.Error(t, err, "this operation should fail because we are offline") err = api.Routing().Put(ctx, ipns.NamespacePrefix+name.String(), data, options.Routing.AllowOffline(true)) require.NoError(t, err) } func (tp *TestSuite) TestRoutingFindPeer(t *testing.T) { ctx := t.Context() apis, err := tp.MakeAPISwarm(t, ctx, 5) if err != nil { t.Fatal(err) } self0, err := apis[0].Key().Self(ctx) if err != nil { t.Fatal(err) } laddrs0, err := apis[0].Swarm().LocalAddrs(ctx) if err != nil { t.Fatal(err) } if len(laddrs0) != 1 { t.Fatal("unexpected number of local addrs") } time.Sleep(3 * time.Second) pi, err := apis[2].Routing().FindPeer(ctx, self0.ID()) if err != nil { t.Fatal(err) } if pi.Addrs[0].String() != laddrs0[0].String() { t.Errorf("got unexpected address from FindPeer: %s", pi.Addrs[0].String()) } self2, err := apis[2].Key().Self(ctx) if err != nil { t.Fatal(err) } pi, err = apis[1].Routing().FindPeer(ctx, self2.ID()) if err != nil { t.Fatal(err) } laddrs2, err := apis[2].Swarm().LocalAddrs(ctx) if err != nil { t.Fatal(err) } if len(laddrs2) != 1 { t.Fatal("unexpected number of local addrs") } if pi.Addrs[0].String() != laddrs2[0].String() { t.Errorf("got unexpected address from FindPeer: %s", pi.Addrs[0].String()) } } func (tp *TestSuite) TestRoutingFindProviders(t *testing.T) { ctx := t.Context() apis, err := tp.MakeAPISwarm(t, ctx, 5) if err != nil { t.Fatal(err) } p, err := addTestObject(ctx, apis[0]) if err != nil { t.Fatal(err) } // Pin so that it is provided, given that providing strategy is // "roots" and addTestObject does not pin. err = apis[0].Pin().Add(ctx, p) if err != nil { t.Fatal(err) } time.Sleep(3 * time.Second) out, err := apis[2].Routing().FindProviders(ctx, p, options.Routing.NumProviders(1)) if err != nil { t.Fatal(err) } provider := <-out self0, err := apis[0].Key().Self(ctx) if err != nil { t.Fatal(err) } if provider.ID.String() != self0.ID().String() { t.Errorf("got wrong provider: %s != %s", provider.ID.String(), self0.ID().String()) } } func (tp *TestSuite) TestRoutingProvide(t *testing.T) { ctx := t.Context() apis, err := tp.MakeAPISwarm(t, ctx, 5) if err != nil { t.Fatal(err) } off0, err := apis[0].WithOptions(options.Api.Offline(true)) if err != nil { t.Fatal(err) } s, err := off0.Block().Put(ctx, &io.LimitedReader{R: rnd, N: 4092}) if err != nil { t.Fatal(err) } p := s.Path() time.Sleep(3 * time.Second) out, err := apis[2].Routing().FindProviders(ctx, p, options.Routing.NumProviders(1)) if err != nil { t.Fatal(err) } _, ok := <-out if ok { t.Fatal("did not expect to find any providers") } self0, err := apis[0].Key().Self(ctx) if err != nil { t.Fatal(err) } err = apis[0].Routing().Provide(ctx, p) if err != nil { t.Fatal(err) } maxAttempts := 5 success := false for range maxAttempts { // We may need to try again as Provide() doesn't block until the CID is // actually provided. out, err = apis[2].Routing().FindProviders(ctx, p, options.Routing.NumProviders(1)) if err != nil { t.Fatal(err) } provider := <-out if provider.ID.String() == self0.ID().String() { success = true break } if len(provider.ID.String()) > 0 { t.Errorf("got wrong provider: %s != %s", provider.ID.String(), self0.ID().String()) } time.Sleep(time.Second) } if !success { t.Errorf("missing provider after %d attempts", maxAttempts) } } ================================================ FILE: core/coreiface/tests/unixfs.go ================================================ package tests import ( "bytes" "context" "encoding/hex" "fmt" "io" "math" "math/rand" "os" "strconv" "strings" "sync" "testing" "github.com/ipfs/boxo/path" coreiface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/core/coreiface/options" "github.com/ipfs/boxo/files" mdag "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" ipld "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" ) func (tp *TestSuite) TestUnixfs(t *testing.T) { tp.hasApi(t, func(api coreiface.CoreAPI) error { if api.Unixfs() == nil { return errAPINotImplemented } return nil }) t.Run("TestAdd", tp.TestAdd) t.Run("TestAddPinned", tp.TestAddPinned) t.Run("TestAddHashOnly", tp.TestAddHashOnly) t.Run("TestGetEmptyFile", tp.TestGetEmptyFile) t.Run("TestGetDir", tp.TestGetDir) t.Run("TestGetNonUnixfs", tp.TestGetNonUnixfs) t.Run("TestLs", tp.TestLs) t.Run("TestEntriesExpired", tp.TestEntriesExpired) t.Run("TestLsEmptyDir", tp.TestLsEmptyDir) t.Run("TestLsNonUnixfs", tp.TestLsNonUnixfs) t.Run("TestAddCloses", tp.TestAddCloses) t.Run("TestGetSeek", tp.TestGetSeek) t.Run("TestGetReadAt", tp.TestGetReadAt) } // `echo -n 'hello, world!' | ipfs add` var ( hello = "/ipfs/QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk" helloStr = "hello, world!" ) // `echo -n | ipfs add` var emptyFile = "/ipfs/QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH" func strFile(data string) func() files.Node { return func() files.Node { return files.NewBytesFile([]byte(data)) } } func twoLevelDir() func() files.Node { return func() files.Node { return files.NewMapDirectory(map[string]files.Node{ "abc": files.NewMapDirectory(map[string]files.Node{ "def": files.NewBytesFile([]byte("world")), }), "bar": files.NewBytesFile([]byte("hello2")), "foo": files.NewBytesFile([]byte("hello1")), }) } } func flatDir() files.Node { return files.NewMapDirectory(map[string]files.Node{ "bar": files.NewBytesFile([]byte("hello2")), "foo": files.NewBytesFile([]byte("hello1")), }) } func wrapped(names ...string) func(f files.Node) files.Node { return func(f files.Node) files.Node { for i := range names { f = files.NewMapDirectory(map[string]files.Node{ names[len(names)-i-1]: f, }) } return f } } func (tp *TestSuite) TestAdd(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } p := func(h string) path.ImmutablePath { c, err := cid.Parse(h) if err != nil { t.Fatal(err) } return path.FromCid(c) } rf, err := os.CreateTemp(os.TempDir(), "unixfs-add-real") if err != nil { t.Fatal(err) } rfp := rf.Name() if _, err := rf.Write([]byte(helloStr)); err != nil { t.Fatal(err) } stat, err := rf.Stat() if err != nil { t.Fatal(err) } if err := rf.Close(); err != nil { t.Fatal(err) } defer os.Remove(rfp) realFile := func() files.Node { n, err := files.NewReaderPathFile(rfp, io.NopCloser(strings.NewReader(helloStr)), stat) if err != nil { t.Fatal(err) } return n } cases := []struct { name string data func() files.Node expect func(files.Node) files.Node apiOpts []options.ApiOption path string err string wrap string events []coreiface.AddEvent opts []options.UnixfsAddOption }{ // Simple cases { name: "simpleAdd", data: strFile(helloStr), path: hello, opts: []options.UnixfsAddOption{}, }, { name: "addEmpty", data: strFile(""), path: emptyFile, }, // CIDv1 version / rawLeaves { name: "addCidV1", data: strFile(helloStr), path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(1)}, }, { name: "addCidV1NoLeaves", data: strFile(helloStr), path: "/ipfs/bafybeibhbcn7k7o2m6xsqkrlfiokod3nxwe47viteynhruh6uqx7hvkjfu", opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(1), options.Unixfs.RawLeaves(false)}, }, // Non sha256 hash vs CID { name: "addCidSha3", data: strFile(helloStr), path: "/ipfs/bafkrmichjflejeh6aren53o7pig7zk3m3vxqcoc2i5dv326k3x6obh7jry", opts: []options.UnixfsAddOption{options.Unixfs.Hash(mh.SHA3_256)}, }, { name: "addCidSha3Cid0", data: strFile(helloStr), err: "CIDv0 only supports sha2-256", opts: []options.UnixfsAddOption{options.Unixfs.CidVersion(0), options.Unixfs.Hash(mh.SHA3_256)}, }, // Inline { name: "addInline", data: strFile(helloStr), path: "/ipfs/bafyaafikcmeaeeqnnbswy3dpfqqho33snrsccgan", opts: []options.UnixfsAddOption{options.Unixfs.Inline(true)}, }, { name: "addInlineLimit", data: strFile(helloStr), path: "/ipfs/bafyaafikcmeaeeqnnbswy3dpfqqho33snrsccgan", opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32), options.Unixfs.Inline(true)}, }, { name: "addInlineZero", data: strFile(""), path: "/ipfs/bafkqaaa", opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(0), options.Unixfs.Inline(true), options.Unixfs.RawLeaves(true)}, }, { // TODO: after coreapi add is used in `ipfs add`, consider making this default for inline name: "addInlineRaw", data: strFile(helloStr), path: "/ipfs/bafkqadlimvwgy3zmeb3w64tmmqqq", opts: []options.UnixfsAddOption{options.Unixfs.InlineLimit(32), options.Unixfs.Inline(true), options.Unixfs.RawLeaves(true)}, }, // Chunker / Layout { name: "addChunks", data: strFile(strings.Repeat("aoeuidhtns", 200)), path: "/ipfs/QmRo11d4QJrST47aaiGVJYwPhoNA4ihRpJ5WaxBWjWDwbX", opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4")}, }, { name: "addChunksTrickle", data: strFile(strings.Repeat("aoeuidhtns", 200)), path: "/ipfs/QmNNhDGttafX3M1wKWixGre6PrLFGjnoPEDXjBYpTv93HP", opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4"), options.Unixfs.Layout(options.TrickleLayout)}, }, // Local { name: "addLocal", // better cases in sharness data: strFile(helloStr), path: hello, apiOpts: []options.ApiOption{options.Api.Offline(true)}, }, { name: "hashOnly", // test (non)fetchability data: strFile(helloStr), path: hello, opts: []options.UnixfsAddOption{options.Unixfs.HashOnly(true)}, }, // multi file { name: "simpleDirNoWrap", data: flatDir, path: "/ipfs/QmRKGpFfR32FVXdvJiHfo4WJ5TDYBsM1P9raAp1p6APWSp", }, { name: "simpleDir", data: flatDir, wrap: "t", expect: wrapped("t"), path: "/ipfs/Qmc3nGXm1HtUVCmnXLQHvWcNwfdZGpfg2SRm1CxLf7Q2Rm", }, { name: "twoLevelDir", data: twoLevelDir(), wrap: "t", expect: wrapped("t"), path: "/ipfs/QmPwsL3T5sWhDmmAWZHAzyjKtMVDS9a11aHNRqb3xoVnmg", }, // wrapped { name: "addWrapped", path: "/ipfs/QmVE9rNpj5doj7XHzp5zMUxD7BJgXEqx4pe3xZ3JBReWHE", data: func() files.Node { return files.NewBytesFile([]byte(helloStr)) }, wrap: "foo", expect: wrapped("foo"), }, // hidden { name: "hiddenFilesAdded", data: func() files.Node { return files.NewMapDirectory(map[string]files.Node{ ".bar": files.NewBytesFile([]byte("hello2")), "bar": files.NewBytesFile([]byte("hello2")), "foo": files.NewBytesFile([]byte("hello1")), }) }, wrap: "t", expect: wrapped("t"), path: "/ipfs/QmPXLSBX382vJDLrGakcbrZDkU3grfkjMox7EgSC9KFbtQ", }, // NoCopy { name: "simpleNoCopy", data: realFile, path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", opts: []options.UnixfsAddOption{options.Unixfs.Nocopy(true)}, }, { name: "noCopyNoRaw", data: realFile, path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", opts: []options.UnixfsAddOption{options.Unixfs.Nocopy(true), options.Unixfs.RawLeaves(false)}, err: "nocopy option requires '--raw-leaves' to be enabled as well", }, { name: "noCopyNoPath", data: strFile(helloStr), path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", opts: []options.UnixfsAddOption{options.Unixfs.Nocopy(true)}, err: helpers.ErrMissingFsRef.Error(), }, // Events / Progress { name: "simpleAddEvent", data: strFile(helloStr), path: "/ipfs/bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", events: []coreiface.AddEvent{ {Name: "bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa", Path: p("bafkreidi4zlleupgp2bvrpxyja5lbvi4mym7hz5bvhyoowby2qp7g2hxfa"), Size: strconv.Itoa(len(helloStr))}, }, opts: []options.UnixfsAddOption{options.Unixfs.RawLeaves(true)}, }, { name: "silentAddEvent", data: twoLevelDir(), path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", events: []coreiface.AddEvent{ {Name: "abc", Path: p("QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt"), Size: "62"}, {Name: "", Path: p("QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr"), Size: "229"}, }, opts: []options.UnixfsAddOption{options.Unixfs.Silent(true)}, }, { name: "dirAddEvents", data: twoLevelDir(), path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", events: []coreiface.AddEvent{ {Name: "abc/def", Path: p("QmNyJpQkU1cEkBwMDhDNFstr42q55mqG5GE5Mgwug4xyGk"), Size: "13"}, {Name: "bar", Path: p("QmS21GuXiRMvJKHos4ZkEmQDmRBqRaF5tQS2CQCu2ne9sY"), Size: "14"}, {Name: "foo", Path: p("QmfAjGiVpTN56TXi6SBQtstit5BEw3sijKj1Qkxn6EXKzJ"), Size: "14"}, {Name: "abc", Path: p("QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt"), Size: "62"}, {Name: "", Path: p("QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr"), Size: "229"}, }, }, { name: "progress1M", data: func() files.Node { return files.NewReaderFile(bytes.NewReader(bytes.Repeat([]byte{0}, 1000000))) }, path: "/ipfs/QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", events: []coreiface.AddEvent{ {Name: "", Bytes: 262144}, {Name: "", Bytes: 524288}, {Name: "", Bytes: 786432}, {Name: "", Bytes: 1000000}, {Name: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Path: p("QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD"), Size: "1000256"}, }, wrap: "", opts: []options.UnixfsAddOption{options.Unixfs.Progress(true)}, }, } for _, testCase := range cases { t.Run(testCase.name, func(t *testing.T) { ctx, cancel := context.WithCancel(ctx) defer cancel() // recursive logic data := testCase.data() if testCase.wrap != "" { data = files.NewMapDirectory(map[string]files.Node{ testCase.wrap: data, }) } // handle events if relevant to test case opts := testCase.opts eventOut := make(chan any) var evtWg sync.WaitGroup if len(testCase.events) > 0 { opts = append(opts, options.Unixfs.Events(eventOut)) evtWg.Go(func() { expected := testCase.events for evt := range eventOut { event, ok := evt.(*coreiface.AddEvent) if !ok { t.Error("unexpected event type") continue } if len(expected) < 1 { t.Error("got more events than expected") continue } if expected[0].Size != event.Size { t.Errorf("Event.Size didn't match, %s != %s", expected[0].Size, event.Size) } if expected[0].Name != event.Name { t.Errorf("Event.Name didn't match, %s != %s", expected[0].Name, event.Name) } if (expected[0].Path != path.ImmutablePath{} && event.Path != path.ImmutablePath{}) { if expected[0].Path.RootCid().String() != event.Path.RootCid().String() { t.Errorf("Event.Hash didn't match, %s != %s", expected[0].Path, event.Path) } } else if event.Path != expected[0].Path { t.Errorf("Event.Hash didn't match, %s != %s", expected[0].Path, event.Path) } if expected[0].Bytes != event.Bytes { t.Errorf("Event.Bytes didn't match, %d != %d", expected[0].Bytes, event.Bytes) } expected = expected[1:] } if len(expected) > 0 { t.Errorf("%d event(s) didn't arrive", len(expected)) } }) } tapi, err := api.WithOptions(testCase.apiOpts...) if err != nil { t.Fatal(err) } // Add! p, err := tapi.Unixfs().Add(ctx, data, opts...) close(eventOut) evtWg.Wait() if testCase.err != "" { if err == nil { t.Fatalf("expected an error: %s", testCase.err) } if err.Error() != testCase.err { t.Fatalf("expected an error: '%s' != '%s'", err.Error(), testCase.err) } return } if err != nil { t.Fatal(err) } if p.String() != testCase.path { t.Errorf("expected path %s, got: %s", testCase.path, p) } // compare file structure with Unixfs().Get var cmpFile func(origName string, orig files.Node, gotName string, got files.Node) cmpFile = func(origName string, orig files.Node, gotName string, got files.Node) { _, origDir := orig.(files.Directory) _, gotDir := got.(files.Directory) if origName != gotName { t.Errorf("file name mismatch, orig='%s', got='%s'", origName, gotName) } if origDir != gotDir { t.Fatalf("file type mismatch on %s", origName) } if !gotDir { defer orig.Close() defer got.Close() do, err := io.ReadAll(orig.(files.File)) if err != nil { t.Fatal(err) } dg, err := io.ReadAll(got.(files.File)) if err != nil { t.Fatal(err) } if !bytes.Equal(do, dg) { t.Fatal("data not equal") } return } origIt := orig.(files.Directory).Entries() gotIt := got.(files.Directory).Entries() for { if origIt.Next() { if !gotIt.Next() { t.Fatal("gotIt out of entries before origIt") } } else { if gotIt.Next() { t.Fatal("origIt out of entries before gotIt") } break } cmpFile(origIt.Name(), origIt.Node(), gotIt.Name(), gotIt.Node()) } if origIt.Err() != nil { t.Fatal(origIt.Err()) } if gotIt.Err() != nil { t.Fatal(gotIt.Err()) } } f, err := tapi.Unixfs().Get(ctx, p) if err != nil { t.Fatal(err) } orig := testCase.data() if testCase.expect != nil { orig = testCase.expect(orig) } cmpFile("", orig, "", f) }) } } func (tp *TestSuite) TestAddPinned(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } _, err = api.Unixfs().Add(ctx, strFile(helloStr)(), options.Unixfs.Pin(true, "")) if err != nil { t.Fatal(err) } pins, err := accPins(ctx, api) if err != nil { t.Fatal(err) } if len(pins) != 1 { t.Fatalf("expected 1 pin, got %d", len(pins)) } if pins[0].Path().String() != "/ipfs/QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk" { t.Fatalf("got unexpected pin: %s", pins[0].Path().String()) } } func (tp *TestSuite) TestAddHashOnly(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } p, err := api.Unixfs().Add(ctx, strFile(helloStr)(), options.Unixfs.HashOnly(true)) if err != nil { t.Fatal(err) } if p.String() != hello { t.Errorf("unexpected path: %s", p.String()) } _, err = api.Block().Get(ctx, p) if err == nil { t.Fatal("expected an error") } if !ipld.IsNotFound(err) { t.Errorf("unexpected error: %s", err.Error()) } } func (tp *TestSuite) TestGetEmptyFile(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } _, err = api.Unixfs().Add(ctx, files.NewBytesFile([]byte{})) if err != nil { t.Fatal(err) } emptyFilePath, err := path.NewPath(emptyFile) if err != nil { t.Fatal(err) } r, err := api.Unixfs().Get(ctx, emptyFilePath) if err != nil { t.Fatal(err) } buf := make([]byte, 1) // non-zero so that Read() actually tries to read n, err := io.ReadFull(r.(files.File), buf) if err != nil && err != io.EOF { t.Error(err) } if !bytes.HasPrefix(buf, []byte{0x00}) { t.Fatalf("expected empty data, got [%s] [read=%d]", buf, n) } } func (tp *TestSuite) TestGetDir(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } edir := unixfs.EmptyDirNode() err = api.Dag().Add(ctx, edir) if err != nil { t.Fatal(err) } p := path.FromCid(edir.Cid()) if p.String() != path.FromCid(edir.Cid()).String() { t.Fatalf("expected path %s, got: %s", edir.Cid(), p.String()) } r, err := api.Unixfs().Get(ctx, path.FromCid(edir.Cid())) if err != nil { t.Fatal(err) } if _, ok := r.(files.Directory); !ok { t.Fatalf("expected a directory") } } func (tp *TestSuite) TestGetNonUnixfs(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } nd := new(mdag.ProtoNode) err = api.Dag().Add(ctx, nd) if err != nil { t.Fatal(err) } _, err = api.Unixfs().Get(ctx, path.FromCid(nd.Cid())) if !strings.Contains(err.Error(), "proto:") || !strings.Contains(err.Error(), "required field") { t.Fatalf("expected \"proto: required field\", got: %q", err) } } func (tp *TestSuite) TestLs(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } r := strings.NewReader("content-of-file") p, err := api.Unixfs().Add(ctx, files.NewMapDirectory(map[string]files.Node{ "name-of-file": files.NewReaderFile(r), "name-of-symlink": files.NewLinkFile("/foo/bar", nil), })) if err != nil { t.Fatal(err) } errCh := make(chan error, 1) entries := make(chan coreiface.DirEntry) go func() { errCh <- api.Unixfs().Ls(ctx, p, entries) }() entry, ok := <-entries if !ok { t.Fatal("expected another entry") } if entry.Size != 15 { t.Errorf("expected size = 15, got %d", entry.Size) } if entry.Name != "name-of-file" { t.Errorf("expected name = name-of-file, got %s", entry.Name) } if entry.Type != coreiface.TFile { t.Errorf("wrong type %s", entry.Type) } if entry.Cid.String() != "QmX3qQVKxDGz3URVC3861Z3CKtQKGBn6ffXRBBWGMFz9Lr" { t.Errorf("expected cid = QmX3qQVKxDGz3URVC3861Z3CKtQKGBn6ffXRBBWGMFz9Lr, got %s", entry.Cid) } entry, ok = <-entries if !ok { t.Fatal("expected another entry") } if entry.Type != coreiface.TSymlink { t.Errorf("wrong type %s", entry.Type) } if entry.Name != "name-of-symlink" { t.Errorf("expected name = name-of-symlink, got %s", entry.Name) } if entry.Target != "/foo/bar" { t.Errorf("expected symlink target to be /foo/bar, got %s", entry.Target) } _, ok = <-entries if ok { t.Errorf("didn't expect a another link") } if err = <-errCh; err != nil { t.Error(err) } } func (tp *TestSuite) TestEntriesExpired(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } r := strings.NewReader("content-of-file") p, err := api.Unixfs().Add(ctx, files.NewMapDirectory(map[string]files.Node{ "name-of-file": files.NewReaderFile(r), })) if err != nil { t.Fatal(err) } ctx, cancel = context.WithCancel(ctx) nd, err := api.Unixfs().Get(ctx, p) if err != nil { t.Fatal(err) } cancel() it := files.ToDir(nd).Entries() if it == nil { t.Fatal("it was nil") } if it.Next() { t.Fatal("Next succeeded") } if it.Err() != context.Canceled { t.Fatalf("unexpected error %s", it.Err()) } if it.Next() { t.Fatal("Next succeeded") } } func (tp *TestSuite) TestLsEmptyDir(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } p, err := api.Unixfs().Add(ctx, files.NewSliceDirectory([]files.DirEntry{})) if err != nil { t.Fatal(err) } errCh := make(chan error, 1) links := make(chan coreiface.DirEntry) go func() { errCh <- api.Unixfs().Ls(ctx, p, links) }() var count int for range links { count++ } if err = <-errCh; err != nil { t.Fatal(err) } if count != 0 { t.Fatalf("expected 0 links, got %d", count) } } // TODO(lgierth) this should test properly, with len(links) > 0 func (tp *TestSuite) TestLsNonUnixfs(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } nd, err := cbor.WrapObject(map[string]any{"foo": "bar"}, math.MaxUint64, -1) if err != nil { t.Fatal(err) } err = api.Dag().Add(ctx, nd) if err != nil { t.Fatal(err) } errCh := make(chan error, 1) links := make(chan coreiface.DirEntry) go func() { errCh <- api.Unixfs().Ls(ctx, path.FromCid(nd.Cid()), links) }() var count int for range links { count++ } if err = <-errCh; err != nil { t.Fatal(err) } if count != 0 { t.Fatalf("expected 0 links, got %d", count) } } type closeTestF struct { files.File closed bool t *testing.T } type closeTestD struct { files.Directory closed bool t *testing.T } func (f *closeTestD) Close() error { f.t.Helper() if f.closed { f.t.Fatal("already closed") } f.closed = true return nil } func (f *closeTestF) Close() error { if f.closed { f.t.Fatal("already closed") } f.closed = true return nil } func (tp *TestSuite) TestAddCloses(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } n4 := &closeTestF{files.NewBytesFile([]byte("foo")), false, t} d3 := &closeTestD{files.NewMapDirectory(map[string]files.Node{ "sub": n4, }), false, t} n2 := &closeTestF{files.NewBytesFile([]byte("bar")), false, t} n1 := &closeTestF{files.NewBytesFile([]byte("baz")), false, t} d0 := &closeTestD{files.NewMapDirectory(map[string]files.Node{ "a": d3, "b": n1, "c": n2, }), false, t} _, err = api.Unixfs().Add(ctx, d0) if err != nil { t.Fatal(err) } for i, n := range []*closeTestF{n1, n2, n4} { if !n.closed { t.Errorf("file %d not closed!", i) } } for i, n := range []*closeTestD{d0, d3} { if !n.closed { t.Errorf("dir %d not closed!", i) } } } func (tp *TestSuite) TestGetSeek(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } dataSize := int64(100000) tf := files.NewReaderFile(io.LimitReader(rand.New(rand.NewSource(1403768328)), dataSize)) p, err := api.Unixfs().Add(ctx, tf, options.Unixfs.Chunker("size-100")) if err != nil { t.Fatal(err) } r, err := api.Unixfs().Get(ctx, p) if err != nil { t.Fatal(err) } f := files.ToFile(r) if f == nil { t.Fatal("not a file") } orig := make([]byte, dataSize) if _, err := io.ReadFull(f, orig); err != nil { t.Fatal(err) } f.Close() origR := bytes.NewReader(orig) r, err = api.Unixfs().Get(ctx, p) if err != nil { t.Fatal(err) } f = files.ToFile(r) if f == nil { t.Fatal("not a file") } test := func(offset int64, whence int, read int, expect int64, shouldEof bool) { t.Run(fmt.Sprintf("seek%d+%d-r%d-%d", whence, offset, read, expect), func(t *testing.T) { n, err := f.Seek(offset, whence) if err != nil { t.Fatal(err) } origN, err := origR.Seek(offset, whence) if err != nil { t.Fatal(err) } if n != origN { t.Fatalf("offsets didn't match, expected %d, got %d", origN, n) } buf := make([]byte, read) origBuf := make([]byte, read) origRead, err := origR.Read(origBuf) if err != nil { t.Fatalf("orig: %s", err) } r, err := io.ReadFull(f, buf) switch { case shouldEof && err != nil && err != io.ErrUnexpectedEOF: fallthrough case !shouldEof && err != nil: t.Fatalf("f: %s", err) case shouldEof: _, err := f.Read([]byte{0}) if err != io.EOF { t.Fatal("expected EOF") } _, err = origR.Read([]byte{0}) if err != io.EOF { t.Fatal("expected EOF (orig)") } } if int64(r) != expect { t.Fatal("read wrong amount of data") } if r != origRead { t.Fatal("read different amount of data than bytes.Reader") } if !bytes.Equal(buf, origBuf) { fmt.Fprintf(os.Stderr, "original:\n%s\n", hex.Dump(origBuf)) fmt.Fprintf(os.Stderr, "got:\n%s\n", hex.Dump(buf)) t.Fatal("data didn't match") } }) } test(3, io.SeekCurrent, 10, 10, false) test(3, io.SeekCurrent, 10, 10, false) test(500, io.SeekCurrent, 10, 10, false) test(350, io.SeekStart, 100, 100, false) test(-123, io.SeekCurrent, 100, 100, false) test(0, io.SeekStart, int(dataSize), dataSize, false) test(dataSize-50, io.SeekStart, 100, 50, true) test(-5, io.SeekEnd, 100, 5, true) } func (tp *TestSuite) TestGetReadAt(t *testing.T) { ctx := t.Context() api, err := tp.makeAPI(t, ctx) if err != nil { t.Fatal(err) } dataSize := int64(100000) tf := files.NewReaderFile(io.LimitReader(rand.New(rand.NewSource(1403768328)), dataSize)) p, err := api.Unixfs().Add(ctx, tf, options.Unixfs.Chunker("size-100")) if err != nil { t.Fatal(err) } r, err := api.Unixfs().Get(ctx, p) if err != nil { t.Fatal(err) } f, ok := r.(interface { files.File io.ReaderAt }) if !ok { t.Skip("ReaderAt not implemented") } orig := make([]byte, dataSize) if _, err := io.ReadFull(f, orig); err != nil { t.Fatal(err) } f.Close() origR := bytes.NewReader(orig) if _, err := api.Unixfs().Get(ctx, p); err != nil { t.Fatal(err) } test := func(offset int64, read int, expect int64, shouldEof bool) { t.Run(fmt.Sprintf("readat%d-r%d-%d", offset, read, expect), func(t *testing.T) { origBuf := make([]byte, read) origRead, err := origR.ReadAt(origBuf, offset) if err != nil && err != io.EOF { t.Fatalf("orig: %s", err) } buf := make([]byte, read) r, err := f.ReadAt(buf, offset) if shouldEof { if err != io.EOF { t.Fatal("expected EOF, got: ", err) } } else if err != nil { t.Fatal("got: ", err) } if int64(r) != expect { t.Fatal("read wrong amount of data") } if r != origRead { t.Fatal("read different amount of data than bytes.Reader") } if !bytes.Equal(buf, origBuf) { fmt.Fprintf(os.Stderr, "original:\n%s\n", hex.Dump(origBuf)) fmt.Fprintf(os.Stderr, "got:\n%s\n", hex.Dump(buf)) t.Fatal("data didn't match") } }) } test(3, 10, 10, false) test(13, 10, 10, false) test(513, 10, 10, false) test(350, 100, 100, false) test(0, int(dataSize), dataSize, false) test(dataSize-50, 100, 50, true) } ================================================ FILE: core/coreiface/unixfs.go ================================================ package iface import ( "context" "iter" "os" "time" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" "github.com/ipfs/kubo/core/coreiface/options" ) type AddEvent struct { Name string Path path.ImmutablePath Bytes int64 `json:",omitempty"` Size string `json:",omitempty"` Mode os.FileMode `json:",omitempty"` Mtime int64 `json:",omitempty"` MtimeNsecs int `json:",omitempty"` } // FileType is an enum of possible UnixFS file types. type FileType int32 const ( // TUnknown means the file type isn't known (e.g., it hasn't been // resolved). TUnknown FileType = iota // TFile is a regular file. TFile // TDirectory is a directory. TDirectory // TSymlink is a symlink. TSymlink ) func (t FileType) String() string { switch t { case TUnknown: return "unknown" case TFile: return "file" case TDirectory: return "directory" case TSymlink: return "symlink" default: return "" } } // DirEntry is a directory entry returned by `Ls`. type DirEntry struct { Name string Cid cid.Cid // Only filled when asked to resolve the directory entry. Size uint64 // The size of the file in bytes (or the size of the symlink). Type FileType // The type of the file. Target string // The symlink target (if a symlink). Mode os.FileMode ModTime time.Time } // UnixfsAPI is the basic interface to immutable files in IPFS // NOTE: This API is heavily WIP, things are guaranteed to break frequently type UnixfsAPI interface { // Add imports the data from the reader into merkledag file // // TODO: a long useful comment on how to use this for many different scenarios Add(context.Context, files.Node, ...options.UnixfsAddOption) (path.ImmutablePath, error) // Get returns a read-only handle to a file tree referenced by a path // // Note that some implementations of this API may apply the specified context // to operations performed on the returned file Get(context.Context, path.Path) (files.Node, error) // Ls writes the links in a directory to the DirEntry channel. Links aren't // guaranteed to be returned in order. If an error occurs or the context is // canceled, the DirEntry channel is closed and an error is returned. // // Example: // // dirs := make(chan DirEntry) // lsErr := make(chan error, 1) // go func() { // lsErr <- Ls(ctx, p, dirs) // }() // for dirEnt := range dirs { // fmt.Println("Dir name:", dirEnt.Name) // } // err := <-lsErr // if err != nil { // return fmt.Errorf("error listing directory: %w", err) // } Ls(context.Context, path.Path, chan<- DirEntry, ...options.UnixfsLsOption) error } // LsIter returns a go iterator that allows ranging over DirEntry results. // Iteration stops if the context is canceled or if the iterator yields an // error. // // Example: // // for dirEnt, err := LsIter(ctx, ufsAPI, p) { // if err != nil { // return fmt.Errorf("error listing directory: %w", err) // } // fmt.Println("Dir name:", dirEnt.Name) // } func LsIter(ctx context.Context, api UnixfsAPI, p path.Path, opts ...options.UnixfsLsOption) iter.Seq2[DirEntry, error] { return func(yield func(DirEntry, error) bool) { ctx, cancel := context.WithCancel(ctx) defer cancel() // cancel Ls if done iterating early dirs := make(chan DirEntry) lsErr := make(chan error, 1) go func() { lsErr <- api.Ls(ctx, p, dirs, opts...) }() for dirEnt := range dirs { if !yield(dirEnt, nil) { return } } if err := <-lsErr; err != nil { yield(DirEntry{}, err) } } } ================================================ FILE: core/coreiface/util.go ================================================ package iface import ( "context" "io" ) type Reader interface { ReadSeekCloser Size() uint64 CtxReadFull(context.Context, []byte) (int, error) } // A ReadSeekCloser implements interfaces to read, copy, seek and close. type ReadSeekCloser interface { io.Reader io.Seeker io.Closer io.WriterTo } ================================================ FILE: core/corerepo/gc.go ================================================ package corerepo import ( "bytes" "context" "errors" "time" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/gc" "github.com/ipfs/kubo/repo" "github.com/dustin/go-humanize" "github.com/ipfs/boxo/mfs" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("corerepo") var ErrMaxStorageExceeded = errors.New("maximum storage limit exceeded. Try to unpin some files") type GC struct { Node *core.IpfsNode Repo repo.Repo StorageMax uint64 StorageGC uint64 SlackGB uint64 Storage uint64 } func NewGC(n *core.IpfsNode) (*GC, error) { r := n.Repo cfg, err := r.Config() if err != nil { return nil, err } // check if cfg has these fields initialized // TODO: there should be a general check for all of the cfg fields // maybe distinguish between user config file and default struct? if cfg.Datastore.StorageMax == "" { if err := r.SetConfigKey("Datastore.StorageMax", "10GB"); err != nil { return nil, err } cfg.Datastore.StorageMax = "10GB" } if cfg.Datastore.StorageGCWatermark == 0 { if err := r.SetConfigKey("Datastore.StorageGCWatermark", 90); err != nil { return nil, err } cfg.Datastore.StorageGCWatermark = 90 } storageMax, err := humanize.ParseBytes(cfg.Datastore.StorageMax) if err != nil { return nil, err } storageGC := storageMax * uint64(cfg.Datastore.StorageGCWatermark) / 100 // calculate the slack space between StorageMax and StorageGCWatermark // used to limit GC duration slackGB := max((storageMax-storageGC)/10e9, 1) return &GC{ Node: n, Repo: r, StorageMax: storageMax, StorageGC: storageGC, SlackGB: slackGB, }, nil } func BestEffortRoots(filesRoot *mfs.Root) ([]cid.Cid, error) { rootDag, err := filesRoot.GetDirectory().GetNode() if err != nil { return nil, err } return []cid.Cid{rootDag.Cid()}, nil } func GarbageCollect(n *core.IpfsNode, ctx context.Context) error { roots, err := BestEffortRoots(n.FilesRoot) if err != nil { return err } rmed := gc.GC(ctx, n.Blockstore, n.Repo.Datastore(), n.Pinning, roots) return CollectResult(ctx, rmed, nil) } // CollectResult collects the output of a garbage collection run and calls the // given callback for each object removed. It also collects all errors into a // MultiError which is returned after the gc is completed. func CollectResult(ctx context.Context, gcOut <-chan gc.Result, cb func(cid.Cid)) error { var errors []error loop: for { select { case res, ok := <-gcOut: if !ok { break loop } if res.Error != nil { errors = append(errors, res.Error) } else if res.KeyRemoved.Defined() && cb != nil { cb(res.KeyRemoved) } case <-ctx.Done(): errors = append(errors, ctx.Err()) break loop } } switch len(errors) { case 0: return nil case 1: return errors[0] default: return NewMultiError(errors...) } } // NewMultiError creates a new MultiError object from a given slice of errors. func NewMultiError(errs ...error) *MultiError { return &MultiError{errs[:len(errs)-1], errs[len(errs)-1]} } // MultiError contains the results of multiple errors. type MultiError struct { Errors []error Summary error } func (e *MultiError) Error() string { var buf bytes.Buffer for _, err := range e.Errors { buf.WriteString(err.Error()) buf.WriteString("; ") } buf.WriteString(e.Summary.Error()) return buf.String() } func GarbageCollectAsync(n *core.IpfsNode, ctx context.Context) <-chan gc.Result { roots, err := BestEffortRoots(n.FilesRoot) if err != nil { out := make(chan gc.Result) out <- gc.Result{Error: err} close(out) return out } return gc.GC(ctx, n.Blockstore, n.Repo.Datastore(), n.Pinning, roots) } func PeriodicGC(ctx context.Context, node *core.IpfsNode) error { cfg, err := node.Repo.Config() if err != nil { return err } if cfg.Datastore.GCPeriod == "" { cfg.Datastore.GCPeriod = "1h" } period, err := time.ParseDuration(cfg.Datastore.GCPeriod) if err != nil { return err } if int64(period) == 0 { // if duration is 0, it means GC is disabled. return nil } gc, err := NewGC(node) if err != nil { return err } for { select { case <-ctx.Done(): return nil case <-time.After(period): // the private func maybeGC doesn't compute storageMax, storageGC, slackGC so that they are not re-computed for every cycle if err := gc.maybeGC(ctx, 0); err != nil { log.Error(err) } } } } func ConditionalGC(ctx context.Context, node *core.IpfsNode, offset uint64) error { gc, err := NewGC(node) if err != nil { return err } return gc.maybeGC(ctx, offset) } func (gc *GC) maybeGC(ctx context.Context, offset uint64) error { storage, err := gc.Repo.GetStorageUsage(ctx) if err != nil { return err } if storage+offset > gc.StorageGC { if storage+offset > gc.StorageMax { log.Warnf("pre-GC: %s", ErrMaxStorageExceeded) } // Do GC here log.Info("Watermark exceeded. Starting repo GC...") if err := GarbageCollect(gc.Node, ctx); err != nil { return err } log.Infof("Repo GC done. See `ipfs repo stat` to see how much space got freed.\n") } return nil } ================================================ FILE: core/corerepo/stat.go ================================================ package corerepo import ( "fmt" "math" context "context" "github.com/ipfs/kubo/core" fsrepo "github.com/ipfs/kubo/repo/fsrepo" humanize "github.com/dustin/go-humanize" ) // SizeStat wraps information about the repository size and its limit. type SizeStat struct { RepoSize uint64 // size in bytes StorageMax uint64 // size in bytes } // Stat wraps information about the objects stored on disk. type Stat struct { SizeStat NumObjects uint64 RepoPath string Version string } // NoLimit represents the value for unlimited storage const NoLimit uint64 = math.MaxUint64 // RepoStat returns a *Stat object with all the fields set. func RepoStat(ctx context.Context, n *core.IpfsNode) (Stat, error) { sizeStat, err := RepoSize(ctx, n) if err != nil { return Stat{}, err } allKeys, err := n.Blockstore.AllKeysChan(ctx) if err != nil { return Stat{}, err } count := uint64(0) for range allKeys { count++ } path, err := fsrepo.BestKnownPath() if err != nil { return Stat{}, err } return Stat{ SizeStat: SizeStat{ RepoSize: sizeStat.RepoSize, StorageMax: sizeStat.StorageMax, }, NumObjects: count, RepoPath: path, Version: fmt.Sprintf("fs-repo@%d", fsrepo.RepoVersion), }, nil } // RepoSize returns a *Stat object with the RepoSize and StorageMax fields set. func RepoSize(ctx context.Context, n *core.IpfsNode) (SizeStat, error) { r := n.Repo cfg, err := r.Config() if err != nil { return SizeStat{}, err } usage, err := r.GetStorageUsage(ctx) if err != nil { return SizeStat{}, err } storageMax := NoLimit if cfg.Datastore.StorageMax != "" { storageMax, err = humanize.ParseBytes(cfg.Datastore.StorageMax) if err != nil { return SizeStat{}, err } } return SizeStat{ RepoSize: usage, StorageMax: storageMax, }, nil } ================================================ FILE: core/coreunix/add.go ================================================ package coreunix import ( "context" "errors" "fmt" "io" "os" gopath "path" "strconv" "time" bstore "github.com/ipfs/boxo/blockstore" chunker "github.com/ipfs/boxo/chunker" "github.com/ipfs/boxo/files" posinfo "github.com/ipfs/boxo/filestore/posinfo" dag "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/boxo/ipld/unixfs/importer/balanced" ihelper "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" "github.com/ipfs/boxo/ipld/unixfs/importer/trickle" uio "github.com/ipfs/boxo/ipld/unixfs/io" "github.com/ipfs/boxo/mfs" "github.com/ipfs/boxo/path" pin "github.com/ipfs/boxo/pinning/pinner" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" "github.com/ipfs/kubo/config" coreiface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/tracing" ) var log = logging.Logger("coreunix") // how many bytes of progress to wait before sending a progress update message const progressReaderIncrement = 1024 * 256 var liveCacheSize = uint64(256 << 10) type Link struct { Name, Hash string Size uint64 } type syncer interface { Sync() error } // NewAdder Returns a new Adder used for a file add operation. func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCLocker, ds ipld.DAGService) (*Adder, error) { bufferedDS := ipld.NewBufferedDAG(ctx, ds) return &Adder{ ctx: ctx, pinning: p, gcLocker: bs, dagService: ds, bufferedDS: bufferedDS, Progress: false, Pin: true, Trickle: false, MaxLinks: ihelper.DefaultLinksPerBlock, MaxHAMTFanout: uio.DefaultShardWidth, Chunker: "", IncludeEmptyDirs: config.DefaultUnixFSIncludeEmptyDirs, }, nil } // Adder holds the switches passed to the `add` command. type Adder struct { ctx context.Context pinning pin.Pinner gcLocker bstore.GCLocker dagService ipld.DAGService bufferedDS *ipld.BufferedDAG Out chan<- any Progress bool Pin bool PinName string Trickle bool RawLeaves bool MaxLinks int MaxDirectoryLinks int MaxHAMTFanout int SizeEstimationMode *uio.SizeEstimationMode Silent bool NoCopy bool Chunker string mroot *mfs.Root unlocker bstore.Unlocker tempRoot cid.Cid CidBuilder cid.Builder liveNodes uint64 PreserveMode bool PreserveMtime bool FileMode os.FileMode FileMtime time.Time IncludeEmptyDirs bool } func (adder *Adder) mfsRoot() (*mfs.Root, error) { if adder.mroot != nil { return adder.mroot, nil } // Note, this adds it to DAGService already. mr, err := mfs.NewEmptyRoot(adder.ctx, adder.dagService, nil, nil, mfs.MkdirOpts{ CidBuilder: adder.CidBuilder, MaxLinks: adder.MaxDirectoryLinks, MaxHAMTFanout: adder.MaxHAMTFanout, SizeEstimationMode: adder.SizeEstimationMode, }) if err != nil { return nil, err } adder.mroot = mr return adder.mroot, nil } // SetMfsRoot sets `r` as the root for Adder. func (adder *Adder) SetMfsRoot(r *mfs.Root) { adder.mroot = r } // Constructs a node from reader's data, and adds it. Doesn't pin. func (adder *Adder) add(reader io.Reader) (ipld.Node, error) { chnk, err := chunker.FromString(reader, adder.Chunker) if err != nil { return nil, err } maxLinks := ihelper.DefaultLinksPerBlock if adder.MaxLinks > 0 { maxLinks = adder.MaxLinks } params := ihelper.DagBuilderParams{ Dagserv: adder.bufferedDS, RawLeaves: adder.RawLeaves, Maxlinks: maxLinks, NoCopy: adder.NoCopy, CidBuilder: adder.CidBuilder, FileMode: adder.FileMode, FileModTime: adder.FileMtime, } db, err := params.New(chnk) if err != nil { return nil, err } var nd ipld.Node if adder.Trickle { nd, err = trickle.Layout(db) } else { nd, err = balanced.Layout(db) } if err != nil { return nil, err } return nd, adder.bufferedDS.Commit() } // RootNode returns the mfs root node func (adder *Adder) curRootNode() (ipld.Node, error) { mr, err := adder.mfsRoot() if err != nil { return nil, err } root, err := mr.GetDirectory().GetNode() if err != nil { return nil, err } // if one root file, use that hash as root. if len(root.Links()) == 1 { nd, err := root.Links()[0].GetNode(adder.ctx, adder.dagService) if err != nil { return nil, err } root = nd } return root, err } // PinRoot recursively pins the root node of Adder with an optional name and // writes the pin state to the backing datastore. If name is empty, the pin // will be created without a name. func (adder *Adder) PinRoot(ctx context.Context, root ipld.Node, name string) error { ctx, span := tracing.Span(ctx, "CoreUnix.Adder", "PinRoot") defer span.End() if !adder.Pin { return nil } rnk := root.Cid() err := adder.dagService.Add(ctx, root) if err != nil { return err } if adder.tempRoot.Defined() { err := adder.pinning.Unpin(ctx, adder.tempRoot, true) if err != nil { return err } adder.tempRoot = rnk } err = adder.pinning.PinWithMode(ctx, rnk, pin.Recursive, name) if err != nil { return err } return adder.pinning.Flush(ctx) } func (adder *Adder) outputDirs(path string, fsn mfs.FSNode) error { switch fsn := fsn.(type) { case *mfs.File: return nil case *mfs.Directory: names, err := fsn.ListNames(adder.ctx) if err != nil { return err } for _, name := range names { child, err := fsn.Child(name) if err != nil { return err } childpath := gopath.Join(path, name) err = adder.outputDirs(childpath, child) if err != nil { return err } fsn.Uncache(name) } nd, err := fsn.GetNode() if err != nil { return err } return outputDagnode(adder.Out, path, nd) default: return fmt.Errorf("unrecognized fsn type: %#v", fsn) } } func (adder *Adder) addNode(node ipld.Node, path string) error { // patch it into the root if path == "" { path = node.Cid().String() } if pi, ok := node.(*posinfo.FilestoreNode); ok { node = pi.Node } mr, err := adder.mfsRoot() if err != nil { return err } dir := gopath.Dir(path) if dir != "." { opts := mfs.MkdirOpts{ Mkparents: true, Flush: false, CidBuilder: adder.CidBuilder, MaxLinks: adder.MaxDirectoryLinks, MaxHAMTFanout: adder.MaxHAMTFanout, SizeEstimationMode: adder.SizeEstimationMode, } if err := mfs.Mkdir(mr, dir, opts); err != nil { return err } } if err := mfs.PutNode(mr, path, node); err != nil { return err } if !adder.Silent { return outputDagnode(adder.Out, path, node) } return nil } // AddAllAndPin adds the given request's files and pin them. func (adder *Adder) AddAllAndPin(ctx context.Context, file files.Node) (ipld.Node, error) { ctx, span := tracing.Span(ctx, "CoreUnix.Adder", "AddAllAndPin") defer span.End() if adder.Pin { adder.unlocker = adder.gcLocker.PinLock(ctx) } defer func() { if adder.unlocker != nil { adder.unlocker.Unlock(ctx) } }() if err := adder.addFileNode(ctx, "", file, true); err != nil { return nil, err } // get root mr, err := adder.mfsRoot() if err != nil { return nil, err } var root mfs.FSNode rootdir := mr.GetDirectory() root = rootdir err = root.Flush() if err != nil { return nil, err } // if adding a file without wrapping, swap the root to it (when adding a // directory, mfs root is the directory) _, dir := file.(files.Directory) var name string if !dir { children, err := rootdir.ListNames(adder.ctx) if err != nil { return nil, err } if len(children) == 0 { return nil, fmt.Errorf("expected at least one child dir, got none") } // Replace root with the first child name = children[0] root, err = rootdir.Child(name) if err != nil { return nil, err } } err = mr.Close() if err != nil { return nil, err } nd, err := root.GetNode() if err != nil { return nil, err } // output directory events err = adder.outputDirs(name, root) if err != nil { return nil, err } if asyncDagService, ok := adder.dagService.(syncer); ok { err = asyncDagService.Sync() if err != nil { return nil, err } } if !adder.Pin { return nd, nil } if err := adder.PinRoot(ctx, nd, adder.PinName); err != nil { return nil, err } return nd, nil } func (adder *Adder) addFileNode(ctx context.Context, path string, file files.Node, toplevel bool) error { ctx, span := tracing.Span(ctx, "CoreUnix.Adder", "AddFileNode") defer span.End() defer file.Close() err := adder.maybePauseForGC(ctx) if err != nil { return err } if adder.PreserveMtime { adder.FileMtime = file.ModTime() } if adder.PreserveMode { adder.FileMode = file.Mode() } if adder.liveNodes >= liveCacheSize { // TODO: A smarter cache that uses some sort of lru cache with an eviction handler mr, err := adder.mfsRoot() if err != nil { return err } if err := mr.FlushMemFree(adder.ctx); err != nil { return err } adder.liveNodes = 0 } adder.liveNodes++ switch f := file.(type) { case files.Directory: return adder.addDir(ctx, path, f, toplevel) case *files.Symlink: return adder.addSymlink(ctx, path, f) case files.File: return adder.addFile(path, f) default: return errors.New("unknown file type") } } func (adder *Adder) addSymlink(ctx context.Context, path string, l *files.Symlink) error { sdata, err := unixfs.SymlinkData(l.Target) if err != nil { return err } if !adder.FileMtime.IsZero() { fsn, err := unixfs.FSNodeFromBytes(sdata) if err != nil { return err } fsn.SetModTime(adder.FileMtime) if sdata, err = fsn.GetBytes(); err != nil { return err } } dagnode := dag.NodeWithData(sdata) err = dagnode.SetCidBuilder(adder.CidBuilder) if err != nil { return err } err = adder.dagService.Add(adder.ctx, dagnode) if err != nil { return err } return adder.addNode(dagnode, path) } func (adder *Adder) addFile(path string, file files.File) error { // if the progress flag was specified, wrap the file so that we can send // progress updates to the client (over the output channel) var reader io.Reader = file if adder.Progress { rdr := &progressReader{file: reader, path: path, out: adder.Out} if fi, ok := file.(files.FileInfo); ok { reader = &progressReader2{rdr, fi} } else { reader = rdr } } dagnode, err := adder.add(reader) if err != nil { return err } // patch it into the root return adder.addNode(dagnode, path) } func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory, toplevel bool) error { log.Infof("adding directory: %s", path) // Peek at first entry to check if directory is empty. // We advance the iterator once here and continue from this position // in the processing loop below. This avoids allocating a slice to // collect all entries just to check for emptiness. it := dir.Entries() hasEntry := it.Next() if !hasEntry { if err := it.Err(); err != nil { return err } // Directory is empty. Skip it unless IncludeEmptyDirs is set or // this is the toplevel directory (we always include the root). if !adder.IncludeEmptyDirs && !toplevel { log.Debugf("skipping empty directory: %s", path) return nil } } // if we need to store mode or modification time then create a new root which includes that data if toplevel && (adder.FileMode != 0 || !adder.FileMtime.IsZero()) { mr, err := mfs.NewEmptyRoot(ctx, adder.dagService, nil, nil, mfs.MkdirOpts{ CidBuilder: adder.CidBuilder, MaxLinks: adder.MaxDirectoryLinks, MaxHAMTFanout: adder.MaxHAMTFanout, ModTime: adder.FileMtime, Mode: adder.FileMode, SizeEstimationMode: adder.SizeEstimationMode, }) if err != nil { return err } adder.SetMfsRoot(mr) } if !(toplevel && path == "") { mr, err := adder.mfsRoot() if err != nil { return err } err = mfs.Mkdir(mr, path, mfs.MkdirOpts{ Mkparents: true, Flush: false, CidBuilder: adder.CidBuilder, Mode: adder.FileMode, ModTime: adder.FileMtime, MaxLinks: adder.MaxDirectoryLinks, MaxHAMTFanout: adder.MaxHAMTFanout, SizeEstimationMode: adder.SizeEstimationMode, }) if err != nil { return err } } // Process directory entries. The iterator was already advanced once above // to peek for emptiness, so we start from that position. for hasEntry { fpath := gopath.Join(path, it.Name()) if err := adder.addFileNode(ctx, fpath, it.Node(), false); err != nil { return err } hasEntry = it.Next() } return it.Err() } func (adder *Adder) maybePauseForGC(ctx context.Context) error { ctx, span := tracing.Span(ctx, "CoreUnix.Adder", "MaybePauseForGC") defer span.End() if adder.unlocker != nil && adder.gcLocker.GCRequested(ctx) { rn, err := adder.curRootNode() if err != nil { return err } err = adder.PinRoot(ctx, rn, "") if err != nil { return err } adder.unlocker.Unlock(ctx) adder.unlocker = adder.gcLocker.PinLock(ctx) } return nil } // outputDagnode sends dagnode info over the output channel func outputDagnode(out chan<- any, name string, dn ipld.Node) error { if out == nil { return nil } o, err := getOutput(dn) if err != nil { return err } out <- &coreiface.AddEvent{ Path: o.Path, Name: name, Size: o.Size, } return nil } // from core/commands/object.go func getOutput(dagnode ipld.Node) (*coreiface.AddEvent, error) { c := dagnode.Cid() s, err := dagnode.Size() if err != nil { return nil, err } output := &coreiface.AddEvent{ Path: path.FromCid(c), Size: strconv.FormatUint(s, 10), } return output, nil } type progressReader struct { file io.Reader path string out chan<- any bytes int64 lastProgress int64 } func (i *progressReader) Read(p []byte) (int, error) { n, err := i.file.Read(p) i.bytes += int64(n) if i.bytes-i.lastProgress >= progressReaderIncrement || err == io.EOF { i.lastProgress = i.bytes i.out <- &coreiface.AddEvent{ Name: i.path, Bytes: i.bytes, } } return n, err } type progressReader2 struct { *progressReader files.FileInfo } func (i *progressReader2) Read(p []byte) (int, error) { return i.progressReader.Read(p) } ================================================ FILE: core/coreunix/add_test.go ================================================ package coreunix import ( "bytes" "context" "io" "math/rand" "os" "path/filepath" "testing" "time" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/gc" "github.com/ipfs/kubo/repo" "github.com/ipfs/boxo/blockservice" blockstore "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/files" pi "github.com/ipfs/boxo/filestore/posinfo" dag "github.com/ipfs/boxo/ipld/merkledag" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" syncds "github.com/ipfs/go-datastore/sync" config "github.com/ipfs/kubo/config" coreiface "github.com/ipfs/kubo/core/coreiface" ) const testPeerID = "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe" func TestAddMultipleGCLive(t *testing.T) { ctx := t.Context() r := &repo.Mock{ C: config.Config{ Identity: config.Identity{ PeerID: testPeerID, // required by offline node }, }, D: syncds.MutexWrap(datastore.NewMapDatastore()), } node, err := core.NewNode(ctx, &core.BuildCfg{Repo: r}) if err != nil { t.Fatal(err) } out := make(chan any, 10) adder, err := NewAdder(ctx, node.Pinning, node.Blockstore, node.DAG) if err != nil { t.Fatal(err) } adder.Out = out // make two files with pipes so we can 'pause' the add for timing of the test piper1, pipew1 := io.Pipe() hangfile1 := files.NewReaderFile(piper1) piper2, pipew2 := io.Pipe() hangfile2 := files.NewReaderFile(piper2) rfc := files.NewBytesFile([]byte("testfileA")) slf := files.NewMapDirectory(map[string]files.Node{ "a": hangfile1, "b": hangfile2, "c": rfc, }) go func() { defer close(out) _, _ = adder.AddAllAndPin(ctx, slf) // Ignore errors for clarity - the real bug would be gc'ing files while adding them, not this resultant error }() // Start writing the first file but don't close the stream if _, err := pipew1.Write([]byte("some data for file a")); err != nil { t.Fatal(err) } var gc1out <-chan gc.Result gc1started := make(chan struct{}) go func() { defer close(gc1started) gc1out = gc.GC(ctx, node.Blockstore, node.Repo.Datastore(), node.Pinning, nil) }() // Give GC goroutine time to reach GCLock (will block there waiting for adder) time.Sleep(time.Millisecond * 100) // GC shouldn't get the lock until after the file is completely added select { case <-gc1started: t.Fatal("gc shouldn't have started yet") default: } // finish write and unblock gc pipew1.Close() // Wait for GC to acquire the lock // The adder needs to finish processing file 'a' and call maybePauseForGC // when starting file 'b' before GC can proceed select { case <-gc1started: // GC got the lock as expected case <-time.After(5 * time.Second): t.Fatal("timeout waiting for GC to start - possible deadlock") } removedHashes := make(map[string]struct{}) for r := range gc1out { if r.Error != nil { t.Fatal(err) } removedHashes[r.KeyRemoved.String()] = struct{}{} } if _, err := pipew2.Write([]byte("some data for file b")); err != nil { t.Fatal(err) } var gc2out <-chan gc.Result gc2started := make(chan struct{}) go func() { defer close(gc2started) gc2out = gc.GC(ctx, node.Blockstore, node.Repo.Datastore(), node.Pinning, nil) }() // Give GC goroutine time to reach GCLock time.Sleep(time.Millisecond * 100) select { case <-gc2started: t.Fatal("gc shouldn't have started yet") default: } pipew2.Close() // Wait for second GC to acquire the lock // The adder needs to finish processing file 'b' and call maybePauseForGC // when starting file 'c' before GC can proceed select { case <-gc2started: // GC got the lock as expected case <-time.After(5 * time.Second): t.Fatal("timeout waiting for second GC to start - possible deadlock") } for r := range gc2out { if r.Error != nil { t.Fatal(err) } removedHashes[r.KeyRemoved.String()] = struct{}{} } for o := range out { if _, ok := removedHashes[o.(*coreiface.AddEvent).Path.RootCid().String()]; ok { t.Fatal("gc'ed a hash we just added") } } } func TestAddGCLive(t *testing.T) { ctx := t.Context() r := &repo.Mock{ C: config.Config{ Identity: config.Identity{ PeerID: testPeerID, // required by offline node }, }, D: syncds.MutexWrap(datastore.NewMapDatastore()), } node, err := core.NewNode(ctx, &core.BuildCfg{Repo: r}) if err != nil { t.Fatal(err) } out := make(chan any) adder, err := NewAdder(ctx, node.Pinning, node.Blockstore, node.DAG) if err != nil { t.Fatal(err) } adder.Out = out rfa := files.NewBytesFile([]byte("testfileA")) // make two files with pipes so we can 'pause' the add for timing of the test piper, pipew := io.Pipe() hangfile := files.NewReaderFile(piper) rfd := files.NewBytesFile([]byte("testfileD")) slf := files.NewMapDirectory(map[string]files.Node{ "a": rfa, "b": hangfile, "d": rfd, }) addDone := make(chan struct{}) go func() { defer close(addDone) defer close(out) _, err := adder.AddAllAndPin(ctx, slf) if err != nil { t.Error(err) } }() addedHashes := make(map[string]struct{}) select { case o := <-out: addedHashes[o.(*coreiface.AddEvent).Path.RootCid().String()] = struct{}{} case <-addDone: t.Fatal("add shouldn't complete yet") } var gcout <-chan gc.Result gcstarted := make(chan struct{}) go func() { defer close(gcstarted) gcout = gc.GC(ctx, node.Blockstore, node.Repo.Datastore(), node.Pinning, nil) }() // gc shouldn't start until we let the add finish its current file. if _, err := pipew.Write([]byte("some data for file b")); err != nil { t.Fatal(err) } select { case <-gcstarted: t.Fatal("gc shouldn't have started yet") default: } time.Sleep(time.Millisecond * 100) // make sure gc gets to requesting lock // finish write and unblock gc pipew.Close() // receive next object from adder o := <-out addedHashes[o.(*coreiface.AddEvent).Path.RootCid().String()] = struct{}{} <-gcstarted for r := range gcout { if r.Error != nil { t.Fatal(err) } if _, ok := addedHashes[r.KeyRemoved.String()]; ok { t.Fatal("gc'ed a hash we just added") } } var last cid.Cid for a := range out { // wait for it to finish c, err := cid.Decode(a.(*coreiface.AddEvent).Path.RootCid().String()) if err != nil { t.Fatal(err) } last = c } set := cid.NewSet() err = dag.Walk(ctx, dag.GetLinksWithDAG(node.DAG), last, set.Visit) if err != nil { t.Fatal(err) } } func testAddWPosInfo(t *testing.T, rawLeaves bool) { r := &repo.Mock{ C: config.Config{ Identity: config.Identity{ PeerID: testPeerID, // required by offline node }, }, D: syncds.MutexWrap(datastore.NewMapDatastore()), } node, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r}) if err != nil { t.Fatal(err) } bs := &testBlockstore{GCBlockstore: node.Blockstore, expectedPath: filepath.Join(os.TempDir(), "foo.txt"), t: t} bserv := blockservice.New(bs, node.Exchange) dserv := dag.NewDAGService(bserv) adder, err := NewAdder(context.Background(), node.Pinning, bs, dserv) if err != nil { t.Fatal(err) } out := make(chan any) adder.Out = out adder.Progress = true adder.RawLeaves = rawLeaves adder.NoCopy = true data := make([]byte, 5*1024*1024) rand.New(rand.NewSource(2)).Read(data) // Rand.Read never returns an error fileData := io.NopCloser(bytes.NewBuffer(data)) fileInfo := dummyFileInfo{"foo.txt", int64(len(data)), time.Now()} file, _ := files.NewReaderPathFile(filepath.Join(os.TempDir(), "foo.txt"), fileData, &fileInfo) go func() { defer close(adder.Out) _, err = adder.AddAllAndPin(context.Background(), file) if err != nil { t.Error(err) } }() for range out { } exp := 0 nonOffZero := 0 if rawLeaves { exp = 1 nonOffZero = 19 } if bs.countAtOffsetZero != exp { t.Fatalf("expected %d blocks with an offset at zero (one root and one leaf), got %d", exp, bs.countAtOffsetZero) } if bs.countAtOffsetNonZero != nonOffZero { // note: the exact number will depend on the size and the sharding algo. used t.Fatalf("expected %d blocks with an offset > 0, got %d", nonOffZero, bs.countAtOffsetNonZero) } } func TestAddWPosInfo(t *testing.T) { testAddWPosInfo(t, false) } func TestAddWPosInfoAndRawLeafs(t *testing.T) { testAddWPosInfo(t, true) } type testBlockstore struct { blockstore.GCBlockstore expectedPath string t *testing.T countAtOffsetZero int countAtOffsetNonZero int } func (bs *testBlockstore) Put(ctx context.Context, block blocks.Block) error { bs.CheckForPosInfo(block) return bs.GCBlockstore.Put(ctx, block) } func (bs *testBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error { for _, blk := range blocks { bs.CheckForPosInfo(blk) } return bs.GCBlockstore.PutMany(ctx, blocks) } func (bs *testBlockstore) CheckForPosInfo(block blocks.Block) { fsn, ok := block.(*pi.FilestoreNode) if ok { posInfo := fsn.PosInfo if posInfo.FullPath != bs.expectedPath { bs.t.Fatal("PosInfo does not have the expected path") } if posInfo.Offset == 0 { bs.countAtOffsetZero += 1 } else { bs.countAtOffsetNonZero += 1 } } } type dummyFileInfo struct { name string size int64 modTime time.Time } func (fi *dummyFileInfo) Name() string { return fi.name } func (fi *dummyFileInfo) Size() int64 { return fi.size } func (fi *dummyFileInfo) Mode() os.FileMode { return 0 } func (fi *dummyFileInfo) ModTime() time.Time { return fi.modTime } func (fi *dummyFileInfo) IsDir() bool { return false } func (fi *dummyFileInfo) Sys() any { return nil } ================================================ FILE: core/coreunix/metadata.go ================================================ package coreunix import ( dag "github.com/ipfs/boxo/ipld/merkledag" ft "github.com/ipfs/boxo/ipld/unixfs" cid "github.com/ipfs/go-cid" core "github.com/ipfs/kubo/core" ) func AddMetadataTo(n *core.IpfsNode, skey string, m *ft.Metadata) (string, error) { c, err := cid.Decode(skey) if err != nil { return "", err } nd, err := n.DAG.Get(n.Context(), c) if err != nil { return "", err } mdnode := new(dag.ProtoNode) mdata, err := ft.BytesForMetadata(m) if err != nil { return "", err } mdnode.SetData(mdata) if err := mdnode.AddNodeLink("file", nd); err != nil { return "", err } err = n.DAG.Add(n.Context(), mdnode) if err != nil { return "", err } return mdnode.Cid().String(), nil } func Metadata(n *core.IpfsNode, skey string) (*ft.Metadata, error) { c, err := cid.Decode(skey) if err != nil { return nil, err } nd, err := n.DAG.Get(n.Context(), c) if err != nil { return nil, err } pbnd, ok := nd.(*dag.ProtoNode) if !ok { return nil, dag.ErrNotProtobuf } return ft.MetadataFromBytes(pbnd.Data()) } ================================================ FILE: core/coreunix/metadata_test.go ================================================ package coreunix import ( "bytes" "context" "io" "testing" bserv "github.com/ipfs/boxo/blockservice" merkledag "github.com/ipfs/boxo/ipld/merkledag" ft "github.com/ipfs/boxo/ipld/unixfs" importer "github.com/ipfs/boxo/ipld/unixfs/importer" uio "github.com/ipfs/boxo/ipld/unixfs/io" core "github.com/ipfs/kubo/core" bstore "github.com/ipfs/boxo/blockstore" chunker "github.com/ipfs/boxo/chunker" offline "github.com/ipfs/boxo/exchange/offline" cid "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-test/random" ) func getDagserv(t *testing.T) ipld.DAGService { db := dssync.MutexWrap(ds.NewMapDatastore()) bs := bstore.NewBlockstore(db) blockserv := bserv.New(bs, offline.Exchange(bs)) return merkledag.NewDAGService(blockserv) } func TestMetadata(t *testing.T) { ctx := context.Background() // Make some random node ds := getDagserv(t) data := make([]byte, 1000) _, err := io.ReadFull(random.NewRand(), data) if err != nil { t.Fatal(err) } r := bytes.NewReader(data) nd, err := importer.BuildDagFromReader(ds, chunker.DefaultSplitter(r)) if err != nil { t.Fatal(err) } c := nd.Cid() m := new(ft.Metadata) m.MimeType = "THIS IS A TEST" // Such effort, many compromise ipfsnode := &core.IpfsNode{DAG: ds} mdk, err := AddMetadataTo(ipfsnode, c.String(), m) if err != nil { t.Fatal(err) } rec, err := Metadata(ipfsnode, mdk) if err != nil { t.Fatal(err) } if rec.MimeType != m.MimeType { t.Fatalf("something went wrong in conversion: '%s' != '%s'", rec.MimeType, m.MimeType) } cdk, err := cid.Decode(mdk) if err != nil { t.Fatal(err) } retnode, err := ds.Get(ctx, cdk) if err != nil { t.Fatal(err) } rtnpb, ok := retnode.(*merkledag.ProtoNode) if !ok { t.Fatal("expected protobuf node") } ndr, err := uio.NewDagReader(ctx, rtnpb, ds) if err != nil { t.Fatal(err) } out, err := io.ReadAll(ndr) if err != nil { t.Fatal(err) } if !bytes.Equal(out, data) { t.Fatal("read incorrect data") } } ================================================ FILE: core/coreunix/test/data/colors/orange ================================================ orange ================================================ FILE: core/coreunix/test/data/corps/apple ================================================ apple ================================================ FILE: core/coreunix/test/data/fruits/apple ================================================ apple ================================================ FILE: core/coreunix/test/data/fruits/orange ================================================ orange ================================================ FILE: core/mock/mock.go ================================================ package coremock import ( "context" "fmt" "io" libp2p2 "github.com/ipfs/kubo/core/node/libp2p" "github.com/ipfs/kubo/commands" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/repo" "github.com/ipfs/go-datastore" syncds "github.com/ipfs/go-datastore/sync" config "github.com/ipfs/kubo/config" "github.com/libp2p/go-libp2p" testutil "github.com/libp2p/go-libp2p-testing/net" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" pstore "github.com/libp2p/go-libp2p/core/peerstore" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" ) // NewMockNode constructs an IpfsNode for use in tests. func NewMockNode() (*core.IpfsNode, error) { // effectively offline, only peer in its network return core.NewNode(context.Background(), &core.BuildCfg{ Online: true, Host: MockHostOption(mocknet.New()), }) } func MockHostOption(mn mocknet.Mocknet) libp2p2.HostOption { return func(id peer.ID, ps pstore.Peerstore, opts ...libp2p.Option) (host.Host, error) { var cfg libp2p.Config if err := cfg.Apply(opts...); err != nil { return nil, err } // The mocknet does not use the provided libp2p.Option. This options include // the listening addresses we want our peer listening on. Therefore, we have // to manually parse the configuration and add them here. ps.AddAddrs(id, cfg.ListenAddrs, pstore.PermanentAddrTTL) return mn.AddPeerWithPeerstore(id, ps) } } func MockCmdsCtx() (commands.Context, error) { // Generate Identity ident, err := testutil.RandIdentity() if err != nil { return commands.Context{}, err } p := ident.ID() conf := config.Config{ Identity: config.Identity{ PeerID: p.String(), }, } r := &repo.Mock{ D: syncds.MutexWrap(datastore.NewMapDatastore()), C: conf, } node, err := core.NewNode(context.Background(), &core.BuildCfg{ Repo: r, }) if err != nil { return commands.Context{}, err } return commands.Context{ ConfigRoot: "/tmp/.mockipfsconfig", ConstructNode: func() (*core.IpfsNode, error) { return node, nil }, }, nil } func MockPublicNode(ctx context.Context, mn mocknet.Mocknet) (*core.IpfsNode, error) { ds := syncds.MutexWrap(datastore.NewMapDatastore()) cfg, err := config.Init(io.Discard, 2048) if err != nil { return nil, err } count := len(mn.Peers()) cfg.Addresses.Swarm = []string{ fmt.Sprintf("/ip4/18.0.%d.%d/tcp/4001", count>>16, count&0xFF), } cfg.Datastore = config.Datastore{} return core.NewNode(ctx, &core.BuildCfg{ Online: true, Routing: libp2p2.DHTServerOption, Repo: &repo.Mock{ C: *cfg, D: ds, }, Host: MockHostOption(mn), }) } ================================================ FILE: core/node/bitswap.go ================================================ package node import ( "context" "errors" "io" "time" "github.com/dustin/go-humanize" "github.com/ipfs/boxo/bitswap" "github.com/ipfs/boxo/bitswap/client" "github.com/ipfs/boxo/bitswap/network" bsnet "github.com/ipfs/boxo/bitswap/network/bsnet" "github.com/ipfs/boxo/bitswap/network/httpnet" blockstore "github.com/ipfs/boxo/blockstore" exchange "github.com/ipfs/boxo/exchange" rpqm "github.com/ipfs/boxo/routing/providerquerymanager" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" version "github.com/ipfs/kubo" "github.com/ipfs/kubo/config" "github.com/libp2p/go-libp2p/core/host" peer "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "go.uber.org/fx" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/kubo/core/node/helpers" ) // Docs: https://github.com/ipfs/kubo/blob/master/docs/config.md#internalbitswap const ( DefaultEngineBlockstoreWorkerCount = 128 DefaultTaskWorkerCount = 8 DefaultEngineTaskWorkerCount = 8 DefaultMaxOutstandingBytesPerPeer = 1 << 20 DefaultProviderSearchDelay = 1000 * time.Millisecond DefaultMaxProviders = 10 // matching BitswapClientDefaultMaxProviders from https://github.com/ipfs/boxo/blob/v0.29.1/bitswap/internal/defaults/defaults.go#L15 DefaultWantHaveReplaceSize = 1024 ) type bitswapOptionsOut struct { fx.Out BitswapOpts []bitswap.Option `group:"bitswap-options,flatten"` } // BitswapOptions creates configuration options for Bitswap from the config file // and whether to provide data. func BitswapOptions(cfg *config.Config) any { return func() bitswapOptionsOut { var internalBsCfg config.InternalBitswap if cfg.Internal.Bitswap != nil { internalBsCfg = *cfg.Internal.Bitswap } opts := []bitswap.Option{ bitswap.ProviderSearchDelay(internalBsCfg.ProviderSearchDelay.WithDefault(DefaultProviderSearchDelay)), // See https://github.com/ipfs/go-ipfs/issues/8807 for rationale bitswap.EngineBlockstoreWorkerCount(int(internalBsCfg.EngineBlockstoreWorkerCount.WithDefault(DefaultEngineBlockstoreWorkerCount))), bitswap.TaskWorkerCount(int(internalBsCfg.TaskWorkerCount.WithDefault(DefaultTaskWorkerCount))), bitswap.EngineTaskWorkerCount(int(internalBsCfg.EngineTaskWorkerCount.WithDefault(DefaultEngineTaskWorkerCount))), bitswap.MaxOutstandingBytesPerPeer(int(internalBsCfg.MaxOutstandingBytesPerPeer.WithDefault(DefaultMaxOutstandingBytesPerPeer))), bitswap.WithWantHaveReplaceSize(int(internalBsCfg.WantHaveReplaceSize.WithDefault(DefaultWantHaveReplaceSize))), } return bitswapOptionsOut{BitswapOpts: opts} } } type bitswapIn struct { fx.In Mctx helpers.MetricsCtx Cfg *config.Config Host host.Host Discovery routing.ContentDiscovery Bs blockstore.GCBlockstore BitswapOpts []bitswap.Option `group:"bitswap-options"` } // Bitswap creates the BitSwap server/client instance. // If Bitswap.ServerEnabled is false, the node will act only as a client // using an empty blockstore to prevent serving blocks to other peers. func Bitswap(serverEnabled, libp2pEnabled, httpEnabled bool) any { return func(in bitswapIn, lc fx.Lifecycle) (*bitswap.Bitswap, error) { var bitswapNetworks, bitswapLibp2p network.BitSwapNetwork var bitswapBlockstore blockstore.Blockstore = in.Bs connEvtMgr := network.NewConnectEventManager() libp2pEnabled := in.Cfg.Bitswap.Libp2pEnabled.WithDefault(config.DefaultBitswapLibp2pEnabled) if libp2pEnabled { bitswapLibp2p = bsnet.NewFromIpfsHost( in.Host, bsnet.WithConnectEventManager(connEvtMgr), ) } if httpEnabled { httpCfg := in.Cfg.HTTPRetrieval maxBlockSize, err := humanize.ParseBytes(httpCfg.MaxBlockSize.WithDefault(config.DefaultHTTPRetrievalMaxBlockSize)) if err != nil { return nil, err } logger.Infof("HTTP Retrieval enabled: Allowlist: %t. Denylist: %t", httpCfg.Allowlist != nil, httpCfg.Denylist != nil, ) bitswapHTTP := httpnet.New(in.Host, httpnet.WithHTTPWorkers(int(httpCfg.NumWorkers.WithDefault(config.DefaultHTTPRetrievalNumWorkers))), httpnet.WithAllowlist(httpCfg.Allowlist), httpnet.WithDenylist(httpCfg.Denylist), httpnet.WithInsecureSkipVerify(httpCfg.TLSInsecureSkipVerify.WithDefault(config.DefaultHTTPRetrievalTLSInsecureSkipVerify)), httpnet.WithMaxBlockSize(int64(maxBlockSize)), httpnet.WithUserAgent(version.GetUserAgentVersion()), httpnet.WithMetricsLabelsForEndpoints(httpCfg.Allowlist), httpnet.WithConnectEventManager(connEvtMgr), ) bitswapNetworks = network.New(in.Host.Peerstore(), bitswapLibp2p, bitswapHTTP) } else if libp2pEnabled { bitswapNetworks = bitswapLibp2p } else { return nil, errors.New("invalid configuration: Bitswap.Libp2pEnabled and HTTPRetrieval.Enabled are both disabled, unable to initialize Bitswap") } // Kubo uses own, customized ProviderQueryManager in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.WithDefaultProviderQueryManager(false))) var maxProviders int = DefaultMaxProviders var bcDisposition string if in.Cfg.Internal.Bitswap != nil { maxProviders = int(in.Cfg.Internal.Bitswap.ProviderSearchMaxResults.WithDefault(DefaultMaxProviders)) if in.Cfg.Internal.Bitswap.BroadcastControl != nil { bcCfg := in.Cfg.Internal.Bitswap.BroadcastControl bcEnable := bcCfg.Enable.WithDefault(config.DefaultBroadcastControlEnable) in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlEnable(bcEnable))) if bcEnable { bcDisposition = "enabled" bcMaxPeers := int(bcCfg.MaxPeers.WithDefault(config.DefaultBroadcastControlMaxPeers)) in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlMaxPeers(bcMaxPeers))) bcLocalPeers := bcCfg.LocalPeers.WithDefault(config.DefaultBroadcastControlLocalPeers) in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlLocalPeers(bcLocalPeers))) bcPeeredPeers := bcCfg.PeeredPeers.WithDefault(config.DefaultBroadcastControlPeeredPeers) in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlPeeredPeers(bcPeeredPeers))) bcMaxRandomPeers := int(bcCfg.MaxRandomPeers.WithDefault(config.DefaultBroadcastControlMaxRandomPeers)) in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlMaxRandomPeers(bcMaxRandomPeers))) bcSendToPendingPeers := bcCfg.SendToPendingPeers.WithDefault(config.DefaultBroadcastControlSendToPendingPeers) in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlSendToPendingPeers(bcSendToPendingPeers))) } else { bcDisposition = "disabled" } } } // If broadcast control is not configured, then configure with defaults. if bcDisposition == "" { in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlEnable(config.DefaultBroadcastControlEnable))) if config.DefaultBroadcastControlEnable { bcDisposition = "enabled" in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlMaxPeers(config.DefaultBroadcastControlMaxPeers))) in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlLocalPeers(config.DefaultBroadcastControlLocalPeers))) in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlPeeredPeers(config.DefaultBroadcastControlPeeredPeers))) in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlMaxRandomPeers(config.DefaultBroadcastControlMaxRandomPeers))) in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlSendToPendingPeers(config.DefaultBroadcastControlSendToPendingPeers))) } else { bcDisposition = "enabled" } } logger.Infof("bitswap client broadcast control %s", bcDisposition) ignoredPeerIDs := make([]peer.ID, 0, len(in.Cfg.Routing.IgnoreProviders)) for _, str := range in.Cfg.Routing.IgnoreProviders { pid, err := peer.Decode(str) if err != nil { return nil, err } ignoredPeerIDs = append(ignoredPeerIDs, pid) } providerQueryMgr, err := rpqm.New(bitswapNetworks, in.Discovery, rpqm.WithMaxProviders(maxProviders), rpqm.WithIgnoreProviders(ignoredPeerIDs...), ) if err != nil { return nil, err } // Explicitly enable/disable server in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(serverEnabled)) bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetworks, providerQueryMgr, bitswapBlockstore, in.BitswapOpts...) lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return bs.Close() }, }) return bs, nil } } // OnlineExchange creates new LibP2P backed block exchange. // Returns a no-op exchange if Bitswap is disabled. func OnlineExchange(isBitswapActive bool) any { return func(in *bitswap.Bitswap, lc fx.Lifecycle) exchange.Interface { if !isBitswapActive { return &noopExchange{closer: in} } lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return in.Close() }, }) return in } } type noopExchange struct { closer io.Closer } func (e *noopExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { return nil, ipld.ErrNotFound{Cid: c} } func (e *noopExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { ch := make(chan blocks.Block) close(ch) return ch, nil } func (e *noopExchange) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { return nil } func (e *noopExchange) Close() error { return e.closer.Close() } ================================================ FILE: core/node/builder.go ================================================ package node import ( "context" "crypto/rand" "encoding/base64" "go.uber.org/fx" "github.com/ipfs/boxo/autoconf" "github.com/ipfs/kubo/core/node/helpers" "github.com/ipfs/kubo/core/node/libp2p" "github.com/ipfs/kubo/repo" ds "github.com/ipfs/go-datastore" dsync "github.com/ipfs/go-datastore/sync" cfg "github.com/ipfs/kubo/config" "github.com/libp2p/go-libp2p/core/crypto" peer "github.com/libp2p/go-libp2p/core/peer" ) type BuildCfg struct { // If online is set, the node will have networking enabled Online bool // ExtraOpts is a map of extra options used to configure the ipfs nodes creation ExtraOpts map[string]bool // If permanent then node should run more expensive processes // that will improve performance in long run Permanent bool // DisableEncryptedConnections disables connection encryption *entirely*. // DO NOT SET THIS UNLESS YOU'RE TESTING. DisableEncryptedConnections bool Routing libp2p.RoutingOption Host libp2p.HostOption Repo repo.Repo } func (cfg *BuildCfg) getOpt(key string) bool { if cfg.ExtraOpts == nil { return false } return cfg.ExtraOpts[key] } func (cfg *BuildCfg) fillDefaults() error { if cfg.Repo == nil { r, err := defaultRepo(dsync.MutexWrap(ds.NewMapDatastore())) if err != nil { return err } cfg.Repo = r } if cfg.Routing == nil { cfg.Routing = libp2p.DHTOption } if cfg.Host == nil { cfg.Host = libp2p.DefaultHostOption } return nil } // options creates fx option group from this build config func (cfg *BuildCfg) options(ctx context.Context) (fx.Option, *cfg.Config) { err := cfg.fillDefaults() if err != nil { return fx.Error(err), nil } repoOption := fx.Provide(func(lc fx.Lifecycle) repo.Repo { lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return cfg.Repo.Close() }, }) return cfg.Repo }) metricsCtx := fx.Provide(func() helpers.MetricsCtx { return helpers.MetricsCtx(ctx) }) hostOption := fx.Provide(func() libp2p.HostOption { return cfg.Host }) routingOption := fx.Provide(func() libp2p.RoutingOption { return cfg.Routing }) conf, err := cfg.Repo.Config() if err != nil { return fx.Error(err), nil } return fx.Options( repoOption, hostOption, routingOption, metricsCtx, ), conf } func defaultRepo(dstore repo.Datastore) (repo.Repo, error) { c := cfg.Config{} priv, pub, err := crypto.GenerateKeyPairWithReader(crypto.RSA, 2048, rand.Reader) if err != nil { return nil, err } pid, err := peer.IDFromPublicKey(pub) if err != nil { return nil, err } privkeyb, err := crypto.MarshalPrivateKey(priv) if err != nil { return nil, err } c.Bootstrap = autoconf.FallbackBootstrapPeers c.Addresses.Swarm = []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/udp/4001/quic-v1"} c.Identity.PeerID = pid.String() c.Identity.PrivKey = base64.StdEncoding.EncodeToString(privkeyb) return &repo.Mock{ D: dstore, C: c, }, nil } ================================================ FILE: core/node/core.go ================================================ package node import ( "context" "errors" "fmt" "github.com/ipfs/boxo/blockservice" blockstore "github.com/ipfs/boxo/blockstore" exchange "github.com/ipfs/boxo/exchange" offline "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/fetcher" bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/filestore" "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/boxo/mfs" pathresolver "github.com/ipfs/boxo/path/resolver" pin "github.com/ipfs/boxo/pinning/pinner" "github.com/ipfs/boxo/pinning/pinner/dspinner" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-unixfsnode" dagpb "github.com/ipld/go-codec-dagpb" "go.uber.org/fx" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/node/helpers" "github.com/ipfs/kubo/repo" ) // FilesRootDatastoreKey is the datastore key for the MFS files root CID. var FilesRootDatastoreKey = datastore.NewKey("/local/filesroot") // BlockService creates new blockservice which provides an interface to fetch content-addressable blocks func BlockService(cfg *config.Config) func(lc fx.Lifecycle, bs blockstore.Blockstore, rem exchange.Interface) blockservice.BlockService { return func(lc fx.Lifecycle, bs blockstore.Blockstore, rem exchange.Interface) blockservice.BlockService { bsvc := blockservice.New(bs, rem, blockservice.WriteThrough(cfg.Datastore.WriteThrough.WithDefault(config.DefaultWriteThrough)), ) lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return bsvc.Close() }, }) return bsvc } } // Pinning creates new pinner which tells GC which blocks should be kept func Pinning(strategy string) func(bstore blockstore.Blockstore, ds format.DAGService, repo repo.Repo, prov DHTProvider) (pin.Pinner, error) { // Parse strategy at function creation time (not inside the returned function) // This happens before the provider is created, which is why we pass the strategy // string and parse it here, rather than using fx-provided ProvidingStrategy. strategyFlag := config.ParseProvideStrategy(strategy) return func(bstore blockstore.Blockstore, ds format.DAGService, repo repo.Repo, prov DHTProvider, ) (pin.Pinner, error) { rootDS := repo.Datastore() syncFn := func(ctx context.Context) error { if err := rootDS.Sync(ctx, blockstore.BlockPrefix); err != nil { return err } return rootDS.Sync(ctx, filestore.FilestorePrefix) } syncDs := &syncDagService{ds, syncFn} ctx := context.TODO() var opts []dspinner.Option roots := (strategyFlag & config.ProvideStrategyRoots) != 0 pinned := (strategyFlag & config.ProvideStrategyPinned) != 0 // Important: Only one of WithPinnedProvider or WithRootsProvider should be active. // Having both would cause duplicate root advertisements since "pinned" includes all // pinned content (roots + children), while "roots" is just the root CIDs. // We prioritize "pinned" if both are somehow set (though this shouldn't happen // with proper strategy parsing). if pinned { opts = append(opts, dspinner.WithPinnedProvider(prov)) } else if roots { opts = append(opts, dspinner.WithRootsProvider(prov)) } pinning, err := dspinner.New(ctx, rootDS, syncDs, opts...) if err != nil { return nil, err } return pinning, nil } } var ( _ merkledag.SessionMaker = new(syncDagService) _ format.DAGService = new(syncDagService) ) // syncDagService is used by the Pinner to ensure data gets persisted to the underlying datastore type syncDagService struct { format.DAGService syncFn func(context.Context) error } func (s *syncDagService) Sync(ctx context.Context) error { return s.syncFn(ctx) } func (s *syncDagService) Session(ctx context.Context) format.NodeGetter { return merkledag.NewSession(ctx, s.DAGService) } // FetchersOut allows injection of fetchers. type FetchersOut struct { fx.Out IPLDFetcher fetcher.Factory `name:"ipldFetcher"` UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"` OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"` } // FetchersIn allows using fetchers for other dependencies. type FetchersIn struct { fx.In IPLDFetcher fetcher.Factory `name:"ipldFetcher"` UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"` OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"` } // FetcherConfig returns a fetcher config that can build new fetcher instances func FetcherConfig(bs blockservice.BlockService) FetchersOut { ipldFetcher := bsfetcher.NewFetcherConfig(bs) ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser) unixFSFetcher := ipldFetcher.WithReifier(unixfsnode.Reify) // Construct offline versions which we can safely use in contexts where // path resolution should not fetch new blocks via exchange. offlineBs := blockservice.New(bs.Blockstore(), offline.Exchange(bs.Blockstore())) offlineIpldFetcher := bsfetcher.NewFetcherConfig(offlineBs) offlineIpldFetcher.SkipNotFound = true // carries onto the UnixFSFetcher below offlineIpldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser) offlineUnixFSFetcher := offlineIpldFetcher.WithReifier(unixfsnode.Reify) return FetchersOut{ IPLDFetcher: ipldFetcher, UnixfsFetcher: unixFSFetcher, OfflineIPLDFetcher: offlineIpldFetcher, OfflineUnixfsFetcher: offlineUnixFSFetcher, } } // PathResolversOut allows injection of path resolvers type PathResolversOut struct { fx.Out IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` } // PathResolverConfig creates path resolvers with the given fetchers. func PathResolverConfig(fetchers FetchersIn) PathResolversOut { return PathResolversOut{ IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher), UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher), OfflineIPLDPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineIPLDFetcher), OfflineUnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineUnixfsFetcher), } } // Dag creates new DAGService func Dag(bs blockservice.BlockService) format.DAGService { return merkledag.NewDAGService(bs) } // Files loads persisted MFS root func Files(strategy string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo repo.Repo, dag format.DAGService, bs blockstore.Blockstore, prov DHTProvider) (*mfs.Root, error) { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo repo.Repo, dag format.DAGService, bs blockstore.Blockstore, prov DHTProvider) (*mfs.Root, error) { pf := func(ctx context.Context, c cid.Cid) error { rootDS := repo.Datastore() if err := rootDS.Sync(ctx, blockstore.BlockPrefix); err != nil { return err } if err := rootDS.Sync(ctx, filestore.FilestorePrefix); err != nil { return err } if err := rootDS.Put(ctx, FilesRootDatastoreKey, c.Bytes()); err != nil { return err } return rootDS.Sync(ctx, FilesRootDatastoreKey) } var nd *merkledag.ProtoNode ctx := helpers.LifecycleCtx(mctx, lc) val, err := repo.Datastore().Get(ctx, FilesRootDatastoreKey) switch { case errors.Is(err, datastore.ErrNotFound): nd = unixfs.EmptyDirNode() err := dag.Add(ctx, nd) if err != nil { return nil, fmt.Errorf("failure writing filesroot to dagstore: %s", err) } case err == nil: c, err := cid.Cast(val) if err != nil { return nil, err } offlineDag := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) rnd, err := offlineDag.Get(ctx, c) if err != nil { return nil, fmt.Errorf("error loading filesroot from dagservice: %s", err) } pbnd, ok := rnd.(*merkledag.ProtoNode) if !ok { return nil, merkledag.ErrNotProtobuf } nd = pbnd default: return nil, err } // MFS (Mutable File System) provider integration: Only pass the provider // to MFS when the strategy includes "mfs". MFS will call StartProviding() // on every DAGService.Add() operation, which is sufficient for the "mfs" // strategy - it ensures all MFS content gets announced as it's added or // modified. For non-mfs strategies, we set provider to nil to avoid // unnecessary providing. strategyFlag := config.ParseProvideStrategy(strategy) if strategyFlag&config.ProvideStrategyMFS == 0 { prov = nil } // Get configured settings from Import config cfg, err := repo.Config() if err != nil { return nil, fmt.Errorf("failed to get config: %w", err) } chunkerGen := cfg.Import.UnixFSSplitterFunc() maxDirLinks := int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks)) maxHAMTFanout := int(cfg.Import.UnixFSHAMTDirectoryMaxFanout.WithDefault(config.DefaultUnixFSHAMTDirectoryMaxFanout)) hamtShardingSize := int(cfg.Import.UnixFSHAMTDirectorySizeThreshold.WithDefault(config.DefaultUnixFSHAMTDirectorySizeThreshold)) sizeEstimationMode := cfg.Import.HAMTSizeEstimationMode() root, err := mfs.NewRoot(ctx, dag, nd, pf, prov, mfs.WithChunker(chunkerGen), mfs.WithMaxLinks(maxDirLinks), mfs.WithMaxHAMTFanout(maxHAMTFanout), mfs.WithHAMTShardingSize(hamtShardingSize), mfs.WithSizeEstimationMode(sizeEstimationMode), ) if err != nil { return nil, fmt.Errorf("failed to initialize MFS root from %s stored at %s: %w. "+ "If corrupted, use 'ipfs files chroot' to reset (see --help)", nd.Cid(), FilesRootDatastoreKey, err) } lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return root.Close() }, }) return root, err } } ================================================ FILE: core/node/dns.go ================================================ package node import ( "math" "time" "github.com/ipfs/boxo/gateway" config "github.com/ipfs/kubo/config" doh "github.com/libp2p/go-doh-resolver" madns "github.com/multiformats/go-multiaddr-dns" ) // Compile-time interface check: *madns.Resolver (returned by gateway.NewDNSResolver // and madns.NewResolver) must implement madns.BasicResolver for p2pForgeResolver fallback. var _ madns.BasicResolver = (*madns.Resolver)(nil) func DNSResolver(cfg *config.Config) (*madns.Resolver, error) { var dohOpts []doh.Option if !cfg.DNS.MaxCacheTTL.IsDefault() { dohOpts = append(dohOpts, doh.WithMaxCacheTTL(cfg.DNS.MaxCacheTTL.WithDefault(time.Duration(math.MaxUint32)*time.Second))) } // Replace "auto" DNS resolver placeholders with autoconf values resolvers := cfg.DNSResolversWithAutoConf() // Get base resolver from boxo (handles custom DoH resolvers per eTLD) baseResolver, err := gateway.NewDNSResolver(resolvers, dohOpts...) if err != nil { return nil, err } // Check if we should skip network DNS lookups for p2p-forge domains skipAutoTLSDNS := cfg.AutoTLS.SkipDNSLookup.WithDefault(config.DefaultAutoTLSSkipDNSLookup) if !skipAutoTLSDNS { // Local resolution disabled, use network DNS for everything return baseResolver, nil } // Build list of p2p-forge domains to resolve locally without network I/O. // AutoTLS hostnames encode IP addresses directly (e.g., 1-2-3-4.peerID.libp2p.direct), // so DNS lookups are wasteful. We resolve these in-memory when possible. forgeDomains := []string{config.DefaultDomainSuffix} customDomain := cfg.AutoTLS.DomainSuffix.WithDefault(config.DefaultDomainSuffix) if customDomain != config.DefaultDomainSuffix { forgeDomains = append(forgeDomains, customDomain) } forgeResolver := NewP2PForgeResolver(forgeDomains, baseResolver) // Register p2p-forge resolver for each domain, fallback to baseResolver for others opts := []madns.Option{madns.WithDefaultResolver(baseResolver)} for _, domain := range forgeDomains { opts = append(opts, madns.WithDomainResolver(domain+".", forgeResolver)) } return madns.NewResolver(opts...) } ================================================ FILE: core/node/groups.go ================================================ package node import ( "context" "errors" "fmt" "regexp" "strings" "time" blockstore "github.com/ipfs/boxo/blockstore" offline "github.com/ipfs/boxo/exchange/offline" uio "github.com/ipfs/boxo/ipld/unixfs/io" util "github.com/ipfs/boxo/util" "github.com/ipfs/go-log/v2" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/node/libp2p" "github.com/ipfs/kubo/p2p" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p-pubsub/timecache" "github.com/libp2p/go-libp2p/core/peer" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "go.uber.org/fx" ) var logger = log.Logger("core:constructor") var BaseLibP2P = fx.Options( fx.Provide(libp2p.PNet), fx.Provide(libp2p.ConnectionManager), fx.Provide(libp2p.Host), fx.Provide(libp2p.MultiaddrResolver), fx.Provide(libp2p.DiscoveryHandler), fx.Invoke(libp2p.PNetChecker), ) func LibP2P(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.PartialLimitConfig) fx.Option { var connmgr fx.Option // set connmgr based on Swarm.ConnMgr.Type connMgrType := cfg.Swarm.ConnMgr.Type.WithDefault(config.DefaultConnMgrType) switch connMgrType { case "none": connmgr = fx.Options() // noop case "", "basic": grace := cfg.Swarm.ConnMgr.GracePeriod.WithDefault(config.DefaultConnMgrGracePeriod) low := int(cfg.Swarm.ConnMgr.LowWater.WithDefault(config.DefaultConnMgrLowWater)) high := int(cfg.Swarm.ConnMgr.HighWater.WithDefault(config.DefaultConnMgrHighWater)) silence := cfg.Swarm.ConnMgr.SilencePeriod.WithDefault(config.DefaultConnMgrSilencePeriod) connmgr = fx.Provide(libp2p.ConnectionManager(low, high, grace, silence)) default: return fx.Error(fmt.Errorf("unrecognized Swarm.ConnMgr.Type: %q", connMgrType)) } // parse PubSub config ps, disc := fx.Options(), fx.Options() if bcfg.getOpt("pubsub") || bcfg.getOpt("ipnsps") { disc = fx.Provide(libp2p.TopicDiscovery()) var pubsubOptions []pubsub.Option pubsubOptions = append( pubsubOptions, pubsub.WithMessageSigning(!cfg.Pubsub.DisableSigning), pubsub.WithSeenMessagesTTL(cfg.Pubsub.SeenMessagesTTL.WithDefault(pubsub.TimeCacheDuration)), ) var seenMessagesStrategy timecache.Strategy configSeenMessagesStrategy := cfg.Pubsub.SeenMessagesStrategy.WithDefault(config.DefaultSeenMessagesStrategy) switch configSeenMessagesStrategy { case config.LastSeenMessagesStrategy: seenMessagesStrategy = timecache.Strategy_LastSeen case config.FirstSeenMessagesStrategy: seenMessagesStrategy = timecache.Strategy_FirstSeen default: return fx.Error(fmt.Errorf("unsupported Pubsub.SeenMessagesStrategy %q", configSeenMessagesStrategy)) } pubsubOptions = append(pubsubOptions, pubsub.WithSeenMessagesStrategy(seenMessagesStrategy)) switch cfg.Pubsub.Router { case "": fallthrough case "gossipsub": ps = fx.Provide(libp2p.GossipSub(pubsubOptions...)) case "floodsub": ps = fx.Provide(libp2p.FloodSub(pubsubOptions...)) default: return fx.Error(fmt.Errorf("unknown pubsub router %s", cfg.Pubsub.Router)) } } autonat := fx.Options() switch cfg.AutoNAT.ServiceMode { default: panic("BUG: unhandled autonat service mode") case config.AutoNATServiceDisabled: case config.AutoNATServiceUnset: // TODO // // We're enabling the AutoNAT service by default on _all_ nodes // for the moment. // // We should consider disabling it by default if the dht is set // to dhtclient. fallthrough case config.AutoNATServiceEnabled: autonat = fx.Provide(libp2p.AutoNATService(cfg.AutoNAT.Throttle, false)) case config.AutoNATServiceEnabledV1Only: autonat = fx.Provide(libp2p.AutoNATService(cfg.AutoNAT.Throttle, true)) } enableTCPTransport := cfg.Swarm.Transports.Network.TCP.WithDefault(true) enableWebsocketTransport := cfg.Swarm.Transports.Network.Websocket.WithDefault(true) enableRelayTransport := cfg.Swarm.Transports.Network.Relay.WithDefault(true) // nolint enableRelayService := cfg.Swarm.RelayService.Enabled.WithDefault(enableRelayTransport) enableRelayClient := cfg.Swarm.RelayClient.Enabled.WithDefault(enableRelayTransport) enableAutoTLS := cfg.AutoTLS.Enabled.WithDefault(config.DefaultAutoTLSEnabled) enableAutoWSS := cfg.AutoTLS.AutoWSS.WithDefault(config.DefaultAutoWSS) atlsLog := log.Logger("autotls") // Log error when relay subsystem could not be initialized due to missing dependency if !enableRelayTransport { if enableRelayService { logger.Fatal("Failed to enable `Swarm.RelayService`, it requires `Swarm.Transports.Network.Relay` to be true.") } if enableRelayClient { logger.Fatal("Failed to enable `Swarm.RelayClient`, it requires `Swarm.Transports.Network.Relay` to be true.") } } switch { case enableAutoTLS && enableTCPTransport && enableWebsocketTransport: // AutoTLS for Secure WebSockets: ensure WSS listeners are in place (manual or automatic) wssWildcard := fmt.Sprintf("/tls/sni/*.%s/ws", cfg.AutoTLS.DomainSuffix.WithDefault(config.DefaultDomainSuffix)) wssWildcardPresent := false customWsPresent := false customWsRegex := regexp.MustCompile(`/wss?$`) tcpRegex := regexp.MustCompile(`/tcp/\d+$`) // inspect listeners defined in config at Addresses.Swarm var tcpListeners []string for _, listener := range cfg.Addresses.Swarm { // detect if user manually added /tls/sni/.../ws listener matching AutoTLS.DomainSuffix if strings.Contains(listener, wssWildcard) { atlsLog.Infof("found compatible wildcard listener in Addresses.Swarm. AutoTLS will be used on %s", listener) wssWildcardPresent = true break } // detect if user manually added own /ws or /wss listener that is // not related to AutoTLS feature if customWsRegex.MatchString(listener) { atlsLog.Infof("found custom /ws listener set by user in Addresses.Swarm. AutoTLS will not be used on %s.", listener) customWsPresent = true break } // else, remember /tcp listeners that can be reused for /tls/sni/../ws if tcpRegex.MatchString(listener) { tcpListeners = append(tcpListeners, listener) } } // Append AutoTLS's wildcard listener // if no manual /ws listener was set by the user if enableAutoWSS && !wssWildcardPresent && !customWsPresent { if len(tcpListeners) == 0 { logger.Error("Invalid configuration, AutoTLS will be disabled: AutoTLS.AutoWSS=true requires at least one /tcp listener present in Addresses.Swarm, see https://github.com/ipfs/kubo/blob/master/docs/config.md#autotls") enableAutoTLS = false } for _, tcpListener := range tcpListeners { wssListener := tcpListener + wssWildcard cfg.Addresses.Swarm = append(cfg.Addresses.Swarm, wssListener) atlsLog.Infof("appended AutoWSS listener: %s", wssListener) } } if !wssWildcardPresent && !enableAutoWSS { logger.Error(fmt.Sprintf("Invalid configuration, AutoTLS will be disabled: AutoTLS.Enabled=true requires a /tcp listener ending with %q to be present in Addresses.Swarm or AutoTLS.AutoWSS=true, see https://github.com/ipfs/kubo/blob/master/docs/config.md#autotls", wssWildcard)) enableAutoTLS = false } case enableAutoTLS && !enableTCPTransport: logger.Error("Invalid configuration: AutoTLS.Enabled=true requires Swarm.Transports.Network.TCP to be true as well. AutoTLS will be disabled.") enableAutoTLS = false case enableAutoTLS && !enableWebsocketTransport: logger.Error("Invalid configuration: AutoTLS.Enabled=true requires Swarm.Transports.Network.Websocket to be true as well. AutoTLS will be disabled.") enableAutoTLS = false } // Gather all the options opts := fx.Options( BaseLibP2P, // identify's AgentVersion (incl. optional agent-version-suffix) fx.Provide(libp2p.UserAgent()), // Services (resource management) fx.Provide(libp2p.ResourceManager(bcfg.Repo.Path(), cfg.Swarm, userResourceOverrides)), maybeProvide(libp2p.P2PForgeCertMgr(bcfg.Repo.Path(), cfg.AutoTLS, atlsLog), enableAutoTLS), maybeInvoke(libp2p.StartP2PAutoTLS, enableAutoTLS), fx.Provide(libp2p.AddrFilters(cfg.Swarm.AddrFilters)), fx.Provide(libp2p.AddrsFactory(cfg.Addresses.Announce, cfg.Addresses.AppendAnnounce, cfg.Addresses.NoAnnounce)), fx.Provide(libp2p.SmuxTransport(cfg.Swarm.Transports)), fx.Provide(libp2p.RelayTransport(enableRelayTransport)), fx.Provide(libp2p.RelayService(enableRelayService, cfg.Swarm.RelayService)), fx.Provide(libp2p.Transports(cfg.Swarm.Transports)), fx.Provide(libp2p.ListenOn(cfg.Addresses.Swarm)), fx.Invoke(libp2p.SetupDiscovery(cfg.Discovery.MDNS.Enabled)), fx.Provide(libp2p.ForceReachability(cfg.Internal.Libp2pForceReachability)), fx.Provide(libp2p.HolePunching(cfg.Swarm.EnableHolePunching, enableRelayClient)), fx.Provide(libp2p.Security(!bcfg.DisableEncryptedConnections, cfg.Swarm.Transports)), fx.Provide(libp2p.Routing), fx.Provide(libp2p.ContentRouting), fx.Provide(libp2p.ContentDiscovery), fx.Provide(libp2p.BaseRouting(cfg)), maybeProvide(libp2p.PubsubRouter, bcfg.getOpt("ipnsps")), maybeProvide(libp2p.BandwidthCounter, !cfg.Swarm.DisableBandwidthMetrics), maybeProvide(libp2p.NatPortMap, !cfg.Swarm.DisableNatPortMap), libp2p.MaybeAutoRelay(cfg.Swarm.RelayClient.StaticRelays, cfg.Peering, enableRelayClient), autonat, connmgr, ps, disc, ) return opts } // Storage groups units which setup datastore based persistence and blockstore layers func Storage(bcfg *BuildCfg, cfg *config.Config) fx.Option { cacheOpts := blockstore.DefaultCacheOpts() cacheOpts.HasBloomFilterSize = cfg.Datastore.BloomFilterSize cacheOpts.HasTwoQueueCacheSize = int(cfg.Datastore.BlockKeyCacheSize.WithDefault(config.DefaultBlockKeyCacheSize)) if !bcfg.Permanent { cacheOpts.HasBloomFilterSize = 0 } finalBstore := fx.Provide(GcBlockstoreCtor) if cfg.Experimental.FilestoreEnabled || cfg.Experimental.UrlstoreEnabled { finalBstore = fx.Provide(FilestoreBlockstoreCtor) } return fx.Options( fx.Provide(RepoConfig), fx.Provide(Datastore), fx.Provide(BaseBlockstoreCtor( cacheOpts, cfg.Datastore.HashOnRead, cfg.Datastore.WriteThrough.WithDefault(config.DefaultWriteThrough), cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy), )), finalBstore, ) } // Identity groups units providing cryptographic identity func Identity(cfg *config.Config) fx.Option { // PeerID cid := cfg.Identity.PeerID if cid == "" { return fx.Error(errors.New("identity was not set in config (was 'ipfs init' run?)")) } if len(cid) == 0 { return fx.Error(errors.New("no peer ID in config! (was 'ipfs init' run?)")) } id, err := peer.Decode(cid) if err != nil { return fx.Error(fmt.Errorf("peer ID invalid: %s", err)) } // Private Key if cfg.Identity.PrivKey == "" { return fx.Options( // No PK (usually in tests) fx.Provide(PeerID(id)), fx.Provide(libp2p.Peerstore), ) } sk, err := cfg.Identity.DecodePrivateKey("passphrase todo!") if err != nil { return fx.Error(err) } return fx.Options( // Full identity fx.Provide(PeerID(id)), fx.Provide(PrivateKey(sk)), fx.Provide(libp2p.Peerstore), fx.Invoke(libp2p.PstoreAddSelfKeys), ) } // IPNS groups namesys related units var IPNS = fx.Options( fx.Provide(RecordValidator), ) // Online groups online-only units func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.PartialLimitConfig) fx.Option { // Namesys params ipnsCacheSize := cfg.Ipns.ResolveCacheSize if ipnsCacheSize == 0 { ipnsCacheSize = DefaultIpnsCacheSize } if ipnsCacheSize < 0 { return fx.Error(errors.New("cannot specify negative resolve cache size")) } // Republisher params var repubPeriod, recordLifetime time.Duration if cfg.Ipns.RepublishPeriod != "" { d, err := time.ParseDuration(cfg.Ipns.RepublishPeriod) if err != nil { return fx.Error(fmt.Errorf("failure to parse config setting IPNS.RepublishPeriod: %s", err)) } if !util.Debug && (d < time.Minute || d > (time.Hour*24)) { return fx.Error(fmt.Errorf("config setting IPNS.RepublishPeriod is not between 1min and 1day: %s", d)) } repubPeriod = d } if cfg.Ipns.RecordLifetime != "" { d, err := time.ParseDuration(cfg.Ipns.RecordLifetime) if err != nil { return fx.Error(fmt.Errorf("failure to parse config setting IPNS.RecordLifetime: %s", err)) } recordLifetime = d } isBitswapLibp2pEnabled := cfg.Bitswap.Libp2pEnabled.WithDefault(config.DefaultBitswapLibp2pEnabled) isBitswapServerEnabled := cfg.Bitswap.ServerEnabled.WithDefault(config.DefaultBitswapServerEnabled) isHTTPRetrievalEnabled := cfg.HTTPRetrieval.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled) // The Provide system handles both new CID announcements and periodic re-announcements. // Disabling is controlled by Provide.Enabled=false or setting Interval to 0. isProviderEnabled := cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled) && cfg.Provide.DHT.Interval.WithDefault(config.DefaultProvideDHTInterval) != 0 return fx.Options( fx.Provide(BitswapOptions(cfg)), fx.Provide(Bitswap(isBitswapServerEnabled, isBitswapLibp2pEnabled, isHTTPRetrievalEnabled)), fx.Provide(OnlineExchange(isBitswapLibp2pEnabled)), fx.Provide(DNSResolver), fx.Provide(Namesys(ipnsCacheSize, cfg.Ipns.MaxCacheTTL.WithDefault(config.DefaultIpnsMaxCacheTTL))), fx.Provide(Peering), PeerWith(cfg.Peering.Peers...), fx.Invoke(IpnsRepublisher(repubPeriod, recordLifetime)), fx.Provide(p2p.New), LibP2P(bcfg, cfg, userResourceOverrides), OnlineProviders(isProviderEnabled, cfg), ) } // Offline groups offline alternatives to Online units func Offline(cfg *config.Config) fx.Option { return fx.Options( fx.Provide(offline.Exchange), fx.Provide(DNSResolver), fx.Provide(Namesys(0, 0)), fx.Provide(libp2p.Routing), fx.Provide(libp2p.ContentRouting), fx.Provide(libp2p.OfflineRouting), fx.Provide(libp2p.ContentDiscovery), OfflineProviders(), ) } // Core groups basic IPFS services var Core = fx.Options( fx.Provide(Dag), fx.Provide(FetcherConfig), fx.Provide(PathResolverConfig), ) func Networked(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.PartialLimitConfig) fx.Option { if bcfg.Online { return Online(bcfg, cfg, userResourceOverrides) } return Offline(cfg) } // IPFS builds a group of fx Options based on the passed BuildCfg func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option { if bcfg == nil { bcfg = new(BuildCfg) } bcfgOpts, cfg := bcfg.options(ctx) if cfg == nil { return bcfgOpts // error } userResourceOverrides, err := bcfg.Repo.UserResourceOverrides() if err != nil { return fx.Error(err) } // Migrate users of deprecated Experimental.ShardingEnabled flag if cfg.Experimental.ShardingEnabled { logger.Fatal("The `Experimental.ShardingEnabled` field is no longer used, please remove it from the config. Use Import.UnixFSHAMTDirectorySizeThreshold instead.") } if !cfg.Internal.UnixFSShardingSizeThreshold.IsDefault() { msg := "The `Internal.UnixFSShardingSizeThreshold` field was renamed to `Import.UnixFSHAMTDirectorySizeThreshold`. Please update your config.\n" if !cfg.Import.UnixFSHAMTDirectorySizeThreshold.IsDefault() { logger.Fatal(msg) // conflicting values, hard fail } logger.Error(msg) // Migrate the old OptionalString value to the new OptionalBytes field. // Since OptionalBytes embeds OptionalString, we can construct it directly // with the old value, preserving the user's original string (e.g., "256KiB"). cfg.Import.UnixFSHAMTDirectorySizeThreshold = config.OptionalBytes{OptionalString: *cfg.Internal.UnixFSShardingSizeThreshold} } // Validate Import configuration if err := config.ValidateImportConfig(&cfg.Import); err != nil { return fx.Error(err) } // Validate Provide configuration if err := config.ValidateProvideConfig(&cfg.Provide); err != nil { return fx.Error(err) } // Directory sharding settings from Import config. // These globals affect both `ipfs add` and MFS (`ipfs files` API). shardSizeThreshold := cfg.Import.UnixFSHAMTDirectorySizeThreshold.WithDefault(config.DefaultUnixFSHAMTDirectorySizeThreshold) shardMaxFanout := cfg.Import.UnixFSHAMTDirectoryMaxFanout.WithDefault(config.DefaultUnixFSHAMTDirectoryMaxFanout) uio.HAMTShardingSize = int(shardSizeThreshold) uio.DefaultShardWidth = int(shardMaxFanout) uio.HAMTSizeEstimation = cfg.Import.HAMTSizeEstimationMode() providerStrategy := cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy) return fx.Options( bcfgOpts, Storage(bcfg, cfg), Identity(cfg), IPNS, Networked(bcfg, cfg, userResourceOverrides), fx.Provide(BlockService(cfg)), fx.Provide(Pinning(providerStrategy)), fx.Provide(Files(providerStrategy)), Core, ) } ================================================ FILE: core/node/helpers/helpers.go ================================================ package helpers import ( "context" "go.uber.org/fx" ) type MetricsCtx context.Context // LifecycleCtx creates a context which will be canceled when lifecycle stops // // This is a hack which we need because most of our services use contexts in a // wrong way func LifecycleCtx(mctx MetricsCtx, lc fx.Lifecycle) context.Context { ctx, cancel := context.WithCancel(mctx) lc.Append(fx.Hook{ OnStop: func(_ context.Context) error { cancel() return nil }, }) return ctx } ================================================ FILE: core/node/helpers.go ================================================ package node import ( "context" "errors" "go.uber.org/fx" ) type lcStartStop struct { fx.In LC fx.Lifecycle } // Append wraps a function into a fx.Hook and appends it to the fx.Lifecycle. func (lcss *lcStartStop) Append(f func() func()) { // Hooks are guaranteed to run in sequence. If a hook fails to start, its // OnStop won't be executed. var stopFunc func() lcss.LC.Append(fx.Hook{ OnStart: func(ctx context.Context) error { if ctx.Err() != nil { return nil } stopFunc = f() return nil }, OnStop: func(ctx context.Context) error { if ctx.Err() != nil { return nil } if stopFunc == nil { // Theoretically this shouldn't ever happen return errors.New("lcStatStop: stopFunc was nil") } stopFunc() return nil }, }) } func maybeProvide(opt any, enable bool) fx.Option { if enable { return fx.Provide(opt) } return fx.Options() } // nolint unused func maybeInvoke(opt any, enable bool) fx.Option { if enable { return fx.Invoke(opt) } return fx.Options() } ================================================ FILE: core/node/identity.go ================================================ package node import ( "fmt" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" ) func PeerID(id peer.ID) func() peer.ID { return func() peer.ID { return id } } // PrivateKey loads the private key from config func PrivateKey(sk crypto.PrivKey) func(id peer.ID) (crypto.PrivKey, error) { return func(id peer.ID) (crypto.PrivKey, error) { id2, err := peer.IDFromPrivateKey(sk) if err != nil { return nil, err } if id2 != id { return nil, fmt.Errorf("private key in config does not match id: %s != %s", id, id2) } return sk, nil } } ================================================ FILE: core/node/ipns.go ================================================ package node import ( "fmt" "time" "github.com/ipfs/boxo/ipns" util "github.com/ipfs/boxo/util" record "github.com/libp2p/go-libp2p-record" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peerstore" madns "github.com/multiformats/go-multiaddr-dns" "github.com/ipfs/boxo/namesys" "github.com/ipfs/boxo/namesys/republisher" "github.com/ipfs/kubo/repo" irouting "github.com/ipfs/kubo/routing" ) const DefaultIpnsCacheSize = 128 // RecordValidator provides namesys compatible routing record validator func RecordValidator(ps peerstore.Peerstore) record.Validator { return record.NamespacedValidator{ "pk": record.PublicKeyValidator{}, "ipns": ipns.Validator{KeyBook: ps}, } } // Namesys creates new name system func Namesys(cacheSize int, cacheMaxTTL time.Duration) func(rt irouting.ProvideManyRouter, rslv *madns.Resolver, repo repo.Repo) (namesys.NameSystem, error) { return func(rt irouting.ProvideManyRouter, rslv *madns.Resolver, repo repo.Repo) (namesys.NameSystem, error) { opts := []namesys.Option{ namesys.WithDatastore(repo.Datastore()), namesys.WithDNSResolver(rslv), namesys.WithMaxCacheTTL(cacheMaxTTL), } if cacheSize > 0 { opts = append(opts, namesys.WithCache(cacheSize)) } return namesys.NewNameSystem(rt, opts...) } } // IpnsRepublisher runs new IPNS republisher service func IpnsRepublisher(repubPeriod time.Duration, recordLifetime time.Duration) func(lcStartStop, namesys.NameSystem, repo.Repo, crypto.PrivKey) error { return func(lc lcStartStop, namesys namesys.NameSystem, repo repo.Repo, privKey crypto.PrivKey) error { repub := republisher.NewRepublisher(namesys, repo.Datastore(), privKey, repo.Keystore()) if repubPeriod != 0 { if !util.Debug && (repubPeriod < time.Minute || repubPeriod > (time.Hour*24)) { return fmt.Errorf("config setting IPNS.RepublishPeriod is not between 1min and 1day: %s", repubPeriod) } repub.Interval = repubPeriod } if recordLifetime != 0 { repub.RecordLifetime = recordLifetime } lc.Append(repub.Run) return nil } } ================================================ FILE: core/node/libp2p/addrs.go ================================================ package libp2p import ( "context" "fmt" "os" "path/filepath" "time" logging "github.com/ipfs/go-log/v2" version "github.com/ipfs/kubo" "github.com/ipfs/kubo/config" p2pforge "github.com/ipshipyard/p2p-forge/client" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/host" p2pbhost "github.com/libp2p/go-libp2p/p2p/host/basic" ma "github.com/multiformats/go-multiaddr" mamask "github.com/whyrusleeping/multiaddr-filter" "github.com/caddyserver/certmagic" "go.uber.org/fx" ) func AddrFilters(filters []string) func() (*ma.Filters, Libp2pOpts, error) { return func() (filter *ma.Filters, opts Libp2pOpts, err error) { filter = ma.NewFilters() opts.Opts = append(opts.Opts, libp2p.ConnectionGater((*filtersConnectionGater)(filter))) for _, s := range filters { f, err := mamask.NewMask(s) if err != nil { return filter, opts, fmt.Errorf("incorrectly formatted address filter in config: %s", s) } filter.AddFilter(*f, ma.ActionDeny) } return filter, opts, nil } } func makeAddrsFactory(announce []string, appendAnnounce []string, noAnnounce []string) (p2pbhost.AddrsFactory, error) { var err error // To assign to the slice in the for loop existing := make(map[string]bool) // To avoid duplicates annAddrs := make([]ma.Multiaddr, len(announce)) for i, addr := range announce { annAddrs[i], err = ma.NewMultiaddr(addr) if err != nil { return nil, err } existing[addr] = true } var appendAnnAddrs []ma.Multiaddr for _, addr := range appendAnnounce { if existing[addr] { // skip AppendAnnounce that is on the Announce list already continue } appendAddr, err := ma.NewMultiaddr(addr) if err != nil { return nil, err } appendAnnAddrs = append(appendAnnAddrs, appendAddr) } filters := ma.NewFilters() noAnnAddrs := map[string]bool{} for _, addr := range noAnnounce { f, err := mamask.NewMask(addr) if err == nil { filters.AddFilter(*f, ma.ActionDeny) continue } maddr, err := ma.NewMultiaddr(addr) if err != nil { return nil, err } noAnnAddrs[string(maddr.Bytes())] = true } return func(allAddrs []ma.Multiaddr) []ma.Multiaddr { var addrs []ma.Multiaddr if len(annAddrs) > 0 { addrs = annAddrs } else { addrs = allAddrs } addrs = append(addrs, appendAnnAddrs...) var out []ma.Multiaddr for _, maddr := range addrs { // check for exact matches ok := noAnnAddrs[string(maddr.Bytes())] // check for /ipcidr matches if !ok && !filters.AddrBlocked(maddr) { out = append(out, maddr) } } return out }, nil } func AddrsFactory(announce []string, appendAnnounce []string, noAnnounce []string) any { return func(params struct { fx.In ForgeMgr *p2pforge.P2PForgeCertMgr `optional:"true"` }, ) (opts Libp2pOpts, err error) { var addrsFactory p2pbhost.AddrsFactory announceAddrsFactory, err := makeAddrsFactory(announce, appendAnnounce, noAnnounce) if err != nil { return opts, err } if params.ForgeMgr == nil { addrsFactory = announceAddrsFactory } else { addrsFactory = func(multiaddrs []ma.Multiaddr) []ma.Multiaddr { forgeProcessing := params.ForgeMgr.AddressFactory()(multiaddrs) announceProcessing := announceAddrsFactory(forgeProcessing) return announceProcessing } } opts.Opts = append(opts.Opts, libp2p.AddrsFactory(addrsFactory)) return } } func ListenOn(addresses []string) any { return func() (opts Libp2pOpts) { return Libp2pOpts{ Opts: []libp2p.Option{ libp2p.ListenAddrStrings(addresses...), }, } } } func P2PForgeCertMgr(repoPath string, cfg config.AutoTLS, atlsLog *logging.ZapEventLogger) any { return func() (*p2pforge.P2PForgeCertMgr, error) { storagePath := filepath.Join(repoPath, "p2p-forge-certs") rawLogger := atlsLog.Desugar() // TODO: this should not be necessary after // https://github.com/ipshipyard/p2p-forge/pull/42 but keep it here for // now to help tracking down any remaining conditions causing // https://github.com/ipshipyard/p2p-forge/issues/8 certmagic.Default.Logger = rawLogger.Named("default_fixme") certmagic.DefaultACME.Logger = rawLogger.Named("default_acme_client_fixme") registrationDelay := cfg.RegistrationDelay.WithDefault(config.DefaultAutoTLSRegistrationDelay) if cfg.Enabled == config.True && cfg.RegistrationDelay.IsDefault() { // Skip delay if user explicitly enabled AutoTLS.Enabled in config // and did not set custom AutoTLS.RegistrationDelay registrationDelay = 0 * time.Second } certStorage := &certmagic.FileStorage{Path: storagePath} certMgr, err := p2pforge.NewP2PForgeCertMgr( p2pforge.WithLogger(rawLogger.Sugar()), p2pforge.WithForgeDomain(cfg.DomainSuffix.WithDefault(config.DefaultDomainSuffix)), p2pforge.WithForgeRegistrationEndpoint(cfg.RegistrationEndpoint.WithDefault(config.DefaultRegistrationEndpoint)), p2pforge.WithRegistrationDelay(registrationDelay), p2pforge.WithCAEndpoint(cfg.CAEndpoint.WithDefault(config.DefaultCAEndpoint)), p2pforge.WithForgeAuth(cfg.RegistrationToken.WithDefault(os.Getenv(p2pforge.ForgeAuthEnv))), p2pforge.WithUserAgent(version.GetUserAgentVersion()), p2pforge.WithCertificateStorage(certStorage), p2pforge.WithShortForgeAddrs(cfg.ShortAddrs.WithDefault(config.DefaultAutoTLSShortAddrs)), ) if err != nil { return nil, err } return certMgr, nil } } func StartP2PAutoTLS(lc fx.Lifecycle, certMgr *p2pforge.P2PForgeCertMgr, h host.Host) { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { certMgr.ProvideHost(h) return certMgr.Start() }, OnStop: func(ctx context.Context) error { certMgr.Stop() return nil }, }) } ================================================ FILE: core/node/libp2p/discovery.go ================================================ package libp2p import ( "context" "time" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/p2p/discovery/mdns" "go.uber.org/fx" "github.com/ipfs/kubo/core/node/helpers" ) const discoveryConnTimeout = time.Second * 30 type discoveryHandler struct { ctx context.Context host host.Host } func (dh *discoveryHandler) HandlePeerFound(p peer.AddrInfo) { log.Info("connecting to discovered peer: ", p) ctx, cancel := context.WithTimeout(dh.ctx, discoveryConnTimeout) defer cancel() if err := dh.host.Connect(ctx, p); err != nil { log.Warnf("failed to connect to peer %s found by discovery: %s", p.ID, err) } } func DiscoveryHandler(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host) *discoveryHandler { return &discoveryHandler{ ctx: helpers.LifecycleCtx(mctx, lc), host: host, } } func SetupDiscovery(useMdns bool) func(helpers.MetricsCtx, fx.Lifecycle, host.Host, *discoveryHandler) error { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, handler *discoveryHandler) error { if useMdns { service := mdns.NewMdnsService(host, mdns.ServiceName, handler) if err := service.Start(); err != nil { log.Error("error starting mdns service: ", err) return nil } } return nil } } ================================================ FILE: core/node/libp2p/dns.go ================================================ package libp2p import ( "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/p2p/net/swarm" madns "github.com/multiformats/go-multiaddr-dns" ) func MultiaddrResolver(rslv *madns.Resolver) (opts Libp2pOpts, err error) { opts.Opts = append(opts.Opts, libp2p.MultiaddrResolver(swarm.ResolverFromMaDNS{Resolver: rslv})) return opts, nil } ================================================ FILE: core/node/libp2p/fd/sys_not_unix.go ================================================ //go:build !linux && !darwin && !windows package fd func GetNumFDs() int { return 0 } ================================================ FILE: core/node/libp2p/fd/sys_unix.go ================================================ //go:build linux || darwin package fd import ( "golang.org/x/sys/unix" ) func GetNumFDs() int { var l unix.Rlimit if err := unix.Getrlimit(unix.RLIMIT_NOFILE, &l); err != nil { return 0 } return int(l.Cur) } ================================================ FILE: core/node/libp2p/fd/sys_windows.go ================================================ //go:build windows package fd import ( "math" ) func GetNumFDs() int { return math.MaxInt } ================================================ FILE: core/node/libp2p/filters.go ================================================ package libp2p import ( "github.com/libp2p/go-libp2p/core/connmgr" "github.com/libp2p/go-libp2p/core/control" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" ma "github.com/multiformats/go-multiaddr" ) // filtersConnectionGater is an adapter that turns multiaddr.Filter into a // connmgr.ConnectionGater. type filtersConnectionGater ma.Filters var _ connmgr.ConnectionGater = (*filtersConnectionGater)(nil) func (f *filtersConnectionGater) InterceptAddrDial(_ peer.ID, addr ma.Multiaddr) (allow bool) { return !(*ma.Filters)(f).AddrBlocked(addr) } func (f *filtersConnectionGater) InterceptPeerDial(p peer.ID) (allow bool) { return true } func (f *filtersConnectionGater) InterceptAccept(connAddr network.ConnMultiaddrs) (allow bool) { return !(*ma.Filters)(f).AddrBlocked(connAddr.RemoteMultiaddr()) } func (f *filtersConnectionGater) InterceptSecured(_ network.Direction, _ peer.ID, connAddr network.ConnMultiaddrs) (allow bool) { return !(*ma.Filters)(f).AddrBlocked(connAddr.RemoteMultiaddr()) } func (f *filtersConnectionGater) InterceptUpgraded(_ network.Conn) (allow bool, reason control.DisconnectReason) { return true, 0 } ================================================ FILE: core/node/libp2p/host.go ================================================ package libp2p import ( "context" "github.com/libp2p/go-libp2p" record "github.com/libp2p/go-libp2p-record" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/peerstore" "github.com/libp2p/go-libp2p/core/routing" routedhost "github.com/libp2p/go-libp2p/p2p/host/routed" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/node/helpers" "github.com/ipfs/kubo/repo" "go.uber.org/fx" ) type P2PHostIn struct { fx.In Repo repo.Repo Validator record.Validator HostOption HostOption RoutingOption RoutingOption ID peer.ID Peerstore peerstore.Peerstore Opts [][]libp2p.Option `group:"libp2p"` } type P2PHostOut struct { fx.Out Host host.Host Routing routing.Routing `name:"initialrouting"` } func Host(mctx helpers.MetricsCtx, lc fx.Lifecycle, params P2PHostIn) (out P2PHostOut, err error) { opts := []libp2p.Option{libp2p.NoListenAddrs} for _, o := range params.Opts { opts = append(opts, o...) } ctx := helpers.LifecycleCtx(mctx, lc) cfg, err := params.Repo.Config() if err != nil { return out, err } // Use auto-config resolution for actual connectivity bootstrappers, err := cfg.BootstrapPeersWithAutoConf() if err != nil { return out, err } // Optimistic provide is enabled either via dedicated expierimental flag, or when DHT Provide Sweep is enabled. // When DHT Provide Sweep is enabled, all provide operations go through the // `SweepingProvider`, hence the provides don't use the optimistic provide // logic. Provides use `SweepingProvider.StartProviding()` and not // `IpfsDHT.Provide()`, which is where the optimistic provide logic is // implemented. However, `IpfsDHT.Provide()` is used to quickly provide roots // when user manually adds content with the `--fast-provide` flag enabled. In // this case we want to use optimistic provide logic to quickly announce the // content to the network. This should be the only use case of // `IpfsDHT.Provide()` when DHT Provide Sweep is enabled. optimisticProvide := cfg.Experimental.OptimisticProvide || cfg.Provide.DHT.SweepEnabled.WithDefault(config.DefaultProvideDHTSweepEnabled) routingOptArgs := RoutingOptionArgs{ Ctx: ctx, Datastore: params.Repo.Datastore(), Validator: params.Validator, BootstrapPeers: bootstrappers, OptimisticProvide: optimisticProvide, OptimisticProvideJobsPoolSize: cfg.Experimental.OptimisticProvideJobsPoolSize, LoopbackAddressesOnLanDHT: cfg.Routing.LoopbackAddressesOnLanDHT.WithDefault(config.DefaultLoopbackAddressesOnLanDHT), } opts = append(opts, libp2p.Routing(func(h host.Host) (routing.PeerRouting, error) { args := routingOptArgs args.Host = h r, err := params.RoutingOption(args) out.Routing = r return r, err })) out.Host, err = params.HostOption(params.ID, params.Peerstore, opts...) if err != nil { return P2PHostOut{}, err } routingOptArgs.Host = out.Host // this code is necessary just for tests: mock network constructions // ignore the libp2p constructor options that actually construct the routing! if out.Routing == nil { r, err := params.RoutingOption(routingOptArgs) if err != nil { return P2PHostOut{}, err } out.Routing = r out.Host = routedhost.Wrap(out.Host, out.Routing) } lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return out.Host.Close() }, }) return out, err } ================================================ FILE: core/node/libp2p/hostopt.go ================================================ package libp2p import ( "fmt" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/peerstore" ) type HostOption func(id peer.ID, ps peerstore.Peerstore, options ...libp2p.Option) (host.Host, error) var DefaultHostOption HostOption = constructPeerHost // isolates the complex initialization steps func constructPeerHost(id peer.ID, ps peerstore.Peerstore, options ...libp2p.Option) (host.Host, error) { pkey := ps.PrivKey(id) if pkey == nil { return nil, fmt.Errorf("missing private key for node ID: %s", id) } options = append([]libp2p.Option{libp2p.Identity(pkey), libp2p.Peerstore(ps)}, options...) return libp2p.New(options...) } ================================================ FILE: core/node/libp2p/libp2p.go ================================================ package libp2p import ( "fmt" "sort" "time" version "github.com/ipfs/kubo" config "github.com/ipfs/kubo/config" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/peerstore" "github.com/libp2p/go-libp2p/p2p/net/connmgr" "go.uber.org/fx" ) var log = logging.Logger("p2pnode") type Libp2pOpts struct { fx.Out Opts []libp2p.Option `group:"libp2p"` } func ConnectionManager(low, high int, grace, silence time.Duration) func() (opts Libp2pOpts, err error) { return func() (opts Libp2pOpts, err error) { cm, err := connmgr.NewConnManager(low, high, connmgr.WithGracePeriod(grace), connmgr.WithSilencePeriod(silence), ) if err != nil { return opts, err } opts.Opts = append(opts.Opts, libp2p.ConnectionManager(cm)) return } } func PstoreAddSelfKeys(id peer.ID, sk crypto.PrivKey, ps peerstore.Peerstore) error { if err := ps.AddPubKey(id, sk.GetPublic()); err != nil { return err } return ps.AddPrivKey(id, sk) } func UserAgent() func() (opts Libp2pOpts, err error) { return simpleOpt(libp2p.UserAgent(version.GetUserAgentVersion())) } func simpleOpt(opt libp2p.Option) func() (opts Libp2pOpts, err error) { return func() (opts Libp2pOpts, err error) { opts.Opts = append(opts.Opts, opt) return } } type priorityOption struct { priority, defaultPriority config.Priority opt libp2p.Option } func prioritizeOptions(opts []priorityOption) libp2p.Option { type popt struct { priority int64 // lower priority values mean higher priority opt libp2p.Option } enabledOptions := make([]popt, 0, len(opts)) for _, o := range opts { if prio, ok := o.priority.WithDefault(o.defaultPriority); ok { enabledOptions = append(enabledOptions, popt{ priority: prio, opt: o.opt, }) } } sort.Slice(enabledOptions, func(i, j int) bool { return enabledOptions[i].priority < enabledOptions[j].priority }) p2pOpts := make([]libp2p.Option, len(enabledOptions)) for i, opt := range enabledOptions { p2pOpts[i] = opt.opt } return libp2p.ChainOptions(p2pOpts...) } func ForceReachability(val *config.OptionalString) func() (opts Libp2pOpts, err error) { return func() (opts Libp2pOpts, err error) { if val.IsDefault() { return } v := val.WithDefault("unrecognized") switch v { case "public": opts.Opts = append(opts.Opts, libp2p.ForceReachabilityPublic()) case "private": opts.Opts = append(opts.Opts, libp2p.ForceReachabilityPrivate()) default: return opts, fmt.Errorf("unrecognized reachability option: %s", v) } return } } ================================================ FILE: core/node/libp2p/libp2p_test.go ================================================ package libp2p import ( "fmt" "strconv" "testing" "github.com/libp2p/go-libp2p" ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/require" ) func TestPrioritize(t *testing.T) { // The option is encoded into the port number of a TCP multiaddr. // By extracting the port numbers obtained from the applied option, we can make sure that // prioritization sorted the options correctly. newOption := func(num int) libp2p.Option { return func(cfg *libp2p.Config) error { cfg.ListenAddrs = append(cfg.ListenAddrs, ma.StringCast(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", num))) return nil } } extractNums := func(cfg *libp2p.Config) []int { addrs := cfg.ListenAddrs nums := make([]int, 0, len(addrs)) for _, addr := range addrs { _, comp := ma.SplitLast(addr) num, err := strconv.Atoi(comp.Value()) require.NoError(t, err) nums = append(nums, num) } return nums } t.Run("using default priorities", func(t *testing.T) { opts := []priorityOption{ {defaultPriority: 200, opt: newOption(200)}, {defaultPriority: 1, opt: newOption(1)}, {defaultPriority: 300, opt: newOption(300)}, } var cfg libp2p.Config require.NoError(t, prioritizeOptions(opts)(&cfg)) require.Equal(t, extractNums(&cfg), []int{1, 200, 300}) }) t.Run("using custom priorities", func(t *testing.T) { opts := []priorityOption{ {defaultPriority: 200, priority: 1, opt: newOption(1)}, {defaultPriority: 1, priority: 300, opt: newOption(300)}, {defaultPriority: 300, priority: 20, opt: newOption(20)}, } var cfg libp2p.Config require.NoError(t, prioritizeOptions(opts)(&cfg)) require.Equal(t, extractNums(&cfg), []int{1, 20, 300}) }) } ================================================ FILE: core/node/libp2p/nat.go ================================================ package libp2p import ( "time" config "github.com/ipfs/kubo/config" "github.com/libp2p/go-libp2p" ) var NatPortMap = simpleOpt(libp2p.NATPortMap()) func AutoNATService(throttle *config.AutoNATThrottleConfig, v1only bool) func() Libp2pOpts { return func() (opts Libp2pOpts) { opts.Opts = append(opts.Opts, libp2p.EnableNATService()) if throttle != nil { opts.Opts = append(opts.Opts, libp2p.AutoNATServiceRateLimit( throttle.GlobalLimit, throttle.PeerLimit, throttle.Interval.WithDefault(time.Minute), ), ) } // While V1 still exists and V2 rollout is in progress // (https://github.com/ipfs/kubo/issues/10091) we check a flag that // allows users to disable V2 and run V1-only mode if !v1only { opts.Opts = append(opts.Opts, libp2p.EnableAutoNATv2()) } return opts } } ================================================ FILE: core/node/libp2p/peerstore.go ================================================ package libp2p import ( "context" "github.com/libp2p/go-libp2p/core/peerstore" "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" "go.uber.org/fx" ) func Peerstore(lc fx.Lifecycle) (peerstore.Peerstore, error) { pstore, err := pstoremem.NewPeerstore() if err != nil { return nil, err } lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return pstore.Close() }, }) return pstore, nil } ================================================ FILE: core/node/libp2p/pnet.go ================================================ package libp2p import ( "bytes" "context" "fmt" "time" "github.com/ipfs/kubo/repo" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/pnet" "go.uber.org/fx" "golang.org/x/crypto/salsa20" "golang.org/x/crypto/sha3" ) type PNetFingerprint []byte func PNet(repo repo.Repo) (opts Libp2pOpts, fp PNetFingerprint, err error) { swarmkey, err := repo.SwarmKey() if err != nil || swarmkey == nil { return opts, nil, err } psk, err := pnet.DecodeV1PSK(bytes.NewReader(swarmkey)) if err != nil { return opts, nil, fmt.Errorf("failed to configure private network: %s", err) } opts.Opts = append(opts.Opts, libp2p.PrivateNetwork(psk)) return opts, pnetFingerprint(psk), nil } func PNetChecker(repo repo.Repo, ph host.Host, lc fx.Lifecycle) error { // TODO: better check? swarmkey, err := repo.SwarmKey() if err != nil || swarmkey == nil { return err } done := make(chan struct{}) lc.Append(fx.Hook{ OnStart: func(_ context.Context) error { go func() { t := time.NewTicker(30 * time.Second) defer t.Stop() <-t.C // swallow one tick for { select { case <-t.C: if len(ph.Network().Peers()) == 0 { log.Warn("We are in private network and have no peers.") log.Warn("This might be configuration mistake.") } case <-done: return } } }() return nil }, OnStop: func(_ context.Context) error { close(done) return nil }, }) return nil } func pnetFingerprint(psk pnet.PSK) []byte { var pskArr [32]byte copy(pskArr[:], psk) enc := make([]byte, 64) zeros := make([]byte, 64) out := make([]byte, 16) // We encrypt data first so we don't feed PSK to hash function. // Salsa20 function is not reversible thus increasing our security margin. salsa20.XORKeyStream(enc, zeros, []byte("finprint"), &pskArr) // Then do Shake-128 hash to reduce its length. // This way if for some reason Shake is broken and Salsa20 preimage is possible, // attacker has only half of the bytes necessary to recreate psk. sha3.ShakeSum128(out, enc) return out } ================================================ FILE: core/node/libp2p/pubsub.go ================================================ package libp2p import ( "context" "errors" "log/slog" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/discovery" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "go.uber.org/fx" "github.com/ipfs/kubo/core/node/helpers" "github.com/ipfs/kubo/repo" ) type pubsubParams struct { fx.In Repo repo.Repo Host host.Host Discovery discovery.Discovery } func FloodSub(pubsubOptions ...pubsub.Option) any { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, params pubsubParams) (service *pubsub.PubSub, err error) { return pubsub.NewFloodSub( helpers.LifecycleCtx(mctx, lc), params.Host, append(pubsubOptions, pubsub.WithDiscovery(params.Discovery), pubsub.WithDefaultValidator(newSeqnoValidator(params.Repo.Datastore())))..., ) } } func GossipSub(pubsubOptions ...pubsub.Option) any { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, params pubsubParams) (service *pubsub.PubSub, err error) { return pubsub.NewGossipSub( helpers.LifecycleCtx(mctx, lc), params.Host, append(pubsubOptions, pubsub.WithDiscovery(params.Discovery), pubsub.WithFloodPublish(true), // flood own publications to all peers for reliable IPNS delivery pubsub.WithDefaultValidator(newSeqnoValidator(params.Repo.Datastore())))..., ) } } func newSeqnoValidator(ds datastore.Datastore) pubsub.ValidatorEx { return pubsub.NewBasicSeqnoValidator(&seqnoStore{ds: ds}, slog.New(logging.SlogHandler()).With("logger", "pubsub")) } // SeqnoStorePrefix is the datastore prefix for pubsub seqno validator state. const SeqnoStorePrefix = "/pubsub/seqno/" // seqnoStore implements pubsub.PeerMetadataStore using the repo datastore. // It stores the maximum seen sequence number per peer to prevent message // cycles when network diameter exceeds the timecache span. type seqnoStore struct { ds datastore.Datastore } var _ pubsub.PeerMetadataStore = (*seqnoStore)(nil) // Get returns the stored seqno for a peer, or (nil, nil) if the peer is unknown. // Returning (nil, nil) for unknown peers allows BasicSeqnoValidator to accept // the first message from any peer. func (s *seqnoStore) Get(ctx context.Context, p peer.ID) ([]byte, error) { key := datastore.NewKey(SeqnoStorePrefix + p.String()) val, err := s.ds.Get(ctx, key) if errors.Is(err, datastore.ErrNotFound) { return nil, nil } return val, err } // Put stores the seqno for a peer. func (s *seqnoStore) Put(ctx context.Context, p peer.ID, val []byte) error { key := datastore.NewKey(SeqnoStorePrefix + p.String()) return s.ds.Put(ctx, key, val) } ================================================ FILE: core/node/libp2p/pubsub_test.go ================================================ package libp2p import ( "encoding/binary" "testing" "github.com/ipfs/go-datastore" syncds "github.com/ipfs/go-datastore/sync" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" ) // TestSeqnoStore tests the seqnoStore implementation which backs the // BasicSeqnoValidator. The validator prevents message cycles when network // diameter exceeds the timecache span by tracking the maximum sequence number // seen from each peer. func TestSeqnoStore(t *testing.T) { ctx := t.Context() ds := syncds.MutexWrap(datastore.NewMapDatastore()) store := &seqnoStore{ds: ds} peerA, err := peer.Decode("12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5") require.NoError(t, err) peerB, err := peer.Decode("12D3KooWJRqDKTRjvXeGdUEgwkHNsoghYMBUagNYgLPdA4mqdTeo") require.NoError(t, err) // BasicSeqnoValidator expects Get to return (nil, nil) for unknown peers, // not an error. This allows the validator to accept the first message from // any peer without special-casing. t.Run("unknown peer returns nil without error", func(t *testing.T) { val, err := store.Get(ctx, peerA) require.NoError(t, err) require.Nil(t, val, "unknown peer should return nil, not empty slice") }) // Verify basic store/retrieve functionality with a sequence number encoded // as big-endian uint64, matching the format used by BasicSeqnoValidator. t.Run("stores and retrieves seqno", func(t *testing.T) { seqno := uint64(12345) data := make([]byte, 8) binary.BigEndian.PutUint64(data, seqno) err := store.Put(ctx, peerA, data) require.NoError(t, err) val, err := store.Get(ctx, peerA) require.NoError(t, err) require.Equal(t, seqno, binary.BigEndian.Uint64(val)) }) // Each peer must have isolated storage. If peer data leaked between peers, // the validator would incorrectly reject valid messages or accept replays. t.Run("isolates seqno per peer", func(t *testing.T) { seqnoA := uint64(100) seqnoB := uint64(200) dataA := make([]byte, 8) dataB := make([]byte, 8) binary.BigEndian.PutUint64(dataA, seqnoA) binary.BigEndian.PutUint64(dataB, seqnoB) err := store.Put(ctx, peerA, dataA) require.NoError(t, err) err = store.Put(ctx, peerB, dataB) require.NoError(t, err) valA, err := store.Get(ctx, peerA) require.NoError(t, err) require.Equal(t, seqnoA, binary.BigEndian.Uint64(valA)) valB, err := store.Get(ctx, peerB) require.NoError(t, err) require.Equal(t, seqnoB, binary.BigEndian.Uint64(valB)) }) // The validator updates the stored seqno when accepting messages with // higher seqnos. This test verifies that updates work correctly. t.Run("updates seqno to higher value", func(t *testing.T) { seqno1 := uint64(1000) seqno2 := uint64(2000) data1 := make([]byte, 8) data2 := make([]byte, 8) binary.BigEndian.PutUint64(data1, seqno1) binary.BigEndian.PutUint64(data2, seqno2) err := store.Put(ctx, peerA, data1) require.NoError(t, err) err = store.Put(ctx, peerA, data2) require.NoError(t, err) val, err := store.Get(ctx, peerA) require.NoError(t, err) require.Equal(t, seqno2, binary.BigEndian.Uint64(val)) }) // Verify the datastore key format. This is important for: // 1. Debugging: operators can inspect/clear pubsub state // 2. Migrations: future changes need to know the key format t.Run("uses expected datastore key format", func(t *testing.T) { seqno := uint64(42) data := make([]byte, 8) binary.BigEndian.PutUint64(data, seqno) err := store.Put(ctx, peerA, data) require.NoError(t, err) // Verify we can read directly from datastore with expected key expectedKey := datastore.NewKey("/pubsub/seqno/" + peerA.String()) val, err := ds.Get(ctx, expectedKey) require.NoError(t, err) require.Equal(t, seqno, binary.BigEndian.Uint64(val)) }) // Verify data persists when creating a new store instance with the same // underlying datastore. This simulates node restart. t.Run("persists across store instances", func(t *testing.T) { seqno := uint64(99999) data := make([]byte, 8) binary.BigEndian.PutUint64(data, seqno) err := store.Put(ctx, peerB, data) require.NoError(t, err) // Create new store instance with same datastore store2 := &seqnoStore{ds: ds} val, err := store2.Get(ctx, peerB) require.NoError(t, err) require.Equal(t, seqno, binary.BigEndian.Uint64(val)) }) } ================================================ FILE: core/node/libp2p/rcmgr.go ================================================ package libp2p import ( "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/node/helpers" "github.com/ipfs/kubo/repo" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/multiformats/go-multiaddr" "go.uber.org/fx" ) var rcmgrLogger = logging.Logger("rcmgr") const NetLimitTraceFilename = "rcmgr.json.gz" var ErrNoResourceMgr = errors.New("missing ResourceMgr: make sure the daemon is running with Swarm.ResourceMgr.Enabled") func ResourceManager(repoPath string, cfg config.SwarmConfig, userResourceOverrides rcmgr.PartialLimitConfig) any { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo repo.Repo) (network.ResourceManager, Libp2pOpts, error) { var manager network.ResourceManager var opts Libp2pOpts enabled := cfg.ResourceMgr.Enabled.WithDefault(true) // ENV overrides Config (if present) switch os.Getenv("LIBP2P_RCMGR") { case "0", "false": enabled = false case "1", "true": enabled = true } if enabled { log.Debug("libp2p resource manager is enabled") limitConfig, msg, err := LimitConfig(cfg, userResourceOverrides) if err != nil { return nil, opts, fmt.Errorf("creating final Resource Manager config: %w", err) } if !isPartialConfigEmpty(userResourceOverrides) { rcmgrLogger.Info(` libp2p-resource-limit-overrides.json has been loaded, "default" fields will be filled in with autocomputed defaults.`) } // We want to see this message on startup, that's why we are using fmt instead of log. rcmgrLogger.Info(msg) if err := ensureConnMgrMakeSenseVsResourceMgr(limitConfig, cfg); err != nil { return nil, opts, err } str, err := rcmgr.NewStatsTraceReporter() if err != nil { return nil, opts, err } ropts := []rcmgr.Option{ rcmgr.WithTraceReporter(str), rcmgr.WithLimitPerSubnet( nil, []rcmgr.ConnLimitPerSubnet{ { ConnCount: 16, PrefixLength: 56, }, { ConnCount: 8 * 16, PrefixLength: 48, }, }), } if len(cfg.ResourceMgr.Allowlist) > 0 { var mas []multiaddr.Multiaddr for _, maStr := range cfg.ResourceMgr.Allowlist { ma, err := multiaddr.NewMultiaddr(maStr) if err != nil { log.Errorf("failed to parse multiaddr=%v for allowlist, skipping. err=%v", maStr, err) continue } mas = append(mas, ma) } ropts = append(ropts, rcmgr.WithAllowlistedMultiaddrs(mas)) log.Infof("Setting allowlist to: %v", mas) } if os.Getenv("LIBP2P_DEBUG_RCMGR") != "" { traceFilePath := filepath.Join(repoPath, NetLimitTraceFilename) ropts = append(ropts, rcmgr.WithTrace(traceFilePath)) } limiter := rcmgr.NewFixedLimiter(limitConfig) manager, err = rcmgr.NewResourceManager(limiter, ropts...) if err != nil { return nil, opts, fmt.Errorf("creating libp2p resource manager: %w", err) } lrm := &loggingResourceManager{ logger: &logging.Logger("resourcemanager").SugaredLogger, delegate: manager, } lrm.start(helpers.LifecycleCtx(mctx, lc)) manager = lrm } else { rcmgrLogger.Info("go-libp2p resource manager protection disabled") manager = &network.NullResourceManager{} } opts.Opts = append(opts.Opts, libp2p.ResourceManager(manager)) lc.Append(fx.Hook{ OnStop: func(_ context.Context) error { return manager.Close() }, }) return manager, opts, nil } } func isPartialConfigEmpty(cfg rcmgr.PartialLimitConfig) bool { var emptyResourceConfig rcmgr.ResourceLimits if cfg.System != emptyResourceConfig || cfg.Transient != emptyResourceConfig || cfg.AllowlistedSystem != emptyResourceConfig || cfg.AllowlistedTransient != emptyResourceConfig || cfg.ServiceDefault != emptyResourceConfig || cfg.ServicePeerDefault != emptyResourceConfig || cfg.ProtocolDefault != emptyResourceConfig || cfg.ProtocolPeerDefault != emptyResourceConfig || cfg.PeerDefault != emptyResourceConfig || cfg.Conn != emptyResourceConfig || cfg.Stream != emptyResourceConfig { return false } for _, v := range cfg.Service { if v != emptyResourceConfig { return false } } for _, v := range cfg.ServicePeer { if v != emptyResourceConfig { return false } } for _, v := range cfg.Protocol { if v != emptyResourceConfig { return false } } for _, v := range cfg.ProtocolPeer { if v != emptyResourceConfig { return false } } for _, v := range cfg.Peer { if v != emptyResourceConfig { return false } } return true } // LimitConfig returns the union of the Computed Default Limits and the User Supplied Override Limits. func LimitConfig(cfg config.SwarmConfig, userResourceOverrides rcmgr.PartialLimitConfig) (limitConfig rcmgr.ConcreteLimitConfig, logMessageForStartup string, err error) { limitConfig, msg, err := createDefaultLimitConfig(cfg) if err != nil { return rcmgr.ConcreteLimitConfig{}, msg, err } // The logic for defaults and overriding with specified userResourceOverrides // is documented in docs/libp2p-resource-management.md. // Any changes here should be reflected there. // This effectively overrides the computed default LimitConfig with any non-"useDefault" values from the userResourceOverrides file. // Because of how how Build works, any rcmgr.Default value in userResourceOverrides // will be overridden with a computed default value. limitConfig = userResourceOverrides.Build(limitConfig) return limitConfig, msg, nil } type ResourceLimitsAndUsage struct { // This is duplicated from rcmgr.ResourceResourceLimits but adding *Usage fields. Memory rcmgr.LimitVal64 MemoryUsage int64 FD rcmgr.LimitVal FDUsage int Conns rcmgr.LimitVal ConnsUsage int ConnsInbound rcmgr.LimitVal ConnsInboundUsage int ConnsOutbound rcmgr.LimitVal ConnsOutboundUsage int Streams rcmgr.LimitVal StreamsUsage int StreamsInbound rcmgr.LimitVal StreamsInboundUsage int StreamsOutbound rcmgr.LimitVal StreamsOutboundUsage int } func (u ResourceLimitsAndUsage) ToResourceLimits() rcmgr.ResourceLimits { return rcmgr.ResourceLimits{ Memory: u.Memory, FD: u.FD, Conns: u.Conns, ConnsInbound: u.ConnsInbound, ConnsOutbound: u.ConnsOutbound, Streams: u.Streams, StreamsInbound: u.StreamsInbound, StreamsOutbound: u.StreamsOutbound, } } type LimitsConfigAndUsage struct { // This is duplicated from rcmgr.ResourceManagerStat but using ResourceLimitsAndUsage // instead of network.ScopeStat. System ResourceLimitsAndUsage Transient ResourceLimitsAndUsage Services map[string]ResourceLimitsAndUsage `json:",omitempty"` Protocols map[protocol.ID]ResourceLimitsAndUsage `json:",omitempty"` Peers map[peer.ID]ResourceLimitsAndUsage `json:",omitempty"` } func (u LimitsConfigAndUsage) MarshalJSON() ([]byte, error) { // we want to marshal the encoded peer id encodedPeerMap := make(map[string]ResourceLimitsAndUsage, len(u.Peers)) for p, v := range u.Peers { encodedPeerMap[p.String()] = v } type Alias LimitsConfigAndUsage return json.Marshal(&struct { *Alias Peers map[string]ResourceLimitsAndUsage `json:",omitempty"` }{ Alias: (*Alias)(&u), Peers: encodedPeerMap, }) } func (u LimitsConfigAndUsage) ToPartialLimitConfig() (result rcmgr.PartialLimitConfig) { result.System = u.System.ToResourceLimits() result.Transient = u.Transient.ToResourceLimits() result.Service = make(map[string]rcmgr.ResourceLimits, len(u.Services)) for s, l := range u.Services { result.Service[s] = l.ToResourceLimits() } result.Protocol = make(map[protocol.ID]rcmgr.ResourceLimits, len(u.Protocols)) for p, l := range u.Protocols { result.Protocol[p] = l.ToResourceLimits() } result.Peer = make(map[peer.ID]rcmgr.ResourceLimits, len(u.Peers)) for p, l := range u.Peers { result.Peer[p] = l.ToResourceLimits() } return } func MergeLimitsAndStatsIntoLimitsConfigAndUsage(l rcmgr.ConcreteLimitConfig, stats rcmgr.ResourceManagerStat) LimitsConfigAndUsage { limits := l.ToPartialLimitConfig() return LimitsConfigAndUsage{ System: mergeResourceLimitsAndScopeStatToResourceLimitsAndUsage(limits.System, stats.System), Transient: mergeResourceLimitsAndScopeStatToResourceLimitsAndUsage(limits.Transient, stats.Transient), Services: mergeLimitsAndStatsMapIntoLimitsConfigAndUsageMap(limits.Service, stats.Services), Protocols: mergeLimitsAndStatsMapIntoLimitsConfigAndUsageMap(limits.Protocol, stats.Protocols), Peers: mergeLimitsAndStatsMapIntoLimitsConfigAndUsageMap(limits.Peer, stats.Peers), } } func mergeLimitsAndStatsMapIntoLimitsConfigAndUsageMap[K comparable](limits map[K]rcmgr.ResourceLimits, stats map[K]network.ScopeStat) map[K]ResourceLimitsAndUsage { r := make(map[K]ResourceLimitsAndUsage, maxInt(len(limits), len(stats))) for p, s := range stats { var l rcmgr.ResourceLimits if limits != nil { if rl, ok := limits[p]; ok { l = rl } } r[p] = mergeResourceLimitsAndScopeStatToResourceLimitsAndUsage(l, s) } for p, s := range limits { if _, ok := stats[p]; ok { continue // we already processed this element in the loop above } r[p] = mergeResourceLimitsAndScopeStatToResourceLimitsAndUsage(s, network.ScopeStat{}) } return r } func maxInt(x, y int) int { if x > y { return x } return y } func mergeResourceLimitsAndScopeStatToResourceLimitsAndUsage(rl rcmgr.ResourceLimits, ss network.ScopeStat) ResourceLimitsAndUsage { return ResourceLimitsAndUsage{ Memory: rl.Memory, MemoryUsage: ss.Memory, FD: rl.FD, FDUsage: ss.NumFD, Conns: rl.Conns, ConnsUsage: ss.NumConnsOutbound + ss.NumConnsInbound, ConnsOutbound: rl.ConnsOutbound, ConnsOutboundUsage: ss.NumConnsOutbound, ConnsInbound: rl.ConnsInbound, ConnsInboundUsage: ss.NumConnsInbound, Streams: rl.Streams, StreamsUsage: ss.NumStreamsOutbound + ss.NumStreamsInbound, StreamsOutbound: rl.StreamsOutbound, StreamsOutboundUsage: ss.NumStreamsOutbound, StreamsInbound: rl.StreamsInbound, StreamsInboundUsage: ss.NumStreamsInbound, } } type ResourceInfos []ResourceInfo type ResourceInfo struct { ScopeName string LimitName string LimitValue rcmgr.LimitVal64 CurrentUsage int64 } // LimitConfigsToInfo gets limits and stats and generates a list of scopes and limits to be printed. func LimitConfigsToInfo(stats LimitsConfigAndUsage) ResourceInfos { result := ResourceInfos{} result = append(result, resourceLimitsAndUsageToResourceInfo(config.ResourceMgrSystemScope, stats.System)...) result = append(result, resourceLimitsAndUsageToResourceInfo(config.ResourceMgrTransientScope, stats.Transient)...) for i, s := range stats.Services { result = append(result, resourceLimitsAndUsageToResourceInfo( config.ResourceMgrServiceScopePrefix+i, s, )...) } for i, p := range stats.Protocols { result = append(result, resourceLimitsAndUsageToResourceInfo( config.ResourceMgrProtocolScopePrefix+string(i), p, )...) } for i, p := range stats.Peers { result = append(result, resourceLimitsAndUsageToResourceInfo( config.ResourceMgrPeerScopePrefix+i.String(), p, )...) } return result } const ( limitNameMemory = "Memory" limitNameFD = "FD" limitNameConns = "Conns" limitNameConnsInbound = "ConnsInbound" limitNameConnsOutbound = "ConnsOutbound" limitNameStreams = "Streams" limitNameStreamsInbound = "StreamsInbound" limitNameStreamsOutbound = "StreamsOutbound" ) var limits = []string{ limitNameMemory, limitNameFD, limitNameConns, limitNameConnsInbound, limitNameConnsOutbound, limitNameStreams, limitNameStreamsInbound, limitNameStreamsOutbound, } func resourceLimitsAndUsageToResourceInfo(scopeName string, stats ResourceLimitsAndUsage) ResourceInfos { result := ResourceInfos{} for _, l := range limits { ri := ResourceInfo{ ScopeName: scopeName, } switch l { case limitNameMemory: ri.LimitName = limitNameMemory ri.LimitValue = stats.Memory ri.CurrentUsage = stats.MemoryUsage case limitNameFD: ri.LimitName = limitNameFD ri.LimitValue = rcmgr.LimitVal64(stats.FD) ri.CurrentUsage = int64(stats.FDUsage) case limitNameConns: ri.LimitName = limitNameConns ri.LimitValue = rcmgr.LimitVal64(stats.Conns) ri.CurrentUsage = int64(stats.ConnsUsage) case limitNameConnsInbound: ri.LimitName = limitNameConnsInbound ri.LimitValue = rcmgr.LimitVal64(stats.ConnsInbound) ri.CurrentUsage = int64(stats.ConnsInboundUsage) case limitNameConnsOutbound: ri.LimitName = limitNameConnsOutbound ri.LimitValue = rcmgr.LimitVal64(stats.ConnsOutbound) ri.CurrentUsage = int64(stats.ConnsOutboundUsage) case limitNameStreams: ri.LimitName = limitNameStreams ri.LimitValue = rcmgr.LimitVal64(stats.Streams) ri.CurrentUsage = int64(stats.StreamsUsage) case limitNameStreamsInbound: ri.LimitName = limitNameStreamsInbound ri.LimitValue = rcmgr.LimitVal64(stats.StreamsInbound) ri.CurrentUsage = int64(stats.StreamsInboundUsage) case limitNameStreamsOutbound: ri.LimitName = limitNameStreamsOutbound ri.LimitValue = rcmgr.LimitVal64(stats.StreamsOutbound) ri.CurrentUsage = int64(stats.StreamsOutboundUsage) } if ri.LimitValue == rcmgr.Unlimited64 || ri.LimitValue == rcmgr.DefaultLimit64 { // ignore unlimited and unset limits to remove noise from output. continue } result = append(result, ri) } return result } func ensureConnMgrMakeSenseVsResourceMgr(concreteLimits rcmgr.ConcreteLimitConfig, cfg config.SwarmConfig) error { if cfg.ConnMgr.Type.WithDefault(config.DefaultConnMgrType) == "none" || len(cfg.ResourceMgr.Allowlist) != 0 { // no connmgr OR // If an allowlist is set, a user may be enacting some form of DoS defense. // We don't want want to modify the System.ConnsInbound in that case for example // as it may make sense for it to be (and stay) as "blockAll" // so that only connections within the allowlist of multiaddrs get established. return nil } rcm := concreteLimits.ToPartialLimitConfig() highWater := cfg.ConnMgr.HighWater.WithDefault(config.DefaultConnMgrHighWater) if (rcm.System.Conns > rcmgr.DefaultLimit || rcm.System.Conns == rcmgr.BlockAllLimit) && int64(rcm.System.Conns) <= highWater { // nolint return fmt.Errorf(` Unable to initialize libp2p due to conflicting resource manager limit configuration. resource manager System.Conns (%d) must be bigger than ConnMgr.HighWater (%d) See: https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md#how-does-the-resource-manager-resourcemgr-relate-to-the-connection-manager-connmgr `, rcm.System.Conns, highWater) } if (rcm.System.ConnsInbound > rcmgr.DefaultLimit || rcm.System.ConnsInbound == rcmgr.BlockAllLimit) && int64(rcm.System.ConnsInbound) <= highWater { // nolint return fmt.Errorf(` Unable to initialize libp2p due to conflicting resource manager limit configuration. resource manager System.ConnsInbound (%d) must be bigger than ConnMgr.HighWater (%d) See: https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md#how-does-the-resource-manager-resourcemgr-relate-to-the-connection-manager-connmgr `, rcm.System.ConnsInbound, highWater) } if (rcm.System.Streams > rcmgr.DefaultLimit || rcm.System.Streams == rcmgr.BlockAllLimit) && int64(rcm.System.Streams) <= highWater { // nolint return fmt.Errorf(` Unable to initialize libp2p due to conflicting resource manager limit configuration. resource manager System.Streams (%d) must be bigger than ConnMgr.HighWater (%d) See: https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md#how-does-the-resource-manager-resourcemgr-relate-to-the-connection-manager-connmgr `, rcm.System.Streams, highWater) } if (rcm.System.StreamsInbound > rcmgr.DefaultLimit || rcm.System.StreamsInbound == rcmgr.BlockAllLimit) && int64(rcm.System.StreamsInbound) <= highWater { // nolint return fmt.Errorf(` Unable to initialize libp2p due to conflicting resource manager limit configuration. resource manager System.StreamsInbound (%d) must be bigger than ConnMgr.HighWater (%d) See: https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md#how-does-the-resource-manager-resourcemgr-relate-to-the-connection-manager-connmgr `, rcm.System.StreamsInbound, highWater) } return nil } ================================================ FILE: core/node/libp2p/rcmgr_defaults.go ================================================ package libp2p import ( "fmt" "github.com/dustin/go-humanize" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/node/libp2p/fd" "github.com/libp2p/go-libp2p" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/pbnjay/memory" ) var infiniteResourceLimits = rcmgr.InfiniteLimits.ToPartialLimitConfig().System // This file defines implicit limit defaults used when Swarm.ResourceMgr.Enabled // createDefaultLimitConfig creates LimitConfig to pass to libp2p's resource manager. // The defaults follow the documentation in docs/libp2p-resource-management.md. // Any changes in the logic here should be reflected there. func createDefaultLimitConfig(cfg config.SwarmConfig) (limitConfig rcmgr.ConcreteLimitConfig, logMessageForStartup string, err error) { maxMemoryDefault := uint64(memory.TotalMemory()) / 2 maxMemory := cfg.ResourceMgr.MaxMemory.WithDefault(maxMemoryDefault) maxMemoryMB := maxMemory / (1024 * 1024) maxFD := int(cfg.ResourceMgr.MaxFileDescriptors.WithDefault(int64(fd.GetNumFDs()) / 2)) // At least as of 2023-01-25, it's possible to open a connection that // doesn't ask for any memory usage with the libp2p Resource Manager/Accountant // (see https://github.com/libp2p/go-libp2p/issues/2010#issuecomment-1404280736). // As a result, we can't currently rely on Memory limits to full protect us. // Until https://github.com/libp2p/go-libp2p/issues/2010 is addressed, // we take a proxy now of restricting to 1 inbound connection per MB. // Note: this is more generous than go-libp2p's default autoscaled limits which do // 64 connections per 1GB // (see https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit_defaults.go#L357 ). systemConnsInbound := int(1 * maxMemoryMB) partialLimits := rcmgr.PartialLimitConfig{ System: rcmgr.ResourceLimits{ Memory: rcmgr.LimitVal64(maxMemory), FD: rcmgr.LimitVal(maxFD), Conns: rcmgr.Unlimited, ConnsInbound: rcmgr.LimitVal(systemConnsInbound), ConnsOutbound: rcmgr.Unlimited, Streams: rcmgr.Unlimited, StreamsOutbound: rcmgr.Unlimited, StreamsInbound: rcmgr.Unlimited, }, // Transient connections won't cause any memory to be accounted for by the resource manager/accountant. // Only established connections do. // As a result, we can't rely on System.Memory to protect us from a bunch of transient connection being opened. // We limit the same values as the System scope, but only allow the Transient scope to take 25% of what is allowed for the System scope. Transient: rcmgr.ResourceLimits{ Memory: rcmgr.LimitVal64(maxMemory / 4), FD: rcmgr.LimitVal(maxFD / 4), Conns: rcmgr.Unlimited, ConnsInbound: rcmgr.LimitVal(systemConnsInbound / 4), ConnsOutbound: rcmgr.Unlimited, Streams: rcmgr.Unlimited, StreamsInbound: rcmgr.Unlimited, StreamsOutbound: rcmgr.Unlimited, }, // Lets get out of the way of the allow list functionality. // If someone specified "Swarm.ResourceMgr.Allowlist" we should let it go through. AllowlistedSystem: infiniteResourceLimits, AllowlistedTransient: infiniteResourceLimits, // Keep it simple by not having Service, ServicePeer, Protocol, ProtocolPeer, Conn, or Stream limits. ServiceDefault: infiniteResourceLimits, ServicePeerDefault: infiniteResourceLimits, ProtocolDefault: infiniteResourceLimits, ProtocolPeerDefault: infiniteResourceLimits, Conn: infiniteResourceLimits, Stream: infiniteResourceLimits, // Limit the resources consumed by a peer. // This doesn't protect us against intentional DoS attacks since an attacker can easily spin up multiple peers. // We specify this limit against unintentional DoS attacks (e.g., a peer has a bug and is sending too much traffic intentionally). // In that case we want to keep that peer's resource consumption contained. // To keep this simple, we only constrain inbound connections and streams. PeerDefault: rcmgr.ResourceLimits{ Memory: rcmgr.Unlimited64, FD: rcmgr.Unlimited, Conns: rcmgr.Unlimited, ConnsInbound: rcmgr.DefaultLimit, ConnsOutbound: rcmgr.Unlimited, Streams: rcmgr.Unlimited, StreamsInbound: rcmgr.DefaultLimit, StreamsOutbound: rcmgr.Unlimited, }, } scalingLimitConfig := rcmgr.DefaultLimits libp2p.SetDefaultServiceLimits(&scalingLimitConfig) // Anything set above in partialLimits that had a value of rcmgr.DefaultLimit will be overridden. // Anything in scalingLimitConfig that wasn't defined in partialLimits above will be added (e.g., libp2p's default service limits). partialLimits = partialLimits.Build(scalingLimitConfig.Scale(int64(maxMemory), maxFD)).ToPartialLimitConfig() // Simple checks to override autoscaling ensuring limits make sense versus the connmgr values. // There are ways to break this, but this should catch most problems already. // We might improve this in the future. // See: https://github.com/ipfs/kubo/issues/9545 if partialLimits.System.ConnsInbound > rcmgr.DefaultLimit && cfg.ConnMgr.Type.WithDefault(config.DefaultConnMgrType) != "none" { maxInboundConns := int64(partialLimits.System.ConnsInbound) if connmgrHighWaterTimesTwo := cfg.ConnMgr.HighWater.WithDefault(config.DefaultConnMgrHighWater) * 2; maxInboundConns < connmgrHighWaterTimesTwo { maxInboundConns = connmgrHighWaterTimesTwo } if maxInboundConns < config.DefaultResourceMgrMinInboundConns { maxInboundConns = config.DefaultResourceMgrMinInboundConns } // Scale System.StreamsInbound as well, but use the existing ratio of StreamsInbound to ConnsInbound if partialLimits.System.StreamsInbound > rcmgr.DefaultLimit { partialLimits.System.StreamsInbound = rcmgr.LimitVal(maxInboundConns * int64(partialLimits.System.StreamsInbound) / int64(partialLimits.System.ConnsInbound)) } partialLimits.System.ConnsInbound = rcmgr.LimitVal(maxInboundConns) } msg := fmt.Sprintf(` Computed default go-libp2p Resource Manager limits based on: - 'Swarm.ResourceMgr.MaxMemory': %q - 'Swarm.ResourceMgr.MaxFileDescriptors': %d These can be inspected with 'ipfs swarm resources'. `, humanize.Bytes(maxMemory), maxFD) // We already have a complete value thus pass in an empty ConcreteLimitConfig. return partialLimits.Build(rcmgr.ConcreteLimitConfig{}), msg, nil } ================================================ FILE: core/node/libp2p/rcmgr_logging.go ================================================ package libp2p import ( "context" "errors" "net" "sync" "time" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" ma "github.com/multiformats/go-multiaddr" "go.uber.org/zap" ) type loggingResourceManager struct { logger *zap.SugaredLogger delegate network.ResourceManager logInterval time.Duration mut sync.Mutex limitExceededErrs map[string]int } type loggingScope struct { logger *zap.SugaredLogger delegate network.ResourceScope countErrs func(error) } var ( _ network.ResourceManager = (*loggingResourceManager)(nil) _ rcmgr.ResourceManagerState = (*loggingResourceManager)(nil) ) func (n *loggingResourceManager) start(ctx context.Context) { logInterval := n.logInterval if logInterval == 0 { logInterval = 10 * time.Second } ticker := time.NewTicker(logInterval) go func() { defer ticker.Stop() for { select { case <-ticker.C: n.mut.Lock() errs := n.limitExceededErrs n.limitExceededErrs = make(map[string]int) for e, count := range errs { n.logger.Warnf("Protected from exceeding resource limits %d times. libp2p message: %q.", count, e) } if len(errs) != 0 { n.logger.Warnf("Learn more about potential actions to take at: https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md") } n.mut.Unlock() case <-ctx.Done(): return } } }() } func (n *loggingResourceManager) countErrs(err error) { if errors.Is(err, network.ErrResourceLimitExceeded) { n.mut.Lock() if n.limitExceededErrs == nil { n.limitExceededErrs = make(map[string]int) } // we need to unwrap the error to get the limit scope and the kind of reached limit eout := errors.Unwrap(err) if eout != nil { n.limitExceededErrs[eout.Error()]++ } n.mut.Unlock() } } func (n *loggingResourceManager) ViewSystem(f func(network.ResourceScope) error) error { return n.delegate.ViewSystem(f) } func (n *loggingResourceManager) ViewTransient(f func(network.ResourceScope) error) error { return n.delegate.ViewTransient(func(s network.ResourceScope) error { return f(&loggingScope{logger: n.logger, delegate: s, countErrs: n.countErrs}) }) } func (n *loggingResourceManager) ViewService(svc string, f func(network.ServiceScope) error) error { return n.delegate.ViewService(svc, func(s network.ServiceScope) error { return f(&loggingScope{logger: n.logger, delegate: s, countErrs: n.countErrs}) }) } func (n *loggingResourceManager) ViewProtocol(p protocol.ID, f func(network.ProtocolScope) error) error { return n.delegate.ViewProtocol(p, func(s network.ProtocolScope) error { return f(&loggingScope{logger: n.logger, delegate: s, countErrs: n.countErrs}) }) } func (n *loggingResourceManager) ViewPeer(p peer.ID, f func(network.PeerScope) error) error { return n.delegate.ViewPeer(p, func(s network.PeerScope) error { return f(&loggingScope{logger: n.logger, delegate: s, countErrs: n.countErrs}) }) } func (n *loggingResourceManager) OpenConnection(dir network.Direction, usefd bool, remote ma.Multiaddr) (network.ConnManagementScope, error) { connMgmtScope, err := n.delegate.OpenConnection(dir, usefd, remote) n.countErrs(err) return connMgmtScope, err } func (n *loggingResourceManager) OpenStream(p peer.ID, dir network.Direction) (network.StreamManagementScope, error) { connMgmtScope, err := n.delegate.OpenStream(p, dir) n.countErrs(err) return connMgmtScope, err } func (n *loggingResourceManager) Close() error { return n.delegate.Close() } func (n *loggingResourceManager) ListServices() []string { rapi, ok := n.delegate.(rcmgr.ResourceManagerState) if !ok { return nil } return rapi.ListServices() } func (n *loggingResourceManager) ListProtocols() []protocol.ID { rapi, ok := n.delegate.(rcmgr.ResourceManagerState) if !ok { return nil } return rapi.ListProtocols() } func (n *loggingResourceManager) ListPeers() []peer.ID { rapi, ok := n.delegate.(rcmgr.ResourceManagerState) if !ok { return nil } return rapi.ListPeers() } func (n *loggingResourceManager) Stat() rcmgr.ResourceManagerStat { rapi, ok := n.delegate.(rcmgr.ResourceManagerState) if !ok { return rcmgr.ResourceManagerStat{} } return rapi.Stat() } func (n *loggingResourceManager) VerifySourceAddress(addr net.Addr) bool { return n.delegate.VerifySourceAddress(addr) } func (s *loggingScope) ReserveMemory(size int, prio uint8) error { err := s.delegate.ReserveMemory(size, prio) s.countErrs(err) return err } func (s *loggingScope) ReleaseMemory(size int) { s.delegate.ReleaseMemory(size) } func (s *loggingScope) Stat() network.ScopeStat { return s.delegate.Stat() } func (s *loggingScope) BeginSpan() (network.ResourceScopeSpan, error) { return s.delegate.BeginSpan() } func (s *loggingScope) Done() { s.delegate.(network.ResourceScopeSpan).Done() } func (s *loggingScope) Name() string { return s.delegate.(network.ServiceScope).Name() } func (s *loggingScope) Protocol() protocol.ID { return s.delegate.(network.ProtocolScope).Protocol() } func (s *loggingScope) Peer() peer.ID { return s.delegate.(network.PeerScope).Peer() } func (s *loggingScope) PeerScope() network.PeerScope { return s.delegate.(network.PeerScope) } func (s *loggingScope) SetPeer(p peer.ID) error { err := s.delegate.(network.ConnManagementScope).SetPeer(p) s.countErrs(err) return err } func (s *loggingScope) ProtocolScope() network.ProtocolScope { return s.delegate.(network.ProtocolScope) } func (s *loggingScope) SetProtocol(proto protocol.ID) error { err := s.delegate.(network.StreamManagementScope).SetProtocol(proto) s.countErrs(err) return err } func (s *loggingScope) ServiceScope() network.ServiceScope { return s.delegate.(network.ServiceScope) } func (s *loggingScope) SetService(srv string) error { err := s.delegate.(network.StreamManagementScope).SetService(srv) s.countErrs(err) return err } func (s *loggingScope) Limit() rcmgr.Limit { return s.delegate.(rcmgr.ResourceScopeLimiter).Limit() } func (s *loggingScope) SetLimit(limit rcmgr.Limit) { s.delegate.(rcmgr.ResourceScopeLimiter).SetLimit(limit) } ================================================ FILE: core/node/libp2p/rcmgr_logging_test.go ================================================ package libp2p import ( "testing" "testing/synctest" "time" "github.com/libp2p/go-libp2p/core/network" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest/observer" ) func TestLoggingResourceManager(t *testing.T) { synctest.Test(t, func(t *testing.T) { orig := rcmgr.DefaultLimits.AutoScale() limits := orig.ToPartialLimitConfig() limits.System.Conns = 1 limits.System.ConnsInbound = 1 limits.System.ConnsOutbound = 1 limiter := rcmgr.NewFixedLimiter(limits.Build(orig)) rm, err := rcmgr.NewResourceManager(limiter) if err != nil { t.Fatal(err) } defer rm.Close() oCore, oLogs := observer.New(zap.WarnLevel) oLogger := zap.New(oCore) lrm := &loggingResourceManager{ logger: oLogger.Sugar(), delegate: rm, logInterval: 1 * time.Second, } // 2 of these should result in resource limit exceeded errors and subsequent log messages for range 3 { _, _ = lrm.OpenConnection(network.DirInbound, false, ma.StringCast("/ip4/127.0.0.1/tcp/1234")) } // run the logger which will write an entry for those errors ctx := t.Context() lrm.start(ctx) time.Sleep(3 * time.Second) timer := time.NewTimer(1 * time.Second) for { select { case <-timer.C: t.Fatalf("expected logs never arrived") default: if oLogs.Len() == 0 { continue } require.Equal(t, "Protected from exceeding resource limits 2 times. libp2p message: \"system: cannot reserve inbound connection: resource limit exceeded\".", oLogs.All()[0].Message) return } } }) } ================================================ FILE: core/node/libp2p/relay.go ================================================ package libp2p import ( "context" "github.com/ipfs/kubo/config" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/p2p/host/autorelay" "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay" "go.uber.org/fx" ) func RelayTransport(enableRelay bool) func() (opts Libp2pOpts, err error) { return func() (opts Libp2pOpts, err error) { if enableRelay { opts.Opts = append(opts.Opts, libp2p.EnableRelay()) } else { opts.Opts = append(opts.Opts, libp2p.DisableRelay()) } return } } func RelayService(enable bool, relayOpts config.RelayService) func() (opts Libp2pOpts, err error) { return func() (opts Libp2pOpts, err error) { if enable { def := relay.DefaultResources() // Real defaults live in go-libp2p. // Here we apply any overrides from user config. opts.Opts = append(opts.Opts, libp2p.EnableRelayService(relay.WithResources(relay.Resources{ Limit: &relay.RelayLimit{ Data: relayOpts.ConnectionDataLimit.WithDefault(def.Limit.Data), Duration: relayOpts.ConnectionDurationLimit.WithDefault(def.Limit.Duration), }, MaxCircuits: int(relayOpts.MaxCircuits.WithDefault(int64(def.MaxCircuits))), BufferSize: int(relayOpts.BufferSize.WithDefault(int64(def.BufferSize))), ReservationTTL: relayOpts.ReservationTTL.WithDefault(def.ReservationTTL), MaxReservations: int(relayOpts.MaxReservations.WithDefault(int64(def.MaxReservations))), MaxReservationsPerIP: int(relayOpts.MaxReservationsPerIP.WithDefault(int64(def.MaxReservationsPerIP))), MaxReservationsPerASN: int(relayOpts.MaxReservationsPerASN.WithDefault(int64(def.MaxReservationsPerASN))), }))) } return } } func MaybeAutoRelay(staticRelays []string, cfgPeering config.Peering, enabled bool) fx.Option { if !enabled { return fx.Options() } if len(staticRelays) > 0 { return fx.Provide(func() (opts Libp2pOpts, err error) { if len(staticRelays) > 0 { static := make([]peer.AddrInfo, 0, len(staticRelays)) for _, s := range staticRelays { var addr *peer.AddrInfo addr, err = peer.AddrInfoFromString(s) if err != nil { return } static = append(static, *addr) } opts.Opts = append(opts.Opts, libp2p.EnableAutoRelayWithStaticRelays(static)) } return }) } peerChan := make(chan peer.AddrInfo) return fx.Options( // Provide AutoRelay option fx.Provide(func() (opts Libp2pOpts, err error) { opts.Opts = append(opts.Opts, libp2p.EnableAutoRelayWithPeerSource( func(ctx context.Context, numPeers int) <-chan peer.AddrInfo { // TODO(9257): make this code smarter (have a state and actually try to grow the search outward) instead of a long running task just polling our K cluster. r := make(chan peer.AddrInfo) go func() { defer close(r) for ; numPeers != 0; numPeers-- { select { case v, ok := <-peerChan: if !ok { return } select { case r <- v: case <-ctx.Done(): return } case <-ctx.Done(): return } } }() return r }, autorelay.WithMinInterval(0), )) return }), autoRelayFeeder(cfgPeering, peerChan), ) } func HolePunching(flag config.Flag, hasRelayClient bool) func() (opts Libp2pOpts, err error) { return func() (opts Libp2pOpts, err error) { if flag.WithDefault(true) { if !hasRelayClient { // If hole punching is explicitly enabled but the relay client is disabled then panic, // otherwise just silently disable hole punching if flag != config.Default { log.Fatal("Failed to enable `Swarm.EnableHolePunching`, it requires `Swarm.RelayClient.Enabled` to be true.") } else { log.Info("HolePunching has been disabled due to the RelayClient being disabled.") } return } opts.Opts = append(opts.Opts, libp2p.EnableHolePunching()) } return } } ================================================ FILE: core/node/libp2p/routing.go ================================================ package libp2p import ( "context" "fmt" "runtime/debug" "sort" "time" "github.com/cenkalti/backoff/v4" offroute "github.com/ipfs/boxo/routing/offline" ds "github.com/ipfs/go-datastore" dht "github.com/libp2p/go-libp2p-kad-dht" ddht "github.com/libp2p/go-libp2p-kad-dht/dual" "github.com/libp2p/go-libp2p-kad-dht/fullrt" pubsub "github.com/libp2p/go-libp2p-pubsub" namesys "github.com/libp2p/go-libp2p-pubsub-router" record "github.com/libp2p/go-libp2p-record" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "go.uber.org/fx" config "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/node/helpers" "github.com/ipfs/kubo/repo" irouting "github.com/ipfs/kubo/routing" ) type Router struct { routing.Routing Priority int // less = more important } type p2pRouterOut struct { fx.Out Router Router `group:"routers"` } type processInitialRoutingIn struct { fx.In Router routing.Routing `name:"initialrouting"` // For setting up experimental DHT client Host host.Host Repo repo.Repo Validator record.Validator } type processInitialRoutingOut struct { fx.Out Router Router `group:"routers"` ContentRouter routing.ContentRouting `group:"content-routers"` DHT *ddht.DHT DHTClient routing.Routing `name:"dhtc"` } type AddrInfoChan chan peer.AddrInfo func BaseRouting(cfg *config.Config) any { return func(lc fx.Lifecycle, in processInitialRoutingIn) (out processInitialRoutingOut, err error) { var dualDHT *ddht.DHT if dht, ok := in.Router.(*ddht.DHT); ok { dualDHT = dht lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return dualDHT.Close() }, }) } if cr, ok := in.Router.(routinghelpers.ComposableRouter); ok { for _, r := range cr.Routers() { if dht, ok := r.(*ddht.DHT); ok { dualDHT = dht lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return dualDHT.Close() }, }) break } } } if dualDHT != nil && cfg.Routing.AcceleratedDHTClient.WithDefault(config.DefaultAcceleratedDHTClient) { cfg, err := in.Repo.Config() if err != nil { return out, err } // Use auto-config resolution for actual connectivity bspeers, err := cfg.BootstrapPeersWithAutoConf() if err != nil { return out, err } fullRTClient, err := fullrt.NewFullRT(in.Host, dht.DefaultPrefix, fullrt.DHTOption( dht.Validator(in.Validator), dht.Datastore(in.Repo.Datastore()), dht.BootstrapPeers(bspeers...), dht.BucketSize(20), ), ) if err != nil { return out, err } lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return fullRTClient.Close() }, }) // we want to also use the default HTTP routers, so wrap the FullRT client // in a parallel router that calls them in parallel addrFunc := httpRouterAddrFunc(in.Host, cfg.Addresses) httpRouters, err := constructDefaultHTTPRouters(cfg, addrFunc) if err != nil { return out, err } routers := []*routinghelpers.ParallelRouter{ {Router: fullRTClient, DoNotWaitForSearchValue: true}, } routers = append(routers, httpRouters...) router := routinghelpers.NewComposableParallel(routers) return processInitialRoutingOut{ Router: Router{ Priority: 1000, Routing: router, }, DHT: dualDHT, DHTClient: fullRTClient, ContentRouter: fullRTClient, }, nil } return processInitialRoutingOut{ Router: Router{ Priority: 1000, Routing: in.Router, }, DHT: dualDHT, DHTClient: dualDHT, ContentRouter: in.Router, }, nil } } type p2pOnlineContentRoutingIn struct { fx.In ContentRouter []routing.ContentRouting `group:"content-routers"` } // ContentRouting will get all routers that can do contentRouting and add them // all together using a TieredRouter. It will be used for topic discovery. func ContentRouting(in p2pOnlineContentRoutingIn) routing.ContentRouting { var routers []routing.Routing for _, cr := range in.ContentRouter { routers = append(routers, &routinghelpers.Compose{ ContentRouting: cr, }, ) } return routinghelpers.Tiered{ Routers: routers, } } // ContentDiscovery narrows down the given content routing facility so that it // only does discovery. func ContentDiscovery(in irouting.ProvideManyRouter) routing.ContentDiscovery { return in } type p2pOnlineRoutingIn struct { fx.In Routers []Router `group:"routers"` Validator record.Validator } // Routing will get all routers obtained from different methods (delegated // routers, pub-sub, and so on) and add them all together using a ParallelRouter. func Routing(in p2pOnlineRoutingIn) irouting.ProvideManyRouter { routers := in.Routers sort.SliceStable(routers, func(i, j int) bool { return routers[i].Priority < routers[j].Priority }) var cRouters []*routinghelpers.ParallelRouter for _, v := range routers { cRouters = append(cRouters, &routinghelpers.ParallelRouter{ IgnoreError: true, DoNotWaitForSearchValue: true, Router: v.Routing, }) } return routinghelpers.NewComposableParallel(cRouters) } // OfflineRouting provides a special Router to the routers list when we are // creating an offline node. func OfflineRouting(dstore ds.Datastore, validator record.Validator) p2pRouterOut { return p2pRouterOut{ Router: Router{ Routing: offroute.NewOfflineRouter(dstore, validator), Priority: 10000, }, } } type p2pPSRoutingIn struct { fx.In Validator record.Validator Host host.Host PubSub *pubsub.PubSub `optional:"true"` } func PubsubRouter(mctx helpers.MetricsCtx, lc fx.Lifecycle, in p2pPSRoutingIn) (p2pRouterOut, *namesys.PubsubValueStore, error) { psRouter, err := namesys.NewPubsubValueStore( helpers.LifecycleCtx(mctx, lc), in.Host, in.PubSub, in.Validator, namesys.WithRebroadcastInterval(time.Minute), ) if err != nil { return p2pRouterOut{}, nil, err } return p2pRouterOut{ Router: Router{ Routing: &routinghelpers.Compose{ ValueStore: &routinghelpers.LimitedValueStore{ ValueStore: psRouter, Namespaces: []string{"ipns"}, }, }, Priority: 100, }, }, psRouter, nil } func autoRelayFeeder(cfgPeering config.Peering, peerChan chan<- peer.AddrInfo) fx.Option { return fx.Invoke(func(lc fx.Lifecycle, h host.Host, dht *ddht.DHT) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) defer func() { if r := recover(); r != nil { fmt.Println("Recovering from unexpected error in AutoRelayFeeder:", r) debug.PrintStack() } }() go func() { defer close(done) // Feed peers more often right after the bootstrap, then backoff bo := backoff.NewExponentialBackOff() bo.InitialInterval = 15 * time.Second bo.Multiplier = 3 bo.MaxInterval = 1 * time.Hour bo.MaxElapsedTime = 0 // never stop t := backoff.NewTicker(bo) defer t.Stop() for { select { case <-t.C: case <-ctx.Done(): return } // Always feed trusted IDs (Peering.Peers in the config) for _, trustedPeer := range cfgPeering.Peers { if len(trustedPeer.Addrs) == 0 { continue } select { case peerChan <- trustedPeer: case <-ctx.Done(): return } } // Additionally, feed closest peers discovered via DHT if dht != nil { closestPeers, err := dht.WAN.GetClosestPeers(ctx, h.ID().String()) if err == nil { for _, p := range closestPeers { addrs := h.Peerstore().Addrs(p) if len(addrs) == 0 { continue } dhtPeer := peer.AddrInfo{ID: p, Addrs: addrs} select { case peerChan <- dhtPeer: case <-ctx.Done(): return } } } } // Additionally, feed all connected swarm peers as potential relay candidates. // This includes peers from HTTP routing, manual swarm connect, mDNS discovery, etc. // (fixes https://github.com/ipfs/kubo/issues/10899) connectedPeers := h.Network().Peers() for _, p := range connectedPeers { addrs := h.Peerstore().Addrs(p) if len(addrs) == 0 { continue } swarmPeer := peer.AddrInfo{ID: p, Addrs: addrs} select { case peerChan <- swarmPeer: case <-ctx.Done(): return } } } }() lc.Append(fx.Hook{ OnStop: func(_ context.Context) error { cancel() <-done return nil }, }) }) } ================================================ FILE: core/node/libp2p/routingopt.go ================================================ package libp2p import ( "context" "fmt" "os" "slices" "strings" "time" "github.com/ipfs/boxo/autoconf" "github.com/ipfs/go-datastore" "github.com/ipfs/kubo/config" irouting "github.com/ipfs/kubo/routing" dht "github.com/libp2p/go-libp2p-kad-dht" dual "github.com/libp2p/go-libp2p-kad-dht/dual" record "github.com/libp2p/go-libp2p-record" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" host "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" routing "github.com/libp2p/go-libp2p/core/routing" basichost "github.com/libp2p/go-libp2p/p2p/host/basic" ma "github.com/multiformats/go-multiaddr" ) type RoutingOptionArgs struct { Ctx context.Context Host host.Host Datastore datastore.Batching Validator record.Validator BootstrapPeers []peer.AddrInfo OptimisticProvide bool OptimisticProvideJobsPoolSize int LoopbackAddressesOnLanDHT bool } type RoutingOption func(args RoutingOptionArgs) (routing.Routing, error) var noopRouter = routinghelpers.Null{} // EndpointSource tracks where a URL came from to determine appropriate capabilities type EndpointSource struct { URL string SupportsRead bool // came from DelegatedRoutersWithAutoConf (Read operations) SupportsWrite bool // came from DelegatedPublishersWithAutoConf (Write operations) } // determineCapabilities determines endpoint capabilities based on URL path and source func determineCapabilities(endpoint EndpointSource) (string, autoconf.EndpointCapabilities, error) { parsed, err := autoconf.DetermineKnownCapabilities(endpoint.URL, endpoint.SupportsRead, endpoint.SupportsWrite) if err != nil { log.Debugf("Skipping endpoint %q: %v", endpoint.URL, err) return "", autoconf.EndpointCapabilities{}, nil // Return empty caps, not error } return parsed.BaseURL, parsed.Capabilities, nil } // collectAllEndpoints gathers URLs from both router and publisher sources func collectAllEndpoints(cfg *config.Config) []EndpointSource { var endpoints []EndpointSource // Get router URLs (Read operations) var routerURLs []string if envRouters := os.Getenv(config.EnvHTTPRouters); envRouters != "" { // Use environment variable override if set (space or comma separated) splitFunc := func(r rune) bool { return r == ',' || r == ' ' } routerURLs = strings.FieldsFunc(envRouters, splitFunc) log.Warnf("Using HTTP routers from %s environment variable instead of config/autoconf: %v", config.EnvHTTPRouters, routerURLs) } else { // Use delegated routers from autoconf routerURLs = cfg.DelegatedRoutersWithAutoConf() // No fallback - if autoconf doesn't provide endpoints, use empty list // This exposes any autoconf issues rather than masking them with hardcoded defaults } // Add router URLs to collection for _, url := range routerURLs { endpoints = append(endpoints, EndpointSource{ URL: url, SupportsRead: true, SupportsWrite: false, }) } // Get publisher URLs (Write operations) publisherURLs := cfg.DelegatedPublishersWithAutoConf() // Add publisher URLs, merging with existing router URLs if they match for _, url := range publisherURLs { found := false for i, existing := range endpoints { if existing.URL == url { endpoints[i].SupportsWrite = true found = true break } } if !found { endpoints = append(endpoints, EndpointSource{ URL: url, SupportsRead: false, SupportsWrite: true, }) } } return endpoints } func constructDefaultHTTPRouters(cfg *config.Config, addrFunc func() []ma.Multiaddr) ([]*routinghelpers.ParallelRouter, error) { var routers []*routinghelpers.ParallelRouter httpRetrievalEnabled := cfg.HTTPRetrieval.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled) // Collect URLs from both router and publisher sources endpoints := collectAllEndpoints(cfg) // Group endpoints by origin (base URL) and aggregate capabilities originCapabilities := make(map[string]autoconf.EndpointCapabilities) for _, endpoint := range endpoints { // Parse endpoint and determine capabilities based on source baseURL, capabilities, err := determineCapabilities(endpoint) if err != nil { return nil, fmt.Errorf("failed to parse endpoint %q: %w", endpoint.URL, err) } // Aggregate capabilities for this origin existing := originCapabilities[baseURL] existing.Merge(capabilities) originCapabilities[baseURL] = existing } // Create single HTTP router and composer per origin for baseURL, capabilities := range originCapabilities { // Construct HTTP router using base URL (without path) httpRouter, err := irouting.ConstructHTTPRouter(baseURL, cfg.Identity.PeerID, addrFunc, cfg.Identity.PrivKey, httpRetrievalEnabled) if err != nil { return nil, err } // Configure router operations based on aggregated capabilities // https://specs.ipfs.tech/routing/http-routing-v1/ composer := &irouting.Composer{ GetValueRouter: noopRouter, // Default disabled, enabled below based on capabilities PutValueRouter: noopRouter, // Default disabled, enabled below based on capabilities ProvideRouter: noopRouter, // we don't have spec for sending provides to /routing/v1 (revisit once https://github.com/ipfs/specs/pull/378 or similar is ratified) FindPeersRouter: noopRouter, // Default disabled, enabled below based on capabilities FindProvidersRouter: noopRouter, // Default disabled, enabled below based on capabilities } // Enable specific capabilities if capabilities.IPNSGet { composer.GetValueRouter = httpRouter // GET /routing/v1/ipns for IPNS resolution } if capabilities.IPNSPut { composer.PutValueRouter = httpRouter // PUT /routing/v1/ipns for IPNS publishing } if capabilities.Peers { composer.FindPeersRouter = httpRouter // GET /routing/v1/peers } if capabilities.Providers { composer.FindProvidersRouter = httpRouter // GET /routing/v1/providers } // Handle special cases and backward compatibility if baseURL == config.CidContactRoutingURL { // Special-case: cid.contact only supports /routing/v1/providers/cid endpoint // Override any capabilities detected from URL path to ensure only providers is enabled // TODO: Consider moving this to configuration or removing once cid.contact adds more capabilities composer.GetValueRouter = noopRouter composer.PutValueRouter = noopRouter composer.ProvideRouter = noopRouter composer.FindPeersRouter = noopRouter composer.FindProvidersRouter = httpRouter // Only providers supported } routers = append(routers, &routinghelpers.ParallelRouter{ Router: composer, IgnoreError: true, // https://github.com/ipfs/kubo/pull/9475#discussion_r1042507387 Timeout: 15 * time.Second, // 5x server value from https://github.com/ipfs/kubo/pull/9475#discussion_r1042428529 DoNotWaitForSearchValue: true, ExecuteAfter: 0, }) } return routers, nil } // ConstructDelegatedOnlyRouting returns routers used when Routing.Type is set to "delegated" // This provides HTTP-only routing without DHT, using only delegated routers and IPNS publishers. // Useful for environments where DHT connectivity is not available or desired func ConstructDelegatedOnlyRouting(cfg *config.Config) RoutingOption { return func(args RoutingOptionArgs) (routing.Routing, error) { // Use only HTTP routers (includes both read and write capabilities) - no DHT var routers []*routinghelpers.ParallelRouter // Add HTTP delegated routers (includes both router and publisher capabilities) addrFunc := httpRouterAddrFunc(args.Host, cfg.Addresses) httpRouters, err := constructDefaultHTTPRouters(cfg, addrFunc) if err != nil { return nil, err } routers = append(routers, httpRouters...) // Validate that we have at least one router configured if len(routers) == 0 { return nil, fmt.Errorf("no delegated routers or publishers configured for 'delegated' routing mode") } routing := routinghelpers.NewComposableParallel(routers) return routing, nil } } // ConstructDefaultRouting returns routers used when Routing.Type is unset or set to "auto" func ConstructDefaultRouting(cfg *config.Config, routingOpt RoutingOption) RoutingOption { return func(args RoutingOptionArgs) (routing.Routing, error) { // Defined routers will be queried in parallel (optimizing for response speed) // Different trade-offs can be made by setting Routing.Type = "custom" with own Routing.Routers var routers []*routinghelpers.ParallelRouter dhtRouting, err := routingOpt(args) if err != nil { return nil, err } routers = append(routers, &routinghelpers.ParallelRouter{ Router: dhtRouting, IgnoreError: false, DoNotWaitForSearchValue: true, ExecuteAfter: 0, }) addrFunc := httpRouterAddrFunc(args.Host, cfg.Addresses) httpRouters, err := constructDefaultHTTPRouters(cfg, addrFunc) if err != nil { return nil, err } routers = append(routers, httpRouters...) routing := routinghelpers.NewComposableParallel(routers) return routing, nil } } // constructDHTRouting is used when Routing.Type = "dht" func constructDHTRouting(mode dht.ModeOpt) RoutingOption { return func(args RoutingOptionArgs) (routing.Routing, error) { dhtOpts := []dht.Option{ dht.Concurrency(10), dht.Mode(mode), dht.Datastore(args.Datastore), dht.Validator(args.Validator), } if args.OptimisticProvide { dhtOpts = append(dhtOpts, dht.EnableOptimisticProvide()) } if args.OptimisticProvideJobsPoolSize != 0 { dhtOpts = append(dhtOpts, dht.OptimisticProvideJobsPoolSize(args.OptimisticProvideJobsPoolSize)) } wanOptions := []dht.Option{ dht.BootstrapPeers(args.BootstrapPeers...), } lanOptions := []dht.Option{} if args.LoopbackAddressesOnLanDHT { lanOptions = append(lanOptions, dht.AddressFilter(nil)) } return dual.New( args.Ctx, args.Host, dual.DHTOption(dhtOpts...), dual.WanDHTOption(wanOptions...), dual.LanDHTOption(lanOptions...), ) } } // ConstructDelegatedRouting is used when Routing.Type = "custom" func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs config.Addresses, privKey string, httpRetrieval bool) RoutingOption { return func(args RoutingOptionArgs) (routing.Routing, error) { addrFunc := httpRouterAddrFunc(args.Host, addrs) return irouting.Parse(routers, methods, &irouting.ExtraDHTParams{ BootstrapPeers: args.BootstrapPeers, Host: args.Host, Validator: args.Validator, Datastore: args.Datastore, Context: args.Ctx, }, &irouting.ExtraHTTPParams{ PeerID: peerID, AddrFunc: addrFunc, PrivKeyB64: privKey, HTTPRetrieval: httpRetrieval, }, ) } } func constructNilRouting(_ RoutingOptionArgs) (routing.Routing, error) { return routinghelpers.Null{}, nil } var ( DHTOption RoutingOption = constructDHTRouting(dht.ModeAuto) DHTClientOption = constructDHTRouting(dht.ModeClient) DHTServerOption = constructDHTRouting(dht.ModeServer) NilRouterOption = constructNilRouting ) // confirmedAddrsHost matches libp2p hosts that support AutoNAT V2 address confirmation. type confirmedAddrsHost interface { ConfirmedAddrs() (reachable, unreachable, unknown []ma.Multiaddr) } // Compile-time check: BasicHost must satisfy confirmedAddrsHost. // ConfirmedAddrs is not part of the core host.Host interface and is marked // experimental in go-libp2p. If BasicHost ever drops or changes this method, // this assertion will fail at build time. In that case, update // httpRouterAddrFunc (this file) and the swarm autonat command // (core/commands/swarm_addrs_autonat.go) which both type-assert to this // interface. var _ confirmedAddrsHost = (*basichost.BasicHost)(nil) // httpRouterAddrFunc returns a function that resolves provider addresses for // HTTP routers at provide-time. // // Resolution logic: // - If Announce is set, use it as a static override (no dynamic resolution). // - Otherwise, prefer AutoNAT V2 confirmed reachable addresses when available, // falling back to static Swarm addresses (filtered by NoAnnounce). // - AppendAnnounce addresses are always appended. func httpRouterAddrFunc(h host.Host, cfgAddrs config.Addresses) func() []ma.Multiaddr { appendAddrs := parseMultiaddrs(cfgAddrs.AppendAnnounce) // If Announce is explicitly set, use it as a static override. if len(cfgAddrs.Announce) > 0 { staticAddrs := slices.Concat(parseMultiaddrs(cfgAddrs.Announce), appendAddrs) return func() []ma.Multiaddr { return staticAddrs } } // Precompute fallback: Swarm minus NoAnnounce plus AppendAnnounce. fallbackStrs := cfgAddrs.Swarm if len(cfgAddrs.NoAnnounce) > 0 { noAnnounce := map[string]struct{}{} for _, a := range cfgAddrs.NoAnnounce { noAnnounce[a] = struct{}{} } filtered := make([]string, 0, len(fallbackStrs)) for _, a := range fallbackStrs { if _, skip := noAnnounce[a]; !skip { filtered = append(filtered, a) } } fallbackStrs = filtered } fallbackResult := slices.Concat(parseMultiaddrs(fallbackStrs), appendAddrs) ch, hasConfirmed := h.(confirmedAddrsHost) return func() []ma.Multiaddr { if hasConfirmed { reachable, _, _ := ch.ConfirmedAddrs() if len(reachable) > 0 { if len(appendAddrs) == 0 { return reachable } return slices.Concat(reachable, appendAddrs) } } return fallbackResult } } func parseMultiaddrs(strs []string) []ma.Multiaddr { addrs := make([]ma.Multiaddr, 0, len(strs)) for _, s := range strs { a, err := ma.NewMultiaddr(s) if err != nil { log.Errorf("ignoring invalid multiaddr %q: %s", s, err) continue } addrs = append(addrs, a) } return addrs } ================================================ FILE: core/node/libp2p/routingopt_test.go ================================================ package libp2p import ( "context" "testing" "github.com/ipfs/boxo/autoconf" config "github.com/ipfs/kubo/config" "github.com/libp2p/go-libp2p/core/connmgr" "github.com/libp2p/go-libp2p/core/event" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/peerstore" "github.com/libp2p/go-libp2p/core/protocol" ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestDetermineCapabilities(t *testing.T) { tests := []struct { name string endpoint EndpointSource expectedBaseURL string expectedCapabilities autoconf.EndpointCapabilities expectError bool }{ { name: "URL with no path should have all Read capabilities", endpoint: EndpointSource{ URL: "https://example.com", SupportsRead: true, SupportsWrite: false, }, expectedBaseURL: "https://example.com", expectedCapabilities: autoconf.EndpointCapabilities{ Providers: true, Peers: true, IPNSGet: true, IPNSPut: false, }, expectError: false, }, { name: "URL with trailing slash should have all Read capabilities", endpoint: EndpointSource{ URL: "https://example.com/", SupportsRead: true, SupportsWrite: false, }, expectedBaseURL: "https://example.com", expectedCapabilities: autoconf.EndpointCapabilities{ Providers: true, Peers: true, IPNSGet: true, IPNSPut: false, }, expectError: false, }, { name: "URL with IPNS path should have only IPNS capabilities", endpoint: EndpointSource{ URL: "https://example.com/routing/v1/ipns", SupportsRead: true, SupportsWrite: true, }, expectedBaseURL: "https://example.com", expectedCapabilities: autoconf.EndpointCapabilities{ Providers: false, Peers: false, IPNSGet: true, IPNSPut: true, }, expectError: false, }, { name: "URL with providers path should have only Providers capability", endpoint: EndpointSource{ URL: "https://example.com/routing/v1/providers", SupportsRead: true, SupportsWrite: false, }, expectedBaseURL: "https://example.com", expectedCapabilities: autoconf.EndpointCapabilities{ Providers: true, Peers: false, IPNSGet: false, IPNSPut: false, }, expectError: false, }, { name: "URL with peers path should have only Peers capability", endpoint: EndpointSource{ URL: "https://example.com/routing/v1/peers", SupportsRead: true, SupportsWrite: false, }, expectedBaseURL: "https://example.com", expectedCapabilities: autoconf.EndpointCapabilities{ Providers: false, Peers: true, IPNSGet: false, IPNSPut: false, }, expectError: false, }, { name: "URL with Write support only should enable IPNSPut for no-path endpoint", endpoint: EndpointSource{ URL: "https://example.com", SupportsRead: false, SupportsWrite: true, }, expectedBaseURL: "https://example.com", expectedCapabilities: autoconf.EndpointCapabilities{ Providers: false, Peers: false, IPNSGet: false, IPNSPut: true, }, expectError: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { baseURL, capabilities, err := determineCapabilities(tt.endpoint) if tt.expectError { assert.Error(t, err) return } require.NoError(t, err) assert.Equal(t, tt.expectedBaseURL, baseURL) assert.Equal(t, tt.expectedCapabilities, capabilities) }) } } func TestEndpointCapabilitiesReadWriteLogic(t *testing.T) { t.Run("Read endpoint with no path should enable read capabilities", func(t *testing.T) { endpoint := EndpointSource{ URL: "https://example.com", SupportsRead: true, SupportsWrite: false, } _, capabilities, err := determineCapabilities(endpoint) require.NoError(t, err) // Read endpoint with no path should enable all read capabilities assert.True(t, capabilities.Providers) assert.True(t, capabilities.Peers) assert.True(t, capabilities.IPNSGet) assert.False(t, capabilities.IPNSPut) // Write capability should be false }) t.Run("Write endpoint with no path should enable write capabilities", func(t *testing.T) { endpoint := EndpointSource{ URL: "https://example.com", SupportsRead: false, SupportsWrite: true, } _, capabilities, err := determineCapabilities(endpoint) require.NoError(t, err) // Write endpoint with no path should only enable IPNS write capability assert.False(t, capabilities.Providers) assert.False(t, capabilities.Peers) assert.False(t, capabilities.IPNSGet) assert.True(t, capabilities.IPNSPut) // Only write capability should be true }) t.Run("Specific path should only enable matching capabilities", func(t *testing.T) { endpoint := EndpointSource{ URL: "https://example.com/routing/v1/ipns", SupportsRead: true, SupportsWrite: true, } _, capabilities, err := determineCapabilities(endpoint) require.NoError(t, err) // Specific IPNS path should only enable IPNS capabilities based on source assert.False(t, capabilities.Providers) assert.False(t, capabilities.Peers) assert.True(t, capabilities.IPNSGet) // Read capability enabled assert.True(t, capabilities.IPNSPut) // Write capability enabled }) t.Run("Unsupported paths should result in empty capabilities", func(t *testing.T) { endpoint := EndpointSource{ URL: "https://example.com/routing/v1/unsupported", SupportsRead: true, SupportsWrite: false, } _, capabilities, err := determineCapabilities(endpoint) require.NoError(t, err) // Unsupported paths should result in no capabilities assert.False(t, capabilities.Providers) assert.False(t, capabilities.Peers) assert.False(t, capabilities.IPNSGet) assert.False(t, capabilities.IPNSPut) }) } // stubHost is a minimal host.Host stub for testing httpRouterAddrFunc. // Only the methods checked via type assertion (confirmedAddrsHost) matter; // all other methods panic if called. type stubHost struct { reachable []ma.Multiaddr } func (h *stubHost) ConfirmedAddrs() (reachable, unreachable, unknown []ma.Multiaddr) { return h.reachable, nil, nil } func (h *stubHost) ID() peer.ID { panic("unused") } func (h *stubHost) Addrs() []ma.Multiaddr { panic("unused") } func (h *stubHost) Peerstore() peerstore.Peerstore { panic("unused") } func (h *stubHost) Network() network.Network { panic("unused") } func (h *stubHost) Mux() protocol.Switch { panic("unused") } func (h *stubHost) Connect(context.Context, peer.AddrInfo) error { panic("unused") } func (h *stubHost) SetStreamHandler(protocol.ID, network.StreamHandler) { panic("unused") } func (h *stubHost) SetStreamHandlerMatch(protocol.ID, func(protocol.ID) bool, network.StreamHandler) { panic("unused") } func (h *stubHost) RemoveStreamHandler(protocol.ID) { panic("unused") } func (h *stubHost) NewStream(context.Context, peer.ID, ...protocol.ID) (network.Stream, error) { panic("unused") } func (h *stubHost) Close() error { panic("unused") } func (h *stubHost) ConnManager() connmgr.ConnManager { panic("unused") } func (h *stubHost) EventBus() event.Bus { panic("unused") } func TestHttpRouterAddrFunc(t *testing.T) { tests := []struct { name string reachable []string // autonat confirmed addrs (nil = none) cfg config.Addresses want []string }{ { name: "prefers autonat confirmed reachable addrs over swarm fallback", reachable: []string{"/ip4/1.2.3.4/tcp/4001", "/ip4/1.2.3.4/udp/4001/quic-v1"}, cfg: config.Addresses{Swarm: []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/udp/4001/quic-v1"}}, want: []string{"/ip4/1.2.3.4/tcp/4001", "/ip4/1.2.3.4/udp/4001/quic-v1"}, }, { name: "falls back to swarm when autonat has no confirmed addrs", cfg: config.Addresses{Swarm: []string{"/ip4/0.0.0.0/tcp/4001"}}, want: []string{"/ip4/0.0.0.0/tcp/4001"}, }, { name: "Announce overrides autonat and swarm", reachable: []string{"/ip4/1.2.3.4/tcp/4001"}, cfg: config.Addresses{Swarm: []string{"/ip4/0.0.0.0/tcp/4001"}, Announce: []string{"/ip4/5.6.7.8/tcp/4001"}}, want: []string{"/ip4/5.6.7.8/tcp/4001"}, }, { name: "AppendAnnounce added to autonat addrs", reachable: []string{"/ip4/1.2.3.4/tcp/4001"}, cfg: config.Addresses{Swarm: []string{"/ip4/0.0.0.0/tcp/4001"}, AppendAnnounce: []string{"/ip4/10.0.0.1/tcp/4001"}}, want: []string{"/ip4/1.2.3.4/tcp/4001", "/ip4/10.0.0.1/tcp/4001"}, }, { name: "AppendAnnounce added to swarm fallback", cfg: config.Addresses{Swarm: []string{"/ip4/0.0.0.0/tcp/4001"}, AppendAnnounce: []string{"/ip4/10.0.0.1/tcp/4001"}}, want: []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/10.0.0.1/tcp/4001"}, }, { name: "NoAnnounce filters swarm fallback", cfg: config.Addresses{Swarm: []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/udp/4001/quic-v1"}, NoAnnounce: []string{"/ip4/0.0.0.0/tcp/4001"}}, want: []string{"/ip4/0.0.0.0/udp/4001/quic-v1"}, }, { name: "AppendAnnounce added to Announce", cfg: config.Addresses{Swarm: []string{"/ip4/0.0.0.0/tcp/4001"}, Announce: []string{"/ip4/5.6.7.8/tcp/4001"}, AppendAnnounce: []string{"/ip4/10.0.0.1/tcp/4001"}}, want: []string{"/ip4/5.6.7.8/tcp/4001", "/ip4/10.0.0.1/tcp/4001"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { h := &stubHost{reachable: parseMultiaddrs(tt.reachable)} fn := httpRouterAddrFunc(h, tt.cfg) assert.Equal(t, parseMultiaddrs(tt.want), fn()) }) } } ================================================ FILE: core/node/libp2p/sec.go ================================================ package libp2p import ( "github.com/ipfs/kubo/config" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/p2p/security/noise" tls "github.com/libp2p/go-libp2p/p2p/security/tls" ) func Security(enabled bool, tptConfig config.Transports) any { if !enabled { return func() (opts Libp2pOpts) { log.Errorf(`Your IPFS node has been configured to run WITHOUT ENCRYPTED CONNECTIONS. You will not be able to connect to any nodes configured to use encrypted connections`) opts.Opts = append(opts.Opts, libp2p.NoSecurity) return opts } } // Using the new config options. return func() (opts Libp2pOpts) { opts.Opts = append(opts.Opts, prioritizeOptions([]priorityOption{{ priority: tptConfig.Security.TLS, defaultPriority: 100, opt: libp2p.Security(tls.ID, tls.New), }, { priority: tptConfig.Security.Noise, defaultPriority: 200, opt: libp2p.Security(noise.ID, noise.New), }})) return opts } } ================================================ FILE: core/node/libp2p/smux.go ================================================ package libp2p import ( "errors" "os" "github.com/ipfs/kubo/config" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/p2p/muxer/yamux" ) func makeSmuxTransportOption(tptConfig config.Transports) (libp2p.Option, error) { if prefs := os.Getenv("LIBP2P_MUX_PREFS"); prefs != "" { return nil, errors.New("configuring muxers with LIBP2P_MUX_PREFS is no longer supported, use Swarm.Transports.Multiplexers") } if tptConfig.Multiplexers.Yamux < 0 { return nil, errors.New("running libp2p with Swarm.Transports.Multiplexers.Yamux disabled is not supported") } return libp2p.Muxer(yamux.ID, yamux.DefaultTransport), nil } func SmuxTransport(tptConfig config.Transports) func() (opts Libp2pOpts, err error) { return func() (opts Libp2pOpts, err error) { res, err := makeSmuxTransportOption(tptConfig) if err != nil { return opts, err } opts.Opts = append(opts.Opts, res) return opts, nil } } ================================================ FILE: core/node/libp2p/topicdiscovery.go ================================================ package libp2p import ( "math/rand" "time" "github.com/libp2p/go-libp2p/core/discovery" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/p2p/discovery/backoff" disc "github.com/libp2p/go-libp2p/p2p/discovery/routing" "github.com/libp2p/go-libp2p/core/routing" ) func TopicDiscovery() any { return func(host host.Host, cr routing.ContentRouting) (service discovery.Discovery, err error) { baseDisc := disc.NewRoutingDiscovery(cr) minBackoff, maxBackoff := time.Second*60, time.Hour rng := rand.New(rand.NewSource(rand.Int63())) d, err := backoff.NewBackoffDiscovery( baseDisc, backoff.NewExponentialBackoff(minBackoff, maxBackoff, backoff.FullJitter, time.Second, 5.0, 0, rng), ) if err != nil { return nil, err } return d, nil } } ================================================ FILE: core/node/libp2p/transport.go ================================================ package libp2p import ( "fmt" "os" "github.com/ipfs/kubo/config" "github.com/ipshipyard/p2p-forge/client" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/metrics" quic "github.com/libp2p/go-libp2p/p2p/transport/quic" "github.com/libp2p/go-libp2p/p2p/transport/tcp" webrtc "github.com/libp2p/go-libp2p/p2p/transport/webrtc" "github.com/libp2p/go-libp2p/p2p/transport/websocket" webtransport "github.com/libp2p/go-libp2p/p2p/transport/webtransport" "go.uber.org/fx" ) func Transports(tptConfig config.Transports) any { return func(params struct { fx.In Fprint PNetFingerprint `optional:"true"` ForgeMgr *client.P2PForgeCertMgr `optional:"true"` }, ) (opts Libp2pOpts, err error) { privateNetworkEnabled := params.Fprint != nil tcpEnabled := tptConfig.Network.TCP.WithDefault(true) wsEnabled := tptConfig.Network.Websocket.WithDefault(true) if tcpEnabled { // TODO(9290): Make WithMetrics configurable opts.Opts = append(opts.Opts, libp2p.Transport(tcp.NewTCPTransport, tcp.WithMetrics())) } if wsEnabled { if params.ForgeMgr == nil { opts.Opts = append(opts.Opts, libp2p.Transport(websocket.New)) } else { opts.Opts = append(opts.Opts, libp2p.Transport(websocket.New, websocket.WithTLSConfig(params.ForgeMgr.TLSConfig()))) } } if tcpEnabled && wsEnabled && os.Getenv("LIBP2P_TCP_MUX") != "false" { if privateNetworkEnabled { log.Error("libp2p.ShareTCPListener() is not supported in private networks, please disable Swarm.Transports.Network.Websocket or run with LIBP2P_TCP_MUX=false to make this message go away") } else { opts.Opts = append(opts.Opts, libp2p.ShareTCPListener()) } } if tptConfig.Network.QUIC.WithDefault(!privateNetworkEnabled) { if privateNetworkEnabled { return opts, fmt.Errorf( "QUIC transport does not support private networks, please disable Swarm.Transports.Network.QUIC", ) } opts.Opts = append(opts.Opts, libp2p.Transport(quic.NewTransport)) } if tptConfig.Network.WebTransport.WithDefault(!privateNetworkEnabled) { if privateNetworkEnabled { return opts, fmt.Errorf( "WebTransport transport does not support private networks, please disable Swarm.Transports.Network.WebTransport", ) } opts.Opts = append(opts.Opts, libp2p.Transport(webtransport.New)) } if tptConfig.Network.WebRTCDirect.WithDefault(!privateNetworkEnabled) { if privateNetworkEnabled { return opts, fmt.Errorf( "WebRTC Direct transport does not support private networks, please disable Swarm.Transports.Network.WebRTCDirect", ) } opts.Opts = append(opts.Opts, libp2p.Transport(webrtc.New)) } return opts, nil } } func BandwidthCounter() (opts Libp2pOpts, reporter *metrics.BandwidthCounter) { reporter = metrics.NewBandwidthCounter() opts.Opts = append(opts.Opts, libp2p.BandwidthReporter(reporter)) return opts, reporter } ================================================ FILE: core/node/p2pforge_resolver.go ================================================ package node import ( "context" "net" "net/netip" "strings" "github.com/libp2p/go-libp2p/core/peer" madns "github.com/multiformats/go-multiaddr-dns" ) // p2pForgeResolver implements madns.BasicResolver for deterministic resolution // of p2p-forge domains (e.g., *.libp2p.direct) without network I/O for A/AAAA queries. // // p2p-forge encodes IP addresses in DNS hostnames: // - IPv4: 1-2-3-4.peerID.libp2p.direct -> 1.2.3.4 // - IPv6: 2001-db8--1.peerID.libp2p.direct -> 2001:db8::1 // // When local parsing fails (invalid format, invalid peerID, etc.), the resolver // falls back to network DNS. This ensures future .libp2p.direct records // can still resolve if the authoritative DNS adds support for them. // // TXT queries always delegate to the fallback resolver. This is important for // p2p-forge/client ACME DNS-01 challenges to work correctly, as Let's Encrypt // needs to verify TXT records at _acme-challenge.peerID.libp2p.direct. // // See: https://github.com/ipshipyard/p2p-forge type p2pForgeResolver struct { suffixes []string fallback madns.BasicResolver } // Compile-time check that p2pForgeResolver implements madns.BasicResolver. var _ madns.BasicResolver = (*p2pForgeResolver)(nil) // NewP2PForgeResolver creates a resolver for the given p2p-forge domain suffixes. // Each suffix should be a bare domain like "libp2p.direct" (without leading dot). // When local IP parsing fails, queries fall back to the provided resolver. // TXT queries always delegate to the fallback resolver for ACME compatibility. func NewP2PForgeResolver(suffixes []string, fallback madns.BasicResolver) *p2pForgeResolver { normalized := make([]string, len(suffixes)) for i, s := range suffixes { normalized[i] = strings.ToLower(strings.TrimSuffix(s, ".")) } return &p2pForgeResolver{suffixes: normalized, fallback: fallback} } // LookupIPAddr parses IP addresses encoded in the hostname. // // Format: .. // - IPv4: 192-168-1-1.peerID.libp2p.direct -> [192.168.1.1] // - IPv6: 2001-db8--1.peerID.libp2p.direct -> [2001:db8::1] // // If the hostname doesn't match the expected format (wrong suffix, invalid peerID, // invalid IP encoding, or peerID-only), the lookup falls back to network DNS. // This allows future DNS records like .libp2p.direct to resolve normally. func (r *p2pForgeResolver) LookupIPAddr(ctx context.Context, hostname string) ([]net.IPAddr, error) { // DNS is case-insensitive, normalize to lowercase hostname = strings.ToLower(strings.TrimSuffix(hostname, ".")) // find matching suffix and extract subdomain var subdomain string for _, suffix := range r.suffixes { if sub, found := strings.CutSuffix(hostname, "."+suffix); found { subdomain = sub break } } if subdomain == "" { // not a p2p-forge domain, fallback to network return r.fallback.LookupIPAddr(ctx, hostname) } // split subdomain into parts: should be [ip-prefix, peerID] parts := strings.Split(subdomain, ".") if len(parts) != 2 { // not the expected . format, fallback to network return r.fallback.LookupIPAddr(ctx, hostname) } encodedIP := parts[0] peerIDStr := parts[1] // validate peerID (same validation as libp2p.direct DNS server) if _, err := peer.Decode(peerIDStr); err != nil { // invalid peerID, fallback to network return r.fallback.LookupIPAddr(ctx, hostname) } // RFC 1123: hostname labels cannot start or end with hyphen if len(encodedIP) == 0 || encodedIP[0] == '-' || encodedIP[len(encodedIP)-1] == '-' { // invalid hostname label, fallback to network return r.fallback.LookupIPAddr(ctx, hostname) } // try parsing as IPv4 first: segments joined by "-" become "." segments := strings.Split(encodedIP, "-") if len(segments) == 4 { ipv4Str := strings.Join(segments, ".") if ip, err := netip.ParseAddr(ipv4Str); err == nil && ip.Is4() { return []net.IPAddr{{IP: ip.AsSlice()}}, nil } } // try parsing as IPv6: segments joined by "-" become ":" ipv6Str := strings.Join(segments, ":") if ip, err := netip.ParseAddr(ipv6Str); err == nil && ip.Is6() { return []net.IPAddr{{IP: ip.AsSlice()}}, nil } // IP parsing failed, fallback to network return r.fallback.LookupIPAddr(ctx, hostname) } // LookupTXT delegates to the fallback resolver to support ACME DNS-01 challenges // and any other TXT record lookups on p2p-forge domains. func (r *p2pForgeResolver) LookupTXT(ctx context.Context, hostname string) ([]string, error) { return r.fallback.LookupTXT(ctx, hostname) } ================================================ FILE: core/node/p2pforge_resolver_test.go ================================================ package node import ( "context" "errors" "net" "testing" "github.com/ipfs/kubo/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // Test constants matching p2p-forge production format const ( // testPeerID is a valid peerID in CIDv1 base36 format as used by p2p-forge. // Base36 is lowercase-only, making it safe for case-insensitive DNS. // Corresponds to 12D3KooWDpJ7As7BWAwRMfu1VU2WCqNjvq387JEYKDBj4kx6nXTN in base58btc. testPeerID = "k51qzi5uqu5dhnwe629wdlncpql6frppdpwnz4wtlcw816aysd5wwlk63g4wmh" // domainSuffix is the default p2p-forge domain used in tests. domainSuffix = config.DefaultDomainSuffix ) // mockResolver implements madns.BasicResolver for testing type mockResolver struct { txtRecords map[string][]string ipRecords map[string][]net.IPAddr ipErr error } func (m *mockResolver) LookupIPAddr(_ context.Context, hostname string) ([]net.IPAddr, error) { if m.ipErr != nil { return nil, m.ipErr } if m.ipRecords != nil { return m.ipRecords[hostname], nil } return nil, nil } func (m *mockResolver) LookupTXT(_ context.Context, name string) ([]string, error) { if m.txtRecords != nil { return m.txtRecords[name], nil } return nil, nil } // newTestResolver creates a p2pForgeResolver with default suffix. func newTestResolver(t *testing.T) *p2pForgeResolver { t.Helper() return NewP2PForgeResolver([]string{domainSuffix}, &mockResolver{}) } // assertLookupIP verifies that hostname resolves to wantIP. func assertLookupIP(t *testing.T, r *p2pForgeResolver, hostname, wantIP string) { t.Helper() addrs, err := r.LookupIPAddr(t.Context(), hostname) require.NoError(t, err) require.Len(t, addrs, 1) assert.Equal(t, wantIP, addrs[0].IP.String()) } func TestP2PForgeResolver_LookupIPAddr(t *testing.T) { r := newTestResolver(t) tests := []struct { name string hostname string wantIP string }{ // IPv4 {"ipv4/basic", "192-168-1-1." + testPeerID + "." + domainSuffix, "192.168.1.1"}, {"ipv4/zeros", "0-0-0-0." + testPeerID + "." + domainSuffix, "0.0.0.0"}, {"ipv4/max", "255-255-255-255." + testPeerID + "." + domainSuffix, "255.255.255.255"}, {"ipv4/trailing dot", "10-0-0-1." + testPeerID + "." + domainSuffix + ".", "10.0.0.1"}, {"ipv4/uppercase suffix", "192-168-1-1." + testPeerID + ".LIBP2P.DIRECT", "192.168.1.1"}, // IPv6 {"ipv6/full", "2001-db8-0-0-0-0-0-1." + testPeerID + "." + domainSuffix, "2001:db8::1"}, {"ipv6/compressed", "2001-db8--1." + testPeerID + "." + domainSuffix, "2001:db8::1"}, {"ipv6/loopback", "0--1." + testPeerID + "." + domainSuffix, "::1"}, {"ipv6/all zeros", "0--0." + testPeerID + "." + domainSuffix, "::"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { assertLookupIP(t, r, tt.hostname, tt.wantIP) }) } } func TestP2PForgeResolver_LookupIPAddr_MultipleSuffixes(t *testing.T) { r := NewP2PForgeResolver([]string{domainSuffix, "custom.example.com"}, &mockResolver{}) tests := []struct { hostname string wantIP string }{ {"192-168-1-1." + testPeerID + "." + domainSuffix, "192.168.1.1"}, {"10-0-0-1." + testPeerID + ".custom.example.com", "10.0.0.1"}, } for _, tt := range tests { t.Run(tt.hostname, func(t *testing.T) { assertLookupIP(t, r, tt.hostname, tt.wantIP) }) } } func TestP2PForgeResolver_LookupIPAddr_FallbackToNetwork(t *testing.T) { fallbackIP := []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}} tests := []struct { name string hostname string }{ {"peerID only", testPeerID + "." + domainSuffix}, {"invalid peerID", "192-168-1-1.invalid-peer-id." + domainSuffix}, {"invalid IP encoding", "not-an-ip." + testPeerID + "." + domainSuffix}, {"leading hyphen", "-192-168-1-1." + testPeerID + "." + domainSuffix}, {"too many parts", "extra.192-168-1-1." + testPeerID + "." + domainSuffix}, {"wrong suffix", "192-168-1-1." + testPeerID + ".example.com"}, } // Build fallback records from test cases ipRecords := make(map[string][]net.IPAddr, len(tests)) for _, tt := range tests { ipRecords[tt.hostname] = fallbackIP } fallback := &mockResolver{ipRecords: ipRecords} r := NewP2PForgeResolver([]string{domainSuffix}, fallback) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { addrs, err := r.LookupIPAddr(t.Context(), tt.hostname) require.NoError(t, err) require.Len(t, addrs, 1, "should fallback to network") assert.Equal(t, "93.184.216.34", addrs[0].IP.String()) }) } } func TestP2PForgeResolver_LookupIPAddr_FallbackError(t *testing.T) { expectedErr := errors.New("network error") r := NewP2PForgeResolver([]string{domainSuffix}, &mockResolver{ipErr: expectedErr}) // peerID-only triggers fallback, which returns error _, err := r.LookupIPAddr(t.Context(), testPeerID+"."+domainSuffix) require.ErrorIs(t, err, expectedErr) } func TestP2PForgeResolver_LookupTXT(t *testing.T) { t.Run("delegates to fallback for ACME DNS-01", func(t *testing.T) { acmeHost := "_acme-challenge." + testPeerID + "." + domainSuffix fallback := &mockResolver{ txtRecords: map[string][]string{acmeHost: {"acme-token-value"}}, } r := NewP2PForgeResolver([]string{domainSuffix}, fallback) records, err := r.LookupTXT(t.Context(), acmeHost) require.NoError(t, err) assert.Equal(t, []string{"acme-token-value"}, records) }) t.Run("returns empty when fallback has no records", func(t *testing.T) { r := NewP2PForgeResolver([]string{domainSuffix}, &mockResolver{}) records, err := r.LookupTXT(t.Context(), "anything."+domainSuffix) require.NoError(t, err) assert.Empty(t, records) }) } ================================================ FILE: core/node/peering.go ================================================ package node import ( "context" "github.com/ipfs/boxo/peering" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "go.uber.org/fx" ) // Peering constructs the peering service and hooks it into fx's lifetime // management system. func Peering(lc fx.Lifecycle, host host.Host) *peering.PeeringService { ps := peering.NewPeeringService(host) lc.Append(fx.Hook{ OnStart: func(context.Context) error { return ps.Start() }, OnStop: func(context.Context) error { ps.Stop() return nil }, }) return ps } // PeerWith configures the peering service to peer with the specified peers. func PeerWith(peers ...peer.AddrInfo) fx.Option { return fx.Invoke(func(ps *peering.PeeringService) { for _, ai := range peers { ps.AddPeer(ai) } }) } ================================================ FILE: core/node/provider.go ================================================ package node import ( "context" "errors" "fmt" "os" "path/filepath" "time" "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/fetcher" "github.com/ipfs/boxo/mfs" pin "github.com/ipfs/boxo/pinning/pinner" "github.com/ipfs/boxo/pinning/pinner/dspinner" "github.com/ipfs/boxo/provider" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/mount" "github.com/ipfs/go-datastore/namespace" "github.com/ipfs/go-datastore/query" log "github.com/ipfs/go-log/v2" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/repo" "github.com/ipfs/kubo/repo/fsrepo" irouting "github.com/ipfs/kubo/routing" dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p-kad-dht/amino" "github.com/libp2p/go-libp2p-kad-dht/dual" "github.com/libp2p/go-libp2p-kad-dht/fullrt" dht_pb "github.com/libp2p/go-libp2p-kad-dht/pb" dhtprovider "github.com/libp2p/go-libp2p-kad-dht/provider" "github.com/libp2p/go-libp2p-kad-dht/provider/buffered" ddhtprovider "github.com/libp2p/go-libp2p-kad-dht/provider/dual" "github.com/libp2p/go-libp2p-kad-dht/provider/keystore" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/host" peer "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" ma "github.com/multiformats/go-multiaddr" mh "github.com/multiformats/go-multihash" "go.uber.org/fx" ) const ( // The size of a batch that will be used for calculating average announcement // time per CID, inside of boxo/provider.ThroughputReport // and in 'ipfs stats provide' report. // Used when Provide.DHT.SweepEnabled=false sampledBatchSize = 1000 // Datastore key used to store previous reprovide strategy. reprovideStrategyKey = "/reprovideStrategy" // KeystoreDatastorePath is the base directory for the provider keystore datastores. KeystoreDatastorePath = "provider-keystore" ) var ( // Datastore namespace key for provider data. providerDatastoreKey = datastore.NewKey("provider") // Datastore namespace key for provider keystore data. keystoreDatastoreKey = datastore.NewKey("keystore") ) var errAcceleratedDHTNotReady = errors.New("AcceleratedDHTClient: routing table not ready") // validateKeystoreSuffix rejects any suffix other than "0" or "1". // The upstream library uses these two values as alternating namespace // identifiers. Validating here prevents accidental deletion of unrelated // directories via os.RemoveAll if the upstream ever changes its scheme. func validateKeystoreSuffix(suffix string) error { if suffix != "0" && suffix != "1" { return fmt.Errorf("unexpected keystore suffix %q, expected \"0\" or \"1\"", suffix) } return nil } // Interval between reprovide queue monitoring checks for slow reprovide alerts. // Used when Provide.DHT.SweepEnabled=true const reprovideAlertPollInterval = 15 * time.Minute // Number of consecutive polling intervals with sustained queue growth before // triggering a slow reprovide alert (3 intervals = 45 minutes). // Used when Provide.DHT.SweepEnabled=true const consecutiveAlertsThreshold = 3 // DHTProvider is an interface for providing keys to a DHT swarm. It holds a // state of keys to be advertised, and is responsible for periodically // publishing provider records for these keys to the DHT swarm before the // records expire. type DHTProvider interface { // StartProviding ensures keys are periodically advertised to the DHT swarm. // // If the `keys` aren't currently being reprovided, they are added to the // queue to be provided to the DHT swarm as soon as possible, and scheduled // to be reprovided periodically. If `force` is set to true, all keys are // provided to the DHT swarm, regardless of whether they were already being // reprovided in the past. `keys` keep being reprovided until `StopProviding` // is called. // // This operation is asynchronous, it returns as soon as the `keys` are added // to the provide queue, and provides happens asynchronously. // // Returns an error if the keys couldn't be added to the provide queue. This // can happen if the provider is closed or if the node is currently Offline // (either never bootstrapped, or disconnected since more than `OfflineDelay`). // The schedule and provide queue depend on the network size, hence recent // network connectivity is essential. StartProviding(force bool, keys ...mh.Multihash) error // ProvideOnce sends provider records for the specified keys to the DHT swarm // only once. It does not automatically reprovide those keys afterward. // // Add the supplied multihashes to the provide queue, and return immediately. // The provide operation happens asynchronously. // // Returns an error if the keys couldn't be added to the provide queue. This // can happen if the provider is closed or if the node is currently Offline // (either never bootstrapped, or disconnected since more than `OfflineDelay`). // The schedule and provide queue depend on the network size, hence recent // network connectivity is essential. ProvideOnce(keys ...mh.Multihash) error // Clear clears the all the keys from the provide queue and returns the number // of keys that were cleared. // // The keys are not deleted from the keystore, so they will continue to be // reprovided as scheduled. Clear() int // RefreshSchedule scans the Keystore for any keys that are not currently // scheduled for reproviding. If such keys are found, it schedules their // associated keyspace region to be reprovided. // // This function doesn't remove prefixes that have no keys from the schedule. // This is done automatically during the reprovide operation if a region has no // keys. // // Returns an error if the provider is closed or if the node is currently // Offline (either never bootstrapped, or disconnected since more than // `OfflineDelay`). The schedule depends on the network size, hence recent // network connectivity is essential. RefreshSchedule() error Close() error } var ( _ DHTProvider = &ddhtprovider.SweepingProvider{} _ DHTProvider = &dhtprovider.SweepingProvider{} _ DHTProvider = &NoopProvider{} _ DHTProvider = &LegacyProvider{} ) // NoopProvider is a no-operation provider implementation that does nothing. // It is used when providing is disabled or when no DHT is available. // All methods return successfully without performing any actual operations. type NoopProvider struct{} func (r *NoopProvider) StartProviding(bool, ...mh.Multihash) error { return nil } func (r *NoopProvider) ProvideOnce(...mh.Multihash) error { return nil } func (r *NoopProvider) Clear() int { return 0 } func (r *NoopProvider) RefreshSchedule() error { return nil } func (r *NoopProvider) Close() error { return nil } // LegacyProvider is a wrapper around the boxo/provider.System that implements // the DHTProvider interface. This provider manages reprovides using a burst // strategy where it sequentially reprovides all keys at once during each // reprovide interval, rather than spreading the load over time. // // This is the legacy provider implementation that can cause resource spikes // during reprovide operations. For more efficient providing, consider using // the SweepingProvider which spreads the load over the reprovide interval. type LegacyProvider struct { provider.System } func (r *LegacyProvider) StartProviding(force bool, keys ...mh.Multihash) error { return r.ProvideOnce(keys...) } func (r *LegacyProvider) ProvideOnce(keys ...mh.Multihash) error { if many, ok := r.System.(routinghelpers.ProvideManyRouter); ok { return many.ProvideMany(context.Background(), keys) } for _, k := range keys { if err := r.Provide(context.Background(), cid.NewCidV1(cid.Raw, k), true); err != nil { return err } } return nil } func (r *LegacyProvider) Clear() int { return r.System.Clear() } func (r *LegacyProvider) RefreshSchedule() error { return nil } // LegacyProviderOpt creates a LegacyProvider to be used as provider in the // IpfsNode func LegacyProviderOpt(reprovideInterval time.Duration, strategy string, acceleratedDHTClient bool, provideWorkerCount int) fx.Option { system := fx.Provide( fx.Annotate(func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, repo repo.Repo) (*LegacyProvider, error) { // Initialize provider.System first, before pinner/blockstore/etc. // The KeyChanFunc will be set later via SetKeyProvider() once we have // created the pinner, blockstore and other dependencies. opts := []provider.Option{ provider.Online(cr), provider.ReproviderInterval(reprovideInterval), provider.ProvideWorkerCount(provideWorkerCount), } if !acceleratedDHTClient && reprovideInterval > 0 { // The estimation kinda suck if you are running with accelerated DHT client, // given this message is just trying to push people to use the acceleratedDHTClient // let's not report on through if it's in use opts = append(opts, provider.ThroughputReport(func(reprovide bool, complete bool, keysProvided uint, duration time.Duration) bool { avgProvideSpeed := duration / time.Duration(keysProvided) count := uint64(keysProvided) if !reprovide || !complete { // We don't know how many CIDs we have to provide, try to fetch it from the blockstore. // But don't try for too long as this might be very expensive if you have a huge datastore. ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) defer cancel() // FIXME: I want a running counter of blocks so size of blockstore can be an O(1) lookup. // Note: talk to datastore directly, as to not depend on Blockstore here. qr, err := repo.Datastore().Query(ctx, query.Query{ Prefix: blockstore.BlockPrefix.String(), KeysOnly: true, }) if err != nil { logger.Errorf("fetching AllKeysChain in provider ThroughputReport: %v", err) return false } defer qr.Close() count = 0 countLoop: for { select { case _, ok := <-qr.Next(): if !ok { break countLoop } count++ case <-ctx.Done(): // really big blockstore mode // how many blocks would be in a 10TiB blockstore with 128KiB blocks. const probableBigBlockstore = (10 * 1024 * 1024 * 1024 * 1024) / (128 * 1024) // How long per block that lasts us. expectedProvideSpeed := reprovideInterval / probableBigBlockstore if avgProvideSpeed > expectedProvideSpeed { logger.Errorf(` 🔔🔔🔔 Reprovide Operations Too Slow 🔔🔔🔔 Your node may be falling behind on DHT reprovides, which could affect content availability. Observed: %d keys at %v per key Estimated: Assuming 10TiB blockstore, would take %v to complete ⏰ Must finish within %v (Provide.DHT.Interval) Solutions (try in order): 1. Enable Provide.DHT.SweepEnabled=true (recommended) 2. Increase Provide.DHT.MaxWorkers if needed 3. Enable Routing.AcceleratedDHTClient=true (last resort, resource intensive) Learn more: https://github.com/ipfs/kubo/blob/master/docs/config.md#provide`, keysProvided, avgProvideSpeed, avgProvideSpeed*probableBigBlockstore, reprovideInterval) return false } } } } // How long per block that lasts us. expectedProvideSpeed := reprovideInterval if count > 0 { expectedProvideSpeed = reprovideInterval / time.Duration(count) } if avgProvideSpeed > expectedProvideSpeed { logger.Errorf(` 🔔🔔🔔 Reprovide Operations Too Slow 🔔🔔🔔 Your node is falling behind on DHT reprovides, which will affect content availability. Observed: %d keys at %v per key Confirmed: ~%d total CIDs requiring %v to complete ⏰ Must finish within %v (Provide.DHT.Interval) Solutions (try in order): 1. Enable Provide.DHT.SweepEnabled=true (recommended) 2. Increase Provide.DHT.MaxWorkers if needed 3. Enable Routing.AcceleratedDHTClient=true (last resort, resource intensive) Learn more: https://github.com/ipfs/kubo/blob/master/docs/config.md#provide`, keysProvided, avgProvideSpeed, count, avgProvideSpeed*time.Duration(count), reprovideInterval) } return false }, sampledBatchSize)) } sys, err := provider.New(repo.Datastore(), opts...) if err != nil { return nil, err } lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return sys.Close() }, }) prov := &LegacyProvider{sys} handleStrategyChange(strategy, prov, repo.Datastore()) return prov, nil }, fx.As(new(provider.System)), fx.As(new(DHTProvider)), ), ) setKeyProvider := fx.Invoke(func(lc fx.Lifecycle, system provider.System, keyProvider provider.KeyChanFunc) { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { // SetKeyProvider breaks the circular dependency between provider, blockstore, and pinner. // We cannot create the blockstore without the provider (it needs to provide blocks), // and we cannot determine the reproviding strategy without the pinner/blockstore. // This deferred initialization allows us to create provider.System first, // then set the actual key provider function after all dependencies are ready. system.SetKeyProvider(keyProvider) return nil }, }) }) return fx.Options( system, setKeyProvider, ) } type dhtImpl interface { routing.Routing GetClosestPeers(context.Context, string) ([]peer.ID, error) Host() host.Host MessageSender() dht_pb.MessageSender } type fullrtRouter struct { *fullrt.FullRT ready bool logger *log.ZapEventLogger } func newFullRTRouter(fr *fullrt.FullRT, loggerName string) *fullrtRouter { return &fullrtRouter{ FullRT: fr, ready: true, logger: log.Logger(loggerName), } } // GetClosestPeers overrides fullrt.FullRT's GetClosestPeers and returns an // error if the fullrt's initial network crawl isn't complete yet. func (fr *fullrtRouter) GetClosestPeers(ctx context.Context, key string) ([]peer.ID, error) { if fr.ready { if !fr.Ready() { fr.ready = false fr.logger.Info("AcceleratedDHTClient: waiting for routing table initialization (5-10 min, depends on DHT size and network) to complete before providing") return nil, errAcceleratedDHTNotReady } } else { if fr.Ready() { fr.ready = true fr.logger.Info("AcceleratedDHTClient: routing table ready, providing can begin") } else { return nil, errAcceleratedDHTNotReady } } return fr.FullRT.GetClosestPeers(ctx, key) } var ( _ dhtImpl = &dht.IpfsDHT{} _ dhtImpl = &fullrtRouter{} ) type addrsFilter interface { FilteredAddrs() []ma.Multiaddr } // findRootDatastoreSpec extracts the leaf datastore spec for the root ("/") // mount from the repo's Datastore.Spec config. It unwraps mount (picks the "/" // mountpoint), measure, and log wrappers to find the actual backend spec // (e.g., levelds, pebbleds). func findRootDatastoreSpec(spec map[string]any) map[string]any { if spec == nil { return nil } switch spec["type"] { case "mount": mounts, ok := spec["mounts"].([]any) if !ok { return spec } for _, m := range mounts { mnt, ok := m.(map[string]any) if !ok { continue } if mnt["mountpoint"] == "/" { return findRootDatastoreSpec(mnt) } } // No root mount found; return nil so callers fall back gracefully // (in-memory datastore or skip mounting) rather than passing a // mount-type spec to openDatastoreAt which expects a leaf backend. return nil case "measure", "log": if child, ok := spec["child"].(map[string]any); ok { return findRootDatastoreSpec(child) } return spec default: if _, hasChild := spec["child"]; hasChild { logger.Warnw("unrecognized datastore wrapper type, using as-is", "type", spec["type"]) } return spec } } // MountKeystoreDatastores opens any provider keystore datastores that exist on // disk and returns them as mount.Mount entries ready to be combined with the // main repo datastore. The caller must call the returned cleanup function when // done. Returns nil mounts and a no-op closer if no keystores exist. func MountKeystoreDatastores(repo repo.Repo) ([]mount.Mount, func(), error) { cfg, err := repo.Config() if err != nil { return nil, nil, fmt.Errorf("reading repo config: %w", err) } rootSpec := findRootDatastoreSpec(cfg.Datastore.Spec) if rootSpec == nil { return nil, func() {}, nil } keystoreBasePath := filepath.Join(repo.Path(), KeystoreDatastorePath) var mounts []mount.Mount var closers []func() for _, suffix := range []string{"0", "1"} { dir := filepath.Join(keystoreBasePath, suffix) if _, err := os.Stat(dir); err != nil { continue } ds, err := openDatastoreAt(rootSpec, dir) if err != nil { for _, c := range closers { c() } return nil, nil, err } prefix := providerDatastoreKey.Child(keystoreDatastoreKey).ChildString(suffix) mounts = append(mounts, mount.Mount{Prefix: prefix, Datastore: ds}) closers = append(closers, func() { ds.Close() }) } closer := func() { for _, c := range closers { c() } } return mounts, closer, nil } // openDatastoreAt opens a datastore using the given spec at the specified path. // It deep-copies the spec to avoid mutating the original. func openDatastoreAt(rootSpec map[string]any, path string) (datastore.Batching, error) { spec := copySpec(rootSpec) spec["path"] = path dsc, err := fsrepo.AnyDatastoreConfig(spec) if err != nil { return nil, fmt.Errorf("creating datastore config for %s: %w", path, err) } return dsc.Create("") } // copySpec deep-copies a datastore spec map so modifications (e.g., changing // the path) don't affect the original. func copySpec(spec map[string]any) map[string]any { if spec == nil { return nil } cp := make(map[string]any, len(spec)) for k, v := range spec { switch val := v.(type) { case map[string]any: cp[k] = copySpec(val) case []any: s := make([]any, len(val)) for i, elem := range val { if m, ok := elem.(map[string]any); ok { s[i] = copySpec(m) } else { s[i] = elem } } cp[k] = s default: cp[k] = v } } return cp } // purgeBatchSize is the number of keys deleted per batch commit during // orphaned keystore cleanup. Each commit is a cancellation checkpoint. const purgeBatchSize = 1 << 12 // 4096 // purgeOrphanedKeystoreData deletes all keys under /provider/keystore/ from the // shared repo datastore. These were written by older Kubo versions that stored // provider keystore data inline in the shared datastore. The new code uses // separate filesystem datastores under /{KeystoreDatastorePath}/ instead. // // The operation is idempotent and safe to interrupt: partial completion is // fine because already-deleted keys are no-ops on re-run. func purgeOrphanedKeystoreData(ctx context.Context, ds datastore.Batching) error { orphanedPrefix := providerDatastoreKey.Child(keystoreDatastoreKey).String() syncKey := datastore.NewKey(orphanedPrefix) results, err := ds.Query(ctx, query.Query{ Prefix: orphanedPrefix, KeysOnly: true, }) if err != nil { return fmt.Errorf("querying orphaned keystore data: %w", err) } defer results.Close() var batch datastore.Batch var count, pending int for result := range results.Next() { if ctx.Err() != nil { return ctx.Err() } if result.Error != nil { return fmt.Errorf("iterating orphaned keystore data: %w", result.Error) } if batch == nil { batch, err = ds.Batch(ctx) if err != nil { return fmt.Errorf("creating batch for orphaned keystore cleanup: %w", err) } } if err := batch.Delete(ctx, datastore.NewKey(result.Key)); err != nil { return fmt.Errorf("batch deleting orphaned key %s: %w", result.Key, err) } count++ pending++ if pending >= purgeBatchSize { if err := batch.Commit(ctx); err != nil { return fmt.Errorf("committing orphaned keystore cleanup batch: %w", err) } if err := ds.Sync(ctx, syncKey); err != nil { return fmt.Errorf("syncing orphaned keystore cleanup: %w", err) } batch = nil pending = 0 } } if pending > 0 { if err := batch.Commit(ctx); err != nil { return fmt.Errorf("committing orphaned keystore cleanup batch: %w", err) } if err := ds.Sync(ctx, syncKey); err != nil { return fmt.Errorf("syncing orphaned keystore cleanup: %w", err) } } if count > 0 { logger.Infow("purged orphaned provider keystore data from shared datastore", "keys", count) } return nil } func SweepingProviderOpt(cfg *config.Config) fx.Option { reprovideInterval := cfg.Provide.DHT.Interval.WithDefault(config.DefaultProvideDHTInterval) type providerInput struct { fx.In DHT routing.Routing `name:"dhtc"` Repo repo.Repo Lc fx.Lifecycle } sweepingReprovider := fx.Provide(func(in providerInput) (DHTProvider, *keystore.ResettableKeystore, error) { ds := namespace.Wrap(in.Repo.Datastore(), providerDatastoreKey) // Get repo path and config to determine datastore type repoPath := in.Repo.Path() repoCfg, err := in.Repo.Config() if err != nil { return nil, nil, fmt.Errorf("getting repo config: %w", err) } // Find the root datastore type (levelds, pebbleds, etc.) rootSpec := findRootDatastoreSpec(repoCfg.Datastore.Spec) // Keystore datastores live at /{KeystoreDatastorePath}/ keystoreBasePath := filepath.Join(repoPath, KeystoreDatastorePath) createDs := func(suffix string) (datastore.Batching, error) { if err := validateKeystoreSuffix(suffix); err != nil { return nil, err } // When no datastore spec is configured (e.g., test/mock repos), // fall back to an in-memory datastore. if rootSpec == nil { return datastore.NewMapDatastore(), nil } if err := os.MkdirAll(keystoreBasePath, 0o755); err != nil { return nil, fmt.Errorf("creating keystore base directory: %w", err) } ds, err := openDatastoreAt(rootSpec, filepath.Join(keystoreBasePath, suffix)) if err != nil { return nil, err } logger.Infow("provider keystore: opened datastore", "suffix", suffix, "path", filepath.Join(keystoreBasePath, suffix)) return ds, nil } destroyDs := func(suffix string) error { if err := validateKeystoreSuffix(suffix); err != nil { return err } logger.Infow("provider keystore: removing datastore from disk", "suffix", suffix, "path", filepath.Join(keystoreBasePath, suffix)) return os.RemoveAll(filepath.Join(keystoreBasePath, suffix)) } // One-time cleanup of stale keystore data left by older Kubo in the // shared repo datastore under /provider/keystore/. New code stores // bulk key data in separate filesystem datastores under // /{KeystoreDatastorePath}/ while still using the same // /provider/keystore/ namespace in the shared datastore for metadata. // // The absence of the keystoreBasePath directory signals a first run // after upgrade: the directory is created later by createDs on first // use, so it doubles as a "cleanup done" flag. If the process dies // mid-purge the directory still won't exist and the cleanup re-runs // on next start (it is idempotent). Must run synchronously before // NewResettableKeystore to avoid racing with reads on the same // namespace. if _, statErr := os.Stat(keystoreBasePath); os.IsNotExist(statErr) { logger.Infow("migrating provider keystore data from shared datastore to separate filesystem datastores", "path", keystoreBasePath) // Create a cancellable context for the purge. The OnStop hook // below calls purgeCancel when the node receives a shutdown // signal (e.g., SIGINT), which interrupts the purge loop // instead of blocking indefinitely. purgeCtx, purgeCancel := context.WithCancel(context.Background()) in.Lc.Append(fx.Hook{ OnStop: func(_ context.Context) error { purgeCancel() return nil }, }) if purgeErr := purgeOrphanedKeystoreData(purgeCtx, in.Repo.Datastore()); purgeErr != nil { if purgeCtx.Err() != nil { logger.Infow("provider keystore migration interrupted by shutdown, will resume on next start") } else { logger.Warnw("provider keystore migration failed, will retry on next start", "error", purgeErr) } } else { logger.Infow("provider keystore migration completed") } purgeCancel() } keystoreDs := namespace.Wrap(ds, keystoreDatastoreKey) ks, err := keystore.NewResettableKeystore(keystoreDs, keystore.WithDatastoreFactory(createDs, destroyDs), keystore.KeystoreOption( keystore.WithPrefixBits(16), keystore.WithBatchSize(int(cfg.Provide.DHT.KeystoreBatchSize.WithDefault(config.DefaultProvideDHTKeystoreBatchSize))), ), ) if err != nil { return nil, nil, err } // Constants for buffered provider configuration // These values match the upstream defaults from go-libp2p-kad-dht and have been battle-tested const ( // bufferedDsName is the datastore namespace used by the buffered provider. // The dsqueue persists operations here to handle large data additions without // being memory-bound, allowing operations on hardware with limited RAM and // enabling core operations to return instantly while processing happens async. bufferedDsName = "bprov" // bufferedBatchSize controls how many operations are dequeued and processed // together from the datastore queue. The worker processes up to this many // operations at once, grouping them by type for efficiency. bufferedBatchSize = 1 << 10 // 1024 items // bufferedIdleWriteTime is an implementation detail of go-dsqueue that controls // how long the datastore buffer waits for new multihashes to arrive before // flushing in-memory items to the datastore. This does NOT affect providing speed - // provides happen as fast as possible via a dedicated worker that continuously // processes the queue regardless of this timing. bufferedIdleWriteTime = time.Minute // loggerName is the name of the go-log logger used by the provider. loggerName = dhtprovider.DefaultLoggerName ) bufferedProviderOpts := []buffered.Option{ buffered.WithBatchSize(bufferedBatchSize), buffered.WithDsName(bufferedDsName), buffered.WithIdleWriteTime(bufferedIdleWriteTime), } var impl dhtImpl switch inDht := in.DHT.(type) { case *dht.IpfsDHT: if inDht != nil { impl = inDht } case *dual.DHT: if inDht != nil { prov, err := ddhtprovider.New(inDht, ddhtprovider.WithKeystore(ks), ddhtprovider.WithDatastore(ds), ddhtprovider.WithResumeCycle(cfg.Provide.DHT.ResumeEnabled.WithDefault(config.DefaultProvideDHTResumeEnabled)), ddhtprovider.WithReprovideInterval(reprovideInterval), ddhtprovider.WithMaxReprovideDelay(time.Hour), ddhtprovider.WithOfflineDelay(cfg.Provide.DHT.OfflineDelay.WithDefault(config.DefaultProvideDHTOfflineDelay)), ddhtprovider.WithConnectivityCheckOnlineInterval(1*time.Minute), ddhtprovider.WithMaxWorkers(int(cfg.Provide.DHT.MaxWorkers.WithDefault(config.DefaultProvideDHTMaxWorkers))), ddhtprovider.WithDedicatedPeriodicWorkers(int(cfg.Provide.DHT.DedicatedPeriodicWorkers.WithDefault(config.DefaultProvideDHTDedicatedPeriodicWorkers))), ddhtprovider.WithDedicatedBurstWorkers(int(cfg.Provide.DHT.DedicatedBurstWorkers.WithDefault(config.DefaultProvideDHTDedicatedBurstWorkers))), ddhtprovider.WithMaxProvideConnsPerWorker(int(cfg.Provide.DHT.MaxProvideConnsPerWorker.WithDefault(config.DefaultProvideDHTMaxProvideConnsPerWorker))), ddhtprovider.WithLoggerName(loggerName), ) if err != nil { return nil, nil, err } return buffered.New(prov, ds, bufferedProviderOpts...), ks, nil } case *fullrt.FullRT: if inDht != nil { impl = newFullRTRouter(inDht, loggerName) } } if impl == nil { return &NoopProvider{}, nil, nil } var selfAddrsFunc func() []ma.Multiaddr if imlpFilter, ok := impl.(addrsFilter); ok { selfAddrsFunc = imlpFilter.FilteredAddrs } else { selfAddrsFunc = func() []ma.Multiaddr { return impl.Host().Addrs() } } opts := []dhtprovider.Option{ dhtprovider.WithKeystore(ks), dhtprovider.WithDatastore(ds), dhtprovider.WithResumeCycle(cfg.Provide.DHT.ResumeEnabled.WithDefault(config.DefaultProvideDHTResumeEnabled)), dhtprovider.WithHost(impl.Host()), dhtprovider.WithRouter(impl), dhtprovider.WithMessageSender(impl.MessageSender()), dhtprovider.WithSelfAddrs(selfAddrsFunc), dhtprovider.WithAddLocalRecord(func(h mh.Multihash) error { return impl.Provide(context.Background(), cid.NewCidV1(cid.Raw, h), false) }), dhtprovider.WithReplicationFactor(amino.DefaultBucketSize), dhtprovider.WithReprovideInterval(reprovideInterval), dhtprovider.WithMaxReprovideDelay(time.Hour), dhtprovider.WithOfflineDelay(cfg.Provide.DHT.OfflineDelay.WithDefault(config.DefaultProvideDHTOfflineDelay)), dhtprovider.WithConnectivityCheckOnlineInterval(1 * time.Minute), dhtprovider.WithMaxWorkers(int(cfg.Provide.DHT.MaxWorkers.WithDefault(config.DefaultProvideDHTMaxWorkers))), dhtprovider.WithDedicatedPeriodicWorkers(int(cfg.Provide.DHT.DedicatedPeriodicWorkers.WithDefault(config.DefaultProvideDHTDedicatedPeriodicWorkers))), dhtprovider.WithDedicatedBurstWorkers(int(cfg.Provide.DHT.DedicatedBurstWorkers.WithDefault(config.DefaultProvideDHTDedicatedBurstWorkers))), dhtprovider.WithMaxProvideConnsPerWorker(int(cfg.Provide.DHT.MaxProvideConnsPerWorker.WithDefault(config.DefaultProvideDHTMaxProvideConnsPerWorker))), dhtprovider.WithLoggerName(loggerName), } prov, err := dhtprovider.New(opts...) if err != nil { return nil, nil, err } return buffered.New(prov, ds, bufferedProviderOpts...), ks, nil }) type keystoreInput struct { fx.In Provider DHTProvider Keystore *keystore.ResettableKeystore KeyProvider provider.KeyChanFunc } initKeystore := fx.Invoke(func(lc fx.Lifecycle, in keystoreInput) { // Skip keystore initialization for NoopProvider if _, ok := in.Provider.(*NoopProvider); ok { return } var ( cancel context.CancelFunc done = make(chan struct{}) ) syncKeystore := func(ctx context.Context) error { kcf, err := in.KeyProvider(ctx) if err != nil { return err } if err := in.Keystore.ResetCids(ctx, kcf); err != nil { return err } if err := in.Provider.RefreshSchedule(); err != nil { logger.Infow("refreshing provider schedule", "err", err) } return nil } lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { // Set the KeyProvider as a garbage collection function for the // keystore. Periodically purge the Keystore from all its keys and // replace them with the keys that needs to be reprovided, coming from // the KeyChanFunc. So far, this is the less worse way to remove CIDs // that shouldn't be reprovided from the provider's state. go func() { // Sync the keystore once at startup. This operation is async since // we need to walk the DAG of objects matching the provide strategy, // which can take a while. strategy := cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy) logger.Infow("provider keystore sync started", "strategy", strategy) if err := syncKeystore(ctx); err != nil { if ctx.Err() == nil { logger.Errorw("provider keystore sync failed", "err", err, "strategy", strategy) } else { logger.Debugw("provider keystore sync interrupted by shutdown", "err", err, "strategy", strategy) } return } logger.Infow("provider keystore sync completed", "strategy", strategy) }() gcCtx, c := context.WithCancel(context.Background()) cancel = c go func() { // garbage collection loop for cids to reprovide defer close(done) ticker := time.NewTicker(reprovideInterval) defer ticker.Stop() for { select { case <-gcCtx.Done(): return case <-ticker.C: if err := syncKeystore(gcCtx); err != nil { logger.Errorw("provider keystore sync", "err", err) } } } }() return nil }, OnStop: func(ctx context.Context) error { if cancel != nil { cancel() } select { case <-done: case <-ctx.Done(): return ctx.Err() } // Keystore will be closed by ensureProviderClosesBeforeKeystore hook // to guarantee provider closes before keystore. return nil }, }) }) // ensureProviderClosesBeforeKeystore manages the shutdown order between // provider and keystore to prevent race conditions. // // The provider's worker goroutines may call keystore methods during their // operation. If keystore closes while these operations are in-flight, we get // "keystore is closed" errors. By closing the provider first, we ensure all // worker goroutines exit and complete any pending keystore operations before // the keystore itself closes. type providerKeystoreShutdownInput struct { fx.In Provider DHTProvider Keystore *keystore.ResettableKeystore } ensureProviderClosesBeforeKeystore := fx.Invoke(func(lc fx.Lifecycle, in providerKeystoreShutdownInput) { // Skip for NoopProvider if _, ok := in.Provider.(*NoopProvider); ok { return } lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { // Close provider first - waits for all worker goroutines to exit. // This ensures no code can access keystore after this returns. if err := in.Provider.Close(); err != nil { logger.Errorw("error closing provider during shutdown", "error", err) } // Close keystore - safe now, provider is fully shut down return in.Keystore.Close() }, }) }) // extractSweepingProvider extracts a SweepingProvider from the given provider interface. // It handles unwrapping buffered and dual providers, always selecting WAN for dual DHT. // Returns nil if the provider is not a sweeping provider type. var extractSweepingProvider func(prov any) *dhtprovider.SweepingProvider extractSweepingProvider = func(prov any) *dhtprovider.SweepingProvider { switch p := prov.(type) { case *dhtprovider.SweepingProvider: return p case *ddhtprovider.SweepingProvider: return p.WAN case *buffered.SweepingProvider: // Recursively extract from the inner provider return extractSweepingProvider(p.Provider) default: return nil } } type alertInput struct { fx.In Provider DHTProvider } reprovideAlert := fx.Invoke(func(lc fx.Lifecycle, in alertInput) { prov := extractSweepingProvider(in.Provider) if prov == nil { return } var ( cancel context.CancelFunc done = make(chan struct{}) ) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { gcCtx, c := context.WithCancel(context.Background()) cancel = c go func() { defer close(done) ticker := time.NewTicker(reprovideAlertPollInterval) defer ticker.Stop() var ( queueSize, prevQueueSize int64 queuedWorkers, prevQueuedWorkers bool count int ) for { select { case <-gcCtx.Done(): return case <-ticker.C: } stats := prov.Stats() queuedWorkers = stats.Workers.QueuedPeriodic > 0 queueSize = int64(stats.Queues.PendingRegionReprovides) // Alert if reprovide queue keeps growing and all periodic workers are busy. // Requires consecutiveAlertsThreshold intervals of sustained growth. if prevQueuedWorkers && queuedWorkers && queueSize > prevQueueSize { count++ if count >= consecutiveAlertsThreshold { logger.Errorf(` 🔔🔔🔔 Reprovide Operations Too Slow 🔔🔔🔔 Your node is falling behind on DHT reprovides, which will affect content availability. Keyspace regions enqueued for reprovide: %s ago:\t%d Now:\t%d All periodic workers are busy! Active workers:\t%d / %d (max) Active workers types:\t%d periodic, %d burst Dedicated workers:\t%d periodic, %d burst Solutions (try in order): 1. Increase Provide.DHT.MaxWorkers (current %d) 2. Increase Provide.DHT.DedicatedPeriodicWorkers (current %d) 3. Set Provide.DHT.SweepEnabled=false and Routing.AcceleratedDHTClient=true (last resort, not recommended) See how the reprovide queue is processed in real-time with 'watch ipfs provide stat --all --compact' See docs: https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtmaxworkers`, reprovideAlertPollInterval.Truncate(time.Minute).String(), prevQueueSize, queueSize, stats.Workers.Active, stats.Workers.Max, stats.Workers.ActivePeriodic, stats.Workers.ActiveBurst, stats.Workers.DedicatedPeriodic, stats.Workers.DedicatedBurst, stats.Workers.Max, stats.Workers.DedicatedPeriodic) } } else if !queuedWorkers { count = 0 } prevQueueSize, prevQueuedWorkers = queueSize, queuedWorkers } }() return nil }, OnStop: func(ctx context.Context) error { // Cancel the alert loop if cancel != nil { cancel() } select { case <-done: case <-ctx.Done(): return ctx.Err() } return nil }, }) }) return fx.Options( sweepingReprovider, initKeystore, ensureProviderClosesBeforeKeystore, reprovideAlert, ) } // ONLINE/OFFLINE // hasDHTRouting checks if the routing configuration includes a DHT component. // Returns false for HTTP-only custom routing configurations (e.g., Routing.Type="custom" // with only HTTP routers). This is used to determine whether SweepingProviderOpt // can be used, since it requires a DHT client. func hasDHTRouting(cfg *config.Config) bool { routingType := cfg.Routing.Type.WithDefault(config.DefaultRoutingType) switch routingType { case "auto", "autoclient", "dht", "dhtclient", "dhtserver": return true case "custom": // Check if any router in custom config is DHT-based for _, router := range cfg.Routing.Routers { if routerIncludesDHT(router, cfg) { return true } } return false default: // "none", "delegated" return false } } // routerIncludesDHT recursively checks if a router configuration includes DHT. // Handles parallel and sequential composite routers by checking their children. func routerIncludesDHT(rp config.RouterParser, cfg *config.Config) bool { switch rp.Type { case config.RouterTypeDHT: return true case config.RouterTypeParallel, config.RouterTypeSequential: if children, ok := rp.Parameters.(*config.ComposableRouterParams); ok { for _, child := range children.Routers { if childRouter, exists := cfg.Routing.Routers[child.RouterName]; exists { if routerIncludesDHT(childRouter, cfg) { return true } } } } } return false } // OnlineProviders groups units managing provide routing records online func OnlineProviders(provide bool, cfg *config.Config) fx.Option { if !provide { return OfflineProviders() } providerStrategy := cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy) strategyFlag := config.ParseProvideStrategy(providerStrategy) if strategyFlag == 0 { return fx.Error(fmt.Errorf("provider: unknown strategy %q", providerStrategy)) } opts := []fx.Option{ fx.Provide(setReproviderKeyProvider(providerStrategy)), } sweepEnabled := cfg.Provide.DHT.SweepEnabled.WithDefault(config.DefaultProvideDHTSweepEnabled) dhtAvailable := hasDHTRouting(cfg) // Use SweepingProvider only when both sweep is enabled AND DHT is available. // For HTTP-only routing (e.g., Routing.Type="custom" with only HTTP routers), // fall back to LegacyProvider which works with ProvideManyRouter. // See https://github.com/ipfs/kubo/issues/11089 if sweepEnabled && dhtAvailable { opts = append(opts, SweepingProviderOpt(cfg)) } else { reprovideInterval := cfg.Provide.DHT.Interval.WithDefault(config.DefaultProvideDHTInterval) acceleratedDHTClient := cfg.Routing.AcceleratedDHTClient.WithDefault(config.DefaultAcceleratedDHTClient) provideWorkerCount := int(cfg.Provide.DHT.MaxWorkers.WithDefault(config.DefaultProvideDHTMaxWorkers)) opts = append(opts, LegacyProviderOpt(reprovideInterval, providerStrategy, acceleratedDHTClient, provideWorkerCount)) } return fx.Options(opts...) } // OfflineProviders groups units managing provide routing records offline func OfflineProviders() fx.Option { return fx.Provide(func() DHTProvider { return &NoopProvider{} }) } func mfsProvider(mfsRoot *mfs.Root, fetcher fetcher.Factory) provider.KeyChanFunc { return func(ctx context.Context) (<-chan cid.Cid, error) { err := mfsRoot.FlushMemFree(ctx) if err != nil { return nil, fmt.Errorf("provider: error flushing MFS, cannot provide MFS: %w", err) } rootNode, err := mfsRoot.GetDirectory().GetNode() if err != nil { return nil, fmt.Errorf("provider: error loading MFS root, cannot provide MFS: %w", err) } kcf := provider.NewDAGProvider(rootNode.Cid(), fetcher) return kcf(ctx) } } type provStrategyIn struct { fx.In Pinner pin.Pinner Blockstore blockstore.Blockstore OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"` OfflineUnixFSFetcher fetcher.Factory `name:"offlineUnixfsFetcher"` MFSRoot *mfs.Root Repo repo.Repo } type provStrategyOut struct { fx.Out ProvidingStrategy config.ProvideStrategy ProvidingKeyChanFunc provider.KeyChanFunc } // createKeyProvider creates the appropriate KeyChanFunc based on strategy. // Each strategy has different behavior: // - "roots": Only root CIDs of pinned content // - "pinned": All pinned content (roots + children) // - "mfs": Only MFS content // - "all": all blocks func createKeyProvider(strategyFlag config.ProvideStrategy, in provStrategyIn) provider.KeyChanFunc { switch strategyFlag { case config.ProvideStrategyRoots: return provider.NewBufferedProvider(dspinner.NewPinnedProvider(true, in.Pinner, in.OfflineIPLDFetcher)) case config.ProvideStrategyPinned: return provider.NewBufferedProvider(dspinner.NewPinnedProvider(false, in.Pinner, in.OfflineIPLDFetcher)) case config.ProvideStrategyPinned | config.ProvideStrategyMFS: return provider.NewPrioritizedProvider( provider.NewBufferedProvider(dspinner.NewPinnedProvider(false, in.Pinner, in.OfflineIPLDFetcher)), mfsProvider(in.MFSRoot, in.OfflineUnixFSFetcher), ) case config.ProvideStrategyMFS: return mfsProvider(in.MFSRoot, in.OfflineUnixFSFetcher) default: // "all", "", "flat" (compat) return in.Blockstore.AllKeysChan } } // detectStrategyChange checks if the reproviding strategy has changed from what's persisted. // Returns: (previousStrategy, hasChanged, error) func detectStrategyChange(ctx context.Context, strategy string, ds datastore.Datastore) (string, bool, error) { strategyKey := datastore.NewKey(reprovideStrategyKey) prev, err := ds.Get(ctx, strategyKey) if err != nil { if errors.Is(err, datastore.ErrNotFound) { return "", strategy != "", nil } return "", false, err } previousStrategy := string(prev) return previousStrategy, previousStrategy != strategy, nil } // persistStrategy saves the current reproviding strategy to the datastore. // Empty string strategies are deleted rather than stored. func persistStrategy(ctx context.Context, strategy string, ds datastore.Datastore) error { strategyKey := datastore.NewKey(reprovideStrategyKey) if strategy == "" { return ds.Delete(ctx, strategyKey) } return ds.Put(ctx, strategyKey, []byte(strategy)) } // handleStrategyChange manages strategy change detection and queue clearing. // Strategy change detection: when the reproviding strategy changes, // we clear the provide queue to avoid unexpected behavior from mixing // strategies. This ensures a clean transition between different providing modes. func handleStrategyChange(strategy string, provider DHTProvider, ds datastore.Datastore) { ctx := context.Background() previous, changed, err := detectStrategyChange(ctx, strategy, ds) if err != nil { logger.Error("cannot read previous reprovide strategy", "err", err) return } if !changed { return } logger.Infow("Provide.Strategy changed, clearing provide queue", "previous", previous, "current", strategy) provider.Clear() if err := persistStrategy(ctx, strategy, ds); err != nil { logger.Error("cannot update reprovide strategy", "err", err) } } func setReproviderKeyProvider(strategy string) func(in provStrategyIn) provStrategyOut { strategyFlag := config.ParseProvideStrategy(strategy) return func(in provStrategyIn) provStrategyOut { // Create the appropriate key provider based on strategy kcf := createKeyProvider(strategyFlag, in) return provStrategyOut{ ProvidingStrategy: strategyFlag, ProvidingKeyChanFunc: kcf, } } } ================================================ FILE: core/node/storage.go ================================================ package node import ( blockstore "github.com/ipfs/boxo/blockstore" "github.com/ipfs/go-datastore" config "github.com/ipfs/kubo/config" "go.uber.org/fx" "github.com/ipfs/boxo/filestore" "github.com/ipfs/kubo/core/node/helpers" "github.com/ipfs/kubo/repo" "github.com/ipfs/kubo/thirdparty/verifbs" ) // RepoConfig loads configuration from the repo func RepoConfig(repo repo.Repo) (*config.Config, error) { cfg, err := repo.Config() return cfg, err } // Datastore provides the datastore func Datastore(repo repo.Repo) datastore.Datastore { return repo.Datastore() } // BaseBlocks is the lower level blockstore without GC or Filestore layers type BaseBlocks blockstore.Blockstore // BaseBlockstoreCtor creates cached blockstore backed by the provided datastore func BaseBlockstoreCtor( cacheOpts blockstore.CacheOpts, hashOnRead bool, writeThrough bool, providingStrategy string, ) func(mctx helpers.MetricsCtx, repo repo.Repo, prov DHTProvider, lc fx.Lifecycle) (bs BaseBlocks, err error) { return func(mctx helpers.MetricsCtx, repo repo.Repo, prov DHTProvider, lc fx.Lifecycle) (bs BaseBlocks, err error) { opts := []blockstore.Option{blockstore.WriteThrough(writeThrough)} // Blockstore providing integration: // When strategy includes "all" the blockstore directly provides blocks as they're Put. // Important: Provide calls from blockstore are intentionally BLOCKING. // The Provider implementation (not the blockstore) should handle concurrency/queuing. // This avoids spawning unbounded goroutines for concurrent block additions. strategyFlag := config.ParseProvideStrategy(providingStrategy) if strategyFlag&config.ProvideStrategyAll != 0 { opts = append(opts, blockstore.Provider(prov)) } // hash security bs = blockstore.NewBlockstore( repo.Datastore(), opts..., ) bs = &verifbs.VerifBS{Blockstore: bs} bs, err = blockstore.CachedBlockstore(helpers.LifecycleCtx(mctx, lc), bs, cacheOpts) if err != nil { return nil, err } bs = blockstore.NewIdStore(bs) if hashOnRead { bs = &blockstore.ValidatingBlockstore{Blockstore: bs} } return } } // GcBlockstoreCtor wraps the base blockstore with GC and Filestore layers func GcBlockstoreCtor(bb BaseBlocks) (gclocker blockstore.GCLocker, gcbs blockstore.GCBlockstore, bs blockstore.Blockstore) { gclocker = blockstore.NewGCLocker() gcbs = blockstore.NewGCBlockstore(bb, gclocker) bs = gcbs return } // FilestoreBlockstoreCtor wraps GcBlockstore and adds Filestore support func FilestoreBlockstoreCtor(repo repo.Repo, bb BaseBlocks, prov DHTProvider) (gclocker blockstore.GCLocker, gcbs blockstore.GCBlockstore, bs blockstore.Blockstore, fstore *filestore.Filestore) { gclocker = blockstore.NewGCLocker() // hash security fstore = filestore.NewFilestore(bb, repo.FileManager(), prov) gcbs = blockstore.NewGCBlockstore(fstore, gclocker) gcbs = &verifbs.VerifBSGC{GCBlockstore: gcbs} bs = gcbs return } ================================================ FILE: coverage/.gitignore ================================================ unitcover sharnesscover ipfs ================================================ FILE: coverage/Rules.mk ================================================ include mk/header.mk GOCC ?= go $(d)/coverage_deps: $$(DEPS_GO) cmd/ipfs/ipfs rm -rf $(@D)/sharnesscover && mkdir $(@D)/sharnesscover .PHONY: $(d)/coverage_deps # unit tests coverage is now produced by test_unit target in mk/golang.mk # (outputs coverage/unit_tests.coverprofile and test/unit/gotest.json) TGTS_$(d) := # sharness tests coverage $(d)/ipfs: GOTAGS += testrunmain $(d)/ipfs: $(d)/main $(go-build-relative) CLEAN += $(d)/ipfs ifneq ($(filter coverage%,$(MAKECMDGOALS)),) # this is quite hacky but it is best way I could figure out DEPS_test/sharness += cmd/ipfs/ipfs-test-cover $(d)/coverage_deps $(d)/ipfs endif export IPFS_COVER_DIR:= $(realpath $(d))/sharnesscover/ $(d)/sharness_tests.coverprofile: export TEST_PLUGIN=0 $(d)/sharness_tests.coverprofile: $(d)/ipfs cmd/ipfs/ipfs-test-cover $(d)/coverage_deps test/bin/gocovmerge test_sharness (cd $(@D)/sharnesscover && find . -type f | gocovmerge -list -) > $@ PATH := $(realpath $(d)):$(PATH) TGTS_$(d) += $(d)/sharness_tests.coverprofile CLEAN += $(TGTS_$(d)) COVERAGE += $(TGTS_$(d)) include mk/footer.mk ================================================ FILE: coverage/main/main.go ================================================ //go:build testrunmain package main import ( "fmt" "io" "os" "os/exec" "os/signal" "strconv" "syscall" ) func main() { coverDir := os.Getenv("IPFS_COVER_DIR") if len(coverDir) == 0 { fmt.Println("IPFS_COVER_DIR not defined") os.Exit(1) } coverFile, err := os.CreateTemp(coverDir, "coverage-") if err != nil { fmt.Println(err.Error()) os.Exit(1) } retFile, err := os.CreateTemp("", "cover-ret-file") if err != nil { fmt.Println(err.Error()) os.Exit(1) } args := []string{"-test.run", "^TestRunMain$", "-test.coverprofile=" + coverFile.Name(), "--"} args = append(args, os.Args[1:]...) p := exec.Command("ipfs-test-cover", args...) p.Stdin = os.Stdin p.Stdout = os.Stdout p.Stderr = os.Stderr p.Env = append(os.Environ(), "IPFS_COVER_RET_FILE="+retFile.Name()) p.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGTERM, } sig := make(chan os.Signal, 10) start := make(chan struct{}) go func() { <-start for { p.Process.Signal(<-sig) } }() signal.Notify(sig, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) err = p.Start() if err != nil { fmt.Println(err.Error()) os.Exit(1) } close(start) err = p.Wait() if err != nil { fmt.Println(err.Error()) os.Exit(1) } b, err := io.ReadAll(retFile) if err != nil { fmt.Println(err.Error()) os.Exit(1) } b = b[:len(b)-1] d, err := strconv.Atoi(string(b)) if err != nil { fmt.Println(err.Error()) os.Exit(1) } os.Exit(d) } ================================================ FILE: doc.go ================================================ /* IPFS is a global, versioned, peer-to-peer filesystem */ package ipfs ================================================ FILE: docker-compose.yaml ================================================ version: '3.8' services: ipfs: build: . restart: unless-stopped volumes: - ipfs_path:/data/ipfs - ipfs_fuse:/ipfs - ipns_fuse:/ipns environment: - IPFS_PATH=/data/ipfs ports: # Swarm listens on all interfaces, so is remotely reachable. - 4001:4001/tcp - 4001:4001/udp # The following ports only listen on the loopback interface, so are not remotely reachable by default. # If you want to override these or add more ports, see https://docs.docker.com/compose/extends/ . # API port, which includes admin operations, so you probably don't want this remotely accessible. - 127.0.0.1:5001:5001 # HTTP Gateway - 127.0.0.1:8080:8080 volumes: ipfs_path: ipfs_fuse: ipns_fuse: ================================================ FILE: docs/EARLY_TESTERS.md ================================================ # EARLY TESTERS PROGRAMME ## What is it? The early testers programme allows groups using Kubo in production to self-volunteer to help test `kubo` release candidates to ensure that no regressions that might affect production systems make it into the final release. While we invite the _entire_ community to help test releases, members of the early testers program are expected to participate directly and actively in every release. ## What are the expectations? Members of the early tester program are expected to work closely with us to: * Provide high quality, actionable feedback. * Work directly with us to debug regressions in the release. * Help ensure a rock-solid, timely release. We will ask early testers to participate at two points in the process: * When Kubo enters the second release stage (public beta), early testers will be asked to test Kubo on non-production infrastructure. This may involve things like: - Running integration tests against the release candidate. - Running simulations/benchmarks on the release candidate. - Manually testing the release candidate to check for regressions. * When Kubo enters the third release stage (soft release), early testers will be asked to partially deploy the release candidate to production infrastructure. Release candidates at this stage are expected to be identical to the final release. However, this stage allows the Kubo team to fix any last-minute regressions without cutting an entirely new release. ## Who has signed up? - [ ] Charity Engine (@rytiss, @tristanolive) - [ ] Fission (@bmann) - [ ] Infura (@MichaelMure) - [ ] OrbitDB (@haydenyoung) - [ ] Pinata (@obo20) - [ ] Shipyard (@cewood, @ns4plabs) - [ ] Siderus (@koalalorenzo) - [ ] Textile (@sanderpick) - [ ] @RubenKelevra ## How to sign up? Simply submit a PR to this document by adding your project name and contact. ================================================ FILE: docs/README.md ================================================ # Developer Documentation and Guides If you're looking for User Documentation & Guides, visit [docs.ipfs.tech](https://docs.ipfs.tech/). If you're experiencing an issue with IPFS, please [file an issue](https://github.com/ipfs/kubo/issues/new/choose) in this repository. ## Configuration - [Configuration reference](config.md) - [Datastore configuration](datastores.md) - [Experimental features](experimental-features.md) - [Environment variables](environment-variables.md) ## Running Kubo - [Gateway configuration](gateway.md) - [Delegated routing](delegated-routing.md) - [Content blocking](content-blocking.md) (for public node operators) - [libp2p resource management](libp2p-resource-management.md) - [Mounting IPFS with FUSE](fuse.md) ## Metrics & Monitoring - [Prometheus metrics](metrics.md) - [Telemetry plugin](telemetry.md) - [Provider statistics](provide-stats.md) - [Performance debugging](debug-guide.md) ## Development - **[Developer Guide](developer-guide.md)** - prerequisites, build, test, and contribute - **[AGENTS.md](../AGENTS.md)** - instructions for AI coding agents - Contributing Guidelines [for IPFS projects](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) and for [Go code specifically](https://github.com/ipfs/community/blob/master/CONTRIBUTING_GO.md) - [Building on Windows](windows.md) - [Customizing Kubo](customizing.md) - [Installing plugins](plugins.md) - [Release checklist](releases.md) ## Guides - [Transferring files over IPFS](file-transfer.md) - [How to implement an API client](implement-api-bindings.md) - [HTTP/RPC clients](http-rpc-clients.md) - [Websocket transports](transports.md) - [Command completion](command-completion.md) ## Production - [Reverse proxy setup](production/reverse-proxy.md) ## Specifications - [Repository structure](specifications/repository.md) - [Filesystem datastore](specifications/repository_fs.md) - [Keystore](specifications/keystore.md) ## Examples - [Kubo as a library](examples/kubo-as-a-library/README.md) ================================================ FILE: docs/RELEASE_CHECKLIST.md ================================================ # ✅ Release Checklist (vX.Y.Z[-rcN]) **Release types:** RC (Release Candidate) | FINAL | PATCH ## Prerequisites - [ ] [GPG signature](https://docs.github.com/en/authentication/managing-commit-signature-verification) configured in local git and GitHub - [ ] [Docker](https://docs.docker.com/get-docker/) installed on your system - [ ] [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) installed on your system - [ ] kubo repository cloned locally - [ ] **non-PATCH:** Upgrade Go in CI to latest patch from ## 1. Prepare Release Branch - [ ] Fetch latest changes: `git fetch origin master release` - [ ] Create branch `release-vX.Y.Z` (base from: `master` if Z=0 for new minor/major, `release` if Z>0 for patch) - [ ] **RC1 only:** Switch to `master` branch and prepare for next release cycle: - [ ] Update [version.go](https://github.com/ipfs/kubo/blob/master/version.go) to `vX.Y+1.0-dev` (⚠️ double-check Y+1 is correct) ([example PR](https://github.com/ipfs/kubo/pull/9305)) - [ ] Create `./docs/changelogs/vX.Y+1.md` and add link in [CHANGELOG.md](https://github.com/ipfs/kubo/blob/master/CHANGELOG.md) - [ ] Switch to `release-vX.Y.Z` branch and update [version.go](https://github.com/ipfs/kubo/blob/master/version.go) to `vX.Y.Z(-rcN)` (⚠️ double-check Y matches release) ([example](https://github.com/ipfs/kubo/pull/9394)) - [ ] Create draft PR: `release-vX.Y.Z` → `release` ([example](https://github.com/ipfs/kubo/pull/9306)) - [ ] Cherry-pick commits from `master` into `release-vX.Y.Z`: `git cherry-pick -x ` ([example](https://github.com/ipfs/kubo/pull/10636/commits/033de22e3bc6191dbb024ad6472f5b96b34e3ccf)) - ⚠️ **NOTE:** `-x` flag records original commit SHA for traceability and cleaner merge history - [ ] Verify all CI checks on the PR are passing - [ ] **FINAL only:** Replace `Changelog` and `Contributors` sections in `release-vX.Y.Z` with `./bin/mkreleaselog` stdout (do **NOT** copy stderr) - [ ] **FINAL only:** Merge PR (`release-vX.Y.Z` → `release`) using `Create a merge commit` - ⚠️ do **NOT** use `Squash and merge` nor `Rebase and merge` -- we want the releaser's GPG signature on the merge commit - ⚠️ do **NOT** delete the `release-vX.Y.Z` branch (needed for future patch releases and git history) ## 2. Tag & Publish ### Create Tag ⚠️ **POINT OF NO RETURN:** Once pushed, tags trigger automatic Docker/NPM publishing and are irreversible! If you're making a release for the first time, do pair programming and have the release reviewer verify all commands. - [ ] **RC:** From `release-vX.Y.Z` branch: `git tag -s vX.Y.Z-rcN -m 'Prerelease X.Y.Z-rcN'` - [ ] **FINAL:** After PR merge, from `release` branch: `git tag -s vX.Y.Z -m 'Release X.Y.Z'` - [ ] ⚠️ Verify tag is signed and correct: `git show vX.Y.Z(-rcN)` - [ ] Push tag: `git push origin vX.Y.Z(-rcN)` - ⚠️ do **NOT** use `git push --tags` (pushes all local tags, polluting the repo with noise) - [ ] **STOP:** Wait for [Docker build](https://github.com/ipfs/kubo/actions/workflows/docker-image.yml) to complete before proceeding ### Publish Artifacts > **Parallelism:** Docker and dist.ipfs.tech only depend on the pushed tag and can be started in parallel. > NPM and GitHub Release both depend on dist.ipfs.tech completing first. - [ ] **Docker:** Verify [docker-image CI](https://github.com/ipfs/kubo/actions/workflows/docker-image.yml) passed and image is available on [Docker Hub → tags](https://hub.docker.com/r/ipfs/kubo/tags) - [ ] **dist.ipfs.tech:** Publish to [dist.ipfs.tech](https://dist.ipfs.tech) - [ ] Check out [ipfs/distributions](https://github.com/ipfs/distributions) - [ ] Create branch: `git checkout -b release-kubo-X.Y.Z(-rcN)` - [ ] Verify `.tool-versions` golang matches [Kubo's CI](https://github.com/ipfs/kubo/blob/master/.github/workflows/gotest.yml) `go-version:` (update if needed) - [ ] Run: `./dist.sh add-version kubo vX.Y.Z(-rcN)` ([usage](https://github.com/ipfs/distributions#usage)) - [ ] Create and merge PR (updates `dists/kubo/versions`, **FINAL** also updates `dists/kubo/current` - [example](https://github.com/ipfs/distributions/pull/1125)) - [ ] Wait for [CI workflow](https://github.com/ipfs/distributions/actions/workflows/main.yml) triggered by merge - [ ] Verify release on [dist.ipfs.tech](https://dist.ipfs.tech/#kubo) - [ ] **NPM:** Publish to [NPM](https://www.npmjs.com/package/kubo?activeTab=versions) - [ ] Manually dispatch [Release to npm](https://github.com/ipfs/npm-kubo/actions/workflows/main.yml) workflow if not auto-triggered - [ ] Verify release on [NPM](https://www.npmjs.com/package/kubo?activeTab=versions) - [ ] **GitHub Release:** Publish to [GitHub](https://github.com/ipfs/kubo/releases) - [ ] [Create release](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository#creating-a-release) ([RC example](https://github.com/ipfs/kubo/releases/tag/v0.36.0-rc1), [FINAL example](https://github.com/ipfs/kubo/releases/tag/v0.35.0)) - [ ] Use tag `vX.Y.Z(-rcN)` - [ ] Link to release issue - [ ] **RC:** Link to changelog, check `This is a pre-release` - [ ] **FINAL:** Copy changelog content (without header), do **NOT** check pre-release - [ ] Run [sync-release-assets](https://github.com/ipfs/kubo/actions/workflows/sync-release-assets.yml) workflow (requires dist.ipfs.tech) - [ ] Verify assets are attached to the GitHub release ## 3. Post-Release ### Technical Tasks - [ ] **FINAL only:** Merge `release` → `master` - [ ] Create branch `merge-release-vX.Y.Z` from `release` - [ ] Merge `master` to `merge-release-vX.Y.Z` first, and resolve conflict in `version.go` - ⚠️ **NOTE:** keep the `-dev` version from `master` in [version.go](https://github.com/ipfs/kubo/blob/master/version.go), discard version from `release` - [ ] Create and merge PR from `merge-release-vX.Y.Z` to `master` using `Create a merge commit` - ⚠️ do **NOT** use `Squash and merge` nor `Rebase and merge` -- only `Create a merge commit` preserves commit history and audit trail of what was merged where - [ ] Update [ipshipyard/waterworks-infra](https://github.com/ipshipyard/waterworks-infra) - [ ] Update Kubo staging environment ([Running Kubo tests on staging](https://www.notion.so/Running-Kubo-tests-on-staging-488578bb46154f9bad982e4205621af8)) - [ ] **RC:** Test last release against current RC - [ ] **FINAL:** Latest release on both boxes - [ ] **FINAL:** Update collab cluster boxes to the tagged release - [ ] **FINAL:** Update libp2p bootstrappers to the tagged release - [ ] Update [ipfs-desktop](https://github.com/ipfs/ipfs-desktop) - [ ] Create PR updating kubo version in `package.json` and `package-lock.json` - [ ] Smoke test with [IPFS Companion Browser Extension](https://docs.ipfs.tech/install/ipfs-companion/) against the PR build - [ ] **FINAL:** Merge PR and ship new ipfs-desktop release - [ ] **FINAL only:** Update [docs.ipfs.tech](https://docs.ipfs.tech/): run [update-on-new-ipfs-tag.yml](https://github.com/ipfs/ipfs-docs/actions/workflows/update-on-new-ipfs-tag.yml) workflow and merge the PR ### Promotion - [ ] Create [IPFS Discourse](https://discuss.ipfs.tech) topic ([RC example](https://discuss.ipfs.tech/t/kubo-v0-38-0-rc2-is-out/19772), [FINAL example](https://discuss.ipfs.tech/t/kubo-v0-38-0-is-out/19795)) - [ ] Title: `Kubo vX.Y.Z(-rcN) is out!`, tag: `kubo` - [ ] Use title as heading (`##`) in description - [ ] Include: GitHub release link, IPNS binaries, docker pull command, release notes - [ ] Pin topic globally (make banner if no existing banner) - [ ] Verify bot posted to [#ipfs-chatter](https://discord.com/channels/669268347736686612/669268347736686615) (Discord) or [#ipfs-chatter:ipfs.io](https://matrix.to/#/#ipfs-chatter:ipfs.io) (Matrix) - [ ] **RC only:** Comment on release issue mentioning early testers ([example](https://github.com/ipfs/kubo/issues/9319#issuecomment-1311002478)) - [ ] **FINAL only:** Comment on release issue with link ([example](https://github.com/ipfs/kubo/issues/9417#issuecomment-1400740975)) - [ ] **FINAL only:** Create [blog.ipfs.tech](https://blog.ipfs.tech) entry ([example](https://github.com/ipfs/ipfs-blog/commit/32040d1e90279f21bad56b924fe4710bba5ba043)) - [ ] **FINAL non-PATCH:** (optional) Post on social media ([bsky](https://bsky.app/profile/ipshipyard.com/post/3ltxcsrbn5s2k), [x.com](https://x.com/ipshipyard/status/1944867893226635603), [Reddit](https://www.reddit.com/r/ipfs/comments/1lzy6ze/release_v0360_ipfskubo/)) ### Final Steps - [ ] **FINAL non-PATCH:** Create dependency update PR - [ ] Review direct dependencies from root `go.mod` (⚠️ do **NOT** run `go get -u` as it will upgrade indirect dependencies which may cause problems) - [ ] Run `make mod_tidy` - [ ] Create PR with `go.mod` and `go.sum` updates - [ ] Add PR to next release milestone - [ ] **FINAL non-PATCH:** Create next release issue ([example](https://github.com/ipfs/kubo/issues/10816)) - [ ] **FINAL only:** Close release issue ================================================ FILE: docs/RELEASE_ISSUE_TEMPLATE.md ================================================ # Items to do upon creating the release issue - [ ] Fill in the Meta section - [ ] Assign the issue to the release owner and reviewer. - [ ] Name the issue "Release vX.Y.Z" - [ ] Set the proper values for X.Y.Z - [ ] Pin the issue # Meta * Release owner: @who * Release reviewer: @who * Expected RC date: week of YYYY-MM-DD * 🚢 Expected final release date: YYYY-MM-DD * Release PR: * Accompanying PR for improving the release process: ([example](https://github.com/ipfs/kubo/pull/9391)) * Changelog: https://github.com/ipfs/kubo/blob/master/docs/changelogs/vX.Y.md # Items In Scope ## Required ## Nice To Have (Optional) ================================================ FILE: docs/add-code-flow.md ================================================ # How `ipfs add` Works This document explains what happens when you run `ipfs add` to import files into IPFS. Understanding this flow helps when debugging, optimizing imports, or building applications on top of IPFS. - [The Big Picture](#the-big-picture) - [Try It Yourself](#try-it-yourself) - [Step by Step](#step-by-step) - [Step 1: Chunking](#step-1-chunking) - [Step 2: Building the DAG](#step-2-building-the-dag) - [Step 3: Storing Blocks](#step-3-storing-blocks) - [Step 4: Pinning](#step-4-pinning) - [Alternative: Organizing with MFS](#alternative-organizing-with-mfs) - [Options](#options) - [UnixFS Format](#unixfs-format) - [Code Architecture](#code-architecture) - [Key Files](#key-files) - [The Adder](#the-adder) - [Further Reading](#further-reading) ## The Big Picture When you add a file to IPFS, three main things happen: 1. **Chunking** - The file is split into smaller pieces 2. **DAG Building** - Those pieces are organized into a tree structure (a [Merkle DAG](https://docs.ipfs.tech/concepts/merkle-dag/)) 3. **Pinning** - The root of the tree is pinned so it persists in your local node The result is a Content Identifier (CID) - a hash that uniquely identifies your content and can be used to retrieve it from anywhere in the IPFS network. ```mermaid flowchart LR A["Your File
(bytes)"] --> B["Chunker
(split data)"] B --> C["DAG Builder
(tree)"] C --> D["CID
(hash)"] ``` ## Try It Yourself ```bash # Add a simple file echo "Hello World" > hello.txt ipfs add hello.txt # added QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u hello.txt # See what's inside ipfs cat QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u # Hello World # View the DAG structure ipfs dag get QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u ``` ## Step by Step ### Step 1: Chunking Big files are split into chunks because: - Large files need to be broken down for efficient transfer - Identical chunks across files are stored only once (deduplication) - You can fetch parts of a file without downloading the whole thing **Chunking strategies** (set with `--chunker`): | Strategy | Description | Best For | |----------|-------------|----------| | `size-N` | Fixed size chunks | General use | | `rabin` | Content-defined chunks using rolling hash | Deduplication across similar files | | `buzhash` | Alternative content-defined chunking | Similar to rabin | See `ipfs add --help` for current defaults, or [Import](config.md#import) for making them permanent. Content-defined chunking (rabin/buzhash) finds natural boundaries in the data. This means if you edit the middle of a file, only the changed chunks need to be re-stored - the rest can be deduplicated. ### Step 2: Building the DAG Each chunk becomes a leaf node in a tree. If a file has many chunks, intermediate nodes group them together. This creates a Merkle DAG (Directed Acyclic Graph) where: - Each node is identified by a hash of its contents - Parent nodes contain links (hashes) to their children - The root node's hash becomes the file's CID **Layout strategies**: **Balanced layout** (default): ```mermaid graph TD Root --> Node1[Node] Root --> Node2[Node] Node1 --> Leaf1[Leaf] Node1 --> Leaf2[Leaf] Node2 --> Leaf3[Leaf] ``` All leaves at similar depth. Good for random access - you can jump to any part of the file efficiently. **Trickle layout** (`--trickle`): ```mermaid graph TD Root --> Leaf1[Leaf] Root --> Node1[Node] Root --> Node2[Node] Node1 --> Leaf2[Leaf] Node2 --> Leaf3[Leaf] ``` Leaves added progressively. Good for streaming - you can start reading before the whole file is added. ### Step 3: Storing Blocks As the DAG is built, each node is stored in the blockstore: - **Normal mode**: Data is copied into IPFS's internal storage (`~/.ipfs/blocks/`) - **Filestore mode** (`--nocopy`): Only references to the original file are stored (saves disk space but the original file must remain in place) ### Step 4: Pinning By default, added content is pinned (`ipfs add --pin=true`). This tells your IPFS node to keep this data - without pinning, content may eventually be removed to free up space. ### Alternative: Organizing with MFS Instead of pinning, you can use the [Mutable File System (MFS)](https://docs.ipfs.tech/concepts/file-systems/#mutable-file-system-mfs) to organize content using familiar paths like `/photos/vacation.jpg` instead of raw CIDs: ```bash # Add directly to MFS path ipfs add --to-files=/backups/ myfile.txt # Or copy an existing CID into MFS ipfs files cp /ipfs/QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u /docs/hello.txt ``` Content in MFS is implicitly pinned and stays organized across node restarts. ## Options Run `ipfs add --help` to see all available options for controlling chunking, DAG layout, CID format, pinning behavior, and more. ## UnixFS Format IPFS uses [UnixFS](https://specs.ipfs.tech/unixfs/) to represent files and directories. UnixFS is an abstraction layer that: - Gives names to raw data blobs (so you can have `/foo/bar.txt` instead of just hashes) - Represents directories as lists of named links to other nodes - Organizes large files as trees of smaller chunks - Makes these structures cryptographically verifiable - any tampering is detectable because it would change the hashes With `--raw-leaves`, leaf nodes store raw data without the UnixFS wrapper. This is more efficient and is the default when using CIDv1. ## Code Architecture The add flow spans several layers: ```mermaid flowchart TD subgraph CLI ["CLI Layer (kubo)"] A["core/commands/add.go
parses flags, shows progress"] end subgraph API ["CoreAPI Layer (kubo)"] B["core/coreapi/unixfs.go
UnixfsAPI.Add() entry point"] end subgraph Adder ["Adder (kubo)"] C["core/coreunix/add.go
orchestrates chunking, DAG building, MFS, pinning"] end subgraph Boxo ["boxo libraries"] D["chunker/ - splits data into chunks"] E["ipld/unixfs/ - DAG layout and UnixFS format"] F["mfs/ - mutable filesystem abstraction"] G["pinning/ - pin management"] H["blockstore/ - block storage"] end A --> B --> C --> Boxo ``` ### Key Files | Component | Location | |-----------|----------| | CLI command | `core/commands/add.go` | | API implementation | `core/coreapi/unixfs.go` | | Adder logic | `core/coreunix/add.go` | | Chunking | [boxo/chunker](https://github.com/ipfs/boxo/tree/main/chunker) | | DAG layouts | [boxo/ipld/unixfs/importer](https://github.com/ipfs/boxo/tree/main/ipld/unixfs/importer) | | MFS | [boxo/mfs](https://github.com/ipfs/boxo/tree/main/mfs) | | Pinning | [boxo/pinning/pinner](https://github.com/ipfs/boxo/tree/main/pinning/pinner) | ### The Adder The `Adder` type in `core/coreunix/add.go` is the workhorse. It: 1. **Creates an MFS root** - temporary in-memory filesystem for building the DAG 2. **Processes files recursively** - chunks each file and builds DAG nodes 3. **Commits to blockstore** - persists all blocks 4. **Pins the result** - keeps content from being removed 5. **Returns the root CID** Key methods: - `AddAllAndPin()` - main entry point - `addFileNode()` - handles a single file or directory - `add()` - chunks data and builds the DAG using boxo's layout builders ## Further Reading - [UnixFS specification](https://specs.ipfs.tech/unixfs/) - [IPLD and Merkle DAGs](https://docs.ipfs.tech/concepts/merkle-dag/) - [Pinning](https://docs.ipfs.tech/concepts/persistence/) - [MFS (Mutable File System)](https://docs.ipfs.tech/concepts/file-systems/#mutable-file-system-mfs) ================================================ FILE: docs/changelogs/v0.10.md ================================================ # go-ipfs changelog v0.10 ## v0.10.0 2021-09-30 We're happy to announce go-ipfs 0.10.0. This release brings some big changes to the IPLD internals of go-ipfs that make working with non-UnixFS DAGs easier than ever. There are also a variety of new commands and configuration options available. As usual, this release includes important fixes, some of which may be critical for security. Unless the fix addresses a bug being exploited in the wild, the fix will _not_ be called out in the release notes. Please make sure to update ASAP. See our [release process](https://github.com/ipfs/go-ipfs/tree/master/docs/releases.md#security-fix-policy) for details. ### 🛠 TLDR: BREAKING CHANGES - `ipfs dag get` - default output changed to [`dag-json`](https://ipld.io/specs/codecs/dag-json/spec/) - dag-pb (e.g. unixfs) field names changed - impacts userland code that works with `dag-pb` objects returned by `dag get` - no longer emits an additional new-line character at the end of the data output - `ipfs dag put` - defaults changed to reduce ambiguity and surprises: input is now assumed to be [`dag-json`](https://ipld.io/specs/codecs/dag-json/spec/), and data is serialized to [`dag-cbor`](https://ipld.io/specs/codecs/dag-cbor/spec/) at rest. - `--format` and `--input-enc` were removed and replaced with `--store-codec` and `--input-codec` - codec names now match the ones defined in the [multicodec table](https://github.com/multiformats/multicodec/blob/master/table.csv) - dag-pb (e.g. unixfs) field names changed - impacts userland code that works with `dag-pb` objects stored via `dag put` Keep reading to learn more details. ### 🔦 Highlights #### 🌲 IPLD Levels Up The handling of data serialization as well as many aspects of DAG traversal and pathing have been migrated from older libraries, including [go-merkledag](https://github.com/ipfs/go-merkledag) and [go-ipld-format](https://github.com/ipfs/go-ipld-format) to the new **[go-ipld-prime](https://github.com/ipld/go-ipld-prime)** library and its components. This allows us to use many of the newer tools afforded by go-ipld-prime, stricter and more uniform codec implementations, support for additional (pluggable) codecs, and some minor performance improvements. This is significant refactor of a core component that touches many parts of IPFS, and does come with some **breaking changes**: * **IPLD plugins**: * The `PluginIPLD` interface has been changed to utilize go-ipld-prime. There is a demonstration of the change in the [bundled git plugin](./plugin/plugins/git/). * **The semantics of `dag put` and `dag get` change**: * `dag get` now takes the `output-codec` option which accepts a [multicodec](https://docs.ipfs.tech/concepts/glossary/#multicodec) name used to encode the output. By default this is `dag-json`, which is a strict and deterministic subset of JSON created by the IPLD team. Users may notice differences from the previously plain Go JSON output, particularly where bytes are concerned which are now encoded using a form similar to CIDs: `{"/":{"bytes":"unpadded-base64-bytes"}}` rather than the previously Go-specific plain padded base64 string. See the [dag-json specification](https://ipld.io/specs/codecs/dag-json/spec/) for an explanation of these forms. * `dag get` no longer prints an additional new-line character at the end of the encoded block output. This means that the output as presented by `dag get` are the exact bytes of the requested node. A round-trip of such bytes back in through `dag put` using the same codec should result in the same CID. * `dag put` uses the `input-codec` option to specify the multicodec name of the format data is being provided in, and the `store-codec` option to specify the multicodec name of the format the data should be stored in at rest. These formerly defaulted to `json` and `cbor` respectively. They now default to `dag-json` and `dag-cbor` respectively but may be changed to any supported codec (bundled or loaded via plugin) by its [multicodec name](https://github.com/multiformats/multicodec/blob/master/table.csv). * The `json` and `cbor` multicodec names (as used by `input-enc` and `format` options) are now no longer aliases for `dag-json` and `dag-cbor` respectively. Instead, they now refer to their proper [multicodec](https://github.com/multiformats/multicodec/blob/master/table.csv) types. `cbor` refers to a plain CBOR format, which will not encode CIDs and does not have strict deterministic encoding rules. `json` is a plain JSON format, which also won't encode CIDs and will encode bytes in the Go-specific padded base64 string format rather than the dag-json method of byte encoding. See https://ipld.io/specs/codecs/ for more information on IPLD codecs. * `protobuf` is no longer used as the codec name for `dag-pb` * The codec name `raw` is used to mean Bytes in the [IPLD Data Model](https://github.com/ipld/specs/blob/master/data-model-layer/data-model.md#bytes-kind) * **UnixFS refactor**. The **dag-pb codec**, which is used to encode UnixFS data for IPFS, is now represented through the `dag` API in a form that mirrors the protobuf schema used to define the binary format. This unifies the implementations and specification of dag-pb across the IPLD and IPFS stacks. Previously, additional layers of code for file and directory handling within IPFS between protobuf serialization and UnixFS obscured the protobuf representation. Much of this code has now been replaced and there are fewer layers of transformation. This means that interacting with dag-pb data via the `dag` API will use different forms: * Previously, using `dag get` on a dag-pb block would present the block serialized as JSON as `{"data":"padded-base64-bytes","links":[{"Name":"foo","Size":100,"Cid":{"/":"Qm..."}},...]}`. * Now, the dag-pb data with dag-json codec for output will be serialized using the data model from the [dag-pb specification](https://ipld.io/specs/codecs/dag-pb/spec/): `{"Data":{"/":{"bytes":"unpadded-base64-bytes"}},"Links":[{"Name":"foo","Tsize":100,"Hash":{"/":"Qm..."}},...]}`. Aside from the change in byte formatting, most field names have changed: `data` → `Data`, `links` → `Links`, `Size` → `Tsize`, `Cid` → `Hash`. Note that this output can be changed now using the `output-codec` option to specify an alternative codec. * Similarly, using `dag put` and a `store-codec` option of `dag-pb` now requires that the input conform to this dag-pb specified form. Previously, input using `{"data":"...","links":[...]}` was accepted, now it must be `{"Data":"...","Links":[...]}`. * Previously it was not possible to use paths to navigate to any of these properties of a dag-pb node, the only possible paths were named links, e.g. `dag get QmFoo/NamedLink` where `NamedLink` was one of the links whose name was `NamedLink`. This functionality remains the same, but by prefixing the path with `/ipld/` we enter data model pathing semantics and can `dag get /ipld/QmFoo/Links/0/Hash` to navigate to links or `/ipld/QmFoo/Data` to simply retrieve the data section of the node, for example. * ℹ See the [dag-pb specification](https://ipld.io/specs/codecs/dag-pb/) for details on the codec and its data model representation. * ℹ See this [detailed write-up](https://github.com/ipld/ipld/blob/master/design/tricky-choices/dag-pb-forms-impl-and-use.md) for further background on these changes. #### Ⓜ Multibase Command go-ipfs now provides utility commands for working with [multibase](https://docs.ipfs.tech/concepts/glossary/#multibase): ```console $ echo -n hello | ipfs multibase encode -b base16 > file-mbase16 $ cat file-mbase16 f68656c6c6f $ ipfs multibase decode file-mbase16 hello $ cat file-mbase16 | ipfs multibase decode hello $ ipfs multibase transcode -b base2 file-mbase16 00110100001100101011011000110110001101111 ``` See `ipfs multibase --help` for more examples. #### 🔨 Bitswap now supports greater configurability This release adds an [`Internal` section](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#internal) to the configuration file that is designed to help advanced users optimize their setups without needing a custom binary. The `Internal` section is not guaranteed to be the same from release to release and may not be covered by migrations. If you use the `Internal` section you should be making sure to check the config documentation between releases for any changes. #### 🐚 Programmatic shell completions command `ipfs commands completion bash` will generate a bash completion script for go-ipfs commands #### 📜 Profile collection command Performance profiles can now be collected using `ipfs diag profile`. If you need to do some debugging or have an issue to submit the collected profiles are very useful to have around. #### 🍎 Mac OS notarized binaries The go-ipfs and related migration binaries (for both Intel and Apple Silicon) are now signed and notarized to make Mac OS installation easier. #### 👨‍👩‍👦 Improved MDNS There is a completed implementation of the revised libp2p MDNS spec. This should result in better MDNS discovery and better local/offline operation as a result. #### 🚗 CAR import statistics `dag import` command now supports `--stats` option which will include the number of imported blocks and their total size in the output. #### 🕸 Peering command This release adds `swarm peering` command for easy management of the peering subsystem. Peer in the peering subsystem is maintained to be connected at all times, and gets reconnected on disconnect with a back-off. See `ipfs swarm peering --help` for more details. ### Changelog - github.com/ipfs/go-ipfs: - fuse: load unixfs adls as their dagpb substrates - enable the legacy mDNS implementation - test: add dag get --output-codec test - change ipfs dag get flag name from format to output-codec - test: check behavior of loading UnixFS sharded directories with missing shards - remove dag put option shortcuts - change names of ipfs dag put flags to make changes clearer - feat: dag import --stats (#8237) ([ipfs/go-ipfs#8237](https://github.com/ipfs/go-ipfs/pull/8237)) - feat: ipfs-webui v2.13.0 (#8430) ([ipfs/go-ipfs#8430](https://github.com/ipfs/go-ipfs/pull/8430)) - feat(cli): add daemon option --agent-version-suffix (#8419) ([ipfs/go-ipfs#8419](https://github.com/ipfs/go-ipfs/pull/8419)) - feat: multibase transcode command (#8403) ([ipfs/go-ipfs#8403](https://github.com/ipfs/go-ipfs/pull/8403)) - fix: take the lock while listing peers - feature: 'ipfs swarm peering' command (#8147) ([ipfs/go-ipfs#8147](https://github.com/ipfs/go-ipfs/pull/8147)) - fix(sharness): add extra check in flush=false in files write - chore: update IPFS Desktop testing steps (#8393) ([ipfs/go-ipfs#8393](https://github.com/ipfs/go-ipfs/pull/8393)) - add more buttons; remove some sections covered in the docs; general cleanup - Cosmetic fixups in examples (#8325) ([ipfs/go-ipfs#8325](https://github.com/ipfs/go-ipfs/pull/8325)) - perf: use performance-enhancing FUSE mount options - ci: publish Docker images for bifrost-* branches - chore: add comments to peerlog plugin about being unsupported - test: add unit tests for peerlog config parsing - ci: preload peerlog plugin, disable by default - fix(mkreleaselog): specify the parent commit when diffing - update go-libp2p to v0.15.0-rc.1 ([ipfs/go-ipfs#8354](https://github.com/ipfs/go-ipfs/pull/8354)) - feat: add 'ipfs multibase' commands (#8180) ([ipfs/go-ipfs#8180](https://github.com/ipfs/go-ipfs/pull/8180)) - support bitswap configurability (#8268) ([ipfs/go-ipfs#8268](https://github.com/ipfs/go-ipfs/pull/8268)) - IPLD Prime In IPFS: Target Merge Branch (#7976) ([ipfs/go-ipfs#7976](https://github.com/ipfs/go-ipfs/pull/7976)) - ci: upgrade to Go 1.16.7 on CI ([ipfs/go-ipfs#8324](https://github.com/ipfs/go-ipfs/pull/8324)) - Add flag to create parent directories in files cp command ([ipfs/go-ipfs#8340](https://github.com/ipfs/go-ipfs/pull/8340)) - fix: avoid out of bounds error when rendering short hashes ([ipfs/go-ipfs#8318](https://github.com/ipfs/go-ipfs/pull/8318)) - fix: remove some deprecated calls ([ipfs/go-ipfs#8296](https://github.com/ipfs/go-ipfs/pull/8296)) - perf: set an appropriate capacity ([ipfs/go-ipfs#8244](https://github.com/ipfs/go-ipfs/pull/8244)) - Fix: Use a pointer type on IpfsNode.Peering ([ipfs/go-ipfs#8331](https://github.com/ipfs/go-ipfs/pull/8331)) - fix: macos notarized fs-repo-migrations (#8333) ([ipfs/go-ipfs#8333](https://github.com/ipfs/go-ipfs/pull/8333)) - README.md: Add MacPorts to install section ([ipfs/go-ipfs#8220](https://github.com/ipfs/go-ipfs/pull/8220)) - feat: register first block metric by default ([ipfs/go-ipfs#8332](https://github.com/ipfs/go-ipfs/pull/8332)) - Build a go-ipfs:extras docker image ([ipfs/go-ipfs#8142](https://github.com/ipfs/go-ipfs/pull/8142)) - fix/go-ipfs-as-a-library ([ipfs/go-ipfs#8266](https://github.com/ipfs/go-ipfs/pull/8266)) - Expose additional migration APIs (#8153) ([ipfs/go-ipfs#8153](https://github.com/ipfs/go-ipfs/pull/8153)) - point ipfs to pinner that syncs on every pin (#8231) ([ipfs/go-ipfs#8231](https://github.com/ipfs/go-ipfs/pull/8231)) - docs: chocolatey package name - Disambiguate online/offline naming in sharness tests ([ipfs/go-ipfs#8254](https://github.com/ipfs/go-ipfs/pull/8254)) - Rename DOCKER_HOST to TEST_DOCKER_HOST to avoid conflicts ([ipfs/go-ipfs#8283](https://github.com/ipfs/go-ipfs/pull/8283)) - feat: add an "ipfs diag profile" command ([ipfs/go-ipfs#8291](https://github.com/ipfs/go-ipfs/pull/8291)) - Merge branch 'release' - feat: improve mkreleaslog ([ipfs/go-ipfs#8290](https://github.com/ipfs/go-ipfs/pull/8290)) - Add test with expected failure for #3503 ([ipfs/go-ipfs#8280](https://github.com/ipfs/go-ipfs/pull/8280)) - Create PATCH_RELEASE_TEMPLATE.md - fix document error ([ipfs/go-ipfs#8271](https://github.com/ipfs/go-ipfs/pull/8271)) - feat: webui v2.12.4 - programmatic shell completions ([ipfs/go-ipfs#8043](https://github.com/ipfs/go-ipfs/pull/8043)) - test: gateway response for bafkqaaa - doc(README): update chat links (and misc fixes) ([ipfs/go-ipfs#8222](https://github.com/ipfs/go-ipfs/pull/8222)) - link to the actual doc (#8126) ([ipfs/go-ipfs#8126](https://github.com/ipfs/go-ipfs/pull/8126)) - Improve peer hints for pin remote add (#8143) ([ipfs/go-ipfs#8143](https://github.com/ipfs/go-ipfs/pull/8143)) - fix(mkreleaselog): support multiple commit authors ([ipfs/go-ipfs#8214](https://github.com/ipfs/go-ipfs/pull/8214)) - fix(mkreleaselog): handle commit 0 ([ipfs/go-ipfs#8121](https://github.com/ipfs/go-ipfs/pull/8121)) - bump snap to build with Go 1.16 - chore: update CHANGELOG - chore: switch tar-utils dep to ipfs org - feat: print error on bootstrap failure ([ipfs/go-ipfs#8166](https://github.com/ipfs/go-ipfs/pull/8166)) - fix: typo in migration error - refactor: improved humanNumber and humanSI - feat: humanized durations in stat provide - feat: humanized numbers in stat provide - feat: add a text output encoding for the stats provide command - fix: webui-2.12.3 - refactor(pinmfs): log error if pre-existing pin failed (#8056) ([ipfs/go-ipfs#8056](https://github.com/ipfs/go-ipfs/pull/8056)) - config.md: fix typos/improve wording ([ipfs/go-ipfs#8031](https://github.com/ipfs/go-ipfs/pull/8031)) - fix(peering_test) : Fix the peering_test to check the connection explicitly added ([ipfs/go-ipfs#8140](https://github.com/ipfs/go-ipfs/pull/8140)) - build: ignore generated files in changelog ([ipfs/go-ipfs#7712](https://github.com/ipfs/go-ipfs/pull/7712)) - update version to 0.10.0-dev ([ipfs/go-ipfs#8136](https://github.com/ipfs/go-ipfs/pull/8136)) - github.com/ipfs/go-bitswap (v0.3.4 -> v0.4.0): - More stats, knobs and tunings (#514) ([ipfs/go-bitswap#514](https://github.com/ipfs/go-bitswap/pull/514)) - fix: fix a map access race condition in the want index ([ipfs/go-bitswap#523](https://github.com/ipfs/go-bitswap/pull/523)) - fix: make blockstore cancel test less timing dependent ([ipfs/go-bitswap#507](https://github.com/ipfs/go-bitswap/pull/507)) - fix(decision): fix a datarace on disconnect ([ipfs/go-bitswap#508](https://github.com/ipfs/go-bitswap/pull/508)) - optimize the lookup which peers are waiting for a given block ([ipfs/go-bitswap#486](https://github.com/ipfs/go-bitswap/pull/486)) - fix: hold the task worker lock when starting task workers ([ipfs/go-bitswap#504](https://github.com/ipfs/go-bitswap/pull/504)) - fix: Nil dereference while using SetSendDontHaves ([ipfs/go-bitswap#488](https://github.com/ipfs/go-bitswap/pull/488)) - Fix flaky tests in message queue ([ipfs/go-bitswap#497](https://github.com/ipfs/go-bitswap/pull/497)) - Fix flaky DontHaveTimeoutManger tests ([ipfs/go-bitswap#495](https://github.com/ipfs/go-bitswap/pull/495)) - sync: update CI config files ([ipfs/go-bitswap#485](https://github.com/ipfs/go-bitswap/pull/485)) - github.com/ipfs/go-blockservice (v0.1.4 -> v0.1.7): - update go-bitswap to v0.3.4 ([ipfs/go-blockservice#78](https://github.com/ipfs/go-blockservice/pull/78)) - fix staticcheck ([ipfs/go-blockservice#75](https://github.com/ipfs/go-blockservice/pull/75)) - fix: handle missing session exchange in Session ([ipfs/go-blockservice#73](https://github.com/ipfs/go-blockservice/pull/73)) - github.com/ipfs/go-datastore (v0.4.5 -> v0.4.6): - sync: update CI config files ([ipfs/go-datastore#175](https://github.com/ipfs/go-datastore/pull/175)) - speedup tests ([ipfs/go-datastore#177](https://github.com/ipfs/go-datastore/pull/177)) - test: reduce element count when the race detector is enabled ([ipfs/go-datastore#176](https://github.com/ipfs/go-datastore/pull/176)) - fix staticcheck ([ipfs/go-datastore#173](https://github.com/ipfs/go-datastore/pull/173)) - remove Makefile ([ipfs/go-datastore#172](https://github.com/ipfs/go-datastore/pull/172)) - github.com/ipfs/go-ds-badger (v0.2.6 -> v0.2.7): - Log start and end of GC rounds ([ipfs/go-ds-badger#115](https://github.com/ipfs/go-ds-badger/pull/115)) - github.com/ipfs/go-fs-lock (v0.0.6 -> v0.0.7): - chore: update log ([ipfs/go-fs-lock#24](https://github.com/ipfs/go-fs-lock/pull/24)) - sync: update CI config files ([ipfs/go-fs-lock#21](https://github.com/ipfs/go-fs-lock/pull/21)) - fix TestLockedByOthers on Windows ([ipfs/go-fs-lock#19](https://github.com/ipfs/go-fs-lock/pull/19)) - github.com/ipfs/go-ipfs-config (v0.14.0 -> v0.16.0): - feat: add Internal and Internal.Bitswap config options - feat: add an OptionalInteger type - fix: make sure the Priority type properly implements the JSON marshal/unmarshal interfaces - fix: remove deprecated calls ([ipfs/go-ipfs-config#138](https://github.com/ipfs/go-ipfs-config/pull/138)) - sync: update CI config files ([ipfs/go-ipfs-config#132](https://github.com/ipfs/go-ipfs-config/pull/132)) - remove period, fix staticcheck ([ipfs/go-ipfs-config#131](https://github.com/ipfs/go-ipfs-config/pull/131)) - github.com/ipfs/go-ipfs-pinner (v0.1.1 -> v0.1.2): - Fix/minimize rebuild (#15) ([ipfs/go-ipfs-pinner#15](https://github.com/ipfs/go-ipfs-pinner/pull/15)) - Define ErrNotPinned alongside the Pinner interface - fix staticcheck ([ipfs/go-ipfs-pinner#11](https://github.com/ipfs/go-ipfs-pinner/pull/11)) - fix: remove the rest of the pb backed pinner ([ipfs/go-ipfs-pinner#9](https://github.com/ipfs/go-ipfs-pinner/pull/9)) - Remove old ipldpinner that has been replaced by dspinner ([ipfs/go-ipfs-pinner#7](https://github.com/ipfs/go-ipfs-pinner/pull/7)) - optimize CheckIfPinned ([ipfs/go-ipfs-pinner#6](https://github.com/ipfs/go-ipfs-pinner/pull/6)) - github.com/ipfs/go-ipfs-provider (v0.5.1 -> v0.6.1): - Update to IPLD Prime (#32) ([ipfs/go-ipfs-provider#32](https://github.com/ipfs/go-ipfs-provider/pull/32)) - github.com/ipfs/go-ipld-git (v0.0.4 -> v0.1.1): - return ErrUnexpectedEOF when Decode input is too short - Update go-ipld-git to a go-ipld-prime codec (#46) ([ipfs/go-ipld-git#46](https://github.com/ipfs/go-ipld-git/pull/46)) - fix staticcheck ([ipfs/go-ipld-git#49](https://github.com/ipfs/go-ipld-git/pull/49)) - change WriteTo to the standard signature ([ipfs/go-ipld-git#47](https://github.com/ipfs/go-ipld-git/pull/47)) - don't copy mutexes ([ipfs/go-ipld-git#48](https://github.com/ipfs/go-ipld-git/pull/48)) - github.com/ipfs/go-ipns (v0.1.0 -> v0.1.2): - fix: remove deprecated calls ([ipfs/go-ipns#30](https://github.com/ipfs/go-ipns/pull/30)) - remove Makefile ([ipfs/go-ipns#27](https://github.com/ipfs/go-ipns/pull/27)) - github.com/ipfs/go-log/v2 (v2.1.3 -> v2.3.0): - Stop defaulting to color output on non-TTY ([ipfs/go-log#116](https://github.com/ipfs/go-log/pull/116)) - feat: add ability to use custom zap core ([ipfs/go-log#114](https://github.com/ipfs/go-log/pull/114)) - fix staticcheck ([ipfs/go-log#112](https://github.com/ipfs/go-log/pull/112)) - test: fix flaky label test ([ipfs/go-log#111](https://github.com/ipfs/go-log/pull/111)) - per-subsystem log-levels ([ipfs/go-log#109](https://github.com/ipfs/go-log/pull/109)) - fix: don't panic on invalid log labels ([ipfs/go-log#110](https://github.com/ipfs/go-log/pull/110)) - github.com/ipfs/go-merkledag (v0.3.2 -> v0.4.0): - Use IPLD-prime: target merge branch ([ipfs/go-merkledag#67](https://github.com/ipfs/go-merkledag/pull/67)) - sync: update CI config files ([ipfs/go-merkledag#70](https://github.com/ipfs/go-merkledag/pull/70)) - staticcheck ([ipfs/go-merkledag#69](https://github.com/ipfs/go-merkledag/pull/69)) - Fix bug in dagutils MergeDiffs. (#59) ([ipfs/go-merkledag#59](https://github.com/ipfs/go-merkledag/pull/59)) - chore: add tests to verify allowable data layouts ([ipfs/go-merkledag#58](https://github.com/ipfs/go-merkledag/pull/58)) - github.com/ipfs/go-namesys (v0.3.0 -> v0.3.1): - fix: remove deprecated call to pk.Bytes ([ipfs/go-namesys#19](https://github.com/ipfs/go-namesys/pull/19)) - github.com/ipfs/go-path (v0.0.9 -> v0.1.2): - fix: give one minute timeouts to function calls instead of block retrievals ([ipfs/go-path#44](https://github.com/ipfs/go-path/pull/44)) - IPLD Prime In IPFS: Target Merge Branch (#36) ([ipfs/go-path#36](https://github.com/ipfs/go-path/pull/36)) - remove Makefile ([ipfs/go-path#40](https://github.com/ipfs/go-path/pull/40)) - sync: update CI config files ([ipfs/go-path#39](https://github.com/ipfs/go-path/pull/39)) - github.com/ipfs/go-peertaskqueue (v0.2.0 -> v0.4.0): - add stats - Have a configurable maximum active work per peer ([ipfs/go-peertaskqueue#10](https://github.com/ipfs/go-peertaskqueue/pull/10)) - sync: update CI config files ([ipfs/go-peertaskqueue#13](https://github.com/ipfs/go-peertaskqueue/pull/13)) - fix staticcheck ([ipfs/go-peertaskqueue#12](https://github.com/ipfs/go-peertaskqueue/pull/12)) - fix go vet ([ipfs/go-peertaskqueue#11](https://github.com/ipfs/go-peertaskqueue/pull/11)) - github.com/ipfs/go-unixfsnode (null -> v1.1.3): - make UnixFSHAMTShard implement the ADL interface (#11) ([ipfs/go-unixfsnode#11](https://github.com/ipfs/go-unixfsnode/pull/11)) - github.com/ipfs/interface-go-ipfs-core (v0.4.0 -> v0.5.1): - IPLD In IPFS: Target Merge Branch (#67) ([ipfs/interface-go-ipfs-core#67](https://github.com/ipfs/interface-go-ipfs-core/pull/67)) - fix staticcheck ([ipfs/interface-go-ipfs-core#72](https://github.com/ipfs/interface-go-ipfs-core/pull/72)) - remove Makefile ([ipfs/interface-go-ipfs-core#70](https://github.com/ipfs/interface-go-ipfs-core/pull/70)) - github.com/ipld/go-codec-dagpb (v1.2.0 -> v1.3.0): - fix staticcheck warnings ([ipld/go-codec-dagpb#29](https://github.com/ipld/go-codec-dagpb/pull/29)) - update go-ipld-prime, use go:generate - allow decoding PBNode fields in any order - expose APIs without Reader/Writer overhead - preallocate 1KiB on the stack for marshals - encode directly with a []byte - decode directly with a []byte - remove unnecessary xerrors dep - github.com/ipld/go-ipld-prime (v0.9.1-0.20210324083106-dc342a9917db -> v0.12.2): - Printer feature ([ipld/go-ipld-prime#238](https://github.com/ipld/go-ipld-prime/pull/238)) - schema: keep TypeSystem names ordered - schema/dmt: redesign with bindnode and add Compile - codec: make cbor and json codecs use ErrUnexpectedEOF - bindnode: fix for stringjoin struct emission when first field is the empty string ([ipld/go-ipld-prime#239](https://github.com/ipld/go-ipld-prime/pull/239)) - schema: typekind names are not capitalized. - Bindnode fixes continued ([ipld/go-ipld-prime#233](https://github.com/ipld/go-ipld-prime/pull/233)) - helper methods for encoding and decoding ([ipld/go-ipld-prime#232](https://github.com/ipld/go-ipld-prime/pull/232)) - mark v0.12.0 - Major refactor: extract datamodel package. ([ipld/go-ipld-prime#228](https://github.com/ipld/go-ipld-prime/pull/228)) - Fix ExploreRecursive stopAt condition, add tests, add error return to Explore (#229) ([ipld/go-ipld-prime#229](https://github.com/ipld/go-ipld-prime/pull/229)) - selector: add tests which are driven by language-agnostic spec fixtures. ([ipld/go-ipld-prime#231](https://github.com/ipld/go-ipld-prime/pull/231)) - selector: Improve docs for implementers. (#227) ([ipld/go-ipld-prime#227](https://github.com/ipld/go-ipld-prime/pull/227)) - Bindnode fixes of opportunity ([ipld/go-ipld-prime#226](https://github.com/ipld/go-ipld-prime/pull/226)) - node/bindnode: redesign the shape of unions in Go ([ipld/go-ipld-prime#223](https://github.com/ipld/go-ipld-prime/pull/223)) - summary of the v0.11.0 changelog should holler even more about how cool bindnode is. - mark v0.11.0 - node/bindnode: mark as experimental in its godoc. - codecs: more docs, a terminology guide, consistency in options. ([ipld/go-ipld-prime#221](https://github.com/ipld/go-ipld-prime/pull/221)) - Changelog backfill. - selectors: docs enhancements, new construction helpers. ([ipld/go-ipld-prime#199](https://github.com/ipld/go-ipld-prime/pull/199)) - Changelog backfill. - Allow parsing of single Null tokens from refmt - Add link conditions for 'stop-at' expression in ExploreRecursive selector ([ipld/go-ipld-prime#214](https://github.com/ipld/go-ipld-prime/pull/214)) - Remove base64 padding for dag-json bytes as per spec - node/bindnode: temporarily skip Links schema test - test: add test for traversal of typed node links - fix: typed links LinkTargetNodePrototype should return ReferencedType - Make `go vet` happy - Add MapSortMode to MarshalOptions - Add {Unm,M}arshalOptions for explicit mode switching for cbor vs dagcbor - Sort map entries marshalling dag-cbor - node/bindnode: first pass at inferring IPLD schemas - Add {Unm,M}arshalOptions for explicit mode switching for json vs dagjson - Make tests pass with sorted dag-json output - Sort map entries marshalling dag-json - Simplify refmt usage - Fix failing test using dagjson encoding - Fix some failing tests using dagjson - Remove pretty-printing - Update readme linking to specs and meta repo. - Fix example names so they render on go.pkg.dev. - fluent/quip: remove in favor of qp - node/basic: add Chooser - schema: add TypedPrototype - node/bindnode: rethink and better document APIs - node/tests: cover yet more interface methods - node/tests: cover more error cases for scalar kinds - node/tests: add more extensive scalar kind tests - node/bindnode: start running all schema tests - mark v0.10.0 - More changelog grooming. - Changelog grooming. - node/tests: put most of the schema test cases here - Add more explicit discussion of indices to ListIterator. - node/bindnode: start of a reflect-based Node implementation - add DeepEqual and start using it in tests - Add enumerate methods to the multicodec registries. ([ipld/go-ipld-prime#176](https://github.com/ipld/go-ipld-prime/pull/176)) - Make a multicodec.Registry type available. ([ipld/go-ipld-prime#172](https://github.com/ipld/go-ipld-prime/pull/172)) - fluent/qp: don't panic on string panics - Allow emitting & parsing of bytes per dagjson codec spec ([ipld/go-ipld-prime#166](https://github.com/ipld/go-ipld-prime/pull/166)) - Package docs for dag-cbor. - Update package docs. - schema/gen/go: apply gofmt automatically ([ipld/go-ipld-prime#163](https://github.com/ipld/go-ipld-prime/pull/163)) - schema/gen/go: fix remaining vet warnings on generated code - schema/gen/go: batch file writes via a bytes.Buffer ([ipld/go-ipld-prime#161](https://github.com/ipld/go-ipld-prime/pull/161)) - schema/gen/go: avoid Maybe pointers for small types - fix readme formatting typo - feat(linksystem): add reification to LinkSystem ([ipld/go-ipld-prime#158](https://github.com/ipld/go-ipld-prime/pull/158)) - github.com/libp2p/go-addr-util (v0.0.2 -> v0.1.0): - stop using the deprecated go-multiaddr-net package ([libp2p/go-addr-util#34](https://github.com/libp2p/go-addr-util/pull/34)) - Remove `IsFDCostlyTransport` ([libp2p/go-addr-util#31](https://github.com/libp2p/go-addr-util/pull/31)) - github.com/libp2p/go-libp2p (v0.14.3 -> v0.15.0): - chore: update go-tcp-transport to v0.2.8 - implement the new mDNS spec, move the old mDNS implementation (#1161) ([libp2p/go-libp2p#1161](https://github.com/libp2p/go-libp2p/pull/1161)) - remove deprecated basichost.New constructor ([libp2p/go-libp2p#1156](https://github.com/libp2p/go-libp2p/pull/1156)) - Make BasicHost.evtLocalAddrsUpdated event emitter stateful. ([libp2p/go-libp2p#1147](https://github.com/libp2p/go-libp2p/pull/1147)) - fix: deflake multipro echo test ([libp2p/go-libp2p#1149](https://github.com/libp2p/go-libp2p/pull/1149)) - fix(basic_host): stream not closed when context done ([libp2p/go-libp2p#1148](https://github.com/libp2p/go-libp2p/pull/1148)) - chore: update deps ([libp2p/go-libp2p#1141](https://github.com/libp2p/go-libp2p/pull/1141)) - remove secio from examples ([libp2p/go-libp2p#1143](https://github.com/libp2p/go-libp2p/pull/1143)) - remove deprecated Filter option ([libp2p/go-libp2p#1132](https://github.com/libp2p/go-libp2p/pull/1132)) - fix: remove deprecated call ([libp2p/go-libp2p#1136](https://github.com/libp2p/go-libp2p/pull/1136)) - test: fix flaky example test ([libp2p/go-libp2p#1135](https://github.com/libp2p/go-libp2p/pull/1135)) - remove deprecated identify.ClientVersion ([libp2p/go-libp2p#1133](https://github.com/libp2p/go-libp2p/pull/1133)) - remove Go version requirement and note about Go modules from README ([libp2p/go-libp2p#1126](https://github.com/libp2p/go-libp2p/pull/1126)) - Error assignment fix ([libp2p/go-libp2p#1124](https://github.com/libp2p/go-libp2p/pull/1124)) - perf/basic_host: Don't handle address change if we hasn't anyone ([libp2p/go-libp2p#1115](https://github.com/libp2p/go-libp2p/pull/1115)) - github.com/libp2p/go-libp2p-core (v0.8.5 -> v0.9.0): - feat: remove unused metrics (#208) ([libp2p/go-libp2p-core#208](https://github.com/libp2p/go-libp2p-core/pull/208)) - feat: keep addresses for longer (#207) ([libp2p/go-libp2p-core#207](https://github.com/libp2p/go-libp2p-core/pull/207)) - remove deprecated key stretching struct / function (#203) ([libp2p/go-libp2p-core#203](https://github.com/libp2p/go-libp2p-core/pull/203)) - remove deprecated Bytes method from the Key interface (#204) ([libp2p/go-libp2p-core#204](https://github.com/libp2p/go-libp2p-core/pull/204)) - remove deprecated functions in the peer package (#205) ([libp2p/go-libp2p-core#205](https://github.com/libp2p/go-libp2p-core/pull/205)) - remove deprecated constructor for the insecure transport (#206) ([libp2p/go-libp2p-core#206](https://github.com/libp2p/go-libp2p-core/pull/206)) - feat: add helper functions for working with addr infos (#202) ([libp2p/go-libp2p-core#202](https://github.com/libp2p/go-libp2p-core/pull/202)) - fix: make timestamps strictly increasing (#201) ([libp2p/go-libp2p-core#201](https://github.com/libp2p/go-libp2p-core/pull/201)) - ci: use github-actions for compatibility testing (#200) ([libp2p/go-libp2p-core#200](https://github.com/libp2p/go-libp2p-core/pull/200)) - sync: update CI config files (#189) ([libp2p/go-libp2p-core#189](https://github.com/libp2p/go-libp2p-core/pull/189)) - remove minimum Go version from README (#199) ([libp2p/go-libp2p-core#199](https://github.com/libp2p/go-libp2p-core/pull/199)) - remove flaky tests (#194) ([libp2p/go-libp2p-core#194](https://github.com/libp2p/go-libp2p-core/pull/194)) - reduce default timeouts to 15s (#192) ([libp2p/go-libp2p-core#192](https://github.com/libp2p/go-libp2p-core/pull/192)) - fix benchmark of key verifications (#190) ([libp2p/go-libp2p-core#190](https://github.com/libp2p/go-libp2p-core/pull/190)) - fix staticcheck errors (#191) ([libp2p/go-libp2p-core#191](https://github.com/libp2p/go-libp2p-core/pull/191)) - doc: document Close on Transport (#188) ([libp2p/go-libp2p-core#188](https://github.com/libp2p/go-libp2p-core/pull/188)) - add a helper function to go directly from a string to an AddrInfo (#184) ([libp2p/go-libp2p-core#184](https://github.com/libp2p/go-libp2p-core/pull/184)) - github.com/libp2p/go-libp2p-http (v0.2.0 -> v0.2.1): - remove Makefile ([libp2p/go-libp2p-http#70](https://github.com/libp2p/go-libp2p-http/pull/70)) - fix staticcheck ([libp2p/go-libp2p-http#67](https://github.com/libp2p/go-libp2p-http/pull/67)) - Revert "increase buffer size" - Increase read buffer size to reduce poll system calls ([libp2p/go-libp2p-http#66](https://github.com/libp2p/go-libp2p-http/pull/66)) - github.com/libp2p/go-libp2p-kad-dht (v0.12.2 -> v0.13.1): - Extract validation from ProtocolMessenger ([libp2p/go-libp2p-kad-dht#741](https://github.com/libp2p/go-libp2p-kad-dht/pull/741)) - remove codecov.yml ([libp2p/go-libp2p-kad-dht#742](https://github.com/libp2p/go-libp2p-kad-dht/pull/742)) - integrate some basic opentelemetry tracing ([libp2p/go-libp2p-kad-dht#734](https://github.com/libp2p/go-libp2p-kad-dht/pull/734)) - feat: delete GetValues ([libp2p/go-libp2p-kad-dht#728](https://github.com/libp2p/go-libp2p-kad-dht/pull/728)) - chore: skip flaky test when race detector is enabled ([libp2p/go-libp2p-kad-dht#731](https://github.com/libp2p/go-libp2p-kad-dht/pull/731)) - Dont count connection times in usefulness ([libp2p/go-libp2p-kad-dht#660](https://github.com/libp2p/go-libp2p-kad-dht/pull/660)) - Routing table refresh should NOT block ([libp2p/go-libp2p-kad-dht#705](https://github.com/libp2p/go-libp2p-kad-dht/pull/705)) - update bootstrapPeers to be func() []peer.AddrInfo (#716) ([libp2p/go-libp2p-kad-dht#716](https://github.com/libp2p/go-libp2p-kad-dht/pull/716)) - github.com/libp2p/go-libp2p-noise (v0.2.0 -> v0.2.2): - remove note about go modules in README ([libp2p/go-libp2p-noise#100](https://github.com/libp2p/go-libp2p-noise/pull/100)) - fix: remove deprecated call to pk.Bytes ([libp2p/go-libp2p-noise#99](https://github.com/libp2p/go-libp2p-noise/pull/99)) - github.com/libp2p/go-libp2p-peerstore (v0.2.7 -> v0.2.8): - Fix performance issue in updating addr book ([libp2p/go-libp2p-peerstore#141](https://github.com/libp2p/go-libp2p-peerstore/pull/141)) - Fix test flakes ([libp2p/go-libp2p-peerstore#164](https://github.com/libp2p/go-libp2p-peerstore/pull/164)) - Only remove records during GC ([libp2p/go-libp2p-peerstore#135](https://github.com/libp2p/go-libp2p-peerstore/pull/135)) - sync: update CI config files ([libp2p/go-libp2p-peerstore#160](https://github.com/libp2p/go-libp2p-peerstore/pull/160)) - fix: fix some race conditions in the ds address book ([libp2p/go-libp2p-peerstore#161](https://github.com/libp2p/go-libp2p-peerstore/pull/161)) - address lints and test failures ([libp2p/go-libp2p-peerstore#159](https://github.com/libp2p/go-libp2p-peerstore/pull/159)) - stop using the deprecated go-multiaddr-net package ([libp2p/go-libp2p-peerstore#158](https://github.com/libp2p/go-libp2p-peerstore/pull/158)) - github.com/libp2p/go-libp2p-pubsub (v0.4.2 -> v0.5.4): - make slowness a warning, with a user configurable threshold - reduce log spam from empty heartbeat messages - fix: code review - add support for custom protocol matching function - fix: remove deprecated Bytes call (#436) ([libp2p/go-libp2p-pubsub#436](https://github.com/libp2p/go-libp2p-pubsub/pull/436)) - cleanup: fix vet and staticcheck failures (#435) ([libp2p/go-libp2p-pubsub#435](https://github.com/libp2p/go-libp2p-pubsub/pull/435)) - Revert noisy newline changes - fix: avoid panic when peer is blacklisted after connection - release priority locks early when handling batches - don't respawn writer if we fail to open a stream; declare it a peer error - batch process dead peer notifications - use a priority lock instead of a semaphore - do the notification in a goroutine - emit new peer notification without holding the semaphore - use a semaphore for new peer notifications so that we don't block the event loop - don't accumulate pending goroutines from new connections - rename RawTracer's DroppedInSubscribe into UndeliverableMessage - add a new RawTracer event to track messages dropped in Subscribe - add an option to configure the Subscription output queue length - fix some comments - expose more events for RawTracer - Make close concurrent safe - Fix close of closed channel - Update README to point to correct example directory (#424) ([libp2p/go-libp2p-pubsub#424](https://github.com/libp2p/go-libp2p-pubsub/pull/424)) - fix: remove deprecated and never used topic descriptors (#423) ([libp2p/go-libp2p-pubsub#423](https://github.com/libp2p/go-libp2p-pubsub/pull/423)) - Refactor Gossipsub Parameters To Make Them More Configurable (#421) ([libp2p/go-libp2p-pubsub#421](https://github.com/libp2p/go-libp2p-pubsub/pull/421)) - add tests for gs features and custom protocols - add support for custom gossipsub protocols and feature tests - RIP travis, Long Live CircleCI (#414) ([libp2p/go-libp2p-pubsub#414](https://github.com/libp2p/go-libp2p-pubsub/pull/414)) - Ignore transient connections (#412) ([libp2p/go-libp2p-pubsub#412](https://github.com/libp2p/go-libp2p-pubsub/pull/412)) - demote log spam to debug - fix bug - add last amount of validation - add threshold validation - strengthen validation - rename checkSignature to checkSigningPolicy - rename validation.Publish to PushLocal - fix TestValidate, add TestValidate2 - skip flaky test until we can fix it - implement synchronous validation for locally published messages - expose internalTracer as RawTracer - export rejection named string constants - more intelligent handling of ip whitelist check - remove obsolete explicit IP whitelisting in favor of subnets - add subnet whitelisting for IPColocation - github.com/libp2p/go-libp2p-quic-transport (v0.11.2 -> v0.12.0): - sync: update CI config files (#228) ([libp2p/go-libp2p-quic-transport#228](https://github.com/libp2p/go-libp2p-quic-transport/pull/228)) - fix closing of streams in example ([libp2p/go-libp2p-quic-transport#221](https://github.com/libp2p/go-libp2p-quic-transport/pull/221)) - close all UDP connections when the reuse is closed ([libp2p/go-libp2p-quic-transport#216](https://github.com/libp2p/go-libp2p-quic-transport/pull/216)) - fix staticcheck ([libp2p/go-libp2p-quic-transport#217](https://github.com/libp2p/go-libp2p-quic-transport/pull/217)) - sync: update CI config files (#214) ([libp2p/go-libp2p-quic-transport#214](https://github.com/libp2p/go-libp2p-quic-transport/pull/214)) - implement a Transport.Close that waits for the reuse's GC to finish ([libp2p/go-libp2p-quic-transport#211](https://github.com/libp2p/go-libp2p-quic-transport/pull/211)) - don't compare peer IDs when hole punching ([libp2p/go-libp2p-quic-transport#210](https://github.com/libp2p/go-libp2p-quic-transport/pull/210)) - add hole punching support (#194) ([libp2p/go-libp2p-quic-transport#194](https://github.com/libp2p/go-libp2p-quic-transport/pull/194)) - github.com/libp2p/go-libp2p-swarm (v0.5.0 -> v0.5.3): - sync: update CI config files ([libp2p/go-libp2p-swarm#263](https://github.com/libp2p/go-libp2p-swarm/pull/263)) - remove incorrect call to InterceptAddrDial ([libp2p/go-libp2p-swarm#260](https://github.com/libp2p/go-libp2p-swarm/pull/260)) - speed up the TestFDLimitUnderflow test ([libp2p/go-libp2p-swarm#262](https://github.com/libp2p/go-libp2p-swarm/pull/262)) - sync: update CI config files (#248) ([libp2p/go-libp2p-swarm#248](https://github.com/libp2p/go-libp2p-swarm/pull/248)) - github.com/libp2p/go-libp2p-testing (v0.4.0 -> v0.4.2): - fix deadlock in the transport's serve function ([libp2p/go-libp2p-testing#35](https://github.com/libp2p/go-libp2p-testing/pull/35)) - fix: cleanup transport suite ([libp2p/go-libp2p-testing#34](https://github.com/libp2p/go-libp2p-testing/pull/34)) - Address `go vet` and `saticcheck` issues ([libp2p/go-libp2p-testing#33](https://github.com/libp2p/go-libp2p-testing/pull/33)) - Defer closing stream for reading ([libp2p/go-libp2p-testing#32](https://github.com/libp2p/go-libp2p-testing/pull/32)) - github.com/libp2p/go-libp2p-tls (v0.1.3 -> v0.2.0): - fix: don't fail the handshake when the libp2p extension is critical ([libp2p/go-libp2p-tls#88](https://github.com/libp2p/go-libp2p-tls/pull/88)) - fix deprecated call to key.Bytes ([libp2p/go-libp2p-tls#86](https://github.com/libp2p/go-libp2p-tls/pull/86)) - fix usage of deprecated peer.IDB58Decode ([libp2p/go-libp2p-tls#77](https://github.com/libp2p/go-libp2p-tls/pull/77)) - remove setting of the TLS 1.3 GODEBUG flag ([libp2p/go-libp2p-tls#68](https://github.com/libp2p/go-libp2p-tls/pull/68)) - improve the error message returned when peer verification fails ([libp2p/go-libp2p-tls#57](https://github.com/libp2p/go-libp2p-tls/pull/57)) - update to Go 1.14 ([libp2p/go-libp2p-tls#54](https://github.com/libp2p/go-libp2p-tls/pull/54)) - Update deps and fix tests ([libp2p/go-libp2p-tls#43](https://github.com/libp2p/go-libp2p-tls/pull/43)) - github.com/libp2p/go-libp2p-transport-upgrader (v0.4.2 -> v0.4.6): - chore: update deps ([libp2p/go-libp2p-transport-upgrader#78](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/78)) - fix typo in error message ([libp2p/go-libp2p-transport-upgrader#77](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/77)) - fix staticcheck ([libp2p/go-libp2p-transport-upgrader#74](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/74)) - don't listen on all interfaces in tests ([libp2p/go-libp2p-transport-upgrader#73](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/73)) - stop using the deprecated go-multiaddr-net ([libp2p/go-libp2p-transport-upgrader#72](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/72)) - github.com/libp2p/go-libp2p-xor (v0.0.0-20200501025846-71e284145d58 -> v0.0.0-20210714161855-5c005aca55db): - Add immutable remove operation ([libp2p/go-libp2p-xor#14](https://github.com/libp2p/go-libp2p-xor/pull/14)) - fix go vet and staticcheck ([libp2p/go-libp2p-xor#11](https://github.com/libp2p/go-libp2p-xor/pull/11)) - github.com/libp2p/go-reuseport-transport (v0.0.4 -> v0.0.5): - remove note about Go modules in README ([libp2p/go-reuseport-transport#32](https://github.com/libp2p/go-reuseport-transport/pull/32)) - stop using the deprecated go-multiaddr-net package ([libp2p/go-reuseport-transport#30](https://github.com/libp2p/go-reuseport-transport/pull/30)) - github.com/libp2p/go-socket-activation (v0.0.2 -> v0.1.0): - chore: stop using the deprecated go-multiaddr-net package ([libp2p/go-socket-activation#16](https://github.com/libp2p/go-socket-activation/pull/16)) - fix staticcheck ([libp2p/go-socket-activation#13](https://github.com/libp2p/go-socket-activation/pull/13)) - github.com/libp2p/go-tcp-transport (v0.2.4 -> v0.2.8): - disable metrics collection on Windows ([libp2p/go-tcp-transport#93](https://github.com/libp2p/go-tcp-transport/pull/93)) - sync: update CI config files (#90) ([libp2p/go-tcp-transport#90](https://github.com/libp2p/go-tcp-transport/pull/90)) - chore: update go-libp2p-transport-upgrader and go-reuseport-transport ([libp2p/go-tcp-transport#84](https://github.com/libp2p/go-tcp-transport/pull/84)) - github.com/libp2p/go-ws-transport (v0.4.0 -> v0.5.0): - chore: update go-libp2p-transport-upgrader and go-libp2p-core ([libp2p/go-ws-transport#103](https://github.com/libp2p/go-ws-transport/pull/103)) - remove deprecated type ([libp2p/go-ws-transport#102](https://github.com/libp2p/go-ws-transport/pull/102)) - sync: update CI config files ([libp2p/go-ws-transport#101](https://github.com/libp2p/go-ws-transport/pull/101)) - chore: various cleanups required to get vet/staticcheck/test to pass ([libp2p/go-ws-transport#100](https://github.com/libp2p/go-ws-transport/pull/100)) - github.com/lucas-clemente/quic-go (v0.21.2 -> v0.23.0): - update to Go 1.17.x ([lucas-clemente/quic-go#3258](https://github.com/lucas-clemente/quic-go/pull/3258)) - quicvarint: export Min and Max (#3253) ([lucas-clemente/quic-go#3253](https://github.com/lucas-clemente/quic-go/pull/3253)) - drop support for Go 1.15 ([lucas-clemente/quic-go#3247](https://github.com/lucas-clemente/quic-go/pull/3247)) - quicvarint: add Reader and Writer interfaces (#3233) ([lucas-clemente/quic-go#3233](https://github.com/lucas-clemente/quic-go/pull/3233)) - fix race when stream.Read and CancelRead are called concurrently ([lucas-clemente/quic-go#3241](https://github.com/lucas-clemente/quic-go/pull/3241)) - also count coalesced 0-RTT packets in the integration tests ([lucas-clemente/quic-go#3251](https://github.com/lucas-clemente/quic-go/pull/3251)) - remove draft versions 32 and 34 from README (#3244) ([lucas-clemente/quic-go#3244](https://github.com/lucas-clemente/quic-go/pull/3244)) - update Changelog ([lucas-clemente/quic-go#3245](https://github.com/lucas-clemente/quic-go/pull/3245)) - optimize hasOutstandingCryptoPackets in sentPacketHandler ([lucas-clemente/quic-go#3230](https://github.com/lucas-clemente/quic-go/pull/3230)) - permit underlying conn to implement batch interface directly ([lucas-clemente/quic-go#3237](https://github.com/lucas-clemente/quic-go/pull/3237)) - cancel the PTO timer when all Handshake packets are acknowledged ([lucas-clemente/quic-go#3231](https://github.com/lucas-clemente/quic-go/pull/3231)) - fix flaky INVALID_TOKEN server test ([lucas-clemente/quic-go#3223](https://github.com/lucas-clemente/quic-go/pull/3223)) - drop support for QUIC draft version 32 and 34 ([lucas-clemente/quic-go#3217](https://github.com/lucas-clemente/quic-go/pull/3217)) - fix flaky 0-RTT integration test ([lucas-clemente/quic-go#3224](https://github.com/lucas-clemente/quic-go/pull/3224)) - use batched reads ([lucas-clemente/quic-go#3142](https://github.com/lucas-clemente/quic-go/pull/3142)) - add a config option to disable sending of Version Negotiation packets ([lucas-clemente/quic-go#3216](https://github.com/lucas-clemente/quic-go/pull/3216)) - remove the RetireBugBackwardsCompatibilityMode ([lucas-clemente/quic-go#3213](https://github.com/lucas-clemente/quic-go/pull/3213)) - remove outdated ackhandler test case ([lucas-clemente/quic-go#3212](https://github.com/lucas-clemente/quic-go/pull/3212)) - remove unused StripGreasedVersions function ([lucas-clemente/quic-go#3214](https://github.com/lucas-clemente/quic-go/pull/3214)) - fix incorrect usage of errors.Is ([lucas-clemente/quic-go#3215](https://github.com/lucas-clemente/quic-go/pull/3215)) - return error on SendMessage when session is closed ([lucas-clemente/quic-go#3218](https://github.com/lucas-clemente/quic-go/pull/3218)) - remove a redundant error check ([lucas-clemente/quic-go#3210](https://github.com/lucas-clemente/quic-go/pull/3210)) - update golangci-lint to v1.41.1 ([lucas-clemente/quic-go#3205](https://github.com/lucas-clemente/quic-go/pull/3205)) - Update doc for dialer in http3.RoundTripper ([lucas-clemente/quic-go#3208](https://github.com/lucas-clemente/quic-go/pull/3208)) - github.com/multiformats/go-multiaddr (v0.3.3 -> v0.4.0): - remove forced dependency on deprecated go-maddr-filter ([multiformats/go-multiaddr#162](https://github.com/multiformats/go-multiaddr/pull/162)) - remove deprecated SwapToP2pMultiaddrs ([multiformats/go-multiaddr#161](https://github.com/multiformats/go-multiaddr/pull/161)) - remove Makefile ([multiformats/go-multiaddr#163](https://github.com/multiformats/go-multiaddr/pull/163)) - remove deprecated filter functions ([multiformats/go-multiaddr#157](https://github.com/multiformats/go-multiaddr/pull/157)) - remove deprecated NetCodec ([multiformats/go-multiaddr#159](https://github.com/multiformats/go-multiaddr/pull/159)) - add Noise ([multiformats/go-multiaddr#156](https://github.com/multiformats/go-multiaddr/pull/156)) - Add TLS protocol ([multiformats/go-multiaddr#153](https://github.com/multiformats/go-multiaddr/pull/153)) - github.com/multiformats/go-multicodec (v0.2.0 -> v0.3.0): - Export reserved range constants (#53) ([multiformats/go-multicodec#53](https://github.com/multiformats/go-multicodec/pull/53)) - make Code.Set accept valid code numbers - replace Of with Code.Set, implementing flag.Value - add multiformats/multicodec as a git submodule - update the generator with the "status" CSV column - Run `go generate` to generate the latest codecs - Add lookup for multicodec code by string name ([multiformats/go-multicodec#40](https://github.com/multiformats/go-multicodec/pull/40)) ### Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Daniel Martí | 42 | +8549/-6587 | 170 | | Eric Myhre | 55 | +5883/-6715 | 395 | | Marten Seemann | 100 | +1814/-2028 | 275 | | Steven Allen | 80 | +1573/-1998 | 127 | | hannahhoward | 18 | +1721/-671 | 53 | | Will | 2 | +1114/-1217 | 18 | | Andrew Gillis | 2 | +1220/-720 | 14 | | gammazero | 3 | +43/-1856 | 10 | | Masih H. Derkani | 3 | +960/-896 | 8 | | Adin Schmahmann | 25 | +1458/-313 | 44 | | vyzo | 27 | +986/-353 | 60 | | Will Scott | 6 | +852/-424 | 16 | | Rod Vagg | 19 | +983/-255 | 66 | | Petar Maymounkov | 6 | +463/-179 | 22 | | web3-bot | 10 | +211/-195 | 24 | | adlrocha | 1 | +330/-75 | 15 | | RubenKelevra | 2 | +128/-210 | 2 | | Ian Davis | 3 | +200/-109 | 17 | | Cory Schwartz | 3 | +231/-33 | 7 | | Keenan Nemetz | 1 | +184/-71 | 2 | | Randy Reddig | 2 | +187/-53 | 8 | | Takashi Matsuda | 3 | +201/-2 | 7 | | guseggert | 4 | +161/-20 | 9 | | Lucas Molas | 5 | +114/-47 | 27 | | nisdas | 4 | +115/-45 | 7 | | Michael Muré | 6 | +107/-33 | 24 | | Richard Ramos | 2 | +113/-9 | 3 | | Marcin Rataj | 12 | +88/-24 | 13 | | Ondrej Prazak | 2 | +104/-6 | 4 | | Michal Dobaczewski | 2 | +77/-28 | 3 | | Jorropo | 3 | +9/-75 | 4 | | Andey Robins | 1 | +70/-3 | 3 | | Gus Eggert | 10 | +34/-31 | 12 | | noot | 1 | +54/-9 | 5 | | Maxim Merzhanov | 1 | +29/-24 | 1 | | Adrian Lanzafame | 1 | +30/-13 | 2 | | Bogdan Stirbat | 1 | +22/-16 | 2 | | Shad Sterling | 1 | +28/-3 | 1 | | Jesse Bouwman | 5 | +30/-0 | 5 | | Pavel Karpy | 1 | +19/-7 | 2 | | lasiar | 5 | +14/-10 | 5 | | Dennis Trautwein | 1 | +20/-4 | 2 | | Louis Thibault | 1 | +22/-1 | 2 | | whyrusleeping | 2 | +21/-1 | 2 | | aarshkshah1992 | 3 | +12/-8 | 3 | | Peter Rabbitson | 2 | +20/-0 | 2 | | bt90 | 2 | +17/-2 | 2 | | Dominic Della Valle | 1 | +13/-1 | 2 | | Audrius Butkevicius | 1 | +12/-1 | 1 | | Brian Strauch | 1 | +9/-3 | 1 | | Aarsh Shah | 2 | +1/-11 | 2 | | Whyrusleeping | 1 | +11/-0 | 1 | | Max | 1 | +7/-3 | 1 | | vallder | 1 | +3/-5 | 1 | | Michael Burns | 3 | +2/-6 | 3 | | Lasse Johnsen | 1 | +4/-4 | 2 | | snyh | 1 | +5/-2 | 1 | | Hector Sanjuan | 2 | +3/-2 | 2 | | 市川恭佑 (ebi) | 1 | +1/-3 | 1 | | godcong | 2 | +2/-1 | 2 | | Mathis Engelbart | 1 | +1/-2 | 1 | | folbrich | 1 | +1/-1 | 1 | | Med Mouine | 1 | +1/-1 | 1 | ================================================ FILE: docs/changelogs/v0.11.md ================================================ # go-ipfs changelog v0.11 ## v0.11.1 2022-04-08 This patch release fixes a security issue wherein traversing some malformed DAGs can cause the node to panic. See also the security advisory: https://github.com/ipfs/go-ipfs/security/advisories/GHSA-mcq2-w56r-5w2w Note: the v0.11.1 patch release contains the Docker compose fix from v0.12.1 as well ### Changelog
Full Changelog - github.com/ipld/go-codec-dagpb (v1.3.0 -> v1.3.2): - fix: use protowire for Links bytes decoding
### ❤ Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Rod Vagg | 1 | +34/-19 | 2 | ## v0.11.0 2021-12-08 We're happy to announce go-ipfs 0.11.0. This release comes with improvements to the UnixFS Sharding and PubSub experiments as well as support for Circuit-Relay v2 which sets the network up for decentralized hole punching support. As usual, this release includes important fixes, some of which may be critical for security. Unless the fix addresses a bug being exploited in the wild, the fix will _not_ be called out in the release notes. Please make sure to update ASAP. See our [release process](https://github.com/ipfs/go-ipfs/tree/master/docs/releases.md#security-fix-policy) for details. ### 🛠 BREAKING CHANGES - UnixFS sharding is now automatic and enabled by default - HAMT-based sharding is applied to large directories (i.e. those that would serialize into [block](https://docs.ipfs.tech/concepts/glossary/#block) larger than ~256KiB)s. This means importing data via commands like `ipfs add -r ` may result in different [CID](https://docs.ipfs.tech/concepts/glossary/#cid)s due to the different [DAG](https://docs.ipfs.tech/concepts/glossary/#dag) representations. - Support for `Experimental.ShardingEnabled` is removed. - go-ipfs can no longer act as a [Circuit Relay](https://docs.ipfs.tech/concepts/glossary/#circuit-relay) v1 - Node will refuse to start if `Swarm.EnableRelayHop` is set to `true` - If you depend on v1 relay service provider, see "Removal of v1 relay service" section for available migration options. - HTTP RPC wire format for experimental commands at `/api/v0/pubsub` changed. - If you use [js-ipfs-http-client](https://www.npmjs.com/package/ipfs-http-client) or [go-ipfs-http-client](https://github.com/ipfs/go-ipfs-http-client), just update to their latest version. - If you use something else, see "Multibase in PubSub" section below for migration details. Keep reading to learn more details. ### 🔦 Highlights #### 🗃 Automatic UnixFS sharding Truly big directories can have so many items, that the root block with all of their names is too big to be exchanged with other peers. This was partially solved by [HAMT-sharding](https://docs.ipfs.tech/concepts/glossary/#hamt-sharding), which was introduced a while ago as opt-in. The main downside of the implementation was that it was a global flag that sharded all imported directories (big and small). This release solves that inconvenience by making UnixFS sharding smarter and applies it only to larger directories (i.e. directories that would be at least ~256KiB). This is now the default behavior in `ipfs add` and `ipfs files` commands, where UnixFS sharding works out-of-the-box. #### 🔁 Circuit Relay v2 This release adds support for the [circuit relay v2](https://github.com/libp2p/specs/blob/master/relay/circuit-v2.md) protocol based on the reference implementation from [go-libp2p 0.16](https://github.com/libp2p/go-libp2p/releases/tag/v0.16.0). This is the cornerstone for maximizing p2p connections between IPFS peers. Every publicly dialable peer can now act as a limited relay v2, which can be used for [hole punching](https://docs.ipfs.tech/concepts/glossary/#hole-punching) and other decentralized signaling protocols. ##### Limited relay v2 configuration options go-ipfs can now be configured to act as a [`RelayClient`](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmrelayclient) that uses other peers for autorelay functionality when behind a NAT, or provide a limited [`RelayService`](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmrelayservice) to other peers on the network. Starting with go-ipfs v0.11 every publicly dialable go-ipfs (based on AutoNAT determination) will start a limited `RelayService`. `RelayClient` remains disabled by default for now, as we want the network to update and get enough v2 service providers first. Note: the limited Circuit Relay v2 provided with this release only allows low-bandwidth protocols (identify, ping, holepunch) over transient connections. If you want to relay things like bitswap sessions, you need to set up a v1 relay by some other means. See details below. ##### Removal of unlimited v1 relay service provider Switching to v2 of the relay spec means removal or deprecation of configuration keys that were specific to v1. - Relay transport and client support circuit-relay v2: - `Swarm.EnableAutoRelay` was replaced by `Swarm.RelayClient.Enabled`. - `Swarm.DisableRelay` is deprecated, relay transport can be now disabled globally (both client and service) by setting `Swarm.Transports.Network.Relay` to `false` - Relay v1 service provider was replaced by v2: - `Swarm.EnableRelayHop` no longer starts an unlimited v1 relay. If you have it set to `true` the node will refuse to start and display an error message. - Existing users who choose to continue running a v1 relay should migrate their setups to relay v1 based on js-ipfs running in node, or the standalone [libp2p-relay-daemon](https://dist.ipfs.tech/#libp2p-relay-daemon) [configured](https://github.com/libp2p/go-libp2p-relay-daemon/#configuration) with `RelayV1.Enabled` set to `true`. Be mindful that v1 relays are unlimited, and one may want to set up some ACL based either on PeerIDs or Subnets. #### 🕳 Decentralized Hole Punching (DCUtR protocol client) We are working towards enabling hole punching for NAT traversal when port forwarding is not possible. [go-libp2p 0.16](https://github.com/libp2p/go-libp2p/releases/tag/v0.16.0) provides an implementation of the [DCUtR (decentralized hole punching)](https://github.com/libp2p/specs/blob/master/relay/DCUtR.md) protocol. It is hidden behind the `Swarm.EnableHolePunching` configuration flag. When enabled, go-ipfs will coordinate with the counterparty using a [relayed v2 connection](https://github.com/libp2p/specs/blob/master/relay/circuit-v2.md), to [upgrade to a direct connection](https://github.com/libp2p/specs/blob/master/relay/DCUtR.md) through a NAT/firewall whenever possible. This feature is disabled by default in this release, but we hope to enable it by default as soon the network updates to go-ipfs v0.11 and gains a healthy set of limited v2 relays. #### 💬 Multibase in PubSub HTTP RPC API This release fixed some edge cases that were reported by users of the PubSub experiment, getting it closer to becoming a stable feature of go-ipfs. Some PubSub users will notice that the plaintext limitation is lifted: one can now use line breaks in messages published to non-ascii topic names, or even publish arbitrary bytes to arbitrary topics. It required a change to the wire format used when pubsub commands are executed over the HTTP RPC API at `/api/v0/pubsub/*`, and also modified the behavior of the `ipfs pubsub pub` command, which now is publishing only a single pubsub message with data read from a file or stdin. ##### PubSub client migration tips If you use the HTTP RPC API with the [go-ipfs-http-client](https://github.com/ipfs/go-ipfs-http-client) library, make sure to update to the latest version. The next version of [js-ipfs-http-client](https://www.npmjs.com/package/ipfs-http-client) will use the new wire format as well, so you don't need to do anything. If you use `/api/v0/pubsub/*` directly or maintain your own client library, you must adjust your HTTP client code. Byte fields and URL args are now encoded in `base64url` [Multibase](https://docs.ipfs.tech/concepts/glossary/#multibase). Encode/decode bytes using the `ipfs multibase --help` commands, or use the multiformats libraries ([js-multiformats](https://github.com/multiformats/js-multiformats#readme), [go-multibase](https://github.com/multiformats/go-multibase)). Low level changes: - `topic` passed as URL `arg` in requests to `/api/v0/pubsub/*` must be encoded in URL-safe multibase (`base64url`) - `data`, `from`, `seqno` and `topicIDs` returned in JSON responses are now encoded in multibase - Peer IDs returned in `from` now use the same default text representation from go-libp2p and peerid encoder/decoder from libp2p. This means the same text representation as in as in `swarm peers`, which makes it possible to compare them without decoding multibase. - `/api/v0/pubsub/pub` no longer accepts `data` to be passed as URL, it has to be sent as `multipart/form-data`. This removes size limitations based on URL length, and enables regular HTTP clients to publish data to PubSub topics. For example, to publish `some-file` to topic named `test-topic` using vanilla `curl`, one would execute: `curl -X POST -v -F "stdin=@some-file" 'http://127.0.0.1:5001/api/v0/pubsub/pub?arg=$(echo -n test-topic | ipfs multibase encode -b base64url)'` - `ipfs pubsub pub` on the command line no longer accepts variadic `data` arguments. Instead, it expects a single file input or stream of bytes from stdin. This ensures arbitrary stream of bytes can be published, removing limitation around messages that include `\n` or `\r\n`. #### ⚙ New configuration flags - [`Addresses.AppendAnnounce`](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#addressesappendannounce) is an array of multiaddrs, similar to `Addresses.Announce`, except it does not override inferred swarm addresses, but appends custom ones to the list. - Pubsub experiments can now be enabled via config, removing the need for CLI flag to be passed every time daemon starts: - [`Pubsub.Enabled`](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#pubsubenabled) enables the pubsub system. - [`Ipns.UsePubsub`](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#ipnsusepubsub) enables IPFS over pubsub experiment for publishing IPNS records in real time. #### 🔐 Support for DAG-JOSE IPLD codec JOSE is a [standard](https://datatracker.ietf.org/wg/jose/documents/) for signing and encrypting JSON objects. [DAG-JOSE](https://ipld.io/specs/codecs/dag-jose/spec/) is an IPLD codec based on JOSE and represented in CBOR. Upon encountering the `dag-jose` multicodec indicator, implementations can expect that the block contains dag-cbor encoded data which matches the IPLD schema from the [DAG-JOSE spec](https://ipld.io/specs/codecs/dag-jose/spec/). This work was [contributed](https://github.com/ipfs/go-ipfs/pull/8569) by [Ceramic](https://ceramic.network/) and acts as a template for future IPFS improvements driven by the real world needs of the IPFS community. ### Changelog - github.com/ipfs/go-ipfs: - docs: update changelog for v0.11.0 - Release v0.11.0-rc2 - fix(corehttp): adjust peer counting metrics (#8577) ([ipfs/go-ipfs#8577](https://github.com/ipfs/go-ipfs/pull/8577)) - Release v0.11.0-rc1 - feat: Swarm.EnableHolePunching flag (#8562) ([ipfs/go-ipfs#8562](https://github.com/ipfs/go-ipfs/pull/8562)) - feat: enabling pubsub and ipns-pubsub via config flags (#8510) ([ipfs/go-ipfs#8510](https://github.com/ipfs/go-ipfs/pull/8510)) - Integrate go-dag-jose plugin (#8569) ([ipfs/go-ipfs#8569](https://github.com/ipfs/go-ipfs/pull/8569)) - feat: Addresses.AppendAnnounce (#8177) ([ipfs/go-ipfs#8177](https://github.com/ipfs/go-ipfs/pull/8177)) - fix: multibase in pubsub http rpc (#8183) ([ipfs/go-ipfs#8183](https://github.com/ipfs/go-ipfs/pull/8183)) - refactor: remove dir-index-html submodule ([ipfs/go-ipfs#8555](https://github.com/ipfs/go-ipfs/pull/8555)) - feat: hard deprecation of IPFS_REUSEPORT - feat: go-libp2p 0.16, UnixFS autosharding and go-datastore with contexts (#8563) ([ipfs/go-ipfs#8563](https://github.com/ipfs/go-ipfs/pull/8563)) - chore: fix link in README.md (#8551) ([ipfs/go-ipfs#8551](https://github.com/ipfs/go-ipfs/pull/8551)) - Updating release template based off some 0.10 learnings (#8491) ([ipfs/go-ipfs#8491](https://github.com/ipfs/go-ipfs/pull/8491)) - fix: multiple subdomain gateways on same domain (#8556) ([ipfs/go-ipfs#8556](https://github.com/ipfs/go-ipfs/pull/8556)) - Fix typos (#8548) ([ipfs/go-ipfs#8548](https://github.com/ipfs/go-ipfs/pull/8548)) - Add support for multiple files to `ipfs files rm`. - add a docker-compose file (#8387) ([ipfs/go-ipfs#8387](https://github.com/ipfs/go-ipfs/pull/8387)) - fix(sharness): use -Q option instead of pipe to tail cmd - Add Homebrew installation method. ([ipfs/go-ipfs#8545](https://github.com/ipfs/go-ipfs/pull/8545)) - docs: fix ipfs files cp examples (#8533) ([ipfs/go-ipfs#8533](https://github.com/ipfs/go-ipfs/pull/8533)) - fix(unixfs): check for errors before dereferencing the link ([ipfs/go-ipfs#8508](https://github.com/ipfs/go-ipfs/pull/8508)) - chore: replace go-merkledag walk with go-ipld-prime traversal for dag export (#8506) ([ipfs/go-ipfs#8506](https://github.com/ipfs/go-ipfs/pull/8506)) - test: add sharness test for reading ADLs with FUSE - fix: allow the levelds compression level to be unspecified - ([ipfs/go-ipfs#8457](https://github.com/ipfs/go-ipfs/pull/8457)) - ([ipfs/go-ipfs#8482](https://github.com/ipfs/go-ipfs/pull/8482)) - Added the missing heathcheck for the container (#8429) ([ipfs/go-ipfs#8429](https://github.com/ipfs/go-ipfs/pull/8429)) - chore: update dir-index-html to v1.2.2 - Update RELEASE_ISSUE_TEMPLATE.md - Update RELEASE_ISSUE_TEMPLATE.md - add more logging to flaky TestPeersTotal - Update RELEASE_ISSUE_TEMPLATE.md - Update RELEASE_ISSUE_TEMPLATE.md - Updating chocolatey to reference go-ipfs - chore: update changelog for v0.10.0 - add testground plans to bitswap on CI - ci: move Docker image build to Actions (#8467) ([ipfs/go-ipfs#8467](https://github.com/ipfs/go-ipfs/pull/8467)) - fix(cli): object add-link: do not allow blocks over BS limit (#8414) ([ipfs/go-ipfs#8414](https://github.com/ipfs/go-ipfs/pull/8414)) - fuse: load unixfs adls as their dagpb substrates - enable the legacy mDNS implementation - change ipfs dag get flag name from format to output-codec ([ipfs/go-ipfs#8440](https://github.com/ipfs/go-ipfs/pull/8440)) - change names of ipfs dag put flags to make changes clearer ([ipfs/go-ipfs#8439](https://github.com/ipfs/go-ipfs/pull/8439)) - test: check behavior of loading UnixFS sharded directories with missing shards - ([ipfs/go-ipfs#8432](https://github.com/ipfs/go-ipfs/pull/8432)) - feat: dag import --stats (#8237) ([ipfs/go-ipfs#8237](https://github.com/ipfs/go-ipfs/pull/8237)) - feat: ipfs-webui v2.13.0 (#8430) ([ipfs/go-ipfs#8430](https://github.com/ipfs/go-ipfs/pull/8430)) - feat(cli): add daemon option --agent-version-suffix (#8419) ([ipfs/go-ipfs#8419](https://github.com/ipfs/go-ipfs/pull/8419)) - feat: multibase transcode command (#8403) ([ipfs/go-ipfs#8403](https://github.com/ipfs/go-ipfs/pull/8403)) - fix: take the lock while listing peers - feature: 'ipfs swarm peering' command (#8147) ([ipfs/go-ipfs#8147](https://github.com/ipfs/go-ipfs/pull/8147)) - chore: update IPFS Desktop testing steps (#8393) ([ipfs/go-ipfs#8393](https://github.com/ipfs/go-ipfs/pull/8393)) - add more buttons; remove some sections covered in the docs; general cleanup ([ipfs/go-ipfs#8274](https://github.com/ipfs/go-ipfs/pull/8274)) - Cosmetic fixups in examples (#8325) ([ipfs/go-ipfs#8325](https://github.com/ipfs/go-ipfs/pull/8325)) - perf: use performance-enhancing FUSE mount options - ci: publish Docker images for bifrost-* branches - chore: add comments to peerlog plugin about being unsupported - test: add unit tests for peerlog config parsing - ci: preload peerlog plugin, disable by default - fix(mkreleaselog): specify the parent commit when diffing - update version to v0.11.0-dev - github.com/ipfs/go-bitswap (v0.4.0 -> v0.5.1): - Version 0.5.1 - Change incorrect function name in README (#541) ([ipfs/go-bitswap#541](https://github.com/ipfs/go-bitswap/pull/541)) - Version 0.5.0 (#540) ([ipfs/go-bitswap#540](https://github.com/ipfs/go-bitswap/pull/540)) - feat: plumb through contexts (#539) ([ipfs/go-bitswap#539](https://github.com/ipfs/go-bitswap/pull/539)) - sync: update CI config files (#538) ([ipfs/go-bitswap#538](https://github.com/ipfs/go-bitswap/pull/538)) - fix: optimize handling for peers with lots of tasks ([ipfs/go-bitswap#537](https://github.com/ipfs/go-bitswap/pull/537)) - Enable custom task prioritization logic ([ipfs/go-bitswap#535](https://github.com/ipfs/go-bitswap/pull/535)) - feat: cache the materialized wantlist ([ipfs/go-bitswap#530](https://github.com/ipfs/go-bitswap/pull/530)) - fix: reduce receive contention ([ipfs/go-bitswap#536](https://github.com/ipfs/go-bitswap/pull/536)) - Fix ProviderQueryManager test timings ([ipfs/go-bitswap#534](https://github.com/ipfs/go-bitswap/pull/534)) - fix: rename wiretap to tracer ([ipfs/go-bitswap#531](https://github.com/ipfs/go-bitswap/pull/531)) - fix: fix race on "responsive" check ([ipfs/go-bitswap#528](https://github.com/ipfs/go-bitswap/pull/528)) - fix: reduce log verbosity - github.com/ipfs/go-blockservice (v0.1.7 -> v0.2.1): - Version 0.2.1 - Version 0.2.0 (#87) ([ipfs/go-blockservice#87](https://github.com/ipfs/go-blockservice/pull/87)) - feat: add context to interfaces (#86) ([ipfs/go-blockservice#86](https://github.com/ipfs/go-blockservice/pull/86)) - sync: update CI config files (#85) ([ipfs/go-blockservice#85](https://github.com/ipfs/go-blockservice/pull/85)) - chore: update log ([ipfs/go-blockservice#84](https://github.com/ipfs/go-blockservice/pull/84)) - github.com/ipfs/go-cid (v0.0.7 -> v0.1.0): - amend the CidFromReader slice extension math - implement CidFromReader - chore: fixups from running go vet, go fmt and staticcheck ([ipfs/go-cid#122](https://github.com/ipfs/go-cid/pull/122)) - s/characters/bytes - Fix inaccurate comment for uvarint - coverage: more tests for cid - coverage: more tests for varint - coverage: more tests for builder - fix: make tests run with Go 1.15 - Add the dagjose multiformat - github.com/ipfs/go-datastore (v0.4.6 -> v0.5.1): - Release v0.5.1 - chore: add lots of interface assertions - fix: make NullDatastore satisfy the Batching interface again - Update version.json (#183) ([ipfs/go-datastore#183](https://github.com/ipfs/go-datastore/pull/183)) - feat: add context to interfaces (#181) ([ipfs/go-datastore#181](https://github.com/ipfs/go-datastore/pull/181)) - sync: update CI config files ([ipfs/go-datastore#182](https://github.com/ipfs/go-datastore/pull/182)) - github.com/ipfs/go-ds-badger (v0.2.7 -> v0.3.0): - feat: plumb through contexts (#119) ([ipfs/go-ds-badger#119](https://github.com/ipfs/go-ds-badger/pull/119)) - github.com/ipfs/go-ds-flatfs (v0.4.5 -> v0.5.1): - Update version.json - fix: add context to DiskUsage() - Version 0.5.0 (#99) ([ipfs/go-ds-flatfs#99](https://github.com/ipfs/go-ds-flatfs/pull/99)) - feat: add contexts on datastore methods (#98) ([ipfs/go-ds-flatfs#98](https://github.com/ipfs/go-ds-flatfs/pull/98)) - sync: update CI config files (#97) ([ipfs/go-ds-flatfs#97](https://github.com/ipfs/go-ds-flatfs/pull/97)) - sync: update CI config files ([ipfs/go-ds-flatfs#96](https://github.com/ipfs/go-ds-flatfs/pull/96)) - fix staticcheck ([ipfs/go-ds-flatfs#92](https://github.com/ipfs/go-ds-flatfs/pull/92)) - fix typo in readme.go ([ipfs/go-ds-flatfs#89](https://github.com/ipfs/go-ds-flatfs/pull/89)) - github.com/ipfs/go-ds-leveldb (v0.4.2 -> v0.5.0): - Version 0.5.0 (#58) ([ipfs/go-ds-leveldb#58](https://github.com/ipfs/go-ds-leveldb/pull/58)) - feat: plumb through contexts (#57) ([ipfs/go-ds-leveldb#57](https://github.com/ipfs/go-ds-leveldb/pull/57)) - sync: update CI config files (#56) ([ipfs/go-ds-leveldb#56](https://github.com/ipfs/go-ds-leveldb/pull/56)) - fix closing of datastore in tests ([ipfs/go-ds-leveldb#52](https://github.com/ipfs/go-ds-leveldb/pull/52)) - fix staticcheck ([ipfs/go-ds-leveldb#49](https://github.com/ipfs/go-ds-leveldb/pull/49)) - fix typo in function documentation ([ipfs/go-ds-leveldb#46](https://github.com/ipfs/go-ds-leveldb/pull/46)) - github.com/ipfs/go-ds-measure (v0.1.0 -> v0.2.0): - Version 0.2.0 (#39) ([ipfs/go-ds-measure#39](https://github.com/ipfs/go-ds-measure/pull/39)) - feat: add contexts on datastore methods (#38) ([ipfs/go-ds-measure#38](https://github.com/ipfs/go-ds-measure/pull/38)) - sync: update CI config files (#37) ([ipfs/go-ds-measure#37](https://github.com/ipfs/go-ds-measure/pull/37)) - github.com/ipfs/go-fetcher (v1.5.0 -> v1.6.1): - Version 1.6.1 - Version 1.6.0 (#29) ([ipfs/go-fetcher#29](https://github.com/ipfs/go-fetcher/pull/29)) - feat: plumb through context changes (#28) ([ipfs/go-fetcher#28](https://github.com/ipfs/go-fetcher/pull/28)) - sync: update CI config files (#27) ([ipfs/go-fetcher#27](https://github.com/ipfs/go-fetcher/pull/27)) - add a fetcher constructor for the case where we already have a session ([ipfs/go-fetcher#26](https://github.com/ipfs/go-fetcher/pull/26)) - github.com/ipfs/go-filestore (v0.0.3 -> v0.1.0): - feat: plumb through context changes (#56) ([ipfs/go-filestore#56](https://github.com/ipfs/go-filestore/pull/56)) - github.com/ipfs/go-graphsync (v0.8.0 -> v0.11.0): - docs(CHANGELOG): update for v0.11.0 release - Merge branch 'release/v0.10.6' - update to context datastores (#275) ([ipfs/go-graphsync#275](https://github.com/ipfs/go-graphsync/pull/275)) - feat!(requestmanager): remove request allocation backpressure (#272) ([ipfs/go-graphsync#272](https://github.com/ipfs/go-graphsync/pull/272)) - message/pb: stop using gogo/protobuf (#277) ([ipfs/go-graphsync#277](https://github.com/ipfs/go-graphsync/pull/277)) - mark all test helper funcs via t.Helper (#276) ([ipfs/go-graphsync#276](https://github.com/ipfs/go-graphsync/pull/276)) - chore(queryexecutor): remove unused RunTraversal - chore(responsemanager): remove unused workSignal - chore(queryexecutor): fix tests for runtraversal refactor + clean up - feat(queryexecutor): merge RunTraversal into QueryExecutor - feat(responsemanager): QueryExecutor to separate module - use TaskQueue, add tests - Merge branch 'release/v0.10.5' - fix(responseassembler): don't hold block data reference in passed on subscribed block link (#268) ([ipfs/go-graphsync#268](https://github.com/ipfs/go-graphsync/pull/268)) - sync: update CI config files (#266) ([ipfs/go-graphsync#266](https://github.com/ipfs/go-graphsync/pull/266)) - Check IPLD context cancellation error type instead of string comparison - Use `context.CancelFunc` instead of `func()` (#257) ([ipfs/go-graphsync#257](https://github.com/ipfs/go-graphsync/pull/257)) - fix: bail properly when budget exceeded - feat(requestmanager): report inProgressRequestCount on OutgoingRequests event - fix(requestmanager): remove failing racy test select block - feat(requestmanager): add OutgoingRequeustProcessingListener - Merge branch 'release/v0.10.4' - fix(allocator): prevent buffer overflow (#248) ([ipfs/go-graphsync#248](https://github.com/ipfs/go-graphsync/pull/248)) - Merge branch 'release/v0.10.3' - Configure message parameters (#247) ([ipfs/go-graphsync#247](https://github.com/ipfs/go-graphsync/pull/247)) - Stats! (#246) ([ipfs/go-graphsync#246](https://github.com/ipfs/go-graphsync/pull/246)) - Limit simultaneous incoming requests on a per peer basis (#245) ([ipfs/go-graphsync#245](https://github.com/ipfs/go-graphsync/pull/245)) - sync: update CI config files (#191) ([ipfs/go-graphsync#191](https://github.com/ipfs/go-graphsync/pull/191)) - Merge branch 'release/v0.10.2' - test(responsemanager): fix flakiness TestCancellationViaCommand (#243) ([ipfs/go-graphsync#243](https://github.com/ipfs/go-graphsync/pull/243)) - Fix deadlock on notifications (#242) ([ipfs/go-graphsync#242](https://github.com/ipfs/go-graphsync/pull/242)) - Merge branch 'release/v0.10.1' - Free memory on request finish (#240) ([ipfs/go-graphsync#240](https://github.com/ipfs/go-graphsync/pull/240)) - release: v1.10.0 ([ipfs/go-graphsync#238](https://github.com/ipfs/go-graphsync/pull/238)) - Add support for IPLD prime's budgets feature in selectors (#235) ([ipfs/go-graphsync#235](https://github.com/ipfs/go-graphsync/pull/235)) - feat(graphsync): add an index for blocks in the on new block hook (#234) ([ipfs/go-graphsync#234](https://github.com/ipfs/go-graphsync/pull/234)) - Do not send first blocks extension (#230) ([ipfs/go-graphsync#230](https://github.com/ipfs/go-graphsync/pull/230)) - Protect Libp2p Connections (#229) ([ipfs/go-graphsync#229](https://github.com/ipfs/go-graphsync/pull/229)) - test(responsemanager): remove check (#228) ([ipfs/go-graphsync#228](https://github.com/ipfs/go-graphsync/pull/228)) - feat(graphsync): give missing blocks a named error (#227) ([ipfs/go-graphsync#227](https://github.com/ipfs/go-graphsync/pull/227)) - Add request limits (#224) ([ipfs/go-graphsync#224](https://github.com/ipfs/go-graphsync/pull/224)) - Tech Debt Cleanup and Docs Update (#219) ([ipfs/go-graphsync#219](https://github.com/ipfs/go-graphsync/pull/219)) - Release/v0.9.3 ([ipfs/go-graphsync#218](https://github.com/ipfs/go-graphsync/pull/218)) - 0.9.2 release ([ipfs/go-graphsync#217](https://github.com/ipfs/go-graphsync/pull/217)) - fix(requestmanager): remove main thread block on allocation (#216) ([ipfs/go-graphsync#216](https://github.com/ipfs/go-graphsync/pull/216)) - feat(allocator): add debug logging (#213) ([ipfs/go-graphsync#213](https://github.com/ipfs/go-graphsync/pull/213)) - fix: spurious warn log (#210) ([ipfs/go-graphsync#210](https://github.com/ipfs/go-graphsync/pull/210)) - docs(CHANGELOG): update for v0.9.1 release (#212) ([ipfs/go-graphsync#212](https://github.com/ipfs/go-graphsync/pull/212)) - fix(message): fix dropping of response extensions (#211) ([ipfs/go-graphsync#211](https://github.com/ipfs/go-graphsync/pull/211)) - docs(CHANGELOG): update change log ([ipfs/go-graphsync#208](https://github.com/ipfs/go-graphsync/pull/208)) - docs(README): add notice about branch rename - fix(graphsync): make sure linkcontext is passed (#207) ([ipfs/go-graphsync#207](https://github.com/ipfs/go-graphsync/pull/207)) - Merge final v0.6.x commit history, and 0.8.0 changelog (#205) ([ipfs/go-graphsync#205](https://github.com/ipfs/go-graphsync/pull/205)) - Fix broken link to IPLD selector documentation (#189) ([ipfs/go-graphsync#189](https://github.com/ipfs/go-graphsync/pull/189)) - fix: check errors before deferring a close (#200) ([ipfs/go-graphsync#200](https://github.com/ipfs/go-graphsync/pull/200)) - chore: fix checks (#197) ([ipfs/go-graphsync#197](https://github.com/ipfs/go-graphsync/pull/197)) - Merge the v0.6.x commit history (#190) ([ipfs/go-graphsync#190](https://github.com/ipfs/go-graphsync/pull/190)) - Ready for universal CI (#187) ([ipfs/go-graphsync#187](https://github.com/ipfs/go-graphsync/pull/187)) - fix(requestmanager): pass through linksystem (#166) ([ipfs/go-graphsync#166](https://github.com/ipfs/go-graphsync/pull/166)) - fix missing word in section title (#179) ([ipfs/go-graphsync#179](https://github.com/ipfs/go-graphsync/pull/179)) - github.com/ipfs/go-ipfs-blockstore (v0.1.6 -> v0.2.1): - fix: revert back to go-ipfs-ds-help@v0.1.1 (#92) ([ipfs/go-ipfs-blockstore#92](https://github.com/ipfs/go-ipfs-blockstore/pull/92)) - feat: add context to interfaces & plumb through datastore contexts (#89) ([ipfs/go-ipfs-blockstore#89](https://github.com/ipfs/go-ipfs-blockstore/pull/89)) - github.com/ipfs/go-ipfs-config (v0.16.0 -> v0.18.0): - Release v0.18.0 (#159) ([ipfs/go-ipfs-config#159](https://github.com/ipfs/go-ipfs-config/pull/159)) - feat: add Addresses.AppendAnnounce (#135) ([ipfs/go-ipfs-config#135](https://github.com/ipfs/go-ipfs-config/pull/135)) - feat: omitempty Swarm.EnableRelayHop for circuit v1 migration (#157) ([ipfs/go-ipfs-config#157](https://github.com/ipfs/go-ipfs-config/pull/157)) - chore: omitempty Experimental.ShardingEnabled (#158) ([ipfs/go-ipfs-config#158](https://github.com/ipfs/go-ipfs-config/pull/158)) - chore: update comment to match struct - Release v0.17.0 (#156) ([ipfs/go-ipfs-config#156](https://github.com/ipfs/go-ipfs-config/pull/156)) - feat: add a flag to enable the hole punching service (#155) ([ipfs/go-ipfs-config#155](https://github.com/ipfs/go-ipfs-config/pull/155)) - improve AutoRelay configuration, add config option for static relays ([ipfs/go-ipfs-config#154](https://github.com/ipfs/go-ipfs-config/pull/154)) - feat: Swarm.RelayService (circuit v2) (#146) ([ipfs/go-ipfs-config#146](https://github.com/ipfs/go-ipfs-config/pull/146)) - fix: String method on the OptionalString (#153) ([ipfs/go-ipfs-config#153](https://github.com/ipfs/go-ipfs-config/pull/153)) - sync: update CI config files (#152) ([ipfs/go-ipfs-config#152](https://github.com/ipfs/go-ipfs-config/pull/152)) - feat: OptionalString type and UnixFSShardingSizeThreshold (#149) ([ipfs/go-ipfs-config#149](https://github.com/ipfs/go-ipfs-config/pull/149)) - feat: pubsub and ipns pubsub flags (#145) ([ipfs/go-ipfs-config#145](https://github.com/ipfs/go-ipfs-config/pull/145)) - feat: add an OptionalDuration type (#148) ([ipfs/go-ipfs-config#148](https://github.com/ipfs/go-ipfs-config/pull/148)) - github.com/ipfs/go-ipfs-exchange-interface (v0.0.1 -> v0.1.0): - Update version.json (#20) ([ipfs/go-ipfs-exchange-interface#20](https://github.com/ipfs/go-ipfs-exchange-interface/pull/20)) - sync: update CI config files (#19) ([ipfs/go-ipfs-exchange-interface#19](https://github.com/ipfs/go-ipfs-exchange-interface/pull/19)) - feat: add context to interface (#18) ([ipfs/go-ipfs-exchange-interface#18](https://github.com/ipfs/go-ipfs-exchange-interface/pull/18)) - doc: add a lead maintainer - github.com/ipfs/go-ipfs-exchange-offline (v0.0.1 -> v0.1.1): - Version 0.1.1 - Version 0.1.0 (#43) ([ipfs/go-ipfs-exchange-offline#43](https://github.com/ipfs/go-ipfs-exchange-offline/pull/43)) - feat: plumb through contexts (#42) ([ipfs/go-ipfs-exchange-offline#42](https://github.com/ipfs/go-ipfs-exchange-offline/pull/42)) - sync: update CI config files (#41) ([ipfs/go-ipfs-exchange-offline#41](https://github.com/ipfs/go-ipfs-exchange-offline/pull/41)) - fix staticcheck ([ipfs/go-ipfs-exchange-offline#35](https://github.com/ipfs/go-ipfs-exchange-offline/pull/35)) - chore(gx): remove gx - github.com/ipfs/go-ipfs-files (v0.0.8 -> v0.0.9): - sync: update CI config files ([ipfs/go-ipfs-files#40](https://github.com/ipfs/go-ipfs-files/pull/40)) - fix: manually parse the content disposition to preserve directories ([ipfs/go-ipfs-files#42](https://github.com/ipfs/go-ipfs-files/pull/42)) - fix: round timestamps down by truncating them to seconds ([ipfs/go-ipfs-files#41](https://github.com/ipfs/go-ipfs-files/pull/41)) - sync: update CI config files ([ipfs/go-ipfs-files#34](https://github.com/ipfs/go-ipfs-files/pull/34)) - Fix test failure on Windows caused by nil `sys` in mock `FileInfo` ([ipfs/go-ipfs-files#39](https://github.com/ipfs/go-ipfs-files/pull/39)) - fix staticcheck ([ipfs/go-ipfs-files#35](https://github.com/ipfs/go-ipfs-files/pull/35)) - fix linters ([ipfs/go-ipfs-files#33](https://github.com/ipfs/go-ipfs-files/pull/33)) - github.com/ipfs/go-ipfs-pinner (v0.1.2 -> v0.2.1): - feat: plumb through context changes (#18) ([ipfs/go-ipfs-pinner#18](https://github.com/ipfs/go-ipfs-pinner/pull/18)) - github.com/ipfs/go-ipfs-provider (v0.6.1 -> v0.7.1): - Fix go vet and staticcheck ([ipfs/go-ipfs-provider#40](https://github.com/ipfs/go-ipfs-provider/pull/40)) - feat: plumb through datastore contexts (#39) ([ipfs/go-ipfs-provider#39](https://github.com/ipfs/go-ipfs-provider/pull/39)) - github.com/ipfs/go-ipfs-routing (v0.1.0 -> v0.2.1): - Version 0.2.1 - Bump version to 0.2.0 (#29) ([ipfs/go-ipfs-routing#29](https://github.com/ipfs/go-ipfs-routing/pull/29)) - feat: plumb through context changes (#28) ([ipfs/go-ipfs-routing#28](https://github.com/ipfs/go-ipfs-routing/pull/28)) - sync: update CI config files (#27) ([ipfs/go-ipfs-routing#27](https://github.com/ipfs/go-ipfs-routing/pull/27)) - fix staticcheck ([ipfs/go-ipfs-routing#24](https://github.com/ipfs/go-ipfs-routing/pull/24)) - github.com/ipfs/go-merkledag (v0.4.0 -> v0.5.1): - Version 0.5.1 - Version 0.5.0 (#79) ([ipfs/go-merkledag#79](https://github.com/ipfs/go-merkledag/pull/79)) - feat: plumb through contexts (#78) ([ipfs/go-merkledag#78](https://github.com/ipfs/go-merkledag/pull/78)) - sync: update CI config files (#77) ([ipfs/go-merkledag#77](https://github.com/ipfs/go-merkledag/pull/77)) - expose session construction to other callers - fix RawNode incomplete stats - github.com/ipfs/go-mfs (v0.1.2 -> v0.2.1): - Version 0.2.1 - Version 0.2.0 (#96) ([ipfs/go-mfs#96](https://github.com/ipfs/go-mfs/pull/96)) - support threshold based automatic sharding and unsharding of directories (#88) ([ipfs/go-mfs#88](https://github.com/ipfs/go-mfs/pull/88)) - sync: update CI config files (#94) ([ipfs/go-mfs#94](https://github.com/ipfs/go-mfs/pull/94)) - Fix lint errors ([ipfs/go-mfs#90](https://github.com/ipfs/go-mfs/pull/90)) - remove Makefile ([ipfs/go-mfs#89](https://github.com/ipfs/go-mfs/pull/89)) - github.com/ipfs/go-namesys (v0.3.1 -> v0.4.0): - Release v0.4.0 - feat: plumb through datastore contexts - sync: update CI config files (#23) ([ipfs/go-namesys#23](https://github.com/ipfs/go-namesys/pull/23)) - github.com/ipfs/go-path (v0.1.2 -> v0.2.1): - Version 0.2.1 - Version 0.2.0 (#48) ([ipfs/go-path#48](https://github.com/ipfs/go-path/pull/48)) - feat: plumb through context changes (#47) ([ipfs/go-path#47](https://github.com/ipfs/go-path/pull/47)) - sync: update CI config files (#46) ([ipfs/go-path#46](https://github.com/ipfs/go-path/pull/46)) - Revert "feat: plumb through context changes" - feat: plumb through context changes - github.com/ipfs/go-peertaskqueue (v0.4.0 -> v0.7.0): - feat: optimize checking if a new task is "better" ([ipfs/go-peertaskqueue#19](https://github.com/ipfs/go-peertaskqueue/pull/19)) - Adds customizable prioritization logic for peertracker and peertaskqueue ([ipfs/go-peertaskqueue#17](https://github.com/ipfs/go-peertaskqueue/pull/17)) - When priority is equal, use FIFO ([ipfs/go-peertaskqueue#16](https://github.com/ipfs/go-peertaskqueue/pull/16)) - github.com/ipfs/go-unixfs (v0.2.5 -> v0.3.1): - Version 0.3.1 - Version 0.3.0 (#114) ([ipfs/go-unixfs#114](https://github.com/ipfs/go-unixfs/pull/114)) - feat: plumb through datastore context changes - Size-based unsharding (#94) ([ipfs/go-unixfs#94](https://github.com/ipfs/go-unixfs/pull/94)) - sync: update CI config files (#112) ([ipfs/go-unixfs#112](https://github.com/ipfs/go-unixfs/pull/112)) - chore(deps): move bitfield to ipfs org ([ipfs/go-unixfs#98](https://github.com/ipfs/go-unixfs/pull/98)) - fix staticcheck ([ipfs/go-unixfs#95](https://github.com/ipfs/go-unixfs/pull/95)) - fix(directory): initialize size when computing it ([ipfs/go-unixfs#93](https://github.com/ipfs/go-unixfs/pull/93)) - fix: always return upgradeable instead of basic dir (#92) ([ipfs/go-unixfs#92](https://github.com/ipfs/go-unixfs/pull/92)) - feat: switch to HAMT based on size (#91) ([ipfs/go-unixfs#91](https://github.com/ipfs/go-unixfs/pull/91)) - go fmt - fix: add pointer receiver - add test - feat: add UpgradeableDirectory - github.com/ipfs/interface-go-ipfs-core (v0.5.1 -> v0.5.2): - fix: check errors by string ([ipfs/interface-go-ipfs-core#76](https://github.com/ipfs/interface-go-ipfs-core/pull/76)) - github.com/ipfs/tar-utils (v0.0.1 -> v0.0.2): - Release v0.0.2 (#8) ([ipfs/tar-utils#8](https://github.com/ipfs/tar-utils/pull/8)) - sync: update CI config files ([ipfs/tar-utils#7](https://github.com/ipfs/tar-utils/pull/7)) - sync: update CI config files (#6) ([ipfs/tar-utils#6](https://github.com/ipfs/tar-utils/pull/6)) - allow .. in file and directory names ([ipfs/tar-utils#5](https://github.com/ipfs/tar-utils/pull/5)) - github.com/ipld/go-car (v0.3.1 -> v0.3.2): - Expose selector traversal options for SelectiveCar ([ipld/go-car#251](https://github.com/ipld/go-car/pull/251)) - Implement API to allow replacing root CIDs in a CARv1 or CARv2 - blockstore: OpenReadWrite should not modify if it refuses to resume - clarify the relation between StoreIdentityCIDs and SetFullyIndexed - Implement options to handle `IDENTITY` CIDs gracefully - Combine API options for simplicity and logical coherence - Add test script for car verify (#236) ([ipld/go-car#236](https://github.com/ipld/go-car/pull/236)) - cmd/car: add first testscript tests - integrate `car/` cli into `cmd/car` (#233) ([ipld/go-car#233](https://github.com/ipld/go-car/pull/233)) - Add `car get-dag` command (#232) ([ipld/go-car#232](https://github.com/ipld/go-car/pull/232)) - Separate CLI to separate module (#231) ([ipld/go-car#231](https://github.com/ipld/go-car/pull/231)) - add `get block` to car cli (#230) ([ipld/go-car#230](https://github.com/ipld/go-car/pull/230)) - use file size when loading from v1 car (#229) ([ipld/go-car#229](https://github.com/ipld/go-car/pull/229)) - add interface describing iteration (#228) ([ipld/go-car#228](https://github.com/ipld/go-car/pull/228)) - Add `list` and `filter` commands (#227) ([ipld/go-car#227](https://github.com/ipld/go-car/pull/227)) - Add `car split` command (#226) ([ipld/go-car#226](https://github.com/ipld/go-car/pull/226)) - Make `MultihashIndexSorted` the default index codec for CARv2 - Add carve utility for updating the index of a car{v1,v2} file (#219) ([ipld/go-car#219](https://github.com/ipld/go-car/pull/219)) - Ignore records with `IDENTITY` CID in `IndexSorted` - Fix index GetAll infinite loop if function always returns `true` - Expose the ability to iterate over records in `MultihasIndexSorted` - avoid another alloc per read byte - avoid allocating on every byte read - Implement new index type that also includes mutltihash code - Return `nil` as Index reader when reading indexless CARv2 - Assert `OpenReader` from file does not panic after closure - Document performance caveats of `ExtractV1File` and address comments - Implement utility to extract CARv1 from a CARv2 - v2/blockstore: add ReadWrite.Discard - update LICENSE files to point to the new gateway - re-add root LICENSE file - v2: stop using a symlink for LICENSE.md - Update the readme with link to examples - update package godocs and root level README for v2 - blockstore: stop embedding ReadOnly in ReadWrite - Implement version agnostic streaming CAR block iterator - blockstore: use errors when API contracts are broken - add the first read-only benchmarks - Implement reader block iterator over CARv1 or CARv2 - Propagate async `blockstore.AllKeysChan` errors via context - Add zero-length sections as EOF option to internal CARv1 reader - Improve error handing in tests - Allow `ReadOption`s to be set when getting or generating index - Use `ioutil.TempFile` to simplify file creation in index example - Avoid writing to files in testdata - blockstore: implement UseWholeCIDs - Merge wip-v2 into master (#178) ([ipld/go-car#178](https://github.com/ipld/go-car/pull/178)) - github.com/ipld/go-ipld-prime (v0.12.2 -> v0.14.2): - dagcbor: coerce undef to null. ([ipld/go-ipld-prime#308](https://github.com/ipld/go-ipld-prime/pull/308)) - fluent: add toInterface (#304) ([ipld/go-ipld-prime#304](https://github.com/ipld/go-ipld-prime/pull/304)) - traversal: s/Walk/WalkLocal/ - traversal: add a primitive walk function. - Remove dependency to `go-wish` - mark v0.14.0 - ([ipld/go-ipld-prime#279](https://github.com/ipld/go-ipld-prime/pull/279)) - Port `traversal` package tests to quicktest - Port `codec` package tests to quicktest - changelog: backfill. - Gracefully handle `TypedNode` with `nil` type of kind `Map` - Gracefully print typed nodes with `nil` type - Implement handling of `Link` and `[]byte` in `printer` (#294) ([ipld/go-ipld-prime#294](https://github.com/ipld/go-ipld-prime/pull/294)) - changelog: backfill for the v0.12.x series. - readme: introduce a migration guide. - Port `fluent` package tests to quicktest - Port `datamodel` package tests to quicktest - Port `adl` package tests to quicktest - Port `node` package tests to quicktest - node/bindnode: support links in ProduceGoTypes - bump CI to Go 1.16 and 1.17 - node/bindnode: support links in schema-type verification - node/bindnode: export ProduceGoTypes - all: fix "an" typos after the ipld->datamodel refactor - node/bindnode: fix test code after two PR merges - add LoadSchema APIs to the root package - storage: add 'Has' feature. ([ipld/go-ipld-prime#276](https://github.com/ipld/go-ipld-prime/pull/276)) - node/bindnode: start verifying schema compatibility - linking: add LoadRaw and LoadPlusRaw functions to LinkSystem. ([ipld/go-ipld-prime#267](https://github.com/ipld/go-ipld-prime/pull/267)) - node/bindnode: add support for lists behind kinded unions - node/bindnode: also run TestPrototype with just schemas - node/bindnode: polish a few TODO panics away - node/bindnode: add support for all scalars behind kinded unions - node/bindnode: get closer to passing the Links schema tests - start using Rod's schema tests from ipld/ipld - fully support parsing, encoding, and decoding the schema-schema - node/bindnode: add native support for cid.Cid - A more Featureful Approach to Storage APIs ([ipld/go-ipld-prime#265](https://github.com/ipld/go-ipld-prime/pull/265)) - Add a cidlink.Memory storage option (#266) ([ipld/go-ipld-prime#266](https://github.com/ipld/go-ipld-prime/pull/266)) - Improve docs for AssignNode; and datamodel.Copy function. ([ipld/go-ipld-prime#264](https://github.com/ipld/go-ipld-prime/pull/264)) - schemadsl: assign the struct representation. - schema,tests,gen/go: more tests, gen union fixes. ([ipld/go-ipld-prime#257](https://github.com/ipld/go-ipld-prime/pull/257)) - fix: deal with LinkRevisit->LinkVisitOnlyOnce change - traversal: the link-visit-only-once behavior should require opt-in, rather than defaulting to on. - chore: add LinkRevisit:false traversal test - traversal: track seen links, and revisit only if configured to do so. - fix: use datamodel.Node selectors - Revert encode round-trip to leave unencoded node test intact - Add more walk tests, including tests for use of SkipMe - Round-trip test nodes through custom codec to ensure stability - Don't abort block processing when encountering SkipMe - traversal: implement monotonically decrementing budgets. ([ipld/go-ipld-prime#260](https://github.com/ipld/go-ipld-prime/pull/260)) - Use datamodel.Node for "Common" selector variants - schema/dmt: first pass at a parser ([ipld/go-ipld-prime#253](https://github.com/ipld/go-ipld-prime/pull/253)) - drop codectools. - drop jst codec. It lives in https://github.com/warpfork/go-jst/ now. - drop dagjson2. - fix(traversal): properly wrap errors - printer: empty maps and lists and structs should stay on one line. - schema: turn TypeName into an alias - schema/dmt: sync with schema-schema changes, finish Compile - schema: add ways to set and access the ImplicitValue for a struct field. - schema: accessor for TypeEnum.Representation. - schema: finish minimum viable support for describing enum types. - github.com/libp2p/go-conn-security-multistream (v0.2.1 -> v0.3.0): - use the new SecureTransport and SecureMuxer interfaces (#36) ([libp2p/go-conn-security-multistream#36](https://github.com/libp2p/go-conn-security-multistream/pull/36)) - fix go vet and staticcheck ([libp2p/go-conn-security-multistream#33](https://github.com/libp2p/go-conn-security-multistream/pull/33)) - github.com/libp2p/go-libp2p (v0.15.0 -> v0.16.0): - release v0.16.0 ([libp2p/go-libp2p#1246](https://github.com/libp2p/go-libp2p/pull/1246)) - allow the ping protocol on transient connections ([libp2p/go-libp2p#1244](https://github.com/libp2p/go-libp2p/pull/1244)) - make the Type field required in the HolePunch protobuf ([libp2p/go-libp2p#1241](https://github.com/libp2p/go-libp2p/pull/1241)) - reject hole punching attempts when we don't have any public addresses ([libp2p/go-libp2p#1214](https://github.com/libp2p/go-libp2p/pull/1214)) - refactor the AutoRelay code ([libp2p/go-libp2p#1240](https://github.com/libp2p/go-libp2p/pull/1240)) - remove dead API link in README ([libp2p/go-libp2p#1233](https://github.com/libp2p/go-libp2p/pull/1233)) - pass static relays to EnableAutoRelay, deprecate libp2p.StaticRelays and libp2p.DefaultStaticRelays ([libp2p/go-libp2p#1239](https://github.com/libp2p/go-libp2p/pull/1239)) - feat: plumb through peerstore context changes (#1237) ([libp2p/go-libp2p#1237](https://github.com/libp2p/go-libp2p/pull/1237)) - emit the EvtPeerConnectednessChanged event ([libp2p/go-libp2p#1230](https://github.com/libp2p/go-libp2p/pull/1230)) - update go-libp2p-swarm to v0.7.0 ([libp2p/go-libp2p#1226](https://github.com/libp2p/go-libp2p/pull/1226)) - sync: update CI config files (#1225) ([libp2p/go-libp2p#1225](https://github.com/libp2p/go-libp2p/pull/1225)) - simplify circuitv2 package structure ([libp2p/go-libp2p#1224](https://github.com/libp2p/go-libp2p/pull/1224)) - use a random string for the mDNS peer-name ([libp2p/go-libp2p#1222](https://github.com/libp2p/go-libp2p/pull/1222)) - remove {Un}RegisterNotifee functions from mDNS service ([libp2p/go-libp2p#1220](https://github.com/libp2p/go-libp2p/pull/1220)) - fix structured logging in holepunch coordination ([libp2p/go-libp2p#1213](https://github.com/libp2p/go-libp2p/pull/1213)) - fix flaky TestStBackpressureStreamWrite test ([libp2p/go-libp2p#1212](https://github.com/libp2p/go-libp2p/pull/1212)) - properly close hosts in mDNS tests ([libp2p/go-libp2p#1216](https://github.com/libp2p/go-libp2p/pull/1216)) - close the ObserverAddrManager when the ID service is closed ([libp2p/go-libp2p#1218](https://github.com/libp2p/go-libp2p/pull/1218)) - make it possible to pass options to a transport constructor ([libp2p/go-libp2p#1205](https://github.com/libp2p/go-libp2p/pull/1205)) - remove goprocess from the NATManager ([libp2p/go-libp2p#1193](https://github.com/libp2p/go-libp2p/pull/1193)) - add an option to start the relay v2 ([libp2p/go-libp2p#1197](https://github.com/libp2p/go-libp2p/pull/1197)) - fix flaky TestFastDisconnect identify test ([libp2p/go-libp2p#1200](https://github.com/libp2p/go-libp2p/pull/1200)) - chore: update go-tcp-transport to v0.3.0 ([libp2p/go-libp2p#1203](https://github.com/libp2p/go-libp2p/pull/1203)) - fix: skip variadic params in constructors ([libp2p/go-libp2p#1204](https://github.com/libp2p/go-libp2p/pull/1204)) - fix flaky BasicHost tests ([libp2p/go-libp2p#1202](https://github.com/libp2p/go-libp2p/pull/1202)) - remove dependency on github.com/ipfs/go-detect-race ([libp2p/go-libp2p#1201](https://github.com/libp2p/go-libp2p/pull/1201)) - fix flaky TestEndToEndSimConnect holepunching test ([libp2p/go-libp2p#1191](https://github.com/libp2p/go-libp2p/pull/1191)) - autorelay support for circuitv2 relays (#1198) ([libp2p/go-libp2p#1198](https://github.com/libp2p/go-libp2p/pull/1198)) - reject circuitv2 reservations with nonsensical expiration times ([libp2p/go-libp2p#1199](https://github.com/libp2p/go-libp2p/pull/1199)) - Tag relay hops in relay implementations ([libp2p/go-libp2p#1188](https://github.com/libp2p/go-libp2p/pull/1188)) - Add standalone implementation of v1 Relay (#1186) ([libp2p/go-libp2p#1186](https://github.com/libp2p/go-libp2p/pull/1186)) - remove the context from the libp2p and the Host constructor ([libp2p/go-libp2p#1190](https://github.com/libp2p/go-libp2p/pull/1190)) - don't use a context to shut down the circuitv2 ([libp2p/go-libp2p#1185](https://github.com/libp2p/go-libp2p/pull/1185)) - fix: remove v1 go-log dep ([libp2p/go-libp2p#1189](https://github.com/libp2p/go-libp2p/pull/1189)) - don't use the context to shut down the relay ([libp2p/go-libp2p#1184](https://github.com/libp2p/go-libp2p/pull/1184)) - Use circuitv2 code (#1183) ([libp2p/go-libp2p#1183](https://github.com/libp2p/go-libp2p/pull/1183)) - clean up badges in README ([libp2p/go-libp2p#1179](https://github.com/libp2p/go-libp2p/pull/1179)) - remove recommendation about Go module proxy from README ([libp2p/go-libp2p#1180](https://github.com/libp2p/go-libp2p/pull/1180)) - merge branch 'hole-punching' - don't use a context for closing the ObservedAddrManager ([libp2p/go-libp2p#1175](https://github.com/libp2p/go-libp2p/pull/1175)) - move the circuit v2 code here ([libp2p/go-libp2p#1174](https://github.com/libp2p/go-libp2p/pull/1174)) - make QUIC a default transport ([libp2p/go-libp2p#1128](https://github.com/libp2p/go-libp2p/pull/1128)) - stop using jbenet/go-cienv ([libp2p/go-libp2p#1176](https://github.com/libp2p/go-libp2p/pull/1176)) - fix flaky TestObsAddrSet test ([libp2p/go-libp2p#1172](https://github.com/libp2p/go-libp2p/pull/1172)) - clean up messy defer logic in IDService.sendIdentifyResp ([libp2p/go-libp2p#1169](https://github.com/libp2p/go-libp2p/pull/1169)) - remove secio from README, add noise ([libp2p/go-libp2p#1165](https://github.com/libp2p/go-libp2p/pull/1165)) - github.com/libp2p/go-libp2p-asn-util (v0.0.0-20200825225859-85005c6cf052 -> v0.1.0): - Update from upstream and make regeneration easier (#17) ([libp2p/go-libp2p-asn-util#17](https://github.com/libp2p/go-libp2p-asn-util/pull/17)) - add license file so it can be found by go-licenses ([libp2p/go-libp2p-asn-util#10](https://github.com/libp2p/go-libp2p-asn-util/pull/10)) - refactor: rename ASN table files ([libp2p/go-libp2p-asn-util#9](https://github.com/libp2p/go-libp2p-asn-util/pull/9)) - Library for IP -> ASN mapping ([libp2p/go-libp2p-asn-util#1](https://github.com/libp2p/go-libp2p-asn-util/pull/1)) - github.com/libp2p/go-libp2p-autonat (v0.4.2 -> v0.6.0): - Version 0.6.0 (#112) ([libp2p/go-libp2p-autonat#112](https://github.com/libp2p/go-libp2p-autonat/pull/112)) - feat: plumb through contexts from peerstore (#111) ([libp2p/go-libp2p-autonat#111](https://github.com/libp2p/go-libp2p-autonat/pull/111)) - sync: update CI config files (#110) ([libp2p/go-libp2p-autonat#110](https://github.com/libp2p/go-libp2p-autonat/pull/110)) - remove context from constructor, implement a proper Close method ([libp2p/go-libp2p-autonat#109](https://github.com/libp2p/go-libp2p-autonat/pull/109)) - fix stream deadlines ([libp2p/go-libp2p-autonat#107](https://github.com/libp2p/go-libp2p-autonat/pull/107)) - disable failing integration test ([libp2p/go-libp2p-autonat#108](https://github.com/libp2p/go-libp2p-autonat/pull/108)) - fix staticcheck ([libp2p/go-libp2p-autonat#103](https://github.com/libp2p/go-libp2p-autonat/pull/103)) - github.com/libp2p/go-libp2p-core (v0.9.0 -> v0.11.0): - release v0.11.0 (#217) ([libp2p/go-libp2p-core#217](https://github.com/libp2p/go-libp2p-core/pull/217)) - remove the ConnHandler (#214) ([libp2p/go-libp2p-core#214](https://github.com/libp2p/go-libp2p-core/pull/214)) - sync: update CI config files (#216) ([libp2p/go-libp2p-core#216](https://github.com/libp2p/go-libp2p-core/pull/216)) - remove the Process from the Network interface (#212) ([libp2p/go-libp2p-core#212](https://github.com/libp2p/go-libp2p-core/pull/212)) - pass the peer ID to SecureInbound in the SecureTransport and SecureMuxer (#211) ([libp2p/go-libp2p-core#211](https://github.com/libp2p/go-libp2p-core/pull/211)) - save the role (client, server) in the simultaneous connect context (#210) ([libp2p/go-libp2p-core#210](https://github.com/libp2p/go-libp2p-core/pull/210)) - sync: update CI config files (#209) ([libp2p/go-libp2p-core#209](https://github.com/libp2p/go-libp2p-core/pull/209)) - github.com/libp2p/go-libp2p-discovery (v0.5.1 -> v0.6.0): - feat: plumb peerstore contexts changes through (#75) ([libp2p/go-libp2p-discovery#75](https://github.com/libp2p/go-libp2p-discovery/pull/75)) - remove deprecated types ([libp2p/go-libp2p-discovery#73](https://github.com/libp2p/go-libp2p-discovery/pull/73)) - github.com/libp2p/go-libp2p-kad-dht (v0.13.1 -> v0.15.0): - Bump version to 0.15.0 (#755) ([libp2p/go-libp2p-kad-dht#755](https://github.com/libp2p/go-libp2p-kad-dht/pull/755)) - sync: update CI config files (#754) ([libp2p/go-libp2p-kad-dht#754](https://github.com/libp2p/go-libp2p-kad-dht/pull/754)) - feat: plumb through datastore contexts (#753) ([libp2p/go-libp2p-kad-dht#753](https://github.com/libp2p/go-libp2p-kad-dht/pull/753)) - custom ProviderManager that brokers AddrInfos (#751) ([libp2p/go-libp2p-kad-dht#751](https://github.com/libp2p/go-libp2p-kad-dht/pull/751)) - feat: make compatible with go-libp2p 0.15 ([libp2p/go-libp2p-kad-dht#747](https://github.com/libp2p/go-libp2p-kad-dht/pull/747)) - sync: update CI config files ([libp2p/go-libp2p-kad-dht#743](https://github.com/libp2p/go-libp2p-kad-dht/pull/743)) - Disallow GetPublicKey when DisableValues is passed ([libp2p/go-libp2p-kad-dht#604](https://github.com/libp2p/go-libp2p-kad-dht/pull/604)) - github.com/libp2p/go-libp2p-nat (v0.0.6 -> v0.1.0): - remove Codecov config file ([libp2p/go-libp2p-nat#39](https://github.com/libp2p/go-libp2p-nat/pull/39)) - stop using goprocess for shutdown ([libp2p/go-libp2p-nat#38](https://github.com/libp2p/go-libp2p-nat/pull/38)) - chore: update go-log ([libp2p/go-libp2p-nat#37](https://github.com/libp2p/go-libp2p-nat/pull/37)) - remove unused field permanent from mapping ([libp2p/go-libp2p-nat#33](https://github.com/libp2p/go-libp2p-nat/pull/33)) - github.com/libp2p/go-libp2p-noise (v0.2.2 -> v0.3.0): - add the peer ID to SecureInbound ([libp2p/go-libp2p-noise#104](https://github.com/libp2p/go-libp2p-noise/pull/104)) - update go-libp2p-core, remove integration test ([libp2p/go-libp2p-noise#102](https://github.com/libp2p/go-libp2p-noise/pull/102)) - github.com/libp2p/go-libp2p-peerstore (v0.2.8 -> v0.4.0): - Update version.json (#178) ([libp2p/go-libp2p-peerstore#178](https://github.com/libp2p/go-libp2p-peerstore/pull/178)) - limit the number of protocols we store per peer ([libp2p/go-libp2p-peerstore#172](https://github.com/libp2p/go-libp2p-peerstore/pull/172)) - sync: update CI config files (#177) ([libp2p/go-libp2p-peerstore#177](https://github.com/libp2p/go-libp2p-peerstore/pull/177)) - feat: plumb through datastore contexts (#176) ([libp2p/go-libp2p-peerstore#176](https://github.com/libp2p/go-libp2p-peerstore/pull/176)) - remove leftover peerstore implementation in the root package ([libp2p/go-libp2p-peerstore#173](https://github.com/libp2p/go-libp2p-peerstore/pull/173)) - fix: replace deprecated call ([libp2p/go-libp2p-peerstore#168](https://github.com/libp2p/go-libp2p-peerstore/pull/168)) - feat: remove queue ([libp2p/go-libp2p-peerstore#166](https://github.com/libp2p/go-libp2p-peerstore/pull/166)) - remove deprecated types ([libp2p/go-libp2p-peerstore#165](https://github.com/libp2p/go-libp2p-peerstore/pull/165)) - github.com/libp2p/go-libp2p-pubsub (v0.5.4 -> v0.6.0): - feat: plumb through context changes (#459) ([libp2p/go-libp2p-pubsub#459](https://github.com/libp2p/go-libp2p-pubsub/pull/459)) - support MinTopicSize without a discovery mechanism - clear peerPromises map when fulfilling a promise - README: remove obsolete notice, fix example code for tracing. - remove peer filter check from subscriptions (#453) ([libp2p/go-libp2p-pubsub#453](https://github.com/libp2p/go-libp2p-pubsub/pull/453)) - Create peer filter option - github.com/libp2p/go-libp2p-pubsub-router (v0.4.0 -> v0.5.0): - Version 0.5.0 - feat: plumb through datastore contexts - sync: update CI config files (#86) ([libp2p/go-libp2p-pubsub-router#86](https://github.com/libp2p/go-libp2p-pubsub-router/pull/86)) - Remove arbitrary sleeps from tests ([libp2p/go-libp2p-pubsub-router#87](https://github.com/libp2p/go-libp2p-pubsub-router/pull/87)) - cleanup: fix staticcheck failures ([libp2p/go-libp2p-pubsub-router#84](https://github.com/libp2p/go-libp2p-pubsub-router/pull/84)) - Add WithDatastore option. ([libp2p/go-libp2p-pubsub-router#82](https://github.com/libp2p/go-libp2p-pubsub-router/pull/82)) - github.com/libp2p/go-libp2p-quic-transport (v0.12.0 -> v0.15.0): - release v0.15.0 (#241) ([libp2p/go-libp2p-quic-transport#241](https://github.com/libp2p/go-libp2p-quic-transport/pull/241)) - reuse the same router until we change listeners ([libp2p/go-libp2p-quic-transport#240](https://github.com/libp2p/go-libp2p-quic-transport/pull/240)) - release v0.14.0 ([libp2p/go-libp2p-quic-transport#237](https://github.com/libp2p/go-libp2p-quic-transport/pull/237)) - fix error assertions in the tracer ([libp2p/go-libp2p-quic-transport#234](https://github.com/libp2p/go-libp2p-quic-transport/pull/234)) - sync: update CI config files (#235) ([libp2p/go-libp2p-quic-transport#235](https://github.com/libp2p/go-libp2p-quic-transport/pull/235)) - read the client option from the simultaneous connect context ([libp2p/go-libp2p-quic-transport#230](https://github.com/libp2p/go-libp2p-quic-transport/pull/230)) - github.com/libp2p/go-libp2p-swarm (v0.5.3 -> v0.8.0): - Version 0.8.0 (#292) ([libp2p/go-libp2p-swarm#292](https://github.com/libp2p/go-libp2p-swarm/pull/292)) - feat: plumb contexts through from peerstore (#290) ([libp2p/go-libp2p-swarm#290](https://github.com/libp2p/go-libp2p-swarm/pull/290)) - release v0.7.0 ([libp2p/go-libp2p-swarm#289](https://github.com/libp2p/go-libp2p-swarm/pull/289)) - update go-tcp-transport to v0.4.0 ([libp2p/go-libp2p-swarm#287](https://github.com/libp2p/go-libp2p-swarm/pull/287)) - remove the ConnHandler ([libp2p/go-libp2p-swarm#286](https://github.com/libp2p/go-libp2p-swarm/pull/286)) - sync: update CI config files (#288) ([libp2p/go-libp2p-swarm#288](https://github.com/libp2p/go-libp2p-swarm/pull/288)) - remove a lot of incorrect statements from the README ([libp2p/go-libp2p-swarm#284](https://github.com/libp2p/go-libp2p-swarm/pull/284)) - unexport the DialSync ([libp2p/go-libp2p-swarm#281](https://github.com/libp2p/go-libp2p-swarm/pull/281)) - add an error return value to the constructor ([libp2p/go-libp2p-swarm#280](https://github.com/libp2p/go-libp2p-swarm/pull/280)) - use functional options to configure the swarm ([libp2p/go-libp2p-swarm#279](https://github.com/libp2p/go-libp2p-swarm/pull/279)) - stop using goprocess to control teardown ([libp2p/go-libp2p-swarm#278](https://github.com/libp2p/go-libp2p-swarm/pull/278)) - read and use the direction from the simultaneous connect context ([libp2p/go-libp2p-swarm#277](https://github.com/libp2p/go-libp2p-swarm/pull/277)) - simplify the DialSync code ([libp2p/go-libp2p-swarm#272](https://github.com/libp2p/go-libp2p-swarm/pull/272)) - remove redundant self-dialing check, simplify starting of dialWorkerLoop ([libp2p/go-libp2p-swarm#273](https://github.com/libp2p/go-libp2p-swarm/pull/273)) - add a test case for the testing package ([libp2p/go-libp2p-swarm#276](https://github.com/libp2p/go-libp2p-swarm/pull/276)) - simplify limiter by removing the injected isFdConsumingFnc ([libp2p/go-libp2p-swarm#274](https://github.com/libp2p/go-libp2p-swarm/pull/274)) - update badges ([libp2p/go-libp2p-swarm#271](https://github.com/libp2p/go-libp2p-swarm/pull/271)) - remove unused context in Swarm.dialWorkerLoop ([libp2p/go-libp2p-swarm#268](https://github.com/libp2p/go-libp2p-swarm/pull/268)) - remove Codecov config ([libp2p/go-libp2p-swarm#270](https://github.com/libp2p/go-libp2p-swarm/pull/270)) - fix race condition in TestFailFirst ([libp2p/go-libp2p-swarm#269](https://github.com/libp2p/go-libp2p-swarm/pull/269)) - github.com/libp2p/go-libp2p-testing (v0.4.2 -> v0.5.0): - chore: update go-libp2p-core to v0.10.0 ([libp2p/go-libp2p-testing#38](https://github.com/libp2p/go-libp2p-testing/pull/38)) - sync: update CI config files (#37) ([libp2p/go-libp2p-testing#37](https://github.com/libp2p/go-libp2p-testing/pull/37)) - github.com/libp2p/go-libp2p-tls (v0.2.0 -> v0.3.1): - release v0.3.1 ([libp2p/go-libp2p-tls#101](https://github.com/libp2p/go-libp2p-tls/pull/101)) - set a random certificate subject ([libp2p/go-libp2p-tls#100](https://github.com/libp2p/go-libp2p-tls/pull/100)) - sync: update CI config files (#96) ([libp2p/go-libp2p-tls#96](https://github.com/libp2p/go-libp2p-tls/pull/96)) - add the peer ID to SecureInbound ([libp2p/go-libp2p-tls#94](https://github.com/libp2p/go-libp2p-tls/pull/94)) - sync: update CI config files ([libp2p/go-libp2p-tls#91](https://github.com/libp2p/go-libp2p-tls/pull/91)) - github.com/libp2p/go-libp2p-transport-upgrader (v0.4.6 -> v0.5.0): - increase timeout in TestConnectionsClosedIfNotAccepted on CI ([libp2p/go-libp2p-transport-upgrader#85](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/85)) - add the peer ID to SecureInbound ([libp2p/go-libp2p-transport-upgrader#83](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/83)) - github.com/libp2p/go-msgio (v0.0.6 -> v0.1.0): - sync: update CI config files (#27) ([libp2p/go-msgio#27](https://github.com/libp2p/go-msgio/pull/27)) - remove .gxignore file ([libp2p/go-msgio#24](https://github.com/libp2p/go-msgio/pull/24)) - remove Codecov config ([libp2p/go-msgio#26](https://github.com/libp2p/go-msgio/pull/26)) - remove "Chan" type ([libp2p/go-msgio#23](https://github.com/libp2p/go-msgio/pull/23)) - github.com/libp2p/go-nat (v0.0.5 -> v0.1.0): - pass a context to DiscoverGateway ([libp2p/go-nat#23](https://github.com/libp2p/go-nat/pull/23)) - github.com/libp2p/go-reuseport (v0.0.2 -> v0.1.0): - stop using github.com/pkg/errors ([libp2p/go-reuseport#85](https://github.com/libp2p/go-reuseport/pull/85)) - sync: update CI config files (#84) ([libp2p/go-reuseport#84](https://github.com/libp2p/go-reuseport/pull/84)) - github.com/libp2p/go-reuseport-transport (v0.0.5 -> v0.1.0): - remove Codecov config file ([libp2p/go-reuseport-transport#36](https://github.com/libp2p/go-reuseport-transport/pull/36)) - chore: update go-log to v2 ([libp2p/go-reuseport-transport#35](https://github.com/libp2p/go-reuseport-transport/pull/35)) - sync: update CI config files ([libp2p/go-reuseport-transport#31](https://github.com/libp2p/go-reuseport-transport/pull/31)) - github.com/libp2p/go-tcp-transport (v0.2.8 -> v0.4.0): - release v0.4.0 ([libp2p/go-tcp-transport#108](https://github.com/libp2p/go-tcp-transport/pull/108)) - sync: update CI config files (#107) ([libp2p/go-tcp-transport#107](https://github.com/libp2p/go-tcp-transport/pull/107)) - remove the deprecated IPFS_REUSEPORT command line flag ([libp2p/go-tcp-transport#104](https://github.com/libp2p/go-tcp-transport/pull/104)) - add options to the constructor ([libp2p/go-tcp-transport#99](https://github.com/libp2p/go-tcp-transport/pull/99)) - remove the context from the libp2p constructor in README ([libp2p/go-tcp-transport#101](https://github.com/libp2p/go-tcp-transport/pull/101)) - don't use libp2p.ChainOption in README ([libp2p/go-tcp-transport#102](https://github.com/libp2p/go-tcp-transport/pull/102)) - remove incorrect statement about dns addresses in README ([libp2p/go-tcp-transport#100](https://github.com/libp2p/go-tcp-transport/pull/100)) - use the assigned role when upgrading a sim open connection ([libp2p/go-tcp-transport#95](https://github.com/libp2p/go-tcp-transport/pull/95)) - chore: update go-log to v2 ([libp2p/go-tcp-transport#97](https://github.com/libp2p/go-tcp-transport/pull/97)) - simplify dial timeout context ([libp2p/go-tcp-transport#94](https://github.com/libp2p/go-tcp-transport/pull/94)) - github.com/libp2p/go-yamux/v2 (v2.2.0 -> v2.3.0): - limit the number of concurrent incoming streams ([libp2p/go-yamux#66](https://github.com/libp2p/go-yamux/pull/66)) - drastically reduce allocations in ring buffer implementation (#64) ([libp2p/go-yamux#64](https://github.com/libp2p/go-yamux/pull/64)) - sync: update CI config files (#63) ([libp2p/go-yamux#63](https://github.com/libp2p/go-yamux/pull/63)) - remove call to asyncNotify in Stream.Read - github.com/libp2p/zeroconf/v2 (v2.0.0 -> v2.1.1): - fix flaky TTL test ([libp2p/zeroconf#18](https://github.com/libp2p/zeroconf/pull/18)) - implement a clean shutdown of the probe method ([libp2p/zeroconf#16](https://github.com/libp2p/zeroconf/pull/16)) - remove dependency on the backoff library ([libp2p/zeroconf#17](https://github.com/libp2p/zeroconf/pull/17)) - Don't stop browsing after ~15min ([libp2p/zeroconf#13](https://github.com/libp2p/zeroconf/pull/13)) - fix delays when sending initial probe packets ([libp2p/zeroconf#14](https://github.com/libp2p/zeroconf/pull/14)) - improve starting of mDNS service in tests, stop using pkg/errors ([libp2p/zeroconf#15](https://github.com/libp2p/zeroconf/pull/15)) - update import path to include v2 in README ([libp2p/zeroconf#11](https://github.com/libp2p/zeroconf/pull/11)) - github.com/lucas-clemente/quic-go (v0.23.0 -> v0.24.0): - don't unlock the receive stream mutex for copying from STREAM frames ([lucas-clemente/quic-go#3290](https://github.com/lucas-clemente/quic-go/pull/3290)) - List projects using quic-go ([lucas-clemente/quic-go#3266](https://github.com/lucas-clemente/quic-go/pull/3266)) - disable Path MTU Discovery on Windows ([lucas-clemente/quic-go#3276](https://github.com/lucas-clemente/quic-go/pull/3276)) - enter the regular run loop if no undecryptable packet was processed ([lucas-clemente/quic-go#3268](https://github.com/lucas-clemente/quic-go/pull/3268)) - Allow use of custom port value in Alt-Svc header. ([lucas-clemente/quic-go#3272](https://github.com/lucas-clemente/quic-go/pull/3272)) - disable the goconst linter ([lucas-clemente/quic-go#3286](https://github.com/lucas-clemente/quic-go/pull/3286)) - use x/net/ipv{4,6} to construct oob info when writing packets (#3278) ([lucas-clemente/quic-go#3278](https://github.com/lucas-clemente/quic-go/pull/3278)) - run gofmt to add the new go:build tags ([lucas-clemente/quic-go#3277](https://github.com/lucas-clemente/quic-go/pull/3277)) - fix log string in client example ([lucas-clemente/quic-go#3264](https://github.com/lucas-clemente/quic-go/pull/3264)) - github.com/multiformats/go-multiaddr (v0.4.0 -> v0.4.1): - add the plaintextv2 protocol ([multiformats/go-multiaddr#165](https://github.com/multiformats/go-multiaddr/pull/165)) - github.com/multiformats/go-multihash (v0.0.15 -> v0.1.0): - bump version to v0.1.0 ([multiformats/go-multihash#151](https://github.com/multiformats/go-multihash/pull/151)) - add version.json per tooling convention. - murmur3 support (#150) ([multiformats/go-multihash#150](https://github.com/multiformats/go-multihash/pull/150)) - Add variations of sha2 ([multiformats/go-multihash#149](https://github.com/multiformats/go-multihash/pull/149)) - don't use pointers for Multihash.String - Add blake3 hash and sharness tests ([multiformats/go-multihash#147](https://github.com/multiformats/go-multihash/pull/147)) - remove Makefile ([multiformats/go-multihash#142](https://github.com/multiformats/go-multihash/pull/142)) - fix staticcheck ([multiformats/go-multihash#141](https://github.com/multiformats/go-multihash/pull/141)) - New SumStream function reads from io.Reader ([multiformats/go-multihash#138](https://github.com/multiformats/go-multihash/pull/138)) - github.com/warpfork/go-testmark (v0.3.0 -> v0.9.0): - testexec: will now always set up tmpdirs. - testexec: fix typo in error message. - testexec: subtest ("then-*") feature ([warpfork/go-testmark#7](https://github.com/warpfork/go-testmark/pull/7)) - testexec: quote error from child; attribution better via more t.Helper. - Improve documentation of format. - Rename Hunk.BlockTag -> InfoString. - testexec: will now create tmpdirs and files for you if you have an 'fs' entry tree. - testexec: getting exit codes correctly. ([warpfork/go-testmark#6](https://github.com/warpfork/go-testmark/pull/6)) - fix parsing CRLF files, part 3 ([warpfork/go-testmark#5](https://github.com/warpfork/go-testmark/pull/5)) - fix parsing CRLF files, part 2 ([warpfork/go-testmark#4](https://github.com/warpfork/go-testmark/pull/4)) - testexec: support both simple sequence and script mode. - Proper tests for read function. - avoid creeping extra linebreaks at the end of a patched document. - refrain from making double linebreaks when patching with content that ends in a linebreak. - Merge branch 'testexec' - add support for parsing CRLF line endings ([warpfork/go-testmark#3](https://github.com/warpfork/go-testmark/pull/3)) - link to patch example code - More readme; and, parsing recommendations document. - Further improve readme. ### ❤️ Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Will | 13 | +73226/-130481 | 43 | | Masih H. Derkani | 99 | +10549/-5799 | 489 | | hannahhoward | 43 | +5515/-3293 | 233 | | Daniel Martí | 60 | +5312/-2883 | 208 | | Marten Seemann | 175 | +4839/-3254 | 396 | | Eric Myhre | 73 | +3924/-3328 | 175 | | Jessica Schilling | 52 | +2709/-2386 | 75 | | Rod Vagg | 30 | +2719/-1703 | 79 | | vyzo | 10 | +3516/-177 | 87 | | Gus Eggert | 64 | +1677/-1416 | 147 | | Adin Schmahmann | 23 | +1708/-381 | 95 | | Lucas Molas | 14 | +1557/-365 | 48 | | Will Scott | 7 | +1846/-15 | 34 | | Steven Allen | 32 | +537/-897 | 56 | | Cory Schwartz | 3 | +614/-109 | 12 | | rht | 3 | +576/-4 | 7 | | Simon Zhu | 9 | +352/-51 | 16 | | Petar Maymounkov | 7 | +173/-167 | 23 | | RubenKelevra | 1 | +107/-188 | 1 | | jwh | 2 | +212/-80 | 7 | | longfeiW | 1 | +4/-249 | 10 | | guseggert | 5 | +230/-21 | 11 | | Kevin Neaton | 8 | +137/-80 | 13 | | Takashi Matsuda | 1 | +199/-0 | 5 | | Andrey Kostakov | 1 | +107/-49 | 2 | | Jesse Bouwman | 1 | +151/-0 | 7 | | web3-bot | 39 | +136/-3 | 52 | | Marcin Rataj | 16 | +62/-57 | 25 | | Marco Munizaga | 1 | +118/-0 | 2 | | Aaron Riekenberg | 4 | +64/-52 | 6 | | Ian Davis | 4 | +81/-32 | 7 | | Jorropo | 2 | +79/-19 | 6 | | Mohsin Zaidi | 1 | +89/-1 | 20 | | Andey Robins | 1 | +70/-3 | 3 | | gammazero | 3 | +40/-25 | 4 | | Steve Loeppky | 2 | +26/-27 | 3 | | Dimitris Apostolou | 1 | +25/-25 | 15 | | Sudarshan Reddy | 1 | +9/-40 | 1 | | Richard Littauer | 2 | +42/-1 | 3 | | pymq | 1 | +32/-8 | 2 | | Dirk McCormick | 2 | +23/-1 | 2 | | Nicholas Bollweg | 1 | +21/-0 | 1 | | anorth | 1 | +14/-6 | 2 | | Jack Loughran | 1 | +16/-0 | 2 | | whyrusleeping | 2 | +11/-2 | 2 | | bt90 | 1 | +13/-0 | 1 | | Yi Cao | 1 | +10/-0 | 1 | | Max | 1 | +7/-3 | 1 | | Juan Batiz-Benet | 2 | +8/-2 | 2 | | Keenan Nemetz | 1 | +8/-0 | 1 | | muXxer | 1 | +3/-3 | 1 | | galargh | 2 | +3/-3 | 3 | | Didrik Nordström | 1 | +2/-4 | 1 | | Ben Lubar | 1 | +3/-3 | 1 | | arjunraghurama | 1 | +5/-0 | 1 | | Whyrusleeping | 1 | +3/-2 | 1 | | TUSF | 1 | +3/-2 | 3 | | mathew-cf | 1 | +3/-1 | 2 | | Stephen Whitmore | 1 | +2/-2 | 1 | | Song Zhu | 1 | +2/-2 | 1 | | Michael Muré | 1 | +4/-0 | 1 | | Alex Good | 1 | +4/-0 | 2 | | aarshkshah1992 | 1 | +2/-1 | 1 | | susarlanikhilesh | 1 | +1/-1 | 1 | | falstack | 1 | +1/-1 | 1 | | Michael Vorburger ⛑️ | 1 | +1/-1 | 1 | | Ismail Khoffi | 1 | +1/-1 | 1 | | George Xie | 1 | +1/-1 | 1 | | Bryan Stenson | 1 | +1/-1 | 1 | | Lars Gierth | 1 | +1/-0 | 1 | ================================================ FILE: docs/changelogs/v0.12.md ================================================ # go-ipfs changelog v0.12 ## v0.12.2 2022-04-08 This patch release fixes a security issue wherein traversing some malformed DAGs can cause the node to panic. See also the security advisory: https://github.com/ipfs/go-ipfs/security/advisories/GHSA-mcq2-w56r-5w2w Note: the v0.11.1 patch release contains the Docker compose fix from v0.12.1 as well ### Changelog
Full Changelog - github.com/ipld/go-codec-dagpb (v1.3.0 -> v1.3.2): - fix: use protowire for Links bytes decoding
### ❤ Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Rod Vagg | 1 | +34/-19 | 2 | ## v0.12.1 2022-03-17 This patch release [fixes](https://github.com/ipfs/go-ipfs/commit/816a128aaf963d72c4930852ce32b9a4e31924a1) a security issue with the `docker-compose.yaml` file in which the IPFS daemon API listens on all interfaces instead of only the loopback interface, which could allow remote callers to control your IPFS daemon. If you use the included `docker-compose.yaml` file, it is recommended to upgrade. See also the security advisory: https://github.com/ipfs/go-ipfs/security/advisories/GHSA-fx5p-f64h-93xc Thanks to @LynHyper for finding and disclosing this. ### Changelog
Full Changelog - github.com/ipfs/go-ipfs: - fix: listen on loopback for API and gateway ports in docker-compose.yaml
### ❤ Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | guseggert | 1 | +10/-3 | 1 | ## v0.12.0 2022-02-17 We're happy to announce go-ipfs 0.12.0. This release switches the storage of IPLD blocks to be keyed by multihash instead of CID. As usual, this release includes important fixes, some of which may be critical for security. Unless the fix addresses a bug being exploited in the wild, the fix will _not_ be called out in the release notes. Please make sure to update ASAP. See our [release process](https://github.com/ipfs/go-ipfs/tree/master/docs/releases.md#security-fix-policy) for details. ### 🛠 BREAKING CHANGES - `ipfs refs local` will now list all blocks as if they were [raw]() CIDv1 instead of with whatever CID version and IPLD codecs they were stored with. All other functionality should remain the same. Note: This change also effects [ipfs-update](https://github.com/ipfs/ipfs-update) so if you use that tool to manage your go-ipfs installation then grab ipfs-update v1.8.0 from [dist](https://dist.ipfs.tech/#ipfs-update). Keep reading to learn more details. #### 🔦 Highlights There is only one change since 0.11: ##### Blockstore migration from full CID to Multihash keys We are switching the default low level [datastore](https://docs.ipfs.tech/concepts/glossary/#datastore) to be keyed only by the [Multihash](https://docs.ipfs.tech/concepts/glossary/#multihash) part of the [CID](https://docs.ipfs.tech/concepts/glossary/#cid), and deduplicate some [blocks](https://docs.ipfs.tech/concepts/glossary/#block) in the process. The blockstore will become [codec](https://docs.ipfs.tech/concepts/glossary/#codec)-agnostic. ###### Rationale The blockstore/datastore layers are not concerned with data interpretation, only with storage of binary blocks and verification that the Multihash they are addressed with (which comes from the CID), matches the block. In fact, different CIDs, with different codecs prefixes, may be carrying the same multihash, and referencing the same block. Carrying the CID abstraction so low on the stack means potentially fetching and storing the same blocks multiple times just because they are referenced by different CIDs. Prior to this change, a CIDv1 with a `dag-cbor` codec and a CIDv1 with a `raw` codec, both containing the same multihash, would result in two identical blocks stored. A CIDv0 and CIDv1 both being the same `dag-pb` block would also result in two copies. ###### How migration works In order to perform the switch, and start referencing all blocks by their multihash, a migration will occur on update. This migration will take the repository version from 11 (current) to 12. One thing to note is that any content addressed CIDv0 (all the hashes that start with `Qm...`, the current default in go-ipfs), does not need any migration, as CIDv0 are raw multihashes already. This means the migration will be very lightweight for the majority of users. The migration process will take care of re-keying any CIDv1 block so that it is only addressed by its multihash. Large nodes with lots of CIDv1-addressed content will need to go through a heavier process as the migration happens. This is how the migration works: 1. Phase 1: The migration script will perform a pass for every block in the datastore and will add all CIDv1s found to a file named `11-to-12-cids.txt`, in the go-ipfs configuration folder. Nothing is written in this first phase and it only serves to identify keys that will be migrated in phase 2. 2. Phase 2: The migration script will perform a second pass where every CIDv1 block will be read and re-written with its raw-multihash as key. There is 1 worker performing this task, although more can be configured. Every 100MiB-worth of blocks (this is configurable), each worker will trigger a datastore "sync" (to ensure all written data is flushed to disk) and delete the CIDv1-addressed blocks that were just renamed. This provides a good compromise between speed and resources needed to run the migration. At every sync, the migration emits a log message showing how many blocks need to be rewritten and how far the process is. ####### FlatFS specific migration For those using a single FlatFS datastore as their backing blockstore (i.e. the default behavior), the migration (but not reversion) will take advantage of the ability to easily move/rename the blocks to improve migration performance. Unfortunately, other common datastores do not support renames which is what makes this FlatFS specific. If you are running a large custom datastore that supports renames you may want to consider running a fork of [fs-repo-11-to-12](https://github.com/ipfs/fs-repo-migrations/tree/master/fs-repo-11-to-12) specific to your datastore. If you want to disable this behavior, set the environment variable `IPFS_FS_MIGRATION_11_TO_12_ENABLE_FLATFS_FASTPATH` to `false`. ####### Migration configuration For those who want to tune the migration more precisely for their setups, there are two environment variables to configure: - `IPFS_FS_MIGRATION_11_TO_12_NWORKERS` : an integer describing the number of migration workers - defaults to 1 - `IPFS_FS_MIGRATION_11_TO_12_SYNC_SIZE_BYTES` : an integer describing the number of bytes after which migration workers will sync - defaults to 104857600 (i.e. 100MiB) ###### Migration caveats Large repositories with very large numbers of CIDv1s should be mindful of the migration process: * We recommend ensuring that IPFS runs with an appropriate (high) file-descriptor limit, particularly when Badger is use as datastore backend. Badger is known to open many tables when experiencing a high number of writes, which may trigger "too many files open" type of errors during the migrations. If this happens, the migration can be retried with a higher FD limit (see below). * Migrations using the Badger datastore may not immediately reclaim the space freed by the deletion of migrated blocks, thus space requirements may grow considerably. A periodic Badger-GC is run every 2 minutes, which will reclaim space used by deleted and de-duplicated blocks. The last portion of the space will only be reclaimed after go-ipfs starts (the Badger-GC cycle will trigger after 15 minutes). * While there is a revert process detailed below, we recommend keeping a backup of the repository, particularly for very large ones, in case an issue happens, so that the revert can happen immediately and cases of repository corruption due to crashes or unexpected circumstances are not catastrophic. ###### Migration interruptions and retries If a problem occurs during the migration, it is be possible to simply re-start and retry it: 1. Phase 1 will never overwrite the `11-to-12-cids.txt` file, but only append to it (so that a list of things we were supposed to have migrated during our first attempt is not lost - this is important for reverts, see below). 2. Phase 2 will proceed to continue re-keying blocks that were not re-keyed during previous attempts. ###### Migration reverts It is also possible to revert the migration after it has succeeded, for example to go to a previous go-ipfs version (<=[0.11](https://github.com/ipfs/go-ipfs/releases/tag/v0.11.0)), even after starting and using go-ipfs in the new version (>=0.12). The revert process works as follows: 1. The `11-to-12-cids.txt` file is read, which has the list of all the CIDv1s that had to be rewritten for the migration. 2. A CIDv1-addressed block is written for every item on the list. This work is performed by 1 worker (configurable), syncing every 100MiB (configurable). 3. It is ensured that every CIDv1 pin, and every CIDv1 reference in MFS, are also written as CIDV1-addressed blocks, regardless of whether they were part of the original migration or were added later. The revert process does not delete any blocks--it only makes sure that blocks that were accessible with CIDv1s before the migration are again keyed with CIDv1s. This may result in a datastore becoming twice as large (i.e. if all the blocks were CIDv1-addressed before the migration). This is however done this way to cover corner cases: user can add CIDv1s after migration, which may reference blocks that existed as CIDv0 before migration. The revert aims to ensure that no data becomes unavailable on downgrade. While go-ipfs will auto-run the migration for you, it will not run the reversion. To do so you can download the [latest migration binary](https://dist.ipfs.tech/fs-repo-11-to-12) or use [ipfs-update](https://dist.ipfs.tech/#ipfs-update). ###### Custom datastores As with previous migrations if you work with custom datastores and want to leverage the migration you can run a fork of [fs-repo-11-to-12](https://github.com/ipfs/fs-repo-migrations/tree/master/fs-repo-11-to-12) specific to your datastore. The repo includes instructions on building for different datastores. For this migration, if your datastore has fast renames you may want to consider writing some code to leverage the particular efficiencies of your datastore similar to what was done for FlatFS. ### Changelog - github.com/ipfs/go-ipfs: - Release v0.12.0 - docs: v0.12.0 release notes - chore: bump migrations dist.ipfs.tech CID to contain fs-repo-11-to-12 v1.0.2 - feat: refactor Fetcher interface used for downloading migrations (#8728) ([ipfs/go-ipfs#8728](https://github.com/ipfs/go-ipfs/pull/8728)) - feat: log multifetcher errors - Release v0.12.0-rc1 - chore: bump Go version to 1.16.12 - feat: switch to raw multihashes for blocks ([ipfs/go-ipfs#6816](https://github.com/ipfs/go-ipfs/pull/6816)) - chore: add release template snippet for fetching artifact tarball - chore: bump Go version to 1.16.11 - chore: add release steps for upgrading Go - Merge branch 'release' - fix(corehttp): adjust peer counting metrics (#8577) ([ipfs/go-ipfs#8577](https://github.com/ipfs/go-ipfs/pull/8577)) - chore: update version to v0.12.0-dev - github.com/ipfs/go-filestore (v0.1.0 -> v1.1.0): - Version 1.1.0 - feat: plumb through context changes - sync: update CI config files (#54) ([ipfs/go-filestore#54](https://github.com/ipfs/go-filestore/pull/54)) - fix staticcheck ([ipfs/go-filestore#48](https://github.com/ipfs/go-filestore/pull/48)) - Work with Multihashes directly (#21) ([ipfs/go-filestore#21](https://github.com/ipfs/go-filestore/pull/21)) - github.com/ipfs/go-ipfs-blockstore (v0.2.1 -> v1.1.2): - Release v1.1.2 - feat: per-cid locking - Version 1.1.1 - fix: remove context from HashOnRead - Version 1.1.0 (#91) ([ipfs/go-ipfs-blockstore#91](https://github.com/ipfs/go-ipfs-blockstore/pull/91)) - feat: add context to interfaces (#90) ([ipfs/go-ipfs-blockstore#90](https://github.com/ipfs/go-ipfs-blockstore/pull/90)) - sync: update CI config files (#88) ([ipfs/go-ipfs-blockstore#88](https://github.com/ipfs/go-ipfs-blockstore/pull/88)) - add constructor that doesn't mess with datastore keys ([ipfs/go-ipfs-blockstore#83](https://github.com/ipfs/go-ipfs-blockstore/pull/83)) - Use bloom filter in GetSize - fix staticcheck ([ipfs/go-ipfs-blockstore#73](https://github.com/ipfs/go-ipfs-blockstore/pull/73)) - add BenchmarkARCCacheConcurrentOps ([ipfs/go-ipfs-blockstore#70](https://github.com/ipfs/go-ipfs-blockstore/pull/70)) - fix(arc): striped locking on last byte of CID ([ipfs/go-ipfs-blockstore#67](https://github.com/ipfs/go-ipfs-blockstore/pull/67)) - make idstore implement io.Closer. (#60) ([ipfs/go-ipfs-blockstore#60](https://github.com/ipfs/go-ipfs-blockstore/pull/60)) - add View() to all the various blockstores. (#59) ([ipfs/go-ipfs-blockstore#59](https://github.com/ipfs/go-ipfs-blockstore/pull/59)) - Optimize id store ([ipfs/go-ipfs-blockstore#56](https://github.com/ipfs/go-ipfs-blockstore/pull/56)) - add race fix for HashOnRead ([ipfs/go-ipfs-blockstore#50](https://github.com/ipfs/go-ipfs-blockstore/pull/50)) - Add test to maintain Put contract of calling Has first ([ipfs/go-ipfs-blockstore#47](https://github.com/ipfs/go-ipfs-blockstore/pull/47)) - Update readme and license ([ipfs/go-ipfs-blockstore#44](https://github.com/ipfs/go-ipfs-blockstore/pull/44)) - feat: switch to raw multihashes for blocks ([ipfs/go-ipfs-blockstore#38](https://github.com/ipfs/go-ipfs-blockstore/pull/38)) - github.com/ipfs/go-ipfs-ds-help (v0.1.1 -> v1.1.0): - Update version.json (#38) ([ipfs/go-ipfs-ds-help#38](https://github.com/ipfs/go-ipfs-ds-help/pull/38)) - sync: update CI config files (#37) ([ipfs/go-ipfs-ds-help#37](https://github.com/ipfs/go-ipfs-ds-help/pull/37)) - feat: switch to raw multihashes for blocks ([ipfs/go-ipfs-ds-help#18](https://github.com/ipfs/go-ipfs-ds-help/pull/18)) ### ❤️ Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Gus Eggert | 10 | +333/-321 | 24 | | Steven Allen | 7 | +289/-190 | 13 | | Hector Sanjuan | 9 | +134/-109 | 18 | | Adin Schmahmann | 11 | +179/-55 | 21 | | Raúl Kripalani | 2 | +152/-42 | 5 | | Daniel Martí | 1 | +120/-1 | 1 | | frrist | 1 | +95/-13 | 2 | | Alex Trottier | 2 | +22/-11 | 4 | | Andrey Petrov | 1 | +32/-0 | 1 | | Lucas Molas | 1 | +18/-7 | 2 | | Marten Seemann | 2 | +11/-7 | 3 | | whyrusleeping | 1 | +10/-0 | 1 | | web3-bot | 3 | +9/-0 | 3 | | postables | 1 | +5/-3 | 1 | | Dr Ian Preston | 1 | +4/-0 | 1 | ================================================ FILE: docs/changelogs/v0.13.md ================================================ # go-ipfs changelog 2022 ## v0.13.1 2022-07-06 This release includes security fixes for various DOS vectors when importing untrusted user input with `ipfs dag import` and the [`v0/dag/import`](https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-dag-import) endpoint. View the linked [security advisory](https://github.com/ipfs/go-ipfs/security/advisories/GHSA-f2gr-7299-487h) for more information. ### Changelog
Full Changelog - github.com/ipfs/go-ipfs: - chore: update car - github.com/ipld/go-car (v0.3.2 -> v0.4.0) & (v2.1.1 -> v2.4.0): - Bump version in prep for releasing go-car `v0` - Revert changes to `insertionindex` - Revert changes to `index.Index` while keeping most of security fixes - Return error when section length is invalid `varint` - Drop repeated package name from `CarStats` - Benchmark `Reader.Inspect` with and without hash validation - Use consistent CID mismatch error in `Inspect` and `BlockReader.Next` - Use streaming APIs to verify the hash of blocks in CAR `Inspect` - test: add fuzzing for reader#Inspect - feat: add block hash validation to Inspect() - feat: add Reader#Inspect() function to check basic validity of a CAR and return stats - Remove support for `ForEach` enumeration from car-index-sorted - Use a fix code as the multihash code for `CarIndexSorted` - Fix testutil assertion logic and update index generation tests - fix: tighter constraint of singleWidthIndex width, add index recommendation docs - fix: explicitly disable serialization of insertionindex - feat: MaxAllowed{Header,Section}Size option - feat: MaxAllowedSectionSize default to 32M - fix: use CidFromReader() which has overread and OOM protection - fix: staticcheck catches - fix: revert to internalio.NewOffsetReadSeeker in Reader#IndexReader - fix index comparisons - feat: Refactor indexes to put storage considerations on consumers - test: v2 add fuzzing of the index - fix: v2 don't divide by zero in width indexes - fix: v2 don't allocate indexes too big - test: v2 add fuzzing to Reader - fix: v2 don't accept overflowing offsets while reading v2 headers - test: v2 add fuzzing to BlockReader - fix: v2 don't OOM if the header size is too big - test: add fuzzing of NewCarReader - fix: do bound check while checking for CIDv0 - fix: don't OOM if the header size is too big - Add API to regenerate index from CARv1 or CARv2 - PrototypeChooser support (#305) ([ipld/go-car#305](https://github.com/ipld/go-car/pull/305)) - bump to newer blockstore err not found (#301) ([ipld/go-car#301](https://github.com/ipld/go-car/pull/301)) - Car command supports for `largebytes` nodes (#296) ([ipld/go-car#296](https://github.com/ipld/go-car/pull/296)) - fix(test): rootless fixture should have no roots, not null roots - Allow extraction of a raw unixfs file (#284) ([ipld/go-car#284](https://github.com/ipld/go-car/pull/284)) - cmd/car: use a better install command in the README - feat: --version selector for `car create` & update deps - feat: add option to create blockstore that writes a plain CARv1 (#288) ([ipld/go-car#288](https://github.com/ipld/go-car/pull/288)) - add `car detach-index list` to list detached index contents (#287) ([ipld/go-car#287](https://github.com/ipld/go-car/pull/287)) - add `car root` command (#283) ([ipld/go-car#283](https://github.com/ipld/go-car/pull/283)) - make specification of root cid in get-dag command optional (#281) ([ipld/go-car#281](https://github.com/ipld/go-car/pull/281)) - Update `version.json` after manual tag push - Update v2 to context datastores (#275) ([ipld/go-car#275](https://github.com/ipld/go-car/pull/275)) - update context datastore ([ipld/go-car#273](https://github.com/ipld/go-car/pull/273)) - Traversal-based car creation (#269) ([ipld/go-car#269](https://github.com/ipld/go-car/pull/269)) - Seek to start before index generation in `ReadOnly` blockstore - support extraction of unixfs content stored in car files (#263) ([ipld/go-car#263](https://github.com/ipld/go-car/pull/263)) - Add a bare bones readme to the car CLI (#262) ([ipld/go-car#262](https://github.com/ipld/go-car/pull/262)) - sync: update CI config files (#261) ([ipld/go-car#261](https://github.com/ipld/go-car/pull/261)) - fix!: use -version=n instead of -v1 for index command - feat: fix get-dag and add version=1 option - creation of car from file / directory (#246) ([ipld/go-car#246](https://github.com/ipld/go-car/pull/246)) - forEach iterates over index in stable order (#258) ([ipld/go-car#258](https://github.com/ipld/go-car/pull/258)) - github.com/multiformats/go-multicodec (v0.4.1 -> v0.5.0): - Bump version to 0.5.0 - Bump version to 0.4.2 - deps: update stringer version in go generate command - docs(readme): improved usage examples (#66) ([multiformats/go-multicodec#66](https://github.com/multiformats/go-multicodec/pull/66))
### ❤ Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Masih H. Derkani | 27 | +1494/-1446 | 100 | | Rod Vagg | 31 | +2021/-606 | 105 | | Will | 19 | +1898/-151 | 69 | | Jorropo | 27 | +1638/-248 | 76 | | Aayush Rajasekaran | 1 | +130/-100 | 10 | | whyrusleeping | 1 | +24/-22 | 4 | | Marcin Rataj | 1 | +27/-1 | 1 | ## v0.13.0 2022-05-04 We're happy to announce go-ipfs 0.13.0, packed full of changes and improvements! As usual, this release includes important fixes, some of which may be critical for security. Unless the fix addresses a bug being exploited in the wild, the fix will _not_ be called out in the release notes. Please make sure to update ASAP. See our [release process](https://github.com/ipfs/go-ipfs/tree/master/docs/releases.md#security-fix-policy) for details. ### Overview Below is an outline of all that is in this release, so you get a sense of all that's included. - [🛠 BREAKING CHANGES](#---breaking-changes) * [`ipfs block put` command](#-ipfs-block-put--command) * [`ipfs cid codecs` command](#-ipfs-cid-codecs--command) * [`Swarm` configuration](#-swarm--configuration) * [Circuit Relay V1 is deprecated](#circuit-relay-v1-is-deprecated) * [`ls` requests for `/multistream/1.0.0` are removed](#-ls--requests-for---multistream-100--are-removed) * [Gateway Items](#gateway-items) - [🔦 Highlights](#---highlights) * [🧑‍💼 libp2p Network Resource Manager (`Swarm.ResourceMgr`)](#------libp2p-network-resource-manager---swarmresourcemgr--) * [🔃 Relay V2 client with auto discovery (`Swarm.RelayClient`)](#---relay-v2-client-with-auto-discovery---swarmrelayclient--) * [🌉 HTTP Gateway improvements](#---http-gateway-improvements) + [🍱 Support for Block and CAR response formats](#---support-for-block-and-car-response-formats) + [🐎 Fast listing generation for huge directories](#---fast-listing-generation-for-huge--directories) + [🎫 Improved `Etag` and `If-None-Match` for bandwidth savings](#---improved--etag--and--if-none-match--for-bandwidth-savings) + [⛓️ Added X-Ipfs-Roots for smarter HTTP caches](#---added-x-ipfs-roots-for-smarter-http-caches) + [🌡️ Added metrics per response type](#----added-metrics-per-response-type) * [🕵️ OpenTelemetry tracing](#----opentelemetry-tracing) + [How to use Jaeger UI for visual tracing?](#how-to-use-jaeger-ui-for-visual-tracing-) * [🩺 Built-in `ipfs diag profile` to ease debugging](#---built-in--ipfs-diag-profile--to-ease-debugging) * [🔑 Support for PEM/PKCS8 for key import/export](#---support-for-pem-pkcs8-for-key-import-export) * [🧹 Using standard IPLD codec names across the CLI/HTTP API](#---using-standard-ipld-codec-names-across-the-cli-http-api) * [🐳 Custom initialization for Docker](#---custom-initialization-for-docker) * [RPC API docs for experimental and deprecated commands](#rpc-api-docs-for-experimental-and-deprecated-commands) * [Yamux over Mplex](#yamux-over-mplex) ### 🛠 BREAKING CHANGES #### `ipfs block put` command `ipfs block put` command returns a CIDv1 with `raw` codec by default now. - `ipfs block put --cid-codec` makes `block put` return CID with alternative codec - This impacts only the returned CID; it does not trigger any validation or data transformation. - Retrieving a block with a different codec or CID version than it was put with is valid. - Codec names are validated against tables from [go-multicodec](https://github.com/multiformats/go-multicodec) library. - `ipfs block put --format` is deprecated. It used incorrect codec names and should be avoided for new deployments. Use it only if you need the old, invalid behavior, namely: - `ipfs block put --format=v0` will produce CIDv0 (implicit dag-pb) - `ipfs block put --format=cbor` will produce CIDv1 with dag-cbor (!) - `ipfs block put --format=protobuf` will produce CIDv1 with dag-pb (!) #### `ipfs cid codecs` command - Now lists codecs from [go-multicodec](https://github.com/multiformats/go-multicodec) library. - `ipfs cid codecs --supported` can be passed to only show codecs supported in various go-ipfs commands. #### `ipfs cid format` command - `--codec` was removed and replaced with `--mc` to ensure existing users are aware of the following changes: - `--mc protobuf` now correctly points to code `0x50` (was `0x70`, which is `dab-pg`) - `--mc cbor` now correctly points to code `0x51` (was `0x71`, which is `dag-cbor`) #### `Swarm` configuration - Daemon will refuse to start if long-deprecated RelayV1 config key `Swarm.EnableAutoRelay` or `Swarm.DisableRelay` is set to `true`. - If `Swarm.Transports.Network.Relay` is disabled, then `Swarm.RelayService` and `Swarm.RelayClient` are also disabled (unless they have been explicitly enabled). #### Circuit Relay V1 is deprecated - By default, `Swarm.RelayClient` does not use Circuit Relay V1. Circuit V1 support is only enabled when `Swarm.RelayClient.StaticRelays` are specified. #### `ls` requests for `/multistream/1.0.0` are removed - go-libp2p 0.19 removed support for undocumented `ls` command ([PR](https://github.com/multiformats/go-multistream/pull/76)). If you are still using it for internal testing, it is time to refactor ([example](https://github.com/ipfs/go-ipfs/commit/39047bcf61163096d1c965283d671c7c487c9173)) #### Gateway Behavior Directory listings returned by the HTTP Gateway won't have size column if the directory is bigger than [`Gateway.FastDirIndexThreshold`](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#gatewayfastdirindexthreshold) config (default is 100). To understand the wider context why we made these changes, read *Highlights* below. ### 🔦 Highlights #### 🧑‍💼 libp2p Network Resource Manager (`Swarm.ResourceMgr`) *You can now easily bound how much resource usage libp2p consumes! This aids in protecting nodes from consuming more resources then are available to them.* The [libp2p Network Resource Manager](https://github.com/libp2p/go-libp2p-resource-manager#readme) is disabled by default, but can be enabled via: `ipfs config --json Swarm.ResourceMgr.Enabled true` When enabled, it applies some safe defaults that can be inspected and adjusted with: - `ipfs swarm stats --help` - `ipfs swarm limit --help` User changes persist to config at [`Swarm.ResourceMgr`](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmresourcemgr). The Resource Manager will be enabled by default in a future release. #### 🔃 Relay V2 client with auto discovery (`Swarm.RelayClient`) *All the pieces are enabled for [hole-punching](https://blog.ipfs.io/2022-01-20-libp2p-hole-punching/) by default, improving connecting with nodes behind NATs and Firewalls!* This release enables [`Swarm.RelayClient`](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmrelayclient) by default, along with circuit v2 relay discovery provided by go-libp2p [v0.19.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.19.0). This means: 1. go-ipfs will coordinate with the counterparty using a [relayed connection](https://github.com/libp2p/specs/blob/master/relay/circuit-v2.md), to [upgrade to a direct connection](https://github.com/libp2p/specs/blob/master/relay/DCUtR.md) through a NAT/firewall whenever possible. 2. go-ipfs daemon will automatically use [public relays](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmrelayservice) if it detects that it cannot be reached from the public internet (e.g., it's behind a firewall). This results in a `/p2p-circuit` address from a public relay. **Notes:** - [`Swarm.RelayClient`](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmrelayclient) does not use Circuit Relay V1 nodes any more. Circuit V1 support is only enabled when static relays are specified in [`Swarm.RelayClient.StaticRelays`](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmrelayclientstaticrelays). - One can opt-out via [`Swarm.EnableHolePunching`](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmenableholepunching). #### 🌉 HTTP Gateway improvements HTTP Gateway enables seamless interop with the existing Web, clients, user agents, tools, frameworks and libraries. This release ships the first batch of improvements that enable creation of faster and smarter CDNs, and unblocks creation of light clients for Mobile and IoT. Details below. ##### 🍱 Support for Block and CAR response formats *Alternative response formats from Gateway can be requested to avoid needing to trust a gateway.* For now, `{format}` is limited to two options: - `raw` – fetching single block - `car` – fetching entire DAG behind a CID as a [CARv1 stream](https://ipld.io/specs/transport/car/carv1/) When not set, the default UnixFS response is returned. *Why these two formats?* Requesting Block or CAR for `/ipfs/{cid}` allows a client to **use gateways in a trustless fashion**. These types of gateway responses can be verified locally and rejected if digest inside of requested CID does not match received bytes. This enables creation of "light IPFS clients" which use HTTP Gateways as inexpensive transport for [content-addressed](https://docs.ipfs.tech/concepts/content-addressing/) data, unlocking use in Mobile and IoT contexts. Future releases will [add support for dag-json and dag-cbor responses](https://github.com/ipfs/go-ipfs/issues/8823). There are two ways for requesting CID specific response format: 1. HTTP header: `Accept: application/vnd.ipld.{format}` - Examples: [application/vnd.ipld.car](https://www.iana.org/assignments/media-types/application/vnd.ipld.car), [application/vnd.ipld.raw](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw) 2. URL parameter: `?format=` - Useful for creating "Download CAR" links. *Usage examples:* 1. Downloading a single raw Block and manually importing it to the local datastore: ```console $ curl -H 'Accept: application/vnd.ipld.raw' "http://127.0.0.1:8080/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN" --output block.bin $ cat block.bin | ipfs block put $ ipfs cat QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN hello ``` 2. Downloading entire DAG as a CAR file and importing it: ```console $ ipfs resolve -r /ipns/webui.ipfs.io /ipfs/bafybeiednzu62vskme5wpoj4bjjikeg3xovfpp4t7vxk5ty2jxdi4mv4bu $ curl -H 'Accept: application/vnd.ipld.car' "http://127.0.0.1:8080/ipfs/bafybeiednzu62vskme5wpoj4bjjikeg3xovfpp4t7vxk5ty2jxdi4mv4bu" --output webui.car $ ipfs dag import webui.car $ ipfs dag stat bafybeiednzu62vskme5wpoj4bjjikeg3xovfpp4t7vxk5ty2jxdi4mv4bu --offline Size: 27684934, NumBlocks: 394 ``` See also: - [Content Addressable aRchives (CAR / .car) Specifications](https://ipld.io/specs/transport/car/) - [IANA media type](https://www.iana.org/assignments/media-types/media-types.xhtml) definitions: [application/vnd.ipld.car](https://www.iana.org/assignments/media-types/application/vnd.ipld.car), [application/vnd.ipld.raw](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw) - [ipfs-car](https://www.npmjs.com/package/ipfs-car) - CLI tool for verifying and unpacking CAR files - [go-car](https://github.com/ipld/go-car), [js-car](https://github.com/ipld/js-car/) – CAR libraries for GO and JS ##### 🐎 Fast listing generation for huge directories *Added [`Gateway.FastDirIndexThreshold`](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#gatewayfastdirindexthreshold) configuration, which allows for fast listings of big directories, without the linear slowdown caused by reading size metadata from child nodes.* As an example, the CID `bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm` represents UnixFS directory with over 10k (10100) of files. Opening it with go-ipfs 0.12 would require fetching size information of each file, which would take a long long time, most likely causing timeout in the browser or CDN, and introducing unnecessary burden on the gateway node. go-ipfs 0.13 opens it instantly, because the number of items is bigger than the default [`Gateway.FastDirIndexThreshold`](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#gatewayfastdirindexthreshold) and only the root UnixFS node needs to be resolved before the HTML Dir Index is returned to the user. Notes: - The default threshold is 100 items. - Setting to 0 will enable fast listings for all directories. - CLI users will note that this is equivalent to running `ipfs ls -s --size=false --resolve-type=false /ipfs/bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm`. Now the same speed is available on the gateways. ##### 🎫 Improved `Etag` and `If-None-Match` for bandwidth savings *Every response type has an unique [`Etag`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) which can be used by the client or CDN to save bandwidth, as a gateway does not need to resend a full response if the content was not changed.* Gateway evaluates Etags sent by a client in [`If-None-Match`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match) and returns status code 304 (Not Modified) on strong or weak match ([RFC 7232, 2.3](https://datatracker.ietf.org/doc/html/rfc7232#section-2.3)). ##### ⛓️ Added X-Ipfs-Roots for smarter HTTP caches `X-Ipfs-Roots` is now returned with every Gateway response. It is a way to indicate all CIDs required for resolving path segments from `X-Ipfs-Path`. Together, these two headers are meant to improve interop with existing HTTP software (load-balancers, caches, CDNs). This additional information allows HTTP caches and CDNs to make better decisions around cache invalidation: not just invalidate everything under specific IPNS website when the root changes, but do more fine-grained cache invalidation by detecting when only a specific subdirectory (branch of a [DAG](https://docs.ipfs.tech/concepts/glossary/#dag)) changes. ##### 🌡️ Added metrics per response type New metrics can be found at `/debug/metrics/prometheus` on the RPC API port (`127.0.0.1:5001` is the default): - `gw_first_content_block_get_latency_seconds` – the time until the first content block is received on GET from the gateway (no matter the content or response types) - `gw_unixfs_file_get_duration_seconds` – the time to serve an entire UnixFS file from the gateway - `gw_unixfs_gen_dir_listing_get_duration_seconds` – the time to serve a generated UnixFS HTML directory listing from the gateway - `gw_car_stream_get_duration_seconds` – the time to GET an entire CAR stream from the gateway - `gw_raw_block_get_duration_seconds` – The time to GET an entire raw Block from the gateway #### 🕵️ OpenTelemetry tracing *Opt-in tracing support with many spans for tracing the duration of specific tasks performed by go-ipfs.* See [Tracing](https://github.com/ipfs/go-ipfs/blob/master/docs/environment-variables.md#tracing) for details. We will continue to add tracing instrumentation throughout IPFS subcomponents over time. ##### How to use Jaeger UI for visual tracing? One can use the `jaegertracing/all-in-one` Docker image to run a full Jaeger stack and configure go-ipfs to publish traces to it (here, in an ephemeral container): ```console $ docker run --rm -it --name jaeger \ -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \ -p 5775:5775/udp \ -p 6831:6831/udp \ -p 6832:6832/udp \ -p 5778:5778 \ -p 16686:16686 \ -p 14268:14268 \ -p 14250:14250 \ -p 9411:9411 \ jaegertracing/all-in-one ``` Then, in other terminal, start go-ipfs with Jaeger tracing enabled: ``` $ OTEL_TRACES_EXPORTER=jaeger ipfs daemon ``` Finally, the [Jaeger UI](https://github.com/jaegertracing/jaeger-ui#readme) is available at http://localhost:16686 Below are examples of visual tracing for Gateway requests. (Note: this a preview how useful this insight is. Details may look different now, as we are constantly improving tracing annotations across the go-ipfs codebase.) | CAR | Block | File | Directory | | ---- | ---- | ---- | ---- | | ![2022-04-01_01-46](https://user-images.githubusercontent.com/157609/161167986-951d5c8c-9a5e-464d-bc20-81eb5ccbdc22.png) | ![block_2022-04-01_01-47](https://user-images.githubusercontent.com/157609/161167983-e8cac0ce-0575-4271-8cb8-4d44a0d5d786.png) | ![2022-04-01_01-49](https://user-images.githubusercontent.com/157609/161167978-e19aa44c-f5a4-45f4-b7c7-14c313ab1dee.png) | ![dir_2022-04-01_01-48](https://user-images.githubusercontent.com/157609/161167981-456ca52b-3e87-4042-916b-8db149071228.png) | #### 🩺 Built-in `ipfs diag profile` to ease debugging The `diag profile` command has been expanded to include all information that was previously included in the `collect-profiles.sh` script, and the script has been removed. Profiles are now collected in parallel, so that profile collection is much faster. Specific profiles can also be selected for targeted debugging. See `ipfs diag profile --help` for more details. For general debugging information, see [the debug guide](https://github.com/ipfs/go-ipfs/blob/master/docs/debug-guide.md). #### 🔑 Support for PEM/PKCS8 for key import/export It is now possible to import or export private keys wrapped in interoperable [PEM PKCS8](https://en.wikipedia.org/wiki/PKCS_8) by passing `--format=pem-pkcs8-cleartext` to `ipfs key import` and `export` commands. This improved interop allows for key generation outside of the IPFS node: ```console $ openssl genpkey -algorithm ED25519 > ed25519.pem $ ipfs key import test-openssl -f pem-pkcs8-cleartext ed25519.pem ``` Or using external tools like the standard `openssl` to get a PEM file with the public key: ```console $ ipfs key export testkey --format=pem-pkcs8-cleartext -o privkey.pem $ openssl pkey -in privkey.pem -pubout > pubkey.pem ``` #### 🧹 Using standard IPLD codec names across the CLI/HTTP API This release makes necessary (breaking) changes in effort to use canonical codec names from [multicodec/table.csv](https://github.com/multiformats/multicodec/blob/master/table.csv). We also switched to CIDv1 in `block put`. The breaking changes are discussed above. #### 🐳 Custom initialization for Docker Docker images published at https://hub.docker.com/r/ipfs/go-ipfs/ now support custom initialization by mounting scripts in the `/container-init.d` directory in the container. Scripts can set custom configuration using `ipfs config`, or otherwise customize the container before the daemon is started. Scripts are executed sequentially and in lexicographic order, before the IPFS daemon is started and after `ipfs init` is run and the swarm keys are copied (if the IPFS repo needs initialization). For more information, see: - Documentation: [ Run IPFS inside Docker](https://docs.ipfs.tech/how-to/run-ipfs-inside-docker/) - Examples in [ipfs-shipyard/go-ipfs-docker-examples](https://github.com/ipfs-shipyard/go-ipfs-docker-examples). #### RPC API docs for experimental and deprecated commands https://docs.ipfs.tech/reference/kubo/rpc/ now includes separate sections for _experimental_ and _deprecated_ commands. We also display a warning in the command line: ```console $ ipfs name pubsub state --help WARNING: EXPERIMENTAL, command may change in future releases ``` #### Yamux over Mplex The more fully featured yamux stream multiplexer is now prioritized over mplex for outgoing connections. ### Changelog
Full Changelog - github.com/ipfs/go-ipfs: - feat: upgrade to go-libp2p-kad-dht@v0.16.0 (#9005) ([ipfs/go-ipfs#9005](https://github.com/ipfs/go-ipfs/pull/9005)) - docs: fix typo in the `swarm/peering` help text - feat: disable resource manager by default (#9003) ([ipfs/go-ipfs#9003](https://github.com/ipfs/go-ipfs/pull/9003)) - fix: adjust rcmgr limits for accelerated DHT client rt refresh (#8982) ([ipfs/go-ipfs#8982](https://github.com/ipfs/go-ipfs/pull/8982)) - fix(ci): make go-ipfs-as-a-library work without external peers (#8978) ([ipfs/go-ipfs#8978](https://github.com/ipfs/go-ipfs/pull/8978)) - feat: log when resource manager limits are exceeded (#8980) ([ipfs/go-ipfs#8980](https://github.com/ipfs/go-ipfs/pull/8980)) - fix: JS caching via Access-Control-Expose-Headers (#8984) ([ipfs/go-ipfs#8984](https://github.com/ipfs/go-ipfs/pull/8984)) - docs: fix abstractions typo - fix: hanging goroutine in get fileArchive handler - fix(node/libp2p): disable rcmgr checkImplicitDefaults ([ipfs/go-ipfs#8965](https://github.com/ipfs/go-ipfs/pull/8965)) - pubsub multibase encoding (#8933) ([ipfs/go-ipfs#8933](https://github.com/ipfs/go-ipfs/pull/8933)) - 'pin rm' helptext: rewrite description as object is not removed from local storage (immediately) ([ipfs/go-ipfs#8947](https://github.com/ipfs/go-ipfs/pull/8947)) - ([ipfs/go-ipfs#8934](https://github.com/ipfs/go-ipfs/pull/8934)) - Add instructions to resolve repo migration error (#8946) ([ipfs/go-ipfs#8946](https://github.com/ipfs/go-ipfs/pull/8946)) - fix: use path instead of filepath for asset embeds to support Windows - Release v0.13.0-rc1 - docs: v0.13.0 changelog ([ipfs/go-ipfs#8941](https://github.com/ipfs/go-ipfs/pull/8941)) - chore: build with go 1.18.1 ([ipfs/go-ipfs#8932](https://github.com/ipfs/go-ipfs/pull/8932)) - docs(tracing): update env var docs for new tracing env vars - feat: enable Resource Manager by default - chore: Update test/dependencies to match go-ipfs dependencies. (#8928) ([ipfs/go-ipfs#8928](https://github.com/ipfs/go-ipfs/pull/8928)) - chore: fix linting errors (#8930) ([ipfs/go-ipfs#8930](https://github.com/ipfs/go-ipfs/pull/8930)) - docs: Swarm.ResourceMgr.Limits - feat: EnableHolePunching by default ([ipfs/go-ipfs#8748](https://github.com/ipfs/go-ipfs/pull/8748)) - ci: add more golang strictness checks ([ipfs/go-ipfs#8931](https://github.com/ipfs/go-ipfs/pull/8931)) - feat(gateway): Gateway.FastDirIndexThreshold (#8853) ([ipfs/go-ipfs#8853](https://github.com/ipfs/go-ipfs/pull/8853)) - docs: replace all git.io links with their actual URLs - feat: relay v2 discovery (go-libp2p v0.19.0) (#8868) ([ipfs/go-ipfs#8868](https://github.com/ipfs/go-ipfs/pull/8868)) - fix(cmds): add: reject files with different import dir - chore: mark 'log tail' experimental (#8912) ([ipfs/go-ipfs#8912](https://github.com/ipfs/go-ipfs/pull/8912)) - feat: persist limits to Swarm.ResourceMgr.Limits (#8901) ([ipfs/go-ipfs#8901](https://github.com/ipfs/go-ipfs/pull/8901)) - fix: build after Go 1.17 and Prometheus upgrades (#8916) ([ipfs/go-ipfs#8916](https://github.com/ipfs/go-ipfs/pull/8916)) - feat(tracing): use OpenTelemetry env vars where possible (#8875) ([ipfs/go-ipfs#8875](https://github.com/ipfs/go-ipfs/pull/8875)) - feat(cmds): allow to set the configuration file path ([ipfs/go-ipfs#8634](https://github.com/ipfs/go-ipfs/pull/8634)) - chore: deprecate /api/v0/dns (#8893) ([ipfs/go-ipfs#8893](https://github.com/ipfs/go-ipfs/pull/8893)) - fix(cmds): CIDv1 and correct multicodecs in 'block put' and 'cid codecs' (#8568) ([ipfs/go-ipfs#8568](https://github.com/ipfs/go-ipfs/pull/8568)) - feat(gw): improved If-None-Match support (#8891) ([ipfs/go-ipfs#8891](https://github.com/ipfs/go-ipfs/pull/8891)) - Update Go version to 1.17 (#8815) ([ipfs/go-ipfs#8815](https://github.com/ipfs/go-ipfs/pull/8815)) - chore(gw): extract logical functions to improve readability (#8885) ([ipfs/go-ipfs#8885](https://github.com/ipfs/go-ipfs/pull/8885)) - feat(docker): /container-init.d for advanced initialization (#6577) ([ipfs/go-ipfs#6577](https://github.com/ipfs/go-ipfs/pull/6577)) - feat: port collect-profiles.sh to 'ipfs diag profile' (#8786) ([ipfs/go-ipfs#8786](https://github.com/ipfs/go-ipfs/pull/8786)) - fix: assets: correctly use the argument err in the WalkDirFunc Hashing Files - Change `assets.Asset` from a `func` to the embed.FS - Remove gobindata - fix: fix context plumbing in gateway handlers (#8871) ([ipfs/go-ipfs#8871](https://github.com/ipfs/go-ipfs/pull/8871)) - fix(gw): missing return if dir fails to finalize (#8806) ([ipfs/go-ipfs#8806](https://github.com/ipfs/go-ipfs/pull/8806)) - fix(gw): update metrics only when payload data sent (#8827) ([ipfs/go-ipfs#8827](https://github.com/ipfs/go-ipfs/pull/8827)) - Merge branch 'release' - feat: detect changes in go-libp2p-resource-manager (#8857) ([ipfs/go-ipfs#8857](https://github.com/ipfs/go-ipfs/pull/8857)) - feat: opt-in Swarm.ResourceMgr (go-libp2p v0.18) (#8680) ([ipfs/go-ipfs#8680](https://github.com/ipfs/go-ipfs/pull/8680)) - feat(cmds): add support for CAR v2 imports (#8854) ([ipfs/go-ipfs#8854](https://github.com/ipfs/go-ipfs/pull/8854)) - docs(logging): environment variables (#8833) ([ipfs/go-ipfs#8833](https://github.com/ipfs/go-ipfs/pull/8833)) - fix: unknown fetcher type error (#8830) ([ipfs/go-ipfs#8830](https://github.com/ipfs/go-ipfs/pull/8830)) - chore: deprecate tar commands (#8849) ([ipfs/go-ipfs#8849](https://github.com/ipfs/go-ipfs/pull/8849)) - chore: bump go-ipld-format v0.4.0 and fix related sharness tests ([ipfs/go-ipfs#8838](https://github.com/ipfs/go-ipfs/pull/8838)) - feat: add basic gateway tracing (#8595) ([ipfs/go-ipfs#8595](https://github.com/ipfs/go-ipfs/pull/8595)) - fix(cli): ipfs add with multiple files of same name (#8493) ([ipfs/go-ipfs#8493](https://github.com/ipfs/go-ipfs/pull/8493)) - fix(gw): validate requested CAR version (#8835) ([ipfs/go-ipfs#8835](https://github.com/ipfs/go-ipfs/pull/8835)) - feat: re-enable docker sharness tests (#8808) ([ipfs/go-ipfs#8808](https://github.com/ipfs/go-ipfs/pull/8808)) - docs: gateway.md (#8825) ([ipfs/go-ipfs#8825](https://github.com/ipfs/go-ipfs/pull/8825)) - fix(core/commands): do not cache config (#8824) ([ipfs/go-ipfs#8824](https://github.com/ipfs/go-ipfs/pull/8824)) - remove unused import (#8787) ([ipfs/go-ipfs#8787](https://github.com/ipfs/go-ipfs/pull/8787)) - feat(cmds): document deprecated RPC API commands (#8802) ([ipfs/go-ipfs#8802](https://github.com/ipfs/go-ipfs/pull/8802)) - fix(fsrepo): deep merge when setting config ([ipfs/go-ipfs#8793](https://github.com/ipfs/go-ipfs/pull/8793)) - feat: add gateway histogram metrics (#8443) ([ipfs/go-ipfs#8443](https://github.com/ipfs/go-ipfs/pull/8443)) - ErrNotFound changes: bubble tagged libraries. ([ipfs/go-ipfs#8803](https://github.com/ipfs/go-ipfs/pull/8803)) - Fix typos ([ipfs/go-ipfs#8757](https://github.com/ipfs/go-ipfs/pull/8757)) - feat(gateway): Block and CAR response formats (#8758) ([ipfs/go-ipfs#8758](https://github.com/ipfs/go-ipfs/pull/8758)) - fix: allow ipfs-companion browser extension to access RPC API (#8690) ([ipfs/go-ipfs#8690](https://github.com/ipfs/go-ipfs/pull/8690)) - fix(core/node): unwrap fx error in node construction ([ipfs/go-ipfs#8638](https://github.com/ipfs/go-ipfs/pull/8638)) - Update PATCH_RELEASE_TEMPLATE.md - feat: add full goroutine stack dump (#8790) ([ipfs/go-ipfs#8790](https://github.com/ipfs/go-ipfs/pull/8790)) - feat(cmds): extend block size check for dag|block put (#8751) ([ipfs/go-ipfs#8751](https://github.com/ipfs/go-ipfs/pull/8751)) - feat: add endpoint for enabling block profiling (#8469) ([ipfs/go-ipfs#8469](https://github.com/ipfs/go-ipfs/pull/8469)) - fix(cmds): option for progress bar in cat/get (#8686) ([ipfs/go-ipfs#8686](https://github.com/ipfs/go-ipfs/pull/8686)) - docs: note the default reprovider strategy as all (#8603) ([ipfs/go-ipfs#8603](https://github.com/ipfs/go-ipfs/pull/8603)) - fix: listen on loopback for API and gateway ports in docker-compose.yaml (#8773) ([ipfs/go-ipfs#8773](https://github.com/ipfs/go-ipfs/pull/8773)) - fix(discovery): fix daemon not starting due to mdns startup failure (#8704) ([ipfs/go-ipfs#8704](https://github.com/ipfs/go-ipfs/pull/8704)) ([ipfs/go-ipfs#8756](https://github.com/ipfs/go-ipfs/pull/8756)) - feat: ipfs-webui v2.15 (#8712) ([ipfs/go-ipfs#8712](https://github.com/ipfs/go-ipfs/pull/8712)) - feat: X-Ipfs-Roots for smarter HTTP caches (#8720) ([ipfs/go-ipfs#8720](https://github.com/ipfs/go-ipfs/pull/8720)) - chore: add instructions for Chocolatey release - fix prioritization of stream muxers ([ipfs/go-ipfs#8750](https://github.com/ipfs/go-ipfs/pull/8750)) - fix(cmds/keystore): do not allow to import keys we don't generate (#8733) ([ipfs/go-ipfs#8733](https://github.com/ipfs/go-ipfs/pull/8733)) - docs: add Internal.UnixFSShardingSizeThreshold (#8723) ([ipfs/go-ipfs#8723](https://github.com/ipfs/go-ipfs/pull/8723)) - feat(cmd): add silent option for repo gc (#7147) ([ipfs/go-ipfs#7147](https://github.com/ipfs/go-ipfs/pull/7147)) - docs(changelog): update v0.12.0 release notes - Merge branch 'release' - fix: installation without sudo (#8715) ([ipfs/go-ipfs#8715](https://github.com/ipfs/go-ipfs/pull/8715)) - Fix typos (#8726) ([ipfs/go-ipfs#8726](https://github.com/ipfs/go-ipfs/pull/8726)) - fix(build): Recognize Go Beta versions in makefile (#8677) ([ipfs/go-ipfs#8677](https://github.com/ipfs/go-ipfs/pull/8677)) - chore(cmds): encapsulate ipfs rm logic in another function (#8574) ([ipfs/go-ipfs#8574](https://github.com/ipfs/go-ipfs/pull/8574)) - feat: warn user when 'pin remote add' while offline (#8621) ([ipfs/go-ipfs#8621](https://github.com/ipfs/go-ipfs/pull/8621)) - chore(gateway): debug logging for the http requests (#8518) ([ipfs/go-ipfs#8518](https://github.com/ipfs/go-ipfs/pull/8518)) - docker: build for arm cpu (#8633) ([ipfs/go-ipfs#8633](https://github.com/ipfs/go-ipfs/pull/8633)) - feat: refactor Fetcher interface used for downloading migrations (#8728) ([ipfs/go-ipfs#8728](https://github.com/ipfs/go-ipfs/pull/8728)) - feat: log multifetcher errors - docs: optionalInteger|String|Duration (#8729) ([ipfs/go-ipfs#8729](https://github.com/ipfs/go-ipfs/pull/8729)) - feat: DNS.MaxCacheTTL for DNS-over-HTTPS resolvers (#8615) ([ipfs/go-ipfs#8615](https://github.com/ipfs/go-ipfs/pull/8615)) - feat(cmds): ipfs id: support --offline option (#8626) ([ipfs/go-ipfs#8626](https://github.com/ipfs/go-ipfs/pull/8626)) - feat(cmds): add cleartext PEM/PKCS8 for key import/export (#8616) ([ipfs/go-ipfs#8616](https://github.com/ipfs/go-ipfs/pull/8616)) - docs: update badger section in config.md (#8662) ([ipfs/go-ipfs#8662](https://github.com/ipfs/go-ipfs/pull/8662)) - docs: fix typo - Adding PowerShell to Minimal Go Installation - Fixed typos in docs/config.md - docs: add Snap note about customizing IPFS_PATH (#8584) ([ipfs/go-ipfs#8584](https://github.com/ipfs/go-ipfs/pull/8584)) - Fix typo ([ipfs/go-ipfs#8625](https://github.com/ipfs/go-ipfs/pull/8625)) - chore: update version to v0.13.0-dev - github.com/ipfs/go-bitswap (v0.5.1 -> v0.6.0): - v0.6.0 - Use ipld.ErrNotFound - feat: add peer block filter option (#549) ([ipfs/go-bitswap#549](https://github.com/ipfs/go-bitswap/pull/549)) - configurable target message size ([ipfs/go-bitswap#546](https://github.com/ipfs/go-bitswap/pull/546)) - github.com/ipfs/go-blockservice (v0.2.1 -> v0.3.0): - v0.3.0 - s/log/logger - Use ipld.ErrNotFound instead of ErrNotFound - github.com/ipfs/go-cid (v0.1.0 -> v0.2.0): - fix: remove invalid multicodec2string mappings (#137) ([ipfs/go-cid#137](https://github.com/ipfs/go-cid/pull/137)) - sync: update CI config files (#136) ([ipfs/go-cid#136](https://github.com/ipfs/go-cid/pull/136)) - Benchmark existing ways to check for `IDENTITY` CIDs - avoid double alloc in NewCidV1 - sync: update CI config files ([ipfs/go-cid#131](https://github.com/ipfs/go-cid/pull/131)) - github.com/ipfs/go-cidutil (v0.0.2 -> v0.1.0): ([ipfs/go-cidutil#36](https://github.com/ipfs/go-cidutil/pull/36)) - sync: update CI config files ([ipfs/go-cidutil#35](https://github.com/ipfs/go-cidutil/pull/35)) - sync: update CI config files (#34) ([ipfs/go-cidutil#34](https://github.com/ipfs/go-cidutil/pull/34)) - fix staticcheck ([ipfs/go-cidutil#31](https://github.com/ipfs/go-cidutil/pull/31)) - add license file so it can be found by go-licenses ([ipfs/go-cidutil#27](https://github.com/ipfs/go-cidutil/pull/27)) - test: fix for base32 switch ([ipfs/go-cidutil#16](https://github.com/ipfs/go-cidutil/pull/16)) - doc: add a lead maintainer - github.com/ipfs/go-filestore (v1.1.0 -> v1.2.0): - v1.2.0 - refactor: follow the happy left practice in Filestore.DeleteBlock - Use ipld.ErrNotFound - github.com/ipfs/go-graphsync (v0.11.0 -> v0.13.1): - docs(CHANGELOG): update for v0.13.1 - feat(ipld): wrap bindnode with panic protection (#368) ([ipfs/go-graphsync#368](https://github.com/ipfs/go-graphsync/pull/368)) - docs(CHANGELOG): update for v0.13.0 (#366) ([ipfs/go-graphsync#366](https://github.com/ipfs/go-graphsync/pull/366)) - fix(impl): delete file - Minimal alternate metadata type support (#365) ([ipfs/go-graphsync#365](https://github.com/ipfs/go-graphsync/pull/365)) - Fix unixfs fetch (#364) ([ipfs/go-graphsync#364](https://github.com/ipfs/go-graphsync/pull/364)) - [Feature] UUIDs, protocol versioning, v2 protocol w/ dag-cbor messaging (#332) ([ipfs/go-graphsync#332](https://github.com/ipfs/go-graphsync/pull/332)) - feat(CHANGELOG): update for v0.12.0 - Use do not send blocks for pause/resume & prevent processing of blocks on canceled requests (#333) ([ipfs/go-graphsync#333](https://github.com/ipfs/go-graphsync/pull/333)) - Support unixfs reification in default linksystem (#329) ([ipfs/go-graphsync#329](https://github.com/ipfs/go-graphsync/pull/329)) - Don't run hooks on blocks we didn't have (#331) ([ipfs/go-graphsync#331](https://github.com/ipfs/go-graphsync/pull/331)) - feat(responsemanager): trace full messages via links to responses (#325) ([ipfs/go-graphsync#325](https://github.com/ipfs/go-graphsync/pull/325)) - chore(requestmanager): rename processResponses internals for consistency (#328) ([ipfs/go-graphsync#328](https://github.com/ipfs/go-graphsync/pull/328)) - Response message tracing (#327) ([ipfs/go-graphsync#327](https://github.com/ipfs/go-graphsync/pull/327)) - fix(testutil): fix tracing span collection (#324) ([ipfs/go-graphsync#324](https://github.com/ipfs/go-graphsync/pull/324)) - docs(CHANGELOG): update for v0.11.5 release - feat(requestmanager): add tracing for response messages & block processing (#322) ([ipfs/go-graphsync#322](https://github.com/ipfs/go-graphsync/pull/322)) - ipldutil: simplify state synchronization, add docs (#300) ([ipfs/go-graphsync#300](https://github.com/ipfs/go-graphsync/pull/300)) - docs(CHANGELOG): update for v0.11.4 release - Scrub response errors (#320) ([ipfs/go-graphsync#320](https://github.com/ipfs/go-graphsync/pull/320)) - fix(responsemanager): remove unused maxInProcessRequests parameter (#319) ([ipfs/go-graphsync#319](https://github.com/ipfs/go-graphsync/pull/319)) - feat(responsemanager): allow ctx augmentation via queued request hook - make go test with coverpkg=./... - docs(CHANGELOG): update for v0.11.3 - Merge tag 'v0.10.9' - feat: add basic tracing for responses (#291) ([ipfs/go-graphsync#291](https://github.com/ipfs/go-graphsync/pull/291)) - fix(impl): remove accidental legacy field (#310) ([ipfs/go-graphsync#310](https://github.com/ipfs/go-graphsync/pull/310)) - docs(CHANGELOG): update for v0.11.2 - Merge branch 'release/v0.10.8' - feat(taskqueue): fix race on peer state gather (#303) ([ipfs/go-graphsync#303](https://github.com/ipfs/go-graphsync/pull/303)) - feat(responsemanager): clarify response completion (#304) ([ipfs/go-graphsync#304](https://github.com/ipfs/go-graphsync/pull/304)) - docs(CHANGELOG): update for v0.11.1 - Merge branch 'release/v0.10.7' - Expose task queue diagnostics (#302) ([ipfs/go-graphsync#302](https://github.com/ipfs/go-graphsync/pull/302)) - chore: short-circuit unnecessary message processing - Add a bit of logging (#301) ([ipfs/go-graphsync#301](https://github.com/ipfs/go-graphsync/pull/301)) - Peer Stats function (#298) ([ipfs/go-graphsync#298](https://github.com/ipfs/go-graphsync/pull/298)) - fix: use sync.Cond to handle no-task blocking wait (#299) ([ipfs/go-graphsync#299](https://github.com/ipfs/go-graphsync/pull/299)) - ipldutil: use chooser APIs from dagpb and basicnode (#292) ([ipfs/go-graphsync#292](https://github.com/ipfs/go-graphsync/pull/292)) - testutil/chaintypes: simplify maintenance of codegen (#294) ([ipfs/go-graphsync#294](https://github.com/ipfs/go-graphsync/pull/294)) - fix(test): increase 1s timeouts to 2s for slow CI (#289) ([ipfs/go-graphsync#289](https://github.com/ipfs/go-graphsync/pull/289)) - docs(tests): document tracing test helper utilities - feat: add basic OT tracing for incoming requests - fix(responsemanager): make fix more global - fix(responsemanager): fix flaky tests - feat: add WorkerTaskQueue#WaitForNoActiveTasks() for tests (#284) ([ipfs/go-graphsync#284](https://github.com/ipfs/go-graphsync/pull/284)) - github.com/ipfs/go-ipfs-blockstore (v1.1.2 -> v1.2.0): - v0.2.0 ([ipfs/go-ipfs-blockstore#98](https://github.com/ipfs/go-ipfs-blockstore/pull/98)) - s/log/logger - Use ipld.ErrNotFound for NotFound errors - github.com/ipfs/go-ipfs-cmds (v0.6.0 -> v0.8.1): - fix(cli/parse): extract dir before name ([ipfs/go-ipfs-cmds#230](https://github.com/ipfs/go-ipfs-cmds/pull/230)) - Version 0.8.0 ([ipfs/go-ipfs-cmds#228](https://github.com/ipfs/go-ipfs-cmds/pull/228)) - fix(cli): use NewSliceDirectory for duplicate file paths ([ipfs/go-ipfs-cmds#220](https://github.com/ipfs/go-ipfs-cmds/pull/220)) - chore: release v0.7.0 ([ipfs/go-ipfs-cmds#227](https://github.com/ipfs/go-ipfs-cmds/pull/227)) - feat(Command): add status for the helptext ([ipfs/go-ipfs-cmds#225](https://github.com/ipfs/go-ipfs-cmds/pull/225)) - allow header and set header in client ([ipfs/go-ipfs-cmds#226](https://github.com/ipfs/go-ipfs-cmds/pull/226)) - sync: update CI config files (#221) ([ipfs/go-ipfs-cmds#221](https://github.com/ipfs/go-ipfs-cmds/pull/221)) - fix: chanResponseEmitter cancel being ineffective - add: tests for postrun execution - fix: postrun's run condition in Execute - fix: exec deadlock when emitter is not Typer intf - sync: update CI config files ([ipfs/go-ipfs-cmds#207](https://github.com/ipfs/go-ipfs-cmds/pull/207)) - fix: preserve windows file paths ([ipfs/go-ipfs-cmds#214](https://github.com/ipfs/go-ipfs-cmds/pull/214)) - Resolve `staticcheck` issue in prep for unified CI ([ipfs/go-ipfs-cmds#212](https://github.com/ipfs/go-ipfs-cmds/pull/212)) - github.com/ipfs/go-ipfs-exchange-offline (v0.1.1 -> v0.2.0): - v0.2.0 - Improve NotFound error description - github.com/ipfs/go-ipfs-files (v0.0.9 -> v0.1.1): - Release v0.1.1 - fix: add dragonfly build option for filewriter flags - fix: add freebsd build option for filewriter flags - Release v0.1.0 - docs: fix community CONTRIBUTING.md link (#45) ([ipfs/go-ipfs-files#45](https://github.com/ipfs/go-ipfs-files/pull/45)) - chore(filewriter): cleanup writes (#43) ([ipfs/go-ipfs-files#43](https://github.com/ipfs/go-ipfs-files/pull/43)) - sync: update CI config files (#44) ([ipfs/go-ipfs-files#44](https://github.com/ipfs/go-ipfs-files/pull/44)) - github.com/ipfs/go-ipld-format (v0.2.0 -> v0.4.0): - chore: release version v0.4.0 - feat: use new more clearer format in ErrNotFound - chore: bump version to 0.3.1 - fix: make Undef ErrNotFound string consistent with Def version - Version 0.3.0 - ErrNotFound: change error string ([ipfs/go-ipld-format#69](https://github.com/ipfs/go-ipld-format/pull/69)) - Revert "Revert "Add IsErrNotFound() method"" ([ipfs/go-ipld-format#68](https://github.com/ipfs/go-ipld-format/pull/68)) - sync: update CI config files (#67) ([ipfs/go-ipld-format#67](https://github.com/ipfs/go-ipld-format/pull/67)) - ignore statticheck error for EndOfDag ([ipfs/go-ipld-format#62](https://github.com/ipfs/go-ipld-format/pull/62)) - remove Makefile ([ipfs/go-ipld-format#59](https://github.com/ipfs/go-ipld-format/pull/59)) - fix staticcheck ([ipfs/go-ipld-format#60](https://github.com/ipfs/go-ipld-format/pull/60)) - Allowing custom NavigableNode implementations ([ipfs/go-ipld-format#58](https://github.com/ipfs/go-ipld-format/pull/58)) - github.com/ipfs/go-ipld-legacy (v0.1.0 -> v0.1.1): - feat(node): add json.Marshaller method ([ipfs/go-ipld-legacy#7](https://github.com/ipfs/go-ipld-legacy/pull/7)) - github.com/ipfs/go-log/v2 (v2.3.0 -> v2.5.1): - feat: add logger option to skip a number of stack frames ([ipfs/go-log#132](https://github.com/ipfs/go-log/pull/132)) - release v2.5.0 (#131) ([ipfs/go-log#131](https://github.com/ipfs/go-log/pull/131)) - config inspection (#129) ([ipfs/go-log#129](https://github.com/ipfs/go-log/pull/129)) - release v2.4.0 (#127) ([ipfs/go-log#127](https://github.com/ipfs/go-log/pull/127)) - sync: update CI config files (#125) ([ipfs/go-log#125](https://github.com/ipfs/go-log/pull/125)) - fix: cannot call SetPrimaryCore after using a Tee logger ([ipfs/go-log#121](https://github.com/ipfs/go-log/pull/121)) - Document environment variables ([ipfs/go-log#120](https://github.com/ipfs/go-log/pull/120)) - sync: update CI config files (#119) ([ipfs/go-log#119](https://github.com/ipfs/go-log/pull/119)) - Add WithStacktrace utility ([ipfs/go-log#118](https://github.com/ipfs/go-log/pull/118)) - In addition to StdOut/Err check the outfile for TTYness ([ipfs/go-log#117](https://github.com/ipfs/go-log/pull/117)) - github.com/ipfs/go-merkledag (v0.5.1 -> v0.6.0): - v0.6.0 - Improve ErrNotFound - github.com/ipfs/go-namesys (v0.4.0 -> v0.5.0): - Version 0.5.0 - fix: CIDv1 error with go-libp2p 0.19 (#32) ([ipfs/go-namesys#32](https://github.com/ipfs/go-namesys/pull/32)) - feat: add tracing (#30) ([ipfs/go-namesys#30](https://github.com/ipfs/go-namesys/pull/30)) - fix(publisher): fix garbled code output (#28) ([ipfs/go-namesys#28](https://github.com/ipfs/go-namesys/pull/28)) - github.com/ipfs/go-path (v0.2.1 -> v0.3.0): - Release v0.3.0 ([ipfs/go-path#55](https://github.com/ipfs/go-path/pull/55)) - Resolver: convert to interface. ([ipfs/go-path#53](https://github.com/ipfs/go-path/pull/53)) - Release v0.2.2 (#52) ([ipfs/go-path#52](https://github.com/ipfs/go-path/pull/52)) - chore: improve error message for invalid ipfs paths ([ipfs/go-path#51](https://github.com/ipfs/go-path/pull/51)) - github.com/ipfs/go-peertaskqueue (v0.7.0 -> v0.7.1): - Add topic inspector ([ipfs/go-peertaskqueue#20](https://github.com/ipfs/go-peertaskqueue/pull/20)) - github.com/ipfs/go-pinning-service-http-client (v0.1.0 -> v0.1.1): - chore: release v0.1.1 - fix: error handling while enumerating pins - sync: update CI config files (#15) ([ipfs/go-pinning-service-http-client#15](https://github.com/ipfs/go-pinning-service-http-client/pull/15)) - Resolve lint issues prior to CI integration - github.com/ipfs/go-unixfsnode (v1.1.3 -> v1.4.0): - 1.4.0 release ([ipfs/go-unixfsnode#29](https://github.com/ipfs/go-unixfsnode/pull/29)) - Partial file test ([ipfs/go-unixfsnode#26](https://github.com/ipfs/go-unixfsnode/pull/26)) - Add unixfs to UnixFS path selector tail ([ipfs/go-unixfsnode#28](https://github.com/ipfs/go-unixfsnode/pull/28)) - release v1.3.0 ([ipfs/go-unixfsnode#25](https://github.com/ipfs/go-unixfsnode/pull/25)) - add AsLargeBytes support to unixfs files (#24) ([ipfs/go-unixfsnode#24](https://github.com/ipfs/go-unixfsnode/pull/24)) - fix: add extra test to span the shard/no-shard boundary - fix: more Tsize fixes, fix HAMT and make it match go-unixfs output - fix: encode Tsize correctly everywhere (using wrapped LinkSystem) - docs(version): tag 1.2.0 - Update deps for ADL selectors ([ipfs/go-unixfsnode#20](https://github.com/ipfs/go-unixfsnode/pull/20)) - add license (#17) ([ipfs/go-unixfsnode#17](https://github.com/ipfs/go-unixfsnode/pull/17)) - handle empty files (#15) ([ipfs/go-unixfsnode#15](https://github.com/ipfs/go-unixfsnode/pull/15)) - Add ADL/single-node-view of a full unixFS file. (#14) ([ipfs/go-unixfsnode#14](https://github.com/ipfs/go-unixfsnode/pull/14)) - sync: update CI config files (#13) ([ipfs/go-unixfsnode#13](https://github.com/ipfs/go-unixfsnode/pull/13)) - Add builder for unixfs dags (#12) ([ipfs/go-unixfsnode#12](https://github.com/ipfs/go-unixfsnode/pull/12)) - github.com/ipfs/interface-go-ipfs-core (v0.5.2 -> v0.7.0): - refactor(block): CIDv1 and BlockPutSettings CidPrefix (#80) ([ipfs/interface-go-ipfs-core#80](https://github.com/ipfs/interface-go-ipfs-core/pull/80)) - chore: release v0.6.2 - fix: use IPLD.ErrNotFound instead of string comparison in tests - fix: document error (#74) ([ipfs/interface-go-ipfs-core#74](https://github.com/ipfs/interface-go-ipfs-core/pull/74)) - version: release 0.6.1 - v0.6.0 - Update tests to use ipld.IsNotFound to check for notfound errors - sync: update CI config files (#79) ([ipfs/interface-go-ipfs-core#79](https://github.com/ipfs/interface-go-ipfs-core/pull/79)) - github.com/ipld/go-codec-dagpb (v1.3.2 -> v1.4.0): - bump to v1.4.0 given that we updated ipld-prime - add a decode-then-encode roundtrip fuzzer - 1.3.1 - fix: use protowire for Links bytes decoding - delete useless code - sync: update CI config files (#33) ([ipld/go-codec-dagpb#33](https://github.com/ipld/go-codec-dagpb/pull/33)) - github.com/ipld/go-ipld-prime (v0.14.2 -> v0.16.0): - mark v0.16.0 - node/bindnode: enforce pointer requirement for nullable maps - Implement WalkTransforming traversal (#376) ([ipld/go-ipld-prime#376](https://github.com/ipld/go-ipld-prime/pull/376)) - docs(datamodel): add comment to LargeBytesNode - Add partial-match traversal of large bytes (#375) ([ipld/go-ipld-prime#375](https://github.com/ipld/go-ipld-prime/pull/375)) - Implement option to start traversals at a path ([ipld/go-ipld-prime#358](https://github.com/ipld/go-ipld-prime/pull/358)) - add top-level "go value with schema" example - Support optional `LargeBytesNode` interface (#372) ([ipld/go-ipld-prime#372](https://github.com/ipld/go-ipld-prime/pull/372)) - node/bindnode: support pointers to datamodel.Node to bind with Any - fix(bindnode): tuple struct iterator should handle absent fields properly - node/bindnode: make AssignNode work at the repr level - node/bindnode: add support for unsigned integers - node/bindnode: cover even more edge case panics - node/bindnode: polish some more AsT panics - schema/dmt: stop using a fake test to generate code ([ipld/go-ipld-prime#356](https://github.com/ipld/go-ipld-prime/pull/356)) - schema: remove one review note; add another. - fix: minor EncodedLength fixes, add tests to fully exercise - feat: add dagcbor.EncodedLength(Node) to calculate length without encoding - chore: rename Garbage() to Generate() - fix: minor garbage nits - fix: Garbage() takes rand parameter, tweak algorithms, improve docs - feat: add Garbage() Node generator - node/bindnode: introduce an assembler that always errors - node/bindnode: polish panics on invalid AssignT calls - datamodel: don't panic when stringifying an empty KindSet - node/bindnode: start using ipld.LoadSchema APIs - selectors: fix for edge case around recursion clauses with an immediate edge. ([ipld/go-ipld-prime#334](https://github.com/ipld/go-ipld-prime/pull/334)) - node/bindnode: improve support for pointer types - node/bindnode: subtract all absents in Length at the repr level - fix(codecs): error when encoding maps whose lengths don't match entry count - schema: avoid alloc and copy in Struct and Enum methods - node/bindnode: allow mapping int-repr enums with Go integers - schema,node/bindnode: add support for Any - signaling ADLs in selectors (#301) ([ipld/go-ipld-prime#301](https://github.com/ipld/go-ipld-prime/pull/301)) - node/bindnode: add support for enums - schema/...: add support for enum int representations - node/bindnode: allow binding cidlink.Link to links - Update to context datastores (#312) ([ipld/go-ipld-prime#312](https://github.com/ipld/go-ipld-prime/pull/312)) - schema: add support for struct tuple reprs - Allow parsing padding in dag-json bytes fields (#309) ([ipld/go-ipld-prime#309](https://github.com/ipld/go-ipld-prime/pull/309)) - github.com/libp2p/go-doh-resolver (v0.3.1 -> v0.4.0): - Release v0.4.0 (#16) ([libp2p/go-doh-resolver#16](https://github.com/libp2p/go-doh-resolver/pull/16)) - sync: update CI config files (#14) ([libp2p/go-doh-resolver#14](https://github.com/libp2p/go-doh-resolver/pull/14)) - Add a max TTL for cached entries ([libp2p/go-doh-resolver#12](https://github.com/libp2p/go-doh-resolver/pull/12)) - Perform test locally instead of using a live dns resolution ([libp2p/go-doh-resolver#13](https://github.com/libp2p/go-doh-resolver/pull/13)) - sync: update CI config files (#7) ([libp2p/go-doh-resolver#7](https://github.com/libp2p/go-doh-resolver/pull/7)) - fix staticcheck ([libp2p/go-doh-resolver#6](https://github.com/libp2p/go-doh-resolver/pull/6)) - github.com/libp2p/go-libp2p (v0.16.0 -> v0.19.4): - update go-yamux to v3.1.2, release v0.19.4 (#1590) ([libp2p/go-libp2p#1590](https://github.com/libp2p/go-libp2p/pull/1590)) - update quic-go to v0.27.1, release v0.19.3 (#1518) ([libp2p/go-libp2p#1518](https://github.com/libp2p/go-libp2p/pull/1518)) - release v0.19.2 - holepunch: fix incorrect message type for the SYNC message (#1478) ([libp2p/go-libp2p#1478](https://github.com/libp2p/go-libp2p/pull/1478)) - fix race condition in holepunch service, release v0.19.1 ([libp2p/go-libp2p#1474](https://github.com/libp2p/go-libp2p/pull/1474)) - release v0.19.0 (#1408) ([libp2p/go-libp2p#1408](https://github.com/libp2p/go-libp2p/pull/1408)) - Close resource manager when host closes (#1343) ([libp2p/go-libp2p#1343](https://github.com/libp2p/go-libp2p/pull/1343)) - fix flaky reconnect test (#1406) ([libp2p/go-libp2p#1406](https://github.com/libp2p/go-libp2p/pull/1406)) - make sure to not oversubscribe to relays (#1404) ([libp2p/go-libp2p#1404](https://github.com/libp2p/go-libp2p/pull/1404)) - rewrite the reconnect test (#1399) ([libp2p/go-libp2p#1399](https://github.com/libp2p/go-libp2p/pull/1399)) - don't try to reconnect to already connected relays (#1401) ([libp2p/go-libp2p#1401](https://github.com/libp2p/go-libp2p/pull/1401)) - reduce flakiness of AutoRelay TestBackoff test (#1400) ([libp2p/go-libp2p#1400](https://github.com/libp2p/go-libp2p/pull/1400)) - improve AutoRelay v1 handling ([libp2p/go-libp2p#1396](https://github.com/libp2p/go-libp2p/pull/1396)) - remove note about gx from README (#1385) ([libp2p/go-libp2p#1385](https://github.com/libp2p/go-libp2p/pull/1385)) - use the vcs information from ReadBuildInfo in Go 1.18 ([libp2p/go-libp2p#1381](https://github.com/libp2p/go-libp2p/pull/1381)) - fix race condition in AutoRelay candidate handling (#1383) ([libp2p/go-libp2p#1383](https://github.com/libp2p/go-libp2p/pull/1383)) - implement relay v2 discovery ([libp2p/go-libp2p#1368](https://github.com/libp2p/go-libp2p/pull/1368)) - fix go vet error in proxy example (#1377) ([libp2p/go-libp2p#1377](https://github.com/libp2p/go-libp2p/pull/1377)) - Resolve addresses when creating a new stream (#1342) ([libp2p/go-libp2p#1342](https://github.com/libp2p/go-libp2p/pull/1342)) - remove mplex from the list of default muxers (#1344) ([libp2p/go-libp2p#1344](https://github.com/libp2p/go-libp2p/pull/1344)) - refactor the holepunching code ([libp2p/go-libp2p#1355](https://github.com/libp2p/go-libp2p/pull/1355)) - speed up the connmgr tests (#1354) ([libp2p/go-libp2p#1354](https://github.com/libp2p/go-libp2p/pull/1354)) - update go-libp2p-resource manager, release v0.18.0 (#1361) ([libp2p/go-libp2p#1361](https://github.com/libp2p/go-libp2p/pull/1361)) - fix flaky BackoffConnector test (#1353) ([libp2p/go-libp2p#1353](https://github.com/libp2p/go-libp2p/pull/1353)) - release v0.18.0-rc6 (#1350) ([libp2p/go-libp2p#1350](https://github.com/libp2p/go-libp2p/pull/1350)) - release v0.18.0-rc5 ([libp2p/go-libp2p#1341](https://github.com/libp2p/go-libp2p/pull/1341)) - update README (#1330) ([libp2p/go-libp2p#1330](https://github.com/libp2p/go-libp2p/pull/1330)) - fix parsing of IP addresses for zeroconf initialization (#1338) ([libp2p/go-libp2p#1338](https://github.com/libp2p/go-libp2p/pull/1338)) - fix flaky TestBackoffConnector test (#1328) ([libp2p/go-libp2p#1328](https://github.com/libp2p/go-libp2p/pull/1328)) - release v0.18.0-rc4 ([libp2p/go-libp2p#1327](https://github.com/libp2p/go-libp2p/pull/1327)) - fix (and speed up) flaky TestBackoffConnector test (#1316) ([libp2p/go-libp2p#1316](https://github.com/libp2p/go-libp2p/pull/1316)) - fix flaky TestAutoRelay test (#1322) ([libp2p/go-libp2p#1322](https://github.com/libp2p/go-libp2p/pull/1322)) - deflake resource manager tests, take 2 ([libp2p/go-libp2p#1318](https://github.com/libp2p/go-libp2p/pull/1318)) - fix race condition causing TestAutoNATServiceDialError test failure (#1312) ([libp2p/go-libp2p#1312](https://github.com/libp2p/go-libp2p/pull/1312)) - disable flaky relay example test on CI (#1219) ([libp2p/go-libp2p#1219](https://github.com/libp2p/go-libp2p/pull/1219)) - fix flaky resource manager tests ([libp2p/go-libp2p#1315](https://github.com/libp2p/go-libp2p/pull/1315)) - update deps, fixing nil peer scope pointer issues in connection upgrading ([libp2p/go-libp2p#1309](https://github.com/libp2p/go-libp2p/pull/1309)) - release v0.18.0-rc2 ([libp2p/go-libp2p#1306](https://github.com/libp2p/go-libp2p/pull/1306)) - add semaphore to control push/delta concurrency ([libp2p/go-libp2p#1305](https://github.com/libp2p/go-libp2p/pull/1305)) - release v0.18.0-rc1 ([libp2p/go-libp2p#1300](https://github.com/libp2p/go-libp2p/pull/1300)) - default connection manager ([libp2p/go-libp2p#1299](https://github.com/libp2p/go-libp2p/pull/1299)) - Basic resource manager integration tests ([libp2p/go-libp2p#1296](https://github.com/libp2p/go-libp2p/pull/1296)) - use the resource manager ([libp2p/go-libp2p#1275](https://github.com/libp2p/go-libp2p/pull/1275)) - move the go-libp2p-connmgr here ([libp2p/go-libp2p#1297](https://github.com/libp2p/go-libp2p/pull/1297)) - move go-libp2p-discovery here ([libp2p/go-libp2p#1291](https://github.com/libp2p/go-libp2p/pull/1291)) - speed up identify tests ([libp2p/go-libp2p#1294](https://github.com/libp2p/go-libp2p/pull/1294)) - don't close the connection when opening the identify stream fails ([libp2p/go-libp2p#1293](https://github.com/libp2p/go-libp2p/pull/1293)) - use the netutil package that was moved to go-libp2p-testing (#1263) ([libp2p/go-libp2p#1263](https://github.com/libp2p/go-libp2p/pull/1263)) - speed up the autorelay test, fix flaky TestAutoRelay test ([libp2p/go-libp2p#1272](https://github.com/libp2p/go-libp2p/pull/1272)) - fix flaky TestStreamsStress test (#1288) ([libp2p/go-libp2p#1288](https://github.com/libp2p/go-libp2p/pull/1288)) - add an option for the swarm dial timeout ([libp2p/go-libp2p#1271](https://github.com/libp2p/go-libp2p/pull/1271)) - use the transport.Upgrader interface ([libp2p/go-libp2p#1277](https://github.com/libp2p/go-libp2p/pull/1277)) - fix typo in options.go (#1274) ([libp2p/go-libp2p#1274](https://github.com/libp2p/go-libp2p/pull/1274)) - remove direct dependency on libp2p/go-addr-util ([libp2p/go-libp2p#1279](https://github.com/libp2p/go-libp2p/pull/1279)) - fix flaky TestNotifications test ([libp2p/go-libp2p#1278](https://github.com/libp2p/go-libp2p/pull/1278)) - move go-libp2p-autonat to p2p/host/autonat ([libp2p/go-libp2p#1273](https://github.com/libp2p/go-libp2p/pull/1273)) - require the expiration field of the circuit v2 Reservation protobuf ([libp2p/go-libp2p#1269](https://github.com/libp2p/go-libp2p/pull/1269)) - run reconnect test using QUIC ([libp2p/go-libp2p#1268](https://github.com/libp2p/go-libp2p/pull/1268)) - remove goprocess from the mock package ([libp2p/go-libp2p#1266](https://github.com/libp2p/go-libp2p/pull/1266)) - release v0.17.0 (#1265) ([libp2p/go-libp2p#1265](https://github.com/libp2p/go-libp2p/pull/1265)) - use the new network.ConnStats ([libp2p/go-libp2p#1262](https://github.com/libp2p/go-libp2p/pull/1262)) - move the peerstoremanager to the host ([libp2p/go-libp2p#1258](https://github.com/libp2p/go-libp2p/pull/1258)) - reduce the default stream protocol negotiation timeout (#1254) ([libp2p/go-libp2p#1254](https://github.com/libp2p/go-libp2p/pull/1254)) - Doc: QUIC is default when no transports set (#1250) ([libp2p/go-libp2p#1250](https://github.com/libp2p/go-libp2p/pull/1250)) - exclude web3-bot from mkreleaselog ([libp2p/go-libp2p#1248](https://github.com/libp2p/go-libp2p/pull/1248)) - identify: also match observed against listening addresses ([libp2p/go-libp2p#1255](https://github.com/libp2p/go-libp2p/pull/1255)) - make it possible to run the auto relays tests multiple times ([libp2p/go-libp2p#1253](https://github.com/libp2p/go-libp2p/pull/1253)) - github.com/libp2p/go-libp2p-asn-util (v0.1.0 -> v0.2.0): - Release 0.2.0 (#21) ([libp2p/go-libp2p-asn-util#21](https://github.com/libp2p/go-libp2p-asn-util/pull/21)) - perf: replace the ipv6 map by an array of struct (#20) ([libp2p/go-libp2p-asn-util#20](https://github.com/libp2p/go-libp2p-asn-util/pull/20)) - sync: update CI config files (#18) ([libp2p/go-libp2p-asn-util#18](https://github.com/libp2p/go-libp2p-asn-util/pull/18)) - github.com/libp2p/go-libp2p-blankhost (v0.2.0 -> v0.3.0): - add a WithEventBus constructor option ([libp2p/go-libp2p-blankhost#61](https://github.com/libp2p/go-libp2p-blankhost/pull/61)) - emit the EvtPeerConnectednessChanged event ([libp2p/go-libp2p-blankhost#58](https://github.com/libp2p/go-libp2p-blankhost/pull/58)) - chore: update go-log to v2 ([libp2p/go-libp2p-blankhost#59](https://github.com/libp2p/go-libp2p-blankhost/pull/59)) - Remove invalid links ([libp2p/go-libp2p-blankhost#57](https://github.com/libp2p/go-libp2p-blankhost/pull/57)) - fix go vet ([libp2p/go-libp2p-blankhost#53](https://github.com/libp2p/go-libp2p-blankhost/pull/53)) - github.com/libp2p/go-libp2p-circuit (v0.4.0 -> v0.6.0): - release v0.6.0 (#151) ([libp2p/go-libp2p-circuit#151](https://github.com/libp2p/go-libp2p-circuit/pull/151)) - chore: update go-log to v2 (#147) ([libp2p/go-libp2p-circuit#147](https://github.com/libp2p/go-libp2p-circuit/pull/147)) - release v0.5.0 (#150) ([libp2p/go-libp2p-circuit#150](https://github.com/libp2p/go-libp2p-circuit/pull/150)) - use the resource manager ([libp2p/go-libp2p-circuit#148](https://github.com/libp2p/go-libp2p-circuit/pull/148)) - use the transport.Upgrader interface ([libp2p/go-libp2p-circuit#149](https://github.com/libp2p/go-libp2p-circuit/pull/149)) - sync: update CI config files (#144) ([libp2p/go-libp2p-circuit#144](https://github.com/libp2p/go-libp2p-circuit/pull/144)) - ([libp2p/go-libp2p-circuit#143](https://github.com/libp2p/go-libp2p-circuit/pull/143)) - add a Close method, remove the context from the constructor ([libp2p/go-libp2p-circuit#141](https://github.com/libp2p/go-libp2p-circuit/pull/141)) - chore: update go-libp2p-core, go-libp2p-swarm ([libp2p/go-libp2p-circuit#140](https://github.com/libp2p/go-libp2p-circuit/pull/140)) - remove the circuit v2 code ([libp2p/go-libp2p-circuit#139](https://github.com/libp2p/go-libp2p-circuit/pull/139)) - implement circuit v2 ([libp2p/go-libp2p-circuit#136](https://github.com/libp2p/go-libp2p-circuit/pull/136)) - remove deprecated types ([libp2p/go-libp2p-circuit#135](https://github.com/libp2p/go-libp2p-circuit/pull/135)) - fix race condition in TestActiveRelay ([libp2p/go-libp2p-circuit#133](https://github.com/libp2p/go-libp2p-circuit/pull/133)) - minor staticcheck fixes ([libp2p/go-libp2p-circuit#126](https://github.com/libp2p/go-libp2p-circuit/pull/126)) - Timeout Stream Read ([libp2p/go-libp2p-circuit#124](https://github.com/libp2p/go-libp2p-circuit/pull/124)) - github.com/libp2p/go-libp2p-core (v0.11.0 -> v0.15.1): - release v0.15.1 (#246) ([libp2p/go-libp2p-core#246](https://github.com/libp2p/go-libp2p-core/pull/246)) - feat: harden encoding/decoding functions against panics (#243) ([libp2p/go-libp2p-core#243](https://github.com/libp2p/go-libp2p-core/pull/243)) - release v0.15.0 (#242) ([libp2p/go-libp2p-core#242](https://github.com/libp2p/go-libp2p-core/pull/242)) - sync: update CI config files (#241) ([libp2p/go-libp2p-core#241](https://github.com/libp2p/go-libp2p-core/pull/241)) - fix: switch to go-multicodec mappings (#240) ([libp2p/go-libp2p-core#240](https://github.com/libp2p/go-libp2p-core/pull/240)) - chore: add `String()` method to `IDSlice` type (#238) ([libp2p/go-libp2p-core#238](https://github.com/libp2p/go-libp2p-core/pull/238)) - release v0.14.0 (#235) ([libp2p/go-libp2p-core#235](https://github.com/libp2p/go-libp2p-core/pull/235)) - Network Resource Manager interface (#229) ([libp2p/go-libp2p-core#229](https://github.com/libp2p/go-libp2p-core/pull/229)) - introduce a transport.Upgrader interface (#232) ([libp2p/go-libp2p-core#232](https://github.com/libp2p/go-libp2p-core/pull/232)) - remove the transport.AcceptTimeout (#231) ([libp2p/go-libp2p-core#231](https://github.com/libp2p/go-libp2p-core/pull/231)) - remove the DialTimeout (#230) ([libp2p/go-libp2p-core#230](https://github.com/libp2p/go-libp2p-core/pull/230)) - remove duplicate io.Closer on Network interface (#228) ([libp2p/go-libp2p-core#228](https://github.com/libp2p/go-libp2p-core/pull/228)) - release v0.13.0 (#227) ([libp2p/go-libp2p-core#227](https://github.com/libp2p/go-libp2p-core/pull/227)) - rename network.Stat to Stats, introduce ConnStats (#226) ([libp2p/go-libp2p-core#226](https://github.com/libp2p/go-libp2p-core/pull/226)) - release v0.12.0 (#223) ([libp2p/go-libp2p-core#223](https://github.com/libp2p/go-libp2p-core/pull/223)) - generate ecdsa public key from an input public key (#219) ([libp2p/go-libp2p-core#219](https://github.com/libp2p/go-libp2p-core/pull/219)) - add RemovePeer method to PeerMetadata, Metrics, ProtoBook and Keybook (#218) ([libp2p/go-libp2p-core#218](https://github.com/libp2p/go-libp2p-core/pull/218)) - github.com/libp2p/go-libp2p-kad-dht (v0.15.0 -> v0.16.0): - Version 0.16.0 (#774) ([libp2p/go-libp2p-kad-dht#774](https://github.com/libp2p/go-libp2p-kad-dht/pull/774)) - feat: add error log when resource manager throttles crawler (#772) ([libp2p/go-libp2p-kad-dht#772](https://github.com/libp2p/go-libp2p-kad-dht/pull/772)) - fix: incorrect format handling ([libp2p/go-libp2p-kad-dht#771](https://github.com/libp2p/go-libp2p-kad-dht/pull/771)) - Upgrade to go-libp2p v0.16.0 (#756) ([libp2p/go-libp2p-kad-dht#756](https://github.com/libp2p/go-libp2p-kad-dht/pull/756)) - sync: update CI config files ([libp2p/go-libp2p-kad-dht#758](https://github.com/libp2p/go-libp2p-kad-dht/pull/758)) - github.com/libp2p/go-libp2p-mplex (v0.4.1 -> v0.7.0): - release v0.7.0 (#36) ([libp2p/go-libp2p-mplex#36](https://github.com/libp2p/go-libp2p-mplex/pull/36)) - release v0.6.0 (#32) ([libp2p/go-libp2p-mplex#32](https://github.com/libp2p/go-libp2p-mplex/pull/32)) - update mplex (#31) ([libp2p/go-libp2p-mplex#31](https://github.com/libp2p/go-libp2p-mplex/pull/31)) - release v0.5.0 (#30) ([libp2p/go-libp2p-mplex#30](https://github.com/libp2p/go-libp2p-mplex/pull/30)) - implement the new network.MuxedConn interface (#29) ([libp2p/go-libp2p-mplex#29](https://github.com/libp2p/go-libp2p-mplex/pull/29)) - sync: update CI config files (#28) ([libp2p/go-libp2p-mplex#28](https://github.com/libp2p/go-libp2p-mplex/pull/28)) - remove Makefile ([libp2p/go-libp2p-mplex#25](https://github.com/libp2p/go-libp2p-mplex/pull/25)) - github.com/libp2p/go-libp2p-noise (v0.3.0 -> v0.4.0): - release v0.4.0 (#112) ([libp2p/go-libp2p-noise#112](https://github.com/libp2p/go-libp2p-noise/pull/112)) - catch panics during the handshake (#111) ([libp2p/go-libp2p-noise#111](https://github.com/libp2p/go-libp2p-noise/pull/111)) - sync: update CI config files (#106) ([libp2p/go-libp2p-noise#106](https://github.com/libp2p/go-libp2p-noise/pull/106)) - update README to reflect that Noise is enabled by default ([libp2p/go-libp2p-noise#101](https://github.com/libp2p/go-libp2p-noise/pull/101)) - github.com/libp2p/go-libp2p-peerstore (v0.4.0 -> v0.6.0): - release v0.6.0 ([libp2p/go-libp2p-peerstore#189](https://github.com/libp2p/go-libp2p-peerstore/pull/189)) - remove the pstoremanager (will be moved to the Host) ([libp2p/go-libp2p-peerstore#188](https://github.com/libp2p/go-libp2p-peerstore/pull/188)) - release v0.5.0 (#187) ([libp2p/go-libp2p-peerstore#187](https://github.com/libp2p/go-libp2p-peerstore/pull/187)) - remove metadata interning ([libp2p/go-libp2p-peerstore#185](https://github.com/libp2p/go-libp2p-peerstore/pull/185)) - when passed an event bus, automatically clean up disconnected peers ([libp2p/go-libp2p-peerstore#184](https://github.com/libp2p/go-libp2p-peerstore/pull/184)) - implement the RemovePeer method ([libp2p/go-libp2p-peerstore#174](https://github.com/libp2p/go-libp2p-peerstore/pull/174)) - chore: update go-log to v2 (#179) ([libp2p/go-libp2p-peerstore#179](https://github.com/libp2p/go-libp2p-peerstore/pull/179)) - github.com/libp2p/go-libp2p-pubsub (v0.6.0 -> v0.6.1): - add tests for clearing the peerPromises map - properly clear the peerPromises map - more info - add to MinTopicSize godoc re topic size - github.com/libp2p/go-libp2p-quic-transport (v0.15.0 -> v0.17.0): - release v0.17.0 (#269) ([libp2p/go-libp2p-quic-transport#269](https://github.com/libp2p/go-libp2p-quic-transport/pull/269)) - update quic-go to v0.27.0 (#264) ([libp2p/go-libp2p-quic-transport#264](https://github.com/libp2p/go-libp2p-quic-transport/pull/264)) - release v0.16.1 (#261) ([libp2p/go-libp2p-quic-transport#261](https://github.com/libp2p/go-libp2p-quic-transport/pull/261)) - Prevent data race in allowWindowIncrease (#259) ([libp2p/go-libp2p-quic-transport#259](https://github.com/libp2p/go-libp2p-quic-transport/pull/259)) - release v0.16.0 (#258) ([libp2p/go-libp2p-quic-transport#258](https://github.com/libp2p/go-libp2p-quic-transport/pull/258)) - use the Resource Manager ([libp2p/go-libp2p-quic-transport#249](https://github.com/libp2p/go-libp2p-quic-transport/pull/249)) - don't start a Go routine for every connection dialed ([libp2p/go-libp2p-quic-transport#252](https://github.com/libp2p/go-libp2p-quic-transport/pull/252)) - migrate to standard Go tests, stop using Ginkgo ([libp2p/go-libp2p-quic-transport#250](https://github.com/libp2p/go-libp2p-quic-transport/pull/250)) - chore: remove Codecov config (#251) ([libp2p/go-libp2p-quic-transport#251](https://github.com/libp2p/go-libp2p-quic-transport/pull/251)) - reduce the maximum number of incoming streams to 256 (#243) ([libp2p/go-libp2p-quic-transport#243](https://github.com/libp2p/go-libp2p-quic-transport/pull/243)) - chore: update go-log to v2 (#242) ([libp2p/go-libp2p-quic-transport#242](https://github.com/libp2p/go-libp2p-quic-transport/pull/242)) - github.com/libp2p/go-libp2p-swarm (v0.8.0 -> v0.10.2): - bump version to v0.10.2 ([libp2p/go-libp2p-swarm#316](https://github.com/libp2p/go-libp2p-swarm/pull/316)) - Refactor dial worker loop into an object and fix bug ([libp2p/go-libp2p-swarm#315](https://github.com/libp2p/go-libp2p-swarm/pull/315)) - release v0.10.1 ([libp2p/go-libp2p-swarm#313](https://github.com/libp2p/go-libp2p-swarm/pull/313)) - release the stream scope if the conn fails to open a new stream ([libp2p/go-libp2p-swarm#312](https://github.com/libp2p/go-libp2p-swarm/pull/312)) - release v0.10.0 (#311) ([libp2p/go-libp2p-swarm#311](https://github.com/libp2p/go-libp2p-swarm/pull/311)) - add support for the resource manager ([libp2p/go-libp2p-swarm#308](https://github.com/libp2p/go-libp2p-swarm/pull/308)) - use the transport.Upgrader interface ([libp2p/go-libp2p-swarm#309](https://github.com/libp2p/go-libp2p-swarm/pull/309)) - remove dependency on go-addr-util ([libp2p/go-libp2p-swarm#300](https://github.com/libp2p/go-libp2p-swarm/pull/300)) - stop using transport.DialTimeout in tests (#307) ([libp2p/go-libp2p-swarm#307](https://github.com/libp2p/go-libp2p-swarm/pull/307)) - increment active dial counter in dial worker loop ([libp2p/go-libp2p-swarm#305](https://github.com/libp2p/go-libp2p-swarm/pull/305)) - speed up the dial tests ([libp2p/go-libp2p-swarm#301](https://github.com/libp2p/go-libp2p-swarm/pull/301)) - stop using the deprecated libp2p/go-maddr-filter ([libp2p/go-libp2p-swarm#303](https://github.com/libp2p/go-libp2p-swarm/pull/303)) - add constructor options for timeout, stop using transport.DialTimeout ([libp2p/go-libp2p-swarm#302](https://github.com/libp2p/go-libp2p-swarm/pull/302)) - release v0.9.0 (#299) ([libp2p/go-libp2p-swarm#299](https://github.com/libp2p/go-libp2p-swarm/pull/299)) - count the number of streams on a connection for the stats ([libp2p/go-libp2p-swarm#298](https://github.com/libp2p/go-libp2p-swarm/pull/298)) - chore: update go-log to v2 (#294) ([libp2p/go-libp2p-swarm#294](https://github.com/libp2p/go-libp2p-swarm/pull/294)) - github.com/libp2p/go-libp2p-testing (v0.5.0 -> v0.9.2): - release v0.9.2 (#56) ([libp2p/go-libp2p-testing#56](https://github.com/libp2p/go-libp2p-testing/pull/56)) - fix memory allocation check in SubtestStreamReset (#55) ([libp2p/go-libp2p-testing#55](https://github.com/libp2p/go-libp2p-testing/pull/55)) - release v0.9.1 (#54) ([libp2p/go-libp2p-testing#54](https://github.com/libp2p/go-libp2p-testing/pull/54)) - remove stray debug statements for memory allocations - release v0.9.0 (#53) ([libp2p/go-libp2p-testing#53](https://github.com/libp2p/go-libp2p-testing/pull/53)) - add tests for memory management ([libp2p/go-libp2p-testing#52](https://github.com/libp2p/go-libp2p-testing/pull/52)) - release v0.8.0 (#50) ([libp2p/go-libp2p-testing#50](https://github.com/libp2p/go-libp2p-testing/pull/50)) - use io.ReadFull in muxer test, use require.Equal to compare buffers (#49) ([libp2p/go-libp2p-testing#49](https://github.com/libp2p/go-libp2p-testing/pull/49)) - release v0.7.0 (#47) ([libp2p/go-libp2p-testing#47](https://github.com/libp2p/go-libp2p-testing/pull/47)) - add mocks for the resource manager ([libp2p/go-libp2p-testing#46](https://github.com/libp2p/go-libp2p-testing/pull/46)) - merge libp2p/go-libp2p-netutil into this repo ([libp2p/go-libp2p-testing#45](https://github.com/libp2p/go-libp2p-testing/pull/45)) - reduce the number of connections in the stream muxer stress test (#44) ([libp2p/go-libp2p-testing#44](https://github.com/libp2p/go-libp2p-testing/pull/44)) - release v0.6.0 (#42) ([libp2p/go-libp2p-testing#42](https://github.com/libp2p/go-libp2p-testing/pull/42)) - expose a map, not a slice, of muxer tests (#41) ([libp2p/go-libp2p-testing#41](https://github.com/libp2p/go-libp2p-testing/pull/41)) - sync: update CI config files (#40) ([libp2p/go-libp2p-testing#40](https://github.com/libp2p/go-libp2p-testing/pull/40)) - github.com/libp2p/go-libp2p-tls (v0.3.1 -> v0.4.1): - release v0.4.1 (#112) ([libp2p/go-libp2p-tls#112](https://github.com/libp2p/go-libp2p-tls/pull/112)) - feat: catch panics in TLS negotiation ([libp2p/go-libp2p-tls#111](https://github.com/libp2p/go-libp2p-tls/pull/111)) - release v0.4.0 (#110) ([libp2p/go-libp2p-tls#110](https://github.com/libp2p/go-libp2p-tls/pull/110)) - use tls.Conn.HandshakeContext instead of tls.Conn.Handshake (#106) ([libp2p/go-libp2p-tls#106](https://github.com/libp2p/go-libp2p-tls/pull/106)) - remove paragraph about Go modules from README (#104) ([libp2p/go-libp2p-tls#104](https://github.com/libp2p/go-libp2p-tls/pull/104)) - migrate to standard Go tests, stop using Ginkgo ([libp2p/go-libp2p-tls#105](https://github.com/libp2p/go-libp2p-tls/pull/105)) - chore: remove Codecov config (#103) ([libp2p/go-libp2p-tls#103](https://github.com/libp2p/go-libp2p-tls/pull/103)) - github.com/libp2p/go-libp2p-transport-upgrader (v0.5.0 -> v0.7.1): - release v0.7.1 ([libp2p/go-libp2p-transport-upgrader#105](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/105)) - Fix nil peer scope issues ([libp2p/go-libp2p-transport-upgrader#104](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/104)) - release v0.7.0 (#103) ([libp2p/go-libp2p-transport-upgrader#103](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/103)) - use the Resource Manager ([libp2p/go-libp2p-transport-upgrader#99](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/99)) - rename the package to upgrader ([libp2p/go-libp2p-transport-upgrader#101](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/101)) - use the new transport.Upgrader interface ([libp2p/go-libp2p-transport-upgrader#100](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/100)) - reset the temporary error catcher delay after successful accept ([libp2p/go-libp2p-transport-upgrader#97](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/97)) - make the accept timeout configurable, stop using transport.AcceptTimeout ([libp2p/go-libp2p-transport-upgrader#98](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/98)) - release v0.6.0 (#95) ([libp2p/go-libp2p-transport-upgrader#95](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/95)) - remove note about go.mod and Go 1.11 in README (#94) ([libp2p/go-libp2p-transport-upgrader#94](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/94)) - fix flaky TestAcceptQueueBacklogged test ([libp2p/go-libp2p-transport-upgrader#96](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/96)) - chore: remove Codecov config (#93) ([libp2p/go-libp2p-transport-upgrader#93](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/93)) - use the new network.ConnStats ([libp2p/go-libp2p-transport-upgrader#92](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/92)) - sync: update CI config files (#89) ([libp2p/go-libp2p-transport-upgrader#89](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/89)) - chore: update go-log ([libp2p/go-libp2p-transport-upgrader#88](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/88)) - github.com/libp2p/go-libp2p-yamux (v0.6.0 -> v0.9.1): - release v0.9.1 (#55) ([libp2p/go-libp2p-yamux#55](https://github.com/libp2p/go-libp2p-yamux/pull/55)) - release v0.9.0 (#53) ([libp2p/go-libp2p-yamux#53](https://github.com/libp2p/go-libp2p-yamux/pull/53)) - release v0.8.2 (#50) ([libp2p/go-libp2p-yamux#50](https://github.com/libp2p/go-libp2p-yamux/pull/50)) - disable the incoming streams limit (#49) ([libp2p/go-libp2p-yamux#49](https://github.com/libp2p/go-libp2p-yamux/pull/49)) - Release v0.8.1 ([libp2p/go-libp2p-yamux#48](https://github.com/libp2p/go-libp2p-yamux/pull/48)) - release v0.8.0 (#47) ([libp2p/go-libp2p-yamux#47](https://github.com/libp2p/go-libp2p-yamux/pull/47)) - pass the PeerScope to yamux ( satisfying its MemoryManger interface) (#46) ([libp2p/go-libp2p-yamux#46](https://github.com/libp2p/go-libp2p-yamux/pull/46)) - release v0.7.0 (#43) ([libp2p/go-libp2p-yamux#43](https://github.com/libp2p/go-libp2p-yamux/pull/43)) - sync: update CI config files (#42) ([libp2p/go-libp2p-yamux#42](https://github.com/libp2p/go-libp2p-yamux/pull/42)) - reduce the number of max incoming stream to 256 ([libp2p/go-libp2p-yamux#41](https://github.com/libp2p/go-libp2p-yamux/pull/41)) - github.com/libp2p/go-mplex (v0.3.0 -> v0.7.0): - release v0.7.0 (#112) ([libp2p/go-mplex#112](https://github.com/libp2p/go-mplex/pull/112)) - catch panics in handleIncoming and handleOutgoing (#109) ([libp2p/go-mplex#109](https://github.com/libp2p/go-mplex/pull/109)) - remove benchmark tests (#111) ([libp2p/go-mplex#111](https://github.com/libp2p/go-mplex/pull/111)) - release v0.6.0 (#105) ([libp2p/go-mplex#105](https://github.com/libp2p/go-mplex/pull/105)) - fix incorrect reset of timer fired variable (#104) ([libp2p/go-mplex#104](https://github.com/libp2p/go-mplex/pull/104)) - Mplex salvage operations, part II (#102) ([libp2p/go-mplex#102](https://github.com/libp2p/go-mplex/pull/102)) - release v0.5.0 (#100) ([libp2p/go-mplex#100](https://github.com/libp2p/go-mplex/pull/100)) - Salvage mplex in the age of resource management (#99) ([libp2p/go-mplex#99](https://github.com/libp2p/go-mplex/pull/99)) - release v0.4.0 (#97) ([libp2p/go-mplex#97](https://github.com/libp2p/go-mplex/pull/97)) - add a MemoryManager interface to control memory allocations ([libp2p/go-mplex#96](https://github.com/libp2p/go-mplex/pull/96)) - sync: update CI config files (#93) ([libp2p/go-mplex#93](https://github.com/libp2p/go-mplex/pull/93)) - chore: update go-log to v2 ([libp2p/go-mplex#92](https://github.com/libp2p/go-mplex/pull/92)) - chore: remove Codecov config ([libp2p/go-mplex#91](https://github.com/libp2p/go-mplex/pull/91)) - sync: update CI config files (#90) ([libp2p/go-mplex#90](https://github.com/libp2p/go-mplex/pull/90)) - multiplex: add (*Multiplex).CloseChan ([libp2p/go-mplex#89](https://github.com/libp2p/go-mplex/pull/89)) - add a Go Reference badge to the README ([libp2p/go-mplex#88](https://github.com/libp2p/go-mplex/pull/88)) - sync: update CI config files ([libp2p/go-mplex#85](https://github.com/libp2p/go-mplex/pull/85)) - Fix up tests & vet ([libp2p/go-mplex#84](https://github.com/libp2p/go-mplex/pull/84)) - Bump lodash from 4.17.19 to 4.17.21 in /interop/js ([libp2p/go-mplex#83](https://github.com/libp2p/go-mplex/pull/83)) - github.com/libp2p/go-msgio (v0.1.0 -> v0.2.0): - release v0.2.0 (#34) ([libp2p/go-msgio#34](https://github.com/libp2p/go-msgio/pull/34)) - print recovered panics to stderr (#33) ([libp2p/go-msgio#33](https://github.com/libp2p/go-msgio/pull/33)) - catch panics when reading / writing protobuf messages (#31) ([libp2p/go-msgio#31](https://github.com/libp2p/go-msgio/pull/31)) - remove outdated section about channels from README (#32) ([libp2p/go-msgio#32](https://github.com/libp2p/go-msgio/pull/32)) - sync: update CI config files (#28) ([libp2p/go-msgio#28](https://github.com/libp2p/go-msgio/pull/28)) - github.com/libp2p/go-netroute (v0.1.6 -> v0.2.0): - release v0.2.0 (#21) ([libp2p/go-netroute#21](https://github.com/libp2p/go-netroute/pull/21)) - move some functions from go-sockaddr here, remove go-sockaddr dependency ([libp2p/go-netroute#22](https://github.com/libp2p/go-netroute/pull/22)) - ignore the error on the RouteMessage on Darwin ([libp2p/go-netroute#20](https://github.com/libp2p/go-netroute/pull/20)) - sync: update CI config files (#19) ([libp2p/go-netroute#19](https://github.com/libp2p/go-netroute/pull/19)) - sync: update CI config files (#18) ([libp2p/go-netroute#18](https://github.com/libp2p/go-netroute/pull/18)) - skip loopback addr as indication of v6 routes ([libp2p/go-netroute#17](https://github.com/libp2p/go-netroute/pull/17)) - fix staticcheck lint issues ([libp2p/go-netroute#15](https://github.com/libp2p/go-netroute/pull/15)) - github.com/libp2p/go-stream-muxer-multistream (v0.3.0 -> v0.4.0): - release v0.4.0 (#23) ([libp2p/go-stream-muxer-multistream#23](https://github.com/libp2p/go-stream-muxer-multistream/pull/23)) - implement the new Multiplexer.NewConn interface ([libp2p/go-stream-muxer-multistream#22](https://github.com/libp2p/go-stream-muxer-multistream/pull/22)) - sync: update CI config files (#19) ([libp2p/go-stream-muxer-multistream#19](https://github.com/libp2p/go-stream-muxer-multistream/pull/19)) - github.com/libp2p/go-tcp-transport (v0.4.0 -> v0.5.1): - release v0.5.1 (#116) ([libp2p/go-tcp-transport#116](https://github.com/libp2p/go-tcp-transport/pull/116)) - fix: drop raw EINVAL (from keepalives) errors as well (#115) ([libp2p/go-tcp-transport#115](https://github.com/libp2p/go-tcp-transport/pull/115)) - release v0.5.0 (#114) ([libp2p/go-tcp-transport#114](https://github.com/libp2p/go-tcp-transport/pull/114)) - use the ResourceManager ([libp2p/go-tcp-transport#110](https://github.com/libp2p/go-tcp-transport/pull/110)) - use the transport.Upgrader interface ([libp2p/go-tcp-transport#111](https://github.com/libp2p/go-tcp-transport/pull/111)) - describe how to use options in README ([libp2p/go-tcp-transport#105](https://github.com/libp2p/go-tcp-transport/pull/105)) - github.com/libp2p/go-ws-transport (v0.5.0 -> v0.6.0): - release v0.6.0 (#113) ([libp2p/go-ws-transport#113](https://github.com/libp2p/go-ws-transport/pull/113)) - use the resource manager ([libp2p/go-ws-transport#109](https://github.com/libp2p/go-ws-transport/pull/109)) - chore: remove Codecov config (#112) ([libp2p/go-ws-transport#112](https://github.com/libp2p/go-ws-transport/pull/112)) - remove contexts from libp2p constructors in README (#111) ([libp2p/go-ws-transport#111](https://github.com/libp2p/go-ws-transport/pull/111)) - use the transport.Upgrader interface ([libp2p/go-ws-transport#110](https://github.com/libp2p/go-ws-transport/pull/110)) - sync: update CI config files (#108) ([libp2p/go-ws-transport#108](https://github.com/libp2p/go-ws-transport/pull/108)) - sync: update CI config files (#106) ([libp2p/go-ws-transport#106](https://github.com/libp2p/go-ws-transport/pull/106)) - github.com/lucas-clemente/quic-go (v0.24.0 -> v0.27.1): - don't send path MTU probe packets on a timer - stop using the deprecated net.Error.Temporary, update golangci-lint to v1.45.2 ([lucas-clemente/quic-go#3367](https://github.com/lucas-clemente/quic-go/pull/3367)) - add support for serializing Extended CONNECT requests (#3360) ([lucas-clemente/quic-go#3360](https://github.com/lucas-clemente/quic-go/pull/3360)) - improve the error thrown when building with an unsupported Go version ([lucas-clemente/quic-go#3364](https://github.com/lucas-clemente/quic-go/pull/3364)) - remove nextdns from list of projects using quic-go (#3363) ([lucas-clemente/quic-go#3363](https://github.com/lucas-clemente/quic-go/pull/3363)) - rename the Session to Connection ([lucas-clemente/quic-go#3361](https://github.com/lucas-clemente/quic-go/pull/3361)) - respect the request context when dialing ([lucas-clemente/quic-go#3359](https://github.com/lucas-clemente/quic-go/pull/3359)) - update HTTP/3 Datagram to draft-ietf-masque-h3-datagram-07 (#3355) ([lucas-clemente/quic-go#3355](https://github.com/lucas-clemente/quic-go/pull/3355)) - add support for the Extended CONNECT method (#3357) ([lucas-clemente/quic-go#3357](https://github.com/lucas-clemente/quic-go/pull/3357)) - remove the SkipSchemeCheck RoundTripOpt (#3353) ([lucas-clemente/quic-go#3353](https://github.com/lucas-clemente/quic-go/pull/3353)) - remove parser logic for HTTP/3 DUPLICATE_PUSH frame (#3356) ([lucas-clemente/quic-go#3356](https://github.com/lucas-clemente/quic-go/pull/3356)) - improve code coverage of random number generator test (#3358) ([lucas-clemente/quic-go#3358](https://github.com/lucas-clemente/quic-go/pull/3358)) - advertise multiple listeners via Alt-Svc and improve perf of SetQuicHeaders (#3352) ([lucas-clemente/quic-go#3352](https://github.com/lucas-clemente/quic-go/pull/3352)) - avoid recursion when skipping unknown HTTP/3 frames (#3354) ([lucas-clemente/quic-go#3354](https://github.com/lucas-clemente/quic-go/pull/3354)) - Implement http3.Server.ServeListener (#3349) ([lucas-clemente/quic-go#3349](https://github.com/lucas-clemente/quic-go/pull/3349)) - update for Go 1.18 ([lucas-clemente/quic-go#3345](https://github.com/lucas-clemente/quic-go/pull/3345)) - don't print a receive buffer warning for closed connections (#3346) ([lucas-clemente/quic-go#3346](https://github.com/lucas-clemente/quic-go/pull/3346)) - move set DF implementation to separate files & avoid the need for OOBCapablePacketConn (#3334) ([lucas-clemente/quic-go#3334](https://github.com/lucas-clemente/quic-go/pull/3334)) - add env to disable the receive buffer warning (#3339) ([lucas-clemente/quic-go#3339](https://github.com/lucas-clemente/quic-go/pull/3339)) - fix typo (#3333) ([lucas-clemente/quic-go#3333](https://github.com/lucas-clemente/quic-go/pull/3333)) - sendQueue: ignore "datagram too large" error (#3328) ([lucas-clemente/quic-go#3328](https://github.com/lucas-clemente/quic-go/pull/3328)) - add OONI Probe to list of projects in README (#3324) ([lucas-clemente/quic-go#3324](https://github.com/lucas-clemente/quic-go/pull/3324)) - remove build status badges from README (#3325) ([lucas-clemente/quic-go#3325](https://github.com/lucas-clemente/quic-go/pull/3325)) - add a AllowConnectionWindowIncrease config option ([lucas-clemente/quic-go#3317](https://github.com/lucas-clemente/quic-go/pull/3317)) - Update README.md (#3315) ([lucas-clemente/quic-go#3315](https://github.com/lucas-clemente/quic-go/pull/3315)) - fix some typos in documentation and tests - remove unneeded calls to goimports when generating mocks ([lucas-clemente/quic-go#3313](https://github.com/lucas-clemente/quic-go/pull/3313)) - fix comment about congestionWindow value (#3310) ([lucas-clemente/quic-go#3310](https://github.com/lucas-clemente/quic-go/pull/3310)) - fix typo *connections (#3309) ([lucas-clemente/quic-go#3309](https://github.com/lucas-clemente/quic-go/pull/3309)) - add support for Go 1.18 ([lucas-clemente/quic-go#3298](https://github.com/lucas-clemente/quic-go/pull/3298)) - github.com/multiformats/go-base32 (v0.0.3 -> v0.0.4): - optimize encode ([multiformats/go-base32#1](https://github.com/multiformats/go-base32/pull/1)) - Fix `staticcheck` issue - github.com/multiformats/go-multiaddr (v0.4.1 -> v0.5.0): - release v0.5.0 (#171) ([multiformats/go-multiaddr#171](https://github.com/multiformats/go-multiaddr/pull/171)) - remove wrong (and redundant) IsIpv6LinkLocal ([multiformats/go-multiaddr#170](https://github.com/multiformats/go-multiaddr/pull/170)) - move ResolveUnspecifiedAddress(es) and FilterAddrs here from libp2p/go-addr-util ([multiformats/go-multiaddr#168](https://github.com/multiformats/go-multiaddr/pull/168)) - sync: update CI config files (#167) ([multiformats/go-multiaddr#167](https://github.com/multiformats/go-multiaddr/pull/167)) - github.com/multiformats/go-multicodec (v0.3.0 -> v0.4.1): - Version v0.4.1 ([multiformats/go-multicodec#64](https://github.com/multiformats/go-multicodec/pull/64)) - update table with new codecs ([multiformats/go-multicodec#63](https://github.com/multiformats/go-multicodec/pull/63)) - bump version to v0.4.0 - sync: update CI config files (#60) ([multiformats/go-multicodec#60](https://github.com/multiformats/go-multicodec/pull/60)) - add Code.Tag method - add the KnownCodes API - use "go run pkg@version" assuming Go 1.17 or later - update submodule and re-generate - update to newer multicodec table ([multiformats/go-multicodec#57](https://github.com/multiformats/go-multicodec/pull/57)) - Update `multicodec` submodule to `1bcdc08` for CARv2 index codec - github.com/multiformats/go-multistream (v0.2.2 -> v0.3.0): - release v0.3.0 (#82) ([multiformats/go-multistream#82](https://github.com/multiformats/go-multistream/pull/82)) - catch panics (#81) ([multiformats/go-multistream#81](https://github.com/multiformats/go-multistream/pull/81)) - sync: update CI config files (#78) ([multiformats/go-multistream#78](https://github.com/multiformats/go-multistream/pull/78)) - reduce the maximum read buffer size from 64 to 1 kB ([multiformats/go-multistream#77](https://github.com/multiformats/go-multistream/pull/77)) - remove unused ls command ([multiformats/go-multistream#76](https://github.com/multiformats/go-multistream/pull/76)) - chore: remove empty file cases.md ([multiformats/go-multistream#75](https://github.com/multiformats/go-multistream/pull/75)) - chore: remove .gx ([multiformats/go-multistream#72](https://github.com/multiformats/go-multistream/pull/72)) - don't commit the fuzzing binary ([multiformats/go-multistream#74](https://github.com/multiformats/go-multistream/pull/74)) - sync: update CI config files (#71) ([multiformats/go-multistream#71](https://github.com/multiformats/go-multistream/pull/71)) - remove Makefile ([multiformats/go-multistream#67](https://github.com/multiformats/go-multistream/pull/67))
### ❤ Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Marten Seemann | 347 | +14453/-12552 | 842 | | Rod Vagg | 28 | +8848/-4033 | 214 | | vyzo | 133 | +7959/-1783 | 231 | | hannahhoward | 40 | +3761/-1652 | 175 | | Will Scott | 39 | +2885/-1784 | 93 | | Daniel Martí | 32 | +3043/-969 | 103 | | Adin Schmahmann | 48 | +3439/-536 | 121 | | Gus Eggert | 29 | +2644/-788 | 123 | | Steven Allen | 87 | +2417/-840 | 135 | | Marcin Rataj | 29 | +2312/-942 | 75 | | Will | 11 | +2520/-62 | 56 | | Lucas Molas | 28 | +1602/-578 | 90 | | Raúl Kripalani | 18 | +1519/-271 | 38 | | Brian Tiger Chow | 20 | +833/-379 | 40 | | Masih H. Derkani | 5 | +514/-460 | 8 | | Jeromy Johnson | 53 | +646/-302 | 83 | | Łukasz Magiera | 26 | +592/-245 | 43 | | Artem Mikheev | 2 | +616/-120 | 5 | | Franky W | 2 | +49/-525 | 9 | | Laurent Senta | 3 | +468/-82 | 52 | | Hector Sanjuan | 32 | +253/-187 | 62 | | Juan Batiz-Benet | 8 | +285/-80 | 18 | | Justin Johnson | 2 | +181/-88 | 2 | | Thibault Meunier | 5 | +216/-28 | 8 | | James Wetter | 2 | +234/-1 | 2 | | web3-bot | 36 | +158/-66 | 62 | | gammazero | 7 | +140/-84 | 12 | | Rachel Chen | 2 | +165/-57 | 17 | | Jorropo | 18 | +108/-99 | 26 | | Toby | 2 | +107/-86 | 11 | | Antonio Navarro Perez | 4 | +82/-103 | 9 | | Dominic Della Valle | 4 | +148/-33 | 6 | | Ian Davis | 2 | +152/-28 | 6 | | Kyle Huntsman | 2 | +172/-6 | 5 | | huoju | 4 | +127/-41 | 6 | | Jeromy | 19 | +71/-58 | 31 | | Lars Gierth | 12 | +63/-54 | 20 | | Eric Myhre | 3 | +95/-15 | 8 | | Caian Benedicto | 1 | +69/-12 | 6 | | Raúl Kripalani | 2 | +63/-13 | 2 | | Anton Petrov | 1 | +73/-0 | 1 | | hunjixin | 2 | +67/-2 | 5 | | odanado | 1 | +61/-0 | 1 | | Andrew Gillis | 2 | +61/-0 | 3 | | Kevin Atkinson | 6 | +21/-34 | 7 | | Richard Ramos | 1 | +51/-0 | 2 | | Manuel Alonso | 1 | +42/-9 | 2 | | Jakub Sztandera | 10 | +37/-13 | 13 | | Aarsh Shah | 1 | +39/-5 | 2 | | Dave Justice | 1 | +32/-4 | 2 | | Tommi Virtanen | 3 | +23/-9 | 4 | | tarekbadr | 1 | +30/-1 | 1 | | whyrusleeping | 1 | +26/-4 | 3 | | Petar Maymounkov | 2 | +30/-0 | 4 | | rht | 3 | +17/-10 | 4 | | Miguel Mota | 1 | +23/-0 | 1 | | Manfred Touron | 1 | +21/-2 | 2 | | watjurk | 1 | +17/-5 | 1 | | SukkaW | 1 | +11/-11 | 5 | | Nicholas Bollweg | 1 | +21/-0 | 1 | | Ho-Sheng Hsiao | 2 | +11/-10 | 6 | | chblodg | 1 | +18/-2 | 1 | | Friedel Ziegelmayer | 2 | +18/-0 | 2 | | Shu Shen | 2 | +15/-2 | 3 | | Peter Rabbitson | 1 | +15/-2 | 1 | | galargh | 2 | +15/-0 | 2 | | ᴍᴀᴛᴛ ʙᴇʟʟ | 3 | +13/-1 | 4 | | aarshkshah1992 | 3 | +12/-2 | 3 | | RubenKelevra | 4 | +5/-8 | 5 | | Feiran Yang | 1 | +11/-0 | 2 | | zramsay | 2 | +0/-10 | 2 | | Teran McKinney | 1 | +8/-2 | 1 | | Richard Littauer | 2 | +5/-5 | 5 | | Elijah | 1 | +10/-0 | 1 | | Dimitris Apostolou | 2 | +5/-5 | 5 | | Michael Avila | 3 | +8/-1 | 4 | | siiky | 3 | +4/-4 | 3 | | Somajit | 1 | +4/-4 | 1 | | Sherod Taylor | 1 | +0/-8 | 2 | | Eclésio Junior | 1 | +8/-0 | 1 | | godcong | 3 | +4/-3 | 3 | | Piotr Galar | 3 | +3/-4 | 3 | | jwh | 1 | +6/-0 | 2 | | dependabot[bot] | 1 | +3/-3 | 1 | | Volker Mische | 1 | +4/-2 | 1 | | Aayush Rajasekaran | 1 | +3/-3 | 1 | | rene | 2 | +3/-2 | 2 | | keks | 1 | +5/-0 | 1 | | Hlib | 1 | +4/-1 | 2 | | Arash Payan | 1 | +5/-0 | 1 | | Wayback Archiver | 1 | +2/-2 | 1 | | T Mo | 1 | +2/-2 | 1 | | Ju Huo | 1 | +2/-2 | 1 | | Ivan | 2 | +2/-2 | 2 | | Ettore Di Giacinto | 2 | +3/-1 | 2 | | Christian Couder | 1 | +3/-1 | 1 | | ningmingxiao | 1 | +0/-3 | 1 | | 市川恭佑 (ebi) | 1 | +1/-1 | 1 | | star | 1 | +0/-2 | 1 | | alliswell | 1 | +0/-2 | 1 | | Preston Van Loon | 1 | +2/-0 | 1 | | Nguyễn Gia Phong | 1 | +1/-1 | 1 | | Nato Boram | 1 | +1/-1 | 1 | | Mildred Ki'Lya | 1 | +2/-0 | 2 | | Michael Burns | 1 | +1/-1 | 1 | | Glenn | 1 | +1/-1 | 1 | | George Antoniadis | 1 | +1/-1 | 1 | | David Florness | 1 | +1/-1 | 1 | | Daniel Norman | 1 | +1/-1 | 1 | | Coenie Beyers | 1 | +1/-1 | 1 | | Benedikt Spies | 1 | +1/-1 | 1 | | Abdul Rauf | 1 | +1/-1 | 1 | | makeworld | 1 | +1/-0 | 1 | | ignoramous | 1 | +0/-1 | 1 | | Omicron166 | 1 | +0/-1 | 1 | | Jan Winkelmann | 1 | +1/-0 | 1 | | Dr Ian Preston | 1 | +1/-0 | 1 | | Baptiste Jonglez | 1 | +1/-0 | 1 | ================================================ FILE: docs/changelogs/v0.14.md ================================================ # Kubo changelog v0.14 ## v0.14.0 ### Overview Below is an outline of all that is in this release, so you get a sense of all that's included. - [🛠 BREAKING CHANGES](#-breaking-changes) - [Removed `mdns_legacy` implementation](#removed-mdns_legacy-implementation) - [🔦 Highlights](#-highlights) - [🛣️ Delegated Routing](#-delegated-routing) - [👥 Rename to Kubo](#-rename-to-kubo) - [🎒 `ipfs repo migrate`](#-ipfs-repo-migrate) - [🚀 Emoji support in Multibase](#-emoji-support-in-multibase) ### 🛠 BREAKING CHANGES #### Removed `mdns_legacy` implementation The modern DNS-SD compatible [zeroconf implementation](https://github.com/libp2p/zeroconf#readme) (based on [this specification](https://github.com/libp2p/specs/blob/master/discovery/mdns.md)) has been running next to the `mdns_legacy` for a while (since v0.11). During this transitional period Kubo nodes were sending twice as many LAN packets, which ends with this release: we've [removed](https://github.com/ipfs/kubo/pull/9048) the legacy implementation. ### 🔦 Highlights #### 🛣️ Delegated Routing Content routing is the a term used to describe the problem of finding providers for a given piece of content. If you have a hash, or CID of some data, how do you find who has it? In IPFS, until now, only a DHT was used as a decentralized answer to content routing. Now, content routing can be handled by clients implementing the [Reframe protocol](https://github.com/ipfs/specs/tree/main/reframe#readme). Example configuration usage using the [Filecoin Network Indexer](https://docs.cid.contact/filecoin-network-indexer/overview): ``` ipfs config Routing.Routers.CidContact --json '{ "Type": "reframe", "Parameters": { "Endpoint": "https://cid.contact/reframe" } }' ``` #### 👥 Rename to Kubo We've renamed Go-IPFS to Kubo ([details](https://github.com/ipfs/go-ipfs/issues/8959)). Published artifacts use `kubo` now, and are available at: - https://dist.ipfs.tech/kubo/ - https://hub.docker.com/r/ipfs/kubo/ To minimize the impact on infrastructure that autoupdates on a new release, the same binaries are still published under the old name at: - https://dist.ipfs.tech/go-ipfs/ - https://hub.docker.com/r/ipfs/go-ipfs/ The libp2p identify useragent of Kubo has also been changed from `go-ipfs` to `kubo`. #### 🎒 `ipfs repo migrate` This new command allows the you to run the repo migration without starting the daemon. See `ipfs repo migrate --help` for more info. #### 🚀 Emoji support in Multibase Kubo now supports [`base256emoji`](https://github.com/multiformats/multibase/blob/master/rfcs/Base256Emoji.md) encoding in all [Multibase](https://docs.ipfs.tech/concepts/glossary/#multibase) contexts. Use it for testing Unicode support, as visual aid while explaining Multiformats, or just for fun: ```console $ echo -n "test" | ipfs multibase encode -b base256emoji - 🚀😈✋🌈😈 $ echo -n "🚀😈✋🌈😈" | ipfs multibase decode - test $ ipfs cid format -v 1 -b base256emoji bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi 🚀🪐⭐💻😅❓💎🌈🌸🌚💰💍🌒😵🐶💁🤐🌎👼🙃🙅☺🌚😞🤤⭐🚀😃✈🌕😚🍻💜🐷⚽✌😊 ``` [`/ipfs/🚀🪐⭐💻😅❓💎🌈🌸🌚💰💍🌒😵🐶💁🤐🌎👼🙃🙅☺🌚😞🤤⭐🚀😃✈🌕😚🍻💜🐷⚽✌😊`](https://ipfs.io/ipfs/🚀🪐⭐💻😅❓💎🌈🌸🌚💰💍🌒😵🐶💁🤐🌎👼🙃🙅☺🌚😞🤤⭐🚀😃✈🌕😚🍻💜🐷⚽✌😊) ### Changelog
Full Changelog - github.com/ipfs/kubo: - chore: bump to v0.14.0 - docs(changelog): finish v0.14.0 changelog - fix(gw): cache-control of index.html websites - chore(license): fix broken link to apache-2.0 - fix: kubo in daemon and cli stdout - backport: merge commit '839b0848a' into release-v0.14.0 - chore: Release v0.14-rc1 - docs: fix v0.14's changelog format - chore: update go-multibase 🚀 - feat(routing): Delegated Routing (#8997) ([ipfs/kubo#8997](https://github.com/ipfs/kubo/pull/8997)) - chore: changelogs split - feat(gw): Cache-Control: only-if-cached - chore(deps): webui v2.15.1 - Follow-ups after repository rename ([ipfs/kubo#9098](https://github.com/ipfs/kubo/pull/9098)) - docs: refine wording - docs: refine the wording of provider strategies - refactor: rename to kubo ([ipfs/kubo#8958](https://github.com/ipfs/kubo/pull/8958)) - fix: correct cache-control in car responses - docs: v0.13.1 (#9093) ([ipfs/kubo#9093](https://github.com/ipfs/kubo/pull/9093)) - chore: update go-car ([ipfs/kubo#9089](https://github.com/ipfs/kubo/pull/9089)) - update go-libp2p to v0.20.3 ([ipfs/kubo#9038](https://github.com/ipfs/kubo/pull/9038)) - docs: add SECURITY.md (#9062) ([ipfs/kubo#9062](https://github.com/ipfs/kubo/pull/9062)) - fix: remove mdns_legacy & Discovery.MDNS.Interval - refactor: prealloc slices with known sizes (#8892) ([ipfs/kubo#8892](https://github.com/ipfs/kubo/pull/8892)) - docs: fix typo in `cid/base32` - docs: mark Swarm.ResourceMgr as experimental - chore: replace ioutil with io and os (#8969) ([ipfs/kubo#8969](https://github.com/ipfs/kubo/pull/8969)) - feat: add a public function on peering to get the state - fix: honor url filename when downloading as CAR/BLOCK - Merge branch 'release' - chore: GitHub format - fix(cmd/config): make config edit subcommand work on windows - chore: bump Go to 1.18.3 (#9021) ([ipfs/kubo#9021](https://github.com/ipfs/kubo/pull/9021)) - feat: upgrade to go-libp2p-kad-dht@v0.16.0 (#9005) ([ipfs/kubo#9005](https://github.com/ipfs/kubo/pull/9005)) - docs: fix typo in the `swarm/peering` help text - feat: disable resource manager by default (#9003) ([ipfs/kubo#9003](https://github.com/ipfs/kubo/pull/9003)) - fix: adjust rcmgr limits for accelerated DHT client rt refresh (#8982) ([ipfs/kubo#8982](https://github.com/ipfs/kubo/pull/8982)) - fix(ci): make go-ipfs-as-a-library work without external peers (#8978) ([ipfs/kubo#8978](https://github.com/ipfs/kubo/pull/8978)) - feat: log when resource manager limits are exceeded (#8980) ([ipfs/kubo#8980](https://github.com/ipfs/kubo/pull/8980)) - fix: JS caching via Access-Control-Expose-Headers (#8984) ([ipfs/kubo#8984](https://github.com/ipfs/kubo/pull/8984)) - docs: fix abstractions typo - fix: hanging goroutine in get fileArchive handler - chore: mark fuse experimental (#8962) ([ipfs/kubo#8962](https://github.com/ipfs/kubo/pull/8962)) - fix(node/libp2p): disable rcmgr checkImplicitDefaults ([ipfs/kubo#8965](https://github.com/ipfs/kubo/pull/8965)) - Add 'ipfs repo migrate' command (#8428) ([ipfs/kubo#8428](https://github.com/ipfs/kubo/pull/8428)) - pubsub multibase encoding (#8933) ([ipfs/kubo#8933](https://github.com/ipfs/kubo/pull/8933)) - 'pin rm' helptext: rewrite description as object is not removed from local storage (immediately) ([ipfs/kubo#8947](https://github.com/ipfs/kubo/pull/8947)) - ([ipfs/kubo#8934](https://github.com/ipfs/kubo/pull/8934)) - Add instructions to resolve repo migration error (#8946) ([ipfs/kubo#8946](https://github.com/ipfs/kubo/pull/8946)) - fix: use path instead of filepath for asset embeds to support Windows - chore: update version to v0.14.0-dev - github.com/ipfs/go-bitswap (v0.6.0 -> v0.7.0): - chore: release v0.7.0 (#566) ([ipfs/go-bitswap#566](https://github.com/ipfs/go-bitswap/pull/566)) - feat: coalesce and queue connection event handling (#565) ([ipfs/go-bitswap#565](https://github.com/ipfs/go-bitswap/pull/565)) - fix initialisation example in README (#552) ([ipfs/go-bitswap#552](https://github.com/ipfs/go-bitswap/pull/552)) - github.com/ipfs/go-unixfs (v0.3.1 -> v0.4.0): - Set version to v0.3.2 ([ipfs/go-unixfs#122](https://github.com/ipfs/go-unixfs/pull/122)) - Make switchToSharding more efficient - github.com/ipld/go-ipld-prime (v0.16.0 -> v0.17.0): failed to fetch repo - github.com/libp2p/go-libp2p (v0.19.4 -> v0.20.3): - Release 0.20.3 (#1623) ([libp2p/go-libp2p#1623](https://github.com/libp2p/go-libp2p/pull/1623)) - release v0.20.2 - feat: allow dialing wss peers using DNS multiaddrs - update go-yamux to v3.1.2, release v0.20.1 (#1591) ([libp2p/go-libp2p#1591](https://github.com/libp2p/go-libp2p/pull/1591)) - release v0.20.0 (#1530) ([libp2p/go-libp2p#1530](https://github.com/libp2p/go-libp2p/pull/1530)) - update go-libp2p-core, remove stream methods from network.Notifiee (#1521) ([libp2p/go-libp2p#1521](https://github.com/libp2p/go-libp2p/pull/1521)) - autonat: return E_DIAL_REFUSED when skipping dial (#1527) ([libp2p/go-libp2p#1527](https://github.com/libp2p/go-libp2p/pull/1527)) - move go-stream-muxer-multistream here ([libp2p/go-libp2p#1511](https://github.com/libp2p/go-libp2p/pull/1511)) - remove dependency on go-libp2p-testing/suites/sec (#1510) ([libp2p/go-libp2p#1510](https://github.com/libp2p/go-libp2p/pull/1510)) - backoff: fix flaky tests in backoff cache (#1516) ([libp2p/go-libp2p#1516](https://github.com/libp2p/go-libp2p/pull/1516)) - identify: fix flaky tests (#1515) ([libp2p/go-libp2p#1515](https://github.com/libp2p/go-libp2p/pull/1515)) - quic: increase timeout in hole punching test (#1495) ([libp2p/go-libp2p#1495](https://github.com/libp2p/go-libp2p/pull/1495)) - Fix badge image in README (#1517) ([libp2p/go-libp2p#1517](https://github.com/libp2p/go-libp2p/pull/1517)) - move go-libp2p-nat here ([libp2p/go-libp2p#1513](https://github.com/libp2p/go-libp2p/pull/1513)) - move go-reuseport-transport here ([libp2p/go-libp2p#1459](https://github.com/libp2p/go-libp2p/pull/1459)) - holepunch: fix flaky TestEndToEndSimConnect test (#1508) ([libp2p/go-libp2p#1508](https://github.com/libp2p/go-libp2p/pull/1508)) - swarm: fix flaky TestDialExistingConnection test (#1509) ([libp2p/go-libp2p#1509](https://github.com/libp2p/go-libp2p/pull/1509)) - tcp: limit the number of connections in tcp suite test on non-linux hosts (#1507) ([libp2p/go-libp2p#1507](https://github.com/libp2p/go-libp2p/pull/1507)) - increase overly short require.Eventually intervals (#1501) ([libp2p/go-libp2p#1501](https://github.com/libp2p/go-libp2p/pull/1501)) - tls: fix flaky handshake cancellation test (#1503) ([libp2p/go-libp2p#1503](https://github.com/libp2p/go-libp2p/pull/1503)) - merge the transport test suite from go-libp2p-testing here ([libp2p/go-libp2p#1496](https://github.com/libp2p/go-libp2p/pull/1496)) - fix racy connection comparison in TestDialWorkerLoopBasic (#1499) ([libp2p/go-libp2p#1499](https://github.com/libp2p/go-libp2p/pull/1499)) - swarm: fix race condition in TestFailFirst (#1490) ([libp2p/go-libp2p#1490](https://github.com/libp2p/go-libp2p/pull/1490)) - basichost: fix flaky TestSignedPeerRecordWithNoListenAddrs (#1488) ([libp2p/go-libp2p#1488](https://github.com/libp2p/go-libp2p/pull/1488)) - swarm: fix flaky and racy TestDialExistingConnection (#1491) ([libp2p/go-libp2p#1491](https://github.com/libp2p/go-libp2p/pull/1491)) - quic: adjust timeout for reuse garbage collector detection in tests (#1487) ([libp2p/go-libp2p#1487](https://github.com/libp2p/go-libp2p/pull/1487)) - quic: fix flaky TestResourceManagerAcceptDenied (#1485) ([libp2p/go-libp2p#1485](https://github.com/libp2p/go-libp2p/pull/1485)) - quic: deflake the holepunching test (#1484) ([libp2p/go-libp2p#1484](https://github.com/libp2p/go-libp2p/pull/1484)) - holepunch: fix incorrect message type for the SYNC message (#1478) ([libp2p/go-libp2p#1478](https://github.com/libp2p/go-libp2p/pull/1478)) - use real keys in tests instead of go-libp2p-testing/netutil fake keys (#1475) ([libp2p/go-libp2p#1475](https://github.com/libp2p/go-libp2p/pull/1475)) - quic: fix flaky TestResourceManagerAcceptDenied ([libp2p/go-libp2p#1461](https://github.com/libp2p/go-libp2p/pull/1461)) - move go-libp2p-pnet here ([libp2p/go-libp2p#1465](https://github.com/libp2p/go-libp2p/pull/1465)) - move go-libp2p-tls here ([libp2p/go-libp2p#1466](https://github.com/libp2p/go-libp2p/pull/1466)) - fix race condition in relayFinder ([libp2p/go-libp2p#1469](https://github.com/libp2p/go-libp2p/pull/1469)) - fix race condition in holepunch service (#1473) ([libp2p/go-libp2p#1473](https://github.com/libp2p/go-libp2p/pull/1473)) - Update README to include supported Go Versions (#1470) ([libp2p/go-libp2p#1470](https://github.com/libp2p/go-libp2p/pull/1470)) - move go-libp2p-noise here ([libp2p/go-libp2p#1462](https://github.com/libp2p/go-libp2p/pull/1462)) - move go-libp2p-transport-upgrader here ([libp2p/go-libp2p#1463](https://github.com/libp2p/go-libp2p/pull/1463)) - move go-conn-security-multistream here ([libp2p/go-libp2p#1460](https://github.com/libp2p/go-libp2p/pull/1460)) - move go-libp2p-mplex here ([libp2p/go-libp2p#1450](https://github.com/libp2p/go-libp2p/pull/1450)) - use yamux instead of mplex in tests (#1456) ([libp2p/go-libp2p#1456](https://github.com/libp2p/go-libp2p/pull/1456)) - rename the yamux package (#1452) ([libp2p/go-libp2p#1452](https://github.com/libp2p/go-libp2p/pull/1452)) - swarm: don't check return value of str.Close in TestResourceManager (#1453) ([libp2p/go-libp2p#1453](https://github.com/libp2p/go-libp2p/pull/1453)) - move go-libp2p-yamux here ([libp2p/go-libp2p#1439](https://github.com/libp2p/go-libp2p/pull/1439)) - quic: fix flaky TestConnectionGating test (#1442) ([libp2p/go-libp2p#1442](https://github.com/libp2p/go-libp2p/pull/1442)) - quic: fix flaky TestReuseGarbageCollect test (#1446) ([libp2p/go-libp2p#1446](https://github.com/libp2p/go-libp2p/pull/1446)) - quic: fix flaky holepunching test (#1443) ([libp2p/go-libp2p#1443](https://github.com/libp2p/go-libp2p/pull/1443)) - move go-libp2p-quic-transport here ([libp2p/go-libp2p#1424](https://github.com/libp2p/go-libp2p/pull/1424)) - remove flaky TestTcpSimultaneousConnect (#1425) ([libp2p/go-libp2p#1425](https://github.com/libp2p/go-libp2p/pull/1425)) - move go-ws-transport here ([libp2p/go-libp2p#1422](https://github.com/libp2p/go-libp2p/pull/1422)) - update go-multistream, stop using deprecated NegotiateLazy (#1417) ([libp2p/go-libp2p#1417](https://github.com/libp2p/go-libp2p/pull/1417)) - fix flaky TestResourceManagerAcceptStream test (#1420) ([libp2p/go-libp2p#1420](https://github.com/libp2p/go-libp2p/pull/1420)) - move go-tcp-transport here ([libp2p/go-libp2p#1418](https://github.com/libp2p/go-libp2p/pull/1418)) - move the go-libp2p-swarm here ([libp2p/go-libp2p#1414](https://github.com/libp2p/go-libp2p/pull/1414)) - reduce flakiness of backoff cache tests (#1415) ([libp2p/go-libp2p#1415](https://github.com/libp2p/go-libp2p/pull/1415)) - move the go-libp2p-blankhost here ([libp2p/go-libp2p#1411](https://github.com/libp2p/go-libp2p/pull/1411)) - github.com/libp2p/go-libp2p-core (v0.15.1 -> v0.16.1): - release v0.16.1 (#255) ([libp2p/go-libp2p-core#255](https://github.com/libp2p/go-libp2p-core/pull/255)) - force usage of github.com/btcsuite/btcd v0.22.1 or newer (#254) ([libp2p/go-libp2p-core#254](https://github.com/libp2p/go-libp2p-core/pull/254)) - release v0.16.0 (#251) ([libp2p/go-libp2p-core#251](https://github.com/libp2p/go-libp2p-core/pull/251)) - remove OpenedStream and ClosedStream from Notifiee interface (#250) ([libp2p/go-libp2p-core#250](https://github.com/libp2p/go-libp2p-core/pull/250)) - deprecate Negotiator.NegotiateLazy (#249) ([libp2p/go-libp2p-core#249](https://github.com/libp2p/go-libp2p-core/pull/249)) - update btcec dependency (#247) ([libp2p/go-libp2p-core#247](https://github.com/libp2p/go-libp2p-core/pull/247)) - github.com/libp2p/go-libp2p-discovery (v0.6.0 -> v0.7.0): - deprecate this repo (#84) ([libp2p/go-libp2p-discovery#84](https://github.com/libp2p/go-libp2p-discovery/pull/84)) - remove dependency on the go-libp2p-peerstore/addr package (#82) ([libp2p/go-libp2p-discovery#82](https://github.com/libp2p/go-libp2p-discovery/pull/82)) - fix flaky TestBackoffDiscoveryMultipleBackoff test on CI (#80) ([libp2p/go-libp2p-discovery#80](https://github.com/libp2p/go-libp2p-discovery/pull/80)) - chore: update go-log to v2 ([libp2p/go-libp2p-discovery#76](https://github.com/libp2p/go-libp2p-discovery/pull/76)) - sync: update CI config files (#74) ([libp2p/go-libp2p-discovery#74](https://github.com/libp2p/go-libp2p-discovery/pull/74)) - github.com/libp2p/go-libp2p-swarm (v0.10.2 -> v0.11.0): - deprecate this repo (#320) ([libp2p/go-libp2p-swarm#320](https://github.com/libp2p/go-libp2p-swarm/pull/320)) - sync: update CI config files ([libp2p/go-libp2p-swarm#317](https://github.com/libp2p/go-libp2p-swarm/pull/317)) - github.com/libp2p/go-reuseport (v0.1.0 -> v0.2.0): - release v0.2.0 (#90) ([libp2p/go-reuseport#90](https://github.com/libp2p/go-reuseport/pull/90)) - sync: update CI config files (#86) ([libp2p/go-reuseport#86](https://github.com/libp2p/go-reuseport/pull/86)) - github.com/multiformats/go-multibase (v0.0.3 -> v0.1.0): - chore: release v0.1.0 - feat: add UTF-8 support and base256emoji - submodule: spec/ - sync: update CI config files (#48) ([multiformats/go-multibase#48](https://github.com/multiformats/go-multibase/pull/48)) - fix staticcheck ([multiformats/go-multibase#41](https://github.com/multiformats/go-multibase/pull/41)) - Fix vet warnings about conversion of int to string ([multiformats/go-multibase#39](https://github.com/multiformats/go-multibase/pull/39)) - github.com/multiformats/go-multihash (v0.1.0 -> v0.2.0): - chore: replace blake2b implementation by golang.org/x/crypto ([multiformats/go-multihash#157](https://github.com/multiformats/go-multihash/pull/157)) - sync: update CI config files ([multiformats/go-multihash#156](https://github.com/multiformats/go-multihash/pull/156)) - github.com/multiformats/go-multistream (v0.3.0 -> v0.3.3): - Release v0.3.3 ([multiformats/go-multistream#90](https://github.com/multiformats/go-multistream/pull/90)) - Ignore error if can't write back multistream protocol id ([multiformats/go-multistream#89](https://github.com/multiformats/go-multistream/pull/89)) - release v0.3.2 (#88) ([multiformats/go-multistream#88](https://github.com/multiformats/go-multistream/pull/88)) - Ignore error if can't write back echoed protocol in negotiate (#87) ([multiformats/go-multistream#87](https://github.com/multiformats/go-multistream/pull/87)) - release v0.3.1 (#86) ([multiformats/go-multistream#86](https://github.com/multiformats/go-multistream/pull/86)) - deprecate NegotiateLazy (#85) ([multiformats/go-multistream#85](https://github.com/multiformats/go-multistream/pull/85)) - return an ErrNotSupported when lazy negotiation fails (#84) ([multiformats/go-multistream#84](https://github.com/multiformats/go-multistream/pull/84)) - github.com/warpfork/go-testmark (v0.9.0 -> v0.10.0): - testexec: support a hunk named 'input' for stdin. - readme: link to other implementations! - readme: discuss autopatching and fixture regeneration - readme: discuss extensions, and introduce testexec as an example.
### Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Marten Seemann | 376 | +11584/-15055 | 894 | | Jorropo | 18 | +11649/-11249 | 81 | | noot | 43 | +5974/-3332 | 170 | | Steven Allen | 173 | +5206/-3124 | 282 | | Yusef Napora | 49 | +1911/-3606 | 124 | | Juan Batiz-Benet | 14 | +3933/-53 | 48 | | Jeromy | 84 | +2140/-1328 | 240 | | vyzo | 51 | +2057/-1126 | 79 | | Raúl Kripalani | 39 | +1993/-867 | 103 | | Jeromy Johnson | 52 | +1700/-1081 | 233 | | Antonio Navarro Perez | 4 | +1874/-729 | 34 | | Aarsh Shah | 24 | +1428/-504 | 54 | | Marcin Rataj | 19 | +1051/-855 | 251 | | Alex Browne | 25 | +1207/-582 | 49 | | Jakub Sztandera | 29 | +898/-335 | 63 | | Friedel Ziegelmayer | 11 | +491/-284 | 18 | | Will Scott | 6 | +240/-319 | 17 | | Marco Munizaga | 11 | +377/-141 | 17 | | Hlib | 8 | +269/-135 | 15 | | Gus Eggert | 5 | +325/-63 | 19 | | lnykww | 1 | +275/-50 | 4 | | Łukasz Magiera | 3 | +196/-58 | 7 | | Matt Joiner | 14 | +79/-55 | 17 | | Eric Myhre | 4 | +122/-6 | 5 | | Andrew Gillis | 1 | +111/-6 | 4 | | Fazlul Shahriar | 2 | +84/-31 | 5 | | tg | 1 | +70/-15 | 2 | | Cory Schwartz | 4 | +50/-28 | 11 | | Lars Gierth | 3 | +33/-26 | 3 | | Cole Brown | 2 | +37/-16 | 9 | | web3-bot | 7 | +38/-11 | 18 | | Alvin Reyes | 1 | +34/-14 | 1 | | Hector Sanjuan | 4 | +34/-8 | 5 | | Guilhem Fanton | 2 | +28/-10 | 6 | | Brian Meek | 1 | +14/-17 | 4 | | Hlib Kanunnikov | 1 | +25/-3 | 1 | | Adin Schmahmann | 5 | +15/-13 | 5 | | Henrique Dias | 1 | +24/-2 | 4 | | Dennis Trautwein | 1 | +20/-4 | 2 | | galargh | 2 | +18/-2 | 2 | | M. Hawn | 3 | +10/-10 | 7 | | Can ZHANG | 1 | +12/-3 | 1 | | Masih H. Derkani | 1 | +4/-10 | 2 | | gammazero | 1 | +6/-6 | 2 | | Ikko Ashimine | 1 | +6/-6 | 2 | | Daniel N | 2 | +6/-5 | 2 | | watjurk | 1 | +8/-2 | 1 | | John Steidley | 2 | +4/-4 | 3 | | Aaron Bieber | 1 | +6/-2 | 1 | | Kishan Mohanbhai Sagathiya | 1 | +6/-1 | 1 | | siiky | 3 | +3/-3 | 3 | | Lucas Molas | 1 | +5/-1 | 1 | | Kevin Atkinson | 1 | +3/-3 | 1 | | Aayush Rajasekaran | 1 | +5/-1 | 1 | | T Mo | 1 | +2/-2 | 1 | | Piotr Galar | 1 | +2/-2 | 1 | | Arber Avdullahu | 1 | +2/-2 | 1 | | Russell Dempsey | 1 | +2/-1 | 1 | | anders | 1 | +1/-1 | 1 | | RubenKelevra | 1 | +1/-1 | 1 | | Jonathan Rudenberg | 1 | +1/-1 | 1 | | Ettore Di Giacinto | 1 | +2/-0 | 1 | | Daniel Norman | 1 | +1/-1 | 1 | | Chawye Hsu | 1 | +1/-1 | 1 | | Aliabbas Merchant | 1 | +1/-1 | 1 | | can | 1 | +1/-0 | 1 | | Ed Mazurek | 1 | +0/-0 | 1 | ================================================ FILE: docs/changelogs/v0.15.md ================================================ # Kubo changelog v0.15 ## v0.15.0 ### Overview Below is an outline of all that is in this release, so you get a sense of all that's included. - [Kubo changelog v0.15](#kubo-changelog-v015) - [v0.15.0](#v0150) - [Overview](#overview) - [🔦 Highlights](#-highlights) - [#️⃣ Blake 3 support](#️⃣-blake-3-support) - [💉 Fx Options plugin](#-fx-options-plugin) - [📁 `$IPFS_PATH/gateway` file](#-ipfs_pathgateway-file) - [Changelog](#changelog) - [Contributors](#contributors) ### 🔦 Highlights This is a release mainly with bugfixing and library updates. We are improving release speed and cadence trying to have a new release every 5 weeks. #### #️⃣ Blake 3 support You can now use `blake3` as a valid hash function: - `ipfs block put --mhtype=blake3` - `ipfs add --hash=blake3` It uses a 32 bytes default size. And verify up to 128 bytes. Because `blake3` is variable output hash function, you can use a different digest length, set `mhlen`: `ipfs block put --mhtype=blake3 --mhlen=64`, `ipfs add` doesn't have this option yet. #### 💉 Fx Options plugin This adds a plugin interface that lets the plugin modify the fx options that are passed to fx when the app is initialized. This means plugins can inject their own implementations of Kubo interfaces. This enables granular customization of Kubo behavior by plugins, such as: - Bitswap with custom filters (e.g. for CID blocking) - Custom interface implementations such as Pinner or DAGService - Dynamic configuration of libp2p ... Here's an example plugin that overrides the default Pinner with a custom one: ```go func (p *PinnerPlugin) Options(info core.FXNodeInfo) ([]fx.Option, error) { pinner := mypinner.New() return append(info.FXOptions, fx.Replace(fx.Annotate(pinner, fx.As(new(pin.Pinner))))), nil } ``` Extra plugin info [here](https://github.com/ipfs/kubo/blob/master/docs/plugins.md#fx-experimental). #### 📁 `$IPFS_PATH/gateway` file This adds a new file in the `IPFS_PATH` folder similar to `$IPFS_PATH/api` containing an address based on [`Addresses.Gateway`](https://github.com/ipfs/kubo/blob/master/docs/config.md#addressesgateway) configuration. This file is in URL (RFC1738) format. ```console $ cat ~/.ipfs/gateway http://127.0.0.1:8080 ``` ### Changelog
Full Changelog - github.com/ipfs/kubo: - chore: Release v0.15.0-rc1 - Update RELEASE_ISSUE_TEMPLATE.md for 0.15 - docs(add): skip binary name in helptext - docs(cli): clarify CID determinism in add command - docs(cli): clarify CAR format in dag export|import - test(gw): cors preflight with custom header - feat: make corehttp a reusable component ([ipfs/kubo#9070](https://github.com/ipfs/kubo/pull/9070)) - feat: go-libp2p v0.21 (rcmgr auto scaling) ([ipfs/kubo#9074](https://github.com/ipfs/kubo/pull/9074)) - ([ipfs/kubo#9024](https://github.com/ipfs/kubo/pull/9024)) - ([ipfs/kubo#9100](https://github.com/ipfs/kubo/pull/9100)) - ([ipfs/kubo#9095](https://github.com/ipfs/kubo/pull/9095)) - chore(cmd): add shutdown to CLI help ([ipfs/kubo#9194](https://github.com/ipfs/kubo/pull/9194)) - docs: add fx plugin documentation to plugins.md (#9191) ([ipfs/kubo#9191](https://github.com/ipfs/kubo/pull/9191)) - chore: switch to dist.ipfs.tech - feat: add fx options plugin - feat: add blake3 support - Add reference to Experimental config doc (#9181) ([ipfs/kubo#9181](https://github.com/ipfs/kubo/pull/9181)) - feat: add $IPFS_PATH/gateway file - docs: replace `docs.ipfs.io` with `docs.ipfs.tech` (#9158) ([ipfs/kubo#9158](https://github.com/ipfs/kubo/pull/9158)) - chore: fix markdown link syntax typo for AutoNAT.ServiceMode - chore: bump go-blockservice to only do put once - docs: update Arch Linux installation instructions - chore: update kubo-as-a-library example - docs(readme): add maintainer info (#9141) ([ipfs/kubo#9141](https://github.com/ipfs/kubo/pull/9141)) - fix(gw): 404 when a valid DAG is missing link - fix(gw): directory URL normalization ([ipfs/kubo#9123](https://github.com/ipfs/kubo/pull/9123)) - docs(config): add link to someguy router - fix: typo in README - Reproducible Builds: Update GOFLAGS for -trimpath - Merge v0.14.0 back into master - fix(gw): cache-control of index.html websites - chore(license): fix broken link to apache-2.0 - fix: kubo in daemon and cli stdout - docs(readme): move content to docs website (#9102) ([ipfs/kubo#9102](https://github.com/ipfs/kubo/pull/9102)) - fix(gw): no backlink when listing root dir - github.com/ipfs/go-bitswap (v0.7.0 -> v0.9.0): - chore: release v0.9.0 - feat: split client and server ([ipfs/go-bitswap#570](https://github.com/ipfs/go-bitswap/pull/570)) - chore: remove goprocess from blockstoremanager - Don't add blocks to the datastore ([ipfs/go-bitswap#571](https://github.com/ipfs/go-bitswap/pull/571)) - Remove dependency on travis package from go-libp2p-testing ([ipfs/go-bitswap#569](https://github.com/ipfs/go-bitswap/pull/569)) - feat: add basic tracing (#562) ([ipfs/go-bitswap#562](https://github.com/ipfs/go-bitswap/pull/562)) - github.com/ipfs/go-blockservice (v0.3.0 -> v0.4.0): - write blocks retrieved from the exchange to the blockstore ([ipfs/go-blockservice#92](https://github.com/ipfs/go-blockservice/pull/92)) - feat: add basic tracing ([ipfs/go-blockservice#91](https://github.com/ipfs/go-blockservice/pull/91)) - github.com/ipfs/go-ipfs-exchange-interface (v0.1.0 -> v0.2.0): - Rename HasBlock to NotifyNewBlocks, and make it accept multiple blocks ([ipfs/go-ipfs-exchange-interface#23](https://github.com/ipfs/go-ipfs-exchange-interface/pull/23)) - github.com/ipfs/go-ipfs-exchange-offline (v0.2.0 -> v0.3.0): - Exchange don't add blocks on their own anymore ([ipfs/go-ipfs-exchange-offline#47](https://github.com/ipfs/go-ipfs-exchange-offline/pull/47)) - github.com/ipfs/go-verifcid (v0.0.1 -> v0.0.2): - chore: release v0.0.2 - feat: add blake3 as a good hash - sync: update CI config files (#12) ([ipfs/go-verifcid#12](https://github.com/ipfs/go-verifcid/pull/12)) - Add license ([ipfs/go-verifcid#8](https://github.com/ipfs/go-verifcid/pull/8)) - github.com/ipld/go-codec-dagpb (v1.4.0 -> v1.4.1): - v1.4.1 bump - github.com/libp2p/go-buffer-pool (v0.0.2 -> v0.1.0): - release v0.1.0 (#30) ([libp2p/go-buffer-pool#30](https://github.com/libp2p/go-buffer-pool/pull/30)) - panic if a negative length is passed to BufferPool.Get (#28) ([libp2p/go-buffer-pool#28](https://github.com/libp2p/go-buffer-pool/pull/28)) - sync: update CI config files (#22) ([libp2p/go-buffer-pool#22](https://github.com/libp2p/go-buffer-pool/pull/22)) - sync: update CI config files (#20) ([libp2p/go-buffer-pool#20](https://github.com/libp2p/go-buffer-pool/pull/20)) - test: fix gc test on go 1.16 ([libp2p/go-buffer-pool#18](https://github.com/libp2p/go-buffer-pool/pull/18)) - fix staticcheck ([libp2p/go-buffer-pool#16](https://github.com/libp2p/go-buffer-pool/pull/16)) - test: make sure we have the correct number of pools ([libp2p/go-buffer-pool#10](https://github.com/libp2p/go-buffer-pool/pull/10)) - github.com/libp2p/go-libp2p (v0.20.3 -> v0.21.0): - Release v0.21.0 (#1648) ([libp2p/go-libp2p#1648](https://github.com/libp2p/go-libp2p/pull/1648)) - ping: optimize random number generation (#1658) ([libp2p/go-libp2p#1658](https://github.com/libp2p/go-libp2p/pull/1658)) - feat: switch noise to use minio's SHA256 implementation (#1657) ([libp2p/go-libp2p#1657](https://github.com/libp2p/go-libp2p/pull/1657)) - swarm: mark dialing WebTransport addresses as expensive (#1650) ([libp2p/go-libp2p#1650](https://github.com/libp2p/go-libp2p/pull/1650)) - routedhost: fix decoding of relay peer ID (#1644) ([libp2p/go-libp2p#1644](https://github.com/libp2p/go-libp2p/pull/1644)) - Release v0.21.0 RC (#1638) ([libp2p/go-libp2p#1638](https://github.com/libp2p/go-libp2p/pull/1638)) - fix: return the best _acceptable_ conn in NewStream (#1604) ([libp2p/go-libp2p#1604](https://github.com/libp2p/go-libp2p/pull/1604)) - use autoscaling limits (#1637) ([libp2p/go-libp2p#1637](https://github.com/libp2p/go-libp2p/pull/1637)) - docs: point to SetDefaultServiceLimits in ResourceManager option (#1636) ([libp2p/go-libp2p#1636](https://github.com/libp2p/go-libp2p/pull/1636)) - chore: update deps (#1634) ([libp2p/go-libp2p#1634](https://github.com/libp2p/go-libp2p/pull/1634)) - Pass endpoint information to resource manager's OpenConnection (#1633) ([libp2p/go-libp2p#1633](https://github.com/libp2p/go-libp2p/pull/1633)) - Add canonical peer status logs (#1624) ([libp2p/go-libp2p#1624](https://github.com/libp2p/go-libp2p/pull/1624)) - move go-libp2p-circuit here ([libp2p/go-libp2p#1626](https://github.com/libp2p/go-libp2p/pull/1626)) - swarm: fix logging of accepted connections (#1629) ([libp2p/go-libp2p#1629](https://github.com/libp2p/go-libp2p/pull/1629)) - fix: deny connections to peers in the right place (#1627) ([libp2p/go-libp2p#1627](https://github.com/libp2p/go-libp2p/pull/1627)) - ping: fix flaky test (#1617) ([libp2p/go-libp2p#1617](https://github.com/libp2p/go-libp2p/pull/1617)) - chore: use the new multiaddr.Contains function (#1618) ([libp2p/go-libp2p#1618](https://github.com/libp2p/go-libp2p/pull/1618)) - chore: stop using the deprecated mux.MuxedConn (#1614) ([libp2p/go-libp2p#1614](https://github.com/libp2p/go-libp2p/pull/1614)) - logging: Add canonical log for misbehaving peers (#1600) ([libp2p/go-libp2p#1600](https://github.com/libp2p/go-libp2p/pull/1600)) - use multiaddr ipcidr to parse multiaddr filters (#1606) ([libp2p/go-libp2p#1606](https://github.com/libp2p/go-libp2p/pull/1606)) - tcp: unexport TcpTransport.Upgrader (#1596) ([libp2p/go-libp2p#1596](https://github.com/libp2p/go-libp2p/pull/1596)) - muxer: expose func to create MuxedConn from backing Conn (#1609) ([libp2p/go-libp2p#1609](https://github.com/libp2p/go-libp2p/pull/1609)) - remove legacy mDNS implementation (#1192) ([libp2p/go-libp2p#1192](https://github.com/libp2p/go-libp2p/pull/1192)) - feat: allow dialing wss peers using DNS multiaddrs - fix natManager to close natManager.nat (#1468) ([libp2p/go-libp2p#1468](https://github.com/libp2p/go-libp2p/pull/1468)) - Expose DefaultPerPeerRateLimit as var (#1580) ([libp2p/go-libp2p#1580](https://github.com/libp2p/go-libp2p/pull/1580)) - swarm: add ListenClose (#1586) ([libp2p/go-libp2p#1586](https://github.com/libp2p/go-libp2p/pull/1586)) - identify: Fix flaky tests (#1555) ([libp2p/go-libp2p#1555](https://github.com/libp2p/go-libp2p/pull/1555)) - autonat: fix flaky TestAutoNATPrivate (#1581) ([libp2p/go-libp2p#1581](https://github.com/libp2p/go-libp2p/pull/1581)) - pstoremanager: fix test timeout (#1588) ([libp2p/go-libp2p#1588](https://github.com/libp2p/go-libp2p/pull/1588)) - swarm: send notifications synchronously (#1562) ([libp2p/go-libp2p#1562](https://github.com/libp2p/go-libp2p/pull/1562)) - basichost: fix flaky TestSignedPeerRecordWithNoListenAddrs (#1559) ([libp2p/go-libp2p#1559](https://github.com/libp2p/go-libp2p/pull/1559)) - identify: fix flaky TestIdentifyDeltaOnProtocolChange (again) (#1582) ([libp2p/go-libp2p#1582](https://github.com/libp2p/go-libp2p/pull/1582)) - tls: fix flaky TestInvalidCerts on Windows ([libp2p/go-libp2p#1560](https://github.com/libp2p/go-libp2p/pull/1560)) - chore: log autorelay start failure error ([libp2p/go-libp2p#1583](https://github.com/libp2p/go-libp2p/pull/1583)) - Add sanity check assertion (#1570) ([libp2p/go-libp2p#1570](https://github.com/libp2p/go-libp2p/pull/1570)) - swarm: speed up the TestDialWorkerLoopConcurrentFailureStress test (#1573) ([libp2p/go-libp2p#1573](https://github.com/libp2p/go-libp2p/pull/1573)) - chore: update examples to go-libp2p v0.20.0 (#1557) ([libp2p/go-libp2p#1557](https://github.com/libp2p/go-libp2p/pull/1557)) - Wait a couple seconds for ID event (#1568) ([libp2p/go-libp2p#1568](https://github.com/libp2p/go-libp2p/pull/1568)) - remove workspace and packages section from README (#1563) ([libp2p/go-libp2p#1563](https://github.com/libp2p/go-libp2p/pull/1563)) - fix: mkreleaselog exclude autogenerated files (#1567) ([libp2p/go-libp2p#1567](https://github.com/libp2p/go-libp2p/pull/1567)) - move resource manager integration tests to p2p/test/ (#1561) ([libp2p/go-libp2p#1561](https://github.com/libp2p/go-libp2p/pull/1561)) - swarm: only dial a single transport in TestDialWorkerLoopBasic (#1526) ([libp2p/go-libp2p#1526](https://github.com/libp2p/go-libp2p/pull/1526)) - github.com/libp2p/go-libp2p-core (v0.16.1 -> v0.19.1): - Update version.json - Remove btcsuite/btcd dep (#272) ([libp2p/go-libp2p-core#272](https://github.com/libp2p/go-libp2p-core/pull/272)) - Release v0.19.0 (#271) ([libp2p/go-libp2p-core#271](https://github.com/libp2p/go-libp2p-core/pull/271)) - Add endpoint parameter to the OpenConnection method for ResourceManager (#257) ([libp2p/go-libp2p-core#257](https://github.com/libp2p/go-libp2p-core/pull/257)) - Release v0.18.0 (#270) ([libp2p/go-libp2p-core#270](https://github.com/libp2p/go-libp2p-core/pull/270)) - Add canonical peer status logging with sampling (#269) ([libp2p/go-libp2p-core#269](https://github.com/libp2p/go-libp2p-core/pull/269)) - canonicallog: reduce log level to warning (#268) ([libp2p/go-libp2p-core#268](https://github.com/libp2p/go-libp2p-core/pull/268)) - Only log once if we failed to convert from netAddr (#264) ([libp2p/go-libp2p-core#264](https://github.com/libp2p/go-libp2p-core/pull/264)) - remove deprecated mux package (#265) ([libp2p/go-libp2p-core#265](https://github.com/libp2p/go-libp2p-core/pull/265)) - remove the peer.Set (#261) ([libp2p/go-libp2p-core#261](https://github.com/libp2p/go-libp2p-core/pull/261)) - Bump version (#259) ([libp2p/go-libp2p-core#259](https://github.com/libp2p/go-libp2p-core/pull/259)) - Add canonical log for misbehaving peers (#258) ([libp2p/go-libp2p-core#258](https://github.com/libp2p/go-libp2p-core/pull/258)) - github.com/libp2p/go-libp2p-kad-dht (v0.16.0 -> v0.17.0): - Chore: bump version to v0.17.0 - Update go-libp2p to v0.20.3 ([libp2p/go-libp2p-kad-dht#778](https://github.com/libp2p/go-libp2p-kad-dht/pull/778)) - github.com/libp2p/go-libp2p-peerstore (v0.6.0 -> v0.7.1): - Release v0.7.1 ([libp2p/go-libp2p-peerstore#202](https://github.com/libp2p/go-libp2p-peerstore/pull/202)) - stop using the peer.Set (#201) ([libp2p/go-libp2p-peerstore#201](https://github.com/libp2p/go-libp2p-peerstore/pull/201)) - feat: Use a clock interface in pstoreds as well ([libp2p/go-libp2p-peerstore#200](https://github.com/libp2p/go-libp2p-peerstore/pull/200)) - feat: use a clock interface to better support testing for pstoremem ([libp2p/go-libp2p-peerstore#199](https://github.com/libp2p/go-libp2p-peerstore/pull/199)) - pstoremem: fix slice preallocation in GetProtocols (#198) ([libp2p/go-libp2p-peerstore#198](https://github.com/libp2p/go-libp2p-peerstore/pull/198)) - remove all calls to peer.ID.Validate ([libp2p/go-libp2p-peerstore#194](https://github.com/libp2p/go-libp2p-peerstore/pull/194)) - remove the addr package ([libp2p/go-libp2p-peerstore#195](https://github.com/libp2p/go-libp2p-peerstore/pull/195)) - move AddrList to pstoremen, unexport it ([libp2p/go-libp2p-peerstore#193](https://github.com/libp2p/go-libp2p-peerstore/pull/193)) - optimize allocations in the memory address book ([libp2p/go-libp2p-peerstore#191](https://github.com/libp2p/go-libp2p-peerstore/pull/191)) - implement a clean shutdown for the memory address book ([libp2p/go-libp2p-peerstore#192](https://github.com/libp2p/go-libp2p-peerstore/pull/192)) - github.com/libp2p/go-libp2p-resource-manager (v0.3.0 -> v0.5.3): - Chore: release patch v0.5.3 ([libp2p/go-libp2p-resource-manager#77](https://github.com/libp2p/go-libp2p-resource-manager/pull/77)) - Add namespace to metrics ([libp2p/go-libp2p-resource-manager#79](https://github.com/libp2p/go-libp2p-resource-manager/pull/79)) - Fix usage of make to reserve capacity, not values ([libp2p/go-libp2p-resource-manager#76](https://github.com/libp2p/go-libp2p-resource-manager/pull/76)) - Add package docs ([libp2p/go-libp2p-resource-manager#75](https://github.com/libp2p/go-libp2p-resource-manager/pull/75)) - chore: Release v0.5.2 ([libp2p/go-libp2p-resource-manager#74](https://github.com/libp2p/go-libp2p-resource-manager/pull/74)) - Record which direction the resource was blocked ([libp2p/go-libp2p-resource-manager#72](https://github.com/libp2p/go-libp2p-resource-manager/pull/72)) - Simplify mem graphs in stock Grafana dashboard ([libp2p/go-libp2p-resource-manager#73](https://github.com/libp2p/go-libp2p-resource-manager/pull/73)) - feat: Handle multiple instances in stock Grafana dashboard ([libp2p/go-libp2p-resource-manager#70](https://github.com/libp2p/go-libp2p-resource-manager/pull/70)) - Use templated version of Grafana dashboard json ([libp2p/go-libp2p-resource-manager#69](https://github.com/libp2p/go-libp2p-resource-manager/pull/69)) - Release v0.5.1 ([libp2p/go-libp2p-resource-manager#66](https://github.com/libp2p/go-libp2p-resource-manager/pull/66)) - Implement `json.Marshaler` interface for LimitConfig ([libp2p/go-libp2p-resource-manager#67](https://github.com/libp2p/go-libp2p-resource-manager/pull/67)) - Don't wait for a chan that will never close ([libp2p/go-libp2p-resource-manager#65](https://github.com/libp2p/go-libp2p-resource-manager/pull/65)) - release v0.5.0 ([libp2p/go-libp2p-resource-manager#60](https://github.com/libp2p/go-libp2p-resource-manager/pull/60)) - Add docs around WithAllowlistedMultiaddrs. Expose allowlist ([libp2p/go-libp2p-resource-manager#63](https://github.com/libp2p/go-libp2p-resource-manager/pull/63)) - fix marshalling of allowlisted scopes ([libp2p/go-libp2p-resource-manager#62](https://github.com/libp2p/go-libp2p-resource-manager/pull/62)) - docs: describe how the limiter is configured, and how limits are scaled (#59) ([libp2p/go-libp2p-resource-manager#59](https://github.com/libp2p/go-libp2p-resource-manager/pull/59)) - don't limit the number of FDs on Windows (#58) ([libp2p/go-libp2p-resource-manager#58](https://github.com/libp2p/go-libp2p-resource-manager/pull/58)) - Add ability to configure allowlist limits ([libp2p/go-libp2p-resource-manager#57](https://github.com/libp2p/go-libp2p-resource-manager/pull/57)) - rewrite limits to allow auto-scaling ([libp2p/go-libp2p-resource-manager#48](https://github.com/libp2p/go-libp2p-resource-manager/pull/48)) - Release v0.4.0 ([libp2p/go-libp2p-resource-manager#56](https://github.com/libp2p/go-libp2p-resource-manager/pull/56)) - feat: Out of the box metrics for resource manager ([libp2p/go-libp2p-resource-manager#54](https://github.com/libp2p/go-libp2p-resource-manager/pull/54)) - feat: Allowlist ([libp2p/go-libp2p-resource-manager#47](https://github.com/libp2p/go-libp2p-resource-manager/pull/47)) - trace the scope as a JSON object (#52) ([libp2p/go-libp2p-resource-manager#52](https://github.com/libp2p/go-libp2p-resource-manager/pull/52)) - include current limits in debug messages ([libp2p/go-libp2p-resource-manager#42](https://github.com/libp2p/go-libp2p-resource-manager/pull/42)) - add an ID to spans (#44) ([libp2p/go-libp2p-resource-manager#44](https://github.com/libp2p/go-libp2p-resource-manager/pull/44)) - add a DefaultLimitConfig with infinite limits (#41) ([libp2p/go-libp2p-resource-manager#41](https://github.com/libp2p/go-libp2p-resource-manager/pull/41)) - export the TraceEvt (#40) ([libp2p/go-libp2p-resource-manager#40](https://github.com/libp2p/go-libp2p-resource-manager/pull/40)) - trace exact timestamps (#39) ([libp2p/go-libp2p-resource-manager#39](https://github.com/libp2p/go-libp2p-resource-manager/pull/39)) - skip events that don't change anything in tracer (#38) ([libp2p/go-libp2p-resource-manager#38](https://github.com/libp2p/go-libp2p-resource-manager/pull/38)) - fix typos in MetricsReporter docs - fix shadowing of service name (#37) ([libp2p/go-libp2p-resource-manager#37](https://github.com/libp2p/go-libp2p-resource-manager/pull/37)) - add a timestamp to trace events (#34) ([libp2p/go-libp2p-resource-manager#34](https://github.com/libp2p/go-libp2p-resource-manager/pull/34)) - github.com/libp2p/go-libp2p-testing (v0.9.2 -> v0.11.0): - Release v0.11.0 ([libp2p/go-libp2p-testing#64](https://github.com/libp2p/go-libp2p-testing/pull/64)) - Remove unused bench file and dep ([libp2p/go-libp2p-testing#63](https://github.com/libp2p/go-libp2p-testing/pull/63)) - Release v0.10.0 ([libp2p/go-libp2p-testing#62](https://github.com/libp2p/go-libp2p-testing/pull/62)) - Update go-libp2p-core dep ([libp2p/go-libp2p-testing#61](https://github.com/libp2p/go-libp2p-testing/pull/61)) - remove suites (#60) ([libp2p/go-libp2p-testing#60](https://github.com/libp2p/go-libp2p-testing/pull/60)) - don't continue on read / write error in stream suite (#59) ([libp2p/go-libp2p-testing#59](https://github.com/libp2p/go-libp2p-testing/pull/59)) - remove debug logging from stream and muxer suite ([libp2p/go-libp2p-testing#58](https://github.com/libp2p/go-libp2p-testing/pull/58)) - remove Travis package (#57) ([libp2p/go-libp2p-testing#57](https://github.com/libp2p/go-libp2p-testing/pull/57)) - github.com/lucas-clemente/quic-go (v0.27.1 -> v0.28.0): - update for Go 1.19beta1 (#3460) ([lucas-clemente/quic-go#3460](https://github.com/lucas-clemente/quic-go/pull/3460)) - Deduplicate Alt-Svc header values (#3461) ([lucas-clemente/quic-go#3461](https://github.com/lucas-clemente/quic-go/pull/3461)) - only set DF for sockets that can handle it (#3448) ([lucas-clemente/quic-go#3448](https://github.com/lucas-clemente/quic-go/pull/3448)) - fix flaky HTTP/3 request body test (#3447) ([lucas-clemente/quic-go#3447](https://github.com/lucas-clemente/quic-go/pull/3447)) - make the keep alive interval configurable (#3444) ([lucas-clemente/quic-go#3444](https://github.com/lucas-clemente/quic-go/pull/3444)) - implement QUIC v2 ([lucas-clemente/quic-go#3432](https://github.com/lucas-clemente/quic-go/pull/3432)) - allow HTTP clients and servers to take over the request stream ([lucas-clemente/quic-go#3437](https://github.com/lucas-clemente/quic-go/pull/3437)) - remove the http3.DataStreamer (#3435) ([lucas-clemente/quic-go#3435](https://github.com/lucas-clemente/quic-go/pull/3435)) - always reset header buffer, even when QPACK encoding fails (#3436) ([lucas-clemente/quic-go#3436](https://github.com/lucas-clemente/quic-go/pull/3436)) - Change "HTTP/3" to "HTTP/3.0". (#3439) ([lucas-clemente/quic-go#3439](https://github.com/lucas-clemente/quic-go/pull/3439)) - remove stray http3 connection file - pass frame / stream type parsing errors to the hijacker callbacks ([lucas-clemente/quic-go#3429](https://github.com/lucas-clemente/quic-go/pull/3429)) - add test for bidirectional stream hijacker (#3434) ([lucas-clemente/quic-go#3434](https://github.com/lucas-clemente/quic-go/pull/3434)) - make it possible to parse a varint at the end of a reader (#3428) ([lucas-clemente/quic-go#3428](https://github.com/lucas-clemente/quic-go/pull/3428)) - don't ignore errors that occur when the TLS ClientHello is generated ([lucas-clemente/quic-go#3424](https://github.com/lucas-clemente/quic-go/pull/3424)) - don't send path MTU probe packets on a timer (#3423) ([lucas-clemente/quic-go#3423](https://github.com/lucas-clemente/quic-go/pull/3423)) - introduce a http3.RoundTripOpt to prevent closing of request stream (#3411) ([lucas-clemente/quic-go#3411](https://github.com/lucas-clemente/quic-go/pull/3411)) - don't close the request stream when http3.DataStreamer was used (#3413) ([lucas-clemente/quic-go#3413](https://github.com/lucas-clemente/quic-go/pull/3413)) - do not embed http.Server in http3.Server (#3397) ([lucas-clemente/quic-go#3397](https://github.com/lucas-clemente/quic-go/pull/3397)) - remove error return value from ComposeVersionNegotiation (#3410) ([lucas-clemente/quic-go#3410](https://github.com/lucas-clemente/quic-go/pull/3410)) - don't set receive buffer if it is already large enough (#3407) ([lucas-clemente/quic-go#3407](https://github.com/lucas-clemente/quic-go/pull/3407)) - clone TLS conf in newClient (#3400) ([lucas-clemente/quic-go#3400](https://github.com/lucas-clemente/quic-go/pull/3400)) - remove warning comments of stable implementation (#3399) ([lucas-clemente/quic-go#3399](https://github.com/lucas-clemente/quic-go/pull/3399)) - fix parsing of request path for Extended CONNECT requests (#3388) ([lucas-clemente/quic-go#3388](https://github.com/lucas-clemente/quic-go/pull/3388)) - update docs to reflect that we support RFC 9221 (Unreliable Datagrams) (#3382) ([lucas-clemente/quic-go#3382](https://github.com/lucas-clemente/quic-go/pull/3382)) - fix deadlock on concurrent http3.Server.Serve and Close calls (#3387) ([lucas-clemente/quic-go#3387](https://github.com/lucas-clemente/quic-go/pull/3387)) - reduce flakiness of deadline integration tests (#3383) ([lucas-clemente/quic-go#3383](https://github.com/lucas-clemente/quic-go/pull/3383)) - protect against concurrent use of Stream.Write (#3381) ([lucas-clemente/quic-go#3381](https://github.com/lucas-clemente/quic-go/pull/3381)) - protect against concurrent use of Stream.Read (#3380) ([lucas-clemente/quic-go#3380](https://github.com/lucas-clemente/quic-go/pull/3380)) - Expose quic server closed err (#3395) ([lucas-clemente/quic-go#3395](https://github.com/lucas-clemente/quic-go/pull/3395)) - implement HTTP/3 unidirectional stream hijacking (#3389) ([lucas-clemente/quic-go#3389](https://github.com/lucas-clemente/quic-go/pull/3389)) - add LocalAddr and RemoteAddr functions to http3.StreamCreator (#3384) ([lucas-clemente/quic-go#3384](https://github.com/lucas-clemente/quic-go/pull/3384)) - extend the HTTP/3 API for WebTransport support ([lucas-clemente/quic-go#3362](https://github.com/lucas-clemente/quic-go/pull/3362)) - remove unneeded network from custom dial function used in HTTP/3 (#3368) ([lucas-clemente/quic-go#3368](https://github.com/lucas-clemente/quic-go/pull/3368)) - github.com/multiformats/go-multiaddr (v0.5.0 -> v0.6.0): - release v0.6.0 ([multiformats/go-multiaddr#178](https://github.com/multiformats/go-multiaddr/pull/178)) - add WebTransport multiaddr components ([multiformats/go-multiaddr#176](https://github.com/multiformats/go-multiaddr/pull/176)) - add ipcidr support (#177) ([multiformats/go-multiaddr#177](https://github.com/multiformats/go-multiaddr/pull/177)) - add a Contains function (#172) ([multiformats/go-multiaddr#172](https://github.com/multiformats/go-multiaddr/pull/172)) - github.com/multiformats/go-multibase (v0.1.0 -> v0.1.1): - chore: release version 0.1.1 - fix: add new emoji codepoint for Base256Emoji 🐉 - github.com/multiformats/go-multihash (v0.2.0 -> v0.2.1): - chore: release v0.2.1 - feat: adding tests and finish variable sized functions - feat: add support for variable length hash functions - adding blake3 tests and fixing an incorrect error message. (#158) ([multiformats/go-multihash#158](https://github.com/multiformats/go-multihash/pull/158))
### Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Marten Seemann | 129 | +5612/-9895 | 345 | | Marco Munizaga | 109 | +7689/-3221 | 181 | | vyzo | 64 | +3972/-657 | 125 | | Jorropo | 19 | +1977/-1611 | 109 | | Steven Allen | 30 | +633/-593 | 54 | | Jeromy Johnson | 5 | +1032/-64 | 16 | | Marcin Rataj | 21 | +406/-200 | 59 | | Michael Muré | 6 | +335/-250 | 14 | | Gus Eggert | 8 | +336/-104 | 31 | | Claudia Richoux | 3 | +181/-63 | 9 | | Steve Loeppky | 11 | +95/-141 | 11 | | Ian Davis | 4 | +126/-58 | 6 | | hareku | 3 | +172/-6 | 7 | | Ivan Trubach | 1 | +98/-74 | 6 | | Raúl Kripalani | 2 | +69/-62 | 9 | | Seungbae Yu | 1 | +41/-41 | 13 | | Julien Muret | 1 | +60/-7 | 2 | | Mark Gaiser | 1 | +64/-0 | 5 | | Lars Gierth | 1 | +20/-29 | 4 | | Cole Brown | 4 | +27/-19 | 4 | | Chao Fei | 2 | +15/-30 | 9 | | Nuno Diegues | 2 | +25/-18 | 9 | | Jakub Sztandera | 1 | +37/-0 | 3 | | Wiktor Jurkiewicz | 1 | +13/-5 | 1 | | c r | 1 | +11/-6 | 3 | | Christian Stewart | 1 | +15/-2 | 4 | | Matt Robenolt | 1 | +15/-1 | 2 | | aarshkshah1992 | 2 | +8/-2 | 2 | | link2xt | 1 | +4/-4 | 1 | | Aaron Riekenberg | 1 | +4/-4 | 4 | | web3-bot | 3 | +7/-0 | 3 | | Adrian Lanzafame | 1 | +3/-3 | 1 | | Dmitriy Ryajov | 2 | +2/-3 | 2 | | Brendan O'Brien | 1 | +5/-0 | 1 | | millken | 1 | +1/-1 | 1 | | lostystyg | 1 | +1/-1 | 1 | | kpcyrd | 1 | +1/-1 | 1 | | anders | 1 | +1/-1 | 1 | | Rod Vagg | 1 | +1/-1 | 1 | | Matt Joiner | 1 | +1/-1 | 1 | | Leo Balduf | 1 | +1/-1 | 1 | | Didrik Nordström | 1 | +2/-0 | 1 | | Daniel Norman | 1 | +1/-1 | 1 | | Antonio Navarro Perez | 1 | +1/-1 | 1 | | Adin Schmahmann | 1 | +1/-1 | 1 | | Lucas Molas | 1 | +1/-0 | 1 | ================================================ FILE: docs/changelogs/v0.16.md ================================================ # Kubo changelog v0.16 ## v0.16.0 ### Overview Below is an outline of all that is in this release, so you get a sense of all that's included. - [Kubo changelog v0.16](#kubo-changelog-v016) - [v0.16.0](#v0160) - [Overview](#overview) - [🔦 Highlights](#-highlights) - [🛣️ More configurable delegated routing system](#️-more-configurable-delegated-routing-system) - [🌍 WebTransport new experimental Transport](#-webtransport-new-experimental-transport) - [🗃️ Hardened IPNS record verification](#-hardened-ipns-record-verification) - [🌉 Web Gateways now support _redirects files](#-web-gateways-now-support-_redirects-files) - [😻 Add files to MFS with ipfs add --to-files](#-add-files-to-mfs-with-ipfs-add---to-files) - [Changelog](#changelog) - [Contributors](#contributors) ### 🔦 Highlights #### 🛣️ More configurable delegated routing system Since Kubo v0.14.0 [Reframe protocol](https://github.com/ipfs/specs/tree/main/reframe#readme) has been supported as a new routing system. Now, we allow to configure several routers working together, so you can have several `reframe` and `dht` routers making queries. You can use the special `parallel` and `sequential` routers to fill your needs. Example configuration usage using the [Filecoin Network Indexer](https://docs.cid.contact/filecoin-network-indexer/overview) and the DHT, making first a query to the indexer, and timing out after 3 seconds. ```console $ ipfs config Routing.Type --json '"custom"' $ ipfs config Routing.Routers.CidContact --json '{ "Type": "reframe", "Parameters": { "Endpoint": "https://cid.contact/reframe" } }' $ ipfs config Routing.Routers.WanDHT --json '{ "Type": "dht", "Parameters": { "Mode": "auto", "PublicIPNetwork": true, "AcceleratedDHTClient": false } }' $ ipfs config Routing.Routers.ParallelHelper --json '{ "Type": "parallel", "Parameters": { "Routers": [ { "RouterName" : "CidContact", "IgnoreErrors" : true, "Timeout": "3s" }, { "RouterName" : "WanDHT", "IgnoreErrors" : false, "Timeout": "5m", "ExecuteAfter": "2s" } ] } }' $ ipfs config Routing.Methods --json '{ "find-peers": { "RouterName": "ParallelHelper" }, "find-providers": { "RouterName": "ParallelHelper" }, "get-ipns": { "RouterName": "ParallelHelper" }, "provide": { "RouterName": "WanDHT" }, "put-ipns": { "RouterName": "ParallelHelper" } }' ``` ### 🌍 WebTransport new experimental Transport A new feature of [`go-libp2p`](https://github.com/libp2p/go-libp2p/releases/tag/v0.23.0) is [WebTransport](https://github.com/libp2p/go-libp2p/issues/1717). For now it is **disabled by default** and considered **experimental**. If you find issues running it please [report them to us](https://github.com/ipfs/kubo/issues/new). In the future Kubo will listen on WebTransport by default for anyone already listening on QUIC addresses. WebTransport is a new transport protocol currently under development by the [IETF](https://datatracker.ietf.org/wg/webtrans/about/) and the [W3C](https://www.w3.org/TR/webtransport/), and [already implemented by Chrome](https://caniuse.com/webtransport). Conceptually, it’s like WebSocket run over QUIC instead of TCP. Most importantly, it allows browsers to establish (secure!) connections to WebTransport servers without the need for CA-signed certificates, thereby enabling any js-libp2p node running in a browser to connect to any kubo node, with zero manual configuration involved. The previous alternative is websocket secure, which require installing a reverse proxy and TLS certificates manually. #### How to enable WebTransport Those steps are temporary and won't be needed once we make it enabled by default. 1. Enable the WebTransport transport: `ipfs config Swarm.Transports.Network.WebTransport --json true` 1. Add a listener address for WebTransport to your `Addresses.Swarm` key, for example: ```json [ "/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/udp/4001/quic", "/ip4/0.0.0.0/udp/4002/quic/webtransport" ] ``` 1. Restart your daemon to apply the config changes. ### 🗃️ Hardened IPNS record verification Records that do not have a valid IPNS V2 signature, or exceed the max size limit, will no longer pass verification, and will be ignored by Kubo when resolving `/ipns/{libp2p-key}` content paths. Kubo continues publishing backward-compatible V1+V2 records that can be resolved by V1-only (go-ipfs <0.9.0) clients. More details can be found in _Backward Compatibility_, _Record Creation_, and _Record Verification_ sections of the [updated IPNS specification](https://github.com/ipfs/specs/pull/319/files). ### 🌉 Web Gateways now support `_redirects` files This feature enables support for redirects, single-page applications (SPA), custom 404 pages, and moving to IPFS-backed website hosting [without breaking existing HTTP links](https://www.w3.org/Provider/Style/URI). It is limited to websites hosted in web contexts with unique [Origins](https://en.wikipedia.org/wiki/Same-origin_policy), such as [subdomain](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) and [DNSLink](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#dnslink-gateway) gateways. Redirect logic is evaluated only if the requested path is not in the DAG. See more details and usage examples see [docs.ipfs.tech: _Redirects, custom 404s, and SPA support_](https://docs.ipfs.tech/how-to/websites-on-ipfs/redirects-and-custom-404s/). ### 😻 Add files to MFS with `ipfs add --to-files` Users no longer need to call `ipfs files cp` after `ipfs add` to create a reference in [MFS](https://docs.ipfs.tech/concepts/glossary/#mfs), or deal with low level pins if they do not wish to do so. It is now possible to pass MFS path in an optional `--to-files` to add data directly to MFS, without creating a low level pin. Before (Kubo <0.16.0): ```console $ ipfs add cat.jpg QmCID $ ipfs files cp /ipfs/QmCID /mfs-cats/cat.jpg $ ipfs pin rm QmCID # removing low level pin, since MFS is protecting from gc ``` Kubo 0.16.0 collapses the above steps into one: ```console $ ipfs add --pin=false cat.jpg --to-files /mfs-cats/ ``` A recursive add to MFS works too (below line will create `/lots-of-cats/` directory in MFS): ```console $ ipfs add -r ./lots-of-cats/ --to-files / ``` For more information, see `ipfs add --help` and `ipfs files --help`. ### Changelog
Full Changelog - github.com/ipfs/kubo: - fix: Set default Methods value to nil - docs: document remaining 0.16.0 features - docs: add WebTransport docs ([ipfs/kubo#9308](https://github.com/ipfs/kubo/pull/9308)) - chore: bump version to 0.16.0-rc1 - fix: ensure hasher is registered when using a hashing function - feat: add webtransport as an option transport ([ipfs/kubo#9293](https://github.com/ipfs/kubo/pull/9293)) - feat(gateway): _redirects file support (#8890) ([ipfs/kubo#8890](https://github.com/ipfs/kubo/pull/8890)) - docs: fix typo in changelog-v0.16.0.md - Readme: Rewrite introduction and featureset (#9211) ([ipfs/kubo#9211](https://github.com/ipfs/kubo/pull/9211)) - feat: Delegated routing with custom configuration. (#9274) ([ipfs/kubo#9274](https://github.com/ipfs/kubo/pull/9274)) - Add to `ipfs id -h` options (#9229) ([ipfs/kubo#9229](https://github.com/ipfs/kubo/pull/9229)) - chore: bump go-libp2p v0.23.1 ([ipfs/kubo#9285](https://github.com/ipfs/kubo/pull/9285)) - feat(cmds/add): --to-files option automates files cp (#8927) ([ipfs/kubo#8927](https://github.com/ipfs/kubo/pull/8927)) - docs: fix broken ENS DoH example (#9281) ([ipfs/kubo#9281](https://github.com/ipfs/kubo/pull/9281)) - ([ipfs/kubo#9258](https://github.com/ipfs/kubo/pull/9258)) - ([ipfs/kubo#9213](https://github.com/ipfs/kubo/pull/9213)) - docs: small typo in Dockerfile - feat: ipfs-webui v2.18.1 - feat: ipfs-webui v2.18.0 (#9262) ([ipfs/kubo#9262](https://github.com/ipfs/kubo/pull/9262)) - bump go-libp2p v0.22.0 & go1.18&go1.19 ([ipfs/kubo#9244](https://github.com/ipfs/kubo/pull/9244)) - docs: change windows choco install command to point to go-ipfs - fix: pass the repo directory into the ignored_commit function - docs(cmds): daemon: update DHTClient description - fix(gw): send 200 for empty files - docs(readme): official vs unofficial packages - chore: remove Gateway.PathPrefixes - docs(readme): update Docker section - docs: fix markdown syntax typo in v0.15's changelog - chore: Release v0.15.0 ([ipfs/kubo#9236](https://github.com/ipfs/kubo/pull/9236)) - chore: fix undiallable api and gateway files ([ipfs/kubo#9233](https://github.com/ipfs/kubo/pull/9233)) - chore: Bump version to 0.16.0-dev - github.com/ipfs/go-bitswap (v0.9.0 -> v0.10.2): - chore: release v0.10.2 - fix: create a copy of the protocol slice in network.processSettings - chore: release v0.10.1 - fix: incorrect type in the WithTracer polyfill option - chore: fix incorrect log message when a bad option is passed - chore: release v0.10.0 - chore: update go-libp2p v0.22.0 - github.com/ipfs/go-cid (v0.2.0 -> v0.3.2): - chore: release v0.3.2 - Revert "fix: bring back, but deprecate CodecToStr and Codecs" - chore: release v0.2.1 - fix: bring back, but deprecate CodecToStr and Codecs - run gofmt -s - bump go.mod to Go 1.18 and run go fix - chore: release v0.3.0 - fix: return nil Bytes() if the Cid in undef - Add MustParse ([ipfs/go-cid#139](https://github.com/ipfs/go-cid/pull/139)) - github.com/ipfs/go-datastore (v0.5.1 -> v0.6.0): - Release v0.6.0 (#194) ([ipfs/go-datastore#194](https://github.com/ipfs/go-datastore/pull/194)) - feat: add Features + datastore scoping - chore: Fix comment typo (#191) ([ipfs/go-datastore#191](https://github.com/ipfs/go-datastore/pull/191)) - github.com/ipfs/go-delegated-routing (v0.3.0 -> v0.6.0): - ([ipfs/go-delegated-routing#53](https://github.com/ipfs/go-delegated-routing/pull/53)) - ([ipfs/go-delegated-routing#52](https://github.com/ipfs/go-delegated-routing/pull/52)) - Release v0.5.2 (#50) ([ipfs/go-delegated-routing#50](https://github.com/ipfs/go-delegated-routing/pull/50)) - Fixed serialisation issue with multiadds (#49) ([ipfs/go-delegated-routing#49](https://github.com/ipfs/go-delegated-routing/pull/49)) - Upgrade to IPLD `0.18.0` - Release v0.5.0 (#47) ([ipfs/go-delegated-routing#47](https://github.com/ipfs/go-delegated-routing/pull/47)) - feat: use GET for FindProviders (#46) ([ipfs/go-delegated-routing#46](https://github.com/ipfs/go-delegated-routing/pull/46)) - Update provide to take an array of keys, per spec (#45) ([ipfs/go-delegated-routing#45](https://github.com/ipfs/go-delegated-routing/pull/45)) - ([ipfs/go-delegated-routing#44](https://github.com/ipfs/go-delegated-routing/pull/44)) - fix: upgrade edelweiss and rerun 'go generate' (#42) ([ipfs/go-delegated-routing#42](https://github.com/ipfs/go-delegated-routing/pull/42)) - ci: add check to ensure generated files are up-to-date (#41) ([ipfs/go-delegated-routing#41](https://github.com/ipfs/go-delegated-routing/pull/41)) - Add Provide RPC (#37) ([ipfs/go-delegated-routing#37](https://github.com/ipfs/go-delegated-routing/pull/37)) - upgrade to go-log/v2 (#34) ([ipfs/go-delegated-routing#34](https://github.com/ipfs/go-delegated-routing/pull/34)) - github.com/ipfs/go-ipns (v0.1.2 -> v0.3.0): - fix: require V2 signatures ([ipfs/go-ipns#41](https://github.com/ipfs/go-ipns/pull/41)) - update go-libp2p to v0.22.0, release v0.2.0 (#39) ([ipfs/go-ipns#39](https://github.com/ipfs/go-ipns/pull/39)) - use peer.IDFromBytes instead of peer.IDFromString (#38) ([ipfs/go-ipns#38](https://github.com/ipfs/go-ipns/pull/38)) - sync: update CI config files (#34) ([ipfs/go-ipns#34](https://github.com/ipfs/go-ipns/pull/34)) - github.com/ipfs/go-pinning-service-http-client (v0.1.1 -> v0.1.2): - chore: release v0.1.2 - fix: send up to nanosecond precision - refactor: cleanup Sprintf for Bearer token - sync: update CI config files ([ipfs/go-pinning-service-http-client#21](https://github.com/ipfs/go-pinning-service-http-client/pull/21)) - github.com/ipld/edelweiss (v0.1.4 -> v0.2.0): - Release v0.2.0 (#60) ([ipld/edelweiss#60](https://github.com/ipld/edelweiss/pull/60)) - feat: add cacheable modifier to methods (#48) ([ipld/edelweiss#48](https://github.com/ipld/edelweiss/pull/48)) - adding licenses (#52) ([ipld/edelweiss#52](https://github.com/ipld/edelweiss/pull/52)) - sync: update CI config files ([ipld/edelweiss#56](https://github.com/ipld/edelweiss/pull/56)) - chore: replace deprecated ioutil with io/os ([ipld/edelweiss#59](https://github.com/ipld/edelweiss/pull/59)) - Release v0.1.6 - fix: iterate over BlueMap in deterministic order (#57) ([ipld/edelweiss#57](https://github.com/ipld/edelweiss/pull/57)) - fix: wrap DAG-JSON serialization error (#55) ([ipld/edelweiss#55](https://github.com/ipld/edelweiss/pull/55)) - update examples and harness - upgrade to go-log/v2 (#53) ([ipld/edelweiss#53](https://github.com/ipld/edelweiss/pull/53)) - github.com/ipld/go-ipld-prime (v0.17.0 -> v0.18.0): - Prepare v0.18.0 - feat(bindnode): add a BindnodeRegistry utility (#437) ([ipld/go-ipld-prime#437](https://github.com/ipld/go-ipld-prime/pull/437)) - feat(bindnode): support full uint64 range - chore(bindnode): remove typed functions for options - chore(bindnode): docs and minor tweaks - feat(bindnode): make Any converters work for List and Map values - fix(bindnode): shorten converter option names, minor perf improvements - fix(bindnode): only custom convert AssignNull for Any converter - feat(bindnode): pass Null on to nullable custom converters - chore(bindnode): config helper refactor w/ short-circuit - feat(bindnode): add AddCustomTypeAnyConverter() to handle `Any` fields - feat(bindnode): add AddCustomTypeXConverter() options for most scalar kinds - chore(bindnode): back out of reflection for converters - feat(bindnode): switch to converter functions instead of type - feat(bindnode): allow custom type conversions with options - feat: add release checklist (#442) ([ipld/go-ipld-prime#442](https://github.com/ipld/go-ipld-prime/pull/442)) - github.com/libp2p/go-flow-metrics (v0.0.3 -> v0.1.0): - introduce an API to set a mock clock (#20) ([libp2p/go-flow-metrics#20](https://github.com/libp2p/go-flow-metrics/pull/20)) - chore: skip slow tests when short testing is specified ([libp2p/go-flow-metrics#16](https://github.com/libp2p/go-flow-metrics/pull/16)) - github.com/libp2p/go-libp2p (v0.21.0 -> v0.23.2): - release v0.23.2 (#1781) ([libp2p/go-libp2p#1781](https://github.com/libp2p/go-libp2p/pull/1781)) - webtransport: return error before wrapping opened / accepted streams (#1775) ([libp2p/go-libp2p#1775](https://github.com/libp2p/go-libp2p/pull/1775)) - release v0.23.1 (#1773) ([libp2p/go-libp2p#1773](https://github.com/libp2p/go-libp2p/pull/1773)) - websocket: fix nil pointer in tlsClientConf (#1770) ([libp2p/go-libp2p#1770](https://github.com/libp2p/go-libp2p/pull/1770)) - release v0.23.0 (#1764) ([libp2p/go-libp2p#1764](https://github.com/libp2p/go-libp2p/pull/1764)) - noise: switch to proto2, use the new NoiseExtensions protobuf ([libp2p/go-libp2p#1762](https://github.com/libp2p/go-libp2p/pull/1762)) - webtransport: add custom resolver to add SNI (#1761) ([libp2p/go-libp2p#1761](https://github.com/libp2p/go-libp2p/pull/1761)) - swarm: skip dialing WebTransport addresses when we have QUIC addresses (#1756) ([libp2p/go-libp2p#1756](https://github.com/libp2p/go-libp2p/pull/1756)) - webtransport: have the server send the certificates (#1757) ([libp2p/go-libp2p#1757](https://github.com/libp2p/go-libp2p/pull/1757)) - noise: make it possible for the server to send early data (#1750) ([libp2p/go-libp2p#1750](https://github.com/libp2p/go-libp2p/pull/1750)) - swarm: fix selection of transport for dialing (#1653) ([libp2p/go-libp2p#1653](https://github.com/libp2p/go-libp2p/pull/1653)) - autorelay: Add a context.Context to WithPeerSource callback (#1736) ([libp2p/go-libp2p#1736](https://github.com/libp2p/go-libp2p/pull/1736)) - webtransport: add and check the ?type=noise URL parameter (#1749) ([libp2p/go-libp2p#1749](https://github.com/libp2p/go-libp2p/pull/1749)) - webtransport: disable HTTP origin check (#1752) ([libp2p/go-libp2p#1752](https://github.com/libp2p/go-libp2p/pull/1752)) - noise: don't fail handshake when early data is received without handler (#1746) ([libp2p/go-libp2p#1746](https://github.com/libp2p/go-libp2p/pull/1746)) - Add Resolver interface to transport (#1719) ([libp2p/go-libp2p#1719](https://github.com/libp2p/go-libp2p/pull/1719)) - use new /libp2p/go-libp2p/core pkg (#1745) ([libp2p/go-libp2p#1745](https://github.com/libp2p/go-libp2p/pull/1745)) - yamux: pass constructors for peer resource scopes to session constructor (#1739) ([libp2p/go-libp2p#1739](https://github.com/libp2p/go-libp2p/pull/1739)) - tcp: add an option to enable metrics (disabled by default) (#1734) ([libp2p/go-libp2p#1734](https://github.com/libp2p/go-libp2p/pull/1734)) - move go-libp2p-webtransport to p2p/transport/webtransport ([libp2p/go-libp2p#1737](https://github.com/libp2p/go-libp2p/pull/1737)) - autorelay: fix race condition in TestBackoff (#1731) ([libp2p/go-libp2p#1731](https://github.com/libp2p/go-libp2p/pull/1731)) - rcmgr: increase default connection memory limit to 32 MB (#1740) ([libp2p/go-libp2p#1740](https://github.com/libp2p/go-libp2p/pull/1740)) - quic: update quic-go to v0.29.0 (#1723) ([libp2p/go-libp2p#1723](https://github.com/libp2p/go-libp2p/pull/1723)) - noise: implement an API to send and receive early data ([libp2p/go-libp2p#1728](https://github.com/libp2p/go-libp2p/pull/1728)) - identify: make the protocol version configurable (#1724) ([libp2p/go-libp2p#1724](https://github.com/libp2p/go-libp2p/pull/1724)) - Fix threshold calculation (#1722) ([libp2p/go-libp2p#1722](https://github.com/libp2p/go-libp2p/pull/1722)) - connmgr: use clock interface (#1720) ([libp2p/go-libp2p#1720](https://github.com/libp2p/go-libp2p/pull/1720)) - quic: increase the buffer size used for encoding qlogs (#1715) ([libp2p/go-libp2p#1715](https://github.com/libp2p/go-libp2p/pull/1715)) - quic: add a WithMetrics option (#1716) ([libp2p/go-libp2p#1716](https://github.com/libp2p/go-libp2p/pull/1716)) - add default listen addresses for QUIC (#1615) ([libp2p/go-libp2p#1615](https://github.com/libp2p/go-libp2p/pull/1615)) - feat: inject DNS resolver (#1607) ([libp2p/go-libp2p#1607](https://github.com/libp2p/go-libp2p/pull/1607)) - connmgr: prefer peers with no streams when closing connections (#1675) ([libp2p/go-libp2p#1675](https://github.com/libp2p/go-libp2p/pull/1675)) - quic: add DisableReuseport option (#1476) ([libp2p/go-libp2p#1476](https://github.com/libp2p/go-libp2p/pull/1476)) - release v0.22.0 ([libp2p/go-libp2p#1688](https://github.com/libp2p/go-libp2p/pull/1688)) - fix: don't prefer local ports from other addresses when dialing (#1673) ([libp2p/go-libp2p#1673](https://github.com/libp2p/go-libp2p/pull/1673)) - crypto: add better support for alternative backends (#1686) ([libp2p/go-libp2p#1686](https://github.com/libp2p/go-libp2p/pull/1686)) - crypto/secp256k1: Remove btcsuite intermediary. (#1689) ([libp2p/go-libp2p#1689](https://github.com/libp2p/go-libp2p/pull/1689)) - Update resource manager README (#1684) ([libp2p/go-libp2p#1684](https://github.com/libp2p/go-libp2p/pull/1684)) - move go-libp2p-core here ([libp2p/go-libp2p#1683](https://github.com/libp2p/go-libp2p/pull/1683)) - rcmgr: make scaling changes more intuitive (#1685) ([libp2p/go-libp2p#1685](https://github.com/libp2p/go-libp2p/pull/1685)) - move go-eventbus here ([libp2p/go-libp2p#1681](https://github.com/libp2p/go-libp2p/pull/1681)) - basichost: remove usage of MultistreamServerMatcher in test (#1680) ([libp2p/go-libp2p#1680](https://github.com/libp2p/go-libp2p/pull/1680)) - sync: update CI config files (#1678) ([libp2p/go-libp2p#1678](https://github.com/libp2p/go-libp2p/pull/1678)) - move go-libp2p-resource-manager to p2p/host/resource-manager ([libp2p/go-libp2p#1677](https://github.com/libp2p/go-libp2p/pull/1677)) - chore: preallocate slices with known final size (#1679) ([libp2p/go-libp2p#1679](https://github.com/libp2p/go-libp2p/pull/1679)) - autorelay: fix flaky TestMaxAge (#1676) ([libp2p/go-libp2p#1676](https://github.com/libp2p/go-libp2p/pull/1676)) - move go-libp2p-peerstore to p2p/host/peerstore ([libp2p/go-libp2p#1667](https://github.com/libp2p/go-libp2p/pull/1667)) - examples: remove ipfs components from echo (#1672) ([libp2p/go-libp2p#1672](https://github.com/libp2p/go-libp2p/pull/1672)) - chore: update libp2p to v0.21 in examples (#1674) ([libp2p/go-libp2p#1674](https://github.com/libp2p/go-libp2p/pull/1674)) - change the default key type to Ed25519 (#1576) ([libp2p/go-libp2p#1576](https://github.com/libp2p/go-libp2p/pull/1576)) - autorelay: poll for new candidates when needed ([libp2p/go-libp2p#1587](https://github.com/libp2p/go-libp2p/pull/1587)) - examples: fix unresponsive pubsub chat example (#1652) ([libp2p/go-libp2p#1652](https://github.com/libp2p/go-libp2p/pull/1652)) - routed: respect force direct dial context (#1665) ([libp2p/go-libp2p#1665](https://github.com/libp2p/go-libp2p/pull/1665)) - pstoremanager: fix flaky TestClose (#1649) ([libp2p/go-libp2p#1649](https://github.com/libp2p/go-libp2p/pull/1649)) - Allow adding prologue to noise connections (#1663) ([libp2p/go-libp2p#1663](https://github.com/libp2p/go-libp2p/pull/1663)) - connmgr: add nowatchdog go build tag (#1666) ([libp2p/go-libp2p#1666](https://github.com/libp2p/go-libp2p/pull/1666)) - mdns: don't discover ourselves (#1661) ([libp2p/go-libp2p#1661](https://github.com/libp2p/go-libp2p/pull/1661)) - Support generating custom x509 certificates (#1481) ([libp2p/go-libp2p#1481](https://github.com/libp2p/go-libp2p/pull/1481)) - github.com/libp2p/go-libp2p-core (v0.19.1 -> v0.20.1): - chore: release v0.20.1 - feat: forward crypto/pb - release v0.20.0 - deprecate this repo - stop using the deprecated io/ioutil package (#279) ([libp2p/go-libp2p-core#279](https://github.com/libp2p/go-libp2p-core/pull/279)) - use a mock clock in bandwidth tests (#276) ([libp2p/go-libp2p-core#276](https://github.com/libp2p/go-libp2p-core/pull/276)) - remove unused MultistreamSemverMatcher (#277) ([libp2p/go-libp2p-core#277](https://github.com/libp2p/go-libp2p-core/pull/277)) - remove peer.IDFromString (#274) ([libp2p/go-libp2p-core#274](https://github.com/libp2p/go-libp2p-core/pull/274)) - deprecate peer.Encode in favor of peer.ID.String (#275) ([libp2p/go-libp2p-core#275](https://github.com/libp2p/go-libp2p-core/pull/275)) - deprecate peer.ID.Pretty (#273) ([libp2p/go-libp2p-core#273](https://github.com/libp2p/go-libp2p-core/pull/273)) - github.com/libp2p/go-libp2p-kad-dht (v0.17.0 -> v0.18.0): - update go-libp2p to v0.22.0, release v0.18.0 ([libp2p/go-libp2p-kad-dht#788](https://github.com/libp2p/go-libp2p-kad-dht/pull/788)) - sync: update CI config files (#789) ([libp2p/go-libp2p-kad-dht#789](https://github.com/libp2p/go-libp2p-kad-dht/pull/789)) - github.com/libp2p/go-libp2p-peerstore (v0.7.1 -> v0.8.0): - release v0.8.0 - deprecate this repo - fix flaky TestGCDelay (#206) ([libp2p/go-libp2p-peerstore#206](https://github.com/libp2p/go-libp2p-peerstore/pull/206)) - fix flaky EWMA test (#205) ([libp2p/go-libp2p-peerstore#205](https://github.com/libp2p/go-libp2p-peerstore/pull/205)) - github.com/libp2p/go-libp2p-record (v0.1.3 -> v0.2.0): - update go-libp2p to v0.22.0, release v0.2.0 ([libp2p/go-libp2p-record#50](https://github.com/libp2p/go-libp2p-record/pull/50)) - sync: update CI config files (#47) ([libp2p/go-libp2p-record#47](https://github.com/libp2p/go-libp2p-record/pull/47)) - increase RSA key sizes in tests ([libp2p/go-libp2p-record#44](https://github.com/libp2p/go-libp2p-record/pull/44)) - cleanup: fix staticcheck failures ([libp2p/go-libp2p-record#43](https://github.com/libp2p/go-libp2p-record/pull/43)) - github.com/libp2p/go-libp2p-routing-helpers (v0.2.3 -> v0.4.0): - ([libp2p/go-libp2p-routing-helpers#62](https://github.com/libp2p/go-libp2p-routing-helpers/pull/62)) - ([libp2p/go-libp2p-routing-helpers#58](https://github.com/libp2p/go-libp2p-routing-helpers/pull/58)) - Update version.json ([libp2p/go-libp2p-routing-helpers#60](https://github.com/libp2p/go-libp2p-routing-helpers/pull/60)) - update go-libp2p to v0.22.0 ([libp2p/go-libp2p-routing-helpers#59](https://github.com/libp2p/go-libp2p-routing-helpers/pull/59)) - sync: update CI config files (#53) ([libp2p/go-libp2p-routing-helpers#53](https://github.com/libp2p/go-libp2p-routing-helpers/pull/53)) - fix staticcheck ([libp2p/go-libp2p-routing-helpers#49](https://github.com/libp2p/go-libp2p-routing-helpers/pull/49)) - fix error handling in Parallel.search ([libp2p/go-libp2p-routing-helpers#48](https://github.com/libp2p/go-libp2p-routing-helpers/pull/48)) - github.com/libp2p/go-libp2p-testing (v0.11.0 -> v0.12.0): - release v0.12.0 (#67) ([libp2p/go-libp2p-testing#67](https://github.com/libp2p/go-libp2p-testing/pull/67)) - chore: update to go-libp2p v0.22.0 (#66) ([libp2p/go-libp2p-testing#66](https://github.com/libp2p/go-libp2p-testing/pull/66)) - remove the resource manager mocks (#65) ([libp2p/go-libp2p-testing#65](https://github.com/libp2p/go-libp2p-testing/pull/65)) - github.com/libp2p/go-openssl (v0.0.7 -> v0.1.0): - release v0.1.0 (#31) ([libp2p/go-openssl#31](https://github.com/libp2p/go-openssl/pull/31)) - Fix build with OpenSSL 3.0 (#25) ([libp2p/go-openssl#25](https://github.com/libp2p/go-openssl/pull/25)) - sync: update CI config files ([libp2p/go-openssl#24](https://github.com/libp2p/go-openssl/pull/24)) - Add openssl.DialTimeout(network, addr, timeout, ctx, flags) call ([libp2p/go-openssl#26](https://github.com/libp2p/go-openssl/pull/26)) - Add Ctx.SetMinProtoVersion and Ctx.SetMaxProtoVersion wrappers ([libp2p/go-openssl#27](https://github.com/libp2p/go-openssl/pull/27)) - sync: update CI config files ([libp2p/go-openssl#17](https://github.com/libp2p/go-openssl/pull/17)) - fix: unsafe pointer passing ([libp2p/go-openssl#18](https://github.com/libp2p/go-openssl/pull/18)) - Update test RSA cert ([libp2p/go-openssl#15](https://github.com/libp2p/go-openssl/pull/15)) - Fix tests ([libp2p/go-openssl#16](https://github.com/libp2p/go-openssl/pull/16)) - Address `staticcheck` issues ([libp2p/go-openssl#14](https://github.com/libp2p/go-openssl/pull/14)) - Enabled PEM files with CRLF line endings to be used (#10) ([libp2p/go-openssl#11](https://github.com/libp2p/go-openssl/pull/11)) - github.com/libp2p/zeroconf/v2 (v2.1.1 -> v2.2.0): - Fix windows libp2p (#29) ([libp2p/zeroconf#29](https://github.com/libp2p/zeroconf/pull/29)) - Fix compatibility with some IoT devices using avahi 0.8-rc1 (#27) ([libp2p/zeroconf#27](https://github.com/libp2p/zeroconf/pull/27)) - Add TTL server option (#23) ([libp2p/zeroconf#23](https://github.com/libp2p/zeroconf/pull/23)) - github.com/lucas-clemente/quic-go (v0.28.0 -> v0.29.1): - http3: fix double close of chan when using DontCloseRequestStream - add a logging.NullTracer and logging.NullConnectionTracer ([lucas-clemente/quic-go#3512](https://github.com/lucas-clemente/quic-go/pull/3512)) - add support for providing a custom Connection ID generator via Config (#3452) ([lucas-clemente/quic-go#3452](https://github.com/lucas-clemente/quic-go/pull/3452)) - fix typo in README - fix datagram support detection (#3511) ([lucas-clemente/quic-go#3511](https://github.com/lucas-clemente/quic-go/pull/3511)) - use a single Go routine to send copies of CONNECTION_CLOSE packets ([lucas-clemente/quic-go#3514](https://github.com/lucas-clemente/quic-go/pull/3514)) - add YoMo to list of projects in README (#3513) ([lucas-clemente/quic-go#3513](https://github.com/lucas-clemente/quic-go/pull/3513)) - http3: fix listening on both QUIC and TCP (#3465) ([lucas-clemente/quic-go#3465](https://github.com/lucas-clemente/quic-go/pull/3465)) - Disable anti-amplification limit by address validation token (#3326) ([lucas-clemente/quic-go#3326](https://github.com/lucas-clemente/quic-go/pull/3326)) - fix typo in README - implement a new API to let servers control client address verification ([lucas-clemente/quic-go#3501](https://github.com/lucas-clemente/quic-go/pull/3501)) - use a generic streams map for incoming streams ([lucas-clemente/quic-go#3489](https://github.com/lucas-clemente/quic-go/pull/3489)) - fix unreachable code after log.Fatal in fuzzing corpus generator (#3496) ([lucas-clemente/quic-go#3496](https://github.com/lucas-clemente/quic-go/pull/3496)) - use generic Min and Max functions ([lucas-clemente/quic-go#3483](https://github.com/lucas-clemente/quic-go/pull/3483)) - add QPACK (RFC 9204) to the list of supported RFCs (#3485) ([lucas-clemente/quic-go#3485](https://github.com/lucas-clemente/quic-go/pull/3485)) - add a function to distinguish between long and short header packets (#3498) ([lucas-clemente/quic-go#3498](https://github.com/lucas-clemente/quic-go/pull/3498)) - use a generic streams map for outgoing streams (#3488) ([lucas-clemente/quic-go#3488](https://github.com/lucas-clemente/quic-go/pull/3488)) - update golangci-lint action to v3, golangci-lint to v1.48.0 (#3499) ([lucas-clemente/quic-go#3499](https://github.com/lucas-clemente/quic-go/pull/3499)) - use a generic linked list (#3487) ([lucas-clemente/quic-go#3487](https://github.com/lucas-clemente/quic-go/pull/3487)) - drop support for Go 1.16 and 1.17 (#3482) ([lucas-clemente/quic-go#3482](https://github.com/lucas-clemente/quic-go/pull/3482)) - optimize FirstOutstanding in the sent packet history (#3467) ([lucas-clemente/quic-go#3467](https://github.com/lucas-clemente/quic-go/pull/3467)) - update supported RFCs in README (#3456) ([lucas-clemente/quic-go#3456](https://github.com/lucas-clemente/quic-go/pull/3456)) - http3: ignore context after response when using DontCloseRequestStream (#3473) ([lucas-clemente/quic-go#3473](https://github.com/lucas-clemente/quic-go/pull/3473)) - github.com/marten-seemann/webtransport-go (null -> v0.1.1): - release v0.1.1 (#31) ([marten-seemann/webtransport-go#31](https://github.com/marten-seemann/webtransport-go/pull/31)) - fix double close of chan when using DontCloseRequestStream - github.com/multiformats/go-base32 (v0.0.4 -> v0.1.0): - chore: bump version to 0.1.0 - fix: fix staticcheck complaints - run gofmt -s - sync: update CI config files (#5) ([multiformats/go-base32#5](https://github.com/multiformats/go-base32/pull/5)) - github.com/multiformats/go-multiaddr (v0.6.0 -> v0.7.0): - Release v0.7.0 ([multiformats/go-multiaddr#183](https://github.com/multiformats/go-multiaddr/pull/183)) - use decimal numbers for multicodecs ([multiformats/go-multiaddr#184](https://github.com/multiformats/go-multiaddr/pull/184)) - Fix comment on Decapsulate ([multiformats/go-multiaddr#181](https://github.com/multiformats/go-multiaddr/pull/181)) - ([multiformats/go-multiaddr#182](https://github.com/multiformats/go-multiaddr/pull/182)) - sync: update CI config files (#180) ([multiformats/go-multiaddr#180](https://github.com/multiformats/go-multiaddr/pull/180)) - Add webrtc (#179) ([multiformats/go-multiaddr#179](https://github.com/multiformats/go-multiaddr/pull/179)) - github.com/multiformats/go-multicodec (v0.5.0 -> v0.6.0): - chore: version bump 0.6.0 - fix: replace io/ioutil with io - bump go.mod to Go 1.18 and run go fix
### Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Marten Seemann | 236 | +12637/-24326 | 1152 | | Raúl Kripalani | 118 | +11626/-4136 | 422 | | vyzo | 144 | +10129/-3665 | 230 | | galargh | 9 | +5293/-5298 | 26 | | Marco Munizaga | 83 | +7502/-3080 | 147 | | Antonio Navarro Perez | 33 | +4074/-1240 | 78 | | Steven Allen | 98 | +1974/-1693 | 202 | | Cole Brown | 57 | +2169/-1338 | 95 | | Rod Vagg | 21 | +2588/-768 | 56 | | Gus Eggert | 16 | +2011/-1226 | 36 | | Yusef Napora | 6 | +2738/-187 | 43 | | Raúl Kripalani | 2 | +1000/-889 | 18 | | Łukasz Magiera | 26 | +1312/-500 | 54 | | Will | 2 | +1593/-200 | 18 | | Jorropo | 31 | +924/-712 | 204 | | Juan Batiz-Benet | 2 | +1531/-9 | 21 | | Jeromy | 14 | +691/-468 | 51 | | Petar Maymounkov | 4 | +469/-285 | 25 | | Jeromy Johnson | 24 | +474/-204 | 116 | | Justin Johnson | 1 | +582/-93 | 7 | | Aarsh Shah | 24 | +377/-105 | 34 | | web3-bot | 18 | +246/-228 | 93 | | Masih H. Derkani | 2 | +197/-213 | 21 | | Marcin Rataj | 9 | +211/-176 | 16 | | adam | 4 | +235/-49 | 9 | | Jakub Sztandera | 9 | +203/-73 | 13 | | Guilhem Fanton | 1 | +216/-48 | 5 | | Lucas Molas | 1 | +219/-9 | 3 | | Peter Argue | 1 | +166/-36 | 3 | | Vibhav Pant | 4 | +186/-12 | 7 | | Adrian Lanzafame | 3 | +180/-16 | 5 | | Lars Gierth | 5 | +151/-41 | 25 | | João Oliveirinha | 1 | +124/-38 | 11 | | dignifiedquire | 3 | +122/-33 | 6 | | Chinmay Kousik | 2 | +128/-4 | 7 | | Toby | 1 | +89/-36 | 4 | | Oleg Jukovec | 3 | +111/-14 | 8 | | Whyrusleeping | 2 | +120/-0 | 6 | | KevinZønda | 1 | +81/-20 | 2 | | wzp | 2 | +86/-3 | 2 | | Benedikt Spies | 1 | +75/-12 | 8 | | nisainan | 1 | +33/-43 | 12 | | Tshaka Eric Lekholoane | 1 | +57/-19 | 6 | | cpuchip | 1 | +65/-6 | 2 | | Roman Proskuryakov | 2 | +69/-0 | 2 | | Arceliar | 2 | +36/-28 | 2 | | Maxim Merzhanov | 1 | +29/-24 | 1 | | Richard Ramos | 1 | +51/-0 | 2 | | Dave Collins | 1 | +25/-25 | 4 | | Leo Balduf | 2 | +37/-10 | 3 | | David Aronchick | 1 | +42/-0 | 3 | | Didrik Nordström | 1 | +35/-6 | 1 | | Vasco Santos | 1 | +20/-20 | 7 | | Jesse Bouwman | 1 | +19/-21 | 1 | | Ivan Schasny | 2 | +22/-14 | 4 | | MGMCN | 1 | +9/-24 | 2 | | Brian Meek | 1 | +14/-17 | 4 | | Ian Davis | 3 | +21/-9 | 5 | | Mars Zuo | 1 | +7/-18 | 1 | | RubenKelevra | 1 | +10/-10 | 1 | | mojatter | 1 | +9/-8 | 1 | | Cory Schwartz | 1 | +0/-17 | 1 | | Steve Loeppky | 6 | +7/-6 | 6 | | Matt Joiner | 2 | +10/-3 | 2 | | Winterhuman | 2 | +7/-5 | 2 | | Dmitry Yu Okunev | 1 | +5/-7 | 5 | | corverroos | 1 | +7/-4 | 2 | | Marcel Gregoriadis | 1 | +9/-0 | 1 | | Ignacio Hagopian | 2 | +7/-2 | 2 | | Julien Muret | 1 | +4/-4 | 2 | | Eclésio Junior | 1 | +8/-0 | 1 | | Stephan Eberle | 1 | +4/-3 | 1 | | muXxer | 1 | +3/-3 | 1 | | eth-limo | 1 | +3/-3 | 2 | | Russell Dempsey | 2 | +4/-2 | 2 | | Sergey | 1 | +1/-3 | 1 | | Jun10ng | 2 | +2/-2 | 2 | | Jorik Schellekens | 1 | +2/-2 | 1 | | Eli Wang | 1 | +2/-2 | 1 | | Andreas Linde | 1 | +4/-0 | 1 | | whyrusleeping | 1 | +2/-1 | 1 | | xiabin | 1 | +1/-1 | 1 | | star | 1 | +0/-2 | 1 | | fanweixiao | 1 | +1/-1 | 1 | | dbadoy4874 | 1 | +1/-1 | 1 | | bigs | 1 | +1/-1 | 1 | | Tarun Bansal | 1 | +1/-1 | 1 | | Mikerah | 1 | +1/-1 | 1 | | Mike Goelzer | 1 | +2/-0 | 1 | | Max Inden | 1 | +1/-1 | 1 | | Kevin Mai-Husan Chia | 1 | +1/-1 | 1 | | John B Nelson | 1 | +1/-1 | 1 | | Eli Bailey | 1 | +1/-1 | 1 | | Bryan Stenson | 1 | +1/-1 | 1 | | Alex Stokes | 1 | +1/-1 | 1 | | Abirdcfly | 1 | +1/-1 | 1 | ================================================ FILE: docs/changelogs/v0.17.md ================================================ # Kubo changelog v0.17 ## v0.17.0 ### Overview Below is an outline of all that is in this release, so you get a sense of all that's included. - [Kubo changelog v0.17](#kubo-changelog-v017) - [v0.17.0](#v0170) - [Overview](#overview) - [🔦 Highlights](#-highlights) - [libp2p resource management enabled by default](#libp2p-resource-management-enabled-by-default) - [Implicit connection manager limits](#implicit-connection-manager-limits) - [TAR Response Format on Gateways](#tar-response-format-on-gateways) - [Dialling `/wss` peer behind a reverse proxy](#dialling-wss-peer-behind-a-reverse-proxy) - [Changelog](#changelog) - [Contributors](#contributors) ### 🔦 Highlights #### libp2p resource management enabled by default To help protect nodes from DoS (resource exhaustion) and eclipse attacks, go-libp2p released a [Network Resource Manager](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager) with a host of improvements throughout 2022. Kubo first [exposed this functionality in Kubo 0.13](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.13.md#-libp2p-network-resource-manager-swarmresourcemgr), but it was disabled by default. The resource manager is now enabled by default to protect nodes. The defaults balance providing protection from various attacks while still enabling normal usecases to work as expected. If you want to adjust the defaults, then you can: 1. bound the amount of memory and file descriptors that libp2p will use with [Swarm.ResourceMgr.MaxMemory](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmresourcemgrmaxmemory) and [Swarm.ResourceMgr.MaxFileDescriptors](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmresourcemgrmaxfiledescriptors) and/or 2. override any specific resource scopes/limits with [Swarm.ResourceMgr.Limits](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmresourcemgrlimits) See [Swarm.ResourceMgr](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmresourcemgr) for 1. what limits are set by default, 2. example override configuration, 3. how to access Prometheus metrics and view Grafana dashboards of resource usage, and 4. how to set explicit "allow lists" to protect against eclipse attacks. #### Implicit connection manager limits Starting with this release, `ipfs init` will no longer store the default [Connection Manager](https://github.com/ipfs/kubo/blob/master/docs/config.md#swarmconnmgr) limits in the user config under `Swarm.ConnMgr`. Users are still free to use this setting to set custom values, but for most use cases, the defaults provided with the latest Kubo release should be sufficient. To remove any custom limits and switch to the implicit defaults managed by Kubo: ```console $ ipfs config --json Swarm.ConnMgr '{}' ``` We will be adjusting defaults in the future releases. #### TAR Response Format on Gateways Implemented [IPIP-288](https://github.com/ipfs/specs/pull/288) which adds support for requesting deserialized UnixFS directory as a TAR stream. HTTP clients can request TAR response by passing the `?format=tar` URL parameter, or setting `Accept: application/x-tar` HTTP header: ```console $ export DIR_CID=bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq $ curl -H "Accept: application/x-tar" "http://127.0.0.1:8080/ipfs/$DIR_CID" > dir.tar $ curl "http://127.0.0.1:8080/ipfs/$DIR_CID?format=tar" | tar xv bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq/1 - Barrel - Part 1 - alt.txt bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq/1 - Barrel - Part 1 - transcript.txt bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq/1 - Barrel - Part 1.png ``` #### Dialling `/wss` peer behind a reverse proxy This release resolves a regression introduced in Kubo 0.16, making it possible again to connect to a peer over a WebSockets endpoint (`/wss`) that is deployed behind a reverse proxy. More details in [go-libp2p release notes](https://github.com/libp2p/go-libp2p/releases/tag/v0.23.3). ### Changelog
Full Changelog - github.com/ipfs/kubo: - chore: bump version to v0.17.0 ([ipfs/kubo#9427](https://github.com/ipfs/kubo/pull/9427)) - chore: bump version to v0.17.0-rc2 ([ipfs/kubo#9414](https://github.com/ipfs/kubo/pull/9414)) - Doc improvements and changelog for resource manager (#9413) ([ipfs/kubo#9413](https://github.com/ipfs/kubo/pull/9413)) - fix(docs): typo - docs: document /wss fixes in 0.17 - refactor(config): remove Swarm.ConnMgr defaults - fix(config): skip nulls in ResourceMgr - Apply go fmt - Update core/node/libp2p/rcmgr_defaults.go - Remove limitation by HighWater param. - Fix RM errors when acceleratedDHT is active - docs: Deprecate Reframe on docs. (#9401) ([ipfs/kubo#9401](https://github.com/ipfs/kubo/pull/9401)) - chore: bump version to v0.17.0-rc1 ([ipfs/kubo#9394](https://github.com/ipfs/kubo/pull/9394)) - feat: Improve ResourceManager UX (#9338) ([ipfs/kubo#9338](https://github.com/ipfs/kubo/pull/9338)) - feat: ipfs-webui 2.20.0 - docs: note log tail is broken (#9383) ([ipfs/kubo#9383](https://github.com/ipfs/kubo/pull/9383)) - feat(gateway): TAR response format (#9029) ([ipfs/kubo#9029](https://github.com/ipfs/kubo/pull/9029)) - fix: error when using huge json limit file - chore: go-multicodec v0.7.0 - fix: remove old unused buggy coredag code - feat: Add command line completion for fish - chore: delete snap configuration ([ipfs/kubo#9352](https://github.com/ipfs/kubo/pull/9352)) - docs: update scoop package - docs: init release issue template improvement process v0.16.0 ([ipfs/kubo#9283](https://github.com/ipfs/kubo/pull/9283)) - feat: add delegated routing metrics (#9354) ([ipfs/kubo#9354](https://github.com/ipfs/kubo/pull/9354)) - chore: create v0.17.md changelog ([ipfs/kubo#9353](https://github.com/ipfs/kubo/pull/9353)) - docs: pin remote arg - feat: webui@v2.19.0 - test(car): export/import of (dag-)cbor/json codecs - add refs local alias repo ls (#9320) ([ipfs/kubo#9320](https://github.com/ipfs/kubo/pull/9320)) - docs(cmds): Clarify block fetching of refs endpoint. - chore(cmds): dag import: use ipld legacy decode ([ipfs/kubo#9219](https://github.com/ipfs/kubo/pull/9219)) - fix ipfs swarm peering crash in offline mode (#9261) ([ipfs/kubo#9261](https://github.com/ipfs/kubo/pull/9261)) - feat: remove provider delay interval in bitswap (#9053) ([ipfs/kubo#9053](https://github.com/ipfs/kubo/pull/9053)) - feat: --reset flag on swarm limit command (#9310) ([ipfs/kubo#9310](https://github.com/ipfs/kubo/pull/9310)) - fix: add InlineDNSLink flag to PublicGateways config (#9328) ([ipfs/kubo#9328](https://github.com/ipfs/kubo/pull/9328)) - docs: Fix typo and grammar in README - ci: add stylecheck to golangci-lint (#9334) ([ipfs/kubo#9334](https://github.com/ipfs/kubo/pull/9334)) - Fix: `swarm stats all` command - Merge release v0.16.0 back into master ([ipfs/kubo#9324](https://github.com/ipfs/kubo/pull/9324)) - fix: Set default Methods value to nil - docs: add WebTransport docs ([ipfs/kubo#9314](https://github.com/ipfs/kubo/pull/9314)) - chore: bump version to 0.17.0-dev - github.com/ipfs/go-delegated-routing (v0.6.0 -> v0.7.0): - Release v0.7.0 - feat: add latency & count metrics for content routing client (#59) ([ipfs/go-delegated-routing#59](https://github.com/ipfs/go-delegated-routing/pull/59)) - docs: add basic readme ([ipfs/go-delegated-routing#57](https://github.com/ipfs/go-delegated-routing/pull/57)) - sync: update CI config files ([ipfs/go-delegated-routing#40](https://github.com/ipfs/go-delegated-routing/pull/40)) - added link to reframe blog post (#54) ([ipfs/go-delegated-routing#54](https://github.com/ipfs/go-delegated-routing/pull/54)) - github.com/ipfs/go-ipfs-files (v0.1.1 -> v0.2.0): - Release v0.2.0 - fix: error when TAR has files outside of root (#56) ([ipfs/go-ipfs-files#56](https://github.com/ipfs/go-ipfs-files/pull/56)) - sync: update CI config files ([ipfs/go-ipfs-files#55](https://github.com/ipfs/go-ipfs-files/pull/55)) - chore(Directory): add DirIterator API restriction: iterate only once - github.com/ipfs/go-unixfs (v0.4.0 -> v0.4.1): - Update version.json - Fix: panic when childer is nil (#127) ([ipfs/go-unixfs#127](https://github.com/ipfs/go-unixfs/pull/127)) - sync: update CI config files (#125) ([ipfs/go-unixfs#125](https://github.com/ipfs/go-unixfs/pull/125)) - github.com/ipld/go-ipld-prime (v0.18.0 -> v0.19.0): - Prepare v0.19.0 - fix: correct json codec links & bytes handling - test(basicnode): increase test coverage for int and map types (#454) ([ipld/go-ipld-prime#454](https://github.com/ipld/go-ipld-prime/pull/454)) - fix: remove reliance on ioutil - run gofmt -s - bump go.mod to Go 1.18 and run go fix - feat: add kinded union to gendemo - github.com/libp2p/go-libp2p (v0.23.2 -> v0.23.4): - Release v0.23.4 (#1864) ([libp2p/go-libp2p#1864](https://github.com/libp2p/go-libp2p/pull/1864)) - release v0.23.3 - websocket: set the HTTP host header in WSS - github.com/libp2p/go-netroute (v0.2.0 -> v0.2.1): - v0.2.1 ([libp2p/go-netroute#27](https://github.com/libp2p/go-netroute/pull/27)) - fix(phys-addr-length): fix physical address length mismatch ([libp2p/go-netroute#29](https://github.com/libp2p/go-netroute/pull/29)) - compare priority if route rule's dst mask is same size - compare priority if route rule's dst mask is same size - sync: update CI config files (#24) ([libp2p/go-netroute#24](https://github.com/libp2p/go-netroute/pull/24)) - github.com/marten-seemann/qpack (v0.2.1 -> v0.3.0): - update to Ginkgo v2 (#30) ([marten-seemann/qpack#30](https://github.com/marten-seemann/qpack/pull/30)) - return write error when encoding header fields (#28) ([marten-seemann/qpack#28](https://github.com/marten-seemann/qpack/pull/28)) - update Go versions (#29) ([marten-seemann/qpack#29](https://github.com/marten-seemann/qpack/pull/29)) - remove CircleCI build status from README - add link to QPACK RFC to README - remove build constraint from fuzzer ([marten-seemann/qpack#24](https://github.com/marten-seemann/qpack/pull/24)) - github.com/multiformats/go-multicodec (v0.6.0 -> v0.7.0): - feat: update ./multicodec/table.csv ([multiformats/go-multicodec#71](https://github.com/multiformats/go-multicodec/pull/71))
### Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Antonio Navarro Perez | 11 | +780/-987 | 31 | | Marcin Rataj | 14 | +791/-543 | 26 | | web3-bot | 7 | +393/-427 | 71 | | galargh | 20 | +309/-277 | 21 | | Gus Eggert | 5 | +358/-222 | 58 | | Henrique Dias | 3 | +409/-30 | 13 | | Dustin Long | 1 | +314/-0 | 2 | | Marco Munizaga | 2 | +211/-46 | 11 | | Rod Vagg | 4 | +188/-62 | 13 | | Jorropo | 2 | +4/-219 | 5 | | Steve Loeppky | 1 | +115/-72 | 4 | | Andreas Källberg | 1 | +145/-5 | 5 | | Lucas Molas | 3 | +76/-53 | 9 | | snyh | 2 | +36/-18 | 2 | | Piotr Galar | 2 | +31/-4 | 2 | | Ondrej Kokes | 1 | +25/-4 | 2 | | Marten Seemann | 6 | +14/-14 | 14 | | Yann Autissier | 1 | +14/-4 | 1 | | maxos | 1 | +8/-1 | 2 | | reidlw | 1 | +1/-4 | 1 | | Russell Dempsey | 2 | +4/-1 | 2 | | Ian Davis | 1 | +4/-0 | 2 | | Daniel Norman | 1 | +3/-1 | 1 | | Will Scott | 1 | +1/-1 | 1 | | Nikhilesh Susarla | 1 | +2/-0 | 2 | | Jamie Wilkinson | 1 | +1/-1 | 1 | | Will | 1 | +0/-1 | 1 | ================================================ FILE: docs/changelogs/v0.18.md ================================================ # Kubo changelog v0.18 ## v0.18.1 This release includes improvements around Pubsub message deduplication, libp2p resource management, and more. - [Overview](#overview) - [🔦 Highlights](#-highlights) - [New default Pubsub.SeenMessagesStrategy](#new-default-pubsubseenmessagesstrategy) - [Improving libp2p resource management integration](#improving-libp2p-resource-management-integration) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### 🔦 Highlights #### New default `Pubsub.SeenMessagesStrategy` A new optional [`Pubsub.SeenMessagesStrategy`](../config.md#pubsubseenmessagesstrategy) configuration option has been added. This option allows you to choose between two different strategies for deduplicating messages: `first-seen` and `last-seen`. When unset, the default strategy is `last-seen`, which calculates the time-to-live (TTL) countdown based on the last time a message is seen. This means that if a message is received and then seen again within the specified TTL window based on the last time it was seen, it won't be emitted. If you prefer the old behavior, which calculates the TTL countdown based on the first time a message is seen, you can set `Pubsub.SeenMessagesStrategy` to `first-seen`. #### Improving libp2p resource management integration TL;DR: limit autoscaling improved, most users should start with default settings. If you have old configuration, switch to implicit defaults: ``` ipfs config --json -- Swarm.ResourceMgr '{}' ipfs config --json -- Swarm.ConnMgr '{}' ``` IF you run a server and want to utilize more than half of memory and file descriptors to p2p work, adjust [`Swarm.ResourceMgr.MaxMemory`](https://github.com/ipfs/kubo/blob/master/docs/config.md#swarmresourcemgrmaxmemory) and [`Swarm.ResourceMgr.MaxFileDescriptors`](https://github.com/ipfs/kubo/blob/master/docs/config.md#swarmresourcemgrmaxfiledescriptors). The 0.18.1 builds on the default protection nodes get against DoS (resource exhaustion) and eclipse attacks with the [go-libp2p Network Resource Manager/Accountant](https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md) that was fine-tuned in [Kubo 0.18](https://github.com/ipfs/kubo/blob/biglep/resource-manager-example-of-what-want/docs/changelogs/v0.18.md#improving-libp2p-resource-management-integration). Adding default hard-limits from the Resource Manager/Accountant after the fact is tricky, and some additional improvements have been made to improve the [computed defaults](https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md#computed-default-limits). As much as possible, the aim is for a user to only think about how much memory they want to bound libp2p to, and not need to think about translating that to hard numbers for connections, streams, etc. More updates are likely in future Kubo releases, but with this release: 1. ``System.StreamsInbound`` is no longer bounded directly 2. ``System.ConnsInbound``, ``Transient.Memory``, ``Transient.ConnsInbound`` have higher default computed values. ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - Add overview section - Adjust inbound connection limits depending on memory. - feat: Pubsub.SeenMessagesStrategy (#9543) ([ipfs/kubo#9543](https://github.com/ipfs/kubo/pull/9543)) - chore: update version - github.com/libp2p/go-libp2p-pubsub (v0.8.2 -> v0.8.3): - feat: expire messages from the cache based on last seen time (#513) ([libp2p/go-libp2p-pubsub#513](https://github.com/libp2p/go-libp2p-pubsub/pull/513))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Mohsin Zaidi | 2 | +511/-55 | 12 | | Antonio Navarro Perez | 2 | +57/-57 | 5 | | galargh | 1 | +1/-1 | 1 | ## v0.18.0 ### Overview Below is an outline of all that is in this release, so you get a sense of all that's included. - [Overview](#overview) - [🔦 Highlights](#-highlights) - [Content routing](#content-routing) - [Default InterPlanetary Network Indexer](#default-interplanetary-network-indexer) - [Increase provider record republish interval and expiration](#increase-provider-record-republish-interval-and-expiration) - [Gateways](#gateways) - [(DAG-)JSON and (DAG-)CBOR response formats](#dag-json-and-dag-cbor-response-formats) - [🐎 Fast directory listings with DAG sizes](#-fast-directory-listings-with-dag-sizes) - [QUIC and WebTransport](#quic-and-webtransport) - [WebTransport enabled by default](#webtransport-enabled-by-default) - [QUIC and WebTransport share a single port](#quic-and-webtransport-share-a-single-port) - [Differentiating QUIC versions](#differentiating-quic-versions) - [QUICv1 and WebTransport config migration](#quicv1-and-webtransport-config-migration) - [Improving libp2p resource management integration](#improving-libp2p-resource-management-integration) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### 🔦 Highlights #### Content routing ##### Default InterPlanetary Network Indexer Content routing is the process of discovering which peers provide a piece of content. Kubo has traditionally only supported [libp2p's implementation of Kademlia DHT](https://github.com/libp2p/specs/tree/master/kad-dht) for content routing. Kubo can now bridge networks by including support for the [delegated routing HTTP API](https://github.com/ipfs/specs/pull/337). Users can compose content routers using the `Routing.Routers` config to pick content routers with different tradeoffs than a Kademlia DHT (e.g., high-performance and high-capacity centralized endpoints, dedicated Kademlia DHT nodes, routers with unique provider records, privacy-focused content routers). One example is [InterPlanetary Network Indexers](https://github.com/ipni/specs/blob/main/IPNI.md#readme), which are HTTP endpoints that cache records from both the IPFS network and other sources such as web3.storage and Filecoin. This improves not only content availability by enabling Kubo to transparently fetch content directly from Filecoin storage providers, but also improves IPFS content routing latency by an order of magnitude and decreases resource consumption. > *Note:* it's possible to retrieve content stored by Filecoin Storage Providers (SPs) from Kubo if the SPs service Bitswap requests. As of this release, some SPs are advertising Bitswap. You can follow the roadmap progress for IPNIs and Bitswap in SPs [here](https://www.starmaps.app/roadmap/github.com/protocol/bedrock/issues/1). In this release, the default content router is changed from `dht` to `auto`. The `auto` router includes the IPFS DHT in addition to the [cid.contact](https://cid.contact) IPNI instance. In future releases, we plan to expand the functionality of `auto` to encompass automatic discovery of content routers, which will improve performance and content availability (for example, see [IPIP-342](https://github.com/ipfs/specs/pull/342)). Previous behavior can be restored by setting `Routing.Type` to `dht`. Alternative routing rules, including alternative IPNI endpoints, can be configured in `Routing.Routers` after setting `Routing.Type` to `custom`. Learn more in the [`Routing` docs](https://github.com/ipfs/kubo/blob/master/docs/config.md#routing). ##### Increase provider record republish interval and expiration Default `Reprovider.Interval` changed from 12h to 22h to match new defaults for the Provider Record Expiration (48h) in [go-libp2p-kad-dht v0.20.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.20.0). The rationale for increasing this can be found in [RFM 17: Provider Record Liveness Report](https://github.com/protocol/network-measurements/blob/master/results/rfm17-provider-record-liveness.md), [kubo#9326](https://github.com/ipfs/kubo/pull/9326), and the upstream DHT specifications at [libp2p/specs#451](https://github.com/libp2p/specs/pull/451). Learn more in the [`Reprovider` config](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#reprovider). #### Gateways ##### (DAG-)JSON and (DAG-)CBOR response formats The IPFS project has reserved the corresponding media types at IANA: - [`application/vnd.ipld.dag-json`](https://www.iana.org/assignments/media-types/application/vnd.ipld.dag-json) - [`application/vnd.ipld.dag-cbor`](https://www.iana.org/assignments/media-types/application/vnd.ipld.dag-cbor) This release implements them as part of [IPIP-328](https://github.com/ipfs/specs/pull/328) and adds Gateway support for CIDs with `json` (0x0200), `cbor` (0x51), [`dag-json`](https://ipld.io/specs/codecs/dag-json/) (0x0129) and [`dag-cbor`](https://ipld.io/specs/codecs/dag-cbor/spec/) (0x71) codecs. To specify the response `Content-Type` explicitly, the HTTP client can override the codec present in the CID by using the `format` parameter or setting the `Accept` HTTP header: - Plain JSON: `?format=json` or `Accept: application/json` - Plain CBOR: `?format=cbor` or `Accept: application/cbor` - DAG-JSON: `?format=dag-json` or `Accept: application/vnd.ipld.dag-json` - DAG-CBOR: `?format=dag-cbor` or `Accept: application/vnd.ipld.dag-cbor` In addition, when DAG-JSON or DAG-CBOR is requested with the `Accept` header set to `text/html`, the Gateway will return a basic HTML page with download options, improving the user experience in web browsers. ###### Example 1: DAG-CBOR and DAG-JSON Conversion on Gateway The Gateway supports conversion between DAG-CBOR and DAG-JSON for efficient end-to-end data structure management: author in CBOR or JSON, store as binary CBOR and retrieve as JSON via HTTP: ```console $ echo '{"test": "json"}' | ipfs dag put # implicit --input-codec dag-json --store-codec dag-cbor bafyreico7mjtqtqhvawro3yud5uqn6sc33nzqb7b5j2d7pdmzer5nab4t4 $ ipfs block get bafyreico7mjtqtqhvawro3yud5uqn6sc33nzqb7b5j2d7pdmzer5nab4t4 | xxd 00000000: a164 7465 7374 646a 736f 6e .dtestdjson $ ipfs dag get bafyreico7mjtqtqhvawro3yud5uqn6sc33nzqb7b5j2d7pdmzer5nab4t4 # implicit --output-codec dag-json {"test":"json"} $ curl "http://127.0.0.1:8080/ipfs/bafyreico7mjtqtqhvawro3yud5uqn6sc33nzqb7b5j2d7pdmzer5nab4t4?format=dag-json" {"test":"json"} ``` ###### Example 2: Traversing CBOR DAGs Placing a CID in [CBOR Tag 42](https://github.com/ipld/cid-cbor/) enables the creation of arbitrary DAGs. The equivalent DAG-JSON notation for linking to different blocks is represented by `{ "/": "cid" }`. The Gateway supports traversing these links, enabling access to data referenced by structures other than regular UnixFS directories: ```console $ echo '{"test.jpg": {"/": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"}}' | ipfs dag put bafyreihspwy3zlkzgphmec5d3xb5g5njrqwotd46lyubnelbzktnmsxkq4 # dag-cbor document linking to unixfs file $ ipfs resolve /ipfs/bafyreihspwy3zlkzgphmec5d3xb5g5njrqwotd46lyubnelbzktnmsxkq4/test.jpg /ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi $ ipfs dag stat bafyreihspwy3zlkzgphmec5d3xb5g5njrqwotd46lyubnelbzktnmsxkq4 Size: 119827, NumBlocks: 2 $ curl "http://127.0.0.1:8080/ipfs/bafyreihspwy3zlkzgphmec5d3xb5g5njrqwotd46lyubnelbzktnmsxkq4/test.jpg" > test.jpg ``` ###### Example 3: UnixFS directory listing as JSON Finally, Gateway now supports the same [logical format projection](https://ipld.io/specs/codecs/dag-pb/spec/#logical-format) from DAG-PB to DAG-JSON as the `ipfs dag get` command, enabling the retrieval of directory listings as JSON instead of HTML: ```console $ export DIR_CID=bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq $ curl -H "Accept: application/vnd.ipld.dag-json" "http://127.0.0.1:8080/ipfs/$DIR_CID" | jq $ curl "http://127.0.0.1:8080/ipfs/$DIR_CID?format=dag-json" | jq { "Data": { "/": { "bytes": "CAE" } }, "Links": [ { "Hash": { "/": "Qmc3zqKcwzbbvw3MQm3hXdg8BQoFjGdZiGdAfXAyAGGdLi" }, "Name": "1 - Barrel - Part 1 - alt.txt", "Tsize": 21 }, { "Hash": { "/": "QmdMxMx29KVYhHnaCc1icWYxQqXwUNCae6t1wS2NqruiHd" }, "Name": "1 - Barrel - Part 1 - transcript.txt", "Tsize": 195 }, { "Hash": { "/": "QmawceGscqN4o8Y8Fv26UUmB454kn2bnkXV5tEQYc4jBd6" }, "Name": "1 - Barrel - Part 1.png", "Tsize": 24862 } ] } $ ipfs dag get $DIR_CID {"Data":{"/":{"bytes":"CAE"}},"Links":[{"Hash":{"/":"Qmc3zqKcwzbbvw3MQm3hXdg8BQoFjGdZiGdAfXAyAGGdLi"},"Name":"1 - Barrel - Part 1 - alt.txt","Tsize":21},{"Hash":{"/":"QmdMxMx29KVYhHnaCc1icWYxQqXwUNCae6t1wS2NqruiHd"},"Name":"1 - Barrel - Part 1 - transcript.txt","Tsize":195},{"Hash":{"/":"QmawceGscqN4o8Y8Fv26UUmB454kn2bnkXV5tEQYc4jBd6"},"Name":"1 - Barrel - Part 1.png","Tsize":24862}]} ``` ##### 🐎 Fast directory listings with DAG sizes Fast listings are now enabled for _all_ UnixFS directories: big and small. There is no linear slowdown caused by reading size metadata from child nodes, and the size of DAG representing child items is always present. As an example, the CID `bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm` represents a UnixFS directory with over 10k files. Listing big directories was fast since Kubo 0.13, but in this release it will also include the size column. #### QUIC and WebTransport ##### WebTransport enabled by default [WebTransport](https://web.archive.org/web/20260128152314/https://docs.libp2p.io/concepts/transports/webtransport/) is a new libp2p transport that [was introduced in v0.16](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.16.md#-webtransport-new-experimental-transport) that is based on top of QUIC and HTTP3. This allows browser-based nodes to contact Kubo nodes, so now instead of just serving requests for other system-level application nodes, you can also serve requests directly to a node running inside a browser page. For the full story see [connectivity.libp2p.io](https://web.archive.org/web/20251118040510/https://connectivity.libp2p.io/). ##### QUIC and WebTransport share a single port WebTransport is enabled by default in part because [go-libp2p now supports running WebTransport and QUIC transports on the same QUIC listener](https://github.com/libp2p/go-libp2p/issues/1759). No additional port needs to be opened. To use this feature, register two listen addresses on the same `/ipX/.../udp/XXX` prefix. ##### Differentiating QUIC versions go-libp2p now differentiates the first version of QUIC that was originally implemented, `Draft-29`, from the ratified protocol in [RFC9000](https://www.rfc-editor.org/rfc/rfc9000.html), `QUICv1`. This was done for performance (time to first byte) reasons as [outlined here](https://github.com/multiformats/multiaddr/issues/145). This manifests as two different multiaddr components `/quic` (old Draft-29) and `/quic-v1`. go-libp2p do supports listening with both QUIC versions on one single listener. WebTransport has only supported QUICv1. `/webtransport` now needs to be prefixed by a `/quic-v1` component instead of a `/quic` component. Support for QUIC Draft-29 will be removed at some point in 2023 ([tracking issue](https://github.com/ipfs/kubo/issues/9496)). As a result, new deployments should use `/quic-v1` instead of `/quic`. ##### QUICv1 and WebTransport config migration To support QUICv1 and WebTransport by default a new config migration (`v13`) is run which automatically adds entries in addresses-related fields: - Replace all `/quic/webtransport` to `/quic-v1/webtransport`. - For all `/quic` listeners, keep the Draft-29 listener, and on the same ip and port, add `/quic-v1` and `/quic-v1/webtransport` listeners. #### Improving libp2p resource management integration To help protect nodes from DoS (resource exhaustion) and eclipse attacks, Kubo enabled the [go-libp2p Network Resource Manager](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager) by default in [Kubo 0.17](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.17.md#libp2p-resource-management-enabled-by-default). Introducing limits like this by default after the fact is tricky, and various improvements have been made to improve the UX including: 1. [Dedicated docs concerning the resource manager integration](https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md). This is a great place to go to learn more or get your FAQs answered. 2. Increasing the default limits for the resource manager. 3. Enabling the [`Swarm.ConnMgr`](https://github.com/ipfs/kubo/blob/master/docs/config.md#swarmconnmgr) by default and reducing it thresholds so it can intelligently prune connections in many cases before the indiscriminate resource manager kicks in. 4. Adjusted log messages and levels to make clear that the resource manager is likely doing your node a favor by bounding resources. 5. [Other miscellaneous config and command bugs reported by users](https://github.com/ipfs/kubo/issues/9442). ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - fix: clarity: no user supplied rcmgr limits of 0 (#9563) ([ipfs/kubo#9563](https://github.com/ipfs/kubo/pull/9563)) - fix(gateway): undesired conversions to dag-json and friends (#9566) ([ipfs/kubo#9566](https://github.com/ipfs/kubo/pull/9566)) - fix: ensure connmgr is smaller then autoscalled resource limits - fix: typo in ensureConnMgrMakeSenseVsResourcesMgr - docs: clarify browser descriptions for webtransport - fix: update saxon download path - fix: refuse to start if connmgr is smaller than resource limits and not using none connmgr - fix: User-Agent sent to HTTP routers - test: port gateway sharness tests to Go tests - fix: do not download saxon in parallel - docs: improve docs/README (#9539) ([ipfs/kubo#9539](https://github.com/ipfs/kubo/pull/9539)) - test: port CircleCI to GH Actions and improve sharness reporting (#9355) ([ipfs/kubo#9355](https://github.com/ipfs/kubo/pull/9355)) - chore: migrate from go-ipfs-files to go-libipfs/files (#9535) ([ipfs/kubo#9535](https://github.com/ipfs/kubo/pull/9535)) - fix: stats dht command when Routing.Type=auto (#9538) ([ipfs/kubo#9538](https://github.com/ipfs/kubo/pull/9538)) - fix: hint people to changing from RSA peer ids - fix(gateway): JSON when Accept is a list - fix(test): retry flaky t0125-twonode.sh - docs: fix Router config Godoc (#9528) ([ipfs/kubo#9528](https://github.com/ipfs/kubo/pull/9528)) - fix(ci): flaky sharness test - docs(config): ProviderSearchDelay (#9526) ([ipfs/kubo#9526](https://github.com/ipfs/kubo/pull/9526)) - docs: clarify debug environment variables - chore: update version.go - fix(test): stabilize flaky provider tests - feat: port pins CLI test - Removing QRI from early tester ([ipfs/kubo#9503](https://github.com/ipfs/kubo/pull/9503)) - fix: disable provide over HTTP with Routing.Type=auto (#9511) ([ipfs/kubo#9511](https://github.com/ipfs/kubo/pull/9511)) - Update version.go - 'chore: update version.go' - Cleaned up 0.18 changelog for release ([ipfs/kubo#9497](https://github.com/ipfs/kubo/pull/9497)) - feat: turn on WebTransport by default ([ipfs/kubo#9492](https://github.com/ipfs/kubo/pull/9492)) - feat: fast directory listings with DAG Size column (#9481) ([ipfs/kubo#9481](https://github.com/ipfs/kubo/pull/9481)) - feat: add basic CLI tests using Go Test - Changelog: v0.18.0 ([ipfs/kubo#9485](https://github.com/ipfs/kubo/pull/9485)) - feat: go-libp2p-kad-dht with expiration 48h - chore: update go-libp2p to v0.24.1 - fix: remove the imports work-around - fix: replace quic to quic-v1 for webtransport sharness - fix: silence staticcheck warning for fx.Extract usage - update go-libp2p to v0.24.0 - stop using the deprecated go-libp2p-loggables package - docs(readme): update package managers section (#9488) ([ipfs/kubo#9488](https://github.com/ipfs/kubo/pull/9488)) - fix: support /quic-v1 in webui v0.21 - feat: Routing.Type=auto (DHT+IPNI) (#9475) ([ipfs/kubo#9475](https://github.com/ipfs/kubo/pull/9475)) - feat: adjust ConnMgr target to 32-96 (#9483) ([ipfs/kubo#9483](https://github.com/ipfs/kubo/pull/9483)) - feat: increase default Reprovider.Interval (#9326) ([ipfs/kubo#9326](https://github.com/ipfs/kubo/pull/9326)) - feat: add response body limiter to routing HTTP client (#9478) ([ipfs/kubo#9478](https://github.com/ipfs/kubo/pull/9478)) - docs: libp2p resource management (#9468) ([ipfs/kubo#9468](https://github.com/ipfs/kubo/pull/9468)) - chore: upgrade libipfs for routing HTTP API schema changes (#9477) ([ipfs/kubo#9477](https://github.com/ipfs/kubo/pull/9477)) - feat: lower connection pool - Add missing && - Fix sharness test - Added a message when RM is disabled. - Requested changes. - Fix sharness checking daemon output - Update test/sharness/t0060-daemon.sh - Try to fix sharness test. - Fix: RM: Improve init RM message and fix final memory value. - Fix: Resource Manager: Filter stats correctly by % - Apply suggestions from code review - Increase MaxMemory param to use half of total memory. - Update libipfs dependency. - Add sharness tests and documentation - Fix variable name - feature: delegated-routing: Add HTTP delegated routing. - Fix: Change RM log output to WARN level - Fix: RM: Set no-limit value to 1e9 (1000000000). - Partial Revert "Revert "fix: ensure hasher is registered when using a hashing function"" - Add logs to the routing system - fix: apply agent-version-suffix to libp2p identify - chore: migrate ipfs/tar-utils to libipfs - feat(gateway): JSON and CBOR response formats (IPIP-328) (#9335) ([ipfs/kubo#9335](https://github.com/ipfs/kubo/pull/9335)) - ([ipfs/kubo#9318](https://github.com/ipfs/kubo/pull/9318)) - docs: release process updates from v0.17.0 ([ipfs/kubo#9391](https://github.com/ipfs/kubo/pull/9391)) - fix(rcmgr): improve error phrasing - docs: Update CHANGELOG.md adding 0.17 link - feat(config): Pubsub.SeenMessagesTTL (#9372) ([ipfs/kubo#9372](https://github.com/ipfs/kubo/pull/9372)) - docs: remove snap and chocolatey packages - Merge release v0.17.0 ([ipfs/kubo#9431](https://github.com/ipfs/kubo/pull/9431)) - docs: ipfs-http-client -> kubo-rpc-client (#9331) ([ipfs/kubo#9331](https://github.com/ipfs/kubo/pull/9331)) - docs(readme): improve tldr - Update config.md for resource management limits (#9421) ([ipfs/kubo#9421](https://github.com/ipfs/kubo/pull/9421)) - Doc improvements and changelog for resource manager (#9413) ([ipfs/kubo#9413](https://github.com/ipfs/kubo/pull/9413)) - Revert "Doc improvements for rcmgr" - Doc improvements for rcmgr - docs: document /wss fixes in 0.17 - refactor(config): remove Swarm.ConnMgr defaults - fix(config): skip nulls in ResourceMgr - docs: replace tabcat with aphelionz in EARLY_TESTERS.md (#9404) ([ipfs/kubo#9404](https://github.com/ipfs/kubo/pull/9404)) - docs: fix spoiler for 0.13.1 changelog - fix(docs): typo - chore: bump version to v0.18.0-dev ([ipfs/kubo#9393](https://github.com/ipfs/kubo/pull/9393)) - github.com/ipfs/go-bitswap (v0.10.2 -> v0.11.0): - chore: release v0.11.0 - github.com/ipfs/go-blockservice (v0.4.0 -> v0.5.0): - chore: release v0.5.0 - github.com/ipfs/go-graphsync (v0.13.1 -> v0.14.1): - chore: version 0.14.1 (#400) ([ipfs/go-graphsync#400](https://github.com/ipfs/go-graphsync/pull/400)) - chore: migrate files (#399) ([ipfs/go-graphsync#399](https://github.com/ipfs/go-graphsync/pull/399)) - docs(CHANGELOG): update for v0.14.0 release - updates for libp2p v0.22 (#392) ([ipfs/go-graphsync#392](https://github.com/ipfs/go-graphsync/pull/392)) - feat(ipld): use bindnode/registry (#386) ([ipfs/go-graphsync#386](https://github.com/ipfs/go-graphsync/pull/386)) - Accept/Reject requests up front (#384) ([ipfs/go-graphsync#384](https://github.com/ipfs/go-graphsync/pull/384)) - Remove protobuf protocol (#385) ([ipfs/go-graphsync#385](https://github.com/ipfs/go-graphsync/pull/385)) - docs(CHANGELOG): update for v0.13.2 - chore(deps): upgrade libp2p & ipld-prime (#389) ([ipfs/go-graphsync#389](https://github.com/ipfs/go-graphsync/pull/389)) - chore(ipld): switch to using top-level ipld-prime codec helpers (#383) ([ipfs/go-graphsync#383](https://github.com/ipfs/go-graphsync/pull/383)) - feat(requestmanager): read request from context (#381) ([ipfs/go-graphsync#381](https://github.com/ipfs/go-graphsync/pull/381)) - fix: minor typo in error msg - fix(panics): lift panic recovery up to top of network handling - feat: expand use of panic handler to cover network and codec interaction - feat(panics): capture panics from selector execution - github.com/ipfs/go-ipfs-cmds (v0.8.1 -> v0.8.2): - chore: version v0.8.2 (#235) ([ipfs/go-ipfs-cmds#235](https://github.com/ipfs/go-ipfs-cmds/pull/235)) - chore: migrate files (#233) ([ipfs/go-ipfs-cmds#233](https://github.com/ipfs/go-ipfs-cmds/pull/233)) - sync: update CI config files (#229) ([ipfs/go-ipfs-cmds#229](https://github.com/ipfs/go-ipfs-cmds/pull/229)) - github.com/ipfs/go-ipfs-keystore (v0.0.2 -> v0.1.0): - chore: release v0.1.0 - chore: update go-libp2p - sync: update CI config files ([ipfs/go-ipfs-keystore#10](https://github.com/ipfs/go-ipfs-keystore/pull/10)) - sync: update CI config files ([ipfs/go-ipfs-keystore#8](https://github.com/ipfs/go-ipfs-keystore/pull/8)) - Add link to pkg.go.dev - README: this module does not use Gx - github.com/ipfs/go-ipfs-provider (v0.7.1 -> v0.8.1): - chore: release v0.8.1 - fix: make queue 64bits on 32bits platforms too - sync: update CI config files ([ipfs/go-ipfs-provider#36](https://github.com/ipfs/go-ipfs-provider/pull/36)) - chore: update go-lib2p, avoid depending on go-libp2p-core, bump go.mod version - github.com/ipfs/go-ipfs-routing (v0.2.1 -> v0.3.0): - release v0.3.0 ([ipfs/go-ipfs-routing#36](https://github.com/ipfs/go-ipfs-routing/pull/36)) - chore: update go-libp2p to v0.22.0 ([ipfs/go-ipfs-routing#35](https://github.com/ipfs/go-ipfs-routing/pull/35)) - sync: update CI config files (#34) ([ipfs/go-ipfs-routing#34](https://github.com/ipfs/go-ipfs-routing/pull/34)) - github.com/ipfs/go-ipld-cbor (v0.0.5 -> v0.0.6): - Add contexts to IpldBlockstore ([ipfs/go-ipld-cbor#82](https://github.com/ipfs/go-ipld-cbor/pull/82)) - sync: update CI config files (#81) ([ipfs/go-ipld-cbor#81](https://github.com/ipfs/go-ipld-cbor/pull/81)) - sync: update CI config files ([ipfs/go-ipld-cbor#79](https://github.com/ipfs/go-ipld-cbor/pull/79)) - Fix lint errors ([ipfs/go-ipld-cbor#78](https://github.com/ipfs/go-ipld-cbor/pull/78)) - Add notice directing new projects to codec/dagcbor ([ipfs/go-ipld-cbor#77](https://github.com/ipfs/go-ipld-cbor/pull/77)) - github.com/ipfs/go-merkledag (v0.6.0 -> v0.9.0): - chore: bump version to 0.9.0 - chore: bump version to 0.8.1 - feat: remove panic() from non-error methods - feat: improve broken cid.Builder testing for CidBuilder - chore: bump version to 0.8.0 - doc: document potential panics and how to avoid them - fix: simplify Cid generation cache & usage - feat: check links on setting and sanitise on encoding - feat: check that the CidBuilder hasher is usable - chore: bump version to 0.7.0 - fix: remove use of ioutil - run gofmt -s - fix!: keep deserialised state stable until explicit mutation - sync: update CI config files ([ipfs/go-merkledag#84](https://github.com/ipfs/go-merkledag/pull/84)) - github.com/ipfs/go-namesys (v0.5.0 -> v0.6.0): - chore: release v0.6.0 - chore: update go-libp2p to v0.23.4, update go.mod version to 1.18 - stop using the deprecated io/ioutil package - github.com/ipfs/go-peertaskqueue (v0.7.1 -> v0.8.0): - Release v0.8.0 ([ipfs/go-peertaskqueue#25](https://github.com/ipfs/go-peertaskqueue/pull/25)) - chore: update go-libp2p to v0.22.0 ([ipfs/go-peertaskqueue#24](https://github.com/ipfs/go-peertaskqueue/pull/24)) - sync: update CI config files (#23) ([ipfs/go-peertaskqueue#23](https://github.com/ipfs/go-peertaskqueue/pull/23)) - sync: update CI config files (#21) ([ipfs/go-peertaskqueue#21](https://github.com/ipfs/go-peertaskqueue/pull/21)) - github.com/ipfs/go-unixfs (v0.4.1 -> v0.4.2): - chore: version 0.4.2 (#136) ([ipfs/go-unixfs#136](https://github.com/ipfs/go-unixfs/pull/136)) - chore: migrate files (#134) ([ipfs/go-unixfs#134](https://github.com/ipfs/go-unixfs/pull/134)) - ([ipfs/go-unixfs#128](https://github.com/ipfs/go-unixfs/pull/128)) - github.com/ipfs/go-unixfsnode (v1.4.0 -> v1.5.1): - v1.5.1 - fix a possible `index out of range` crash ([ipfs/go-unixfsnode#39](https://github.com/ipfs/go-unixfsnode/pull/39)) - add an ADL to preload hamt loading ([ipfs/go-unixfsnode#38](https://github.com/ipfs/go-unixfsnode/pull/38)) - chore: bump version to 1.5.0 - fix: remove use of ioutil - run gofmt -s - bump go.mod to Go 1.18 and run go fix - test for reader / sizing behavior on large files ([ipfs/go-unixfsnode#34](https://github.com/ipfs/go-unixfsnode/pull/34)) - add helper to approximate test creation pattern from ipfs-files ([ipfs/go-unixfsnode#32](https://github.com/ipfs/go-unixfsnode/pull/32)) - chore: remove Stebalien/go-bitfield in favour of ipfs/go-bitfield - github.com/ipfs/interface-go-ipfs-core (v0.7.0 -> v0.8.2): - chore: version 0.8.2 (#100) ([ipfs/interface-go-ipfs-core#100](https://github.com/ipfs/interface-go-ipfs-core/pull/100)) - chore: migrate files (#97) ([ipfs/interface-go-ipfs-core#97](https://github.com/ipfs/interface-go-ipfs-core/pull/97)) - chore: release v0.8.1 - feat: add UseCumulativeSize UnixfsLs option (#95) ([ipfs/interface-go-ipfs-core#95](https://github.com/ipfs/interface-go-ipfs-core/pull/95)) - chore: release v0.8.0 - chore: update go-libp2p to v0.23.4 - sync: update CI config files (#87) ([ipfs/interface-go-ipfs-core#87](https://github.com/ipfs/interface-go-ipfs-core/pull/87)) - github.com/ipld/go-car/v2 (v2.4.0 -> v2.5.1): - add a `SkipNext` method on block reader (#338) ([ipld/go-car#338](https://github.com/ipld/go-car/pull/338)) - feat: Has() and Get() will respect StoreIdentityCIDs option - chore: bump version to 0.5.0 - fix: remove use of ioutil - run gofmt -s - bump go.mod to Go 1.18 and run go fix - bump go.mod to Go 1.18 and run go fix - OpenReadWriteFile: add test - blockstore: allow to pass a file to write in (#323) ([ipld/go-car#323](https://github.com/ipld/go-car/pull/323)) - feat: add `car inspect` command to cmd pkg (#320) ([ipld/go-car#320](https://github.com/ipld/go-car/pull/320)) - Separate `index.ReadFrom` tests - Only read index codec during inspection - Upgrade to the latest `go-car/v2` - Empty identity CID should be indexed when options are set - github.com/ipld/go-codec-dagpb (v1.4.1 -> v1.5.0): - chore: version bump to 1.5.0 - fix: replace io/ioutil with io - bump go.mod to Go 1.18 and run go fix - v1.4.2 bump - github.com/libp2p/go-libp2p (v0.23.4 -> v0.24.2): - release v0.24.2 (#1969) ([libp2p/go-libp2p#1969](https://github.com/libp2p/go-libp2p/pull/1969)) - webtransport: initialize a NullResourceManager if none is provided (#1962) ([libp2p/go-libp2p#1962](https://github.com/libp2p/go-libp2p/pull/1962)) - release v0.24.1 (#1945) ([libp2p/go-libp2p#1945](https://github.com/libp2p/go-libp2p/pull/1945)) - routed host: return Connect error if FindPeer doesn't yield new addresses (#1946) ([libp2p/go-libp2p#1946](https://github.com/libp2p/go-libp2p/pull/1946)) - chore: update examples to v0.24.0 (#1936) ([libp2p/go-libp2p#1936](https://github.com/libp2p/go-libp2p/pull/1936)) - webtransport: fix flaky accept queue test (#1938) ([libp2p/go-libp2p#1938](https://github.com/libp2p/go-libp2p/pull/1938)) - quic: fix race condition in TestClientCanDialDifferentQUICVersions (#1937) ([libp2p/go-libp2p#1937](https://github.com/libp2p/go-libp2p/pull/1937)) - quic: update quic-go to v0.31.1 (#1942) ([libp2p/go-libp2p#1942](https://github.com/libp2p/go-libp2p/pull/1942)) - release v0.24.0 (#1934) ([libp2p/go-libp2p#1934](https://github.com/libp2p/go-libp2p/pull/1934)) - Disable support for signed/static TLS certificates in WebTransport (#1927) ([libp2p/go-libp2p#1927](https://github.com/libp2p/go-libp2p/pull/1927)) - webtransport: add PSK to constructor, and fail if it is used (#1929) ([libp2p/go-libp2p#1929](https://github.com/libp2p/go-libp2p/pull/1929)) - use a different set of default transports when PSK is enabled (#1921) ([libp2p/go-libp2p#1921](https://github.com/libp2p/go-libp2p/pull/1921)) - transport.Listener,quic: Support multiple QUIC versions with the same Listener. Only return a single multiaddr per listener. (#1923) ([libp2p/go-libp2p#1923](https://github.com/libp2p/go-libp2p/pull/1923)) - quic / webtransport: make it possible to listen on the same address / port (#1905) ([libp2p/go-libp2p#1905](https://github.com/libp2p/go-libp2p/pull/1905)) - autorelay: fix flaky TestReconnectToStaticRelays (#1903) ([libp2p/go-libp2p#1903](https://github.com/libp2p/go-libp2p/pull/1903)) - swarm / rcmgr: synchronize the concurrent outbound dials with limits (#1898) ([libp2p/go-libp2p#1898](https://github.com/libp2p/go-libp2p/pull/1898)) - add QUIC v1 addresses to the default listen addresses (#1914) ([libp2p/go-libp2p#1914](https://github.com/libp2p/go-libp2p/pull/1914)) - webtransport: update webtransport-go to v0.3.0 (#1895) ([libp2p/go-libp2p#1895](https://github.com/libp2p/go-libp2p/pull/1895)) - tls: fix flaky TestHandshakeConnectionCancellations test (#1896) ([libp2p/go-libp2p#1896](https://github.com/libp2p/go-libp2p/pull/1896)) - holepunch: disable the resource manager in tests (#1897) ([libp2p/go-libp2p#1897](https://github.com/libp2p/go-libp2p/pull/1897)) - transports: expose the name of the transport in the ConnectionState (#1911) ([libp2p/go-libp2p#1911](https://github.com/libp2p/go-libp2p/pull/1911)) - respect the user's security protocol preference order ([libp2p/go-libp2p#1912](https://github.com/libp2p/go-libp2p/pull/1912)) - circuitv2: disable the resource manager in tests (#1899) ([libp2p/go-libp2p#1899](https://github.com/libp2p/go-libp2p/pull/1899)) - expose the security protocol on the ConnectionState ([libp2p/go-libp2p#1907](https://github.com/libp2p/go-libp2p/pull/1907)) - fix: autorelay: treat static relays as just another peer source (#1875) ([libp2p/go-libp2p#1875](https://github.com/libp2p/go-libp2p/pull/1875)) - feat: quic,webtransport: enable both quic-draft29 and quic-v1 addrs on quic. only quic-v1 on webtransport (#1881) ([libp2p/go-libp2p#1881](https://github.com/libp2p/go-libp2p/pull/1881)) - holepunch: add multiaddress filter (#1839) ([libp2p/go-libp2p#1839](https://github.com/libp2p/go-libp2p/pull/1839)) - README: remove broken links from table of contents (#1893) ([libp2p/go-libp2p#1893](https://github.com/libp2p/go-libp2p/pull/1893)) - quic: update quic-go to v0.31.0 (#1882) ([libp2p/go-libp2p#1882](https://github.com/libp2p/go-libp2p/pull/1882)) - add an integration test for muxer selection ([libp2p/go-libp2p#1887](https://github.com/libp2p/go-libp2p/pull/1887)) - core/network: fix typo - tls / noise: prefer the client's muxer preferences ([libp2p/go-libp2p#1888](https://github.com/libp2p/go-libp2p/pull/1888)) - upgrader: absorb the muxer_multistream.Transport into the upgrader (#1885) ([libp2p/go-libp2p#1885](https://github.com/libp2p/go-libp2p/pull/1885)) - Apply service peer default (#1878) ([libp2p/go-libp2p#1878](https://github.com/libp2p/go-libp2p/pull/1878)) - webtransport: use deterministic TLS certificates (#1833) ([libp2p/go-libp2p#1833](https://github.com/libp2p/go-libp2p/pull/1833)) - remove deprecated StaticRelays option (#1868) ([libp2p/go-libp2p#1868](https://github.com/libp2p/go-libp2p/pull/1868)) - autorelay: remove the default static relay option (#1867) ([libp2p/go-libp2p#1867](https://github.com/libp2p/go-libp2p/pull/1867)) - core/protocol: remove deprecated Negotiator.NegotiateLazy (#1869) ([libp2p/go-libp2p#1869](https://github.com/libp2p/go-libp2p/pull/1869)) - config: use fx dependency injection to construct transports ([libp2p/go-libp2p#1858](https://github.com/libp2p/go-libp2p/pull/1858)) - noise: add an option to allow unknown peer ID in SecureOutbound (#1823) ([libp2p/go-libp2p#1823](https://github.com/libp2p/go-libp2p/pull/1823)) - Add some guard rails and docs (#1863) ([libp2p/go-libp2p#1863](https://github.com/libp2p/go-libp2p/pull/1863)) - Fix concurrent map access in connmgr (#1860) ([libp2p/go-libp2p#1860](https://github.com/libp2p/go-libp2p/pull/1860)) - fix: return filtered addrs (#1855) ([libp2p/go-libp2p#1855](https://github.com/libp2p/go-libp2p/pull/1855)) - chore: preallocate slices (#1842) ([libp2p/go-libp2p#1842](https://github.com/libp2p/go-libp2p/pull/1842)) - Close ping stream when we exit the loop (#1853) ([libp2p/go-libp2p#1853](https://github.com/libp2p/go-libp2p/pull/1853)) - tls: don't set the deprecated tls.Config.PreferServerCipherSuites field (#1845) ([libp2p/go-libp2p#1845](https://github.com/libp2p/go-libp2p/pull/1845)) - routed host: search for new multi addresses upon connect failure (#1835) ([libp2p/go-libp2p#1835](https://github.com/libp2p/go-libp2p/pull/1835)) - core/peerstore: removed unused provider addr ttl constant (#1848) ([libp2p/go-libp2p#1848](https://github.com/libp2p/go-libp2p/pull/1848)) - basichost: improve protocol negotiation debug message (#1846) ([libp2p/go-libp2p#1846](https://github.com/libp2p/go-libp2p/pull/1846)) - noise: use Noise Extension to negotiate the muxer during the handshake (#1813) ([libp2p/go-libp2p#1813](https://github.com/libp2p/go-libp2p/pull/1813)) - webtransport: use the rcmgr to control flow control window increases ([libp2p/go-libp2p#1832](https://github.com/libp2p/go-libp2p/pull/1832)) - chore: update quic-go to v0.30.0 (#1838) ([libp2p/go-libp2p#1838](https://github.com/libp2p/go-libp2p/pull/1838)) - roadmap: reorder priority, reorganize sections (#1831) ([libp2p/go-libp2p#1831](https://github.com/libp2p/go-libp2p/pull/1831)) - websocket: set the HTTP host header in WSS(#1834) ([libp2p/go-libp2p#1834](https://github.com/libp2p/go-libp2p/pull/1834)) - webtransport: make it possible to record qlogs (controlled by QLOGDIR env) ([libp2p/go-libp2p#1828](https://github.com/libp2p/go-libp2p/pull/1828)) - ipfs /api/v0/id is post (#1819) ([libp2p/go-libp2p#1819](https://github.com/libp2p/go-libp2p/pull/1819)) - examples: connect to all peers in example mdns chat app (#1798) ([libp2p/go-libp2p#1798](https://github.com/libp2p/go-libp2p/pull/1798)) - roadmap: fix header level on "Mid Q4" (#1818) ([libp2p/go-libp2p#1818](https://github.com/libp2p/go-libp2p/pull/1818)) - examples: use circuitv2 in relay example (#1795) ([libp2p/go-libp2p#1795](https://github.com/libp2p/go-libp2p/pull/1795)) - add a roadmap for the next 6 months (#1784) ([libp2p/go-libp2p#1784](https://github.com/libp2p/go-libp2p/pull/1784)) - tls: use ALPN to negotiate the stream multiplexer (#1772) ([libp2p/go-libp2p#1772](https://github.com/libp2p/go-libp2p/pull/1772)) - tls: add tests for test vector from the spec (#1788) ([libp2p/go-libp2p#1788](https://github.com/libp2p/go-libp2p/pull/1788)) - examples: update go-libp2p to v0.23.x (#1803) ([libp2p/go-libp2p#1803](https://github.com/libp2p/go-libp2p/pull/1803)) - Try increasing timeouts if we're in CI for this test (#1796) ([libp2p/go-libp2p#1796](https://github.com/libp2p/go-libp2p/pull/1796)) - Don't use rcmgr in this test (#1799) ([libp2p/go-libp2p#1799](https://github.com/libp2p/go-libp2p/pull/1799)) - Bump timeout in CI for flaky test (#1800) ([libp2p/go-libp2p#1800](https://github.com/libp2p/go-libp2p/pull/1800)) - Bump timeout in CI for flaky test (#1801) ([libp2p/go-libp2p#1801](https://github.com/libp2p/go-libp2p/pull/1801)) - Fix comment in webtransport client auth handshake (#1793) ([libp2p/go-libp2p#1793](https://github.com/libp2p/go-libp2p/pull/1793)) - examples: add basic pubsub-with-rendezvous example (#1738) ([libp2p/go-libp2p#1738](https://github.com/libp2p/go-libp2p/pull/1738)) - quic: speed up the stateless reset test case (#1778) ([libp2p/go-libp2p#1778](https://github.com/libp2p/go-libp2p/pull/1778)) - tls: fix flaky handshake cancellation test (#1779) ([libp2p/go-libp2p#1779](https://github.com/libp2p/go-libp2p/pull/1779)) - github.com/libp2p/go-libp2p-gostream (v0.3.0 -> v0.5.0): - release v0.5.0 (#74) ([libp2p/go-libp2p-gostream#74](https://github.com/libp2p/go-libp2p-gostream/pull/74)) - update go-libp2p to v0.22.0 (#73) ([libp2p/go-libp2p-gostream#73](https://github.com/libp2p/go-libp2p-gostream/pull/73)) - Expose some read-only methods on the underlying Stream interface (#67) ([libp2p/go-libp2p-gostream#67](https://github.com/libp2p/go-libp2p-gostream/pull/67)) - Update libp2p ([libp2p/go-libp2p-gostream#69](https://github.com/libp2p/go-libp2p-gostream/pull/69)) - sync: update CI config files (#65) ([libp2p/go-libp2p-gostream#65](https://github.com/libp2p/go-libp2p-gostream/pull/65)) - sync: update CI config files ([libp2p/go-libp2p-gostream#62](https://github.com/libp2p/go-libp2p-gostream/pull/62)) - fix staticcheck ([libp2p/go-libp2p-gostream#61](https://github.com/libp2p/go-libp2p-gostream/pull/61)) - github.com/libp2p/go-libp2p-http (v0.2.1 -> v0.4.0): - release v0.4.0 ([libp2p/go-libp2p-http#81](https://github.com/libp2p/go-libp2p-http/pull/81)) - sync: update CI config files ([libp2p/go-libp2p-http#79](https://github.com/libp2p/go-libp2p-http/pull/79)) - Update to latest go-libp2p ([libp2p/go-libp2p-http#80](https://github.com/libp2p/go-libp2p-http/pull/80)) - Update to latest go-libp2p ([libp2p/go-libp2p-http#78](https://github.com/libp2p/go-libp2p-http/pull/78)) - sync: update CI config files (#73) ([libp2p/go-libp2p-http#73](https://github.com/libp2p/go-libp2p-http/pull/73)) - github.com/libp2p/go-libp2p-kad-dht (v0.18.0 -> v0.20.0): - release v0.20.0 (#803) ([libp2p/go-libp2p-kad-dht#803](https://github.com/libp2p/go-libp2p-kad-dht/pull/803)) - feat: increase the max record age to 48h (PUT_VALUE, RFM17) (#794) ([libp2p/go-libp2p-kad-dht#794](https://github.com/libp2p/go-libp2p-kad-dht/pull/794)) - feat: increase expiration time for Provider Records to 48h (RFM17) - release v0.19.0 (#801) ([libp2p/go-libp2p-kad-dht#801](https://github.com/libp2p/go-libp2p-kad-dht/pull/801)) - define the ProviderAddrTTL in this repo (#797) ([libp2p/go-libp2p-kad-dht#797](https://github.com/libp2p/go-libp2p-kad-dht/pull/797)) - github.com/libp2p/go-libp2p-kbucket (v0.4.7 -> v0.5.0): - chore: release 0.5.0 (#111) ([libp2p/go-libp2p-kbucket#111](https://github.com/libp2p/go-libp2p-kbucket/pull/111)) - deprecate go-libp2p-core and use go-libp2p instead (#109) ([libp2p/go-libp2p-kbucket#109](https://github.com/libp2p/go-libp2p-kbucket/pull/109)) - sync: update CI config files (#108) ([libp2p/go-libp2p-kbucket#108](https://github.com/libp2p/go-libp2p-kbucket/pull/108)) - sync: update CI config files ([libp2p/go-libp2p-kbucket#107](https://github.com/libp2p/go-libp2p-kbucket/pull/107)) - sync: update CI config files (#104) ([libp2p/go-libp2p-kbucket#104](https://github.com/libp2p/go-libp2p-kbucket/pull/104)) - sync: update CI config files ([libp2p/go-libp2p-kbucket#101](https://github.com/libp2p/go-libp2p-kbucket/pull/101)) - sync: update CI config files ([libp2p/go-libp2p-kbucket#99](https://github.com/libp2p/go-libp2p-kbucket/pull/99)) - fix staticcheck ([libp2p/go-libp2p-kbucket#98](https://github.com/libp2p/go-libp2p-kbucket/pull/98)) - github.com/libp2p/go-libp2p-pubsub (v0.6.1 -> v0.8.2): - Add docstring for WithAppSpecificRPCInspector (#510) ([libp2p/go-libp2p-pubsub#510](https://github.com/libp2p/go-libp2p-pubsub/pull/510)) - Adds Application Specific RPC Inspector (#509) ([libp2p/go-libp2p-pubsub#509](https://github.com/libp2p/go-libp2p-pubsub/pull/509)) - chore: ignore signing keys during WithLocalPublication publishing (#497) ([libp2p/go-libp2p-pubsub#497](https://github.com/libp2p/go-libp2p-pubsub/pull/497)) - improve handling of dead peers (#508) ([libp2p/go-libp2p-pubsub#508](https://github.com/libp2p/go-libp2p-pubsub/pull/508)) - perf: use pooled buffers for message writes (#507) ([libp2p/go-libp2p-pubsub#507](https://github.com/libp2p/go-libp2p-pubsub/pull/507)) - perf: use msgio pooled buffers for received msgs (#500) ([libp2p/go-libp2p-pubsub#500](https://github.com/libp2p/go-libp2p-pubsub/pull/500)) - Enables injectable GossipSub router (#503) ([libp2p/go-libp2p-pubsub#503](https://github.com/libp2p/go-libp2p-pubsub/pull/503)) - Enables non-atomic validation for peer scoring parameters (#499) ([libp2p/go-libp2p-pubsub#499](https://github.com/libp2p/go-libp2p-pubsub/pull/499)) - update go-libp2p to v0.22.0 (#498) ([libp2p/go-libp2p-pubsub#498](https://github.com/libp2p/go-libp2p-pubsub/pull/498)) - fix handling of dead peers (#492) ([libp2p/go-libp2p-pubsub#492](https://github.com/libp2p/go-libp2p-pubsub/pull/492)) - feat: WithLocalPublication option to enable local only publishing on a topic (#481) ([libp2p/go-libp2p-pubsub#481](https://github.com/libp2p/go-libp2p-pubsub/pull/481)) - update pubsub deps (#491) ([libp2p/go-libp2p-pubsub#491](https://github.com/libp2p/go-libp2p-pubsub/pull/491)) - Gossipsub: Unsubscribe backoff (#488) ([libp2p/go-libp2p-pubsub#488](https://github.com/libp2p/go-libp2p-pubsub/pull/488)) - Adds exponential backoff to re-spawning new streams for supposedly dead peers (#483) ([libp2p/go-libp2p-pubsub#483](https://github.com/libp2p/go-libp2p-pubsub/pull/483)) - Publishing option for signing a message with a custom private key (#486) ([libp2p/go-libp2p-pubsub#486](https://github.com/libp2p/go-libp2p-pubsub/pull/486)) - fix unused GossipSubHistoryGossip, make seenMessages ttl configurable, make score params SeenMsgTTL configurable - Update README.md - Add in Backoff Check - Modify comment - Add Backoff For Pruned Peers - tests: new test for WithTopicMsgIdFunction - chore: better name - feat: detach WithMsgIdFunction - fix: use RawID in traceRPCMeta to avoid allocations - feat: extract RawID from ID - chore: hello mister mutex hat - chore: go fmt and return timecache named import - feat: new WithMsgIdFunction topic option to enable topics to have own msg id generation rules - feat: integrate msgIdGenerator - feat: introduce msgIdGenerator and add ID field to Message wrapper - github.com/libp2p/go-libp2p-pubsub-router (v0.5.0 -> v0.6.0): - release v0.6.0 (#99) ([libp2p/go-libp2p-pubsub-router#99](https://github.com/libp2p/go-libp2p-pubsub-router/pull/99)) - sync: update CI config files (#93) ([libp2p/go-libp2p-pubsub-router#93](https://github.com/libp2p/go-libp2p-pubsub-router/pull/93)) - github.com/libp2p/go-libp2p-routing-helpers (v0.4.0 -> v0.6.0): - Update version.json - Change interface name - Add tests - Feat: retrieve routers from composable routers - Update version.json - Update version.json - chore: add regression test for compparallel deadlock - Add logs to composable parallel - Bump version to v0.4.1 - ([libp2p/go-libp2p-routing-helpers#64](https://github.com/libp2p/go-libp2p-routing-helpers/pull/64)) - github.com/lucas-clemente/quic-go (v0.29.1 -> v0.31.1): - qerr: include role (remote / local) in error string representations (#3629) ([lucas-clemente/quic-go#3629](https://github.com/lucas-clemente/quic-go/pull/3629)) - introduce a type for the stateless reset key (#3621) ([lucas-clemente/quic-go#3621](https://github.com/lucas-clemente/quic-go/pull/3621)) - limit the exponential PTO backoff to 60s (#3595) ([lucas-clemente/quic-go#3595](https://github.com/lucas-clemente/quic-go/pull/3595)) - expose the QUIC version of a connection (#3620) ([lucas-clemente/quic-go#3620](https://github.com/lucas-clemente/quic-go/pull/3620)) - expose function to convert byte slice to a connection ID (#3614) ([lucas-clemente/quic-go#3614](https://github.com/lucas-clemente/quic-go/pull/3614)) - use `go run` for mockgen, goimports and ginkgo (#3616) ([lucas-clemente/quic-go#3616](https://github.com/lucas-clemente/quic-go/pull/3616)) - fix client SNI handling (#3613) ([lucas-clemente/quic-go#3613](https://github.com/lucas-clemente/quic-go/pull/3613)) - chore: fix multiple typos in comments (#3612) ([lucas-clemente/quic-go#3612](https://github.com/lucas-clemente/quic-go/pull/3612)) - use the new zero-allocation control message parsing function from x/sys (#3609) ([lucas-clemente/quic-go#3609](https://github.com/lucas-clemente/quic-go/pull/3609)) - http3: add support for parsing and writing HTTP/3 capsules (#3607) ([lucas-clemente/quic-go#3607](https://github.com/lucas-clemente/quic-go/pull/3607)) - http3: add request to response (#3608) ([lucas-clemente/quic-go#3608](https://github.com/lucas-clemente/quic-go/pull/3608)) - fix availability signaling of the send queue (#3597) ([lucas-clemente/quic-go#3597](https://github.com/lucas-clemente/quic-go/pull/3597)) - http3: add a ConnectionState method to the StreamCreator interface (#3600) ([lucas-clemente/quic-go#3600](https://github.com/lucas-clemente/quic-go/pull/3600)) - http3: add a Context method to the StreamCreator interface (#3601) ([lucas-clemente/quic-go#3601](https://github.com/lucas-clemente/quic-go/pull/3601)) - rename the variable in quic.Config.AllowConnectionWindowIncrease (#3602) ([lucas-clemente/quic-go#3602](https://github.com/lucas-clemente/quic-go/pull/3602)) - migrate to Ginkgo v2, remove benchmark test ([lucas-clemente/quic-go#3589](https://github.com/lucas-clemente/quic-go/pull/3589)) - don't drop more than 10 consecutive packets in drop test (#3584) ([lucas-clemente/quic-go#3584](https://github.com/lucas-clemente/quic-go/pull/3584)) - use a monotonous timer for the connection (#3570) ([lucas-clemente/quic-go#3570](https://github.com/lucas-clemente/quic-go/pull/3570)) - http3: add http3.Server.ServeQUICConn to serve a single QUIC connection (#3587) ([lucas-clemente/quic-go#3587](https://github.com/lucas-clemente/quic-go/pull/3587)) - http3: expose ALPN values (#3580) ([lucas-clemente/quic-go#3580](https://github.com/lucas-clemente/quic-go/pull/3580)) - log the size of buffered packets (#3571) ([lucas-clemente/quic-go#3571](https://github.com/lucas-clemente/quic-go/pull/3571)) - ackhandler: reject duplicate packets in ReceivedPacket (#3568) ([lucas-clemente/quic-go#3568](https://github.com/lucas-clemente/quic-go/pull/3568)) - reduce max DATAGRAM frame size, so that DATAGRAMs fit in IPv6 packets (#3581) ([lucas-clemente/quic-go#3581](https://github.com/lucas-clemente/quic-go/pull/3581)) - use a Peek / Pop API for the datagram queue (#3582) ([lucas-clemente/quic-go#3582](https://github.com/lucas-clemente/quic-go/pull/3582)) - http3: handle ErrAbortHandler when the handler panics (#3575) ([lucas-clemente/quic-go#3575](https://github.com/lucas-clemente/quic-go/pull/3575)) - http3: fix double close of chan when using DontCloseRequestStream (#3561) ([lucas-clemente/quic-go#3561](https://github.com/lucas-clemente/quic-go/pull/3561)) - qlog: rename key_retired to key_discarded (#3463) ([lucas-clemente/quic-go#3463](https://github.com/lucas-clemente/quic-go/pull/3463)) - simplify packing of ACK-only packets ([lucas-clemente/quic-go#3545](https://github.com/lucas-clemente/quic-go/pull/3545)) - use a sync.Pool for ACK frames ([lucas-clemente/quic-go#3547](https://github.com/lucas-clemente/quic-go/pull/3547)) - prioritize sending ACKs over sending new DATAGRAM frames (#3544) ([lucas-clemente/quic-go#3544](https://github.com/lucas-clemente/quic-go/pull/3544)) - http3: reduce usage of bytes.Buffer (#3539) ([lucas-clemente/quic-go#3539](https://github.com/lucas-clemente/quic-go/pull/3539)) - use a single bytes.Reader for frame parsing (#3536) ([lucas-clemente/quic-go#3536](https://github.com/lucas-clemente/quic-go/pull/3536)) - split code paths for packing 0-RTT and 1-RTT packets in packet packer (#3540) ([lucas-clemente/quic-go#3540](https://github.com/lucas-clemente/quic-go/pull/3540)) - remove the wire.ShortHeader in favor of more return values (#3535) ([lucas-clemente/quic-go#3535](https://github.com/lucas-clemente/quic-go/pull/3535)) - preallocate the message buffers of the ipv4.Message passed to ReadBatch (#3541) ([lucas-clemente/quic-go#3541](https://github.com/lucas-clemente/quic-go/pull/3541)) - introduce a separate code paths for Short Header packet handling ([lucas-clemente/quic-go#3534](https://github.com/lucas-clemente/quic-go/pull/3534)) - fix usage of ackhandler.Packet pool for non-ack-eliciting packets (#3538) ([lucas-clemente/quic-go#3538](https://github.com/lucas-clemente/quic-go/pull/3538)) - return an error when parsing a too long connection ID from a header (#3533) ([lucas-clemente/quic-go#3533](https://github.com/lucas-clemente/quic-go/pull/3533)) - speed up marshaling of transport parameters (#3531) ([lucas-clemente/quic-go#3531](https://github.com/lucas-clemente/quic-go/pull/3531)) - use a struct containing an array to represent Connection IDs ([lucas-clemente/quic-go#3529](https://github.com/lucas-clemente/quic-go/pull/3529)) - reduce allocations of ackhandler.Packet ([lucas-clemente/quic-go#3525](https://github.com/lucas-clemente/quic-go/pull/3525)) - serialize frames by appending to a byte slice, not to a bytes.Buffer ([lucas-clemente/quic-go#3530](https://github.com/lucas-clemente/quic-go/pull/3530)) - fix datagram RFC number in documentation for quic.Config (#3523) ([lucas-clemente/quic-go#3523](https://github.com/lucas-clemente/quic-go/pull/3523)) - add DPLPMTUD (RFC 8899) to list of supported RFCs in README (#3520) ([lucas-clemente/quic-go#3520](https://github.com/lucas-clemente/quic-go/pull/3520)) - use the null tracers in the tracer integration tests (#3528) ([lucas-clemente/quic-go#3528](https://github.com/lucas-clemente/quic-go/pull/3528)) - github.com/marten-seemann/webtransport-go (v0.1.1 -> v0.4.3): - release v0.4.3 (#57) ([marten-seemann/webtransport-go#57](https://github.com/marten-seemann/webtransport-go/pull/57)) - return the correct error from OpenStreamSync when context is canceled (#55) ([marten-seemann/webtransport-go#55](https://github.com/marten-seemann/webtransport-go/pull/55)) - release v0.4.2 (#54) ([marten-seemann/webtransport-go#54](https://github.com/marten-seemann/webtransport-go/pull/54)) - use a buffered channel in the acceptQueue (#53) ([marten-seemann/webtransport-go#53](https://github.com/marten-seemann/webtransport-go/pull/53)) - add a comment why using (a blocking) Read in the StreamHijacker is fine - release v0.4.1 (#52) ([marten-seemann/webtransport-go#52](https://github.com/marten-seemann/webtransport-go/pull/52)) - release session mutex when an error occurs when closing (#51) ([marten-seemann/webtransport-go#51](https://github.com/marten-seemann/webtransport-go/pull/51)) - release v0.4.0 (#48) ([marten-seemann/webtransport-go#48](https://github.com/marten-seemann/webtransport-go/pull/48)) - add a Server.ServeQUICConn method (#47) ([marten-seemann/webtransport-go#47](https://github.com/marten-seemann/webtransport-go/pull/47)) - release v0.3.0 (#46) ([marten-seemann/webtransport-go#46](https://github.com/marten-seemann/webtransport-go/pull/46)) - read and write CLOSE_WEBTRANSPORT_SESSION capsules ([marten-seemann/webtransport-go#40](https://github.com/marten-seemann/webtransport-go/pull/40)) - implement the SetDeadline method on the stream (#44) ([marten-seemann/webtransport-go#44](https://github.com/marten-seemann/webtransport-go/pull/44)) - expose the QUIC stream ID on the stream interfaces (#43) ([marten-seemann/webtransport-go#43](https://github.com/marten-seemann/webtransport-go/pull/43)) - release v0.2.0 (#38) ([marten-seemann/webtransport-go#38](https://github.com/marten-seemann/webtransport-go/pull/38)) - expose quic-go's connection tracing ID on the Session.Context (#35) ([marten-seemann/webtransport-go#35](https://github.com/marten-seemann/webtransport-go/pull/35)) - add a ConnectionState method to the Session (#33) ([marten-seemann/webtransport-go#33](https://github.com/marten-seemann/webtransport-go/pull/33)) - chore: update quic-go to v0.30.0 (#36) ([marten-seemann/webtransport-go#36](https://github.com/marten-seemann/webtransport-go/pull/36)) - fix interop build (#37) ([marten-seemann/webtransport-go#37](https://github.com/marten-seemann/webtransport-go/pull/37)) - rename session receiver variable (#34) ([marten-seemann/webtransport-go#34](https://github.com/marten-seemann/webtransport-go/pull/34)) - fix double close of chan when using DontCloseRequestStream (#30) ([marten-seemann/webtransport-go#30](https://github.com/marten-seemann/webtransport-go/pull/30)) - add a simple integration test using Selenium and a headless Chrome (#28) ([marten-seemann/webtransport-go#28](https://github.com/marten-seemann/webtransport-go/pull/28)) - use a generic accept queue for uni- and bidirectional streams (#26) ([marten-seemann/webtransport-go#26](https://github.com/marten-seemann/webtransport-go/pull/26)) - github.com/multiformats/go-base36 (v0.1.0 -> v0.2.0): - v0.2.0 - sync: update CI config files (#11) ([multiformats/go-base36#11](https://github.com/multiformats/go-base36/pull/11)) - fix link to documentation (#9) ([multiformats/go-base36#9](https://github.com/multiformats/go-base36/pull/9)) - sync: update CI config files (#8) ([multiformats/go-base36#8](https://github.com/multiformats/go-base36/pull/8)) - Address `staticcheck` issue ([multiformats/go-base36#5](https://github.com/multiformats/go-base36/pull/5)) - Feat/fasterer ([multiformats/go-base36#4](https://github.com/multiformats/go-base36/pull/4)) - github.com/multiformats/go-multiaddr (v0.7.0 -> v0.8.0): - release v0.8.0 ([multiformats/go-multiaddr#187](https://github.com/multiformats/go-multiaddr/pull/187)) - Add quic-v1 component ([multiformats/go-multiaddr#186](https://github.com/multiformats/go-multiaddr/pull/186)) - github.com/multiformats/go-varint (v0.0.6 -> v0.0.7): - v0.0.7 ([multiformats/go-varint#18](https://github.com/multiformats/go-varint/pull/18)) - feat: optimize decoding (#15) ([multiformats/go-varint#15](https://github.com/multiformats/go-varint/pull/15)) - sync: update CI config files (#13) ([multiformats/go-varint#13](https://github.com/multiformats/go-varint/pull/13)) - fix staticcheck ([multiformats/go-varint#9](https://github.com/multiformats/go-varint/pull/9)) - tests for unbounded uvarint streams. ([multiformats/go-varint#7](https://github.com/multiformats/go-varint/pull/7)) - github.com/whyrusleeping/cbor-gen (v0.0.0-20210219115102-f37d292932f2 -> v0.0.0-20221220214510-0333c149dec0): - fix bug in consts code - Allow 'const' fields to be declared ([whyrusleeping/cbor-gen#78](https://github.com/whyrusleeping/cbor-gen/pull/78)) - support omitting empty fields on map encoders ([whyrusleeping/cbor-gen#77](https://github.com/whyrusleeping/cbor-gen/pull/77)) - support string pointers - feat: add the ability to encode a byte array (#76) ([whyrusleeping/cbor-gen#76](https://github.com/whyrusleeping/cbor-gen/pull/76)) - support string slices (#73) ([whyrusleeping/cbor-gen#73](https://github.com/whyrusleeping/cbor-gen/pull/73)) - Feat/size types ([whyrusleeping/cbor-gen#69](https://github.com/whyrusleeping/cbor-gen/pull/69)) - Add CborReader and CborWriter (#67) ([whyrusleeping/cbor-gen#67](https://github.com/whyrusleeping/cbor-gen/pull/67)) - Fix broken TestTimeIsh (#66) ([whyrusleeping/cbor-gen#66](https://github.com/whyrusleeping/cbor-gen/pull/66)) - Return EOF and ErrUnexpectedEOF correctly (#64) ([whyrusleeping/cbor-gen#64](https://github.com/whyrusleeping/cbor-gen/pull/64)) - fix: don't fail if we try to discard nothing at the end of an object ([whyrusleeping/cbor-gen#63](https://github.com/whyrusleeping/cbor-gen/pull/63)) - Make peeker.ReadByte follow buffer.ReadByte semantics ([whyrusleeping/cbor-gen#61](https://github.com/whyrusleeping/cbor-gen/pull/61)) - Fix read bug in readByteBuf ([whyrusleeping/cbor-gen#60](https://github.com/whyrusleeping/cbor-gen/pull/60)) - support for cborgen struct field tags ([whyrusleeping/cbor-gen#58](https://github.com/whyrusleeping/cbor-gen/pull/58)) - feat: take cbor adapters by-value when encoding ([whyrusleeping/cbor-gen#55](https://github.com/whyrusleeping/cbor-gen/pull/55)) - fix: import "math" in generated code for uint8 unmarshalling ([whyrusleeping/cbor-gen#53](https://github.com/whyrusleeping/cbor-gen/pull/53)) - doc: document Write*EncodersToFile functions ([whyrusleeping/cbor-gen#52](https://github.com/whyrusleeping/cbor-gen/pull/52))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Marten Seemann | 154 | +8826/-6369 | 911 | | Gus Eggert | 7 | +2792/-1444 | 40 | | Marco Munizaga | 26 | +2324/-752 | 101 | | hannahhoward | 7 | +695/-1587 | 50 | | Rod Vagg | 30 | +1508/-668 | 106 | | Henrique Dias | 13 | +1321/-431 | 85 | | Yahya Hassanzadeh | 4 | +984/-158 | 9 | | galargh | 17 | +519/-520 | 20 | | Steve Loeppky | 11 | +612/-418 | 25 | | Antonio Navarro Perez | 30 | +742/-88 | 47 | | Marcin Rataj | 19 | +377/-407 | 52 | | Ian Davis | 2 | +419/-307 | 7 | | whyrusleeping | 5 | +670/-28 | 17 | | Piotr Galar | 8 | +211/-417 | 25 | | web3-bot | 28 | +282/-264 | 75 | | Will Scott | 10 | +428/-103 | 19 | | julian88110 | 2 | +367/-55 | 27 | | Will | 5 | +282/-131 | 65 | | Jorropo | 25 | +263/-94 | 38 | | Wondertan | 10 | +203/-87 | 24 | | Mohsin Zaidi | 1 | +269/-0 | 4 | | Dennis Trautwein | 3 | +230/-21 | 7 | | Prithvi Shahi | 1 | +116/-77 | 1 | | Masih H. Derkani | 5 | +130/-37 | 11 | | Iulian Pascalau | 1 | +151/-16 | 2 | | Scott Martin | 1 | +166/-0 | 3 | | Daniel Vernall | 1 | +92/-45 | 2 | | Steven Allen | 7 | +114/-15 | 11 | | Hlib Kanunnikov | 4 | +100/-28 | 6 | | Peter Rabbitson | 4 | +59/-65 | 5 | | Lucas Molas | 1 | +60/-57 | 7 | | nisdas | 3 | +107/-6 | 5 | | why | 2 | +80/-20 | 5 | | ShengTao | 2 | +46/-45 | 16 | | nisainan | 2 | +40/-50 | 12 | | Mikel Cortes | 3 | +44/-36 | 10 | | Chinmay Kousik | 1 | +64/-14 | 6 | | ZenGround0 | 2 | +62/-15 | 6 | | Antonio Navarro | 3 | +58/-3 | 8 | | Michael Muré | 2 | +49/-2 | 2 | | Dirk McCormick | 1 | +3/-42 | 1 | | kixelated | 1 | +20/-20 | 4 | | Russell Dempsey | 1 | +19/-17 | 3 | | Karthik Nallabolu | 1 | +17/-17 | 1 | | protolambda | 1 | +26/-4 | 4 | | cliffc-spirent | 1 | +25/-5 | 2 | | Raúl Kripalani | 1 | +29/-0 | 1 | | Håvard Anda Estensen | 1 | +9/-19 | 6 | | vyzo | 1 | +11/-12 | 1 | | anorth | 1 | +15/-8 | 3 | | shade34321 | 1 | +21/-1 | 2 | | Toby | 2 | +9/-13 | 6 | | Nishant Das | 1 | +9/-9 | 5 | | Jeromy Johnson | 1 | +17/-0 | 3 | | Oleg | 1 | +14/-1 | 1 | | Hector Sanjuan | 6 | +4/-11 | 6 | | Łukasz Magiera | 2 | +10/-4 | 2 | | Aayush Rajasekaran | 1 | +7/-7 | 1 | | Adin Schmahmann | 1 | +4/-3 | 1 | | Stojan Dimitrovski | 1 | +6/-0 | 1 | | Mathew Jacob | 1 | +3/-3 | 1 | | Vladimir Ivanov | 1 | +2/-2 | 1 | | Nex Zhu | 1 | +4/-0 | 2 | | Michele Mastrogiovanni | 1 | +2/-2 | 1 | | Louis Thibault | 1 | +4/-0 | 1 | | Eric Myhre | 1 | +3/-1 | 1 | | Kubo Mage | 2 | +2/-1 | 2 | | tabcat | 1 | +1/-1 | 1 | | Viacheslav | 1 | +1/-1 | 1 | | Max Inden | 1 | +1/-1 | 1 | | Manic Security | 1 | +1/-1 | 1 | | Jc0803kevin | 1 | +1/-1 | 1 | | David Brouwer | 1 | +2/-0 | 2 | | Rong Zhou | 1 | +1/-0 | 1 | | Neel Virdy | 1 | +1/-0 | 1 | ================================================ FILE: docs/changelogs/v0.19.md ================================================ # Kubo changelog v0.19 ## v0.19.2 ### Highlights #### FullRT DHT HTTP Routers The default HTTP routers are now used when the FullRT DHT client is used. This fixes the issue where cid.contact is not being queried by default when the accelerated DHT client was enabled. Read more in ([ipfs/kubo#9841](https://github.com/ipfs/kubo/pull/9841)). ### Changelog
Full Changelog - github.com/ipfs/kubo: - fix: use default HTTP routers when FullRT DHT client is used (#9841) ([ipfs/kubo#9841](https://github.com/ipfs/kubo/pull/9841)) - chore: update version
### Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Gus Eggert | 1 | +65/-53 | 4 | | Henrique Dias | 1 | +1/-1 | 1 | ## v0.19.1 ### 🔦 Highlights #### DHT Timeouts In v0.16.0, Kubo added the ability to configure custom content routers and DHTs with the `custom` router type, and as part of this added a default 5 minute timeout to all DHT operations. In some cases with large repos ([example](https://github.com/ipfs/kubo/issues/9722)), this can cause provide and reprovide operations to fail because the timeout is reached. This release removes these timeouts on DHT operations. If users desire these timeouts, they can be added back using [the `custom` router type](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingrouters-parameters). ### Changelog
Full Changelog - github.com/ipfs/kubo: - chore: update version - fix: remove timeout on default DHT operations (#9783) ([ipfs/kubo#9783](https://github.com/ipfs/kubo/pull/9783)) - chore: update version - github.com/ipfs/go-blockservice (v0.5.0 -> v0.5.1): - chore: release v0.5.1 - fix: remove busyloop in getBlocks by removing batching - github.com/libp2p/go-libp2p (v0.26.3 -> v0.26.4): - release v0.26.4 - autorelay: fix busy loop bug and flaky tests in relay finder (#2208) ([libp2p/go-libp2p#2208](https://github.com/libp2p/go-libp2p/pull/2208)) - github.com/libp2p/go-libp2p-routing-helpers (v0.6.1 -> v0.6.2): - Release v0.6.2 (#73) ([libp2p/go-libp2p-routing-helpers#73](https://github.com/libp2p/go-libp2p-routing-helpers/pull/73)) - feat: zero timeout on composed routers should disable timeout (#72) ([libp2p/go-libp2p-routing-helpers#72](https://github.com/libp2p/go-libp2p-routing-helpers/pull/72))
### Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Marco Munizaga | 1 | +347/-46 | 5 | | Gus Eggert | 3 | +119/-93 | 8 | | Jorropo | 2 | +20/-32 | 2 | | galargh | 2 | +2/-2 | 2 | | Marten Seemann | 1 | +2/-2 | 1 | ## v0.19.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [Improving the libp2p resource management integration](#improving-the-libp2p-resource-management-integration) - [Gateways](#gateways) - [Signed IPNS Record response format](#signed-ipns-record-response-format) - [Example fetch and inspect IPNS record](#example-fetch-and-inspect-ipns-record) - [Addition of "autoclient" router type](#addition-of-autoclient-router-type) - [Deprecation of the `ipfs pubsub` commands and matching HTTP endpoints](#deprecation-of-the-ipfs-pubsub-commands-and-matching-http-endpoints) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### Improving the libp2p resource management integration There are further followups up on libp2p resource manager improvements in Kubo [0.18.0](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.18.md#improving-libp2p-resource-management-integration-1) and [0.18.1](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.18.md#improving-libp2p-resource-management-integration): 1. `ipfs swarm limits` and `ipfs swarm stats` have been replaced by `ipfs swarm resources` to provide a single/combined view for limits and their current usage in a more intuitive ordering. 1. Removal of `Swarm.ResourceMgr.Limits` config. Instead [the power user can specify limits in a .json file that are fed directly to go-libp2p](https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md#user-supplied-override-limits). This allows the power user to take advantage of the [new resource manager types introduced in go-libp2p 0.25](https://github.com/libp2p/go-libp2p/blob/master/CHANGELOG.md#new-resource-manager-types-) including "use default", "unlimited", "block all". - Note: we don't expect most users to need these capabilities, but they are there if so. 1. [Doc updates](https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md). #### Gateways ##### Signed IPNS Record response format This release implements [IPIP-351](https://github.com/ipfs/specs/pull/351) and adds Gateway support for returning signed (verifiable) `ipns-record` (0x0300) when `/ipns/{libp2p-key}` is requested with either `Accept: application/vnd.ipfs.ipns-record` HTTP header or `?format=ipns-record` URL query parameter. The Gateway in Kubo already supported [trustless, verifiable retrieval](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval) of immutable `/ipfs/` namespace. With `?format=ipns-record`, light HTTP clients are now able to get the same level of verifiability for IPNS websites. Tooling is limited at the moment, but we are working on [go-libipfs](https://github.com/ipfs/go-libipfs/) examples that illustrate the verifiable HTTP client pattern. ##### Example: fetch IPNS record over HTTP and inspect it with `ipfs name inspect --verify` ```console $ FILE_CID=$(echo "Hello IPFS" | ipfs add --cid-version 1 -q) $ IPNS_KEY=$(ipfs key gen test) $ ipfs name publish /ipfs/$FILE_CID --key=test --ttl=30m Published to k51q..dvf1: /ipfs/bafk..z244 $ curl "http://127.0.0.1:8080/ipns/$IPNS_KEY?format=ipns-record" > signed.ipns-record $ ipfs name inspect --verify $IPNS_KEY < signed.ipns-record Value: "/ipfs/bafk..." Validity Type: "EOL" Validity: 2023-03-09T23:13:34.032977468Z Sequence: 0 TTL: 1800000000000 PublicKey: "" Signature V1: "m..." Signature V2: "m..." Data: {...} Validation results: Valid: true PublicKey: 12D3... ``` #### Addition of "autoclient" router type A new routing type "autoclient" has been added. This mode is similar to "auto", in that it is a hybrid of content routers (including Kademlia and HTTP routers), but it does not run a DHT server. This is similar to the difference between "dhtclient" and "dht" router types. See the [Routing.Type documentation](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingtype) for more information. #### Deprecation of the `ipfs pubsub` commands and matching HTTP endpoints We are deprecating `ipfs pubsub` and all `/api/v0/pubsub/` RPC endpoints and will remove them in the next release. For more information and rational see [#9717](https://github.com/ipfs/kubo/issues/9717). ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: update version - docs: 0.19 changelog ([ipfs/kubo#9707](https://github.com/ipfs/kubo/pull/9707)) - fix: canonicalize user defined headers - fix: apply API.HTTPHeaders to /webui redirect - feat: add heap allocs to 'ipfs diag profile' - fix: future proof with > rcmgr.DefaultLimit for new enum rcmgr values - test: add test for presarvation of unlimited configs for inbound systems - fix: preserve Unlimited StreamsInbound in connmgr reconciliation - test: fix flaky rcmgr test - chore: deprecate the pubsub api - test: port peering test from sharness to Go - test: use `T.TempDir` to create temporary test directory - fix: --verify forgets the verified key - test: name --verify forgets the verified key - feat: add "autoclient" routing type - test: parallelize more of rcmgr Go tests - test: port legacy DHT tests to Go - fix: t0116-gateway-cache.sh ([ipfs/kubo#9696](https://github.com/ipfs/kubo/pull/9696)) - docs: add bifrost to early testers ([ipfs/kubo#9699](https://github.com/ipfs/kubo/pull/9699)) - fix: typo in documentation for install path - chore: update version - feat: Reduce RM code footprint - Doc updates/additions - ci: replace junit html generation with gh action - test: port rcmgr sharness tests to Go - test(gateway): use deterministic CAR fixtures ([ipfs/kubo#9657](https://github.com/ipfs/kubo/pull/9657)) - feat(gateway): error handling improvements (500, 502, 504) (#9660) ([ipfs/kubo#9660](https://github.com/ipfs/kubo/pull/9660)) - docs: be clear about swarm.addrfilters (#9661) ([ipfs/kubo#9661](https://github.com/ipfs/kubo/pull/9661)) - chore: update go-libp2p to v0.26 (#9656) ([ipfs/kubo#9656](https://github.com/ipfs/kubo/pull/9656)) - feat(pinning): connect some missing go context (#9557) ([ipfs/kubo#9557](https://github.com/ipfs/kubo/pull/9557)) - fix(gateway): return HTTP 500 on ErrResolveFailed (#9589) ([ipfs/kubo#9589](https://github.com/ipfs/kubo/pull/9589)) - docs: bulk spelling edits (#9544) ([ipfs/kubo#9544](https://github.com/ipfs/kubo/pull/9544)) - docs: "remote" errors from resource manager (#9653) ([ipfs/kubo#9653](https://github.com/ipfs/kubo/pull/9653)) - test: remove gateway tests migrated to go-libipfs - fix: update rcmgr for go-libp2p v0.25 - chore: update go-libp2p to v0.25.1 - docs(0.18.1): guide users to clean up limits (#9644) ([ipfs/kubo#9644](https://github.com/ipfs/kubo/pull/9644)) - feat: add NewOptionalInteger function - fix: dereference int64 pointer in OptionalInteger.String() (#9640) ([ipfs/kubo#9640](https://github.com/ipfs/kubo/pull/9640)) - fix: restore wire format for /api/v0/routing/get|put (#9639) ([ipfs/kubo#9639](https://github.com/ipfs/kubo/pull/9639)) - refactor(gw): move Host (DNSLink and subdomain) handling to go-libipfs (#9624) ([ipfs/kubo#9624](https://github.com/ipfs/kubo/pull/9624)) - refactor: new go-libipfs/gateway API, deprecate Gateway.Writable (#9616) ([ipfs/kubo#9616](https://github.com/ipfs/kubo/pull/9616)) - Create Changelog: v0.19 ([ipfs/kubo#9617](https://github.com/ipfs/kubo/pull/9617)) - refactor: use gateway from go-libipfs (#9588) ([ipfs/kubo#9588](https://github.com/ipfs/kubo/pull/9588)) - Merge Release: v0.18.1 ([ipfs/kubo#9613](https://github.com/ipfs/kubo/pull/9613)) - Add overview section - Adjust inbound connection limits depending on memory. - feat: ipfs-webui 2.22.0 - chore: bump go-libipfs remove go-bitswap - docs: DefaultResourceMgrMinInboundConns - feat(gateway): IPNS record response format (IPIP-351) (#9399) ([ipfs/kubo#9399](https://github.com/ipfs/kubo/pull/9399)) - fix(ipns): honour --ttl flag in 'ipfs name publish' (#9471) ([ipfs/kubo#9471](https://github.com/ipfs/kubo/pull/9471)) - feat: Pubsub.SeenMessagesStrategy (#9543) ([ipfs/kubo#9543](https://github.com/ipfs/kubo/pull/9543)) - chore: bump go-libipfs to replace go-block-format - Merge Kubo: v0.18 ([ipfs/kubo#9581](https://github.com/ipfs/kubo/pull/9581)) - fix: clarity: no user supplied rcmgr limits of 0 (#9563) ([ipfs/kubo#9563](https://github.com/ipfs/kubo/pull/9563)) - fix(gateway): undesired conversions to dag-json and friends (#9566) ([ipfs/kubo#9566](https://github.com/ipfs/kubo/pull/9566)) - fix: ensure connmgr is smaller then autoscalled resource limits - fix: typo in ensureConnMgrMakeSenseVsResourcesMgr - docs: clarify browser descriptions for webtransport - fix: update saxon download path - fix: refuse to start if connmgr is smaller than resource limits and not using none connmgr - fix: User-Agent sent to HTTP routers - test: port gateway sharness tests to Go tests - fix: do not download saxon in parallel - docs: improve docs/README (#9539) ([ipfs/kubo#9539](https://github.com/ipfs/kubo/pull/9539)) - test: port CircleCI to GH Actions and improve sharness reporting (#9355) ([ipfs/kubo#9355](https://github.com/ipfs/kubo/pull/9355)) - chore: migrate from go-ipfs-files to go-libipfs/files (#9535) ([ipfs/kubo#9535](https://github.com/ipfs/kubo/pull/9535)) - fix: stats dht command when Routing.Type=auto (#9538) ([ipfs/kubo#9538](https://github.com/ipfs/kubo/pull/9538)) - fix: hint people to changing from RSA peer ids - fix(gateway): JSON when Accept is a list - fix(test): retry flaky t0125-twonode.sh - docs: fix Router config Godoc (#9528) ([ipfs/kubo#9528](https://github.com/ipfs/kubo/pull/9528)) - fix(ci): flaky sharness test - docs(config): ProviderSearchDelay (#9526) ([ipfs/kubo#9526](https://github.com/ipfs/kubo/pull/9526)) - docs: clarify debug environment variables - fix: disable provide over HTTP with Routing.Type=auto (#9511) ([ipfs/kubo#9511](https://github.com/ipfs/kubo/pull/9511)) - fix(test): stabilize flaky provider tests - feat: port pins CLI test - Removing QRI from early tester ([ipfs/kubo#9503](https://github.com/ipfs/kubo/pull/9503)) - Update Version (dev): v0.18 ([ipfs/kubo#9500](https://github.com/ipfs/kubo/pull/9500)) - github.com/ipfs/go-bitfield (v1.0.0 -> v1.1.0): - Merge pull request from GHSA-2h6c-j3gf-xp9r - sync: update CI config files (#3) ([ipfs/go-bitfield#3](https://github.com/ipfs/go-bitfield/pull/3)) - github.com/ipfs/go-block-format (v0.0.3 -> v0.1.1): - chore: release v0.1.1 - docs: fix wrong copy paste in docs - chore: release v0.1.0 - refactor: deprecate and add stub types to go-libipfs/blocks - sync: update CI config files (#34) ([ipfs/go-block-format#34](https://github.com/ipfs/go-block-format/pull/34)) - remove Makefile ([ipfs/go-block-format#31](https://github.com/ipfs/go-block-format/pull/31)) - github.com/ipfs/go-ipfs-files (v0.0.8 -> v0.3.0): - ([ipfs/go-ipfs-files#59](https://github.com/ipfs/go-ipfs-files/pull/59)) - docs: add moved noticed [ci skip] - Release v0.2.0 - fix: error when TAR has files outside of root (#56) ([ipfs/go-ipfs-files#56](https://github.com/ipfs/go-ipfs-files/pull/56)) - sync: update CI config files ([ipfs/go-ipfs-files#55](https://github.com/ipfs/go-ipfs-files/pull/55)) - chore(Directory): add DirIterator API restriction: iterate only once - Release v0.1.1 - fix: add dragonfly build option for filewriter flags - fix: add freebsd build option for filewriter flags - Release v0.1.0 - docs: fix community CONTRIBUTING.md link (#45) ([ipfs/go-ipfs-files#45](https://github.com/ipfs/go-ipfs-files/pull/45)) - chore(filewriter): cleanup writes (#43) ([ipfs/go-ipfs-files#43](https://github.com/ipfs/go-ipfs-files/pull/43)) - sync: update CI config files (#44) ([ipfs/go-ipfs-files#44](https://github.com/ipfs/go-ipfs-files/pull/44)) - sync: update CI config files ([ipfs/go-ipfs-files#40](https://github.com/ipfs/go-ipfs-files/pull/40)) - fix: manually parse the content disposition to preserve directories ([ipfs/go-ipfs-files#42](https://github.com/ipfs/go-ipfs-files/pull/42)) - fix: round timestamps down by truncating them to seconds ([ipfs/go-ipfs-files#41](https://github.com/ipfs/go-ipfs-files/pull/41)) - sync: update CI config files ([ipfs/go-ipfs-files#34](https://github.com/ipfs/go-ipfs-files/pull/34)) - Fix test failure on Windows caused by nil `sys` in mock `FileInfo` ([ipfs/go-ipfs-files#39](https://github.com/ipfs/go-ipfs-files/pull/39)) - fix staticcheck ([ipfs/go-ipfs-files#35](https://github.com/ipfs/go-ipfs-files/pull/35)) - fix linters ([ipfs/go-ipfs-files#33](https://github.com/ipfs/go-ipfs-files/pull/33)) - github.com/ipfs/go-ipfs-pinner (v0.2.1 -> v0.3.0): - chore: release v0.3.0 (#27) ([ipfs/go-ipfs-pinner#27](https://github.com/ipfs/go-ipfs-pinner/pull/27)) - feat!: add and connect missing context, remove RemovePinWithMode (#23) ([ipfs/go-ipfs-pinner#23](https://github.com/ipfs/go-ipfs-pinner/pull/23)) - sync: update CI config files ([ipfs/go-ipfs-pinner#16](https://github.com/ipfs/go-ipfs-pinner/pull/16)) - github.com/ipfs/go-ipfs-pq (v0.0.2 -> v0.0.3): - chore: release v0.0.3 - fix: enable early GC - sync: update CI config files (#10) ([ipfs/go-ipfs-pq#10](https://github.com/ipfs/go-ipfs-pq/pull/10)) - sync: update CI config files ([ipfs/go-ipfs-pq#8](https://github.com/ipfs/go-ipfs-pq/pull/8)) - remove Makefile ([ipfs/go-ipfs-pq#7](https://github.com/ipfs/go-ipfs-pq/pull/7)) - github.com/ipfs/go-libipfs (v0.2.0 -> v0.6.2): - chore: release 0.6.2 (#211) ([ipfs/go-libipfs#211](https://github.com/ipfs/go-libipfs/pull/211)) - fix(gateway): 500 on panic, recover on WithHostname - refactor: use assert in remaining gateway tests - chore: release 0.6.1 - feat: support HTTP 429 with Retry-After (#194) ([ipfs/go-libipfs#194](https://github.com/ipfs/go-libipfs/pull/194)) - docs: fix typo in README.md - fix(gateway): return 500 for all /ip[nf]s/id failures - chore: make gocritic happier - feat(gateway): improved error handling, support for 502 and 504 ([ipfs/go-libipfs#182](https://github.com/ipfs/go-libipfs/pull/182)) - feat: add content path in request context (#184) ([ipfs/go-libipfs#184](https://github.com/ipfs/go-libipfs/pull/184)) - sync: update CI config files ([ipfs/go-libipfs#159](https://github.com/ipfs/go-libipfs/pull/159)) - fix(gateway): return HTTP 500 on namesys.ErrResolveFailed (#150) ([ipfs/go-libipfs#150](https://github.com/ipfs/go-libipfs/pull/150)) - docs(examples): add UnixFS file download over Bitswap (#143) ([ipfs/go-libipfs#143](https://github.com/ipfs/go-libipfs/pull/143)) - bitswap/server/internal/decision: fix: remove unused private type - chore: release v0.6.0 - bitswap/server/internal/decision: add more non flaky tests - bitswap/server/internal/decision: add filtering on CIDs - Ignore cids that are too big. - Kill connection for peers that are using inline CIDs. - bitswap/server/internal/decision: rewrite ledger inversion - docs(readme): various updates for clarity (#171) ([ipfs/go-libipfs#171](https://github.com/ipfs/go-libipfs/pull/171)) - feat: metric for implicit index.html in dirs - fix(gateway): ensure ipfs_http_gw_get_duration_seconds gets updated - test(gateway): migrate Go tests from Kubo ([ipfs/go-libipfs#156](https://github.com/ipfs/go-libipfs/pull/156)) - docs: fix link (#165) ([ipfs/go-libipfs#165](https://github.com/ipfs/go-libipfs/pull/165)) - fix: GetIPNSRecord example gateway implementation (#158) ([ipfs/go-libipfs#158](https://github.com/ipfs/go-libipfs/pull/158)) - chore: release v0.5.0 - chore: update go-libp2p to v0.25.1 - fix(gateway): display correct error with 500 (#160) ([ipfs/go-libipfs#160](https://github.com/ipfs/go-libipfs/pull/160)) - fix: gateway car example dnslink - feat(gateway): add TAR, IPNS Record, DAG-* histograms and spans (#155) ([ipfs/go-libipfs#155](https://github.com/ipfs/go-libipfs/pull/155)) - feat(gateway): migrate subdomain and dnslink code (#153) ([ipfs/go-libipfs#153](https://github.com/ipfs/go-libipfs/pull/153)) - docs: add example of gateway that proxies to ?format=raw (#151) ([ipfs/go-libipfs#151](https://github.com/ipfs/go-libipfs/pull/151)) - docs: add example of gateway backed by CAR file (#147) ([ipfs/go-libipfs#147](https://github.com/ipfs/go-libipfs/pull/147)) - undefined ([ipfs/go-libipfs#145](https://github.com/ipfs/go-libipfs/pull/145)) - Extract Gateway Code From Kubo ([ipfs/go-libipfs#65](https://github.com/ipfs/go-libipfs/pull/65)) - Migrate go-bitswap ([ipfs/go-libipfs#63](https://github.com/ipfs/go-libipfs/pull/63)) - Use `PUT` as method to insert provider records - Migrate `go-block-format` ([ipfs/go-libipfs#58](https://github.com/ipfs/go-libipfs/pull/58)) - chore: add codecov PR comment - chore: add a logo and some basics in the README (#37) ([ipfs/go-libipfs#37](https://github.com/ipfs/go-libipfs/pull/37)) - github.com/ipfs/go-namesys (v0.6.0 -> v0.7.0): - chore: release 0.7.0 (#36) ([ipfs/go-namesys#36](https://github.com/ipfs/go-namesys/pull/36)) - feat: use PublishOptions for publishing IPNS records (#35) ([ipfs/go-namesys#35](https://github.com/ipfs/go-namesys/pull/35)) - github.com/ipfs/go-path (v0.3.0 -> v0.3.1): - chore: release v0.3.1 (#67) ([ipfs/go-path#67](https://github.com/ipfs/go-path/pull/67)) - feat: expose ErrInvalidPath and implement .Is function (#66) ([ipfs/go-path#66](https://github.com/ipfs/go-path/pull/66)) - sync: update CI config files (#60) ([ipfs/go-path#60](https://github.com/ipfs/go-path/pull/60)) - feat: add basic tracing ([ipfs/go-path#59](https://github.com/ipfs/go-path/pull/59)) - github.com/ipfs/go-peertaskqueue (v0.8.0 -> v0.8.1): - chore: release v0.8.1 - feat: add PushTasksTruncated which only push a limited amount of tasks - feat: add (*PeerTaskQueue).Clear which fully removes a peer - sync: update CI config files (#26) ([ipfs/go-peertaskqueue#26](https://github.com/ipfs/go-peertaskqueue/pull/26)) - github.com/ipfs/go-unixfs (v0.4.2 -> v0.4.4): - chore: release v0.4.4 - fix: correctly handle return errors - fix: correctly handle errors in balancedbuilder's Layout - test: fix tests after hamt issues fixes - Merge pull request from GHSA-q264-w97q-q778 - github.com/ipfs/go-unixfsnode (v1.5.1 -> v1.5.2): - Merge pull request from GHSA-4gj3-6r43-3wfc - github.com/ipfs/interface-go-ipfs-core (v0.8.2 -> v0.11.0): - chore: release v0.11.0 - test: basic routing interface test - chore: release v0.10.0 (#102) ([ipfs/interface-go-ipfs-core#102](https://github.com/ipfs/interface-go-ipfs-core/pull/102)) - feat: add RoutingAPI to CoreAPI - chore: release 0.9.0 (#101) ([ipfs/interface-go-ipfs-core#101](https://github.com/ipfs/interface-go-ipfs-core/pull/101)) - feat: add namesys publish options (#94) ([ipfs/interface-go-ipfs-core#94](https://github.com/ipfs/interface-go-ipfs-core/pull/94)) - github.com/ipld/go-car (v0.4.0 -> v0.5.0): - chore: bump version to 0.5.0 - fix: remove use of ioutil - run gofmt -s - bump go.mod to Go 1.18 and run go fix - bump go.mod to Go 1.18 and run go fix - OpenReadWriteFile: add test - blockstore: allow to pass a file to write in (#323) ([ipld/go-car#323](https://github.com/ipld/go-car/pull/323)) - feat: add `car inspect` command to cmd pkg (#320) ([ipld/go-car#320](https://github.com/ipld/go-car/pull/320)) - Separate `index.ReadFrom` tests - Only read index codec during inspection - Upgrade to the latest `go-car/v2` - Empty identity CID should be indexed when options are set - github.com/libp2p/go-libp2p (v0.24.2 -> v0.26.3): - Release v0.26.3 (#2197) ([libp2p/go-libp2p#2197](https://github.com/libp2p/go-libp2p/pull/2197)) - retract v0.26.1, release v0.26.2 (#2153) ([libp2p/go-libp2p#2153](https://github.com/libp2p/go-libp2p/pull/2153)) - rcmgr: fix JSON marshalling of ResourceManagerStat peer map (#2156) ([libp2p/go-libp2p#2156](https://github.com/libp2p/go-libp2p/pull/2156)) - release v0.26.1 ([libp2p/go-libp2p#2146](https://github.com/libp2p/go-libp2p/pull/2146)) - release v0.26.0 (#2133) ([libp2p/go-libp2p#2133](https://github.com/libp2p/go-libp2p/pull/2133)) - identify: add more detailed metrics (#2126) ([libp2p/go-libp2p#2126](https://github.com/libp2p/go-libp2p/pull/2126)) - autorelay: refactor relay finder and start autorelay after identify (#2120) ([libp2p/go-libp2p#2120](https://github.com/libp2p/go-libp2p/pull/2120)) - don't use the time value from the time.Ticker channel (#2127) ([libp2p/go-libp2p#2127](https://github.com/libp2p/go-libp2p/pull/2127)) - Wrap conn with metrics (#2131) ([libp2p/go-libp2p#2131](https://github.com/libp2p/go-libp2p/pull/2131)) - chore: update changelog for 0.26.0 (#2132) ([libp2p/go-libp2p#2132](https://github.com/libp2p/go-libp2p/pull/2132)) - circuitv2: Update proto files to proto3 (#2121) ([libp2p/go-libp2p#2121](https://github.com/libp2p/go-libp2p/pull/2121)) - swarm: remove parallel tests from swarm tests (#2130) ([libp2p/go-libp2p#2130](https://github.com/libp2p/go-libp2p/pull/2130)) - circuitv2: add a relay option to disable limits (#2125) ([libp2p/go-libp2p#2125](https://github.com/libp2p/go-libp2p/pull/2125)) - quic: fix stalled virtual listener (#2122) ([libp2p/go-libp2p#2122](https://github.com/libp2p/go-libp2p/pull/2122)) - swarm: add early muxer selection to swarm metrics (#2119) ([libp2p/go-libp2p#2119](https://github.com/libp2p/go-libp2p/pull/2119)) - metrics: add options to disable metrics and to set Prometheus registerer (#2116) ([libp2p/go-libp2p#2116](https://github.com/libp2p/go-libp2p/pull/2116)) - swarm: add ip_version to metrics (#2114) ([libp2p/go-libp2p#2114](https://github.com/libp2p/go-libp2p/pull/2114)) - Revert mistaken "Bump timeout" - Bump timeout - remove all circuit v1 related code (#2107) ([libp2p/go-libp2p#2107](https://github.com/libp2p/go-libp2p/pull/2107)) - quic: don't send detailed error messages when closing connections (#2112) ([libp2p/go-libp2p#2112](https://github.com/libp2p/go-libp2p/pull/2112)) - metrics: add no alloc metrics for eventbus, swarm, identify (#2108) ([libp2p/go-libp2p#2108](https://github.com/libp2p/go-libp2p/pull/2108)) - chore: fix typo in Changelog (#2111) ([libp2p/go-libp2p#2111](https://github.com/libp2p/go-libp2p/pull/2111)) - chore: update changelog (#2109) ([libp2p/go-libp2p#2109](https://github.com/libp2p/go-libp2p/pull/2109)) - chore: unify dashboard location (#2110) ([libp2p/go-libp2p#2110](https://github.com/libp2p/go-libp2p/pull/2110)) - autonat: add metrics (#2086) ([libp2p/go-libp2p#2086](https://github.com/libp2p/go-libp2p/pull/2086)) - relaymanager: do not start new relay if one already exists (#2093) ([libp2p/go-libp2p#2093](https://github.com/libp2p/go-libp2p/pull/2093)) - autonat: don't emit reachability changed events on address change (#2092) ([libp2p/go-libp2p#2092](https://github.com/libp2p/go-libp2p/pull/2092)) - chore: modify changelog entries (#2101) ([libp2p/go-libp2p#2101](https://github.com/libp2p/go-libp2p/pull/2101)) - Introduce a changelog (#2084) ([libp2p/go-libp2p#2084](https://github.com/libp2p/go-libp2p/pull/2084)) - use atomic.Int32 and atomic.Int64 (#2096) ([libp2p/go-libp2p#2096](https://github.com/libp2p/go-libp2p/pull/2096)) - change atomic.Value to atomic.Pointer (#2088) ([libp2p/go-libp2p#2088](https://github.com/libp2p/go-libp2p/pull/2088)) - use atomic.Bool instead of int32 operations (#2089) ([libp2p/go-libp2p#2089](https://github.com/libp2p/go-libp2p/pull/2089)) - sync: update CI config files (#2073) ([libp2p/go-libp2p#2073](https://github.com/libp2p/go-libp2p/pull/2073)) - chore: update examples to v0.25.1 (#2080) ([libp2p/go-libp2p#2080](https://github.com/libp2p/go-libp2p/pull/2080)) - v0.25.1 (#2082) ([libp2p/go-libp2p#2082](https://github.com/libp2p/go-libp2p/pull/2082)) - Start host in mocknet (#2078) ([libp2p/go-libp2p#2078](https://github.com/libp2p/go-libp2p/pull/2078)) - Release v0.25.0 (#2077) ([libp2p/go-libp2p#2077](https://github.com/libp2p/go-libp2p/pull/2077)) - identify: add some basic metrics (#2069) ([libp2p/go-libp2p#2069](https://github.com/libp2p/go-libp2p/pull/2069)) - p2p/test/quic: use contexts with a timeout for Connect calls (#2070) ([libp2p/go-libp2p#2070](https://github.com/libp2p/go-libp2p/pull/2070)) - feat!: rcmgr: Change LimitConfig to use LimitVal type (#2000) ([libp2p/go-libp2p#2000](https://github.com/libp2p/go-libp2p/pull/2000)) - identify: refactor sending of Identify pushes (#1984) ([libp2p/go-libp2p#1984](https://github.com/libp2p/go-libp2p/pull/1984)) - Update interop to match spec (#2049) ([libp2p/go-libp2p#2049](https://github.com/libp2p/go-libp2p/pull/2049)) - chore: git-ignore various flavors of qlog files (#2064) ([libp2p/go-libp2p#2064](https://github.com/libp2p/go-libp2p/pull/2064)) - rcmgr: add libp2p prefix to all metrics (#2063) ([libp2p/go-libp2p#2063](https://github.com/libp2p/go-libp2p/pull/2063)) - websocket: Replace gorilla websocket transport with nhooyr websocket transport (#1982) ([libp2p/go-libp2p#1982](https://github.com/libp2p/go-libp2p/pull/1982)) - rcmgr: Use prometheus SDK for rcmgr metrics (#2044) ([libp2p/go-libp2p#2044](https://github.com/libp2p/go-libp2p/pull/2044)) - autorelay: Split libp2p.EnableAutoRelay into 2 functions (#2022) ([libp2p/go-libp2p#2022](https://github.com/libp2p/go-libp2p/pull/2022)) - set names for eventbus event subscriptions (#2057) ([libp2p/go-libp2p#2057](https://github.com/libp2p/go-libp2p/pull/2057)) - Test cleanup (#2053) ([libp2p/go-libp2p#2053](https://github.com/libp2p/go-libp2p/pull/2053)) - metrics: use a single slice pool for all metrics tracer (#2054) ([libp2p/go-libp2p#2054](https://github.com/libp2p/go-libp2p/pull/2054)) - eventbus: add metrics (#2038) ([libp2p/go-libp2p#2038](https://github.com/libp2p/go-libp2p/pull/2038)) - quic: disable sending of Version Negotiation packets (#2015) ([libp2p/go-libp2p#2015](https://github.com/libp2p/go-libp2p/pull/2015)) - p2p/test: fix flaky notification test (#2051) ([libp2p/go-libp2p#2051](https://github.com/libp2p/go-libp2p/pull/2051)) - quic, tcp: only register Prometheus counters when metrics are enabled ([libp2p/go-libp2p#1971](https://github.com/libp2p/go-libp2p/pull/1971)) - p2p/test: add test for EvtLocalAddressesUpdated event (#2016) ([libp2p/go-libp2p#2016](https://github.com/libp2p/go-libp2p/pull/2016)) - quic / webtransport: extend test to test dialing draft-29 and v1 (#1957) ([libp2p/go-libp2p#1957](https://github.com/libp2p/go-libp2p/pull/1957)) - holepunch: fix flaky by not remove holepunch protocol handler (#1948) ([libp2p/go-libp2p#1948](https://github.com/libp2p/go-libp2p/pull/1948)) - use quic-go and webtransport-go from quic-go organization (#2040) ([libp2p/go-libp2p#2040](https://github.com/libp2p/go-libp2p/pull/2040)) - Migrate to test-plan composite action (#2039) ([libp2p/go-libp2p#2039](https://github.com/libp2p/go-libp2p/pull/2039)) - chore: remove license files from the eventbus package (#2042) ([libp2p/go-libp2p#2042](https://github.com/libp2p/go-libp2p/pull/2042)) - rcmgr: *: Always close connscope (#2037) ([libp2p/go-libp2p#2037](https://github.com/libp2p/go-libp2p/pull/2037)) - chore: remove textual roadmap in favor for Starmap (#2036) ([libp2p/go-libp2p#2036](https://github.com/libp2p/go-libp2p/pull/2036)) - swarm metrics: fix datasource for dashboard (#2024) ([libp2p/go-libp2p#2024](https://github.com/libp2p/go-libp2p/pull/2024)) - consistently use protocol.ID instead of strings (#2004) ([libp2p/go-libp2p#2004](https://github.com/libp2p/go-libp2p/pull/2004)) - swarm: add a basic metrics tracer (#1973) ([libp2p/go-libp2p#1973](https://github.com/libp2p/go-libp2p/pull/1973)) - Expose muxer ids (#2012) ([libp2p/go-libp2p#2012](https://github.com/libp2p/go-libp2p/pull/2012)) - Clean addresses with peer id before adding to addrbook (#2007) ([libp2p/go-libp2p#2007](https://github.com/libp2p/go-libp2p/pull/2007)) - feat: ci test-plans: Parse test timeout parameter for interop test (#2014) ([libp2p/go-libp2p#2014](https://github.com/libp2p/go-libp2p/pull/2014)) - Export resource manager errors (#2008) ([libp2p/go-libp2p#2008](https://github.com/libp2p/go-libp2p/pull/2008)) - peerstore: make it possible to use an empty peer ID (#2006) ([libp2p/go-libp2p#2006](https://github.com/libp2p/go-libp2p/pull/2006)) - Add ci flakiness score to readme (#2002) ([libp2p/go-libp2p#2002](https://github.com/libp2p/go-libp2p/pull/2002)) - rcmgr: fix: Ignore zero values when marshalling Limits. (#1998) ([libp2p/go-libp2p#1998](https://github.com/libp2p/go-libp2p/pull/1998)) - CI: Fast multidimensional Interop tests (#1991) ([libp2p/go-libp2p#1991](https://github.com/libp2p/go-libp2p/pull/1991)) - feat: add some users to the readme (#1981) ([libp2p/go-libp2p#1981](https://github.com/libp2p/go-libp2p/pull/1981)) - ci: run go generate as part of the go-check workflow (#1986) ([libp2p/go-libp2p#1986](https://github.com/libp2p/go-libp2p/pull/1986)) - switch to Google's Protobuf library, make protobufs compile with go generate ([libp2p/go-libp2p#1979](https://github.com/libp2p/go-libp2p/pull/1979)) - circuitv2: correctly set the transport in the ConnectionState (#1972) ([libp2p/go-libp2p#1972](https://github.com/libp2p/go-libp2p/pull/1972)) - roadmap: remove optimizations of the TCP-based handshake (#1959) ([libp2p/go-libp2p#1959](https://github.com/libp2p/go-libp2p/pull/1959)) - identify: remove support for Identify Delta ([libp2p/go-libp2p#1975](https://github.com/libp2p/go-libp2p/pull/1975)) - core: remove introspection package (#1978) ([libp2p/go-libp2p#1978](https://github.com/libp2p/go-libp2p/pull/1978)) - identify: remove old code targeting Go 1.17 (#1964) ([libp2p/go-libp2p#1964](https://github.com/libp2p/go-libp2p/pull/1964)) - add WebTransport to the list of default transports (#1915) ([libp2p/go-libp2p#1915](https://github.com/libp2p/go-libp2p/pull/1915)) - core/crypto: drop all OpenSSL code paths (#1953) ([libp2p/go-libp2p#1953](https://github.com/libp2p/go-libp2p/pull/1953)) - chore: use generic LRU cache (#1980) ([libp2p/go-libp2p#1980](https://github.com/libp2p/go-libp2p/pull/1980)) - github.com/libp2p/go-libp2p-kad-dht (v0.20.0 -> v0.21.1): - chore: bump to v0.21.1 (#821) ([libp2p/go-libp2p-kad-dht#821](https://github.com/libp2p/go-libp2p-kad-dht/pull/821)) - feat: send FIND_NODE request to peers on routing table refresh (#810) ([libp2p/go-libp2p-kad-dht#810](https://github.com/libp2p/go-libp2p-kad-dht/pull/810)) - chore: release v0.21. - chore: Update to go libp2p v0.25 ([libp2p/go-libp2p-kad-dht#815](https://github.com/libp2p/go-libp2p-kad-dht/pull/815)) - github.com/libp2p/go-libp2p-pubsub (v0.8.3 -> v0.9.0): - chore: update to go-libp2p v0.25 (#517) ([libp2p/go-libp2p-pubsub#517](https://github.com/libp2p/go-libp2p-pubsub/pull/517)) - github.com/libp2p/go-libp2p-routing-helpers (v0.6.0 -> v0.6.1): - chore: release v0.6.1 - fix: cancel parallel routers - github.com/libp2p/go-msgio (v0.2.0 -> v0.3.0): - release v0.3.0 (#39) ([libp2p/go-msgio#39](https://github.com/libp2p/go-msgio/pull/39)) - switch from deprecated gogo to google.golang.org/protobuf ([libp2p/go-msgio#38](https://github.com/libp2p/go-msgio/pull/38)) - sync: update CI config files (#36) ([libp2p/go-msgio#36](https://github.com/libp2p/go-msgio/pull/36)) - github.com/lucas-clemente/quic-go (v0.31.1 -> v0.29.1): - http3: fix double close of chan when using DontCloseRequestStream - github.com/multiformats/go-multistream (v0.3.3 -> v0.4.1): - release v0.4.1 ([multiformats/go-multistream#101](https://github.com/multiformats/go-multistream/pull/101)) - Fix errors Is checking ([multiformats/go-multistream#100](https://github.com/multiformats/go-multistream/pull/100)) - release v0.4.0 (#93) ([multiformats/go-multistream#93](https://github.com/multiformats/go-multistream/pull/93)) - switch to Go's native fuzzing (#96) ([multiformats/go-multistream#96](https://github.com/multiformats/go-multistream/pull/96)) - Add not supported protocols to returned errors (#97) ([multiformats/go-multistream#97](https://github.com/multiformats/go-multistream/pull/97)) - Make MultistreamMuxer and Client APIs generic (#95) ([multiformats/go-multistream#95](https://github.com/multiformats/go-multistream/pull/95)) - remove MultistreamMuxer.NegotiateLazy (#92) ([multiformats/go-multistream#92](https://github.com/multiformats/go-multistream/pull/92)) - sync: update CI config files (#91) ([multiformats/go-multistream#91](https://github.com/multiformats/go-multistream/pull/91)) - github.com/warpfork/go-wish (v0.0.0-20200122115046-b9ea61034e4a -> v0.0.0-20220906213052-39a1cc7a02d0): - Update readme with deprecation info - github.com/whyrusleeping/cbor-gen (v0.0.0-20221220214510-0333c149dec0 -> v0.0.0-20230126041949-52956bd4c9aa): - add setter to allow reuse of cborreader struct - fix typo - allow fields to be ignored ([whyrusleeping/cbor-gen#79](https://github.com/whyrusleeping/cbor-gen/pull/79))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Dirk McCormick | 128 | +16757/-7211 | 387 | | Henrique Dias | 69 | +7599/-10016 | 316 | | hannahhoward | 88 | +8503/-4397 | 271 | | Jeromy Johnson | 244 | +6544/-4034 | 774 | | Marten Seemann | 64 | +4870/-5628 | 266 | | Steven Allen | 296 | +4769/-3517 | 972 | | Brian Tiger Chow | 250 | +5520/-2579 | 435 | | Jorropo | 64 | +4237/-3548 | 302 | | Sukun | 18 | +4327/-1093 | 132 | | Marco Munizaga | 35 | +2809/-1294 | 94 | | Gus Eggert | 20 | +2523/-1476 | 99 | | Adin Schmahmann | 15 | +683/-2625 | 69 | | Marcin Rataj | 73 | +2348/-882 | 133 | | whyrusleeping | 12 | +1683/-1338 | 23 | | Jeromy | 99 | +1754/-1181 | 453 | | Juan Batiz-Benet | 69 | +1182/-678 | 149 | | Lars Gierth | 31 | +827/-358 | 92 | | Paul Wolneykien | 2 | +670/-338 | 9 | | Laurent Senta | 16 | +806/-134 | 53 | | Henry | 19 | +438/-372 | 36 | | Michael Muré | 8 | +400/-387 | 19 | | Łukasz Magiera | 56 | +413/-354 | 117 | | Jakub Sztandera | 40 | +413/-251 | 100 | | Justin Johnson | 2 | +479/-165 | 5 | | Piotr Galar | 7 | +227/-378 | 24 | | Kevin Atkinson | 11 | +252/-232 | 49 | | web3-bot | 17 | +236/-240 | 59 | | Petar Maymounkov | 2 | +348/-84 | 11 | | Hector Sanjuan | 38 | +206/-223 | 85 | | Antonio Navarro Perez | 9 | +259/-95 | 17 | | keks | 22 | +233/-118 | 24 | | Ho-Sheng Hsiao | 3 | +170/-170 | 30 | | Lucas Molas | 6 | +266/-54 | 16 | | Mildred Ki'Lya | 4 | +280/-35 | 7 | | Steve Loeppky | 5 | +147/-156 | 9 | | rht | 14 | +97/-188 | 20 | | Prithvi Shahi | 6 | +89/-193 | 11 | | Ian Davis | 6 | +198/-75 | 11 | | taylor | 1 | +180/-89 | 8 | | ᴍᴀᴛᴛ ʙᴇʟʟ | 14 | +158/-104 | 18 | | Chris Boddy | 6 | +190/-45 | 8 | | Rod Vagg | 3 | +203/-28 | 15 | | Masih H. Derkani | 8 | +165/-61 | 16 | | Kevin Wallace | 4 | +194/-27 | 7 | | Mohsin Zaidi | 1 | +179/-41 | 5 | | ElPaisano | 1 | +110/-110 | 22 | | Simon Zhu | 6 | +177/-32 | 8 | | galargh | 9 | +80/-120 | 14 | | Tomasz Zdybał | 1 | +180/-1 | 4 | | dgrisham | 3 | +176/-2 | 4 | | Michael Avila | 3 | +116/-59 | 8 | | Raúl Kripalani | 2 | +85/-77 | 34 | | Dr Ian Preston | 11 | +101/-48 | 11 | | JP Hastings-Spital | 1 | +145/-0 | 2 | | George Antoniadis | 6 | +59/-58 | 43 | | Kevin Neaton | 2 | +97/-16 | 4 | | Adrian Lanzafame | 6 | +81/-25 | 7 | | Dennis Trautwein | 3 | +89/-9 | 5 | | mathew-cf | 2 | +82/-9 | 5 | | tg | 1 | +41/-33 | 1 | | Eng Zer Jun | 1 | +15/-54 | 5 | | zramsay | 4 | +15/-53 | 12 | | muXxer | 1 | +28/-33 | 4 | | Thomas Eizinger | 1 | +24/-37 | 4 | | Remco Bloemen | 2 | +28/-18 | 3 | | Manuel Alonso | 1 | +36/-9 | 1 | | vyzo | 4 | +26/-12 | 13 | | Djalil Dreamski | 3 | +27/-9 | 3 | | Thomas Gardner | 2 | +32/-3 | 4 | | Jan Winkelmann | 2 | +23/-12 | 8 | | Artem Andreenko | 1 | +16/-19 | 1 | | James Stanley | 1 | +34/-0 | 1 | | Brendan McMillion | 1 | +10/-17 | 3 | | Jack Loughran | 1 | +22/-0 | 3 | | Peter Wu | 2 | +12/-9 | 2 | | Gowtham G | 4 | +14/-7 | 4 | | Tor Arne Vestbø | 3 | +19/-1 | 3 | | Cory Schwartz | 1 | +8/-12 | 5 | | Peter Rabbitson | 1 | +15/-4 | 1 | | David Dias | 1 | +9/-9 | 1 | | Will Scott | 1 | +13/-4 | 2 | | Eric Myhre | 1 | +15/-2 | 1 | | Stephen Whitmore | 1 | +8/-8 | 1 | | Rafael Ramalho | 5 | +11/-5 | 5 | | Christian Couder | 1 | +14/-2 | 1 | | W. Trevor King | 2 | +9/-6 | 3 | | Steven Vandevelde | 1 | +11/-3 | 1 | | Knut Ahlers | 3 | +9/-5 | 3 | | Bob Potter | 1 | +3/-10 | 1 | | Russell Dempsey | 4 | +8/-4 | 4 | | Diogo Silva | 4 | +8/-4 | 4 | | Dave Justice | 1 | +8/-4 | 1 | | Andy Leap | 2 | +2/-10 | 2 | | divingpetrel | 1 | +7/-4 | 2 | | Iaroslav Gridin | 1 | +9/-2 | 1 | | Dominic Della Valle | 3 | +5/-5 | 3 | | Vijayee Kulkaa | 1 | +3/-6 | 1 | | Friedel Ziegelmayer | 3 | +6/-3 | 3 | | Stephen Solka | 1 | +1/-7 | 1 | | Richard Littauer | 3 | +4/-4 | 3 | | Franky W | 2 | +4/-4 | 2 | | Dimitris Apostolou | 2 | +4/-4 | 3 | | Adrian Ulrich | 1 | +8/-0 | 1 | | Masashi Salvador Mitsuzawa | 1 | +5/-1 | 1 | | Gabe | 1 | +3/-3 | 1 | | zuuluuz | 1 | +4/-1 | 1 | | myml | 1 | +5/-0 | 1 | | swedneck | 1 | +3/-1 | 1 | | Wayback Archiver | 1 | +2/-2 | 1 | | Vladimir Ivanov | 1 | +2/-2 | 1 | | Péter Szilágyi | 1 | +2/-2 | 1 | | Karthik Bala | 1 | +2/-2 | 1 | | Etienne Laurin | 1 | +1/-3 | 1 | | Shotaro Yamada | 1 | +2/-1 | 1 | | Robert Carlsen | 1 | +2/-1 | 1 | | Oli Evans | 1 | +2/-1 | 1 | | Dan McQuillan | 1 | +2/-1 | 1 | | susarlanikhilesh | 1 | +1/-1 | 1 | | mateon1 | 1 | +1/-1 | 1 | | kpcyrd | 1 | +1/-1 | 1 | | bbenshoof | 1 | +1/-1 | 1 | | ZenGround0 | 1 | +1/-1 | 1 | | Will Hawkins | 1 | +1/-1 | 1 | | Tommi Virtanen | 1 | +1/-1 | 1 | | Seungbae Yu | 1 | +1/-1 | 1 | | Riishab Joshi | 1 | +1/-1 | 1 | | Kubo Mage | 1 | +1/-1 | 1 | | Ivan | 1 | +1/-1 | 1 | | Guillaume Renault | 1 | +1/-1 | 1 | | Anjor Kanekar | 1 | +1/-1 | 1 | | Andrew Chin | 1 | +1/-1 | 1 | | Abdul Rauf | 1 | +1/-1 | 1 | | makeworld | 1 | +1/-0 | 1 | ================================================ FILE: docs/changelogs/v0.2.md ================================================ # go-ipfs changelog v0.2 ## 0.2.3 - 2015-03-01 * Alpha Release ## 2015-01-31: * bootstrap addresses now have .../ipfs/... in format config file Bootstrap field changed accordingly. users can upgrade cleanly with: ipfs bootstrap >bootstrap_peers ipfs bootstrap rm --all ipfs bootstrap add Full Changelog - github.com/ipfs/kubo: - fix: deadlock on retrieving WebTransport addresses (#9857) ([ipfs/kubo#9857](https://github.com/ipfs/kubo/pull/9857)) - docs(config): remove mentions of relay v1 (#9860) ([ipfs/kubo#9860](https://github.com/ipfs/kubo/pull/9860)) - Merge branch 'master' into merge-release-v0.19.2 - docs: add changelog for v0.19.2 - feat: webui@3.0.0 (#9835) ([ipfs/kubo#9835](https://github.com/ipfs/kubo/pull/9835)) - fix: use default HTTP routers when FullRT DHT client is used (#9841) ([ipfs/kubo#9841](https://github.com/ipfs/kubo/pull/9841)) - chore: update version - docs: add `ipfs pubsub` deprecation reminder to changelog (#9827) ([ipfs/kubo#9827](https://github.com/ipfs/kubo/pull/9827)) - docs: preparing 0.20 changelog for release (#9799) ([ipfs/kubo#9799](https://github.com/ipfs/kubo/pull/9799)) - feat: boxo tracing and traceparent support (#9811) ([ipfs/kubo#9811](https://github.com/ipfs/kubo/pull/9811)) - chore: update version - chore: update version - update go-libp2p to v0.27.0 - docs: add optimistic provide feature description - feat: add experimental optimistic provide - fix(ci): speed up docker build (#9800) ([ipfs/kubo#9800](https://github.com/ipfs/kubo/pull/9800)) - feat(tracing): use OTEL_PROPAGATORS as per OTel spec (#9801) ([ipfs/kubo#9801](https://github.com/ipfs/kubo/pull/9801)) - docs: fix jaeger command (#9797) ([ipfs/kubo#9797](https://github.com/ipfs/kubo/pull/9797)) - Merge Release: v0.19.1 (#9794) ([ipfs/kubo#9794](https://github.com/ipfs/kubo/pull/9794)) - chore: upgrade OpenTelemetry dependencies (#9736) ([ipfs/kubo#9736](https://github.com/ipfs/kubo/pull/9736)) - test: fix flaky content routing over HTTP test (#9772) ([ipfs/kubo#9772](https://github.com/ipfs/kubo/pull/9772)) - feat: allow injecting custom path resolvers (#9750) ([ipfs/kubo#9750](https://github.com/ipfs/kubo/pull/9750)) - feat: add changelog entry for router timeouts for v0.19.1 (#9784) ([ipfs/kubo#9784](https://github.com/ipfs/kubo/pull/9784)) - feat(gw): new metrics and HTTP range support (#9786) ([ipfs/kubo#9786](https://github.com/ipfs/kubo/pull/9786)) - feat!: make --empty-repo default (#9758) ([ipfs/kubo#9758](https://github.com/ipfs/kubo/pull/9758)) - fix: remove timeout on default DHT operations (#9783) ([ipfs/kubo#9783](https://github.com/ipfs/kubo/pull/9783)) - refactor: switch gateway code to new API from go-libipfs (#9681) ([ipfs/kubo#9681](https://github.com/ipfs/kubo/pull/9681)) - test: port remote pinning tests to Go (#9720) ([ipfs/kubo#9720](https://github.com/ipfs/kubo/pull/9720)) - feat: add identify option to swarm peers command - test: port routing DHT tests to Go (#9709) ([ipfs/kubo#9709](https://github.com/ipfs/kubo/pull/9709)) - test: fix autoclient flakiness (#9769) ([ipfs/kubo#9769](https://github.com/ipfs/kubo/pull/9769)) - test: skip flaky pubsub test (#9770) ([ipfs/kubo#9770](https://github.com/ipfs/kubo/pull/9770)) - chore: migrate go-libipfs to boxo - feat: add tracing to the commands client - feat: add client-side metrics for routing-v1 client - test: increase max wait time for peering assertion - feat: remove writable gateway (#9743) ([ipfs/kubo#9743](https://github.com/ipfs/kubo/pull/9743)) - Process Improvement: v0.18.0 ([ipfs/kubo#9484](https://github.com/ipfs/kubo/pull/9484)) - fix: deadlock while racing `ipfs dag import` and `ipfs repo gc` - feat: improve dag/import (#9721) ([ipfs/kubo#9721](https://github.com/ipfs/kubo/pull/9721)) - ci: remove circleci config ([ipfs/kubo#9687](https://github.com/ipfs/kubo/pull/9687)) - docs: use fx.Decorate instead of fx.Replace in examples (#9725) ([ipfs/kubo#9725](https://github.com/ipfs/kubo/pull/9725)) - Create Changelog: v0.20 ([ipfs/kubo#9742](https://github.com/ipfs/kubo/pull/9742)) - Merge Release: v0.19.0 ([ipfs/kubo#9741](https://github.com/ipfs/kubo/pull/9741)) - feat(gateway): invalid CID returns 400 Bad Request (#9726) ([ipfs/kubo#9726](https://github.com/ipfs/kubo/pull/9726)) - fix: remove outdated changelog part ([ipfs/kubo#9739](https://github.com/ipfs/kubo/pull/9739)) - docs: 0.19 changelog ([ipfs/kubo#9707](https://github.com/ipfs/kubo/pull/9707)) - fix: canonicalize user defined headers - fix: apply API.HTTPHeaders to /webui redirect - feat: add heap allocs to 'ipfs diag profile' - fix: future proof with > rcmgr.DefaultLimit for new enum rcmgr values - test: add test for presarvation of unlimited configs for inbound systems - fix: preserve Unlimited StreamsInbound in connmgr reconciliation - test: fix flaky rcmgr test - chore: deprecate the pubsub api - Revert "chore: add hamt directory sharding test" - chore: add hamt directory sharding test - test: port peering test from sharness to Go - test: use `T.TempDir` to create temporary test directory - fix: --verify forgets the verified key - test: name --verify forgets the verified key - chore: fix toc in changelog for 0.18 - feat: add "autoclient" routing type - test: parallelize more of rcmgr Go tests - test: port legacy DHT tests to Go - fix: t0116-gateway-cache.sh ([ipfs/kubo#9696](https://github.com/ipfs/kubo/pull/9696)) - docs: add bifrost to early testers ([ipfs/kubo#9699](https://github.com/ipfs/kubo/pull/9699)) - fix: typo in documentation for install path - docs: fix typos - Update Version: v0.19 ([ipfs/kubo#9698](https://github.com/ipfs/kubo/pull/9698)) - github.com/ipfs/go-block-format (v0.1.1 -> v0.1.2): - chore: release v0.1.2 - Revert deprecation and go-libipfs/blocks stub types - docs: deprecation notice [ci skip] - github.com/ipfs/go-cid (v0.3.2 -> v0.4.1): - v0.4.1 - Add unit test for unexpected eof - Update cid.go - CidFromReader should not wrap valid EOF return. - chore: version 0.4.0 - feat: wrap parsing errors into ErrInvalidCid - fix: use crypto/rand.Read - Fix README.md example error (#146) ([ipfs/go-cid#146](https://github.com/ipfs/go-cid/pull/146)) - github.com/ipfs/go-delegated-routing (v0.7.0 -> v0.8.0): - chore: release v0.8.0 - chore: migrate from go-ipns to boxo - docs: add deprecation notice [ci skip] - github.com/ipfs/go-graphsync (v0.14.1 -> v0.14.4): - Update version to cover latest fixes (#419) ([ipfs/go-graphsync#419](https://github.com/ipfs/go-graphsync/pull/419)) - Bring changes from #412 - Bring changes from #391 - fix: calling message queue Shutdown twice causes panic (because close is called twice on done channel) (#414) ([ipfs/go-graphsync#414](https://github.com/ipfs/go-graphsync/pull/414)) - docs(CHANGELOG): update for v0.14.3 - fix: wire up proper linksystem to traverser (#411) ([ipfs/go-graphsync#411](https://github.com/ipfs/go-graphsync/pull/411)) - sync: update CI config files (#378) ([ipfs/go-graphsync#378](https://github.com/ipfs/go-graphsync/pull/378)) - chore: remove social links (#398) ([ipfs/go-graphsync#398](https://github.com/ipfs/go-graphsync/pull/398)) - Removes `main` branch callout. - release v0.14.2 - github.com/ipfs/go-ipfs-blockstore (v1.2.0 -> v1.3.0): - chore: release v1.3.0 - feat: stub and deprecate NewBlockstoreNoPrefix - Accept options for blockstore: start with WriteThrough and NoPrefix - Allow using a NewWriteThrough() blockstore. - sync: update CI config files (#105) ([ipfs/go-ipfs-blockstore#105](https://github.com/ipfs/go-ipfs-blockstore/pull/105)) - feat: fast-path for PutMany, falling back to Put for single block call (#97) ([ipfs/go-ipfs-blockstore#97](https://github.com/ipfs/go-ipfs-blockstore/pull/97)) - github.com/ipfs/go-ipfs-cmds (v0.8.2 -> v0.9.0): - chore: release v0.9.0 - chore: change go-libipfs to boxo - github.com/ipfs/go-libipfs (v0.6.2 -> v0.7.0): - chore: bump to 0.7.0 (#213) ([ipfs/go-libipfs#213](https://github.com/ipfs/go-libipfs/pull/213)) - feat: return 400 on /ipfs/invalid-cid (#205) ([ipfs/go-libipfs#205](https://github.com/ipfs/go-libipfs/pull/205)) - docs: add note in README that go-libipfs is not comprehensive (#163) ([ipfs/go-libipfs#163](https://github.com/ipfs/go-libipfs/pull/163)) - github.com/ipfs/go-merkledag (v0.9.0 -> v0.10.0): - chore: bump version to 0.10.0 - fix: switch to crypto/rand.Read - stop using the deprecated io/ioutil package - github.com/ipfs/go-unixfs (v0.4.4 -> v0.4.5): - chore: release v0.4.5 - chore: remove go-libipfs dependency - github.com/ipfs/go-unixfsnode (v1.5.2 -> v1.6.0): - chore: bump v1.6.0 - feat: add UnixFSPathSelectorBuilder ([ipfs/go-unixfsnode#45](https://github.com/ipfs/go-unixfsnode/pull/45)) - fix: update state to allow iter continuance on NotFound errors - chore!: make PBLinkItr private - not intended for public use - fix: propagate iteration errors - github.com/ipld/go-car/v2 (v2.5.1 -> v2.9.1-0.20230325062757-fff0e4397a3d): - chore: unmigrate from go-libipfs - Create CODEOWNERS - blockstore: give a direct access to the index for read operations - blockstore: only close the file on error in OpenReadWrite, not OpenReadWriteFile - fix: handle (and test) WholeCID vs not; fast Has() path for storage - ReadWrite: faster Has() by using the in-memory index instead of reading on disk - fix: let `extract` skip missing unixfs shard links - fix: error when no files extracted - fix: make -f optional, read from stdin if omitted - fix: update cmd/car/README with latest description - chore: add test cases for extract modes - feat: extract accepts '-' as an output path for stdout - feat: extract specific path, accept stdin as streaming input - fix: if we don't read the full block data, don't error on !EOF - blockstore: try to close during Finalize(), even in case of previous error - ReadWrite: add an alternative FinalizeReadOnly+Close flow - feat: add WithTrustedCar() reader option (#381) ([ipld/go-car#381](https://github.com/ipld/go-car/pull/381)) - blockstore: fast path for AllKeysChan using the index - fix: switch to crypto/rand.Read - stop using the deprecated io/ioutil package - fix(doc): fix storage package doc formatting - fix: return errors for unsupported operations - chore: move insertionindex into store pkg - chore: add experimental note - fix: minor lint & windows fd test problems - feat: docs for StorageCar interfaces - feat: ReadableWritable; dedupe shared code - feat: add Writable functionality to StorageCar - feat: StorageCar as a Readable storage, separate from blockstore - feat(blockstore): implement a streaming read only storage - feat(cmd): add index create subcommand to create an external carv2 index ([ipld/go-car#350](https://github.com/ipld/go-car/pull/350)) - chore: bump version to 0.6.0 - fix: use goreleaser instead - Allow using WalkOption in WriteCar function ([ipld/go-car#357](https://github.com/ipld/go-car/pull/357)) - fix: update go-block-format to the version that includes the stubs - feat: upgrade from go-block-format to go-libipfs/blocks - cleanup readme a bit to make the cli more discoverable (#353) ([ipld/go-car#353](https://github.com/ipld/go-car/pull/353)) - Update install instructions in README.md - Add a debugging form for car files. (#341) ([ipld/go-car#341](https://github.com/ipld/go-car/pull/341)) - ([ipld/go-car#340](https://github.com/ipld/go-car/pull/340)) - github.com/ipld/go-codec-dagpb (v1.5.0 -> v1.6.0): - Update version.json - github.com/ipld/go-ipld-prime (v0.19.0 -> v0.20.0): - Prepare v0.20.0 - fix(datamodel): add tests to Copy, make it complain on nil - feat(dagcbor): mode to allow parsing undelimited streamed objects - Fix mispatched package declaration. - Add several pieces of docs to schema/dmt. - Additional access to schema/dmt package; schema concatenation feature ([ipld/go-ipld-prime#483](https://github.com/ipld/go-ipld-prime/pull/483)) - Fix hash mismatch error on matching link pointer - feat: support errors.Is for schema errors - github.com/ipld/go-ipld-prime/storage/bsadapter (v0.0.0-20211210234204-ce2a1c70cd73 -> v0.0.0-20230102063945-1a409dc236dd): - build(deps): bump github.com/ipfs/go-blockservice - Fix mispatched package declaration. - Add several pieces of docs to schema/dmt. - Additional access to schema/dmt package; schema concatenation feature ([ipld/go-ipld-prime/storage/bsadapter#483](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/483)) - fix: go mod tidy - build(deps): bump github.com/frankban/quicktest from 1.14.3 to 1.14.4 - Fix hash mismatch error on matching link pointer - build(deps): bump github.com/warpfork/go-testmark from 0.10.0 to 0.11.0 - feat: support errors.Is for schema errors - build(deps): bump github.com/multiformats/go-multicodec - Prepare v0.19.0 - fix: correct json codec links & bytes handling - build(deps): bump github.com/google/go-cmp from 0.5.8 to 0.5.9 (#468) ([ipld/go-ipld-prime/storage/bsadapter#468](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/468)) - build(deps): bump github.com/ipfs/go-cid from 0.3.0 to 0.3.2 (#466) ([ipld/go-ipld-prime/storage/bsadapter#466](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/466)) - build(deps): bump github.com/ipfs/go-cid in /storage/bsrvadapter (#464) ([ipld/go-ipld-prime/storage/bsadapter#464](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/464)) - test(basicnode): increase test coverage for int and map types (#454) ([ipld/go-ipld-prime/storage/bsadapter#454](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/454)) - build(deps): bump github.com/ipfs/go-cid in /storage/bsrvadapter - build(deps): bump github.com/ipfs/go-cid from 0.2.0 to 0.3.0 - build(deps): bump github.com/multiformats/go-multicodec - fix: remove reliance on ioutil - fix: update sub-package modules - build(deps): bump github.com/multiformats/go-multihash - build(deps): bump github.com/ipfs/go-datastore in /storage/dsadapter - update .github/workflows/go-check.yml - update .github/workflows/go-test.yml - run gofmt -s - bump go.mod to Go 1.18 and run go fix - bump go.mod to Go 1.18 and run go fix - bump go.mod to Go 1.18 and run go fix - bump go.mod to Go 1.18 and run go fix - feat: add kinded union to gendemo - fix: go mod 1.17 compat problems - build(deps): bump github.com/ipfs/go-blockservice - Prepare v0.18.0 - fix(deps): update benchmarks go.sum - build(deps): bump github.com/multiformats/go-multihash - feat(bindnode): add a BindnodeRegistry utility (#437) ([ipld/go-ipld-prime/storage/bsadapter#437](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/437)) - feat(bindnode): support full uint64 range - chore(bindnode): remove typed functions for options - chore(bindnode): docs and minor tweaks - feat(bindnode): make Any converters work for List and Map values - fix(bindnode): shorten converter option names, minor perf improvements - fix(bindnode): only custom convert AssignNull for Any converter - feat(bindnode): pass Null on to nullable custom converters - chore(bindnode): config helper refactor w/ short-circuit - feat(bindnode): add AddCustomTypeAnyConverter() to handle `Any` fields - feat(bindnode): add AddCustomTypeXConverter() options for most scalar kinds - chore(bindnode): back out of reflection for converters - feat(bindnode): switch to converter functions instead of type - feat(bindnode): allow custom type conversions with options - feat: add release checklist (#442) ([ipld/go-ipld-prime/storage/bsadapter#442](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/442)) - Prepare v0.17.0 - feat: introduce UIntNode interface, used within DAG-CBOR codec - add option to not parse beyond end of structure (#435) ([ipld/go-ipld-prime/storage/bsadapter#435](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/435)) - sync benchmarks go.sum - build(deps): bump github.com/multiformats/go-multicodec - patch: first draft. ([ipld/go-ipld-prime/storage/bsadapter#350](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/350)) - feat(bindnode): infer links and Any from Go types (#432) ([ipld/go-ipld-prime/storage/bsadapter#432](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/432)) - fix(codecs): error on cid.Undef links in dag{json,cbor} encoding (#433) ([ipld/go-ipld-prime/storage/bsadapter#433](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/433)) - chore(bindnode): add test for sub-node unwrapping - fix(bindnode): more helpful error message for enum value footgun - fix(bindnode): panic early if API has been passed ptr-to-ptr - fix(deps): mod tidy for dependencies - build(deps): bump github.com/warpfork/go-testmark from 0.3.0 to 0.10.0 - build(deps): bump github.com/multiformats/go-multicodec - build(deps): bump github.com/ipfs/go-cid from 0.0.4 to 0.2.0 - build(deps): bump github.com/google/go-cmp from 0.5.7 to 0.5.8 - build(deps): bump github.com/frankban/quicktest from 1.14.2 to 1.14.3 - build(deps): bump github.com/ipfs/go-cid in /storage/bsrvadapter - chore(deps): expand dependabot to sub-modules - chore(deps): add dependabot config - printer: fix printing of floats - add version.json file (#411) ([ipld/go-ipld-prime/storage/bsadapter#411](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/411)) - ci: use GOFLAGS to control test tags - ci: disable coverpkg using custom workflow insertion - ci: add initial web3 unified-ci files - fix: make 32-bit safe and stable & add to CI - ci: add go-check.yml workflow from unified-ci - ci: go mod tidy - fix: staticcheck and govet fixes - test: make tests work on Windows, add Windows to CI (#405) ([ipld/go-ipld-prime/storage/bsadapter#405](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/405)) - schema: enable inline types through dsl parser & compiler (#404) ([ipld/go-ipld-prime/storage/bsadapter#404](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/404)) - node/bindnode: allow nilable types for IPLD optional/nullable - test(ci): enable macos in GitHub Actions - test(gen-go): disable parallelism when testing on macos - storage: update deps - dsl support for stringjoin struct repr and stringprefix union repr ([ipld/go-ipld-prime/storage/bsadapter#397](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/397)) - codec/dagcbor: add DecodeOptions.ExperimentalDeterminism - node/bindnode: add some more docs - start testing on Go 1.18.x, drop Go 1.16.x - readme: getting started pointers. - readme: bindnode definitely needs a mention! - Readme updates! - datamodel: document that repr prototypes produce type nodes - node/bindnode: minor fuzz improvements - gengo: update readme. - fix(dagcbor): don't accept trailing bytes - schema/dmt: reject duplicate or missing union repr members - node/bindnode: actually check schemadmt.Compile errors when fuzzing - node/bindnode: avoid OOM when inferring from cyclic IPLD schemas - schema/dmt: require enum reprs to refer valid members - skip NaN/Inf errors for dag-json - node/bindnode: refuse to decode empty union values - schema/dmt: error in Compile if union reprs refer to unknown members - node/bindnode: start fuzzing with schema/dmt and codec/dagcbor - mark v0.16.0 - node/bindnode: enforce pointer requirement for nullable maps - Implement WalkTransforming traversal (#376) ([ipld/go-ipld-prime/storage/bsadapter#376](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/376)) - docs(datamodel): add comment to LargeBytesNode - Add partial-match traversal of large bytes (#375) ([ipld/go-ipld-prime/storage/bsadapter#375](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/375)) - Implement option to start traversals at a path ([ipld/go-ipld-prime/storage/bsadapter#358](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/358)) - add top-level "go value with schema" example - Support optional `LargeBytesNode` interface (#372) ([ipld/go-ipld-prime/storage/bsadapter#372](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/372)) - node/bindnode: support pointers to datamodel.Node to bind with Any - fix(bindnode): tuple struct iterator should handle absent fields properly - node/bindnode: make AssignNode work at the repr level - node/bindnode: add support for unsigned integers - node/bindnode: cover even more edge case panics - node/bindnode: polish some more AsT panics - schema/dmt: stop using a fake test to generate code ([ipld/go-ipld-prime/storage/bsadapter#356](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/356)) - schema: remove one review note; add another. - fix: minor EncodedLength fixes, add tests to fully exercise - feat: add dagcbor.EncodedLength(Node) to calculate length without encoding - chore: rename Garbage() to Generate() - fix: minor garbage nits - fix: Garbage() takes rand parameter, tweak algorithms, improve docs - feat: add Garbage() Node generator - node/bindnode: introduce an assembler that always errors - node/bindnode: polish panics on invalid AssignT calls - datamodel: don't panic when stringifying an empty KindSet - node/bindnode: start using ipld.LoadSchema APIs - selectors: fix for edge case around recursion clauses with an immediate edge. ([ipld/go-ipld-prime/storage/bsadapter#334](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/334)) - node/bindnode: improve support for pointer types - node/bindnode: subtract all absents in Length at the repr level - fix(codecs): error when encoding maps whose lengths don't match entry count - schema: avoid alloc and copy in Struct and Enum methods - node/bindnode: allow mapping int-repr enums with Go integers - schema,node/bindnode: add support for Any - signaling ADLs in selectors (#301) ([ipld/go-ipld-prime/storage/bsadapter#301](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/301)) - node/bindnode: add support for enums - schema/...: add support for enum int representations - node/bindnode: allow binding cidlink.Link to links - github.com/libp2p/go-libp2p (v0.26.4 -> v0.27.3): - release v0.27.3 - quic virtual listener: don't panic when quic-go's accept call errors (#2276) ([libp2p/go-libp2p#2276](https://github.com/libp2p/go-libp2p/pull/2276)) - Release v0.27.2 (#2270) ([libp2p/go-libp2p#2270](https://github.com/libp2p/go-libp2p/pull/2270)) - release v0.27.1 (#2252) ([libp2p/go-libp2p#2252](https://github.com/libp2p/go-libp2p/pull/2252)) - Infer public webtransport addrs from quic-v1 addrs. (#2251) ([libp2p/go-libp2p#2251](https://github.com/libp2p/go-libp2p/pull/2251)) - basichost: don't allocate when deduplicating multiaddrs (#2206) ([libp2p/go-libp2p#2206](https://github.com/libp2p/go-libp2p/pull/2206)) - identify: fix normalization of interface listen addresses (#2250) ([libp2p/go-libp2p#2250](https://github.com/libp2p/go-libp2p/pull/2250)) - autonat: fix flaky TestAutoNATDialRefused (#2245) ([libp2p/go-libp2p#2245](https://github.com/libp2p/go-libp2p/pull/2245)) - basichost: remove stray print statement in test (#2249) ([libp2p/go-libp2p#2249](https://github.com/libp2p/go-libp2p/pull/2249)) - swarm: fix multiaddr comparison in ListenClose (#2247) ([libp2p/go-libp2p#2247](https://github.com/libp2p/go-libp2p/pull/2247)) - release v0.27.0 (#2242) ([libp2p/go-libp2p#2242](https://github.com/libp2p/go-libp2p/pull/2242)) - add a security policy (#2238) ([libp2p/go-libp2p#2238](https://github.com/libp2p/go-libp2p/pull/2238)) - chore: 0.27.0 changelog entries (#2241) ([libp2p/go-libp2p#2241](https://github.com/libp2p/go-libp2p/pull/2241)) - correctly handle WebTransport addresses without certhashes (#2239) ([libp2p/go-libp2p#2239](https://github.com/libp2p/go-libp2p/pull/2239)) - autorelay: add metrics (#2185) ([libp2p/go-libp2p#2185](https://github.com/libp2p/go-libp2p/pull/2185)) - autonat: don't change status on dial request refused (#2225) ([libp2p/go-libp2p#2225](https://github.com/libp2p/go-libp2p/pull/2225)) - autonat: fix closing of listeners in dialPolicy tests (#2226) ([libp2p/go-libp2p#2226](https://github.com/libp2p/go-libp2p/pull/2226)) - discovery (backoff): fix typo in comment (#2214) ([libp2p/go-libp2p#2214](https://github.com/libp2p/go-libp2p/pull/2214)) - relaysvc: flaky TestReachabilityChangeEvent (#2215) ([libp2p/go-libp2p#2215](https://github.com/libp2p/go-libp2p/pull/2215)) - Add wss transport to interop tester impl (#2178) ([libp2p/go-libp2p#2178](https://github.com/libp2p/go-libp2p/pull/2178)) - tests: add a stream read deadline transport test (#2210) ([libp2p/go-libp2p#2210](https://github.com/libp2p/go-libp2p/pull/2210)) - autorelay: fix busy loop bug and flaky tests in relay finder (#2208) ([libp2p/go-libp2p#2208](https://github.com/libp2p/go-libp2p/pull/2208)) - tests: test mplex and Yamux, Noise and TLS in transport tests (#2209) ([libp2p/go-libp2p#2209](https://github.com/libp2p/go-libp2p/pull/2209)) - tests: add some basic transport integration tests (#2207) ([libp2p/go-libp2p#2207](https://github.com/libp2p/go-libp2p/pull/2207)) - autorelay: remove unused semaphore (#2184) ([libp2p/go-libp2p#2184](https://github.com/libp2p/go-libp2p/pull/2184)) - basichost: prevent duplicate dials (#2196) ([libp2p/go-libp2p#2196](https://github.com/libp2p/go-libp2p/pull/2196)) - websocket: don't set a WSS multiaddr for accepted unencrypted conns (#2199) ([libp2p/go-libp2p#2199](https://github.com/libp2p/go-libp2p/pull/2199)) - websocket: Don't limit message sizes in the websocket reader (#2193) ([libp2p/go-libp2p#2193](https://github.com/libp2p/go-libp2p/pull/2193)) - identify: fix stale comment (#2179) ([libp2p/go-libp2p#2179](https://github.com/libp2p/go-libp2p/pull/2179)) - relay service: add metrics (#2154) ([libp2p/go-libp2p#2154](https://github.com/libp2p/go-libp2p/pull/2154)) - identify: Fix IdentifyWait when Connected events happen out of order (#2173) ([libp2p/go-libp2p#2173](https://github.com/libp2p/go-libp2p/pull/2173)) - chore: fix resource manager's README (#2168) ([libp2p/go-libp2p#2168](https://github.com/libp2p/go-libp2p/pull/2168)) - relay: fix deadlock when closing (#2171) ([libp2p/go-libp2p#2171](https://github.com/libp2p/go-libp2p/pull/2171)) - core: remove LocalPrivateKey method from network.Conn interface (#2144) ([libp2p/go-libp2p#2144](https://github.com/libp2p/go-libp2p/pull/2144)) - routed host: return connection error instead of routing error (#2169) ([libp2p/go-libp2p#2169](https://github.com/libp2p/go-libp2p/pull/2169)) - connmgr: reduce log level for closing connections (#2165) ([libp2p/go-libp2p#2165](https://github.com/libp2p/go-libp2p/pull/2165)) - circuitv2: cleanup relay service properly (#2164) ([libp2p/go-libp2p#2164](https://github.com/libp2p/go-libp2p/pull/2164)) - chore: add patch release to changelog (#2151) ([libp2p/go-libp2p#2151](https://github.com/libp2p/go-libp2p/pull/2151)) - chore: remove superfluous testing section from README (#2150) ([libp2p/go-libp2p#2150](https://github.com/libp2p/go-libp2p/pull/2150)) - autonat: don't use autonat for address discovery (#2148) ([libp2p/go-libp2p#2148](https://github.com/libp2p/go-libp2p/pull/2148)) - swarm metrics: fix connection direction (#2147) ([libp2p/go-libp2p#2147](https://github.com/libp2p/go-libp2p/pull/2147)) - connmgr: Use eventually equal helper in connmgr tests (#2128) ([libp2p/go-libp2p#2128](https://github.com/libp2p/go-libp2p/pull/2128)) - swarm: emit PeerConnectedness event from swarm instead of from hosts (#1574) ([libp2p/go-libp2p#1574](https://github.com/libp2p/go-libp2p/pull/1574)) - relay: initialize the ASN util when starting the service (#2143) ([libp2p/go-libp2p#2143](https://github.com/libp2p/go-libp2p/pull/2143)) - Fix flaky TestMetricsNoAllocNoCover test (#2142) ([libp2p/go-libp2p#2142](https://github.com/libp2p/go-libp2p/pull/2142)) - identify: Bump timeouts/sleep in tests (#2135) ([libp2p/go-libp2p#2135](https://github.com/libp2p/go-libp2p/pull/2135)) - Add sleep to fix flaky test (#2129) ([libp2p/go-libp2p#2129](https://github.com/libp2p/go-libp2p/pull/2129)) - basic_host: Fix flaky tests (#2136) ([libp2p/go-libp2p#2136](https://github.com/libp2p/go-libp2p/pull/2136)) - swarm: Check context once more before dialing (#2139) ([libp2p/go-libp2p#2139](https://github.com/libp2p/go-libp2p/pull/2139)) - github.com/libp2p/go-libp2p-asn-util (v0.2.0 -> v0.3.0): - release v0.3.0 (#26) ([libp2p/go-libp2p-asn-util#26](https://github.com/libp2p/go-libp2p-asn-util/pull/26)) - initialize the store lazily (#25) ([libp2p/go-libp2p-asn-util#25](https://github.com/libp2p/go-libp2p-asn-util/pull/25)) - github.com/libp2p/go-libp2p-gostream (v0.5.0 -> v0.6.0): - Update libp2p ([libp2p/go-libp2p-gostream#80](https://github.com/libp2p/go-libp2p-gostream/pull/80)) - fix typo in README (#75) ([libp2p/go-libp2p-gostream#75](https://github.com/libp2p/go-libp2p-gostream/pull/75)) - github.com/libp2p/go-libp2p-http (v0.4.0 -> v0.5.0): - sync: update CI config files ([libp2p/go-libp2p-http#82](https://github.com/libp2p/go-libp2p-http/pull/82)) - github.com/libp2p/go-libp2p-kad-dht (v0.21.1 -> v0.23.0): - Release v0.23.0 - Specified CODEOWNERS ([libp2p/go-libp2p-kad-dht#828](https://github.com/libp2p/go-libp2p-kad-dht/pull/828)) - fix: optimistic provide ci checks in tests ([libp2p/go-libp2p-kad-dht#833](https://github.com/libp2p/go-libp2p-kad-dht/pull/833)) - feat: add experimental optimistic provide (#783) ([libp2p/go-libp2p-kad-dht#783](https://github.com/libp2p/go-libp2p-kad-dht/pull/783)) - feat: rework tracing a bit - feat: add basic tracing - chore: release v0.22.0 - chore: migrate go-libipfs to boxo - Fix multiple ProviderAddrTTL definitions #795 ([libp2p/go-libp2p-kad-dht#831](https://github.com/libp2p/go-libp2p-kad-dht/pull/831)) - Increase provider Multiaddress TTL ([libp2p/go-libp2p-kad-dht#795](https://github.com/libp2p/go-libp2p-kad-dht/pull/795)) - Make provider manager options configurable in `fullrt` ([libp2p/go-libp2p-kad-dht#829](https://github.com/libp2p/go-libp2p-kad-dht/pull/829)) - Adjust PeerSet logic in the DHT lookup process ([libp2p/go-libp2p-kad-dht#802](https://github.com/libp2p/go-libp2p-kad-dht/pull/802)) - added maintainers in the README ([libp2p/go-libp2p-kad-dht#826](https://github.com/libp2p/go-libp2p-kad-dht/pull/826)) - Allow DHT crawler to be swappable - Introduce options to parameterize config of the accelerated DHT client ([libp2p/go-libp2p-kad-dht#822](https://github.com/libp2p/go-libp2p-kad-dht/pull/822)) - github.com/libp2p/go-libp2p-pubsub (v0.9.0 -> v0.9.3): - Fix Memory Leak In New Timecache Implementations (#528) ([libp2p/go-libp2p-pubsub#528](https://github.com/libp2p/go-libp2p-pubsub/pull/528)) - Default validator support (#525) ([libp2p/go-libp2p-pubsub#525](https://github.com/libp2p/go-libp2p-pubsub/pull/525)) - Refactor timecache implementations (#523) ([libp2p/go-libp2p-pubsub#523](https://github.com/libp2p/go-libp2p-pubsub/pull/523)) - fix(timecache): remove panic in first seen cache on Add (#522) ([libp2p/go-libp2p-pubsub#522](https://github.com/libp2p/go-libp2p-pubsub/pull/522)) - chore: update go version and dependencies (#516) ([libp2p/go-libp2p-pubsub#516](https://github.com/libp2p/go-libp2p-pubsub/pull/516)) - github.com/multiformats/go-multiaddr (v0.8.0 -> v0.9.0): - Release v0.9.0 ([multiformats/go-multiaddr#196](https://github.com/multiformats/go-multiaddr/pull/196)) - Update webrtc protocols after rename ([multiformats/go-multiaddr#195](https://github.com/multiformats/go-multiaddr/pull/195)) - github.com/multiformats/go-multibase (v0.1.1 -> v0.2.0): - chore: bump v0.2.0 - fix: math/rand -> crypto/rand - fuzz: add Decoder fuzzing - github.com/multiformats/go-multicodec (v0.7.0 -> v0.8.1): - Bump version to release `ipns-record` code - chore: update submodules and go generate - deps: upgrade stringer to compatible version - v0.8.0 - chore: update submodules and go generate - github.com/warpfork/go-testmark (v0.10.0 -> v0.11.0): - Quick changelog to note we have an API update. - Index fix ([warpfork/go-testmark#13](https://github.com/warpfork/go-testmark/pull/13)) - Link to python implementation in the readme! ### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Rod Vagg | 40 | +4214/-1400 | 102 | | Sukun | 12 | +3541/-267 | 34 | | Gus Eggert | 22 | +2387/-1160 | 81 | | galargh | 23 | +1331/-1734 | 34 | | Henrique Dias | 23 | +681/-1167 | 79 | | Marco Munizaga | 19 | +1500/-187 | 55 | | Jorropo | 25 | +897/-597 | 180 | | Dennis Trautwein | 4 | +990/-60 | 14 | | Marten Seemann | 18 | +443/-450 | 53 | | vyzo | 2 | +595/-152 | 11 | | Michael Muré | 8 | +427/-182 | 18 | | Will | 2 | +536/-15 | 5 | | Adin Schmahmann | 3 | +327/-125 | 11 | | hannahhoward | 2 | +344/-1 | 4 | | Arthur Gavazza | 1 | +210/-50 | 4 | | Hector Sanjuan | 6 | +181/-77 | 13 | | Masih H. Derkani | 5 | +214/-42 | 12 | | Calvin Behling | 4 | +158/-58 | 11 | | Eric Myhre | 7 | +113/-27 | 15 | | Marcin Rataj | 5 | +72/-30 | 5 | | Steve Loeppky | 2 | +99/-0 | 2 | | Piotr Galar | 9 | +60/-18 | 9 | | gammazero | 4 | +69/-0 | 8 | | Prithvi Shahi | 2 | +55/-14 | 2 | | Eng Zer Jun | 1 | +15/-54 | 5 | | Laurent Senta | 3 | +44/-2 | 3 | | Ian Davis | 1 | +35/-0 | 1 | | web3-bot | 4 | +19/-13 | 7 | | guillaumemichel | 2 | +18/-14 | 3 | | Guillaume Michel - guissou | 4 | +24/-8 | 4 | | omahs | 1 | +9/-9 | 3 | | cortze | 3 | +9/-9 | 3 | | Nishant Das | 1 | +9/-5 | 3 | | Hlib Kanunnikov | 2 | +11/-3 | 3 | | Andrew Gillis | 3 | +6/-8 | 3 | | Johnny | 1 | +0/-10 | 1 | | Rafał Leszko | 1 | +4/-4 | 1 | | Dirk McCormick | 1 | +4/-1 | 1 | | Antonio Navarro Perez | 1 | +4/-1 | 1 | | RichΛrd | 2 | +2/-2 | 2 | | Russell Dempsey | 1 | +2/-1 | 1 | | Winterhuman | 1 | +1/-1 | 1 | | Will Hawkins | 1 | +1/-1 | 1 | | Nikhilesh Susarla | 1 | +1/-1 | 1 | | Kubo Mage | 1 | +1/-1 | 1 | | Bryan White | 1 | +1/-1 | 1 | ================================================ FILE: docs/changelogs/v0.21.md ================================================ # Kubo changelog v0.21 - [v0.21.1](#v0211) - [v0.21.0](#v0210) ## v0.21.1 - Update go-libp2p: - [v0.27.8](https://github.com/libp2p/go-libp2p/releases/tag/v0.27.8) - [v0.27.9](https://github.com/libp2p/go-libp2p/releases/tag/v0.27.9) - Update Boxo to v0.10.3 ([ipfs/boxo#412](https://github.com/ipfs/boxo/pull/412)). ## v0.21.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping) - [Gateway: `DeserializedResponses` config flag](#gateway-deserializedresponses-config-flag) - [`client/rpc` migration of `go-ipfs-http-client`](#clientrpc-migration-of-go-ipfs-http-client) - [Gateway: DAG-CBOR/-JSON previews and improved error pages](#gateway-dag-cbor-json-previews-and-improved-error-pages) - [Gateway: subdomain redirects are now `text/html`](#gateway-subdomain-redirects-are-now-texthtml) - [Gateway: support for partial CAR export parameters (IPIP-402)](#gateway-support-for-partial-car-export-parameters-ipip-402) - [`ipfs dag stat` deduping statistics](#ipfs-dag-stat-deduping-statistics) - [Accelerated DHT Client is no longer experimental](#accelerated-dht-client-is-no-longer-experimental) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### Saving previously seen nodes for later bootstrapping Kubo now stores a subset of connected peers as backup bootstrap nodes ([kubo#8856](https://github.com/ipfs/kubo/pull/8856)). These nodes are used in addition to the explicitly defined bootstrappers in the [`Bootstrap`](https://github.com/ipfs/kubo/blob/master/docs/config.md#bootstrap) configuration. This enhancement improves the resiliency of the system, as it eliminates the necessity of relying solely on the default bootstrappers operated by Protocol Labs for joining the public IPFS swarm. Previously, this level of robustness was only available in LAN contexts with [mDNS peer discovery](https://github.com/ipfs/kubo/blob/master/docs/config.md#discoverymdns) enabled. With this update, the same level of robustness is applied to peers that lack mDNS peers and solely rely on the public DHT. #### Gateway: `DeserializedResponses` config flag This release introduces the [`Gateway.DeserializedResponses`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaydeserializedresponses) configuration flag. With this flag, one can explicitly configure whether the gateway responds to deserialized requests or not. By default, this flag is enabled. Disabling deserialized responses allows the gateway to operate as a [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/) limited to three [verifiable](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval) response types: [application/vnd.ipld.raw](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw), [application/vnd.ipld.car](https://www.iana.org/assignments/media-types/application/vnd.ipld.car), and [application/vnd.ipfs.ipns-record](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record). With deserialized responses disabled, the Kubo gateway can serve as a block backend for other software (like [bifrost-gateway](https://github.com/ipfs/bifrost-gateway#readme), [IPFS in Chromium](https://github.com/little-bear-labs/ipfs-chromium/blob/main/README.md) etc) without the usual risks associated with hosting deserialized data behind third-party CIDs. #### `client/rpc` migration of `go-ipfs-http-client` The [`go-ipfs-http-client`](https://github.com/ipfs/go-ipfs-http-client) RPC has been migrated into [`kubo/client/rpc`](../../client/rpc). With this change the two will be kept in sync, in some previous releases we updated the CoreAPI with new Kubo features but forgot to port those to the http-client, making it impossible to use them together with the same coreapi version. For smooth transition `v0.7.0` of `go-ipfs-http-client` provides updated stubs for Kubo `v0.21`. #### Gateway: DAG-CBOR/-JSON previews and improved error pages In this release, we improved the HTML templates of our HTTP gateway: 1. You can now preview the contents of a DAG-CBOR and DAG-JSON document from your browser, as well as follow any IPLD Links ([CBOR Tag 42](https://github.com/ipld/cid-cbor/)) contained within them. 2. The HTML directory listings now contain [updated, higher-definition icons](https://user-images.githubusercontent.com/5447088/241224419-5385793a-d3bb-40aa-8cb0-0382b5bc56a0.png). 3. On gateway error, instead of a plain text error message, web browsers will now get a friendly HTML response with more details regarding the problem. HTML responses are returned when request's `Accept` header includes `text/html`. | DAG-CBOR Preview | Error Page | | ---- | ---- | | ![DAG-CBOR Preview](https://github.com/ipfs/boxo/assets/5447088/973f05d1-5731-4469-9da5-d1d776891899) | ![Error Page](https://github.com/ipfs/boxo/assets/5447088/14c453df-adbc-4634-b038-133121914550) | #### Gateway: subdomain redirects are now `text/html` HTTP 301 redirects [from path to subdomain](https://specs.ipfs.tech/http-gateways/subdomain-gateway/#migrating-from-path-to-subdomain-gateway) no longer include the target data in the body. The data is returned only once, with the final HTTP 200 returned from the target subdomain. The HTTP 301 body now includes human-readable `text/html` message for clients that do not follow redirects by default: ```console $ curl "https://subdomain-gw.example.net/ipfs/${cid}/" Moved Permanently. ``` Rationale can be found in [kubo#9913](https://github.com/ipfs/kubo/pull/9913). #### Gateway: support for partial CAR export parameters (IPIP-402) The gateway now supports optional CAR export parameters `dag-scope=block|entity|all` and `entity-bytes=from:to` as specified in [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/). Batch block retrieval minimizes round trips, catering to the requirements of light HTTP clients for directory enumeration, range requests, and content path resolution. #### `ipfs dag stat` deduping statistics `ipfs dat stat` now accept multiple CIDs and will dump advanced statistics on the number of shared blocks and size of each CID. ```console $ ipfs dag stat --progress=false QmfXuRxzyVy5H2LssLgtXrKCrNvDY8UBvMp2aoW8LS8AYA QmfZDyu2UFfUhL4VdHaw7Hofivmn5D4DdQj38Lwo86RsnB CID Blocks Size QmfXuRxzyVy5H2LssLgtXrKCrNvDY8UBvMp2aoW8LS8AYA 3 2151 QmfZDyu2UFfUhL4VdHaw7Hofivmn5D4DdQj38Lwo86RsnB 4 3223 Summary Total Size: 3326 Unique Blocks: 5 Shared Size: 2048 Ratio: 1.615755 ``` `ipfs --enc=json dag stat`'s keys are a non breaking change, new keys have been added but old keys with previous semantics are still here. #### Accelerated DHT Client is no longer experimental The [accelerated DHT client](docs/config.md#routingaccelerateddhtclient) is now the main recommended solution for users who are hosting lots of data. By trading some upfront DHT caching and increased memory usage, one gets provider throughput improvements up to 6 millions times bigger dataset. See [the docs](docs/config.md#routingaccelerateddhtclient) for more info. The `Experimental.AcceleratedDHTClient` flag moved to [`Routing.AcceleratedDHTClient`](/docs/config.md#routingaccelerateddhtclient). A config migration has been added to handle this automatically. A new tracker estimates the providing speed and warns users if they should be using AcceleratedDHTClient because they are falling behind. ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - fix: correctly handle migration of configs - fix(gateway): include CORS on subdomain redirects (#9994) ([ipfs/kubo#9994](https://github.com/ipfs/kubo/pull/9994)) - fix: docker repository initialization race condition - chore: update version - ([ipfs/kubo#9981](https://github.com/ipfs/kubo/pull/9981)) - ([ipfs/kubo#9960](https://github.com/ipfs/kubo/pull/9960)) - ([ipfs/kubo#9936](https://github.com/ipfs/kubo/pull/9936)) - github.com/ipfs/boxo (v0.8.1 -> v0.10.2-0.20230629143123-2d3edc552442): - chore: version 0.10.2 - fix(gateway): include CORS on subdomain redirects (#395) ([ipfs/boxo#395](https://github.com/ipfs/boxo/pull/395)) - fix(gateway): ensure 'X-Ipfs-Root' header is valid (#337) ([ipfs/boxo#337](https://github.com/ipfs/boxo/pull/337)) - docs: prepare changelog for next release [ci skip] - chore: version 0.10.1 (#359) ([ipfs/boxo#359](https://github.com/ipfs/boxo/pull/359)) - fix(gateway): allow CAR trustless requests with path - blockstore: replace go.uber.org/atomic with sync/atomic - fix(gateway): remove handleUnsupportedHeaders after go-ipfs 0.13 (#350) ([ipfs/boxo#350](https://github.com/ipfs/boxo/pull/350)) - docs: update RELEASE.md based on 0.9 release (#343) ([ipfs/boxo#343](https://github.com/ipfs/boxo/pull/343)) - chore: v0.10.0 (#345) ([ipfs/boxo#345](https://github.com/ipfs/boxo/pull/345)) - docs(changelog): car params from ipip-402 - docs(changelog): add gateway deserialized responses (#341) ([ipfs/boxo#341](https://github.com/ipfs/boxo/pull/341)) - feat(gateway): implement IPIP-402 extensions for gateway CAR requests (#303) ([ipfs/boxo#303](https://github.com/ipfs/boxo/pull/303)) - chore: release v0.9.0 - changelog: update for 0.8.1 and 0.9.0 - provider: second round of reprovider refactor - feat(unixfs): change protobuf package name to unixfs.v1.pb to prevent collisions with go-unixfs. Also regenerate protobufs with latest gogo - feat(ipld/merkledag): remove use of go-ipld-format global registry - feat(ipld/merkledag): updated to use its own global go-ipld-legacy registry instead of a shared global registry - chore: do not rely on deprecated logger - changelog: add changelog for async pin listing (#336) ([ipfs/boxo#336](https://github.com/ipfs/boxo/pull/336)) - pinner: change the interface to have async pin listing - provider: revert throughput callback and related refactor - fix(gateway): question marks in url.Path when redirecting (#313) ([ipfs/boxo#313](https://github.com/ipfs/boxo/pull/313)) - fix(gateway)!: no duplicate payload during subdomain redirects (#326) ([ipfs/boxo#326](https://github.com/ipfs/boxo/pull/326)) - provider: add breaking changes to the changelog (#330) ([ipfs/boxo#330](https://github.com/ipfs/boxo/pull/330)) - relocated magic numbers, updated Reprovide Interval from 24h to 22h - provider: refactor to only maintain one batched implementation and add throughput callback - feat(gateway): HTML preview for dag-cbor and dag-json (#315) ([ipfs/boxo#315](https://github.com/ipfs/boxo/pull/315)) - coreiface: add a testing.T argument to the provider - feat(gateway): improved templates, user friendly errors (#298) ([ipfs/boxo#298](https://github.com/ipfs/boxo/pull/298)) - feat(gateway)!: deserialised responses turned off by default (#252) ([ipfs/boxo#252](https://github.com/ipfs/boxo/pull/252)) - fix(gw): missing return in error case ([ipfs/boxo#319](https://github.com/ipfs/boxo/pull/319)) - feat(routing/http): pass records limit on routing.FindProviders (#299) ([ipfs/boxo#299](https://github.com/ipfs/boxo/pull/299)) - bitswap/client: fix PeerResponseTrackerProbabilityOneKnownOneUnknownPeer - feat(gw): add ipfs_http_gw_car_stream_fail_duration_seconds (#312) ([ipfs/boxo#312](https://github.com/ipfs/boxo/pull/312)) - feat(gw): add ipfs_http_gw_request_types metric (#311) ([ipfs/boxo#311](https://github.com/ipfs/boxo/pull/311)) - refactor: simplify ipns validation in example - feat: add deprecator - fix(routing/v1): add newline in NDJSON responses (#300) ([ipfs/boxo#300](https://github.com/ipfs/boxo/pull/300)) - feat(gateway): redirect ipns b58mh to cid (#236) ([ipfs/boxo#236](https://github.com/ipfs/boxo/pull/236)) - refactor: replace assert.Nil for assert.NoError - tar: add test cases for validatePlatformPath - feat(ipns): helper ValidateWithPeerID and UnmarshalIpnsEntry (#294) ([ipfs/boxo#294](https://github.com/ipfs/boxo/pull/294)) - Revert "feat: reusable ipns verify (#292)" - feat: reusable ipns verify (#292) ([ipfs/boxo#292](https://github.com/ipfs/boxo/pull/292)) - refactor: remove badger, leveldb dependencies (#286) ([ipfs/boxo#286](https://github.com/ipfs/boxo/pull/286)) - feat(routing/http): add streaming support (#18) ([ipfs/boxo#18](https://github.com/ipfs/boxo/pull/18)) - feat(routing): allow-offline with routing put (#278) ([ipfs/boxo#278](https://github.com/ipfs/boxo/pull/278)) - refactor(gateway): switch to xxhash/v2 (#285) ([ipfs/boxo#285](https://github.com/ipfs/boxo/pull/285)) - github.com/ipfs/go-ipfs-util (v0.0.2 -> v0.0.3): - docs: remove contribution section - chore: bump version - chore: deprecate types and readme - sync: update CI config files (#12) ([ipfs/go-ipfs-util#12](https://github.com/ipfs/go-ipfs-util/pull/12)) - fix staticcheck ([ipfs/go-ipfs-util#9](https://github.com/ipfs/go-ipfs-util/pull/9)) - github.com/ipfs/go-ipld-format (v0.4.0 -> v0.5.0): - chore: release version v0.5.0 - feat: remove block decoding global registry - sync: update CI config files (#75) ([ipfs/go-ipld-format#75](https://github.com/ipfs/go-ipld-format/pull/75)) - sync: update CI config files (#74) ([ipfs/go-ipld-format#74](https://github.com/ipfs/go-ipld-format/pull/74)) - github.com/ipfs/go-ipld-legacy (v0.1.1 -> v0.2.1): - v0.2.1 ([ipfs/go-ipld-legacy#15](https://github.com/ipfs/go-ipld-legacy/pull/15)) - Expose a constructor for making a decoder with an existing link system ([ipfs/go-ipld-legacy#14](https://github.com/ipfs/go-ipld-legacy/pull/14)) - Update to v0.2.0 ([ipfs/go-ipld-legacy#13](https://github.com/ipfs/go-ipld-legacy/pull/13)) - Remove global variable ([ipfs/go-ipld-legacy#12](https://github.com/ipfs/go-ipld-legacy/pull/12)) - sync: update CI config files (#8) ([ipfs/go-ipld-legacy#8](https://github.com/ipfs/go-ipld-legacy/pull/8)) - github.com/ipfs/go-unixfsnode (v1.6.0 -> v1.7.1): - chore: bump to v1.7.1 - test: remove unnecessary t.Log - test: check if reader reads only necessary blocks - fix: do not read extra block if offset = at+childSize - doc: added simple doc for testutil package - bump v1.7.0 - feat(testutil): add test data generation utils (extracted from Lassie) - github.com/libp2p/go-libp2p (v0.27.3 -> v0.27.7): - Release v0.27.7 (#2374) ([libp2p/go-libp2p#2374](https://github.com/libp2p/go-libp2p/pull/2374)) - Release v0.27.6 (#2359) ([libp2p/go-libp2p#2359](https://github.com/libp2p/go-libp2p/pull/2359)) - Release v0.27.5 (#2324) ([libp2p/go-libp2p#2324](https://github.com/libp2p/go-libp2p/pull/2324)) - Bump version to v0.27.4 - identify: reject signed peer records on peer ID mismatch - swarm: change maps with multiaddress keys to use strings (#2284) ([libp2p/go-libp2p#2284](https://github.com/libp2p/go-libp2p/pull/2284)) - identify: avoid spuriously triggering pushes (#2299) ([libp2p/go-libp2p#2299](https://github.com/libp2p/go-libp2p/pull/2299)) - github.com/libp2p/go-libp2p-kad-dht (v0.23.0 -> v0.24.2): - chore: release v0.24.2 - chore: release v0.24.1 - fix: decrease tests noise, update kbucket and fix fixRTIUfNeeded - refactor: remove goprocess - fix: leaking go routines - chore: release v0.24.0 - fix: don't add unresponsive DHT servers to the Routing Table (#820) ([libp2p/go-libp2p-kad-dht#820](https://github.com/libp2p/go-libp2p-kad-dht/pull/820)) - github.com/libp2p/go-libp2p-kbucket (v0.5.0 -> v0.6.3): - fix: fix abba bug in UsefulNewPeer ([libp2p/go-libp2p-kbucket#122](https://github.com/libp2p/go-libp2p-kbucket/pull/122)) - chore: release v0.6.2 ([libp2p/go-libp2p-kbucket#121](https://github.com/libp2p/go-libp2p-kbucket/pull/121)) - Replacing UsefulPeer() with UsefulNewPeer() ([libp2p/go-libp2p-kbucket#120](https://github.com/libp2p/go-libp2p-kbucket/pull/120)) - chore: release 0.6.1 ([libp2p/go-libp2p-kbucket#119](https://github.com/libp2p/go-libp2p-kbucket/pull/119)) - UsefulPeer function ([libp2p/go-libp2p-kbucket#113](https://github.com/libp2p/go-libp2p-kbucket/pull/113)) - Fixed peer replacement with bucket size of 1. ([libp2p/go-libp2p-kbucket#117](https://github.com/libp2p/go-libp2p-kbucket/pull/117)) - GenRandomKey function ([libp2p/go-libp2p-kbucket#116](https://github.com/libp2p/go-libp2p-kbucket/pull/116)) - Removed maintainers from readme ([libp2p/go-libp2p-kbucket#115](https://github.com/libp2p/go-libp2p-kbucket/pull/115)) - Add maintainers ([libp2p/go-libp2p-kbucket#114](https://github.com/libp2p/go-libp2p-kbucket/pull/114)) - sync: update CI config files (#112) ([libp2p/go-libp2p-kbucket#112](https://github.com/libp2p/go-libp2p-kbucket/pull/112)) - github.com/libp2p/go-libp2p-routing-helpers (v0.6.2 -> v0.7.0): - chore: release v0.7.0 - fix: iterate over keys manually in ProvideMany - github.com/libp2p/go-reuseport (v0.2.0 -> v0.3.0): - release v0.3.0 (#103) ([libp2p/go-reuseport#103](https://github.com/libp2p/go-reuseport/pull/103)) - fix error handling when setting socket options (#102) ([libp2p/go-reuseport#102](https://github.com/libp2p/go-reuseport/pull/102)) - minor README updates (#96) ([libp2p/go-reuseport#96](https://github.com/libp2p/go-reuseport/pull/96)) - sync: update CI config files (#94) ([libp2p/go-reuseport#94](https://github.com/libp2p/go-reuseport/pull/94)) - feat: add a DialTimeout function ([libp2p/go-reuseport#92](https://github.com/libp2p/go-reuseport/pull/92)) - github.com/multiformats/go-multicodec (v0.8.1 -> v0.9.0): - Bump v0.9.0 - Bump v0.8.2 - chore: update submodules and go generate - chore: update submodules and go generate - chore: update submodules and go generate - chore: update submodules and go generate - chore: update submodules and go generate - chore: update submodules and go generate - github.com/multiformats/go-multihash (v0.2.1 -> v0.2.3): - chore: release v0.2.3 - perf: outline logic in Decode to allow for stack allocations - chore: release v0.2.2 - sha256: drop minio in favor of crypto/sha256 for go1.21 and above - sync: update CI config files (#169) ([multiformats/go-multihash#169](https://github.com/multiformats/go-multihash/pull/169)) - add handler for hasher.Write returned error ([multiformats/go-multihash#167](https://github.com/multiformats/go-multihash/pull/167)) - sync: update CI config files (#165) ([multiformats/go-multihash#165](https://github.com/multiformats/go-multihash/pull/165)) - test: add benchmark for all hash functions Sum
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Jorropo | 47 | +4394/-4458 | 202 | | Henrique Dias | 48 | +4344/-3962 | 205 | | Łukasz Magiera | 68 | +3604/-886 | 172 | | Adin Schmahmann | 8 | +1754/-1057 | 37 | | galargh | 7 | +1355/-1302 | 15 | | Gus Eggert | 7 | +1566/-655 | 33 | | rvagg | 1 | +396/-389 | 3 | | Michael Muré | 3 | +547/-202 | 14 | | Guillaume Michel - guissou | 5 | +153/-494 | 17 | | guillaumemichel | 15 | +446/-189 | 28 | | Laurent Senta | 4 | +472/-152 | 29 | | Rod Vagg | 6 | +554/-37 | 23 | | Marcin Rataj | 11 | +330/-82 | 21 | | Arthur Gavazza | 1 | +296/-87 | 7 | | Lucas Molas | 1 | +323/-56 | 6 | | Marco Munizaga | 5 | +227/-97 | 17 | | Alex | 8 | +163/-116 | 10 | | Steven Allen | 11 | +154/-114 | 14 | | Marten Seemann | 6 | +214/-41 | 12 | | web3-bot | 9 | +76/-75 | 28 | | Hector Sanjuan | 2 | +5/-96 | 4 | | Sukun | 1 | +83/-17 | 3 | | Steve Loeppky | 2 | +100/-0 | 2 | | Edgar Lee | 1 | +46/-46 | 12 | | Ivan Schasny | 1 | +67/-5 | 4 | | imthe1 | 1 | +65/-3 | 5 | | godcong | 2 | +30/-31 | 5 | | Will Scott | 4 | +36/-23 | 6 | | Petar Maymounkov | 1 | +45/-9 | 1 | | Ross Jones | 1 | +43/-1 | 2 | | William Entriken | 1 | +38/-0 | 1 | | João Pedro | 1 | +35/-0 | 1 | | jhertz | 1 | +21/-0 | 2 | | Nikhilesh Susarla | 1 | +21/-0 | 3 | | Matt Joiner | 1 | +11/-9 | 2 | | Vlad | 2 | +4/-2 | 2 | | Russell Dempsey | 2 | +4/-2 | 2 | | Will | 2 | +2/-2 | 2 | | Piotr Galar | 1 | +1/-1 | 1 | | Joel Gustafson | 1 | +1/-1 | 1 | | Dennis Trautwein | 1 | +1/-1 | 1 | | Bryan Stenson | 1 | +1/-1 | 1 | ================================================ FILE: docs/changelogs/v0.22.md ================================================ # Kubo changelog v0.22 - [v0.22.0](#v0220) ## v0.22.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [Gateway: support for `order=` and `dups=` parameters (IPIP-412)](#gateway-support-for-order-and-dups-parameters-ipip-412) - [`ipfs name publish` now supports V2 only IPNS records](#ipfs-name-publish-now-supports-v2-only-ipns-records) - [IPNS name resolution has been fixed](#ipns-name-resolution-has-been-fixed) - [go-libp2p v0.29.0 update with smart dialing](#go-libp2p-v0290-update-with-smart-dialing) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### Gateway: support for `order=` and `dups=` parameters (IPIP-412) The updated [`boxo/gateway` library](https://github.com/ipfs/boxo/tree/main/gateway) introduces support for ordered CAR responses through the inclusion of optional CAR content type parameters: `order=dfs` and `dups=y|n` from [IPIP-412](https://github.com/ipfs/specs/pull/412). Previously, Kubo already provided CARs in DFS order without duplicate blocks. With the implementation of IPIP-412, this behavior is now explicitly defined rather than implied. In the absence of `dups` or `order` in `Accept` request reader, the default CAR response will have the `Content-Type: application/vnd.ipld.car; version=1; order=dfs; dups=n` and the same blocks as Kubo 0.21. Kubo 0.22 still only supports DFS block ordering (`order=dfs`). However, it is now possible to request a DFS CAR stream with duplicate blocks by opting in via `Accept: application/vnd.ipld.car; order=dfs; dups=y`. This opt-in feature can be beneficial for memory-constrained clients and IoT devices, as it allows for streaming large DAGs without the need to store all previously encountered blocks in memory. #### `ipfs name publish` now supports V2 only IPNS records When publishing an IPNS record, you are now able to create v2 only records by passing `--v1compat=false`. By default, we still create V1+V2 records, such that there is the highest chance of backwards compatibility. The goal is to move to V2 only in the future. For more details, see [IPIP-428](https://specs.ipfs.tech/ipips/ipip-0428/) and the updated [IPNS Record Verification](https://specs.ipfs.tech/ipns/ipns-record/#record-verification) logic. #### IPNS name resolution has been fixed IPNS name resolution had a regression where if IPNS over PubSub was enabled, but the name was not also available via IPNS over PubSub it would take 1 minute to for the lookup to complete (if the record was not yet cached). This has been fixed and as before will give the best record from either the DHT subsystem or IPNS over PubSub, whichever comes back first. For details see [#9927](https://github.com/ipfs/kubo/issues/9927) and [#10020](https://github.com/ipfs/kubo/pull/10020). # go-libp2p v0.29.0 update with smart dialing We updated from [go-libp2p](https://github.com/libp2p/go-libp2p) [v0.27.7](https://github.com/libp2p/go-libp2p/releases/tag/v0.27.7) to [v0.29.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.29.0). This release includes smart dialing, which is a prioritization algorithm that will try to rank addresses and protocols rather than attempting all options in parallel. Anecdotally, we have observed [Kubo nodes make 30% less dials](https://github.com/libp2p/go-libp2p/issues/2326#issuecomment-1644332863) with no to low latency impact. This includes a breaking change to `ipfs id` and some of the `ipfs swarm` commands. We no longer report `ProtocolVersion`. This used to be hardcoded as `ipfs/0.1.0` and sent to other peers but was not providing any distinguishing value. See [libp2p/go-libp2p#2294](https://github.com/libp2p/go-libp2p/issues/2294) for more information. ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: change version to v0.22.0 - chore(misc/README.md): trim duplicated content - Merge branch 'release-v0.21' back into master - docs(readme): unofficial packages badge - chore: remove sharness tests ported to conformance testing (#9999) ([ipfs/kubo#9999](https://github.com/ipfs/kubo/pull/9999)) - ci: switch from testing against js-ipfs to helia (#10042) ([ipfs/kubo#10042](https://github.com/ipfs/kubo/pull/10042)) - chore: merge release back into master - chore: change orbitdb to haydenyoung EARLY_TESTERS - Fix usage numbers - chore: update early testers list (#9218) ([ipfs/kubo#9218](https://github.com/ipfs/kubo/pull/9218)) - docs: changelog v0.21 fixes (#10037) ([ipfs/kubo#10037](https://github.com/ipfs/kubo/pull/10037)) - refactor(ci): simplify Dockerfile and add docker image testing (#10021) ([ipfs/kubo#10021](https://github.com/ipfs/kubo/pull/10021)) - chore: update version - fix(relay): apply user provider options - libp2p: stop reporting ProtocolVersion - chore: update go-libp2p to v0.29.0 - chore: update go-libp2p to v0.28.1 - fix: mark all routers DoNotWaitForSearchValue (#10020) ([ipfs/kubo#10020](https://github.com/ipfs/kubo/pull/10020)) - feat(gateway): support for ipip-412 parameters - docs(commands): explain that swarm connect can reuse existing connections or known addresses (#10015) ([ipfs/kubo#10015](https://github.com/ipfs/kubo/pull/10015)) - docs: add Brave to RELEASE_ISSUE_TEMPLATE.md (#10012) ([ipfs/kubo#10012](https://github.com/ipfs/kubo/pull/10012)) - feat: webui@4.0.2 - ([ipfs/kubo#10008](https://github.com/ipfs/kubo/pull/10008)) - docs: skip check before prepare branch in RELEASE_ISSUE_TEMPLATE.md - docs: update RELEASE_ISSUE_TEMPLATE.md with a warning about npm publish - docs: update refs to kuboreleaser in RELEASE_ISSUE_TEMPLATE.md - docs: Gateway.HTTPHeaders - refactor: replace boxo/ipld/car by ipld/go-car - chore: bump to boxo master - fix: correctly handle migration of configs - fix(gateway): include CORS on subdomain redirects (#9994) ([ipfs/kubo#9994](https://github.com/ipfs/kubo/pull/9994)) - fix: docker repository initialization race condition - feat(ipns): records with V2-only signatures (#9932) ([ipfs/kubo#9932](https://github.com/ipfs/kubo/pull/9932)) - cmds/dag/import: pin roots by default (#9966) ([ipfs/kubo#9966](https://github.com/ipfs/kubo/pull/9966)) - docs: fix 0.21 changelog - feat!: dag import - don't pin roots by default (#9926) ([ipfs/kubo#9926](https://github.com/ipfs/kubo/pull/9926)) - fix(cmd): useful errors in dag import (#9945) ([ipfs/kubo#9945](https://github.com/ipfs/kubo/pull/9945)) - feat: webui@4.0.1 (#9940) ([ipfs/kubo#9940](https://github.com/ipfs/kubo/pull/9940)) - chore(docs): typo http→https - fix: more stable prometheus test (#9944) ([ipfs/kubo#9944](https://github.com/ipfs/kubo/pull/9944)) - ([ipfs/kubo#9937](https://github.com/ipfs/kubo/pull/9937)) - github.com/ipfs/boxo (v0.10.3 -> v0.11.0): - Release v0.11.0 ([ipfs/boxo#417](https://github.com/ipfs/boxo/pull/417)) - github.com/ipfs/go-bitswap (null -> v0.11.0): - chore: release v0.11.0 - chore: release v0.10.2 - fix: create a copy of the protocol slice in network.processSettings - chore: release v0.10.1 - fix: incorrect type in the WithTracer polyfill option - chore: fix incorrect log message when a bad option is passed - chore: release v0.10.0 - chore: update go-libp2p v0.22.0 - chore: release v0.9.0 - feat: split client and server ([ipfs/go-bitswap#570](https://github.com/ipfs/go-bitswap/pull/570)) - chore: remove goprocess from blockstoremanager - Don't add blocks to the datastore ([ipfs/go-bitswap#571](https://github.com/ipfs/go-bitswap/pull/571)) - Remove dependency on travis package from go-libp2p-testing ([ipfs/go-bitswap#569](https://github.com/ipfs/go-bitswap/pull/569)) - feat: add basic tracing (#562) ([ipfs/go-bitswap#562](https://github.com/ipfs/go-bitswap/pull/562)) - chore: release v0.7.0 (#566) ([ipfs/go-bitswap#566](https://github.com/ipfs/go-bitswap/pull/566)) - feat: coalesce and queue connection event handling (#565) ([ipfs/go-bitswap#565](https://github.com/ipfs/go-bitswap/pull/565)) - github.com/ipfs/go-merkledag (v0.10.0 -> v0.11.0): - chore: update v0.11.0 (#106) ([ipfs/go-merkledag#106](https://github.com/ipfs/go-merkledag/pull/106)) - update merkeldag to use the explicit decoder registry (#104) ([ipfs/go-merkledag#104](https://github.com/ipfs/go-merkledag/pull/104)) - Update status in README.md and added CODEOWNERS (#101) ([ipfs/go-merkledag#101](https://github.com/ipfs/go-merkledag/pull/101)) - github.com/ipld/go-car/v2 (v2.9.1-0.20230325062757-fff0e4397a3d -> v2.10.2-0.20230622090957-499d0c909d33): - feat: add inverse and version to filter cmd ([ipld/go-car#457](https://github.com/ipld/go-car/pull/457)) - v0.6.1 bump - chore: update usage of merkledag by go-car (#437) ([ipld/go-car#437](https://github.com/ipld/go-car/pull/437)) - feat(cmd/car): add '--no-wrap' option to 'create' command ([ipld/go-car#432](https://github.com/ipld/go-car/pull/432)) - fix: remove github.com/ipfs/go-ipfs-blockstore dependency - feat: expose index for StorageCar - perf: reduce NewCarReader allocations - fix(deps): update deps for cmd (use master go-car and go-car/v2 for now) - fix: new error strings from go-cid - fix: tests should match stderr for verbose output - fix: reading from stdin should broadcast EOF to block loaders - refactor insertion index to be publicly accessible ([ipld/go-car#408](https://github.com/ipld/go-car/pull/408)) - github.com/libp2p/go-libp2p (v0.27.9 -> v0.29.2): - release v0.29.2 - release v0.29.1 - swarm: don't open new streams over transient connections (#2450) ([libp2p/go-libp2p#2450](https://github.com/libp2p/go-libp2p/pull/2450)) - core/crypto: restrict RSA keys to <= 8192 bits (#2454) ([libp2p/go-libp2p#2454](https://github.com/libp2p/go-libp2p/pull/2454)) - Release version v0.29.0 (#2431) ([libp2p/go-libp2p#2431](https://github.com/libp2p/go-libp2p/pull/2431)) - webtransport: reject listening on a multiaddr with a certhash (#2426) ([libp2p/go-libp2p#2426](https://github.com/libp2p/go-libp2p/pull/2426)) - swarm: deprecate libp2p.DialRanker option (#2430) ([libp2p/go-libp2p#2430](https://github.com/libp2p/go-libp2p/pull/2430)) - quic: Update to quic-go v0.36.2 (#2424) ([libp2p/go-libp2p#2424](https://github.com/libp2p/go-libp2p/pull/2424)) - autonat: fix typo in WithSchedule option comment (#2425) ([libp2p/go-libp2p#2425](https://github.com/libp2p/go-libp2p/pull/2425)) - identify: filter nat64 well-known prefix ipv6 addresses (#2392) ([libp2p/go-libp2p#2392](https://github.com/libp2p/go-libp2p/pull/2392)) - update go-multiaddr to v0.10.1, use Unique function from there (#2407) ([libp2p/go-libp2p#2407](https://github.com/libp2p/go-libp2p/pull/2407)) - swarm: enable smart dialing by default (#2420) ([libp2p/go-libp2p#2420](https://github.com/libp2p/go-libp2p/pull/2420)) - transport integration tests: make TestMoreStreamsThanOurLimits less flaky (#2410) ([libp2p/go-libp2p#2410](https://github.com/libp2p/go-libp2p/pull/2410)) - holepunch: skip racy TestDirectDialWorks (#2419) ([libp2p/go-libp2p#2419](https://github.com/libp2p/go-libp2p/pull/2419)) - swarm: change relay dial delay to 500ms (#2421) ([libp2p/go-libp2p#2421](https://github.com/libp2p/go-libp2p/pull/2421)) - identify: disable racy TestLargeIdentifyMessage with race detector (#2401) ([libp2p/go-libp2p#2401](https://github.com/libp2p/go-libp2p/pull/2401)) - swarm: make black hole detection configurable (#2403) ([libp2p/go-libp2p#2403](https://github.com/libp2p/go-libp2p/pull/2403)) - net/mock: support ConnectionGater in MockNet (#2297) ([libp2p/go-libp2p#2297](https://github.com/libp2p/go-libp2p/pull/2297)) - docs: Add a Github workflow for checking dead links (#2406) ([libp2p/go-libp2p#2406](https://github.com/libp2p/go-libp2p/pull/2406)) - rcmgr: enable metrics by default (#2389) (#2409) ([libp2p/go-libp2p#2409](https://github.com/libp2p/go-libp2p/pull/2409)) - chore: remove outdated info in README and link to libp2p-implementers slack (#2405) ([libp2p/go-libp2p#2405](https://github.com/libp2p/go-libp2p/pull/2405)) - metrics: deduplicate code in examples (#2404) ([libp2p/go-libp2p#2404](https://github.com/libp2p/go-libp2p/pull/2404)) - transport tests: remove mplex tests (#2402) ([libp2p/go-libp2p#2402](https://github.com/libp2p/go-libp2p/pull/2402)) - swarm: implement Happy Eyeballs ranking (#2365) ([libp2p/go-libp2p#2365](https://github.com/libp2p/go-libp2p/pull/2365)) - docs: fix some comments (#2391) ([libp2p/go-libp2p#2391](https://github.com/libp2p/go-libp2p/pull/2391)) - metrics: provide separate docker-compose files for OSX and Linux (#2397) ([libp2p/go-libp2p#2397](https://github.com/libp2p/go-libp2p/pull/2397)) - identify: use zero-alloc slice sorting function (#2396) ([libp2p/go-libp2p#2396](https://github.com/libp2p/go-libp2p/pull/2396)) - rcmgr: move StatsTraceReporter to rcmgr package (#2388) ([libp2p/go-libp2p#2388](https://github.com/libp2p/go-libp2p/pull/2388)) - swarm: implement blackhole detection (#2320) ([libp2p/go-libp2p#2320](https://github.com/libp2p/go-libp2p/pull/2320)) - basichost / blankhost: wrap errors (#2331) ([libp2p/go-libp2p#2331](https://github.com/libp2p/go-libp2p/pull/2331)) - network: don't allocate in DedupAddrs (#2395) ([libp2p/go-libp2p#2395](https://github.com/libp2p/go-libp2p/pull/2395)) - rcmgr: test snapshot defaults and that we keep consistent defaults (#2315) ([libp2p/go-libp2p#2315](https://github.com/libp2p/go-libp2p/pull/2315)) - rcmgr: register prometheus metrics with the libp2p registerer (#2370) ([libp2p/go-libp2p#2370](https://github.com/libp2p/go-libp2p/pull/2370)) - metrics: make it possible to spin up Grafana using docker-compose (#2383) ([libp2p/go-libp2p#2383](https://github.com/libp2p/go-libp2p/pull/2383)) - identify: set stream deadlines for Identify and Identify Push streams (#2382) ([libp2p/go-libp2p#2382](https://github.com/libp2p/go-libp2p/pull/2382)) - fix: in the swarm move Connectedness emit after releasing conns (#2373) ([libp2p/go-libp2p#2373](https://github.com/libp2p/go-libp2p/pull/2373)) - metrics: add example for metrics and dashboard (#2232) ([libp2p/go-libp2p#2232](https://github.com/libp2p/go-libp2p/pull/2232)) - dashboards: finish metrics effort (#2362) ([libp2p/go-libp2p#2362](https://github.com/libp2p/go-libp2p/pull/2362)) - transport tests: many streams and lots of data (#2296) ([libp2p/go-libp2p#2296](https://github.com/libp2p/go-libp2p/pull/2296)) - webtransport: close the challenge stream after the Noise handshake (#2305) ([libp2p/go-libp2p#2305](https://github.com/libp2p/go-libp2p/pull/2305)) - test: document why InstantTimer is required (#2351) ([libp2p/go-libp2p#2351](https://github.com/libp2p/go-libp2p/pull/2351)) - rcmgr: fix link to dashboards in README (#2363) ([libp2p/go-libp2p#2363](https://github.com/libp2p/go-libp2p/pull/2363)) - docs: fix some comments errors (#2356) ([libp2p/go-libp2p#2356](https://github.com/libp2p/go-libp2p/pull/2356)) - release v0.28.0 (#2344) ([libp2p/go-libp2p#2344](https://github.com/libp2p/go-libp2p/pull/2344)) - nat: add HasDiscoveredNAT method for checking NAT environments (#2358) ([libp2p/go-libp2p#2358](https://github.com/libp2p/go-libp2p/pull/2358)) - swarm: fix stale DialBackoff comment (#2353) ([libp2p/go-libp2p#2353](https://github.com/libp2p/go-libp2p/pull/2353)) - swarm: use RLock for DialBackoff reads (#2354) ([libp2p/go-libp2p#2354](https://github.com/libp2p/go-libp2p/pull/2354)) - Clear stream scope if we error (#2345) ([libp2p/go-libp2p#2345](https://github.com/libp2p/go-libp2p/pull/2345)) - changelog: improve description of smart dialing (#2342) ([libp2p/go-libp2p#2342](https://github.com/libp2p/go-libp2p/pull/2342)) - swarm: make smart-dialing opt in (#2340) ([libp2p/go-libp2p#2340](https://github.com/libp2p/go-libp2p/pull/2340)) - swarm: cleanup address filtering logic (#2333) ([libp2p/go-libp2p#2333](https://github.com/libp2p/go-libp2p/pull/2333)) - chore: add 0.28.0 changelog (#2335) ([libp2p/go-libp2p#2335](https://github.com/libp2p/go-libp2p/pull/2335)) - swarm: improve documentation for the DefaultDialRanker (#2336) ([libp2p/go-libp2p#2336](https://github.com/libp2p/go-libp2p/pull/2336)) - holepunch: add metrics (#2246) ([libp2p/go-libp2p#2246](https://github.com/libp2p/go-libp2p/pull/2246)) - swarm: implement smart dialing logic (#2260) ([libp2p/go-libp2p#2260](https://github.com/libp2p/go-libp2p/pull/2260)) - revert "feat:add contexts to all peerstore methods (#2312)" (#2328) ([libp2p/go-libp2p#2328](https://github.com/libp2p/go-libp2p/pull/2328)) - identify: don't save signed peer records (#2325) ([libp2p/go-libp2p#2325](https://github.com/libp2p/go-libp2p/pull/2325)) - feat:add contexts to all peerstore methods (#2312) ([libp2p/go-libp2p#2312](https://github.com/libp2p/go-libp2p/pull/2312)) - swarm: Dedup addresses to dial (#2322) ([libp2p/go-libp2p#2322](https://github.com/libp2p/go-libp2p/pull/2322)) - identify: filter received addresses based on the node's remote address (#2300) ([libp2p/go-libp2p#2300](https://github.com/libp2p/go-libp2p/pull/2300)) - update go-nat to v0.2.0, use context on AddMapping and RemoveMapping (#2319) ([libp2p/go-libp2p#2319](https://github.com/libp2p/go-libp2p/pull/2319)) - transport integration tests: add tests for resource manager (#2285) ([libp2p/go-libp2p#2285](https://github.com/libp2p/go-libp2p/pull/2285)) - identify: reject signed peer records on peer ID mismatch - identify: don't send default protocol version (#2303) ([libp2p/go-libp2p#2303](https://github.com/libp2p/go-libp2p/pull/2303)) - metrics: add instance filter to all dashboards (#2301) ([libp2p/go-libp2p#2301](https://github.com/libp2p/go-libp2p/pull/2301)) - identify: avoid spuriously triggering pushes (#2299) ([libp2p/go-libp2p#2299](https://github.com/libp2p/go-libp2p/pull/2299)) - net/mock: mimic Swarm's event and notification behavior in MockNet (#2287) ([libp2p/go-libp2p#2287](https://github.com/libp2p/go-libp2p/pull/2287)) - examples: fix flaky multipro TestMain (#2289) ([libp2p/go-libp2p#2289](https://github.com/libp2p/go-libp2p/pull/2289)) - swarm: change maps with multiaddress keys to use strings (#2284) ([libp2p/go-libp2p#2284](https://github.com/libp2p/go-libp2p/pull/2284)) - tests: add comprehensive end-to-end tests for connection gating (#2200) ([libp2p/go-libp2p#2200](https://github.com/libp2p/go-libp2p/pull/2200)) - swarm: log unexpected listener errors (#2277) ([libp2p/go-libp2p#2277](https://github.com/libp2p/go-libp2p/pull/2277)) - websocket: switch back to the gorilla library (#2280) ([libp2p/go-libp2p#2280](https://github.com/libp2p/go-libp2p/pull/2280)) - quic: prioritise listen connections for reuse (#2262) ([libp2p/go-libp2p#2262](https://github.com/libp2p/go-libp2p/pull/2262)) - quic virtual listener: don't panic when quic-go's accept call errors (#2276) ([libp2p/go-libp2p#2276](https://github.com/libp2p/go-libp2p/pull/2276)) - tests: add docks for debugging flaky tests (#2216) ([libp2p/go-libp2p#2216](https://github.com/libp2p/go-libp2p/pull/2216)) - webtransport: only add cert hashes if we already started listening (#2271) ([libp2p/go-libp2p#2271](https://github.com/libp2p/go-libp2p/pull/2271)) - Revert "webtransport: initialize the certmanager when creating the transport (#2268)" (#2273) ([libp2p/go-libp2p#2273](https://github.com/libp2p/go-libp2p/pull/2273)) - webtransport: initialize the certmanager when creating the transport (#2268) ([libp2p/go-libp2p#2268](https://github.com/libp2p/go-libp2p/pull/2268)) - move NAT mapping logic out of the host, add tests for NAT handling ([libp2p/go-libp2p#2248](https://github.com/libp2p/go-libp2p/pull/2248)) - githooks: add a githook to check that the test-plans go.mod is tidied (#2256) ([libp2p/go-libp2p#2256](https://github.com/libp2p/go-libp2p/pull/2256)) - quic: fix race condition when generating random holepunch packet (#2263) ([libp2p/go-libp2p#2263](https://github.com/libp2p/go-libp2p/pull/2263)) - swarm: remove unused variable in addrDial (#2257) ([libp2p/go-libp2p#2257](https://github.com/libp2p/go-libp2p/pull/2257)) - github.com/libp2p/go-libp2p-routing-helpers (v0.7.0 -> v0.7.1): - chore: release v0.7.1 - fix: for comparallel never return nil channel for FindProvidersAsync - chore: rename DoNotWaitForStreamingResponses to DoNotWaitForSearchValue - feat: add DoNotWaitForStreamingResponses to ParallelRouter - chore: cleanup error handling in compparallel - fix: correctly handle errors in compparallel - fix: make the ProvideMany docs clearer - perf: remove goroutine that just waits before closing with a synchronous waitgroup - github.com/libp2p/go-nat (v0.1.0 -> v0.2.0): - release v0.2.0 (#30) ([libp2p/go-nat#30](https://github.com/libp2p/go-nat/pull/30)) - update deps, use contexts on UPnP functions (#29) ([libp2p/go-nat#29](https://github.com/libp2p/go-nat/pull/29)) - sync: update CI config files (#28) ([libp2p/go-nat#28](https://github.com/libp2p/go-nat/pull/28)) - sync: update CI config files (#24) ([libp2p/go-nat#24](https://github.com/libp2p/go-nat/pull/24)) - github.com/libp2p/go-yamux/v4 (v4.0.0 -> v4.0.1): - Release v4.0.1 ([libp2p/go-yamux#106](https://github.com/libp2p/go-yamux/pull/106)) - fix: sendWindowUpdate respects deadlines (#105) ([libp2p/go-yamux#105](https://github.com/libp2p/go-yamux/pull/105)) - github.com/multiformats/go-multiaddr (v0.9.0 -> v0.10.1): - release v0.10.1 (#206) ([multiformats/go-multiaddr#206](https://github.com/multiformats/go-multiaddr/pull/206)) - fix nat64 well-known prefix check (#205) ([multiformats/go-multiaddr#205](https://github.com/multiformats/go-multiaddr/pull/205)) - release v0.10.0 (#204) ([multiformats/go-multiaddr#204](https://github.com/multiformats/go-multiaddr/pull/204)) - add a Unique function (#203) ([multiformats/go-multiaddr#203](https://github.com/multiformats/go-multiaddr/pull/203)) - manet: add function to test if address is NAT64 IPv4 converted IPv6 address (#202) ([multiformats/go-multiaddr#202](https://github.com/multiformats/go-multiaddr/pull/202)) - sync: update CI config files (#190) ([multiformats/go-multiaddr#190](https://github.com/multiformats/go-multiaddr/pull/190))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Henrique Dias | 14 | +3735/-17889 | 185 | | Sukun | 28 | +5910/-957 | 100 | | Jorropo | 40 | +2913/-2112 | 205 | | Marten Seemann | 41 | +2926/-1833 | 163 | | Marco Munizaga | 20 | +1559/-586 | 81 | | Prem Chaitanya Prathi | 1 | +757/-740 | 61 | | Laurent Senta | 2 | +69/-1094 | 32 | | Marcin Rataj | 11 | +339/-198 | 22 | | Steven Allen | 2 | +313/-161 | 9 | | Will | 2 | +118/-211 | 9 | | Adin Schmahmann | 4 | +275/-41 | 8 | | Michael Muré | 1 | +113/-164 | 6 | | Rod Vagg | 8 | +228/-46 | 28 | | Gus Eggert | 5 | +156/-93 | 21 | | Adrian Sutton | 1 | +190/-17 | 4 | | Hlib Kanunnikov | 3 | +139/-40 | 9 | | VM | 2 | +80/-79 | 49 | | UnkwUsr | 1 | +0/-124 | 1 | | Piotr Galar | 4 | +51/-59 | 5 | | web3-bot | 3 | +22/-46 | 4 | | Will Scott | 2 | +29/-28 | 6 | | Prithvi Shahi | 2 | +40/-7 | 2 | | Brad Fitzpatrick | 1 | +42/-2 | 2 | | Steve Loeppky | 1 | +6/-23 | 2 | | Sahib Yar | 1 | +4/-4 | 3 | | Russell Dempsey | 2 | +4/-2 | 2 | | Mohamed MHAMDI | 1 | +3/-3 | 1 | | Bryan White | 1 | +2/-2 | 1 | | Dennis Trautwein | 1 | +1/-1 | 1 | | Antonio Navarro Perez | 1 | +0/-1 | 1 | ================================================ FILE: docs/changelogs/v0.23.md ================================================ # Kubo changelog v0.23 - [v0.23.0](#v0230) ## v0.23.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [Mplex deprecation](#mplex-deprecation) - [Gateway: meaningful CAR responses on Not Found errors](#gateway-meaningful-car-responses-on-not-found-errors) - [Gateway: added `Gateway.DisableHTMLErrors` configuration option](#gateway-added-gatewaydisablehtmlerrors-configuration-option) - [Binary characters in file names: no longer works with old clients and new Kubo servers](#binary-characters-in-file-names-no-longer-works-with-old-clients-and-new-kubo-servers) - [Self-hosting `/routing/v1` endpoint for delegated routing needs](#self-hosting-routingv1-endpoint-for-delegated-routing-needs) - [Trustless Gateway Over Libp2p Experiment](#trustless-gateway-over-libp2p-experiment) - [Removal of `/quic` (Draft 29) support](#removal-of-quic-draft-29-support) - [Better Caching of multiaddresses for providers in DHT servers](#better-caching-of-multiaddresses-for-providers-in-dht-servers) - [Fixed FUSE multiblock structures](#fixed-fuse-multiblock-structures) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### Mplex deprecation Mplex is being deprecated, this is because it is unreliable and randomly drop streams when sending data *too fast*. New pieces of code rely on backpressure, that means the stream will dynamically slow down the sending rate if data is getting backed up. Backpressure is provided by **Yamux** and **QUIC**. In case you need compatibility with older implementations that do not ship with Yamux (like default's JS-IPFS) you can turned it back ON in the config with: ```console $ ipfs config --json Swarm.Transports.Multiplexers.Mplex 200 ``` We will completely remove Mplex in v0.24 as it makes protocols very bad to implement, if you are in this situation you need to add yamux support to your other implementation. #### Gateway: meaningful CAR responses on Not Found errors When requesting a CAR from the gateway, the root of the CAR might no longer be meaningful. By default, the CAR root will be the last resolvable segment of the path. However, in situations where the path cannot be resolved, such as when the path does not exist, a CAR will be sent with a root of `bafkqaaa` (empty CID). This CAR will contain all blocks necessary to validate that the path does not exist without having to trust the gateway. #### Gateway: added `Gateway.DisableHTMLErrors` configuration option The `Gateway.DisableHTMLErrors` configuration option forces errors to be displayed in browsers as plain text (`text/plain`) rather than HTML error pages. It's especially beneficial for whitelabel or middleware deployments that wish to avoid IPFS branding and links on error pages in browsers. #### Binary characters in file names: no longer works with old clients and new Kubo servers In this version, we updated Kubo to support Go 1.20+. In Go 1.20, a regression regarding multipart headers was [introduced](https://github.com/golang/go/issues/60674). This only affects `ipfs add` when a file name has binary characters in its name. As a consequence, we had to update the encoding of the file name headers. This is the compatibility table: | | New Client | Old Client | |------------|------------|-------------| | New Server | ✅ | 🟡* | | Old Server | ✅ | ✅ | *Old clients can only send Unicode file paths to the server. #### Self-hosting `/routing/v1` endpoint for delegated routing needs The `Routing` system configured in Kubo can be now exposed on the gateway port as a standard HTTP [Routing V1](https://specs.ipfs.tech/routing/http-routing-v1/) API endpoint. This allows self-hosting and experimentation with custom delegated routers. This is disabled by default, but can be enabled by setting [`Gateway.ExposeRoutingAPI`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewayexposeroutingapi) to `true` . #### Trustless Gateway Over Libp2p Experiment In this update, we've introduced an experimental opt-in feature allowing users to serve a subset of [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/) responses, such as blocks and CARs, over libp2p. This enhancement leverages the ongoing [`/http/1.1` specification work in libp2p](https://github.com/libp2p/specs/pull/508) to make it easier to support HTTP semantics over libp2p streams. This development means that if users wish to utilize the Trustless Gateway API for data transport, they can now do so even in scenarios where standard HTTP might be problematic, such as when the endpoint is behind a firewall or when attempting to serve data to a browser without a CA certificate. See [HTTP Gateway over Libp2p](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#http-gateway-over-libp2p) for details about this experiment. #### Removal of `/quic` (Draft 29) support Kubo no longer supports QUIC Draft 29. This means that older nodes aren't able to connect to newer nodes using QUIC Draft 29. However, they are still able to connect through any other transport that both nodes talk (such as QUIC RFC 9000, or TCP). QUIC Draft 29 was a preliminary implementation of QUIC before the official RFC 9000 was published, and it has now been dropped by [`go-libp2p`](https://github.com/libp2p/go-libp2p/releases/tag/v0.30.0) and therefore Kubo. In [Kubo 0.18](https://github.com/ipfs/kubo/releases/tag/v0.18.0), we shipped a migration to have listeners for both `/quic` (Draft 29) and `/quic-v1` (RFC 9000). Similarly, in this version we are shipping a migration to remove the current `/quic` addresses, maintaining the `/quic-v1` addresses only. For more background information, check [issue #9496](https://github.com/ipfs/kubo/issues/9496). #### Better Caching of multiaddresses for providers in DHT servers Thanks to [probelab.io's RFM17.1](https://github.com/plprobelab/network-measurements/blob/master/results/rfm17.1-sharing-prs-with-multiaddresses.md) DHT servers will [now cache the addresses of content hosts for the lifetime of the provider record](https://github.com/libp2p/go-libp2p-kad-dht/commit/777160f164b8c187c534debd293157031e9f3a02). This means clients who resolve content from these servers get a responses which include both peer id and multiaddresses. In most cases this enables skipping a second query which resolves the peer id to multiaddresses for stable enough peers. This will improve content fetching lantency in the network overtime as servers updates. #### Fixed FUSE multiblock structures `ls`ing directories and reading dag-pb files on a fuse volume have been fixed. [#9044](https://github.com/ipfs/kubo/issues/9044) Thx a lot @bmwiedemann for debugging this issue. ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - fix: align systemd unit file with default IPFS installation path (#10163) ([ipfs/kubo#10163](https://github.com/ipfs/kubo/pull/10163)) - docs: capitalize headers for consistency - Merge commit '695bf66674931a138862b6fa2cb0b16dc2f6ddd8' into release-v0.23.0 - chore: update version - changelog: generalize fuse 9044's entry - changelog: update fuse 9044's entry - Update go-unixfsnode to 1.8.0 to fix FUSE - docs(readme): header improvements (#10144) ([ipfs/kubo#10144](https://github.com/ipfs/kubo/pull/10144)) - fix(docker): allow nofuse builds for MacOS (#10135) ([ipfs/kubo#10135](https://github.com/ipfs/kubo/pull/10135)) - docs: fix typos - docs: s/ipfs dht/amino dht/ - changelog: mention probelab RFM17.1 dht improvement - tests: remove sharness ping tests - perf: make bootstrap saves O(N) - chore: update go-libp2p-kad-dht - chore: webui v4.1.1 (#10120) ([ipfs/kubo#10120](https://github.com/ipfs/kubo/pull/10120)) - core/bootstrap: fix panic without backup bootstrap peer functions (#10029) ([ipfs/kubo#10029](https://github.com/ipfs/kubo/pull/10029)) - feat: add Gateway.DisableHTMLErrors option (#10137) ([ipfs/kubo#10137](https://github.com/ipfs/kubo/pull/10137)) - fix(migrations): use dweb.link (#10133) ([ipfs/kubo#10133](https://github.com/ipfs/kubo/pull/10133)) - docs: add changelog info for QUIC Draft 29 (#10132) ([ipfs/kubo#10132](https://github.com/ipfs/kubo/pull/10132)) - feat: add gateway to http over libp2p ([ipfs/kubo#10108](https://github.com/ipfs/kubo/pull/10108)) - migration: update 14-to-15 to v1.0.1 - chore: update to build with Go 1.21 - refactor: stop using go-libp2p deprecated peer.ID.Pretty - docs(readonly): fix typo - docs(changelog): link to relevant IPIP - fix: hamt traversal in ipld-explorer (webui@4.1.0) (#10025) ([ipfs/kubo#10025](https://github.com/ipfs/kubo/pull/10025)) - refactor: if statement (#10105) ([ipfs/kubo#10105](https://github.com/ipfs/kubo/pull/10105)) - chore: bump repo version to 15 - docs: remove link to deleted #accelerated-dht-client - feat(gateway): expose /routing/v1 server (opt-in) (#9877) ([ipfs/kubo#9877](https://github.com/ipfs/kubo/pull/9877)) - improve error in fuse node failures - chore: update boxo, go-libp2p, and internalize mplex (#10095) ([ipfs/kubo#10095](https://github.com/ipfs/kubo/pull/10095)) - dockerfile: reorder copy order for better layer caching - refactor: using error is instead of == (#10093) ([ipfs/kubo#10093](https://github.com/ipfs/kubo/pull/10093)) - fix: use %-encoded headers in most compatible way - fix: open /dev/null with read write permissions - chore: bump to go 1.20 - docs(readme): new logo and header - docker: change to releases that follow debian's updates - docker: bump debian version to bookworm - chore: restore exec perms for t0116-gateway-cache.sh and fixtures (#10085) ([ipfs/kubo#10085](https://github.com/ipfs/kubo/pull/10085)) - fix(gw): useful IPIP-402 CARs on not found errors (#10084) ([ipfs/kubo#10084](https://github.com/ipfs/kubo/pull/10084)) - feat: add zsh completions (#10040) ([ipfs/kubo#10040](https://github.com/ipfs/kubo/pull/10040)) - style: remove commented imports [skip changelog] - style: gofumpt and godot [skip changelog] (#10081) ([ipfs/kubo#10081](https://github.com/ipfs/kubo/pull/10081)) - chore: bump boxo for verifcid breaking changes - chore: remove outdated comment (#10077) ([ipfs/kubo#10077](https://github.com/ipfs/kubo/pull/10077)) - chore: remove deprecated testground plans - feat: allow users to option again into mplex - feat: remove Mplex - docs(readme): minimal reqs (#10066) ([ipfs/kubo#10066](https://github.com/ipfs/kubo/pull/10066)) - docs: add v0.23.md - docs: get ready for v0.23 - chore: fix link in v0.22 changelog - github.com/ipfs/boxo (v0.11.0 -> v0.13.1): - Release v0.13.1 ([ipfs/boxo#469](https://github.com/ipfs/boxo/pull/469)) - Release v0.13.0 ([ipfs/boxo#465](https://github.com/ipfs/boxo/pull/465)) - Release v0.12 ([ipfs/boxo#446](https://github.com/ipfs/boxo/pull/446)) - github.com/ipfs/go-graphsync (v0.14.4 -> v0.15.1): - v0.15.1 bump - fix: partial revert of 1be7c1a20; make traverser process identity CIDs - v0.15.0 bump - chore: add identity CID parse tests - fix: traverser should skip over identity CIDs - fix(ipld): update ipld deps, only slurp LargeBytesNode when matching - docs(version): update for v0.14.7 - Handle context cancellation properly (#428) ([ipfs/go-graphsync#428](https://github.com/ipfs/go-graphsync/pull/428)) - chore(version.json): update for v0.14.6 - feat: MaxLinks for requests (#420) ([ipfs/go-graphsync#420](https://github.com/ipfs/go-graphsync/pull/420)) - fix(responsemanager): network disconnect reliability (#425) ([ipfs/go-graphsync#425](https://github.com/ipfs/go-graphsync/pull/425)) - Update version to reflect latest fixes (#424) ([ipfs/go-graphsync#424](https://github.com/ipfs/go-graphsync/pull/424)) - Fix shutdown bug in #412 (#422) ([ipfs/go-graphsync#422](https://github.com/ipfs/go-graphsync/pull/422)) - github.com/ipfs/go-ipfs-cmds (v0.9.0 -> v0.10.0): - chore: version 0.10.0 - fix: panic when calling .SetLength for writerResponseEmitter - fix!: client with raw abs path option - doc: clarify flag inheritance explanation - ci: uci/copy-templates ([ipfs/go-ipfs-cmds#242](https://github.com/ipfs/go-ipfs-cmds/pull/242)) - chore: remove dep on github.com/Kubuxu/go-os-helper - github.com/ipfs/go-unixfsnode (v1.7.1 -> v1.8.1): - v1.8.1 bump - testutil: relax DirEntry usage for non-dag-pb - v1.8.0 bump - fix: add cross-impl shard test - files returned from unixfsnode should be traversable back to their substrate - fix: better import name - chore: refactor and add tests with fixtures - fix: proper tsize encoding in sharded files - rel 1.7.4 - Provide path for getting sizes on directory iteration ([ipfs/go-unixfsnode#60](https://github.com/ipfs/go-unixfsnode/pull/60)) - tag 1.7.3 ([ipfs/go-unixfsnode#57](https://github.com/ipfs/go-unixfsnode/pull/57)) - Fail to construct preload hamt shards when traversal fails ([ipfs/go-unixfsnode#55](https://github.com/ipfs/go-unixfsnode/pull/55)) - fix: large files support io.SeekCurrent ([ipfs/go-unixfsnode#56](https://github.com/ipfs/go-unixfsnode/pull/56)) - chore(version): update version number - feat: add entity matcher w/o preload, add matcher fn for consuming bytes ([ipfs/go-unixfsnode#52](https://github.com/ipfs/go-unixfsnode/pull/52)) - github.com/ipld/go-ipld-prime (v0.20.0 -> v0.21.0): - v0.21.0 release - fix(selectors): document ranges in slice matcher - fix(selectors): update ipld/ipld submodule with latest fixtures - fix(selectors): more permissive with slice "from" underflow - chore: extract simpleBytes to testutil package - feat(selectors): negative values for slice matcher's From and To - chore: extract MultiByteNote to testutil package - feat(test): add matcher/slice selector test cases - feat: remove hard-error when slice matcher reaches non-string/bytes node - fix: cache offsets for sequential reads - feat: add inline union representation to schema parser - fix: basic.NewInt returns pointer (like others) - fix(bindnode): listpairs value assembly handles complex reprs - fix(bindnode): listpairs repr assembler handles AssignNode - fix(schema): handle parsing of "listpairs" in the DSL - fix: remove _skipAbsent labels - fix: make listpairs repr [[k1,v1],[k2,v2]...] - feat(bindnode): support listpairs struct representation - fix(windows,test): avoid "already exists" error on codegen tests for Windows - Make traversal.WalkTransforming() work - doc: clean up and expand on traversal pkg docs - doc: add lots of notes about using the preloader and the budget - doc: expand on preloader docs - fix: inline initialPhase() logic for clarity - feat: preload walk using phase state, call preloader once per link - fix: handle Budget & SeenLinks - chore: remove BufferedLoader - fix: recurse preloader at block level - fix: Context->PreloadContext for clarity and consistency with LinkContext - fix: replace ioutil.ReadAll - fix: fix tooling complaints - feat: add BufferedLoader - feat(traversal): allow preloading functionality - fix: address dodgy test case variable capture - stop using the deprecated io/ioutil package - stop using the deprecated io/ioutil package - stop using the deprecated io/ioutil package - fix: make StartAtPath work properly for matching walks - github.com/libp2p/go-libp2p (v0.29.2 -> v0.31.0): - release v0.31.0 (#2543) ([libp2p/go-libp2p#2543](https://github.com/libp2p/go-libp2p/pull/2543)) - dashboards: improve naming for black hole panel (#2539) ([libp2p/go-libp2p#2539](https://github.com/libp2p/go-libp2p/pull/2539)) - reuseport: use DialContext instead of Dial to fail quickly (#2541) ([libp2p/go-libp2p#2541](https://github.com/libp2p/go-libp2p/pull/2541)) - swarm: track dial cancellation reason (#2532) ([libp2p/go-libp2p#2532](https://github.com/libp2p/go-libp2p/pull/2532)) - p2p/http: cache json wellknown mappings in the .well-known handler (#2537) ([libp2p/go-libp2p#2537](https://github.com/libp2p/go-libp2p/pull/2537)) - feat: Implement HTTP spec (#2438) ([libp2p/go-libp2p#2438](https://github.com/libp2p/go-libp2p/pull/2438)) - move libp2p/go-libp2p-gostream to p2p/net/gostream ([libp2p/go-libp2p#2535](https://github.com/libp2p/go-libp2p/pull/2535)) - host: disable black hole detection on autonat dialer (#2529) ([libp2p/go-libp2p#2529](https://github.com/libp2p/go-libp2p/pull/2529)) - identify: disable racy test when running with race detector (#2526) ([libp2p/go-libp2p#2526](https://github.com/libp2p/go-libp2p/pull/2526)) - swarm: return a more meaningful error when dialing QUIC draft-29 (#2524) ([libp2p/go-libp2p#2524](https://github.com/libp2p/go-libp2p/pull/2524)) - swarm: fix Unwrap for DialError, implement Unwrap for TransportError (#2437) ([libp2p/go-libp2p#2437](https://github.com/libp2p/go-libp2p/pull/2437)) - swarm: return errors on filtered addresses when dialing (#2461) ([libp2p/go-libp2p#2461](https://github.com/libp2p/go-libp2p/pull/2461)) - core: add ErrPeerIDMismatch error type to replace ad-hoc errors (#2451) ([libp2p/go-libp2p#2451](https://github.com/libp2p/go-libp2p/pull/2451)) - update quic-go to v0.38.1 (#2506) ([libp2p/go-libp2p#2506](https://github.com/libp2p/go-libp2p/pull/2506)) - quic: don't claim to be able to dial draft-29 in CanDial (#2520) ([libp2p/go-libp2p#2520](https://github.com/libp2p/go-libp2p/pull/2520)) - examples: update go-libp2p to v0.30.0 (#2507) ([libp2p/go-libp2p#2507](https://github.com/libp2p/go-libp2p/pull/2507)) - metrics: update dashboard names from libp2p to go-libp2p (#2512) ([libp2p/go-libp2p#2512](https://github.com/libp2p/go-libp2p/pull/2512)) - chore: be more descriptive about where public dashboards come from (#2508) ([libp2p/go-libp2p#2508](https://github.com/libp2p/go-libp2p/pull/2508)) - release v0.30.0 (#2505) ([libp2p/go-libp2p#2505](https://github.com/libp2p/go-libp2p/pull/2505)) - transport tests: add deadline tests (#2286) ([libp2p/go-libp2p#2286](https://github.com/libp2p/go-libp2p/pull/2286)) - chore: remove unused and outdated package-list.json (#2499) ([libp2p/go-libp2p#2499](https://github.com/libp2p/go-libp2p/pull/2499)) - muxer: remove support for mplex (#2498) ([libp2p/go-libp2p#2498](https://github.com/libp2p/go-libp2p/pull/2498)) - transport tests: refactor workers in TestMoreStreamsThanOurLimits (#2472) ([libp2p/go-libp2p#2472](https://github.com/libp2p/go-libp2p/pull/2472)) - use standard library sha256 implementation for Go 1.21 (#2309) ([libp2p/go-libp2p#2309](https://github.com/libp2p/go-libp2p/pull/2309)) - quic: update quic-go to v0.37.5 (#2497) ([libp2p/go-libp2p#2497](https://github.com/libp2p/go-libp2p/pull/2497)) - cleanup: add continue in case of failure in the (*BasicHost).Addrs certhash loop (#2492) ([libp2p/go-libp2p#2492](https://github.com/libp2p/go-libp2p/pull/2492)) - tests: add a CertHashes testcase in TestInferWebtransportAddrsFromQuic (#2495) ([libp2p/go-libp2p#2495](https://github.com/libp2p/go-libp2p/pull/2495)) - basichost: use byte representation of WebTransport multiaddr as map key (#2494) ([libp2p/go-libp2p#2494](https://github.com/libp2p/go-libp2p/pull/2494)) - webtransport: check for UDP multiaddr component in address matcher (#2491) ([libp2p/go-libp2p#2491](https://github.com/libp2p/go-libp2p/pull/2491)) - swarm: remove unnecessary reqno for pending request tracking (#2460) ([libp2p/go-libp2p#2460](https://github.com/libp2p/go-libp2p/pull/2460)) - quic: drop support for QUIC draft-29 (#2487) ([libp2p/go-libp2p#2487](https://github.com/libp2p/go-libp2p/pull/2487)) - metrics: add links to public dashboards (#2486) ([libp2p/go-libp2p#2486](https://github.com/libp2p/go-libp2p/pull/2486)) - swarm: remove leftover TODO (#2474) ([libp2p/go-libp2p#2474](https://github.com/libp2p/go-libp2p/pull/2474)) - peerstore: deprecate the database-backed peerstore (#2475) ([libp2p/go-libp2p#2475](https://github.com/libp2p/go-libp2p/pull/2475)) - identify: fix sorting of observed addresses (#2476) ([libp2p/go-libp2p#2476](https://github.com/libp2p/go-libp2p/pull/2476)) - update go-multiaddr to v0.11.0 (#2467) ([libp2p/go-libp2p#2467](https://github.com/libp2p/go-libp2p/pull/2467)) - chore: update golang-lru to v2.0.4, fixing semver violation (#2448) ([libp2p/go-libp2p#2448](https://github.com/libp2p/go-libp2p/pull/2448)) - swarm: don't open new streams over transient connections (#2450) ([libp2p/go-libp2p#2450](https://github.com/libp2p/go-libp2p/pull/2450)) - core/crypto: restrict RSA keys to <= 8192 bits (#2454) ([libp2p/go-libp2p#2454](https://github.com/libp2p/go-libp2p/pull/2454)) - chore: add notable project requirement (#2453) ([libp2p/go-libp2p#2453](https://github.com/libp2p/go-libp2p/pull/2453)) - examples: update go-libp2p to v0.29.0 (#2432) ([libp2p/go-libp2p#2432](https://github.com/libp2p/go-libp2p/pull/2432)) - examples: fix description of command line flags for pubsub (#2400) ([libp2p/go-libp2p#2400](https://github.com/libp2p/go-libp2p/pull/2400)) - basichost: remove invalid comment (#2435) ([libp2p/go-libp2p#2435](https://github.com/libp2p/go-libp2p/pull/2435)) - github.com/libp2p/go-libp2p-kad-dht (v0.24.2 -> v0.24.4): - Make v0.24.4 ([libp2p/go-libp2p-kad-dht#931](https://github.com/libp2p/go-libp2p-kad-dht/pull/931)) - github.com/libp2p/go-libp2p-routing-helpers (v0.7.1 -> v0.7.3): - chore: release v0.7.3 - nit: invert if - fix: for getValueOrErrorParallel do not return values if they come with errors - test: add test to make sure we return not found when we get errors back with values - chore: release v0.7.2 - tracing: do not leak goroutines when the context is canceled - tracing: allow for reuse of the tracing - tracing: add tracing to compose parallel's worker - tests: add more tests - tests: mark all tests Parallel - tracing: add highlevel APIs records on the composable routers - github.com/libp2p/go-reuseport (v0.3.0 -> v0.4.0): - release v0.4.0 (#111) ([libp2p/go-reuseport#111](https://github.com/libp2p/go-reuseport/pull/111)) - use SO_REUSEPORT_LB on FreeBSD (#106) ([libp2p/go-reuseport#106](https://github.com/libp2p/go-reuseport/pull/106)) - github.com/multiformats/go-multiaddr (v0.10.1 -> v0.11.0): - release v0.11.0 (#214) ([multiformats/go-multiaddr#214](https://github.com/multiformats/go-multiaddr/pull/214)) - update golang.org/x/exp slice comparison to match standard library version (#210) ([multiformats/go-multiaddr#210](https://github.com/multiformats/go-multiaddr/pull/210)) - github.com/warpfork/go-testmark (v0.11.0 -> v0.12.1): - suite: allow disabling file parallelism. - Suite feature ([warpfork/go-testmark#16](https://github.com/warpfork/go-testmark/pull/16)) - fix unchecked error in a test - accept a simplification suggestion from linters - Trailing whitespace error ([warpfork/go-testmark#15](https://github.com/warpfork/go-testmark/pull/15)) - FS implementation (#11) ([warpfork/go-testmark#11](https://github.com/warpfork/go-testmark/pull/11)) - Add a readme for the testexec extension and its conventions. ([warpfork/go-testmark#14](https://github.com/warpfork/go-testmark/pull/14)) - Strict mode for testexec structure ([warpfork/go-testmark#12](https://github.com/warpfork/go-testmark/pull/12))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Rod Vagg | 48 | +3578/-1789 | 110 | | Henrique Dias | 24 | +3173/-1128 | 104 | | Jorropo | 51 | +1721/-1297 | 252 | | Marco Munizaga | 6 | +1989/-505 | 39 | | Kay | 3 | +487/-474 | 163 | | hannahhoward | 8 | +626/-136 | 23 | | Calvin Behling | 6 | +496/-259 | 20 | | Eric Myhre | 9 | +610/-121 | 16 | | Adin Schmahmann | 17 | +659/-45 | 35 | | Marten Seemann | 17 | +218/-477 | 119 | | Sukun | 11 | +481/-174 | 29 | | CJB | 1 | +639/-2 | 5 | | Hector Sanjuan | 10 | +450/-127 | 21 | | Wondertan | 2 | +203/-127 | 8 | | Marcin Rataj | 11 | +148/-86 | 18 | | Andrew Gillis | 2 | +163/-14 | 5 | | P. Reis | 3 | +120/-4 | 4 | | Will Scott | 4 | +107/-12 | 6 | | Amir Mohammad Fakhimi | 1 | +97/-2 | 5 | | Ed Schouten | 1 | +55/-7 | 2 | | Icarus9913 | 1 | +30/-30 | 18 | | Dirk McCormick | 1 | +3/-42 | 1 | | Raúl Kripalani | 1 | +20/-18 | 4 | | Michael Muré | 1 | +26/-7 | 5 | | Prem Chaitanya Prathi | 1 | +28/-1 | 2 | | ShengTao | 1 | +13/-14 | 4 | | Prithvi Shahi | 3 | +14/-13 | 3 | | web3-bot | 5 | +12/-10 | 9 | | Alejandro Criado-Pérez | 1 | +11/-11 | 6 | | Steven Allen | 2 | +6/-10 | 2 | | Andrej Manduch | 1 | +5/-5 | 3 | | Russell Dempsey | 2 | +4/-2 | 2 | | Johannes Maria Frank | 1 | +4/-1 | 1 | | downIoads | 1 | +2/-2 | 1 | | Will | 2 | +2/-2 | 2 | | Marin Kirkov | 1 | +2/-2 | 2 | | Gus Eggert | 1 | +2/-2 | 1 | | Bernhard M. Wiedemann | 1 | +4/-0 | 1 | | Dennis Trautwein | 1 | +1/-2 | 1 | | “GheisMohammadi” | 1 | +1/-1 | 1 | | cce | 1 | +1/-1 | 1 | | Joao Andrade | 1 | +1/-1 | 1 | | guillaumemichel | 1 | +1/-0 | 1 | | Santiago Botto | 1 | +0/-1 | 1 | ================================================ FILE: docs/changelogs/v0.24.md ================================================ # Kubo changelog v0.24 - [v0.24.0](#v0240) ## v0.24.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [Support for content blocking](#support-for-content-blocking) - [Gateway: the root of the CARs are no longer meaningful](#gateway-the-root-of-the-cars-are-no-longer-meaningful) - [IPNS: improved publishing defaults](#ipns-improved-publishing-defaults) - [IPNS: record TTL is used for caching](#ipns-record-ttl-is-used-for-caching) - [Experimental Transport: WebRTC Direct](#experimental-transport-webrtc-direct) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### Support for content blocking This Kubo release ships with built-in content-blocking subsystem [announced earlier this year](https://blog.ipfs.tech/2023-content-blocking-for-the-ipfs-stack/). Content blocking is an opt-in decision made by the operator of `ipfs daemon`. The official build does not ship with any denylists. Learn more at [`/docs/content-blocking.md`](https://github.com/ipfs/kubo/blob/master/docs/content-blocking.md) #### Gateway: the root of the CARs are no longer meaningful When requesting a CAR from the gateway, the root of the CAR might no longer be meaningful. By default, the CAR root will be the last resolvable segment of the path. However, in situations where the path cannot be resolved, such as when the path does not exist, a CAR will be sent with a root of `bafkqaaa` (empty CID). This CAR will contain all blocks necessary to validate that the path does not exist. #### IPNS: improved publishing defaults This release changes the default values used when publishing IPNS record via `ipfs name publish` command: - Default `--lifetime` increased from `24h` to `48h` to take full advantage of the increased expiration window of Amino DHT ([go-libp2p-kad-dht#793](https://github.com/libp2p/go-libp2p-kad-dht/pull/793)) - Default `--ttl` increased from `1m` to `1h` to improve website caching and follow saner defaults present in similar systems like DNS ([specs#371](https://github.com/ipfs/specs/pull/371)) This change only impacts the implicit defaults, when mentioned parameters are omitted during publishing. Users are free to override the default if different value makes more sense for their use case. #### IPNS: record TTL is used for caching In this release, we've made significant improvements to IPNS caching. Previously, the TTL value in IPNS records was not utilized, and the `boxo/namesys` library maintained a static one-minute resolution cache. With this update, IPNS publishers gain more control over how long a valid IPNS record remains cached before checking an upstream routing system, such as Amino DHT, for updates. The TTL value in the IPNS record now serves as a hint for: - `boxo/namesys`: the internal cache, determining how long the IPNS resolution result is cached before asking upstream routing systems for updates. - `boxo/gateway`: the `Cache-Control` HTTP header in responses to requests made for `/ipns/name` content paths. These changes make it easier for rarely updated IPNS-hosted websites to be cached more efficiently and load faster in browser contexts. #### Experimental Transport: WebRTC Direct This Kubo release includes the initial work towards WebRTC Direct introduced in [`go-libp2p`](https://github.com/libp2p/go-libp2p/releases/tag/v0.32.0) v0.32: > [WebRTC Direct](https://github.com/libp2p/specs/blob/master/webrtc/webrtc-direct.md) > allows browser nodes to connect to go-libp2p nodes directly, > without any configuration (e.g. TLS certificates) needed on the go-libp2p > side. This is useful for browser nodes that aren’t able to use > [WebTransport](https://web.archive.org/web/20260107053250/https://blog.libp2p.io/2022-12-19-libp2p-webtransport/). The `/webrtc-direct` transport is disabled by default in Kubo 0.24, and not ready for production use yet, but we plan to enable it in a future release. See [`Swarm.Transports.Network.WebRTCDirect`](https://github.com/ipfs/kubo/blob/master/docs/config.md#swarmtransportsnetworkwebrtcdirect) to learn how to enable it manually, and what current limitations are. ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: update version - fix: allow event emitting to happen in parallel with getting the query channel - fixes to routing put command (#10205) ([ipfs/kubo#10205](https://github.com/ipfs/kubo/pull/10205)) - docs: fix accelerated-dht-client - docs/config: remove extra commas in PublicGateways example entries - chore: update version - docs: make it clear Web RTC Direct is experimental - feat: add WebRTC Direct support - docs: update EARLY_TESTERS.md (#10194) ([ipfs/kubo#10194](https://github.com/ipfs/kubo/pull/10194)) - Release: v0.24.0-1 ([ipfs/kubo#10190](https://github.com/ipfs/kubo/pull/10190)) - github.com/ipfs/boxo (v0.13.1 -> v0.15.0): - Release v0.15.0 ([ipfs/boxo#505](https://github.com/ipfs/boxo/pull/505)) - Release v0.14.0 ([ipfs/boxo#500](https://github.com/ipfs/boxo/pull/500)) - github.com/ipfs/go-block-format (v0.1.2 -> v0.2.0): - v0.2.0 bump - github.com/ipfs/go-graphsync (v0.15.1 -> v0.16.0): - chore: release 0.16.0 - chore: bump go-libp2p to 0.32.0 - github.com/ipfs/go-ipld-format (v0.5.0 -> v0.6.0): - v0.6.0 bump - chore: update deps - fix: stop using the deprecated io/ioutil package - github.com/libp2p/go-libp2p (v0.31.0 -> v0.32.1): - release v0.32.1 (#2637) ([libp2p/go-libp2p#2637](https://github.com/libp2p/go-libp2p/pull/2637)) - swarm: fix timer Leak in the dial loop (#2636) ([libp2p/go-libp2p#2636](https://github.com/libp2p/go-libp2p/pull/2636)) - release v0.32.0 (#2625) ([libp2p/go-libp2p#2625](https://github.com/libp2p/go-libp2p/pull/2625)) - chore: update js-libp2p examples repo (#2624) ([libp2p/go-libp2p#2624](https://github.com/libp2p/go-libp2p/pull/2624)) - identify: don't filter dns addresses based on remote addr type (#2553) ([libp2p/go-libp2p#2553](https://github.com/libp2p/go-libp2p/pull/2553)) - webrtc: fix race in TestRemoveConnByUfrag (#2620) ([libp2p/go-libp2p#2620](https://github.com/libp2p/go-libp2p/pull/2620)) - swarm: fix recursive resolving of DNS multiaddrs (#2564) ([libp2p/go-libp2p#2564](https://github.com/libp2p/go-libp2p/pull/2564)) - ci: migrate to renamed interop test action (#2617) ([libp2p/go-libp2p#2617](https://github.com/libp2p/go-libp2p/pull/2617)) - quic: update quic-go to v0.39.1, set a static resumption token generator key (#2572) ([libp2p/go-libp2p#2572](https://github.com/libp2p/go-libp2p/pull/2572)) - test/basichost: fix flaky test due to rcmgr (#2613) ([libp2p/go-libp2p#2613](https://github.com/libp2p/go-libp2p/pull/2613)) - swarm: use typed atomics (#2612) ([libp2p/go-libp2p#2612](https://github.com/libp2p/go-libp2p/pull/2612)) - swarm: cleanup stream handler goroutine (#2610) ([libp2p/go-libp2p#2610](https://github.com/libp2p/go-libp2p/pull/2610)) - circuitv2: don't check ASN for private addrs (#2611) ([libp2p/go-libp2p#2611](https://github.com/libp2p/go-libp2p/pull/2611)) - swarm: use happy eyeballs ranking for TCP dials (#2573) ([libp2p/go-libp2p#2573](https://github.com/libp2p/go-libp2p/pull/2573)) - webrtc: fix race in TestMuxedConnection (#2607) ([libp2p/go-libp2p#2607](https://github.com/libp2p/go-libp2p/pull/2607)) - tcp: fix build on riscv64 (#2590) ([libp2p/go-libp2p#2590](https://github.com/libp2p/go-libp2p/pull/2590)) - Fix missing deprecation tag (#2605) ([libp2p/go-libp2p#2605](https://github.com/libp2p/go-libp2p/pull/2605)) - swarm: wait for transient connections to upgrade for NewStream (#2542) ([libp2p/go-libp2p#2542](https://github.com/libp2p/go-libp2p/pull/2542)) - docs: fix typos (#2604) ([libp2p/go-libp2p#2604](https://github.com/libp2p/go-libp2p/pull/2604)) - webrtc: correctly report incoming packet address on muxed connection (#2586) ([libp2p/go-libp2p#2586](https://github.com/libp2p/go-libp2p/pull/2586)) - swarm: add loopback to low timeout filter (#2595) ([libp2p/go-libp2p#2595](https://github.com/libp2p/go-libp2p/pull/2595)) - Fix typos in comments and a test failure message (#2600) ([libp2p/go-libp2p#2600](https://github.com/libp2p/go-libp2p/pull/2600)) - libp2phttp: don't strip `/` suffix when mounting handler (#2552) ([libp2p/go-libp2p#2552](https://github.com/libp2p/go-libp2p/pull/2552)) - interop: fix redis env var (#2585) ([libp2p/go-libp2p#2585](https://github.com/libp2p/go-libp2p/pull/2585)) - quicreuse: remove QUIC metrics tracer (#2582) ([libp2p/go-libp2p#2582](https://github.com/libp2p/go-libp2p/pull/2582)) - config: warn if connmgr limits conflict with rcmgr (#2527) ([libp2p/go-libp2p#2527](https://github.com/libp2p/go-libp2p/pull/2527)) - update gomock to v0.3.0 (#2581) ([libp2p/go-libp2p#2581](https://github.com/libp2p/go-libp2p/pull/2581)) - webrtc: fix deadlock on connection close (#2580) ([libp2p/go-libp2p#2580](https://github.com/libp2p/go-libp2p/pull/2580)) - webrtc: put buffer back to pool (#2574) ([libp2p/go-libp2p#2574](https://github.com/libp2p/go-libp2p/pull/2574)) - webrtc: fail Write early if deadline has exceeded before the call (#2578) ([libp2p/go-libp2p#2578](https://github.com/libp2p/go-libp2p/pull/2578)) - swarm: fix DialPeer behaviour for transient connections (#2547) ([libp2p/go-libp2p#2547](https://github.com/libp2p/go-libp2p/pull/2547)) - websocket: don't resolve /dnsaddr addresses (#2571) ([libp2p/go-libp2p#2571](https://github.com/libp2p/go-libp2p/pull/2571)) - core/peer: remove deprecated ID.Pretty method (#2565) ([libp2p/go-libp2p#2565](https://github.com/libp2p/go-libp2p/pull/2565)) - core/peer: remove deprecated Encode function (#2566) ([libp2p/go-libp2p#2566](https://github.com/libp2p/go-libp2p/pull/2566)) - mock: use go.uber.org/mock (#2540) ([libp2p/go-libp2p#2540](https://github.com/libp2p/go-libp2p/pull/2540)) - add WebRTC Direct transport implementation (#2337) ([libp2p/go-libp2p#2337](https://github.com/libp2p/go-libp2p/pull/2337)) - upgrader: drop support for multistream simultaneous open (#2557) ([libp2p/go-libp2p#2557](https://github.com/libp2p/go-libp2p/pull/2557)) - examples: stop using deprecated peer.ID.Pretty (#2563) ([libp2p/go-libp2p#2563](https://github.com/libp2p/go-libp2p/pull/2563)) - swarm: don't dial unspecified addresses (#2560) ([libp2p/go-libp2p#2560](https://github.com/libp2p/go-libp2p/pull/2560)) - basichost: handle the SetProtocol error in NewStream (#2555) ([libp2p/go-libp2p#2555](https://github.com/libp2p/go-libp2p/pull/2555)) - libp2phttp: don't initialise ServeMux if not nil (#2548) ([libp2p/go-libp2p#2548](https://github.com/libp2p/go-libp2p/pull/2548)) - github.com/libp2p/go-libp2p-pubsub (v0.9.3 -> v0.10.0): - chore: update go-libp2p to v0.32 (#548) ([libp2p/go-libp2p-pubsub#548](https://github.com/libp2p/go-libp2p-pubsub/pull/548)) - remove usage of deprecated peerid.Pretty method (#542) ([libp2p/go-libp2p-pubsub#542](https://github.com/libp2p/go-libp2p-pubsub/pull/542)) - Revert "fix: topicscore params can't be set for dynamically subscribed topic (#540)" (#541) ([libp2p/go-libp2p-pubsub#541](https://github.com/libp2p/go-libp2p-pubsub/pull/541)) - fix: topicscore params can't be set for dynamically subscribed topic (#540) ([libp2p/go-libp2p-pubsub#540](https://github.com/libp2p/go-libp2p-pubsub/pull/540)) - github.com/multiformats/go-multiaddr (v0.11.0 -> v0.12.0): - release v0.12.0 (#223) ([multiformats/go-multiaddr#223](https://github.com/multiformats/go-multiaddr/pull/223)) - net: consider /dns/localhost as private address (#221) ([multiformats/go-multiaddr#221](https://github.com/multiformats/go-multiaddr/pull/221)) - net: consider dns addresses as public (#220) ([multiformats/go-multiaddr#220](https://github.com/multiformats/go-multiaddr/pull/220)) - github.com/multiformats/go-multistream (v0.4.1 -> v0.5.0): - remove support for the simultaneous open extension (#107) ([multiformats/go-multistream#107](https://github.com/multiformats/go-multistream/pull/107))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Henrique Dias | 27 | +4505/-3853 | 244 | | Marten Seemann | 18 | +4260/-1173 | 101 | | Sukun | 24 | +1499/-340 | 79 | | Andrew Gillis | 4 | +169/-1025 | 16 | | Adin Schmahmann | 4 | +788/-184 | 19 | | Hector Sanjuan | 6 | +619/-72 | 19 | | Steven Allen | 11 | +489/-101 | 14 | | Jorropo | 10 | +221/-192 | 28 | | Łukasz Magiera | 2 | +306/-9 | 3 | | Lucas Molas | 1 | +183/-52 | 2 | | Marcin Rataj | 5 | +160/-25 | 6 | | piersy | 1 | +57/-0 | 6 | | Raúl Kripalani | 1 | +25/-25 | 2 | | Alvin Reyes | 1 | +34/-14 | 1 | | Dennis Trautwein | 1 | +1/-40 | 2 | | Icarus9913 | 1 | +14/-14 | 10 | | Takashi Matsuda | 2 | +18/-1 | 3 | | gammazero | 4 | +8/-5 | 7 | | xiaolou86 | 1 | +6/-6 | 5 | | Daniel Martí | 1 | +9/-2 | 1 | | Rod Vagg | 3 | +5/-5 | 4 | | Andrej Manduch | 1 | +5/-5 | 3 | | vuittont60 | 1 | +4/-4 | 3 | | vyzo | 1 | +5/-1 | 1 | | tkzktk | 1 | +3/-3 | 3 | | tk | 1 | +3/-3 | 2 | | Prem Chaitanya Prathi | 1 | +1/-5 | 1 | | Kay | 2 | +2/-3 | 2 | | Thomas Eizinger | 1 | +2/-2 | 1 | | Steve Loeppky | 1 | +2/-2 | 1 | | Jonas Keunecke | 1 | +2/-2 | 1 | | Alejandro Criado-Pérez | 1 | +1/-1 | 1 | | web3-bot | 1 | +1/-0 | 1 | | Eric | 1 | +1/-0 | 1 | ================================================ FILE: docs/changelogs/v0.25.md ================================================ # Kubo changelog v0.25 - [v0.25.0](#v0250) ## v0.25.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [WebUI: Updated Peers View](#webui-updated-peers-view) - [RPC `API.Authorizations`](#rpc-apiauthorizations) - [MPLEX Removal](#mplex-removal) - [Graphsync Experiment Removal](#graphsync-experiment-removal) - [Commands `ipfs key sign` and `ipfs key verify`](#commands-ipfs-key-sign-and-ipfs-key-verify) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### WebUI: Updated Peers View WebUI [v4.2.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.2.0) shipped with updated [ipfs-geoip](https://www.npmjs.com/package/ipfs-geoip) dataset and [ability to filter the peers table](https://github.com/ipfs/ipfs-webui/pull/2181). #### RPC `API.Authorizations` Kubo RPC API now supports optional HTTP Authorization. Granular control over user access to the RPC can be defined in the [`API.Authorizations`](https://github.com/ipfs/kubo/blob/master/docs/config.md#apiauthorizations) map in the configuration file, allowing different users or apps to have unique access secrets and allowed paths. This feature is opt-in. By default, no authorization is set up. For configuration instructions, refer to the [documentation](https://github.com/ipfs/kubo/blob/master/docs/config.md#apiauthorizations). #### MPLEX Removal After deprecating and removing mplex support by default in [v0.23.0](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.23.md#mplex-deprecation). We now fully removed it. If you still need mplex support to talk with other pieces of software, please try updating them, and if they don't support yamux or QUIC [talk to us about it](https://github.com/ipfs/kubo/issues/new/choose). Mplex is unreliable by design, it will drop data and generate errors when sending data *too fast*, yamux and QUIC support backpressure, that means if we send data faster than the remote machine can process it, we slows down to match the remote's speed. #### Graphsync Experiment Removal Currently the Graphsync server is to our knowledge not used due to lack of compatible software. And we are left to have to maintain the go-graphsync implementation when trying to update Kubo because some dependency changed and it fails to build anymore. For more information see https://github.com/ipfs/kubo/pull/9747. ##### Commands `ipfs key sign` and `ipfs key verify` This allows the Kubo node to sign arbitrary bytes to prove ownership of a PeerID or an IPNS Name. To avoid signature reuse, the signed payload is always prefixed with `libp2p-key signed message:`. These commands are also both available through the RPC client and implemented in `client/rpc`. For more information see https://github.com/ipfs/kubo/issues/10230. ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: update version - fix: allow daemon to start correctly if the API is null (#10062) ([ipfs/kubo#10062](https://github.com/ipfs/kubo/pull/10062)) - chore: update version - feat: ipfs key sign|verify (#10235) ([ipfs/kubo#10235](https://github.com/ipfs/kubo/pull/10235)) - docs(cli): fix spelling - feat: webui v4.2.0 (#10241) ([ipfs/kubo#10241](https://github.com/ipfs/kubo/pull/10241)) - Migrate coreiface ([ipfs/kubo#10237](https://github.com/ipfs/kubo/pull/10237)) - docs: clarify WebRTCDirect cannot reuse the same port as QUIC - libp2p: remove mplex - graphsync: remove support for the server - docs: move kubo-specific docs (#10226) ([ipfs/kubo#10226](https://github.com/ipfs/kubo/pull/10226)) - feat(rpc): Opt-in HTTP RPC API Authorization (#10218) ([ipfs/kubo#10218](https://github.com/ipfs/kubo/pull/10218)) - docs: clarify ipfs id agent version - fix: regression in 'ipfs dns' - docs(changelog): clarify webrtc in v0.24 - chore: create next changelog - Merge Release: v0.24.0 ([ipfs/kubo#10209](https://github.com/ipfs/kubo/pull/10209)) - fix: allow event emitting to happen in parallel with getting the query channel - fixes to routing put command (#10205) ([ipfs/kubo#10205](https://github.com/ipfs/kubo/pull/10205)) - docs: fix accelerated-dht-client - docs/config: remove extra commas in PublicGateways example entries - docs: make it clear Web RTC Direct is experimental - feat: add WebRTC Direct support - docs: update EARLY_TESTERS.md (#10194) ([ipfs/kubo#10194](https://github.com/ipfs/kubo/pull/10194)) - Update Version: v0.24 ([ipfs/kubo#10191](https://github.com/ipfs/kubo/pull/10191)) - github.com/ipfs/boxo (v0.15.0 -> v0.16.0): - Release 0.16.0 ([ipfs/boxo#518](https://github.com/ipfs/boxo/pull/518)) - github.com/libp2p/go-libp2p (v0.32.1 -> v0.32.2): - release v0.32.2
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Łukasz Magiera | 149 | +7833/-2505 | 375 | | Henrique Dias | 26 | +2498/-7535 | 210 | | Steven Allen | 48 | +497/-373 | 129 | | Jorropo | 9 | +247/-604 | 49 | | Michael Muré | 6 | +306/-79 | 14 | | Adin Schmahmann | 3 | +275/-8 | 5 | | Lucas Molas | 1 | +181/-56 | 2 | | Laurent Senta | 1 | +109/-24 | 7 | | Lars Gierth | 6 | +82/-18 | 8 | | Petar Maymounkov | 1 | +66/-32 | 3 | | web3-bot | 1 | +47/-42 | 17 | | Marcin Rataj | 6 | +57/-23 | 8 | | Kevin Atkinson | 5 | +31/-31 | 17 | | Marten Seemann | 3 | +27/-28 | 16 | | Hector Sanjuan | 3 | +28/-14 | 10 | | Overbool | 2 | +36/-3 | 3 | | Raúl Kripalani | 1 | +11/-12 | 4 | | hannahhoward | 2 | +11/-7 | 6 | | Jeromy Johnson | 5 | +9/-9 | 5 | | ForrestWeston | 1 | +14/-1 | 1 | | Russell Dempsey | 1 | +10/-2 | 2 | | Will Scott | 1 | +8/-1 | 1 | | Jeromy | 2 | +4/-4 | 2 | | sukun | 1 | +2/-2 | 1 | | Steve Loeppky | 1 | +2/-2 | 1 | | Jonas Keunecke | 1 | +2/-2 | 1 | | Edgar Lee | 1 | +3/-1 | 1 | | Dreamacro | 1 | +2/-2 | 2 | | godcong | 1 | +1/-1 | 1 | | Cole Brown | 1 | +1/-1 | 1 | ================================================ FILE: docs/changelogs/v0.26.md ================================================ # Kubo changelog v0.26 - [v0.26.0](#v0260) ## v0.26.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [Several deprecated commands have been removed](#several-deprecated-commands-have-been-removed) - [Support optional pin names](#support-optional-pin-names) - [`jaeger` trace exporter has been removed](#jaeger-trace-exporter-has-been-removed) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### Kubo binary imports For users of [Kubo preloaded plugins](https://github.com/ipfs/kubo/blob/master/docs/plugins.md#preloaded-plugins) there is now a way to create a kubo instance with your plugins by depending on the `cmd/ipfs/kubo` package rather than rebuilding kubo with the included plugins. See the [customization docs](https://github.com/ipfs/kubo/blob/master/docs/customizing.md) for more information. #### Several deprecated commands have been removed Several deprecated commands have been removed: - `ipfs urlstore` deprecated in [April 2019, Kubo 0.4.21](https://github.com/ipfs/kubo/commit/8beaee63b3fa634c59b85179286ad3873921a535), use `ipfs add -q --nocopy --cid-version=1 {url}` instead. - `ipfs repo fsck` deprecated in [July 2019, Kubo 0.5.0](https://github.com/ipfs/kubo/commit/288a83ce7dcbf4a2498e06e4a95245bbb5e30f45) - `ipfs file` (and `ipfs file ls`) deprecated in [November 2020, Kubo 0.8.0](https://github.com/ipfs/kubo/commit/ec64dc5c396e7114590e15909384fabce0035482), use `ipfs ls` and `ipfs files ls` instead. - `ipfs dns` deprecated in [April 2022, Kubo 0.13](https://github.com/ipfs/kubo/commit/76ae33a9f3f9abd166d1f6f23d6a8a0511510e3c), use `ipfs resolve /ipns/{name}` instead. - `ipfs tar` deprecated [April 2022, Kubo 0.13](https://github.com/ipfs/kubo/pull/8849) #### Support optional pin names You can now add a name to a pin when pinning a CID. To do so, use `ipfs pin add --name "Some Name" bafy...`. You can list your pins, including their names, with `ipfs pin ls --names`. #### `jaeger` trace exporter has been removed `jaeger` exporter has been removed from upstream, you should use `otlp` exporter instead. See the [boxo tracing docs](https://github.com/ipfs/boxo/blob/a391d02102875ee7075a692076154bec1fa871f3/docs/tracing.md) for an example. ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: update version - chore: update version - feat(pinning): allow for overwriting pin name - chore: update otlp - Revert "build,docker: add support for riscv64" - feat: support optional pin names (#10261) ([ipfs/kubo#10261](https://github.com/ipfs/kubo/pull/10261)) - build,docker: add support for riscv64 - feat(cmd/ipfs): Make it possible to depend on cmd/ipfs/kubo for easier preloaded plugin management ([ipfs/kubo#10219](https://github.com/ipfs/kubo/pull/10219)) - docs: fix broken link in HTTP RPC client doc (#10267) ([ipfs/kubo#10267](https://github.com/ipfs/kubo/pull/10267)) - Merge Release: v0.25.0 [skip changelog] ([ipfs/kubo#10260](https://github.com/ipfs/kubo/pull/10260)) - docs: add detail to NOpfs instructions in content-blocking.md - commands: remove several deprecated commands - fix: allow daemon to start correctly if the API is null (#10062) ([ipfs/kubo#10062](https://github.com/ipfs/kubo/pull/10062)) - chore: update version - github.com/ipfs/boxo (v0.16.0 -> v0.17.0): - Release v0.17.0 ([ipfs/boxo#542](https://github.com/ipfs/boxo/pull/542)) - github.com/ipfs/go-ipld-cbor (v0.0.6 -> v0.1.0): - v0.1.0 bump - chore: add or force update version.json - allow configuration of ipldStores default hash function ([ipfs/go-ipld-cbor#86](https://github.com/ipfs/go-ipld-cbor/pull/86)) - sync: update CI config files (#85) ([ipfs/go-ipld-cbor#85](https://github.com/ipfs/go-ipld-cbor/pull/85)) - github.com/ipfs/go-unixfsnode (v1.8.1 -> v1.9.0): - v1.9.0 bump - feat: expose ToDirEntryFrom to allow sub-dag representation - feat: new UnixFS{File,Directory} with options pattern - feat: testutil generator enhancements - github.com/ipld/go-car/v2 (v2.10.2-0.20230622090957-499d0c909d33 -> v2.13.1): - fix: BlockMetadata#Offset should be for section, not block data - fix: add closed check, expose storage.ErrClosed - fix: switch constructor args to match storage.New*, make roots plural - feat: add DeferredCarWriter - feat: fix BlockReader#SkipNext & add SourceOffset property - v0.6.2 ([ipld/go-car#464](https://github.com/ipld/go-car/pull/464)) - fix: opt-in way to allow empty list of roots in CAR headers ([ipld/go-car#461](https://github.com/ipld/go-car/pull/461)) - github.com/libp2p/go-libp2p-asn-util (v0.3.0 -> v0.4.1): - chore: release v0.4.1 - fix: add Init method on backward compat - chore: release v0.4.0 - rewrite representation to a sorted binary list and embed it - docs: fix incorrect markdown === in README - ci: run go generate on CI (#27) ([libp2p/go-libp2p-asn-util#27](https://github.com/libp2p/go-libp2p-asn-util/pull/27)) - github.com/multiformats/go-multiaddr (v0.12.0 -> v0.12.1): - v0.12.1 bump - manet: reduce allocations in resolve unspecified address - github.com/whyrusleeping/cbor-gen (v0.0.0-20230126041949-52956bd4c9aa -> v0.0.0-20240109153615-66e95c3e8a87): - Add a feature to preserve nil slices (#88) ([whyrusleeping/cbor-gen#88](https://github.com/whyrusleeping/cbor-gen/pull/88)) - some cleanup for easier reading ([whyrusleeping/cbor-gen#89](https://github.com/whyrusleeping/cbor-gen/pull/89)) - Support gen for map with value type `string` (#83) ([whyrusleeping/cbor-gen#83](https://github.com/whyrusleeping/cbor-gen/pull/83)) - feat: add support for pointers to CIDs in slices (#86) ([whyrusleeping/cbor-gen#86](https://github.com/whyrusleeping/cbor-gen/pull/86)) - optimize anything using WriteString ([whyrusleeping/cbor-gen#85](https://github.com/whyrusleeping/cbor-gen/pull/85)) - Implement *bool support and support omitempty for slices ([whyrusleeping/cbor-gen#81](https://github.com/whyrusleeping/cbor-gen/pull/81))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Henrique Dias | 11 | +493/-1184 | 48 | | Łukasz Magiera | 3 | +610/-582 | 16 | | Rod Vagg | 11 | +1030/-151 | 18 | | whyrusleeping | 6 | +553/-388 | 14 | | Jorropo | 13 | +561/-348 | 84 | | Jeromy Johnson | 1 | +771/-48 | 6 | | Steven Allen | 2 | +264/-135 | 4 | | Forrest | 1 | +214/-0 | 5 | | Marcin Rataj | 1 | +89/-24 | 2 | | sukun | 1 | +31/-11 | 5 | | Will Scott | 3 | +25/-10 | 3 | | Adin Schmahmann | 3 | +21/-5 | 3 | | web3-bot | 2 | +8/-8 | 3 | | Marten Seemann | 1 | +13/-1 | 1 | | Bumblefudge | 1 | +5/-2 | 1 | | Will | 1 | +1/-1 | 1 | | Nicholas Ericksen | 1 | +1/-1 | 1 | | 0xbasar | 1 | +1/-1 | 1 | ================================================ FILE: docs/changelogs/v0.27.md ================================================ # Kubo changelog v0.27 - [v0.27.0](#v0270) ## v0.27.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [Gateway: support for `/api/v0` is deprecated](#gateway-support-for-apiv0-is-deprecated) - [IPNS resolver cache's TTL can now be configured](#ipns-resolver-caches-ttl-can-now-be-configured) - [RPC client: deprecated DHT API, added Routing API](#rpc-client-deprecated-dht-api-added-routing-api) - [Deprecated DHT commands removed from `/api/v0/dht`](#deprecated-dht-commands-removed-from-apiv0dht) - [Repository migrations are now trustless](#repository-migrations-are-now-trustless) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### Gateway: support for `/api/v0` is deprecated Support for exposing the legacy subset of Kubo RPC via the Gateway port is deprecated and should not be used. It will be removed in the next version. You can read more in . If you have a legacy software that relies on this behavior, and want to expose parts of `/api/v0` next to `/ipfs`, use reverse-proxy in front of Kubo to mount both Gateway and RPC on the same port. NOTE: exposing RPC to the internet comes with security risk: make sure to specify access control via [API.Authorizations](https://github.com/ipfs/kubo/blob/master/docs/config.md#apiauthorizations). #### IPNS resolver cache's TTL can now be configured You can now configure the upper-bound of a cached IPNS entry's Time-To-Live via [`Ipns.MaxCacheTTL`](https://github.com/ipfs/kubo/blob/master/docs/config.md#ipnsmaxcachettl). #### RPC client: deprecated DHT API, added Routing API The RPC client for GO (`kubo/client/rpc`) now includes a Routing API to match the available commands in `/api/v0/routing`. In addition, the DHT API has been marked as deprecated. In the next version, all DHT deprecated methods will be removed from the Go RPC client. #### Deprecated DHT commands removed from `/api/v0/dht` All the DHT commands that were deprecated for over a year were finally removed from `/api/v0/dht`. Users should switch to modern `/api/v0/routing` which works with [both Amino DHT and Delegated Routers](https://github.com/ipfs/kubo/blob/master/docs/config.md#routing). #### Repository migrations are now trustless Kubo now only uses [trustless requests](https://specs.ipfs.tech/http-gateways/trustless-gateway/) (e.g., CAR files) when downloading repository migrations via HTTP. This further strengthens Kubo by not delegating trust to public gateways. The migration binaries are locally verified before being executed. ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: update version - chore: update version - test: cleanup content blocking tests (#10360) ([ipfs/kubo#10360](https://github.com/ipfs/kubo/pull/10360)) - docs: improve release issue template - chore: update version - repo/fsrepo/migrations: verified HTTP migrations (#10324) ([ipfs/kubo#10324](https://github.com/ipfs/kubo/pull/10324)) - chore: fix link - docs: clarify Gateway.ExposeRoutingAPI (#10337) ([ipfs/kubo#10337](https://github.com/ipfs/kubo/pull/10337)) - commands/add: return an error when using --only-hash and --to-files - docs(config): mention routing v1 spec - core/commands: remove 'ipfs dht' commands, except 'query' (#10328) ([ipfs/kubo#10328](https://github.com/ipfs/kubo/pull/10328)) - core: deprecate CoreAPI.Dht, introduce CoreAPI.Routing - refactor: superfluous namespace test redirects (#10322) ([ipfs/kubo#10322](https://github.com/ipfs/kubo/pull/10322)) - feat: add Ipns.MaxCacheTTL - fix(gw): negative entity-bytes beyond file size (#10320) ([ipfs/kubo#10320](https://github.com/ipfs/kubo/pull/10320)) - core/corehttp: wrap gateway with headers, deprecate gateway /api/v0 - docs: add changelog link to release issue template - docs: remove whizzzkid - chore: create next changelog - Merge Release: v0.26.0 [skip changelog] ([ipfs/kubo#10313](https://github.com/ipfs/kubo/pull/10313)) - config: remove all options that are marked as REMOVED - chore: remove Gateway.APICommands - docs(cli): name inspect --verify (#10308) ([ipfs/kubo#10308](https://github.com/ipfs/kubo/pull/10308)) - docs: improve release issue template (#10305) ([ipfs/kubo#10305](https://github.com/ipfs/kubo/pull/10305)) - core/corehttp: wrap hostname option with otelhttp - fix: profiling tests - profile: add trace - docs(config): clarify ReproviderStrategy roots - chore: update version - docs: in RELEASE_ISSUE_TEMPLATE ask releaser to ensure we are using the latest go release on the major branch - github.com/ipfs/boxo (v0.17.0 -> v0.18.0): - Release v0.18.0 ([ipfs/boxo#581](https://github.com/ipfs/boxo/pull/581)) - github.com/libp2p/go-libp2p (v0.32.2 -> v0.33.0): - release v0.33.0 (#2715) ([libp2p/go-libp2p#2715](https://github.com/libp2p/go-libp2p/pull/2715)) - chore: update deps for v0.33 (#2713) ([libp2p/go-libp2p#2713](https://github.com/libp2p/go-libp2p/pull/2713)) - webrtc: wait for FIN_ACK before closing data channels (#2615) ([libp2p/go-libp2p#2615](https://github.com/libp2p/go-libp2p/pull/2615)) - quic: upgrade quic-go to v0.41.0 (#2710) ([libp2p/go-libp2p#2710](https://github.com/libp2p/go-libp2p/pull/2710)) - chore: remove unused GenerateEKeyPair function (#2711) ([libp2p/go-libp2p#2711](https://github.com/libp2p/go-libp2p/pull/2711)) - chore: drop support for go1.20 (#2708) ([libp2p/go-libp2p#2708](https://github.com/libp2p/go-libp2p/pull/2708)) - chore: testify fix got, expected transpositions (#2666) ([libp2p/go-libp2p#2666](https://github.com/libp2p/go-libp2p/pull/2666)) - docs: fix broken link in README - chore: fix typos (#2694) ([libp2p/go-libp2p#2694](https://github.com/libp2p/go-libp2p/pull/2694)) - libp2phttp: fix flaky ExampleHost_listenOnHTTPTransportAndStreams (#2697) ([libp2p/go-libp2p#2697](https://github.com/libp2p/go-libp2p/pull/2697)) - chore(p2p/host): fix typos (#2683) ([libp2p/go-libp2p#2683](https://github.com/libp2p/go-libp2p/pull/2683)) - chore: fix typos (#2689) ([libp2p/go-libp2p#2689](https://github.com/libp2p/go-libp2p/pull/2689)) - defaults: do TLS by default for encryption (#2650) ([libp2p/go-libp2p#2650](https://github.com/libp2p/go-libp2p/pull/2650)) - webrtc: fix flaky TestMaxInFlightRequests (#2682) ([libp2p/go-libp2p#2682](https://github.com/libp2p/go-libp2p/pull/2682)) - chore: remove unnecessary conversions (#2680) ([libp2p/go-libp2p#2680](https://github.com/libp2p/go-libp2p/pull/2680)) - chore: update chat-with-mdns example readme (#2678) ([libp2p/go-libp2p#2678](https://github.com/libp2p/go-libp2p/pull/2678)) - examples: call NewStream from only one side (#2677) ([libp2p/go-libp2p#2677](https://github.com/libp2p/go-libp2p/pull/2677)) - chore: fix typos in comment (#2674) ([libp2p/go-libp2p#2674](https://github.com/libp2p/go-libp2p/pull/2674)) - chore: update go-libp2p-asn-util (#2673) ([libp2p/go-libp2p#2673](https://github.com/libp2p/go-libp2p/pull/2673)) - chore: update go security policy url (#2665) ([libp2p/go-libp2p#2665](https://github.com/libp2p/go-libp2p/pull/2665)) - security: remove separate licenses for Noise and TLS (#2663) ([libp2p/go-libp2p#2663](https://github.com/libp2p/go-libp2p/pull/2663)) - webrtc: clarify that there is no reuseport functionality (#2652) ([libp2p/go-libp2p#2652](https://github.com/libp2p/go-libp2p/pull/2652)) - rcmgr: fix connmgr connection limit conflict warning (#2648) ([libp2p/go-libp2p#2648](https://github.com/libp2p/go-libp2p/pull/2648)) - tcp: fix build on loong64 (#2655) ([libp2p/go-libp2p#2655](https://github.com/libp2p/go-libp2p/pull/2655)) - swarm: fix grafana dashboard templating (#2640) ([libp2p/go-libp2p#2640](https://github.com/libp2p/go-libp2p/pull/2640)) - chore: fix typos (#2608) ([libp2p/go-libp2p#2608](https://github.com/libp2p/go-libp2p/pull/2608)) - chore: add resource manager dashboard to docker-compose (#2641) ([libp2p/go-libp2p#2641](https://github.com/libp2p/go-libp2p/pull/2641)) - pstoremanager: fix race condition when removing peers from peer store (#2644) ([libp2p/go-libp2p#2644](https://github.com/libp2p/go-libp2p/pull/2644)) - examples: remove unused 'SetStreamHandler' (#2598) ([libp2p/go-libp2p#2598](https://github.com/libp2p/go-libp2p/pull/2598)) - Update docs from RSA to Ed25519 (#2606) ([libp2p/go-libp2p#2606](https://github.com/libp2p/go-libp2p/pull/2606)) - github.com/multiformats/go-multiaddr (v0.12.1 -> v0.12.2): - chore: release v0.12.2 - tests: add round trip equality check to fuzz (#232) ([multiformats/go-multiaddr#232](https://github.com/multiformats/go-multiaddr/pull/232)) - fix: correctly parse ports as uint16 and explicitly fail on overflows (#228) ([multiformats/go-multiaddr#228](https://github.com/multiformats/go-multiaddr/pull/228)) - replace custom random tests with testing.F (#227) ([multiformats/go-multiaddr#227](https://github.com/multiformats/go-multiaddr/pull/227))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Henrique Dias | 26 | +1668/-1484 | 96 | | Sukun | 13 | +983/-618 | 68 | | Jorropo | 18 | +501/-222 | 32 | | Marten Seemann | 2 | +17/-244 | 5 | | dozyio | 1 | +117/-132 | 31 | | Marcin Rataj | 7 | +100/-20 | 8 | | Alexandr Burdiyan | 2 | +29/-54 | 2 | | Tyler | 1 | +17/-19 | 2 | | KeienWang | 2 | +14/-14 | 12 | | Håvard Anda Estensen | 1 | +14/-14 | 11 | | Halimao | 2 | +17/-4 | 2 | | hannahhoward | 1 | +14/-6 | 2 | | alex | 1 | +8/-8 | 4 | | shuoer86 | 1 | +7/-7 | 5 | | John Chase | 1 | +0/-12 | 1 | | GoodDaisy | 1 | +5/-5 | 4 | | Michael Muré | 1 | +6/-2 | 1 | | 吴小白 | 1 | +3/-3 | 3 | | Vehorny | 1 | +3/-3 | 2 | | Eric | 1 | +1/-1 | 1 | ================================================ FILE: docs/changelogs/v0.28.md ================================================ # Kubo changelog v0.28 - [v0.28.0](#v0280) ## v0.28.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [RPC client: removed deprecated DHT API](#rpc-client-removed-deprecated-dht-api) - [Gateway: `/api/v0` is removed](#gateway-apiv0-is-removed) - [Removed deprecated Object API commands](#removed-deprecated-object-api-commands) - [No longer publishes loopback and private addresses on DHT](#no-longer-publishes-loopback-and-private-addresses-on-dht) - [Pin roots are now prioritized when announcing](#pin-roots-are-now-prioritized-when-announcing) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview #### RPC client: removed deprecated DHT API The deprecated DHT API commands in the RPC client have been removed. Instead, use the Routing API. #### Gateway: `/api/v0` is removed The legacy subset of the Kubo RPC that was available via the Gateway port and was deprecated is now completely removed. You can read more in . If you have a legacy software that relies on this behavior, and want to expose parts of `/api/v0` next to `/ipfs`, use reverse-proxy in front of Kubo to mount both Gateway and RPC on the same port. NOTE: exposing RPC to the internet comes with security risk: make sure to specify access control via [API.Authorizations](https://github.com/ipfs/kubo/blob/master/docs/config.md#apiauthorizations). #### Removed deprecated Object API commands The Object API commands deprecated back in [2021](https://github.com/ipfs/kubo/issues/7936) have been removed, except for `object diff`, `object patch add-link` and `object patch rm-link`, whose alternatives have not yet been built (see issues [4801](https://github.com/ipfs/kubo/issues/4801) and [4782](https://github.com/ipfs/kubo/issues/4782)). ##### Kubo ignores loopback addresses on LAN DHT and private addresses on WAN DHT Kubo no longer keeps track of loopback and private addresses on the LAN and WAN DHTs, respectively. This means that other nodes will not try to dial likely undialable addresses. To support testing scenarios where multiple Kubo instances run on the same machine, [`Routing.LoopbackAddressesOnLanDHT`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingloopbackaddressesonlandht) is set to `true` when the `test` profile is applied. #### Pin roots are now prioritized when announcing The root CIDs of pinned content are now prioritized when announcing to the Amino DHT with [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy) set to `all` (default) or `pinned`, making the important CIDs accessible faster. ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: update version - chore: update version - core/node: prioritize announcing pin roots, and flat strategy (#10376) ([ipfs/kubo#10376](https://github.com/ipfs/kubo/pull/10376)) - chore: webui v4.2.1 (#10391) ([ipfs/kubo#10391](https://github.com/ipfs/kubo/pull/10391)) - docs(config): clarify RPC vs Gateway - chore: upgrade go-libp2p-kad-dht (#10378) ([ipfs/kubo#10378](https://github.com/ipfs/kubo/pull/10378)) - chore(config): make Routing.AcceleratedDHTClient a Flag (#10384) ([ipfs/kubo#10384](https://github.com/ipfs/kubo/pull/10384)) - fix: switch lowpower profile to autoclient - core: fix some typos (#10382) ([ipfs/kubo#10382](https://github.com/ipfs/kubo/pull/10382)) - docs: fix some typos (#10377) ([ipfs/kubo#10377](https://github.com/ipfs/kubo/pull/10377)) - core/commands!: remove deprecated object APIs (#10375) ([ipfs/kubo#10375](https://github.com/ipfs/kubo/pull/10375)) - docs: update default ipns lifetime - coreapi/unixfs: don't create an additional IpfsNode for --only-hash - chore: cleanup old workaround (#10369) ([ipfs/kubo#10369](https://github.com/ipfs/kubo/pull/10369)) - chore: finish reframe removal - docs: remove repetitive words (#10370) ([ipfs/kubo#10370](https://github.com/ipfs/kubo/pull/10370)) - docs: updated links and refs to external resources (#10368) ([ipfs/kubo#10368](https://github.com/ipfs/kubo/pull/10368)) - core/corehttp!: remove /api/v0 from gateway port - client/rpc!: remove deprecated DHT commands - ci: upgrade to go 1.22 (#10355) ([ipfs/kubo#10355](https://github.com/ipfs/kubo/pull/10355)) - chore: create next changelog - Merge Release: v0.27.0 [skip changelog] ([ipfs/kubo#10362](https://github.com/ipfs/kubo/pull/10362)) - test: cleanup content blocking tests (#10360) ([ipfs/kubo#10360](https://github.com/ipfs/kubo/pull/10360)) - docs: improve release issue template - chore: update version - github.com/ipfs/boxo (v0.18.0 -> v0.19.0): - Release v0.19.0 ([ipfs/boxo#598](https://github.com/ipfs/boxo/pull/598)) - github.com/libp2p/go-libp2p (v0.33.0 -> v0.33.2): - chore: release v0.33.2 (#2755) ([libp2p/go-libp2p#2755](https://github.com/libp2p/go-libp2p/pull/2755)) - Update quic-go to v0.42.0. Release v0.33.1 (#2741) ([libp2p/go-libp2p#2741](https://github.com/libp2p/go-libp2p/pull/2741)) - github.com/libp2p/go-libp2p-kad-dht (v0.24.4 -> v0.25.2): - chore: release v0.25.2 ([libp2p/go-libp2p-kad-dht#961](https://github.com/libp2p/go-libp2p-kad-dht/pull/961)) - add ctx canceled err check ([libp2p/go-libp2p-kad-dht#960](https://github.com/libp2p/go-libp2p-kad-dht/pull/960)) - chore: release v0.25.1 - perf: don't buffer the output of FindProvidersAsync - chore: use go-libp2p-routing-helpers for tracing needs - fix: properly iterate in tracing for protocol messenger - fix: apply addrFilters in the dht (#872) ([libp2p/go-libp2p-kad-dht#872](https://github.com/libp2p/go-libp2p-kad-dht/pull/872)) - Add provider record addresses to peerstore ([libp2p/go-libp2p-kad-dht#870](https://github.com/libp2p/go-libp2p-kad-dht/pull/870)) - chore: release v0.25.0 - tracing: add protocol messages client tracing - Enhance handleNewMessage Server Mode Logging: Convert Error Logs to Debug Level ([libp2p/go-libp2p-kad-dht#860](https://github.com/libp2p/go-libp2p-kad-dht/pull/860)) - tracing: fix DHT keys as string attribute not being valid utf-8 ([libp2p/go-libp2p-kad-dht#859](https://github.com/libp2p/go-libp2p-kad-dht/pull/859)) - merge: fix: issues discovered in kubo v0.21.0-rc2 (#853) ([libp2p/go-libp2p-kad-dht#853](https://github.com/libp2p/go-libp2p-kad-dht/pull/853)) - merge: fix: issues discovered in kubo v0.21.0-rc1 (#851) ([libp2p/go-libp2p-kad-dht#851](https://github.com/libp2p/go-libp2p-kad-dht/pull/851)) - Release v0.24.0 ([libp2p/go-libp2p-kad-dht#844](https://github.com/libp2p/go-libp2p-kad-dht/pull/844)) - fix: don't add unresponsive DHT servers to the Routing Table (#820) ([libp2p/go-libp2p-kad-dht#820](https://github.com/libp2p/go-libp2p-kad-dht/pull/820)) - filter local addresses (for WAN) and localhost addresses (for LAN) ([libp2p/go-libp2p-kad-dht#839](https://github.com/libp2p/go-libp2p-kad-dht/pull/839)) - github.com/multiformats/go-multiaddr (v0.12.2 -> v0.12.3): - chore: release v0.12.3 ([multiformats/go-multiaddr#240](https://github.com/multiformats/go-multiaddr/pull/240)) - chore: Expand comment ForEach ([multiformats/go-multiaddr#238](https://github.com/multiformats/go-multiaddr/pull/238)) - .Decapsulate by Components ([multiformats/go-multiaddr#239](https://github.com/multiformats/go-multiaddr/pull/239)) - github.com/whyrusleeping/cbor-gen (v0.0.0-20240109153615-66e95c3e8a87 -> v0.1.0): - Nullable ints (#93) ([whyrusleeping/cbor-gen#93](https://github.com/whyrusleeping/cbor-gen/pull/93)) - Introduce Gen{} struct for configurability ([whyrusleeping/cbor-gen#94](https://github.com/whyrusleeping/cbor-gen/pull/94)) - Transparent encoding ([whyrusleeping/cbor-gen#91](https://github.com/whyrusleeping/cbor-gen/pull/91)) - turn max length consts into global vars ([whyrusleeping/cbor-gen#92](https://github.com/whyrusleeping/cbor-gen/pull/92))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Henrique Dias | 19 | +867/-2806 | 96 | | Rod Vagg | 7 | +921/-475 | 25 | | Marcin Rataj | 8 | +358/-344 | 18 | | Guillaume Michel - guissou | 1 | +145/-485 | 13 | | Jorropo | 8 | +429/-136 | 22 | | Łukasz Magiera | 4 | +284/-48 | 11 | | whyrusleeping | 1 | +90/-90 | 2 | | Michael Muré | 2 | +48/-73 | 9 | | Marco Munizaga | 6 | +86/-29 | 10 | | guillaumemichel | 3 | +93/-1 | 3 | | Marten Seemann | 1 | +31/-4 | 4 | | godeamon | 3 | +11/-8 | 3 | | shuangcui | 1 | +6/-6 | 5 | | occupyhabit | 1 | +3/-3 | 3 | | crazehang | 1 | +2/-2 | 1 | | Dennis Trautwein | 1 | +1/-2 | 1 | | “GheisMohammadi” | 1 | +1/-1 | 1 | | web3-bot | 1 | +2/-0 | 1 | | Daniel Norman | 1 | +1/-1 | 1 | ================================================ FILE: docs/changelogs/v0.29.md ================================================ # Kubo changelog v0.29 - [v0.29.0](#v0290) ## v0.29.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [Add search functionality for pin names](#add-search-functionality-for-pin-names) - [Customizing `ipfs add` defaults](#customizing-ipfs-add-defaults) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### Add search functionality for pin names It is now possible to search for pins by name via `ipfs pin ls --name "SomeName"`. The search is case-sensitive and will return all pins that contain the specified substring in their name. > [!TIP] > The `ipfs pin ls -n` is now a shorthand for `ipfs pin ls --name`, mirroring the behavior of `ipfs pin add`. > See `ipfs pin ls --help` for more information. #### Customizing `ipfs add` defaults This release supports overriding global data ingestion defaults used by commands like `ipfs add` via user-defined [`Import.*` configuration options](../config.md#import). The hash function, CID version, or UnixFS raw leaves and chunker behaviors can be set once, and used as the new implicit default for `ipfs add`. > [!TIP] > As a convenience, two CID [profiles](../config.md#profile) are provided: `legacy-cid-v0` and `test-cid-v1`. > A test profile that defaults to modern CIDv1 can be applied via `ipfs config profile apply test-cid-v1`. > We encourage users to try it and report any issues in [kubo#4143](https://github.com/ipfs/kubo/issues/4143). ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - fix(cli): unify --name param in ls and add (#10439) ([ipfs/kubo#10439](https://github.com/ipfs/kubo/pull/10439)) - chore: set version to 0.29.0-rc2 - fix(libp2p): streams config validation in resource manager (#10435) ([ipfs/kubo#10435](https://github.com/ipfs/kubo/pull/10435)) - chore: update version - chore: libp2p 0.34.1 (#10429) ([ipfs/kubo#10429](https://github.com/ipfs/kubo/pull/10429)) - refactor: stop using github.com/pkg/errors (#10431) ([ipfs/kubo#10431](https://github.com/ipfs/kubo/pull/10431)) - chore: fix --help text - config: introduce Import section (#10421) ([ipfs/kubo#10421](https://github.com/ipfs/kubo/pull/10421)) - feat: enables searching pins by name (#10412) ([ipfs/kubo#10412](https://github.com/ipfs/kubo/pull/10412)) - fix(fuse): ipfs path parsing (#10243) ([ipfs/kubo#10243](https://github.com/ipfs/kubo/pull/10243)) - core/node: fix divide by zero fatal crash for reprovide rate check (#10411) ([ipfs/kubo#10411](https://github.com/ipfs/kubo/pull/10411)) - chore: bump to go-ipfs-cmds @ v0.11 - chore: create next changelog - Merge Release: v0.28.0 [skip changelog] ([ipfs/kubo#10402](https://github.com/ipfs/kubo/pull/10402)) - docs: update release checklist (#10401) ([ipfs/kubo#10401](https://github.com/ipfs/kubo/pull/10401)) - chore: update version - github.com/ipfs/boxo (v0.19.0 -> v0.20.0): - Release v0.20.0 ([ipfs/boxo#613](https://github.com/ipfs/boxo/pull/613)) - github.com/ipfs/go-blockservice (v0.5.0 -> v0.5.2): - docs: remove contribution section - chore: bump version - chore: deprecate types and readme - chore: release v0.5.1 - fix: remove busyloop in getBlocks by removing batching - github.com/ipfs/go-ipfs-blockstore (v1.3.0 -> v1.3.1): - docs: remove contribution section - chore: bump version - chore: deprecate types and readme - github.com/ipfs/go-ipfs-cmds (v0.10.0 -> v0.11.0): - chore: release v0.11.0 (#253) ([ipfs/go-ipfs-cmds#253](https://github.com/ipfs/go-ipfs-cmds/pull/253)) - chore: update deps (#252) ([ipfs/go-ipfs-cmds#252](https://github.com/ipfs/go-ipfs-cmds/pull/252)) - chore: release 0.10.2 (#251) ([ipfs/go-ipfs-cmds#251](https://github.com/ipfs/go-ipfs-cmds/pull/251)) - fix(http): return error in case of panic (#250) ([ipfs/go-ipfs-cmds#250](https://github.com/ipfs/go-ipfs-cmds/pull/250)) - chore: release v0.10.1 - github.com/ipfs/go-ipfs-ds-help (v1.1.0 -> v1.1.1): - docs: remove contribution section - chore: bump version - chore: deprecate types and readme - github.com/ipfs/go-ipfs-exchange-interface (v0.2.0 -> v0.2.1): - chore: bump version - Deprecate types and readme (#29) ([ipfs/go-ipfs-exchange-interface#29](https://github.com/ipfs/go-ipfs-exchange-interface/pull/29)) - docs: Add proper documentation to the interface. - github.com/ipfs/go-verifcid (v0.0.2 -> v0.0.3): - chore: bump version - chore: deprecate types and readme - Make poseidon hashes good hashes ([ipfs/go-verifcid#19](https://github.com/ipfs/go-verifcid/pull/19)) - sync: update CI config files (#18) ([ipfs/go-verifcid#18](https://github.com/ipfs/go-verifcid/pull/18)) - github.com/ipld/go-car (v0.5.0 -> v0.6.2): - v0.6.2 ([ipld/go-car#464](https://github.com/ipld/go-car/pull/464)) - fix: opt-in way to allow empty list of roots in CAR headers ([ipld/go-car#461](https://github.com/ipld/go-car/pull/461)) - feat: add inverse and version to filter cmd ([ipld/go-car#457](https://github.com/ipld/go-car/pull/457)) - v0.6.1 bump - chore: update usage of merkledag by go-car (#437) ([ipld/go-car#437](https://github.com/ipld/go-car/pull/437)) - feat(cmd/car): add '--no-wrap' option to 'create' command ([ipld/go-car#432](https://github.com/ipld/go-car/pull/432)) - fix: remove github.com/ipfs/go-ipfs-blockstore dependency - feat: expose index for StorageCar - perf: reduce NewCarReader allocations - fix(deps): update deps for cmd (use master go-car and go-car/v2 for now) - fix: new error strings from go-cid - fix: tests should match stderr for verbose output - fix: reading from stdin should broadcast EOF to block loaders - refactor insertion index to be publicly accessible ([ipld/go-car#408](https://github.com/ipld/go-car/pull/408)) - chore: unmigrate from go-libipfs - Create CODEOWNERS - blockstore: give a direct access to the index for read operations - blockstore: only close the file on error in OpenReadWrite, not OpenReadWriteFile - fix: handle (and test) WholeCID vs not; fast Has() path for storage - ReadWrite: faster Has() by using the in-memory index instead of reading on disk - fix: let `extract` skip missing unixfs shard links - fix: error when no files extracted - fix: make -f optional, read from stdin if omitted - fix: update cmd/car/README with latest description - chore: add test cases for extract modes - feat: extract accepts '-' as an output path for stdout - feat: extract specific path, accept stdin as streaming input - fix: if we don't read the full block data, don't error on !EOF - blockstore: try to close during Finalize(), even in case of previous error - ReadWrite: add an alternative FinalizeReadOnly+Close flow - feat: add WithTrustedCar() reader option (#381) ([ipld/go-car#381](https://github.com/ipld/go-car/pull/381)) - blockstore: fast path for AllKeysChan using the index - fix: switch to crypto/rand.Read - stop using the deprecated io/ioutil package - fix(doc): fix storage package doc formatting - fix: return errors for unsupported operations - chore: move insertionindex into store pkg - chore: add experimental note - fix: minor lint & windows fd test problems - feat: docs for StorageCar interfaces - feat: ReadableWritable; dedupe shared code - feat: add Writable functionality to StorageCar - feat: StorageCar as a Readable storage, separate from blockstore - feat(blockstore): implement a streaming read only storage - feat(cmd): add index create subcommand to create an external carv2 index ([ipld/go-car#350](https://github.com/ipld/go-car/pull/350)) - chore: bump version to 0.6.0 - fix: use goreleaser instead - Allow using WalkOption in WriteCar function ([ipld/go-car#357](https://github.com/ipld/go-car/pull/357)) - fix: update go-block-format to the version that includes the stubs - feat: upgrade from go-block-format to go-libipfs/blocks - cleanup readme a bit to make the cli more discoverable (#353) ([ipld/go-car#353](https://github.com/ipld/go-car/pull/353)) - Update install instructions in README.md - Add a debugging form for car files. (#341) ([ipld/go-car#341](https://github.com/ipld/go-car/pull/341)) - ([ipld/go-car#340](https://github.com/ipld/go-car/pull/340)) - add a `SkipNext` method on block reader (#338) ([ipld/go-car#338](https://github.com/ipld/go-car/pull/338)) - feat: Has() and Get() will respect StoreIdentityCIDs option - github.com/libp2p/go-libp2p (v0.33.2 -> v0.34.1): - release v0.34.1 (#2811) ([libp2p/go-libp2p#2811](https://github.com/libp2p/go-libp2p/pull/2811)) - config: fix Insecure security constructor (#2810) ([libp2p/go-libp2p#2810](https://github.com/libp2p/go-libp2p/pull/2810)) - rcmgr: Backwards compatibility if you wrap default impl (#2805) ([libp2p/go-libp2p#2805](https://github.com/libp2p/go-libp2p/pull/2805)) - v0.34.0 (#2795) ([libp2p/go-libp2p#2795](https://github.com/libp2p/go-libp2p/pull/2795)) - swarm: fix addr for TestBlackHoledAddrBlocked (#2803) ([libp2p/go-libp2p#2803](https://github.com/libp2p/go-libp2p/pull/2803)) - Add backwards compatibility with old well-known resource (#2798) ([libp2p/go-libp2p#2798](https://github.com/libp2p/go-libp2p/pull/2798)) - rcmgr: remove a connection only once from the limiter (#2800) ([libp2p/go-libp2p#2800](https://github.com/libp2p/go-libp2p/pull/2800)) - Adhere to request.Context when roundtripping on a stream (#2796) ([libp2p/go-libp2p#2796](https://github.com/libp2p/go-libp2p/pull/2796)) - fix: Set missing deadlines (#2794) ([libp2p/go-libp2p#2794](https://github.com/libp2p/go-libp2p/pull/2794)) - rcmgr: Add conn_limiter to limit number of conns per ip cidr (#2788) ([libp2p/go-libp2p#2788](https://github.com/libp2p/go-libp2p/pull/2788)) - identify: refactor observed address manager to do address mapping at thin waist(IP+TCP/UDP) layer (#2793) ([libp2p/go-libp2p#2793](https://github.com/libp2p/go-libp2p/pull/2793)) - fix: DNS protocol address is not reserved (#2792) ([libp2p/go-libp2p#2792](https://github.com/libp2p/go-libp2p/pull/2792)) - Update github.com/quic-go/quic-go dependency (#2780) ([libp2p/go-libp2p#2780](https://github.com/libp2p/go-libp2p/pull/2780)) - webrtc: add webrtc addresses to host normalizer (#2784) ([libp2p/go-libp2p#2784](https://github.com/libp2p/go-libp2p/pull/2784)) - Add a "Limited" network connectivity state (#2696) ([libp2p/go-libp2p#2696](https://github.com/libp2p/go-libp2p/pull/2696)) - basichost: append certhash for webrtc addresses provided via address factory (#2774) ([libp2p/go-libp2p#2774](https://github.com/libp2p/go-libp2p/pull/2774)) - Fix comment (#2775) ([libp2p/go-libp2p#2775](https://github.com/libp2p/go-libp2p/pull/2775)) - Update: update incomplete readmes (#2767) ([libp2p/go-libp2p#2767](https://github.com/libp2p/go-libp2p/pull/2767)) - libp2phttp: Return connection: close when doing http over streams (#2756) ([libp2p/go-libp2p#2756](https://github.com/libp2p/go-libp2p/pull/2756)) - Identify: emit useful events after identification (#2759) ([libp2p/go-libp2p#2759](https://github.com/libp2p/go-libp2p/pull/2759)) - Update chat with rendezvous example (#2769) ([libp2p/go-libp2p#2769](https://github.com/libp2p/go-libp2p/pull/2769)) - Rename well-known resource (#2757) ([libp2p/go-libp2p#2757](https://github.com/libp2p/go-libp2p/pull/2757)) - quic: make server cmd use RFC 9000 instead of draft-29 (#2753) ([libp2p/go-libp2p#2753](https://github.com/libp2p/go-libp2p/pull/2753)) - autonat: Clean up after close (#2749) ([libp2p/go-libp2p#2749](https://github.com/libp2p/go-libp2p/pull/2749)) - webrtc: run onDone callback immediately on close (#2729) ([libp2p/go-libp2p#2729](https://github.com/libp2p/go-libp2p/pull/2729)) - fix: add NullResourceManager to webrtc, fixes panic (#2752) ([libp2p/go-libp2p#2752](https://github.com/libp2p/go-libp2p/pull/2752)) - feat: add tls KeyLogWriter option (#2750) ([libp2p/go-libp2p#2750](https://github.com/libp2p/go-libp2p/pull/2750)) - Use any port, not a specific one for examples (#2748) ([libp2p/go-libp2p#2748](https://github.com/libp2p/go-libp2p/pull/2748)) - quicreuse: remove workaround for quic-go listener close deadlock (#2746) ([libp2p/go-libp2p#2746](https://github.com/libp2p/go-libp2p/pull/2746)) - use Fx to start and stop the host, swarm, autorelay and quicreuse (#2118) ([libp2p/go-libp2p#2118](https://github.com/libp2p/go-libp2p/pull/2118)) - webrtc: set sctp receive buffer size to 100kB (#2745) ([libp2p/go-libp2p#2745](https://github.com/libp2p/go-libp2p/pull/2745)) - basichost: log more info when protocol selection fails (#2734) ([libp2p/go-libp2p#2734](https://github.com/libp2p/go-libp2p/pull/2734)) - chore: bump quic-go (#2742) ([libp2p/go-libp2p#2742](https://github.com/libp2p/go-libp2p/pull/2742)) - security: remove unnecessary noise code (#2738) ([libp2p/go-libp2p#2738](https://github.com/libp2p/go-libp2p/pull/2738)) - webrtc: increase receive buffer size on listener (#2730) ([libp2p/go-libp2p#2730](https://github.com/libp2p/go-libp2p/pull/2730)) - webrtc: fix bug with logger wrapper (#2727) ([libp2p/go-libp2p#2727](https://github.com/libp2p/go-libp2p/pull/2727)) - dcutr: fix log format to actually print error (#2725) ([libp2p/go-libp2p#2725](https://github.com/libp2p/go-libp2p/pull/2725)) - webrtc: use a common logger for all pion logging (#2718) ([libp2p/go-libp2p#2718](https://github.com/libp2p/go-libp2p/pull/2718)) - chore: remove unreadable code, move a test function to test code, better locking in webrtc control reader - ping: use context.Afterfunc to avoid a lingering goroutine (#2723) ([libp2p/go-libp2p#2723](https://github.com/libp2p/go-libp2p/pull/2723)) - webrtc: close mux when closing listener (#2717) ([libp2p/go-libp2p#2717](https://github.com/libp2p/go-libp2p/pull/2717)) - webrtc: setup datachannel handlers before connecting to a peer (#2716) ([libp2p/go-libp2p#2716](https://github.com/libp2p/go-libp2p/pull/2716)) - github.com/libp2p/go-libp2p-pubsub (v0.10.0 -> v0.11.0): - Fix: Own our CertifiedAddrBook (#555) ([libp2p/go-libp2p-pubsub#555](https://github.com/libp2p/go-libp2p-pubsub/pull/555)) - chores: bump go-libp2p (#558) ([libp2p/go-libp2p-pubsub#558](https://github.com/libp2p/go-libp2p-pubsub/pull/558)) - fix: Don't bother parsing an empty slice (#556) ([libp2p/go-libp2p-pubsub#556](https://github.com/libp2p/go-libp2p-pubsub/pull/556)) - Replace fragmentRPC with appendOrMergeRPC (#557) ([libp2p/go-libp2p-pubsub#557](https://github.com/libp2p/go-libp2p-pubsub/pull/557)) - github.com/multiformats/go-multiaddr (v0.12.3 -> v0.12.4): - Release v0.12.4 ([multiformats/go-multiaddr#245](https://github.com/multiformats/go-multiaddr/pull/245)) - net: restrict unicast ip6 public address space (#235) ([multiformats/go-multiaddr#235](https://github.com/multiformats/go-multiaddr/pull/235)) - github.com/whyrusleeping/cbor-gen (v0.1.0 -> v0.1.1): - fix: reduce memory held by deferred objects (#96) ([whyrusleeping/cbor-gen#96](https://github.com/whyrusleeping/cbor-gen/pull/96))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Henrique Dias | 33 | +4994/-579 | 115 | | Rod Vagg | 29 | +3781/-1367 | 90 | | sukun | 12 | +2026/-1215 | 39 | | Marco Munizaga | 18 | +1482/-382 | 47 | | Will | 5 | +769/-213 | 17 | | Steven Allen | 5 | +540/-115 | 24 | | Sukun | 4 | +274/-194 | 11 | | Michael Muré | 7 | +372/-55 | 16 | | Marten Seemann | 1 | +243/-141 | 10 | | Marcin Rataj | 7 | +244/-134 | 13 | | hannahhoward | 1 | +277/-0 | 2 | | Will Scott | 5 | +54/-38 | 9 | | Hector Sanjuan | 3 | +68/-20 | 5 | | Jorropo | 5 | +34/-47 | 15 | | Andrew Gillis | 2 | +67/-7 | 3 | | IGP | 1 | +59/-8 | 5 | | Adin Schmahmann | 2 | +50/-0 | 3 | | Laurent Senta | 1 | +40/-4 | 2 | | Brad Fitzpatrick | 1 | +42/-2 | 2 | | Fabio Bozzo | 1 | +36/-1 | 3 | | Yolan Romailler | 1 | +15/-19 | 4 | | Hlib Kanunnikov | 2 | +14/-14 | 6 | | Andreas Penzkofer | 1 | +22/-2 | 3 | | Matthias Fasching | 1 | +8/-10 | 1 | | gopherfarm | 2 | +16/-1 | 2 | | Dreamacro | 1 | +1/-10 | 1 | | web3-bot | 2 | +7/-3 | 4 | | Rafał Leszko | 1 | +4/-4 | 1 | | Oleg Kovalov | 1 | +4/-4 | 3 | | dbeal | 1 | +5/-1 | 1 | | Antonio Navarro Perez | 1 | +4/-1 | 1 | | dozyio | 1 | +3/-0 | 1 | | zhiqiangxu | 1 | +1/-1 | 1 | | the harder the luckier | 1 | +1/-1 | 1 | | Lukáš Lukáč | 1 | +1/-1 | 1 | | Steve Loeppky | 1 | +1/-0 | 1 | ================================================ FILE: docs/changelogs/v0.3.md ================================================ # go-ipfs changelog 2015 ## v0.3.11 - 2016-01-12 This is the final ipfs version before the transition to v0.4.0. It introduces a few stability improvements, bugfixes, and increased test coverage. * Features * Add 'get' and 'patch' to the allowed gateway commands (@whyrusleeping) * Updated webui version (@dignifiedquire) * BugFixes * Fix path parsing for add command (@djdv) * namesys: Make paths with multiple segments work. Fixes #2059 (@Kubuxu) * Fix up panic catching in http handler funcs (@whyrusleeping) * Add correct access control headers to the default api config (@dignifiedquire) * Fix closenotify by not sending empty file set (@whyrusleeping) * Tool Changes * Have install.sh use the full path to ipfs binary if detected (@jedahan) * Install daemon system-wide if on El Capitan (@jedahan) * makefile: add -ldflags to install and nofuse tasks (@lgierth) * General Codebase * Clean up http client code (@whyrusleeping) * Move api version check to header (@rht) * Documentation * Improved release checklist (@jbenet) * Added quotes around command in long description (@RichardLitt) * Added a shutdown note to daemon description (@RichardLitt) * Testing * t0080: improve last tests (@chriscool) * t0080: improve 'ipfs refs --unique' test (@chriscool) * Fix t.Fatal usage in goroutines (@chriscool) * Add docker testing support to sharness (@chriscool) * sharness: add t0300-docker-image.sh (@chriscool) * Included more namesys tests. (@Kubuxu) * Add sharness test to verify requests look good (@whyrusleeping) * Re-enable ipns sharness test now that iptb is fixed (@whyrusleeping) * Force use of ipv4 in test (@whyrusleeping) * Travis-CI: use go 1.5.2 (@jbenet) ## v0.3.10 - 2015-12-07 This patch update introduces the 'ipfs update' command which will be used for future ipfs updates along with a few other bug fixes and documentation improvements. * Features * support for 'ipfs update' to call external binary (@whyrusleeping) * cache ipns entries to speed things up a little (@whyrusleeping) * add option to version command to print repo version (@whyrusleeping) * Add in some more notifications to help profile queries (@whyrusleeping) * gateway: add path prefix for directory listings (@lgierth) * gateway: add CurrentCommit to /version (@lgierth) * BugFixes * set data and links nil if not present (@whyrusleeping) * fix log hanging issue, and implement close-notify for commands (@whyrusleeping) * fix dial backoff (@whyrusleeping) * proper ndjson implementation (@whyrusleeping) * seccat: fix secio context (@lgierth) * Add newline to end of the output for a few commands. (@nham) * Add fixed period repo GC + test (@rht) * Tool Changes * Allow `ipfs cat` on ipns path (@rht) * General Codebase * rewrite of backoff mechanism (@whyrusleeping) * refactor net code to use transports, in rough accordance with libp2p (@whyrusleeping) * disable building fuse stuff on windows (@whyrusleeping) * repo: remove Log config (@lgierth) * commands: fix description of --api (@lgierth) * Documentation * --help: Add a note on using IPFS_PATH to the footer of the helptext. (@sahib) * Moved email juan to ipfs/contribute (@richardlitt) * Added commit sign off section (@richardlitt) * Added a security section (@richardlitt) * Moved TODO doc to issue #1929 (@richardlitt) * Testing * gateway: add tests for /version (@lgierth) * Add gc auto test (@rht) * t0020: cleanup dir with bad perms (@chriscool) Note: this commit introduces fixed-period repo gc, which will trigger gc after a fixed period of time. This feature is introduced now, disabled by default, and can be enabled with `ipfs daemon --enable-gc`. If all goes well, in the future, it will be enabled by default. ## v0.3.9 - 2015-10-30 This patch update includes a good number of bugfixes, notably, it fixes builds on windows, and puts newlines between streaming json objects for a proper ndjson format. * Features * Writable gateway enabled again (@cryptix) * Bugfixes * fix windows builds (@whyrusleeping) * content type on command responses default to text (@whyrusleeping) * add check to makefile to ensure windows builds don't fail silently (@whyrusleeping) * put newlines between streaming json output objects (@whyrusleeping) * fix streaming output to flush per write (@whyrusleeping) * purposely fail builds pre go1.5 (@whyrusleeping) * fix ipfs id (@whyrusleeping) * fix a few race conditions in mocknet (@whyrusleeping) * fix makefile failing when not in a git repo (@whyrusleeping) * fix cli flag orders (long, short) (@rht) * fix races in http cors (@miolini) * small webui update (some bugfixes) (@jbenet) * Tool Changes * make swarm connect return an error when it fails (@whyrusleeping) * Add short flag for `ipfs ls --headers` (v for verbose) (@rht) * General Codebase * bitswap: clean log printf and humanize dup data count (@cryptix) * config: update pluto's peerID (@lgierth) * config: update bootstrap list hostname (@lgierth) * Documentation * Pared down contribute to link to new go guidelines (@richardlitt) * Testing * t0010: add tests for 'ipfs commands --flags' (@chriscool) * ipns_test: fix namesys.NewNameSystem() call (@chriscool) * t0060: fail if no nc (@chriscool) ## v0.3.8 - 2015-10-09 This patch update includes changes to make ipns more consistent and reliable, symlink support in unixfs, mild performance improvements, new tooling features, a plethora of bugfixes, and greatly improved tests. NOTICE: Version 0.3.8 also requires golang version 1.5.1 or higher. * Bugfixes * refactor ipns to be more consistent and reliable (@whyrusleeping) * fix 'ipfs refs' json output (@whyrusleeping) * fix setting null config maps (@rht) * fix output of dht commands (@whyrusleeping) * fix NAT spam dialing (@whyrusleeping) * fix random panics on 32 bit systems (@whyrusleeping) * limit total number of network fd's (@whyrusleeping) * fix http api content type (@WeMeetAgain) * fix writing of api file for port zero daemons (@whyrusleeping) * windows connection refused fixes (@mjanczyk) * use go1.5's built in trailers, no more failures (@whyrusleeping) * fix random bitswap hangs (@whyrusleeping) * rate limit fd usage (@whyrusleeping) * fix panic in bitswap rate limiting (@whyrusleeping) * Tool Changes * --empty-repo option for init (@prusnak) * implement symlinks (@whyrusleeping) * improve cmds lib files processing (@rht) * properly return errors through commands (@whyrusleeping) * bitswap unwant command (@whyrusleeping) * tar add/cat commands (@whyrusleeping) * fix gzip compression in get (@klauspost) * bitswap stat logs wasted bytes (@whyrusleeping) * resolve command now uses core.Resolve (@rht) * add `--local` flag to 'name resolve' (@whyrusleeping) * add `ipfs diag sys` command for debugging help (@whyrusleeping) * General Codebase * improvements to dag editor (@whyrusleeping) * swarm IPv6 in default config (Baptiste Jonglez) * improve dir listing css (@rht) * removed elliptic.P224 usage (@prusnak) * improve bitswap providing speed (@jbenet) * print panics that occur in cmds lib (@whyrusleeping) * ipfs api check test fixes (@rht) * update peerstream and datastore (@whyrusleeping) * cleaned up tar-reader code (@jbenet) * write context into coreunix.Cat (@rht) * move assets to separate repo (@rht) * fix proc/ctx wiring in bitswap (@jbenet) * rabin fingerprinting chunker (@whyrusleeping) * better notification on daemon ready (@rht) * coreunix cat cleanup (@rht) * extract logging into go-log (@whyrusleeping) * blockservice.New no longer errors (@whyrusleeping) * refactor ipfs get (@rht) * readonly api on gateway (@rht) * cleanup context usage all over (@rht) * add xml decoding to 'object put' (@ForrestWeston) * replace nodebuilder with NewNode method (@whyrusleeping) * add metrics to http handlers (@lgierth) * rm blockservice workers (@whyrusleeping) * decompose maybeGzWriter (@rht) * makefile sets git commit sha on build (@CaioAlonso) * Documentation * add contribute file (@RichardLitt) * add go devel guide to contribute.md (@whyrusleeping) * Testing * fix mock notifs test (@whyrusleeping) * test utf8 with object cmd (@chriscool) * make mocknet conn close idempotent (@jbenet) * fix fuse tests (@pnelson) * improve sharness test quoting (@chriscool) * sharness tests for chunker and add-cat (@rht) * generalize peerid check in sharness (@chriscool) * test_cmp argument cleanup (@chriscool) ## v0.3.7 - 2015-08-02 This patch update fixes a problem we introduced in 0.3.6 and did not catch: the webui failed to work with out-of-the-box CORS configs. This has been fixed and now should work correctly. @jbenet ## v0.3.6 - 2015-07-30 This patch improves the resource consumption of go-ipfs, introduces a few new options on the CLI, and also fixes (yet again) windows builds. * Resource consumption: * fixed goprocess memory leak @rht * implement batching on datastore @whyrusleeping * Fix bitswap memory leak @whyrusleeping * let bitswap ignore temporary write errors @whyrusleeping * remove logging to disk in favor of api endpoint @whyrusleeping * --only-hash option for add to skip writing to disk @whyrusleeping * Tool changes * improved `ipfs daemon` output with all addresses @jbenet * improved `ipfs id -f` output, added `` and `\n \t` support @jbenet * `ipfs swarm addrs local` now shows the local node's addrs @jbenet * improved config json parsing @rht * improved Dockerfile to use alpine linux @Luzifer @lgierth * improved bash completion @MichaelMure * Improved 404 for gateway @cryptix * add unixfs ls to list correct filesizes @wking * ignore hidden files by default @gatesvp * global --timeout flag @whyrusleeping * fix random API failures by closing resp bodies @whyrusleeping * ipfs swarm filters @whyrusleeping * api returns errors in http trailers @whyrusleeping @jbenet * `ipfs patch` learned to create intermediate nodes @whyrusleeping * `ipfs object stat` now shows Hash @whyrusleeping * `ipfs cat` now clears progressbar on exit @rht * `ipfs add -w -r ` now wraps directories @jbenet * `ipfs add -w ` now wraps with one dir @jbenet * API + Gateway now support arbitrary HTTP Headers from config @jbenet * API now supports CORS properly from config @jbenet * **Deprecated:** `API_ORIGIN` env var (use config, see `ipfs daemon --help`) @jbenet * General Codebase * `nofuse` tag for windows @Luzifer * improved `ipfs add` code @gatesvp * started requiring license trailers @chriscool @jbenet * removed CtxCloser for goprocess @rht * remove deadcode @lgierth @whyrusleeping * reduced number of logging libs to 2 (soon to be 1) @rht * dial address filtering @whyrusleeping * Prometheus metrics @lgierth * new index page for gateway @krl @cryptix * move ping to separate protocol @whyrusleeping * add events to bitswap for a dashboard @whyrusleeping * add latency and bandwidth options to mocknet @heems * levenshtein distance cmd autosuggest @sbruce * refactor/cleanup of cmds http handler @whyrusleeping * cmds http stream reports errors in trailers @whyrusleeping * Bugfixes * fixed path resolution and validation @rht * fixed `ipfs get -C` output and progress bar @rht * Fixed install pkg dist bug @jbenet @Luzifer * Fix `ipfs get` silent failure @whyrusleeping * `ipfs get` tarx no longer times out @jbenet * `ipfs refs -r -u` is now correct @gatesvp * Fix `ipfs add -w -r ` wrapping bugs @jbenet * Fixed FUSE unmount failures @jbenet * Fixed `ipfs log tail` command (api + cli) @whyrusleeping * Testing * sharness updates @chriscool * ability to disable secio for testing @jbenet * fixed many random test failures, more reliable CI @whyrusleeping * Fixed racey notifier failures @whyrusleeping * `ipfs refs -r -u` test cases @jbenet * Fix failing pinning test @jbenet * Better CORS + Referer tests @jbenet * Added reversible gc test @rht * Fixed bugs in FUSE IPNS tests @whyrusleeping * Fixed bugs in FUSE IPFS tests @jbenet * Added `random-files` tool for easier sharness tests @jbenet * Documentation * Add link to init system examples @slang800 * Add CORS documentation to daemon init @carver (Note: this will change soon) ## v0.3.5 - 2015-06-11 This patch improves overall stability and performance * added 'object patch' and 'object new' commands @whyrusleeping * improved symmetric NAT avoidance @jbenet * move util.Key to blocks.Key @whyrusleeping * fix memory leak in provider store @whyrusleeping * updated webui to 0.2.0 @krl * improved bitswap performance @whyrusleeping * update fuse lib @cryptix * fix path resolution @wking * implement test_seq() in sharness @chriscool * improve parsing of stdin for commands @chriscool * fix 'ipfs refs' failing silently @whyrusleeping * fix serial dialing bug @jbenet * improved testing @chriscool @rht @jbenet * fixed domain resolving @luzifer * fix parsing of unwanted stdin @lgierth * added CORS handlers to gateway @NodeGuy * added `ipfs daemon --unrestricted-api` option @krl * general cleanup of dependencies ## v0.3.4 - 2015-05-10 * fix ipns append bug @whyrusleeping * fix out of memory panic @whyrusleeping * add in expvar metrics @tv42 * bitswap improvements @whyrusleeping * fix write-cache in blockstore @tv42 * vendoring cleanup @cryptix * added `launchctl` plist for OSX @grncdr * improved Dockerfile, changed root and mount paths @ehd * improved `pin ls` output to show types @vitorbaptista ## v0.3.3 - 2015-04-28 This patch update fixes various issues, in particular: - windows support (0.3.0 had broken it) - command line parses spaces correctly. * much improved command line parsing by @AtnNn * improved dockerfile by @luzifer * add cmd cleanup by @wking * fix flatfs windows support by @tv42 and @gatesvp * test case improvements by @chriscool * ipns resolution timeout bug fix by @whyrusleeping * new cluster tests with iptb by @whyrusleeping * fix log callstack printing bug by @whyrusleeping * document bash completion by @dylanPowers ## v0.3.2 - 2015-04-22 This patch update implements multicast dns as well as fixing a few test issues. * implement mdns peer discovery @whyrusleeping * fix mounting issues in sharness tests @chriscool ## v0.3.1 - 2015-04-21 This patch update fixes a few bugs: * harden shutdown logic by @torarnv * daemon locking fixes by @travisperson * don't re-add entire dirs by @whyrusleeping * tests now wait for graceful shutdown by @jbenet * default key size is now 2048 by @jbenet ## v0.3.0 - 2015-04-20 We've just released version 0.3.0, which contains many performance improvements, bugfixes, and new features. Perhaps the most noticeable change is moving block storage from leveldb to flat files in the filesystem. What to expect: * _much faster_ performance * Repo format 2 * moved default location from ~/.go-ipfs -> ~/.ipfs * renamed lock filename daemon.lock -> repo.lock * now using a flat-file datastore for local blocks * Fixed lots of bugs * proper ipfs-path in various commands * fixed two pinning bugs (recursive pins) * increased yamux streams window (for speed) * increased bitswap workers (+ env var) * fixed memory leaks * ipfs add error returns * daemon exit bugfix * set proper UID and GID on fuse mounts * Gateway * Added support for HEAD requests * configuration * env var to turn off SO_REUSEPORT: IPFS_REUSEPORT=false * env var to increase bitswap workers: IPFS_BITSWAP_TASK_WORKERS=n * other * bash completion is now available * ipfs stats bw -- bandwidth metering And many more things. ================================================ FILE: docs/changelogs/v0.30.md ================================================ # Kubo changelog v0.30 - [v0.30.0](#v0300) ## v0.30.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [Improved P2P connectivity](#improved-p2p-connectivity) - [Refactored Bitswap and dag-pb chunker](#refactored-bitswap-and-dag-pb-chunker) - [WebRTC-Direct Transport enabled by default](#webrtc-direct-transport-enabled-by-default) - [UnixFS 1.5: Mode and Modification Time Support](#unixfs-15-mode-and-modification-time-support) - [AutoNAT V2 Service Introduced Alongside V1](#autonat-v2-service-introduced-alongside-v1) - [Automated `ipfs version check`](#automated-ipfs-version-check) - [Version Suffix Configuration](#version-suffix-configuration) - [`/unix/` socket support in `Addresses.API`](#unix-socket-support-in-addressesapi) - [Cleaned Up `ipfs daemon` Startup Log](#cleaned-up-ipfs-daemon-startup-log) - [Commands Preserve Specified Hostname](#commands-preserve-specified-hostname) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights This release took longer and is more packed with fixes and features than usual. > [!IMPORTANT] > TLDR: update, it contains many, many fixes. #### Improved P2P connectivity This release comes with significant go-libp2p update from v0.34.1 to v0.36.3 ([release notes](https://github.com/libp2p/go-libp2p/releases/)). It includes multiple fixes to key protocols: [QUIC](https://github.com/libp2p/specs/tree/master/quic)/[Webtransport](https://github.com/libp2p/specs/tree/master/webtransport)/[WebRTC](https://github.com/libp2p/specs/tree/master/webrtc), Connection Upgrades through Relay ([DCUtR](https://github.com/libp2p/specs/blob/master/relay/DCUtR.md)), and [Secure WebSockets](https://github.com/libp2p/specs/pull/624). Also, peers that are behind certain types of NAT will now be more reachable. For this alone, Kubo users are highly encouraged to upgrade. #### Refactored Bitswap and dag-pb chunker Some workloads may experience improved memory profile thanks to optimizations from Boxo SDK [v0.23.0](https://github.com/ipfs/boxo/releases/tag/v0.23.0). > [!IMPORTANT] > Storage providers should upgrade to take advantage of the Bitswap server fix, which resolves the issue of greedy peers depleting available wantlist slots for their PeerID, resulting in stalled downloads. #### WebRTC-Direct Transport enabled by default Kubo now ships with [WebRTC Direct](https://github.com/libp2p/specs/blob/master/webrtc/webrtc-direct.md) listener enabled by default: `/udp/4001/webrtc-direct`. WebRTC Direct complements existing `/wss` (Secure WebSockets) and `/webtransport` transports. Unlike `/wss`, which requires a domain name and a CA-issued TLS certificate, WebRTC Direct works with IPs and can be enabled by default on all Kubo nodes. Learn more: [`Swarm.Transports.Network.WebRTCDirect`](https://github.com/ipfs/kubo/blob/master/docs/config.md#swarmtransportsnetworkwebrtcdirect) > [!NOTE] > Kubo 0.30 includes a migration for existing users that adds `/webrtc-direct` listener on the same UDP port as `/udp/{port}/quic-v1`. This supports the WebRTC-Direct rollout by reusing preexisting UDP firewall settings and port mappings created for QUIC. #### UnixFS 1.5: Mode and Modification Time Support Kubo now allows users to opt-in to store mode and modification time for files, directories, and symbolic links. By default, if you do not opt-in, the old behavior remains unchanged, and the same CIDs will be generated as before. The `ipfs add` CLI options `--preserve-mode` and `--preserve-mtime` can be used to store the original mode and last modified time of the file being added, and `ipfs files stat /ipfs/CID` can be used for inspecting these optional attributes: ```console $ touch ./file $ chmod 654 ./file $ ipfs add --preserve-mode --preserve-mtime -Q ./file QmczQr4XS1rRnWVopyg5Chr9EQ7JKpbhgnrjpb5kTQ1DKQ $ ipfs files stat /ipfs/QmczQr4XS1rRnWVopyg5Chr9EQ7JKpbhgnrjpb5kTQ1DKQ QmczQr4XS1rRnWVopyg5Chr9EQ7JKpbhgnrjpb5kTQ1DKQ Size: 0 CumulativeSize: 22 ChildBlocks: 0 Type: file Mode: -rw-r-xr-- (0654) Mtime: 13 Aug 2024, 21:15:31 UTC ``` The CLI and HTTP RPC options `--mode`, `--mtime` and `--mtime-nsecs` can be used to set them to arbitrary values. Opt-in support for `mode` and `mtime` was also added to MFS (`ipfs files --help`). For more information see `--help` text of `ipfs files touch|stat|chmod` commands. Modification time support was also added to the Gateway. If present, value from file's dag-pb is returned in `Last-Modified` HTTP header and requests made with `If-Modified-Since` can produce HTTP 304 not modified response. > [!NOTE] > Storing `mode` and `mtime` requires root block to be `dag-pb` and disabled `raw-leaves` setting to create envelope for storing the metadata. #### AutoNAT V2 Service Introduced Alongside V1 The AutoNAT service enables nodes to determine their public reachability on the internet. [AutoNAT V2](https://github.com/libp2p/specs/pull/538) enhances this protocol with improved features. In this release, Kubo will offer both V1 and V2 services to other peers, although it will continue to use only V1 when acting as a client. Future releases will phase out V1, transitioning clients to utilize V2 exclusively. For more details, see the [Deployment Plan for AutoNAT V2](https://github.com/ipfs/kubo/issues/10091) and [`AutoNAT`](https://github.com/ipfs/kubo/blob/master/docs/config.md#autonat) configuration options. #### Automated `ipfs version check` Kubo now performs privacy-preserving version checks using the [libp2p identify protocol](https://github.com/libp2p/specs/blob/master/identify/README.md) on peers detected by the Amino DHT client. If more than 5% of Kubo peers seen by your node are running a newer version, you will receive a log message notification. - For manual checks, refer to `ipfs version check --help` for details. - To disable automated checks, set [`Version.SwarmCheckEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#versionswarmcheckenabled) to `false`. #### Version Suffix Configuration Defining the optional agent version suffix is now simpler. The [`Version.AgentSuffix`](https://github.com/ipfs/kubo/blob/master/docs/config.md#agentsuffix) value from the Kubo config takes precedence over any value provided via `ipfs daemon --agent-version-suffix` (which is still supported). > [!NOTE] > Setting a custom version suffix helps with ecosystem analysis, such as Amino DHT reports published at https://stats.ipfs.network #### `/unix/` socket support in `Addresses.API` This release fixes a bug which blocked users from using Unix domain sockets for [Kubo's RPC](https://docs.ipfs.tech/reference/kubo/rpc/) (instead of a local HTTP port). ```console $ ipfs config Addresses.API "/unix/tmp/kubo.socket" $ ipfs daemon # start with rpc socket ... RPC API server listening on /unix/tmp/kubo.socket $ # cli client, in different terminal can find socket via /api file $ cat $IPFS_PATH/api /unix/tmp/kubo.socket $ # or have it passed via --api $ ipfs --api=/unix/tmp/kubo.socket id ``` #### Cleaned Up `ipfs daemon` Startup Log The `ipfs daemon` startup output has been streamlined to enhance clarity and usability: ```console $ ipfs daemon Initializing daemon... Kubo version: 0.30.0 Repo version: 16 System version: amd64/linux Golang version: go1.22.5 PeerID: 12D3KooWQ73s1CQsm4jWwQvdCAtc5w8LatyQt7QLQARk5xdhK9CE Swarm listening on 127.0.0.1:4001 (TCP+UDP) Swarm listening on 192.0.2.10:4001 (TCP+UDP) Swarm listening on [::1]:4001 (TCP+UDP) Swarm listening on [2001:0db8::10]:4001 (TCP+UDP) Run 'ipfs id' to inspect announced and discovered multiaddrs of this node. RPC API server listening on /ip4/127.0.0.1/tcp/5001 WebUI: http://127.0.0.1:5001/webui Gateway server listening on /ip4/127.0.0.1/tcp/8080 Daemon is ready ``` The previous lengthy listing of all listener and announced multiaddrs has been removed due to its complexity, especially with modern libp2p nodes sharing multiple transports and long lists of `/webtransport` and `/webrtc-direct` certhashes. The output now features a simplified list of swarm listeners, displayed in the format `host:port (TCP+UDP)`, which provides essential information for debugging connectivity issues, particularly related to port forwarding. Announced libp2p addresses are no longer printed on startup, because libp2p may change or augment them based on AutoNAT, relay, and UPnP state. Instead, users are prompted to run `ipfs id` to obtain up-to-date list of listeners and announced multiaddrs in libp2p format. #### Commands Preserve Specified Hostname When executing a [CLI command](https://docs.ipfs.tech/reference/kubo/cli/) over [Kubo RPC API](https://docs.ipfs.tech/reference/kubo/rpc/), if a hostname is specified by `--api=/dns4//` the resulting HTTP request now contains the hostname, instead of the the IP address that the hostname resolved to, as was the previous behavior. This makes it easier for those trying to run Kubo behind a reverse proxy using hostname-based rules. #### Commands Preserve Specified Hostname When executing a [CLI command](https://docs.ipfs.tech/reference/kubo/cli/) over [Kubo RPC API](https://docs.ipfs.tech/reference/kubo/rpc/), if a hostname is specified by `--api=/dns4//` the resulting HTTP request now contains the hostname, instead of the the IP address that the hostname resolved to, as was the previous behavior. This makes it easier for those trying to run Kubo behind a reverse proxy using hostname-based rules. ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: set version to 0.30.0 - chore: bump CurrentVersionNumber - chore: boxo v0.23.0 and go-libp2p v0.36.3 (#10507) ([ipfs/kubo#10507](https://github.com/ipfs/kubo/pull/10507)) - fix: switch back to go 1.22 (#10502) ([ipfs/kubo#10502](https://github.com/ipfs/kubo/pull/10502)) - chore: update go-unixfsnode, cmds, and boxo (#10494) ([ipfs/kubo#10494](https://github.com/ipfs/kubo/pull/10494)) - fix(cli): preserve hostname specified with --api in http request headers (#10497) ([ipfs/kubo#10497](https://github.com/ipfs/kubo/pull/10497)) - chore: upgrade to go 1.23 (#10486) ([ipfs/kubo#10486](https://github.com/ipfs/kubo/pull/10486)) - fix: error during config when running benchmarks (#10495) ([ipfs/kubo#10495](https://github.com/ipfs/kubo/pull/10495)) - chore: update version to rc-2 - chore: update version - chore: fix function name (#10481) ([ipfs/kubo#10481](https://github.com/ipfs/kubo/pull/10481)) - feat: Support storing UnixFS 1.5 Mode and ModTime (#10478) ([ipfs/kubo#10478](https://github.com/ipfs/kubo/pull/10478)) - fix(rpc): cross-platform support for /unix/ socket maddrs in Addresses.API ([ipfs/kubo#10019](https://github.com/ipfs/kubo/pull/10019)) - chore(daemon): sort listeners (#10480) ([ipfs/kubo#10480](https://github.com/ipfs/kubo/pull/10480)) - feat(daemon): improve stdout on startup (#10472) ([ipfs/kubo#10472](https://github.com/ipfs/kubo/pull/10472)) - fix(daemon): panic in kubo/daemon.go:595 (#10473) ([ipfs/kubo#10473](https://github.com/ipfs/kubo/pull/10473)) - feat: webui v4.3.0 (#10477) ([ipfs/kubo#10477](https://github.com/ipfs/kubo/pull/10477)) - docs(readme): add Gentoo Linux (#10474) ([ipfs/kubo#10474](https://github.com/ipfs/kubo/pull/10474)) - libp2p: default to preferring TLS ([ipfs/kubo#10227](https://github.com/ipfs/kubo/pull/10227)) - docs: document unofficial Ubuntu PPA ([ipfs/kubo#10467](https://github.com/ipfs/kubo/pull/10467)) - feat: run AutoNAT V2 service in addition to V1 (#10468) ([ipfs/kubo#10468](https://github.com/ipfs/kubo/pull/10468)) - feat: go-libp2p 0.36 and /webrtc-direct listener (#10463) ([ipfs/kubo#10463](https://github.com/ipfs/kubo/pull/10463)) - chore: update dependencies (#10462)(#10466) ([ipfs/kubo#10466](https://github.com/ipfs/kubo/pull/10466)) - feat: periodic version check and json config (#10438) ([ipfs/kubo#10438](https://github.com/ipfs/kubo/pull/10438)) - docs: clarify pnet limitations - docs: "error mounting: could not resolve name" (#10449) ([ipfs/kubo#10449](https://github.com/ipfs/kubo/pull/10449)) - docs: update ipfs-swarm-key-gen example (#10453) ([ipfs/kubo#10453](https://github.com/ipfs/kubo/pull/10453)) - chore: update deps incl. boxo v0.21.0 (#10444) ([ipfs/kubo#10444](https://github.com/ipfs/kubo/pull/10444)) - chore: go-libp2p 0.35.1 (#10430) ([ipfs/kubo#10430](https://github.com/ipfs/kubo/pull/10430)) - docsa: update RELEASE_CHECKLIST.md - chore: create next changelog (#10443) ([ipfs/kubo#10443](https://github.com/ipfs/kubo/pull/10443)) - Merge Release: v0.29.0 [skip changelog] ([ipfs/kubo#10442](https://github.com/ipfs/kubo/pull/10442)) - fix(cli): unify --name param in ls and add (#10439) ([ipfs/kubo#10439](https://github.com/ipfs/kubo/pull/10439)) - fix(libp2p): streams config validation in resource manager (#10435) ([ipfs/kubo#10435](https://github.com/ipfs/kubo/pull/10435)) - chore: fix some typos (#10396) ([ipfs/kubo#10396](https://github.com/ipfs/kubo/pull/10396)) - chore: update version - github.com/ipfs/boxo (v0.20.0 -> v0.23.0): - Release v0.23.0 ([ipfs/boxo#669](https://github.com/ipfs/boxo/pull/669)) - docs(changelog): move entry to correct release - Release v0.22.0 ([ipfs/boxo#654](https://github.com/ipfs/boxo/pull/654)) - Release v0.21.0 ([ipfs/boxo#622](https://github.com/ipfs/boxo/pull/622)) - github.com/ipfs/go-ipfs-cmds (v0.11.0 -> v0.13.0): - chore: release v0.13.0 (#261) ([ipfs/go-ipfs-cmds#261](https://github.com/ipfs/go-ipfs-cmds/pull/261)) - chore: release v0.12.0 (#259) ([ipfs/go-ipfs-cmds#259](https://github.com/ipfs/go-ipfs-cmds/pull/259)) - github.com/ipfs/go-unixfsnode (v1.9.0 -> v1.9.1): - Update release version ([ipfs/go-unixfsnode#76](https://github.com/ipfs/go-unixfsnode/pull/76)) - chore: update dependencies ([ipfs/go-unixfsnode#75](https://github.com/ipfs/go-unixfsnode/pull/75)) - github.com/libp2p/go-libp2p (v0.34.1 -> v0.36.3): - Release v0.36.3 - Fix: WebSocket: Clone TLS config before creating a new listener - fix: enable dctur when interface address is public (#2931) ([libp2p/go-libp2p#2931](https://github.com/libp2p/go-libp2p/pull/2931)) - fix: QUIC/Webtransport Transports now will prefer their owned listeners for dialing out (#2936) ([libp2p/go-libp2p#2936](https://github.com/libp2p/go-libp2p/pull/2936)) - ci: uci/update-go (#2937) ([libp2p/go-libp2p#2937](https://github.com/libp2p/go-libp2p/pull/2937)) - fix: slice append value (#2938) ([libp2p/go-libp2p#2938](https://github.com/libp2p/go-libp2p/pull/2938)) - webrtc: wait for listener context before dropping connection (#2932) ([libp2p/go-libp2p#2932](https://github.com/libp2p/go-libp2p/pull/2932)) - ci: use go1.23, drop go1.21 (#2933) ([libp2p/go-libp2p#2933](https://github.com/libp2p/go-libp2p/pull/2933)) - Fail on any test timeout (#2929) ([libp2p/go-libp2p#2929](https://github.com/libp2p/go-libp2p/pull/2929)) - test: Try to fix test timeout (#2930) ([libp2p/go-libp2p#2930](https://github.com/libp2p/go-libp2p/pull/2930)) - ci: Out of the tarpit (#2923) ([libp2p/go-libp2p#2923](https://github.com/libp2p/go-libp2p/pull/2923)) - Fix proto import paths (#2920) ([libp2p/go-libp2p#2920](https://github.com/libp2p/go-libp2p/pull/2920)) - Release v0.36.2 - webrtc: reduce loglevel for pion logs (#2915) ([libp2p/go-libp2p#2915](https://github.com/libp2p/go-libp2p/pull/2915)) - webrtc: close connection when remote closes (#2914) ([libp2p/go-libp2p#2914](https://github.com/libp2p/go-libp2p/pull/2914)) - basic_host: close swarm on Close (#2916) ([libp2p/go-libp2p#2916](https://github.com/libp2p/go-libp2p/pull/2916)) - Revert "Create funding.json" (#2919) ([libp2p/go-libp2p#2919](https://github.com/libp2p/go-libp2p/pull/2919)) - Create funding.json - Release v0.36.1 - Release v0.36.0 (#2905) ([libp2p/go-libp2p#2905](https://github.com/libp2p/go-libp2p/pull/2905)) - swarm: add a default timeout to conn.NewStream (#2907) ([libp2p/go-libp2p#2907](https://github.com/libp2p/go-libp2p/pull/2907)) - udpmux: Don't log an error if canceled because of shutdown (#2903) ([libp2p/go-libp2p#2903](https://github.com/libp2p/go-libp2p/pull/2903)) - ObsAddrManager: Infer external addresses for transports that share the same listening address. (#2892) ([libp2p/go-libp2p#2892](https://github.com/libp2p/go-libp2p/pull/2892)) - feat: WebRTC reuse QUIC conn (#2889) ([libp2p/go-libp2p#2889](https://github.com/libp2p/go-libp2p/pull/2889)) - examples/chat-with-mdns: default to a random port (#2896) ([libp2p/go-libp2p#2896](https://github.com/libp2p/go-libp2p/pull/2896)) - allow findpeers limit to be 0 (#2894) ([libp2p/go-libp2p#2894](https://github.com/libp2p/go-libp2p/pull/2894)) - quic: add support for quic-go metrics (#2823) ([libp2p/go-libp2p#2823](https://github.com/libp2p/go-libp2p/pull/2823)) - webrtc: remove experimental tag, enable by default (#2887) ([libp2p/go-libp2p#2887](https://github.com/libp2p/go-libp2p/pull/2887)) - config: fix AddrFactory for AutoNAT (#2868) ([libp2p/go-libp2p#2868](https://github.com/libp2p/go-libp2p/pull/2868)) - chore: /quic → /quic-v1 (#2888) ([libp2p/go-libp2p#2888](https://github.com/libp2p/go-libp2p/pull/2888)) - basichost: reset stream if SetProtocol fails (#2875) ([libp2p/go-libp2p#2875](https://github.com/libp2p/go-libp2p/pull/2875)) - feat: libp2phttp `/http-path` (#2850) ([libp2p/go-libp2p#2850](https://github.com/libp2p/go-libp2p/pull/2850)) - readme: update per latest multiversx rename (#2874) ([libp2p/go-libp2p#2874](https://github.com/libp2p/go-libp2p/pull/2874)) - websocket: don't return transport.ErrListenerClosed on closing listener (#2867) ([libp2p/go-libp2p#2867](https://github.com/libp2p/go-libp2p/pull/2867)) - Added tau to README.md (#2870) ([libp2p/go-libp2p#2870](https://github.com/libp2p/go-libp2p/pull/2870)) - basichost: reset new stream if rcmgr blocks (#2869) ([libp2p/go-libp2p#2869](https://github.com/libp2p/go-libp2p/pull/2869)) - transport integration tests: test conn attempt is dropped when the rcmgr blocks for WebRTC (#2856) ([libp2p/go-libp2p#2856](https://github.com/libp2p/go-libp2p/pull/2856)) - webtransport: close underlying h3 connection (#2862) ([libp2p/go-libp2p#2862](https://github.com/libp2p/go-libp2p/pull/2862)) - peerstore: don't intern protocols (#2860) ([libp2p/go-libp2p#2860](https://github.com/libp2p/go-libp2p/pull/2860)) - autonatv2: add server metrics for dial requests (#2848) ([libp2p/go-libp2p#2848](https://github.com/libp2p/go-libp2p/pull/2848)) - PR Comments - Add a transport level test to ensure we close conns after rejecting them by the rcmgr - Close quic conns when wrapping conn fails - libp2p: use rcmgr for autonat dials (#2842) ([libp2p/go-libp2p#2842](https://github.com/libp2p/go-libp2p/pull/2842)) - metricshelper: improve checks for ip and transport (#2849) ([libp2p/go-libp2p#2849](https://github.com/libp2p/go-libp2p/pull/2849)) - Don't reuse the URL, make a new one - Use default transport to make using the Host cheaper - cleanup - Add future test - HTTP Host implements RoundTripper - swarm: improve dial worker performance for common case - pstoremanager: fix connectedness check - autonatv2: implement autonatv2 spec (#2469) ([libp2p/go-libp2p#2469](https://github.com/libp2p/go-libp2p/pull/2469)) - webrtc: add a test for establishing many connections (#2801) ([libp2p/go-libp2p#2801](https://github.com/libp2p/go-libp2p/pull/2801)) - webrtc: fix ufrag prefix for dialing (#2832) ([libp2p/go-libp2p#2832](https://github.com/libp2p/go-libp2p/pull/2832)) - circuitv2: improve voucher validation (#2826) ([libp2p/go-libp2p#2826](https://github.com/libp2p/go-libp2p/pull/2826)) - libp2phttp: workaround for ResponseWriter's CloseNotifier (#2821) ([libp2p/go-libp2p#2821](https://github.com/libp2p/go-libp2p/pull/2821)) - Update README.md (#2830) ([libp2p/go-libp2p#2830](https://github.com/libp2p/go-libp2p/pull/2830)) - identify: add test for observed address handling (#2828) ([libp2p/go-libp2p#2828](https://github.com/libp2p/go-libp2p/pull/2828)) - identify: fix bug in observed address handling (#2825) ([libp2p/go-libp2p#2825](https://github.com/libp2p/go-libp2p/pull/2825)) - identify: Don't filter addr if remote is neither public nor private (#2820) ([libp2p/go-libp2p#2820](https://github.com/libp2p/go-libp2p/pull/2820)) - limit ping duration to 30s (#1358) ([libp2p/go-libp2p#1358](https://github.com/libp2p/go-libp2p/pull/1358)) - Remove out-dated code in example readme (#2818) ([libp2p/go-libp2p#2818](https://github.com/libp2p/go-libp2p/pull/2818)) - v0.35.0 (#2812) ([libp2p/go-libp2p#2812](https://github.com/libp2p/go-libp2p/pull/2812)) - rcmgr: Support specific network prefix in conn limiter (#2807) ([libp2p/go-libp2p#2807](https://github.com/libp2p/go-libp2p/pull/2807)) - github.com/libp2p/go-libp2p-kad-dht (v0.25.2 -> v0.26.1): - Release v0.26.1 ([libp2p/go-libp2p-kad-dht#983](https://github.com/libp2p/go-libp2p-kad-dht/pull/983)) - fix: Unexport hasValidConnectedness to make a patch release ([libp2p/go-libp2p-kad-dht#982](https://github.com/libp2p/go-libp2p-kad-dht/pull/982)) - correctly merging fix from https://github.com/libp2p/go-libp2p-kad-dht/pull/976 ([libp2p/go-libp2p-kad-dht#980](https://github.com/libp2p/go-libp2p-kad-dht/pull/980)) - Release v0.26.0 ([libp2p/go-libp2p-kad-dht#979](https://github.com/libp2p/go-libp2p-kad-dht/pull/979)) - chore: update deps ([libp2p/go-libp2p-kad-dht#974](https://github.com/libp2p/go-libp2p-kad-dht/pull/974)) - Upgrade to go-log v2.5.1 ([libp2p/go-libp2p-kad-dht#971](https://github.com/libp2p/go-libp2p-kad-dht/pull/971)) - Fix: don't perform lookupCheck if not enough peers in routing table ([libp2p/go-libp2p-kad-dht#970](https://github.com/libp2p/go-libp2p-kad-dht/pull/970)) - findnode(self) should return multiple peers ([libp2p/go-libp2p-kad-dht#968](https://github.com/libp2p/go-libp2p-kad-dht/pull/968)) - github.com/libp2p/go-libp2p-routing-helpers (v0.7.3 -> v0.7.4): - chore: release v0.7.4 (#85) ([libp2p/go-libp2p-routing-helpers#85](https://github.com/libp2p/go-libp2p-routing-helpers/pull/85)) - fix: composable parallel router tracing by index (#84) ([libp2p/go-libp2p-routing-helpers#84](https://github.com/libp2p/go-libp2p-routing-helpers/pull/84)) - github.com/multiformats/go-multiaddr (v0.12.4 -> v0.13.0): - Release v0.13.0 ([multiformats/go-multiaddr#248](https://github.com/multiformats/go-multiaddr/pull/248)) - Add support for http-path ([multiformats/go-multiaddr#246](https://github.com/multiformats/go-multiaddr/pull/246)) - github.com/whyrusleeping/cbor-gen (v0.1.1 -> v0.1.2): - properly extend strings (#95) ([whyrusleeping/cbor-gen#95](https://github.com/whyrusleeping/cbor-gen/pull/95)) - ioutil to io (#98) ([whyrusleeping/cbor-gen#98](https://github.com/whyrusleeping/cbor-gen/pull/98))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Andrew Gillis | 14 | +4920/-1714 | 145 | | sukun | 26 | +4402/-448 | 79 | | Marco Munizaga | 32 | +2287/-536 | 73 | | Marcin Rataj | 41 | +685/-193 | 86 | | Patryk | 1 | +312/-10 | 8 | | guillaumemichel | 7 | +134/-105 | 14 | | Adin Schmahmann | 5 | +145/-80 | 9 | | Henrique Dias | 2 | +190/-1 | 6 | | Josh Klopfenstein | 1 | +90/-35 | 27 | | gammazero | 5 | +90/-28 | 11 | | Jeromy Johnson | 1 | +116/-0 | 5 | | Daniel N | 3 | +27/-25 | 9 | | Daniel Norman | 2 | +28/-19 | 4 | | Ivan Shvedunov | 2 | +25/-10 | 2 | | Michael Muré | 2 | +22/-9 | 4 | | Dominic Della Valle | 1 | +23/-4 | 1 | | Andrei Vukolov | 1 | +27/-0 | 1 | | chris erway | 1 | +9/-9 | 9 | | Vitaly Zdanevich | 1 | +12/-0 | 1 | | Guillaume Michel | 1 | +4/-7 | 1 | | swedneck | 1 | +10/-0 | 1 | | Jorropo | 2 | +5/-5 | 3 | | omahs | 1 | +4/-4 | 4 | | THAT ONE GUY | 1 | +3/-5 | 2 | | vyzo | 1 | +5/-2 | 1 | | looklose | 1 | +3/-3 | 2 | | web3-bot | 2 | +2/-3 | 4 | | Dave Huseby | 1 | +5/-0 | 1 | | shenpengfeng | 1 | +1/-1 | 1 | | bytetigers | 1 | +1/-1 | 1 | | Sorin Stanculeanu | 1 | +1/-1 | 1 | | Lukáš Lukáč | 1 | +1/-1 | 1 | | Gabe | 1 | +1/-1 | 1 | | Bryan Stenson | 1 | +1/-1 | 1 | | Samy Fodil | 1 | +1/-0 | 1 | | Lane Rettig | 1 | +1/-0 | 1 | ================================================ FILE: docs/changelogs/v0.31.md ================================================ # Kubo changelog v0.31 - [v0.31.0](#v0310) ## v0.31.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [Experimental Pebble Datastore](#experimental-pebble-datastore) - [New metrics](#new-metrics) - [`lowpower` profile no longer breaks DHT announcements](#lowpower-profile-no-longer-breaks-dht-announcements) - [go 1.23, boxo 0.24 and go-libp2p 0.36.5](#go-123-boxo-024-and-go-libp2p-0365) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### Experimental Pebble Datastore [Pebble](https://github.com/ipfs/kubo/blob/master/docs/config.md#pebbleds-profile) provides a high-performance alternative to leveldb as the datastore, and provides a modern replacement for [legacy badgerv1](https://github.com/ipfs/kubo/blob/master/docs/config.md#badgerds-profile). A fresh Kubo node can be initialized with [`pebbleds` profile](https://github.com/ipfs/kubo/blob/master/docs/config.md#pebbleds-profile) via `ipfs init --profile pebbleds`. There are a number of parameters available for tuning pebble's performance to your specific needs. Default values are used for any parameters that are not configured or are set to their zero-value. For a description of the available tuning parameters, see [kubo/docs/datastores.md#pebbleds](https://github.com/ipfs/kubo/blob/master/docs/datastores.md#pebbleds). #### New metrics - Added 3 new go metrics: `go_gc_gogc_percent`, `go_gc_gomemlimit_bytes` and `go_sched_gomaxprocs_threads` as those are [recommended by the Go team](https://github.com/prometheus/client_golang/pull/1559) - Added [network usage metrics](https://github.com/prometheus/client_golang/pull/1555): `process_network_receive_bytes_total` and `process_network_transmit_bytes_total` - Removed `go_memstat_lookups_total` metric [which was always 0](https://github.com/prometheus/client_golang/pull/1577) #### `lowpower` profile no longer breaks DHT announcements We've notices users were applying `lowpower` profile, and then reporting content routing issues. This was because `lowpower` disabled reprovider system and locally hosted data was no longer announced on Amino DHT. This release changes [`lowpower` profile](https://github.com/ipfs/kubo/blob/master/docs/config.md#lowpower-profile) to not change reprovider settings, ensuring the new users are not sabotaging themselves. It also adds [`announce-on`](https://github.com/ipfs/kubo/blob/master/docs/config.md#announce-on-profile) and [`announce-off`](https://github.com/ipfs/kubo/blob/master/docs/config.md#announce-off-profile) profiles for controlling announcement settings separately. > [!IMPORTANT] > If you've ever applied the `lowpower` profile before, there is a high chance your node is not announcing to DHT anymore. > If you have `Reprovider.Interval` set to `0` you may want to set it to `22h` (or run `ipfs config profile apply announce-on`) to fix your system. > > As a convenience, `ipfs daemon` will warn if reprovide system is disabled, creating oportinity to fix configuration if it was not intentional. #### go 1.23, boxo 0.24 and go-libp2p 0.36.5 Various bugfixes. Please update. ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - fix: go 1.23(.2) (#10540) ([ipfs/kubo#10540](https://github.com/ipfs/kubo/pull/10540)) - chore: bump version to 0.32.0-dev - feat(routing/http): support IPIP-484 and streaming (#10534) ([ipfs/kubo#10534](https://github.com/ipfs/kubo/pull/10534)) - fix(daemon): webui URL when rpc is catch-all (#10520) ([ipfs/kubo#10520](https://github.com/ipfs/kubo/pull/10520)) - chore: update changelog and config doc with more info about pebble (#10533) ([ipfs/kubo#10533](https://github.com/ipfs/kubo/pull/10533)) - feat: pebbleds profile and plugin (#10530) ([ipfs/kubo#10530](https://github.com/ipfs/kubo/pull/10530)) - chore: dependency updates for 0.31 (#10511) ([ipfs/kubo#10511](https://github.com/ipfs/kubo/pull/10511)) - feat: explicit announce-on/off profiles (#10524) ([ipfs/kubo#10524](https://github.com/ipfs/kubo/pull/10524)) - fix(core): look for MFS root in local repo only (#8661) ([ipfs/kubo#8661](https://github.com/ipfs/kubo/pull/8661)) - Fix issue in ResourceManager and nopfsPlugin about repo path (#10492) ([ipfs/kubo#10492](https://github.com/ipfs/kubo/pull/10492)) - feat(bitswap): allow configuring WithWantHaveReplaceSize (#10512) ([ipfs/kubo#10512](https://github.com/ipfs/kubo/pull/10512)) - refactor: simplify logic for MFS pinning (#10506) ([ipfs/kubo#10506](https://github.com/ipfs/kubo/pull/10506)) - docs: clarify Gateway.PublicGateways (#10525) ([ipfs/kubo#10525](https://github.com/ipfs/kubo/pull/10525)) - chore: clarify dep update in RELEASE_CHECKLIST.md (#10518) ([ipfs/kubo#10518](https://github.com/ipfs/kubo/pull/10518)) - feat: ipfs-webui v4.3.2 (#10523) ([ipfs/kubo#10523](https://github.com/ipfs/kubo/pull/10523)) - docs(config): add useful references - docs(config): improve profile descriptions (#10517) ([ipfs/kubo#10517](https://github.com/ipfs/kubo/pull/10517)) - docs: update RELEASE_CHECKLIST.md (#10496) ([ipfs/kubo#10496](https://github.com/ipfs/kubo/pull/10496)) - chore: create next changelog (#10510) ([ipfs/kubo#10510](https://github.com/ipfs/kubo/pull/10510)) - Merge Release: v0.30.0 [skip changelog] ([ipfs/kubo#10508](https://github.com/ipfs/kubo/pull/10508)) - chore: boxo v0.23.0 and go-libp2p v0.36.3 (#10507) ([ipfs/kubo#10507](https://github.com/ipfs/kubo/pull/10507)) - docs: replace outdated package paths described in rpc README (#10505) ([ipfs/kubo#10505](https://github.com/ipfs/kubo/pull/10505)) - fix: switch back to go 1.22 (#10502) ([ipfs/kubo#10502](https://github.com/ipfs/kubo/pull/10502)) - fix(cli): preserve hostname specified with --api in http request headers (#10497) ([ipfs/kubo#10497](https://github.com/ipfs/kubo/pull/10497)) - chore: upgrade to go 1.23 (#10486) ([ipfs/kubo#10486](https://github.com/ipfs/kubo/pull/10486)) - fix: error during config when running benchmarks (#10495) ([ipfs/kubo#10495](https://github.com/ipfs/kubo/pull/10495)) - chore: update go-unixfsnode, cmds, and boxo (#10494) ([ipfs/kubo#10494](https://github.com/ipfs/kubo/pull/10494)) - Docs fix spelling issues (#10493) ([ipfs/kubo#10493](https://github.com/ipfs/kubo/pull/10493)) - chore: update version (#10491) ([ipfs/kubo#10491](https://github.com/ipfs/kubo/pull/10491)) - github.com/ipfs/boxo (v0.23.0 -> v0.24.0): - Release v0.24.0 ([ipfs/boxo#683](https://github.com/ipfs/boxo/pull/683)) - github.com/ipfs/go-ipld-cbor (v0.1.0 -> v0.2.0): - v0.2.0 - deprecate DumpObject() in favor of better named Encode() - add an EncodeWriter method, using the pooled marshallers - fix expCid vs actualCid guard - github.com/ipld/go-car/v2 (v2.13.1 -> v2.14.2): - v2.14.2 bump - fix: goreleaser v2 compat, trigger release-binaries with workflow_run - v2.14.1 bump - chore: update fuzz to Go 1.22 - v2.14.0 bump - fix(cmd): properly pick up --inverse and --cid-file args ([ipld/go-car#531](https://github.com/ipld/go-car/pull/531)) - Re-factor cmd functions to library ([ipld/go-car#524](https://github.com/ipld/go-car/pull/524)) - ci: uci/copy-templates ([ipld/go-car#521](https://github.com/ipld/go-car/pull/521)) - Add a `car ls --unixfs-blocks` to render two-column output ([ipld/go-car#514](https://github.com/ipld/go-car/pull/514)) - github.com/libp2p/go-libp2p (v0.36.3 -> v0.36.5): - chore: remove Roadmap file (#2954) ([libp2p/go-libp2p#2954](https://github.com/libp2p/go-libp2p/pull/2954)) - fix: Release v0.36.5 - autonatv2: recover from panics (#2992) ([libp2p/go-libp2p#2992](https://github.com/libp2p/go-libp2p/pull/2992)) - basichost: ensure no duplicates in Addrs output (#2980) ([libp2p/go-libp2p#2980](https://github.com/libp2p/go-libp2p/pull/2980)) - Release v0.36.4 - peerstore: better GC in membacked peerstore (#2960) ([libp2p/go-libp2p#2960](https://github.com/libp2p/go-libp2p/pull/2960)) - fix: use quic.Version instead of the deprecated quic.VersionNumber (#2955) ([libp2p/go-libp2p#2955](https://github.com/libp2p/go-libp2p/pull/2955)) - tcp: fix metrics for multiple calls to Close (#2953) ([libp2p/go-libp2p#2953](https://github.com/libp2p/go-libp2p/pull/2953)) - github.com/libp2p/go-libp2p-kbucket (v0.6.3 -> v0.6.4): - release v0.6.4 ([libp2p/go-libp2p-kbucket#135](https://github.com/libp2p/go-libp2p-kbucket/pull/135)) - feat: add log printing when peer added and removed table ([libp2p/go-libp2p-kbucket#134](https://github.com/libp2p/go-libp2p-kbucket/pull/134)) - Upgrade to go-log v2.5.1 ([libp2p/go-libp2p-kbucket#132](https://github.com/libp2p/go-libp2p-kbucket/pull/132)) - chore: update go-libp2p-asn-util - github.com/multiformats/go-multiaddr-dns (v0.3.1 -> v0.4.0): - Release v0.4.0 (#64) ([multiformats/go-multiaddr-dns#64](https://github.com/multiformats/go-multiaddr-dns/pull/64)) - Limit total number of resolved addresses from DNS response (#63) ([multiformats/go-multiaddr-dns#63](https://github.com/multiformats/go-multiaddr-dns/pull/63)) - fix!: Only resolve the first DNS-like component (#61) ([multiformats/go-multiaddr-dns#61](https://github.com/multiformats/go-multiaddr-dns/pull/61)) - sync: update CI config files (#43) ([multiformats/go-multiaddr-dns#43](https://github.com/multiformats/go-multiaddr-dns/pull/43)) - remove deprecated types ([multiformats/go-multiaddr-dns#37](https://github.com/multiformats/go-multiaddr-dns/pull/37)) - remove Jenkinsfile ([multiformats/go-multiaddr-dns#40](https://github.com/multiformats/go-multiaddr-dns/pull/40)) - sync: update CI config files (#29) ([multiformats/go-multiaddr-dns#29](https://github.com/multiformats/go-multiaddr-dns/pull/29)) - use net.IP.Equal to compare IP addresses ([multiformats/go-multiaddr-dns#30](https://github.com/multiformats/go-multiaddr-dns/pull/30))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Will Scott | 3 | +731/-581 | 14 | | Daniel N | 17 | +1034/-191 | 33 | | Marco Munizaga | 5 | +721/-404 | 12 | | Andrew Gillis | 9 | +765/-266 | 35 | | Marcin Rataj | 17 | +568/-323 | 41 | | Daniel Norman | 3 | +232/-111 | 10 | | sukun | 4 | +93/-8 | 8 | | Jorropo | 2 | +48/-45 | 5 | | Marten Seemann | 3 | +19/-47 | 5 | | fengzie | 1 | +29/-26 | 5 | | Rod Vagg | 7 | +27/-11 | 9 | | gopherfarm | 1 | +14/-14 | 6 | | web3-bot | 3 | +13/-10 | 3 | | Michael Muré | 2 | +16/-5 | 4 | | i-norden | 1 | +9/-9 | 1 | | Elias Rad | 1 | +7/-7 | 4 | | Prithvi Shahi | 1 | +0/-11 | 2 | | Lucas Molas | 1 | +5/-4 | 1 | | elecbug | 1 | +6/-2 | 1 | | gammazero | 2 | +2/-2 | 2 | | chris erway | 1 | +2/-2 | 2 | | Russell Dempsey | 1 | +2/-1 | 1 | | guillaumemichel | 1 | +1/-1 | 1 | ================================================ FILE: docs/changelogs/v0.32.md ================================================ # Kubo changelog v0.32 - [v0.32.0](#v0320) ## v0.32.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [🎯 AutoTLS: Automatic Certificates for libp2p WebSockets via `libp2p.direct`](#-autotls-automatic-certificates-for-libp2p-websockets-via-libp2pdirect) - [📦️ Dependency updates](#-dependency-updates) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### 🎯 AutoTLS: Automatic Certificates for libp2p WebSockets via `libp2p.direct` This release introduces an experimental feature that significantly improves how browsers ([Helia](https://helia.io/), [Service Worker](https://inbrowser.link)) can connect to Kubo node. Opt-in configuration allows a publicly dialable Kubo nodes (public IP, port forwarding, or NAT with uPnP) to obtain CA-signed TLS certificates for [libp2p Secure WebSocket (WSS)](https://github.com/libp2p/specs/blob/master/websockets/README.md) connections automatically. > [!TIP] > To enable this feature, set `AutoTLS.Enabled` to `true` and add a listener for `/tls/sni/*.libp2p.direct/ws` on a separate TCP port: > ```diff > { > + "AutoTLS": { "Enabled": true }, > "Addresses": { > "Swarm": { > "/ip4/0.0.0.0/tcp/4001", > + "/ip4/0.0.0.0/tcp/4002/tls/sni/*.libp2p.direct/ws", > "/ip6/::/tcp/4001", > + "/ip6/::/tcp/4002/tls/sni/*.libp2p.direct/ws", > ``` > After restarting your node for the first time you may need to wait 5-15 minutes to pass all checks and for the changes to take effect. > We are working on sharing the same TCP port with other transports ([go-libp2p#2984](https://github.com/libp2p/go-libp2p/pull/2984)). See [`AutoTLS` configuration](https://github.com/ipfs/kubo/blob/master/docs/config.md#autotls) for more details how to enable it and what to expect. This is an early preview, we appreciate you testing and filling bug reports or feedback in the tracking issue at [kubo#10560](https://github.com/ipfs/kubo/issues/10560). #### 📦️ Dependency updates - update `ipfs-webui` to [v4.4.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.4.0) - update `boxo` to [v0.24.1](https://github.com/ipfs/boxo/releases/tag/v0.24.1) + [v0.24.2](https://github.com/ipfs/boxo/releases/tag/v0.24.2) + [v0.24.3](https://github.com/ipfs/boxo/releases/tag/v0.24.3) - This includes a number of fixes and bitswap improvements, and support for filtering from [IPIP-484](https://specs.ipfs.tech/ipips/ipip-0484/) in delegated HTTP routing and IPNI queries. - update `go-libp2p` to [v0.37.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.0) - This update required removal of `Swarm.RelayService.MaxReservationsPerPeer` configuration option from Kubo. If you had it set, remove it from your configuration file. - update `go-libp2p-kad-dht` to [v0.27.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.27.0) + [v0.28.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.28.0) + [v0.28.1](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.28.1) - update `go-libp2p-pubsub` to [v0.12.0](https://github.com/libp2p/go-libp2p-pubsub/releases/tag/v0.12.0) - update `p2p-forge/client` to [v0.0.2](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.0.2) - removed `go-homedir` - The `github.com/mitchellh/go-homedir` repo is archived, no longer needed, and no longer maintained. - `homedir.Dir` is replaced by the stdlib `os.UserHomeDir` - `homedir.Expand` is replaced by `fsutil.ExpandHome` in the `github.com/ipfs/kubo/misc/fsutil` package. - The new `github.com/ipfs/kubo/misc/fsutil` package contains file utility code previously located elsewhere in kubo. ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: 0.32.0 - fix: go-libp2p-kad-dht v0.28.0 (#10578) ([ipfs/kubo#10578](https://github.com/ipfs/kubo/pull/10578)) - chore: 0.32.0-rc2 - feat: ipfs-webui v4.4.0 (#10574) ([ipfs/kubo#10574](https://github.com/ipfs/kubo/pull/10574)) - chore: label implicit loggers - chore: boxo v0.24.3 and p2p-forge v0.0.2 (#10572) ([ipfs/kubo#10572](https://github.com/ipfs/kubo/pull/10572)) - chore: stop using go-homedir (#10568) ([ipfs/kubo#10568](https://github.com/ipfs/kubo/pull/10568)) - fix(autotls): store certificates at the location from the repo path (#10566) ([ipfs/kubo#10566](https://github.com/ipfs/kubo/pull/10566)) - chore: 0.32.0-rc1 - docs(autotls): add note about separate port use (#10562) ([ipfs/kubo#10562](https://github.com/ipfs/kubo/pull/10562)) - feat(AutoTLS): opt-in WSS certs from p2p-forge at libp2p.direct (#10521) ([ipfs/kubo#10521](https://github.com/ipfs/kubo/pull/10521)) - chore: upgrade to boxo v0.24.2 (#10559) ([ipfs/kubo#10559](https://github.com/ipfs/kubo/pull/10559)) - refactor: update to go-libp2p v0.37.0 (#10554) ([ipfs/kubo#10554](https://github.com/ipfs/kubo/pull/10554)) - docs(config): explain what multiaddr is - chore: update dependencies (#10548) ([ipfs/kubo#10548](https://github.com/ipfs/kubo/pull/10548)) - chore: update test dependencies (#10555) ([ipfs/kubo#10555](https://github.com/ipfs/kubo/pull/10555)) - chore(ci): adjust verbosity - chore(ci): verbose build of test/bin deps - chore(ci): build docker images for staging branch - Create Changelog: v0.32 ([ipfs/kubo#10546](https://github.com/ipfs/kubo/pull/10546)) - Merge release v0.31.0 ([ipfs/kubo#10545](https://github.com/ipfs/kubo/pull/10545)) - chore: update RELEASE_CHECKLIST.md (#10544) ([ipfs/kubo#10544](https://github.com/ipfs/kubo/pull/10544)) - feat: ipfs-webui v4.3.3 (#10543) ([ipfs/kubo#10543](https://github.com/ipfs/kubo/pull/10543)) - chore: update RELEASE_CHECKLIST.md (#10542) ([ipfs/kubo#10542](https://github.com/ipfs/kubo/pull/10542)) - Add full changelog to release changelog - fix: go 1.23(.2) (#10540) ([ipfs/kubo#10540](https://github.com/ipfs/kubo/pull/10540)) - chore: bump version to 0.32.0-dev - github.com/ipfs/boxo (v0.24.0 -> v0.24.3): - Release v0.24.3 ([ipfs/boxo#713](https://github.com/ipfs/boxo/pull/713)) - Merge branch 'main' into release - Release v0.24.2 ([ipfs/boxo#707](https://github.com/ipfs/boxo/pull/707)) - Release v0.24.1 ([ipfs/boxo#706](https://github.com/ipfs/boxo/pull/706)) - github.com/ipfs/go-ipfs-cmds (v0.13.0 -> v0.14.0): - chore: release v0.14.0 (#269) ([ipfs/go-ipfs-cmds#269](https://github.com/ipfs/go-ipfs-cmds/pull/269)) - github.com/ipfs/go-ipfs-redirects-file (v0.1.1 -> v0.1.2): - chore: v0.1.2 (#29) ([ipfs/go-ipfs-redirects-file#29](https://github.com/ipfs/go-ipfs-redirects-file/pull/29)) - docs(readme): refer specs and ipip - chore: update dependencies (#28) ([ipfs/go-ipfs-redirects-file#28](https://github.com/ipfs/go-ipfs-redirects-file/pull/28)) - github.com/ipfs/go-metrics-prometheus (v0.0.2 -> v0.0.3): - chore: release v0.0.3 (#24) ([ipfs/go-metrics-prometheus#24](https://github.com/ipfs/go-metrics-prometheus/pull/24)) - chore: update deps and update go-log to v2 (#23) ([ipfs/go-metrics-prometheus#23](https://github.com/ipfs/go-metrics-prometheus/pull/23)) - sync: update CI config files (#9) ([ipfs/go-metrics-prometheus#9](https://github.com/ipfs/go-metrics-prometheus/pull/9)) - github.com/ipfs/go-unixfsnode (v1.9.1 -> v1.9.2): - New release version ([ipfs/go-unixfsnode#78](https://github.com/ipfs/go-unixfsnode/pull/78)) - chore: update dependencies - github.com/libp2p/go-flow-metrics (v0.1.0 -> v0.2.0): - chore: release v0.2.0 (#33) ([libp2p/go-flow-metrics#33](https://github.com/libp2p/go-flow-metrics/pull/33)) - chore: cleanup readme (#31) ([libp2p/go-flow-metrics#31](https://github.com/libp2p/go-flow-metrics/pull/31)) - ci: uci/update-go ([libp2p/go-flow-metrics#27](https://github.com/libp2p/go-flow-metrics/pull/27)) - fix(ewma): reduce the chances of fake bandwidth spikes (#8) ([libp2p/go-flow-metrics#8](https://github.com/libp2p/go-flow-metrics/pull/8)) - chore: switch to typed atomics (#24) ([libp2p/go-flow-metrics#24](https://github.com/libp2p/go-flow-metrics/pull/24)) - test: use mock clocks for all tests (#25) ([libp2p/go-flow-metrics#25](https://github.com/libp2p/go-flow-metrics/pull/25)) - ci: uci/copy-templates ([libp2p/go-flow-metrics#21](https://github.com/libp2p/go-flow-metrics/pull/21)) - github.com/libp2p/go-libp2p (v0.36.5 -> v0.37.0): - Release v0.37.0 (#3013) ([libp2p/go-libp2p#3013](https://github.com/libp2p/go-libp2p/pull/3013)) - feat: Add WithFxOption (#2956) ([libp2p/go-libp2p#2956](https://github.com/libp2p/go-libp2p/pull/2956)) - chore: update imports to use slices package (#3007) ([libp2p/go-libp2p#3007](https://github.com/libp2p/go-libp2p/pull/3007)) - Change latency metrics buckets (#3012) ([libp2p/go-libp2p#3012](https://github.com/libp2p/go-libp2p/pull/3012)) - chore: bump deps in preparation for v0.37.0 (#3011) ([libp2p/go-libp2p#3011](https://github.com/libp2p/go-libp2p/pull/3011)) - autonat: fix interaction with autorelay (#2967) ([libp2p/go-libp2p#2967](https://github.com/libp2p/go-libp2p/pull/2967)) - swarm: add a peer dial latency metric (#2959) ([libp2p/go-libp2p#2959](https://github.com/libp2p/go-libp2p/pull/2959)) - peerstore: limit number of non connected peers in addrbook (#2971) ([libp2p/go-libp2p#2971](https://github.com/libp2p/go-libp2p/pull/2971)) - fix: swarm: refactor address resolution (#2990) ([libp2p/go-libp2p#2990](https://github.com/libp2p/go-libp2p/pull/2990)) - Add backoff for updating local IP addresses on error (#2999) ([libp2p/go-libp2p#2999](https://github.com/libp2p/go-libp2p/pull/2999)) - libp2phttp: HTTP Peer ID Authentication (#2854) ([libp2p/go-libp2p#2854](https://github.com/libp2p/go-libp2p/pull/2854)) - relay: make only 1 reservation per peer (#2974) ([libp2p/go-libp2p#2974](https://github.com/libp2p/go-libp2p/pull/2974)) - autonatv2: recover from panics (#2992) ([libp2p/go-libp2p#2992](https://github.com/libp2p/go-libp2p/pull/2992)) - basichost: ensure no duplicates in Addrs output (#2980) ([libp2p/go-libp2p#2980](https://github.com/libp2p/go-libp2p/pull/2980)) - fix(websocket): re-enable websocket transport test (#2987) ([libp2p/go-libp2p#2987](https://github.com/libp2p/go-libp2p/pull/2987)) - feat(websocket): switch the underlying http server logger to use ipfs/go-log (#2985) ([libp2p/go-libp2p#2985](https://github.com/libp2p/go-libp2p/pull/2985)) - peerstore: better GC in membacked peerstore (#2960) ([libp2p/go-libp2p#2960](https://github.com/libp2p/go-libp2p/pull/2960)) - connmgr: reduce log level for untagging untracked peers ([libp2p/go-libp2p#2961](https://github.com/libp2p/go-libp2p/pull/2961)) - fix: use quic.Version instead of the deprecated quic.VersionNumber (#2955) ([libp2p/go-libp2p#2955](https://github.com/libp2p/go-libp2p/pull/2955)) - tcp: fix metrics for multiple calls to Close (#2953) ([libp2p/go-libp2p#2953](https://github.com/libp2p/go-libp2p/pull/2953)) - chore: remove Roadmap file (#2954) ([libp2p/go-libp2p#2954](https://github.com/libp2p/go-libp2p/pull/2954)) - chore: add a funding JSON file to apply for Optimism rPGF round 5 (#2940) ([libp2p/go-libp2p#2940](https://github.com/libp2p/go-libp2p/pull/2940)) - Fix: WebSocket: Clone TLS config before creating a new listener - fix: enable dctur when interface address is public (#2931) ([libp2p/go-libp2p#2931](https://github.com/libp2p/go-libp2p/pull/2931)) - fix: QUIC/Webtransport Transports now will prefer their owned listeners for dialing out (#2936) ([libp2p/go-libp2p#2936](https://github.com/libp2p/go-libp2p/pull/2936)) - ci: uci/update-go (#2937) ([libp2p/go-libp2p#2937](https://github.com/libp2p/go-libp2p/pull/2937)) - fix: slice append value (#2938) ([libp2p/go-libp2p#2938](https://github.com/libp2p/go-libp2p/pull/2938)) - webrtc: wait for listener context before dropping connection (#2932) ([libp2p/go-libp2p#2932](https://github.com/libp2p/go-libp2p/pull/2932)) - ci: use go1.23, drop go1.21 (#2933) ([libp2p/go-libp2p#2933](https://github.com/libp2p/go-libp2p/pull/2933)) - Fail on any test timeout (#2929) ([libp2p/go-libp2p#2929](https://github.com/libp2p/go-libp2p/pull/2929)) - test: Try to fix test timeout (#2930) ([libp2p/go-libp2p#2930](https://github.com/libp2p/go-libp2p/pull/2930)) - ci: Out of the tarpit (#2923) ([libp2p/go-libp2p#2923](https://github.com/libp2p/go-libp2p/pull/2923)) - Make BlackHoleState type public (#2917) ([libp2p/go-libp2p#2917](https://github.com/libp2p/go-libp2p/pull/2917)) - Fix proto import paths (#2920) ([libp2p/go-libp2p#2920](https://github.com/libp2p/go-libp2p/pull/2920)) - github.com/libp2p/go-libp2p-kad-dht (v0.26.1 -> v0.28.0): - chore: release v0.28.0 (#998) ([libp2p/go-libp2p-kad-dht#998](https://github.com/libp2p/go-libp2p-kad-dht/pull/998)) - fix: set context timeout for `queryPeer` (#996) ([libp2p/go-libp2p-kad-dht#996](https://github.com/libp2p/go-libp2p-kad-dht/pull/996)) - refactor: document and expose Amino DHT defaults (#990) ([libp2p/go-libp2p-kad-dht#990](https://github.com/libp2p/go-libp2p-kad-dht/pull/990)) - Use timeout context for NewStream call ([libp2p/go-libp2p-kad-dht#994](https://github.com/libp2p/go-libp2p-kad-dht/pull/994)) - release v0.27.0 ([libp2p/go-libp2p-kad-dht#992](https://github.com/libp2p/go-libp2p-kad-dht/pull/992)) - Add new DHT option to provide custom pb.MessageSender ([libp2p/go-libp2p-kad-dht#991](https://github.com/libp2p/go-libp2p-kad-dht/pull/991)) - fix: replace deprecated Boxo function ([libp2p/go-libp2p-kad-dht#987](https://github.com/libp2p/go-libp2p-kad-dht/pull/987)) - fix(query): reverting changes on TestRTEvictionOnFailedQuery ([libp2p/go-libp2p-kad-dht#984](https://github.com/libp2p/go-libp2p-kad-dht/pull/984)) - github.com/libp2p/go-libp2p-pubsub (v0.11.0 -> v0.12.0): - chore: upgrade go-libp2p (#575) ([libp2p/go-libp2p-pubsub#575](https://github.com/libp2p/go-libp2p-pubsub/pull/575)) - GossipSub v1.2: IDONTWANT control message and priority queue. (#553) ([libp2p/go-libp2p-pubsub#553](https://github.com/libp2p/go-libp2p-pubsub/pull/553)) - Re-enable disabled gossipsub test (#566) ([libp2p/go-libp2p-pubsub#566](https://github.com/libp2p/go-libp2p-pubsub/pull/566)) - chore: staticcheck - chore: update rand usage - chore: go fmt - chore: add or force update version.json - added missing Close call on the AddrBook member of GossipSubRouter (#568) ([libp2p/go-libp2p-pubsub#568](https://github.com/libp2p/go-libp2p-pubsub/pull/568)) - test: test notify protocols updated (#567) ([libp2p/go-libp2p-pubsub#567](https://github.com/libp2p/go-libp2p-pubsub/pull/567)) - Switch to the new peer notify mechanism (#564) ([libp2p/go-libp2p-pubsub#564](https://github.com/libp2p/go-libp2p-pubsub/pull/564)) - test: use the regular libp2p host (#565) ([libp2p/go-libp2p-pubsub#565](https://github.com/libp2p/go-libp2p-pubsub/pull/565)) - Missing flood protection check for number of message IDs when handling `Ihave` messages (#560) ([libp2p/go-libp2p-pubsub#560](https://github.com/libp2p/go-libp2p-pubsub/pull/560))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Marco Munizaga | 16 | +4253/-545 | 81 | | Pop Chunhapanya | 1 | +1423/-137 | 15 | | sukun | 10 | +752/-425 | 35 | | Steven Allen | 11 | +518/-541 | 35 | | Andrew Gillis | 19 | +348/-194 | 50 | | Marcin Rataj | 26 | +343/-132 | 47 | | Adin Schmahmann | 4 | +269/-29 | 12 | | gammazero | 12 | +154/-18 | 13 | | Josh Klopfenstein | 1 | +90/-35 | 27 | | galargh | 3 | +42/-44 | 13 | | Daniel Norman | 2 | +30/-16 | 4 | | Mikel Cortes | 3 | +25/-4 | 4 | | gopherfarm | 1 | +14/-14 | 6 | | Carlos Peliciari | 1 | +12/-12 | 4 | | Prithvi Shahi | 2 | +5/-11 | 3 | | web3-bot | 6 | +12/-3 | 6 | | guillaumemichel | 3 | +7/-6 | 3 | | Jorropo | 1 | +11/-0 | 1 | | Sorin Stanculeanu | 1 | +8/-0 | 1 | | Hlib Kanunnikov | 2 | +6/-2 | 4 | | André Bierlein | 1 | +4/-3 | 1 | | bytetigers | 1 | +1/-1 | 1 | | Wondertan | 2 | +2/-0 | 2 | | Alexandr Burdiyan | 1 | +1/-1 | 1 | | Guillaume Michel | 1 | +0/-1 | 1 | ================================================ FILE: docs/changelogs/v0.33.md ================================================ # Kubo changelog v0.33 - [v0.33.0](#v0330) - [v0.33.1](#v0331) - [v0.33.2](#v0332) ## v0.33.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [Shared TCP listeners](#shared-tcp-listeners) - [AutoTLS takes care of Secure WebSockets setup](#autotls-takes-care-of-secure-websockets-setup) - [Bitswap improvements from Boxo](#bitswap-improvements-from-boxo) - [Using default `libp2p_rcmgr` metrics](#using-default-libp2p_rcmgr--metrics) - [Flatfs does not `sync` on each write](#flatfs-does-not-sync-on-each-write) - [`ipfs add --to-files` no longer works with `--wrap`](#ipfs-add---to-files-no-longer-works-with---wrap) - [`ipfs --api` supports HTTPS RPC endpoints](#ipfs---api-supports-https-rpc-endpoints) - [New options for faster writes: `WriteThrough`, `BlockKeyCacheSize`, `BatchMaxNodes`, `BatchMaxSize`](#new-options-for-faster-writes-writethrough-blockkeycachesize-batchmaxnodes-batchmaxsize) - [MFS stability with large number of writes](#mfs-stability-with-large-number-of-writes) - [New DoH resolvers for non-ICANN DNSLinks](#new-doh-resolvers-for-non-icann-dnslinks) - [Reliability improvements to the WebRTC Direct listener](#reliability-improvements-to-the-webrtc-direct-listener) - [Bitswap improvements from Boxo](#bitswap-improvements-from-boxo-1) - [📦️ Important dependency updates](#-important-dependency-updates) - [Escape Redirect URL for Directory](#escape-redirect-url-for-directory) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### Shared TCP listeners Kubo now supports sharing the same TCP port (`4001` by default) by both [raw TCP](https://github.com/ipfs/kubo/blob/master/docs/config.md#swarmtransportsnetworktcp) and [WebSockets](https://github.com/ipfs/kubo/blob/master/docs/config.md#swarmtransportsnetworkwebsocket) libp2p transports. This feature is not yet compatible with Private Networks and can be disabled by setting `LIBP2P_TCP_MUX=false` if causes any issues. #### AutoTLS takes care of Secure WebSockets setup It is no longer necessary to manually add `/tcp/../ws` listeners to `Addresses.Swarm` when [`AutoTLS.Enabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#autotlsenabled) is set to `true`. Kubo will detect if `/ws` listener is missing and add one on the same port as pre-existing TCP (e.g. `/tcp/4001`), removing the need for any extra configuration. > [!TIP] > Give it a try: > ```console > $ ipfs config --json AutoTLS.Enabled true > ``` > And restart the node. If you are behind NAT, make sure your node is publicly diallable (uPnP or port forwarding), and wait a few minutes to pass all checks and for the changes to take effect. See [`AutoTLS`](https://github.com/ipfs/kubo/blob/master/docs/config.md#autotls) for more information. #### Bitswap improvements from Boxo This release includes some refactorings and improvements affecting Bitswap which should improve reliability. One of the changes affects blocks providing. Previously, the bitswap layer took care itself of announcing new blocks -added or received- with the configured provider (i.e. DHT). This bypassed the "Reprovider", that is, the system that manages precisely "providing" the blocks stored by Kubo. The Reprovider knows how to take advantage of the [AcceleratedDHTClient](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient), is able to handle priorities, logs statistics and is able to resume on daemon reboot where it left off. From now on, Bitswap will not be doing any providing on-the-side and all announcements are managed by the reprovider. In some cases, when the reproviding queue is full with other elements, this may cause additional delays, but more likely this will result in improved block-providing behaviour overall. #### Using default `libp2p_rcmgr` metrics Bespoke rcmgr metrics [were removed](https://github.com/ipfs/kubo/pull/9947), Kubo now exposes only the default `libp2p_rcmgr` metrics from go-libp2p. This makes it easier to compare Kubo with custom implementations based on go-libp2p. If you depended on removed ones, please fill an issue to add them to the upstream [go-libp2p](https://github.com/libp2p/go-libp2p). #### Flatfs does not `sync` on each write New repositories initialized with `flatfs` in `Datastore.Spec` will have `sync` set to `false`. The old default was overly conservative and caused performance issues in big repositories that did a lot of writes. There is usually no need to flush on every block write to disk before continuing. Setting this to false is safe as kubo will automatically flush writes to disk before and after performing critical operations like pinning. However, we still provide users with ability to set this to true to be extra-safe (at the cost of a slowdown when adding files in bulk). #### `ipfs add --to-files` no longer works with `--wrap` Onboarding files and directories with `ipfs add --to-files` now requires non-empty names. due to this, The `--to-files` and `--wrap` options are now mutually exclusive ([#10612](https://github.com/ipfs/kubo/issues/10612)). #### `ipfs --api` supports HTTPS RPC endpoints CLI and RPC client now supports accessing Kubo RPC over `https://` protocol when multiaddr ending with `/https` or `/tls/http` is passed to `ipfs --api`: ```console $ ipfs id --api /dns/kubo-rpc.example.net/tcp/5001/tls/http # → https://kubo-rpc.example.net:5001 ``` #### New options for faster writes: `WriteThrough`, `BlockKeyCacheSize`, `BatchMaxNodes`, `BatchMaxSize` Now that Kubo supports [`pebble`](https://github.com/ipfs/kubo/blob/master/docs/datastores.md#pebbleds) as an _experimental_ datastore backend, it becomes very useful to expose some additional configuration options for how the blockservice/blockstore/datastore combo behaves. Usually, LSM-tree based datastore like Pebble or Badger have very fast write performance (blocks are streamed to disk) while incurring in read-amplification penalties (blocks need to be looked up in the index to know where they are on disk), specially noticeable on spinning disks. Prior to this version, `BlockService` and `Blockstore` implementations performed a `Has(cid)` for every block that was going to be written, skipping the writes altogether if the block was already present in the datastore. The performance impact of this `Has()` call can vary. The `Datastore` implementation itself might include block-caching and things like bloom-filters to speed up lookups and mitigate read-penalties. Our `Blockstore` implementation also supports a bloom-filter (controlled by `BloomFilterSize` and disabled by default), and a two-queue cache for keys and block sizes. If we assume that most of the blocks added to Kubo are new blocks, not already present in the datastore, or that the datastore itself includes mechanisms to optimize writes and avoid writing the same data twice, the calls to `Has()` at both BlockService and Blockstore layers seem superfluous to they point they even harm write performance. For these reasons, from now on, the default is to use a "write-through" mode for the Blockservice and the Blockstore. We have added a new option `Datastore.WriteThrough`, which defaults to `true`. Previous behaviour can be obtained by manually setting it to `false`. We have also made the size of the two-queue blockstore cache configurable with another option: `Datastore.BlockKeyCacheSize`, which defaults to `65536` (64KiB). Additionally, this caching layer can be disabled altogether by setting it to `0`. In particular, this option controls the size of a blockstore caching layer that records whether the blockstore has certain block and their sizes (but does not cache the contents, so it stays relativey small in general). Finally, we have added two new options to the `Import` section to control the maximum size of write-batches: `BatchMaxNodes` and `BatchMaxSize`. These are set by default to `128` nodes and `20MiB`. Increasing them will batch more items together when importing data with `ipfs dag import`, which can speed things up. It is importance to find a balance between available memory (used to hold the batch), disk latencies (when writing the batch) and processing power (when preparing the batch, as nodes are sorted and duplicates removed). As a reminder, details from all the options are explained in the [configuration documentation](https://github.com/ipfs/kubo/blob/master/docs/config.md). We recommend users trying Pebble as a datastore backend to disable both blockstore bloom-filter and key caching layers and enable write through as a way to evaluate the raw performance of the underlying datastore, which includes its own bloom-filter and caching layers (default cache size is `8MiB` and can be configured in the [options](https://github.com/ipfs/kubo/blob/master/docs/datastores.md#pebbleds). #### MFS stability with large number of writes We have fixed a number of issues that were triggered by writing or copying many files onto an MFS folder: increased memory usage first, then CPU, disk usage, and eventually a deadlock on write operations. The details of the fixes can be read at [#10630](https://github.com/ipfs/kubo/pull/10630) and [#10623](https://github.com/ipfs/kubo/pull/10623). The result is that writing large amounts of files to an MFS folder should now be possible without major issues. It is possible, as before, to speed up the operations using the `ipfs files --flush=false ...` flag, but it is recommended to switch to `ipfs files --flush=true ...` regularly, or call `ipfs files flush` on the working directory regularly, as this will flush, clear the directory cache and speed up reads. #### New DoH resolvers for non-ICANN DNSLinks - `.eth` TLD DNSLinks are now resolved via [DNS-over-HTTPS](https://en.wikipedia.org/wiki/DNS_over_HTTPS) endpoint at `https://dns.eth.limo/dns-query` - `.crypto` TLD DNSLinks are now resolved via DoH endpoint at `https://resolver.unstoppable.io/dns-query` #### Reliability improvements to the WebRTC Direct listener Two fixes in go-libp2p improve the reliability of the WebRTC Direct listener in Kubo, and by extension dialability from browsers. Relevant changes in go-libp2p: - [Deprioritising outgoing `/webrtc-direct`](https://github.com/libp2p/go-libp2p/pull/3078) dials. - [Allows more concurrent handshakes by default](https://github.com/libp2p/go-libp2p/pull/3040/). #### Bitswap improvements from Boxo This release includes performance and reliability improvements and fixes for minor resource leaks. #### 📦️ Important dependency updates - update `boxo` to [v0.27.4](https://github.com/ipfs/boxo/releases/tag/v0.27.4) (incl. [v0.25.0](https://github.com/ipfs/boxo/releases/tag/v0.25.0) + [v0.26.0](https://github.com/ipfs/boxo/releases/tag/v0.26.0) + [v0.27.0](https://github.com/ipfs/boxo/releases/tag/v0.27.0) + [v0.27.1](https://github.com/ipfs/boxo/releases/tag/v0.27.1) + [v0.27.2](https://github.com/ipfs/boxo/releases/tag/v0.27.2) + [v0.27.3](https://github.com/ipfs/boxo/releases/tag/v0.27.3)) - update `go-libp2p` to [v0.38.2](https://github.com/libp2p/go-libp2p/releases/tag/v0.38.2) (incl. [v0.37.1](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.1) + [v0.37.2](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.2) + [v0.38.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.38.0) + [v0.38.1](https://github.com/libp2p/go-libp2p/releases/tag/v0.38.1)) - update `go-libp2p-kad-dht` to [v0.28.2](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.28.2) - update `quic-go` to [v0.49.0](https://github.com/quic-go/quic-go/releases/tag/v0.49.0) - update `p2p-forge/client` to [v0.3.0](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.3.0) (incl. [v0.1.0](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.1.0), [v0.2.0](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.2.0), [v0.2.1](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.2.1), [v0.2.2](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.2.2)) - update `ipfs-webui` to [v4.4.2](https://github.com/ipfs/ipfs-webui/releases/tag/v4.4.2) (incl. [v4.4.1](https://github.com/ipfs/ipfs-webui/releases/tag/v4.4.1)) #### Escape Redirect URL for Directory When navigating to a subdirectory, served by the Kubo web server, a subdirectory without a trailing slash gets redirected to a URL with a trailing slash. If there are special characters such as "%" in the subdirectory name then these must be escaped in the redirect URL. Previously this was not being done and was preventing navigation to such subdirectories, requiring the user to manually add a trailing slash to the subdirectory URL. This is now fixed to handle the redirect to URLs with characters that must be escaped. ### 📝 Changelog
Full Changelog v0.33.0 - github.com/ipfs/kubo: - test: fix the socat tests after the ubuntu 24.04 upgrade (#10683) ([ipfs/kubo#10683](https://github.com/ipfs/kubo/pull/10683)) - chore: 0.33.0-rc3 - fix: quic-go v0.49.0 (#10673) ([ipfs/kubo#10673](https://github.com/ipfs/kubo/pull/10673)) - Upgrade to Boxo v0.27.2 (#10672) ([ipfs/kubo#10672](https://github.com/ipfs/kubo/pull/10672)) - chore: 0.33.0-rc2 - Upgrade to Boxo v0.27.1 (#10671) ([ipfs/kubo#10671](https://github.com/ipfs/kubo/pull/10671)) - fix(autotls): renewal and AutoTLS.ShortAddrs (#10669) ([ipfs/kubo#10669](https://github.com/ipfs/kubo/pull/10669)) - update changelog for boxo and go-libp2p (#10668) ([ipfs/kubo#10668](https://github.com/ipfs/kubo/pull/10668)) - Upgrade to Boxo v0.27.0 (#10665) ([ipfs/kubo#10665](https://github.com/ipfs/kubo/pull/10665)) - update dependencies (#10664) ([ipfs/kubo#10664](https://github.com/ipfs/kubo/pull/10664)) - fix(dns): update default DNSLink resolvers (#10655) ([ipfs/kubo#10655](https://github.com/ipfs/kubo/pull/10655)) - chore: p2p-forge v0.2.2 + go-libp2p-kad-dht v0.28.2 (#10663) ([ipfs/kubo#10663](https://github.com/ipfs/kubo/pull/10663)) - fix(cli): support HTTPS in ipfs --api (#10659) ([ipfs/kubo#10659](https://github.com/ipfs/kubo/pull/10659)) - chore: fix typos and comment formatting (#10653) ([ipfs/kubo#10653](https://github.com/ipfs/kubo/pull/10653)) - fix/gateway: escape directory redirect url (#10649) ([ipfs/kubo#10649](https://github.com/ipfs/kubo/pull/10649)) - Add example of setting array to config command help - collection of typo fixes (#10647) ([ipfs/kubo#10647](https://github.com/ipfs/kubo/pull/10647)) - chore: 0.33.0-rc1 - fix: ipfs-webui v4.4.2 (#10635) ([ipfs/kubo#10635](https://github.com/ipfs/kubo/pull/10635)) - feat(libp2p): shared TCP listeners and AutoTLS.AutoWSS (#10565) ([ipfs/kubo#10565](https://github.com/ipfs/kubo/pull/10565)) - feat(flatfs): default to sync=false (#10632) ([ipfs/kubo#10632](https://github.com/ipfs/kubo/pull/10632)) - Minor spelling and wording changes (#10634) ([ipfs/kubo#10634](https://github.com/ipfs/kubo/pull/10634)) - docs: clarify Swarm.ResourceMgr.MaxMemory (#10622) ([ipfs/kubo#10622](https://github.com/ipfs/kubo/pull/10622)) - feat: expose BlockKeyCacheSize and enable WriteThrough datastore options (#10614) ([ipfs/kubo#10614](https://github.com/ipfs/kubo/pull/10614)) - cmd/files: flush parent folders (#10630) ([ipfs/kubo#10630](https://github.com/ipfs/kubo/pull/10630)) - Upgrade to Boxo v0.26.0 (#10631) ([ipfs/kubo#10631](https://github.com/ipfs/kubo/pull/10631)) - [skip changelog] pinmfs: mitigate slow mfs writes when it triggers (#10623) ([ipfs/kubo#10623](https://github.com/ipfs/kubo/pull/10623)) - chore: use errors.New to replace fmt.Errorf with no parameters (#10617) ([ipfs/kubo#10617](https://github.com/ipfs/kubo/pull/10617)) - chore: boxo v0.25.0 (#10619) ([ipfs/kubo#10619](https://github.com/ipfs/kubo/pull/10619)) - fix(cmds/add): disallow --wrap with --to-files (#10612) ([ipfs/kubo#10612](https://github.com/ipfs/kubo/pull/10612)) - refactor(cmds): do not return errors embedded in result type (#10527) ([ipfs/kubo#10527](https://github.com/ipfs/kubo/pull/10527)) - fix: ipfs-webui v4.4.1 (#10608) ([ipfs/kubo#10608](https://github.com/ipfs/kubo/pull/10608)) - chore: fix broken url in comment (#10606) ([ipfs/kubo#10606](https://github.com/ipfs/kubo/pull/10606)) - refactor(rcmgr): use default libp2p rcmgr metrics (#9947) ([ipfs/kubo#9947](https://github.com/ipfs/kubo/pull/9947)) - docs(changelog/v0.33): bitswap reprovide changes (#10604) ([ipfs/kubo#10604](https://github.com/ipfs/kubo/pull/10604)) - tests(cli/harness): use unused Verbose flag to pipe daemon outputs (#10601) ([ipfs/kubo#10601](https://github.com/ipfs/kubo/pull/10601)) - chore: p2p-forge/client v0.1.0 (#10605) ([ipfs/kubo#10605](https://github.com/ipfs/kubo/pull/10605)) - fix: go-libp2p v0.37.2 (#10603) ([ipfs/kubo#10603](https://github.com/ipfs/kubo/pull/10603)) - docs: typos (#10602) ([ipfs/kubo#10602](https://github.com/ipfs/kubo/pull/10602)) - tests/cli: fix flapping tests (#10600) ([ipfs/kubo#10600](https://github.com/ipfs/kubo/pull/10600)) - Update to boxo with refactored providerQueryManager. (#10595) ([ipfs/kubo#10595](https://github.com/ipfs/kubo/pull/10595)) - fix some typos in docs (#10598) ([ipfs/kubo#10598](https://github.com/ipfs/kubo/pull/10598)) - feat(bootstrap): add JS-based va1.bootstrap.libp2p.io (#10575) ([ipfs/kubo#10575](https://github.com/ipfs/kubo/pull/10575)) - fix: increase provider sample size (#10589) ([ipfs/kubo#10589](https://github.com/ipfs/kubo/pull/10589)) - Typos Update config.md (#10591) ([ipfs/kubo#10591](https://github.com/ipfs/kubo/pull/10591)) - refactor: update to boxo without goprocess (#10567) ([ipfs/kubo#10567](https://github.com/ipfs/kubo/pull/10567)) - fix: go-libp2p-kad-dht v0.28.1 (#10581) ([ipfs/kubo#10581](https://github.com/ipfs/kubo/pull/10581)) - docs: update RELEASE_CHECKLIST.md (#10564) ([ipfs/kubo#10564](https://github.com/ipfs/kubo/pull/10564)) - Merge release v0.32.0 ([ipfs/kubo#10579](https://github.com/ipfs/kubo/pull/10579)) - fix: go-libp2p-kad-dht v0.28.0 (#10578) ([ipfs/kubo#10578](https://github.com/ipfs/kubo/pull/10578)) - feat: ipfs-webui v4.4.0 (#10574) ([ipfs/kubo#10574](https://github.com/ipfs/kubo/pull/10574)) - chore: boxo v0.24.3 and p2p-forge v0.0.2 (#10572) ([ipfs/kubo#10572](https://github.com/ipfs/kubo/pull/10572)) - chore: stop using go-homedir (#10568) ([ipfs/kubo#10568](https://github.com/ipfs/kubo/pull/10568)) - fix(autotls): store certificates at the location from the repo path (#10566) ([ipfs/kubo#10566](https://github.com/ipfs/kubo/pull/10566)) - chore: bump master to 0.33.0-dev - github.com/ipfs-shipyard/nopfs (v0.0.12 -> v0.0.14): - Fix error when no doublehash db exists (#42) ([ipfs-shipyard/nopfs#42](https://github.com/ipfs-shipyard/nopfs/pull/42)) - Improve support for IPNS double-hashed entries (#41) ([ipfs-shipyard/nopfs#41](https://github.com/ipfs-shipyard/nopfs/pull/41)) - github.com/ipfs-shipyard/nopfs/ipfs (v0.13.2-0.20231027223058-cde3b5ba964c -> v0.25.0): failed to fetch repo - github.com/ipfs/boxo (v0.24.3 -> v0.27.2): - Release v0.27.2 ([ipfs/boxo#811](https://github.com/ipfs/boxo/pull/811)) - Revert peer exclude cancel ([ipfs/boxo#809](https://github.com/ipfs/boxo/pull/809)) - Release v0.27.1 ([ipfs/boxo#807](https://github.com/ipfs/boxo/pull/807)) - fix sending cancels when excluding peer ([ipfs/boxo#805](https://github.com/ipfs/boxo/pull/805)) - Release v0.27.0 ([ipfs/boxo#802](https://github.com/ipfs/boxo/pull/802)) - Remove want-block sent tracking from sessionWantSender (#759) ([ipfs/boxo#759](https://github.com/ipfs/boxo/pull/759)) - Upgrade to go-libp2p v0.38.2 (#804) ([ipfs/boxo#804](https://github.com/ipfs/boxo/pull/804)) - [skip changelog] Use routing.ContentRouting interface (#803) ([ipfs/boxo#803](https://github.com/ipfs/boxo/pull/803)) - fix potential crash in unixfs directory (#798) ([ipfs/boxo#798](https://github.com/ipfs/boxo/pull/798)) - prefer slices.SortFunc to sort.Sort (#796) ([ipfs/boxo#796](https://github.com/ipfs/boxo/pull/796)) - fix: ipns protobuf namespace conflict (#794) ([ipfs/boxo#794](https://github.com/ipfs/boxo/pull/794)) - update release procedure (#773) ([ipfs/boxo#773](https://github.com/ipfs/boxo/pull/773)) - reduce default number of routing in-process requests (#793) ([ipfs/boxo#793](https://github.com/ipfs/boxo/pull/793)) - Do not return unused values from wantlists (#792) ([ipfs/boxo#792](https://github.com/ipfs/boxo/pull/792)) - Create FUNDING.json [skip changelog] (#795) ([ipfs/boxo#795](https://github.com/ipfs/boxo/pull/795)) - refactor: using slices.Contains to simplify the code (#791) ([ipfs/boxo#791](https://github.com/ipfs/boxo/pull/791)) - do not send cancel message to peer that sent block (#784) ([ipfs/boxo#784](https://github.com/ipfs/boxo/pull/784)) - Define a `go_package` for protobuf, rename to a more unique `ipns-record.proto` ([ipfs/boxo#789](https://github.com/ipfs/boxo/pull/789)) - bitswap: messagequeue: lock only needed sections (#787) ([ipfs/boxo#787](https://github.com/ipfs/boxo/pull/787)) - Update libp2p-kad-dht to v0.28.2 (#786) ([ipfs/boxo#786](https://github.com/ipfs/boxo/pull/786)) - feat(gateway): allow localhost http:// DoH resolvers (#645) ([ipfs/boxo#645](https://github.com/ipfs/boxo/pull/645)) - fix(gateway): update DoH resolver for .crypto DNSLink (#782) ([ipfs/boxo#782](https://github.com/ipfs/boxo/pull/782)) - fix(gateway): update DoH resolver for .eth DNSLink (#781) ([ipfs/boxo#781](https://github.com/ipfs/boxo/pull/781)) - chore: pass options to tracer start (#775) ([ipfs/boxo#775](https://github.com/ipfs/boxo/pull/775)) - escape redirect urls (#783) ([ipfs/boxo#783](https://github.com/ipfs/boxo/pull/783)) - fix/gateway: escape directory redirect url (#779) ([ipfs/boxo#779](https://github.com/ipfs/boxo/pull/779)) - fix spelling in comments (#778) ([ipfs/boxo#778](https://github.com/ipfs/boxo/pull/778)) - trivial spelling changes in comments (#777) ([ipfs/boxo#777](https://github.com/ipfs/boxo/pull/777)) - Release v0.26.0 ([ipfs/boxo#770](https://github.com/ipfs/boxo/pull/770)) - Minor spelling and wording changes (#768) ([ipfs/boxo#768](https://github.com/ipfs/boxo/pull/768)) - update go-libp2p and go-libp2p-kad-dht ([ipfs/boxo#767](https://github.com/ipfs/boxo/pull/767)) - [skip changelog] fix: Drop stream references on Close/Reset ([ipfs/boxo#760](https://github.com/ipfs/boxo/pull/760)) - Update go-libp2p to v0.38.0 (#764) ([ipfs/boxo#764](https://github.com/ipfs/boxo/pull/764)) - Fix leak due to cid queue never getting cleaned up (#756) ([ipfs/boxo#756](https://github.com/ipfs/boxo/pull/756)) - Do not reset the broadcast timer if there are no wants (#758) ([ipfs/boxo#758](https://github.com/ipfs/boxo/pull/758)) - Replace mock time implementation (#762) ([ipfs/boxo#762](https://github.com/ipfs/boxo/pull/762)) - mfs: clean cache on sync ([ipfs/boxo#751](https://github.com/ipfs/boxo/pull/751)) - Remove peer's count of first responses when peer becomes unavailable (#757) ([ipfs/boxo#757](https://github.com/ipfs/boxo/pull/757)) - Remove unnecessary CID copying in SessionInterestManager (#761) ([ipfs/boxo#761](https://github.com/ipfs/boxo/pull/761)) - [bitswap/peermanager] take read-lock for read-only operation (#755) ([ipfs/boxo#755](https://github.com/ipfs/boxo/pull/755)) - bitswap/client/messagequeue: expose dontHaveTimeoutMgr configuration (#750) ([ipfs/boxo#750](https://github.com/ipfs/boxo/pull/750)) - improve mfs republisher (#754) ([ipfs/boxo#754](https://github.com/ipfs/boxo/pull/754)) - blockstore/blockservice: change option to `WriteThrough(enabled bool)` ([ipfs/boxo#749](https://github.com/ipfs/boxo/pull/749)) - Merge release v0.25.0 ([ipfs/boxo#748](https://github.com/ipfs/boxo/pull/748)) - Use deque instead of slice for queues (#742) ([ipfs/boxo#742](https://github.com/ipfs/boxo/pull/742)) - chore: no lifecycle context to shutdown ProviderQueryManager (#734) ([ipfs/boxo#734](https://github.com/ipfs/boxo/pull/734)) - removed Startup function from ProviderQueryManager (#741) ([ipfs/boxo#741](https://github.com/ipfs/boxo/pull/741)) - Re-enable flaky bitswap tests (#740) ([ipfs/boxo#740](https://github.com/ipfs/boxo/pull/740)) - feat(session): do not record erroneous session want sends (#452) ([ipfs/boxo#452](https://github.com/ipfs/boxo/pull/452)) - feat(filestore): add mmap reader option (#665) ([ipfs/boxo#665](https://github.com/ipfs/boxo/pull/665)) - chore: update to latest go-libp2p (#739) ([ipfs/boxo#739](https://github.com/ipfs/boxo/pull/739)) - refactor(remote/pinning): `Ls` to take results channel instead of returning one (#738) ([ipfs/boxo#738](https://github.com/ipfs/boxo/pull/738)) - Bitswap default ProviderQueryManager uses explicit options (#737) ([ipfs/boxo#737](https://github.com/ipfs/boxo/pull/737)) - chore: minor examples cleanup (#736) ([ipfs/boxo#736](https://github.com/ipfs/boxo/pull/736)) - misc comments and spelling (#735) ([ipfs/boxo#735](https://github.com/ipfs/boxo/pull/735)) - chore: fix invalid url in docs (#733) ([ipfs/boxo#733](https://github.com/ipfs/boxo/pull/733)) - [skip changelog] bitswap/client: fix wiring when passing custom providerFinder ([ipfs/boxo#732](https://github.com/ipfs/boxo/pull/732)) - Add debug logging for deduplicated queries (#729) ([ipfs/boxo#729](https://github.com/ipfs/boxo/pull/729)) - [skip changelog] staticcheck fixes / remove unused variables (#730) ([ipfs/boxo#730](https://github.com/ipfs/boxo/pull/730)) - refactor: default to prometheus.DefaultRegisterer (#722) ([ipfs/boxo#722](https://github.com/ipfs/boxo/pull/722)) - chore: minor Improvements to providerquerymanager (#728) ([ipfs/boxo#728](https://github.com/ipfs/boxo/pull/728)) - dspinner: RecursiveKeys(): do not hang on cancellations (#727) ([ipfs/boxo#727](https://github.com/ipfs/boxo/pull/727)) - Tests can signal immediate rebroadcast (#726) ([ipfs/boxo#726](https://github.com/ipfs/boxo/pull/726)) - fix(bitswap/client/msgq): prevent duplicate requests (#691) ([ipfs/boxo#691](https://github.com/ipfs/boxo/pull/691)) - Bitswap: move providing -> Exchange-layer, providerQueryManager -> routing (#641) ([ipfs/boxo#641](https://github.com/ipfs/boxo/pull/641)) - fix(bitswap/client/providerquerymanager): don't end trace span until … (#725) ([ipfs/boxo#725](https://github.com/ipfs/boxo/pull/725)) - fix(routing/http/server): adjust bucket sizes for http metrics ([ipfs/boxo#724](https://github.com/ipfs/boxo/pull/724)) - fix(bitswap/client/providerquerymanager): use non-timed out context for tracing (#721) ([ipfs/boxo#721](https://github.com/ipfs/boxo/pull/721)) - fix(bitswap/server): pass context to server engine to register metrics (#723) ([ipfs/boxo#723](https://github.com/ipfs/boxo/pull/723)) - docs: fix url of tracing env vars (#719) ([ipfs/boxo#719](https://github.com/ipfs/boxo/pull/719)) - feat(routing/http/server): add routing timeout (#720) ([ipfs/boxo#720](https://github.com/ipfs/boxo/pull/720)) - feat(routing/http/server): expose prometheus metrics (#718) ([ipfs/boxo#718](https://github.com/ipfs/boxo/pull/718)) - Remove dependency on goprocess ([ipfs/boxo#710](https://github.com/ipfs/boxo/pull/710)) - Merge release v0.24.3 ([ipfs/boxo#714](https://github.com/ipfs/boxo/pull/714)) - fix(bitswap): log unexpected blocks to debug level (#711) ([ipfs/boxo#711](https://github.com/ipfs/boxo/pull/711)) - Release v0.24.2 ([ipfs/boxo#708](https://github.com/ipfs/boxo/pull/708)) - github.com/ipfs/go-ds-pebble (v0.4.0 -> v0.4.2): - new version (#44) ([ipfs/go-ds-pebble#44](https://github.com/ipfs/go-ds-pebble/pull/44)) - new version for pebble minor version update (#42) ([ipfs/go-ds-pebble#42](https://github.com/ipfs/go-ds-pebble/pull/42)) - github.com/ipfs/go-ipfs-cmds (v0.14.0 -> v0.14.1): - fix(NewClient): support https:// URLs (#277) ([ipfs/go-ipfs-cmds#277](https://github.com/ipfs/go-ipfs-cmds/pull/277)) - github.com/ipfs/go-peertaskqueue (v0.8.1 -> v0.8.2): - new version ([ipfs/go-peertaskqueue#39](https://github.com/ipfs/go-peertaskqueue/pull/39)) - Replace mock time implementation ([ipfs/go-peertaskqueue#37](https://github.com/ipfs/go-peertaskqueue/pull/37)) - fix: staticcheck feedback - github.com/libp2p/go-doh-resolver (v0.4.0 -> v0.5.0): - chore: release v0.5.0 - fix: include url on HTTP error (#29) ([libp2p/go-doh-resolver#29](https://github.com/libp2p/go-doh-resolver/pull/29)) - feat: allow localhost http endpoints (#28) ([libp2p/go-doh-resolver#28](https://github.com/libp2p/go-doh-resolver/pull/28)) - sync: update CI config files (#20) ([libp2p/go-doh-resolver#20](https://github.com/libp2p/go-doh-resolver/pull/20)) - github.com/libp2p/go-libp2p (v0.37.0 -> v0.38.2): - Release v0.38.2 (#3147) ([libp2p/go-libp2p#3147](https://github.com/libp2p/go-libp2p/pull/3147)) - chore: release v0.38.1 - fix(httpauth): Correctly handle concurrent requests on server (#3111) ([libp2p/go-libp2p#3111](https://github.com/libp2p/go-libp2p/pull/3111)) - ci: Install specific protoc version when generating protobufs (#3112) ([libp2p/go-libp2p#3112](https://github.com/libp2p/go-libp2p/pull/3112)) - fix(autorelay): Move relayFinder peer disconnect cleanup to separate goroutine (#3105) ([libp2p/go-libp2p#3105](https://github.com/libp2p/go-libp2p/pull/3105)) - chore: Release v0.38.0 (#3106) ([libp2p/go-libp2p#3106](https://github.com/libp2p/go-libp2p/pull/3106)) - peerstore: remove sync.Pool for expiringAddrs (#3093) ([libp2p/go-libp2p#3093](https://github.com/libp2p/go-libp2p/pull/3093)) - webtransport: close quic conn on dial error (#3104) ([libp2p/go-libp2p#3104](https://github.com/libp2p/go-libp2p/pull/3104)) - peerstore: fix addressbook benchmark timing (#3092) ([libp2p/go-libp2p#3092](https://github.com/libp2p/go-libp2p/pull/3092)) - swarm: record conn metrics only once (#3091) ([libp2p/go-libp2p#3091](https://github.com/libp2p/go-libp2p/pull/3091)) - fix(sampledconn): Correctly handle slow bytes and closed conns (#3080) ([libp2p/go-libp2p#3080](https://github.com/libp2p/go-libp2p/pull/3080)) - peerstore: pass options to addrbook constructor (#3090) ([libp2p/go-libp2p#3090](https://github.com/libp2p/go-libp2p/pull/3090)) - fix(swarm): remove stray print stmt (#3086) ([libp2p/go-libp2p#3086](https://github.com/libp2p/go-libp2p/pull/3086)) - feat(swarm): delay /webrtc-direct dials by 1 second (#3078) ([libp2p/go-libp2p#3078](https://github.com/libp2p/go-libp2p/pull/3078)) - chore: Update dependencies and fix deprecated function in relay example (#3023) ([libp2p/go-libp2p#3023](https://github.com/libp2p/go-libp2p/pull/3023)) - chore: fix broken link to record envelope protobuf file (#3070) ([libp2p/go-libp2p#3070](https://github.com/libp2p/go-libp2p/pull/3070)) - chore(core): fix function name in interface comment (#3056) ([libp2p/go-libp2p#3056](https://github.com/libp2p/go-libp2p/pull/3056)) - basichost: avoid modifying slice returned by AddrsFactory (#3068) ([libp2p/go-libp2p#3068](https://github.com/libp2p/go-libp2p/pull/3068)) - fix(swarm): check after we split for empty multiaddr (#3063) ([libp2p/go-libp2p#3063](https://github.com/libp2p/go-libp2p/pull/3063)) - feat: allow passing options to memoryAddrBook (#3062) ([libp2p/go-libp2p#3062](https://github.com/libp2p/go-libp2p/pull/3062)) - fix(libp2phttp): Return ErrServerClosed on Close (#3050) ([libp2p/go-libp2p#3050](https://github.com/libp2p/go-libp2p/pull/3050)) - chore(dashboard/alertmanager): update api version from v1 to v2 (#3054) ([libp2p/go-libp2p#3054](https://github.com/libp2p/go-libp2p/pull/3054)) - fix(tcpreuse): handle connection that failed to be sampled (#3036) ([libp2p/go-libp2p#3036](https://github.com/libp2p/go-libp2p/pull/3036)) - fix(tcpreuse): remove windows specific code (#3039) ([libp2p/go-libp2p#3039](https://github.com/libp2p/go-libp2p/pull/3039)) - refactor(libp2phttp): don't require specific port for the HTTP host example (#3047) ([libp2p/go-libp2p#3047](https://github.com/libp2p/go-libp2p/pull/3047)) - refactor(core/routing): split ContentRouting interface (#3048) ([libp2p/go-libp2p#3048](https://github.com/libp2p/go-libp2p/pull/3048)) - fix(holepunch/tracer): replace inline peer struct with peerInfo type (#3049) ([libp2p/go-libp2p#3049](https://github.com/libp2p/go-libp2p/pull/3049)) - fix: Defer resource usage cleanup until the very end (#3042) ([libp2p/go-libp2p#3042](https://github.com/libp2p/go-libp2p/pull/3042)) - fix(eventbus): Idempotent wildcardSub close (#3045) ([libp2p/go-libp2p#3045](https://github.com/libp2p/go-libp2p/pull/3045)) - fix: obsaddr: do not record observations over relayed conn (#3043) ([libp2p/go-libp2p#3043](https://github.com/libp2p/go-libp2p/pull/3043)) - fix(identify): push should not dial a new connection (#3035) ([libp2p/go-libp2p#3035](https://github.com/libp2p/go-libp2p/pull/3035)) - webrtc: handshake more connections in parallel (#3040) ([libp2p/go-libp2p#3040](https://github.com/libp2p/go-libp2p/pull/3040)) - eventbus: dont panic on closing Subscription twice (#3034) ([libp2p/go-libp2p#3034](https://github.com/libp2p/go-libp2p/pull/3034)) - fix(swarm): incorrect error message format order (#3037) ([libp2p/go-libp2p#3037](https://github.com/libp2p/go-libp2p/pull/3037)) - feat: eventbus: log error on slow consumers (#3031) ([libp2p/go-libp2p#3031](https://github.com/libp2p/go-libp2p/pull/3031)) - chore: make funding.json uppercase to follow meta convention (#3028) ([libp2p/go-libp2p#3028](https://github.com/libp2p/go-libp2p/pull/3028)) - chore: add drips entry to funding.json for Filecoin rPGF round 2 - tcp: parameterize metrics collector (#3026) ([libp2p/go-libp2p#3026](https://github.com/libp2p/go-libp2p/pull/3026)) - fix: basichost: Use NegotiationTimeout as fallback timeout for NewStream (#3020) ([libp2p/go-libp2p#3020](https://github.com/libp2p/go-libp2p/pull/3020)) - feat(tcpreuse): add options for sharing TCP listeners amongst TCP, WS and WSS transports (#2984) ([libp2p/go-libp2p#2984](https://github.com/libp2p/go-libp2p/pull/2984)) - pnet: wrap underlying error when reading nonce fails (#2975) ([libp2p/go-libp2p#2975](https://github.com/libp2p/go-libp2p/pull/2975)) - github.com/libp2p/go-libp2p-kad-dht (v0.28.1 -> v0.28.2): - Release v0.28.2 (#1010) ([libp2p/go-libp2p-kad-dht#1010](https://github.com/libp2p/go-libp2p-kad-dht/pull/1010)) - accelerated-dht: cleanup peer from message sender on disconnection (#1009) ([libp2p/go-libp2p-kad-dht#1009](https://github.com/libp2p/go-libp2p-kad-dht/pull/1009)) - chore: fix some function names in comment ([libp2p/go-libp2p-kad-dht#1004](https://github.com/libp2p/go-libp2p-kad-dht/pull/1004)) - feat: add more attributes to traces ([libp2p/go-libp2p-kad-dht#1002](https://github.com/libp2p/go-libp2p-kad-dht/pull/1002)) - github.com/libp2p/go-netroute (v0.2.1 -> v0.2.2): - v0.2.2 Includes v4/v6 confusion fix for bsd route parsing - #50, Don't transform v4 routes to their v6 form on bsd ([libp2p/go-netroute#51](https://github.com/libp2p/go-netroute/pull/51)) - Using syscall.RtMsg on Linux ([libp2p/go-netroute#43](https://github.com/libp2p/go-netroute/pull/43)) - add wasi build constraint for netroute_stub ([libp2p/go-netroute#38](https://github.com/libp2p/go-netroute/pull/38)) - Stricter filtering of degenerate routes ([libp2p/go-netroute#33](https://github.com/libp2p/go-netroute/pull/33)) - sync: update CI config files (#30) ([libp2p/go-netroute#30](https://github.com/libp2p/go-netroute/pull/30)) - github.com/multiformats/go-multiaddr (v0.13.0 -> v0.14.0): - Release v0.14.0 ([multiformats/go-multiaddr#258](https://github.com/multiformats/go-multiaddr/pull/258)) - feat: memory multiaddrs ([multiformats/go-multiaddr#256](https://github.com/multiformats/go-multiaddr/pull/256)) - nit: validate ipcidr ([multiformats/go-multiaddr#247](https://github.com/multiformats/go-multiaddr/pull/247)) - check for nil interfaces (#251) ([multiformats/go-multiaddr#251](https://github.com/multiformats/go-multiaddr/pull/251)) - Make it safe to roundtrip SplitXXX and Join (#250) ([multiformats/go-multiaddr#250](https://github.com/multiformats/go-multiaddr/pull/250)) - github.com/multiformats/go-multiaddr-dns (v0.4.0 -> v0.4.1): - Release v0.4.1 - fix: If decapsulating is empty, skip it. (#65) ([multiformats/go-multiaddr-dns#65](https://github.com/multiformats/go-multiaddr-dns/pull/65)) - github.com/multiformats/go-multistream (v0.5.0 -> v0.6.0): - release v0.6.0 ([multiformats/go-multistream#116](https://github.com/multiformats/go-multistream/pull/116)) - fix: finish reading handshake on lazyConn close - feat: New error to highlight unrecognized responses - release v0.5.0 (#108) ([multiformats/go-multistream#108](https://github.com/multiformats/go-multistream/pull/108))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Andrew Gillis | 57 | +1995/-1718 | 191 | | Adin Schmahmann | 7 | +2552/-719 | 84 | | Marco Munizaga | 27 | +1036/-261 | 51 | | Hector Sanjuan | 21 | +789/-362 | 65 | | gammazero | 20 | +407/-419 | 40 | | sukun | 13 | +519/-233 | 30 | | Marcin Rataj | 34 | +426/-142 | 59 | | Marten Seemann | 2 | +11/-261 | 5 | | Dreamacro | 2 | +161/-68 | 5 | | Hlib Kanunnikov | 1 | +34/-65 | 4 | | bashkarev | 1 | +78/-5 | 2 | | Daniel Norman | 4 | +68/-12 | 6 | | Andi | 1 | +37/-32 | 20 | | hannahhoward | 1 | +35/-17 | 7 | | Carlos Peliciari | 2 | +19/-26 | 2 | | Cole Brown | 1 | +32/-0 | 3 | | Will Scott | 2 | +19/-7 | 3 | | Guillaume Michel | 1 | +21/-2 | 4 | | 7sunarni | 1 | +3/-19 | 1 | | Srdjan S | 1 | +11/-2 | 2 | | web3-bot | 2 | +6/-6 | 3 | | dashangcun | 1 | +2/-10 | 1 | | John | 3 | +6/-6 | 5 | | Daniel N | 3 | +8/-3 | 3 | | Ivan Shvedunov | 1 | +4/-6 | 2 | | Piotr Galar | 1 | +4/-4 | 2 | | Derek Nola | 2 | +4/-4 | 4 | | Bryer | 1 | +4/-4 | 1 | | Prithvi Shahi | 2 | +6/-1 | 2 | | Cameron Wood | 1 | +7/-0 | 1 | | wangjingcun | 1 | +3/-3 | 2 | | cuibuwei | 1 | +2/-2 | 2 | | Jorropo | 1 | +1/-3 | 1 | | 未月 | 1 | +1/-1 | 1 | | Ubuntu | 1 | +1/-1 | 1 | | Ryan MacArthur | 1 | +1/-1 | 1 | | Reymon | 1 | +1/-1 | 1 | | guillaumemichel | 1 | +1/-0 | 1 | ## v0.33.1 ### 🔦 Highlights #### Bitswap improvements from Boxo This release includes performance and reliability improvements and fixes for minor resource leaks. One of the performance changes [greatly improves the bitswap clients ability to operate under high load](https://github.com/ipfs/boxo/pull/817#pullrequestreview-2587207745), that could previously result in an out of memory condition. #### Improved IPNS interop Improved compatibility with third-party IPNS publishers by restoring support for compact binary CIDs in the `Value` field of IPNS Records ([IPNS Specs](https://specs.ipfs.tech/ipns/ipns-record/)). As long the signature is valid, Kubo will now resolve such records (likely created by non-Kubo nodes) and convert raw CIDs into valid `/ipfs/cid` content paths. **Note:** This only adds support for resolving externally created records—Kubo’s IPNS record creation remains unchanged. IPNS records with empty `Value` fields default to zero-length `/ipfs/bafkqaaa` to maintain backward compatibility with code expecting a valid content path. #### 📦️ Important dependency updates - update `boxo` to [v0.27.4](https://github.com/ipfs/boxo/releases/tag/v0.27.4) (incl. [v0.27.3](https://github.com/ipfs/boxo/releases/tag/v0.27.3)) ### 📝 Changelog
Full Changelog v0.33.1 - github.com/ipfs/kubo: - chore: v0.33.1 - fix: boxo v0.27.4 (#10692) ([ipfs/kubo#10692](https://github.com/ipfs/kubo/pull/10692)) - docs: add webrtc-direct fixes to 0.33 release changelog (#10688) ([ipfs/kubo#10688](https://github.com/ipfs/kubo/pull/10688)) - fix: config help (#10686) ([ipfs/kubo#10686](https://github.com/ipfs/kubo/pull/10686)) - github.com/ipfs/boxo (v0.27.2 -> v0.27.4): - Release v0.27.4 ([ipfs/boxo#832](https://github.com/ipfs/boxo/pull/832)) - fix(ipns): reading records with raw []byte Value (#830) ([ipfs/boxo#830](https://github.com/ipfs/boxo/pull/830)) - fix(bitswap): blockpresencemanager leak (#833) ([ipfs/boxo#833](https://github.com/ipfs/boxo/pull/833)) - Always send cancels even if peer has no interest (#829) ([ipfs/boxo#829](https://github.com/ipfs/boxo/pull/829)) - tidy changelog ([ipfs/boxo#828](https://github.com/ipfs/boxo/pull/828)) - Update changelog (#827) ([ipfs/boxo#827](https://github.com/ipfs/boxo/pull/827)) - fix(bitswap): filter interests from received messages (#822) ([ipfs/boxo#822](https://github.com/ipfs/boxo/pull/822)) - Reduce unnecessary logging work (#826) ([ipfs/boxo#826](https://github.com/ipfs/boxo/pull/826)) - fix: bitswap lock contention under high load (#817) ([ipfs/boxo#817](https://github.com/ipfs/boxo/pull/817)) - fix: bitswap simplify cancel (#824) ([ipfs/boxo#824](https://github.com/ipfs/boxo/pull/824)) - fix(bitswap): simplify SessionInterestManager (#821) ([ipfs/boxo#821](https://github.com/ipfs/boxo/pull/821)) - feat: Better self-service commands for DHT providing (#815) ([ipfs/boxo#815](https://github.com/ipfs/boxo/pull/815)) - bitswap/client: fewer wantlist iterations in sendCancels (#819) ([ipfs/boxo#819](https://github.com/ipfs/boxo/pull/819)) - style: cleanup code by golangci-lint (#797) ([ipfs/boxo#797](https://github.com/ipfs/boxo/pull/797)) - Move long messagequeue comment to doc.go (#814) ([ipfs/boxo#814](https://github.com/ipfs/boxo/pull/814)) - Describe how bitswap message queue works ([ipfs/boxo#813](https://github.com/ipfs/boxo/pull/813))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Dreamacro | 1 | +304/-376 | 119 | | Andrew Gillis | 7 | +306/-200 | 20 | | Guillaume Michel | 5 | +122/-98 | 14 | | Marcin Rataj | 2 | +113/-7 | 4 | | gammazero | 6 | +41/-11 | 6 | | Sergey Gorbunov | 1 | +14/-2 | 2 | | Daniel Norman | 1 | +9/-0 | 1 | ## v0.33.2 ### 🔦 Highlights #### 📦️ Important dependency updates - update `go-libp2p` to [v0.38.3](https://github.com/libp2p/go-libp2p/releases/tag/v0.38.3) ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: v0.33.2 - github.com/libp2p/go-libp2p (v0.38.2 -> v0.38.3): - Release v0.38.3 (#3184) ([libp2p/go-libp2p#3184](https://github.com/libp2p/go-libp2p/pull/3184))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | sukun | 1 | +122/-23 | 7 | | Marcin Rataj | 1 | +1/-1 | 1 | ================================================ FILE: docs/changelogs/v0.34.md ================================================ # Kubo changelog v0.34 This release was brought to you by the [Shipyard](http://ipshipyard.com/) team. - [v0.34.0](#v0340) - [v0.34.1](#v0341) ## v0.34.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [AutoTLS now enabled by default for nodes with 1 hour uptime](#autotls-now-enabled-by-default-for-nodes-with-1-hour-uptime) - [New WebUI features](#new-webui-features) - [RPC and CLI command changes](#rpc-and-cli-command-changes) - [Bitswap improvements from Boxo](#bitswap-improvements-from-boxo) - [IPNS publishing TTL change](#ipns-publishing-ttl-change) - [`IPFS_LOG_LEVEL` deprecated](#ipfs_log_level-deprecated) - [Pebble datastore format update](#pebble-datastore-format-update) - [Badger datastore update](#badger-datastore-update) - [Datastore Implementation Updates](#datastore-implementation-updates) - [One Multi-error Package](#one-multi-error-package) - [Fix hanging pinset operations during reprovides](#fix-hanging-pinset-operations-during-reprovides) - [📦️ Important dependency updates](#-important-dependency-updates) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### AutoTLS now enabled by default for nodes with 1 hour uptime Starting now, any publicly dialable Kubo node with a `/tcp` listener that remains online for at least one hour will receive a TLS certificate through the [`AutoTLS`](https://github.com/ipfs/kubo/blob/master/docs/config.md#autotls) feature. This occurs automatically, with no need for manual setup. To bypass the 1-hour delay and enable AutoTLS immediately, users can explicitly opt-in by running the following commands: ```console $ ipfs config --json AutoTLS.Enabled true $ ipfs config --json AutoTLS.RegistrationDelay 0 ``` AutoTLS will remain disabled under the following conditions: - The node already has a manually configured `/ws` (WebSocket) listener - A private network is in use with a `swarm.key` - TCP or WebSocket transports are disabled, or there is no `/tcp` listener To troubleshoot, use `GOLOG_LOG_LEVEL="error,autotls=info`. For more details, check out the [`AutoTLS` configuration documentation](https://github.com/ipfs/kubo/blob/master/docs/config.md#autotls) or dive deeper with [AutoTLS libp2p blog post](https://web.archive.org/web/20260112031855/https://blog.libp2p.io/autotls/). #### New WebUI features The WebUI, accessible at http://127.0.0.1:5001/webui/, now includes support for CAR file import and QR code sharing directly from the Files view. Additionally, the Peers screen has been updated with the latest [`ipfs-geoip`](https://www.npmjs.com/package/ipfs-geoip) dataset. #### RPC and CLI command changes - `ipfs config` is now validating json fields ([#10679](https://github.com/ipfs/kubo/pull/10679)). - Deprecated the `bitswap reprovide` command. Make sure to switch to modern `routing reprovide`. ([#10677](https://github.com/ipfs/kubo/pull/10677)) - The `stats reprovide` command now shows additional stats for [`Routing.AcceleratedDHTClient`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient), indicating the last and next `reprovide` times. ([#10677](https://github.com/ipfs/kubo/pull/10677)) - `ipfs files cp` now performs basic codec check and will error when source is not a valid UnixFS (only `dag-pb` and `raw` codecs are allowed in MFS) #### Bitswap improvements from Boxo This release includes performance and reliability improvements and fixes for minor resource leaks. One of the performance changes [greatly improves the bitswap clients ability to operate under high load](https://github.com/ipfs/boxo/pull/817#pullrequestreview-2587207745), that could previously result in an out of memory condition. #### IPNS publishing TTL change Many complaints about IPNS being slow are tied to the default `--ttl` in `ipfs name publish`, which was set to 1 hour. To address this, we’ve lowered the default [IPNS Record TTL](https://specs.ipfs.tech/ipns/ipns-record/#ttl-uint64) during publishing to 5 minutes, matching similar TTL defaults in DNS. This update is now part of `boxo/ipfs` (GO, [boxo#859](https://github.com/ipfs/boxo/pull/859)) and `@helia/ipns` (JS, [helia#749](https://github.com/ipfs/helia/pull/749)). > [!TIP] > IPNS TTL recommendations when even faster update propagation is desired: > - **As a Publisher:** Lower the `--ttl` (e.g., `ipfs name publish --ttl=1m`) to further reduce caching delays. If using DNSLink, ensure the DNS TXT record TTL matches the IPNS record TTL. > - **As a Gateway Operator:** Override publisher TTLs for faster updates using configurations like [`Ipns.MaxCacheTTL`](https://github.com/ipfs/kubo/blob/master/docs/config.md#ipnsmaxcachettl) in Kubo or [`RAINBOW_IPNS_MAX_CACHE_TTL`](https://github.com/ipfs/rainbow/blob/main/docs/environment-variables.md#rainbow_ipns_max_cache_ttl) in [Rainbow](https://github.com/ipfs/rainbow/). #### `IPFS_LOG_LEVEL` deprecated The variable has been deprecated. Please use [`GOLOG_LOG_LEVEL`](https://github.com/ipfs/kubo/blob/master/docs/environment-variables.md#golog_log_level) instead for configuring logging levels. #### Pebble datastore format update If the pebble database format is not explicitly set in the config, then automatically upgrade it to the latest format version supported by the release ob pebble used by kubo. This will ensure that the database format is sufficiently up-to-date to be compatible with a major version upgrade of pebble. This is necessary before upgrading to use pebble v2. #### Badger datastore update An update was made to the badger v1 datastore that avoids use of mmap in 32-bit environments, which has been seen to cause issues on some platforms. Please be aware that this could lead to a performance regression for users of badger in a 32-bit environment. Badger users are advised to move to the flatds or pebble datastore. #### Datastore Implementation Updates The go-ds-xxx datastore implementations have been updated to support the updated `go-datastore` [v0.8.2](https://github.com/ipfs/go-datastore/releases/tag/v0.8.2) query API. This update removes the datastore implementations' dependency on `goprocess` and updates the query API. #### One Multi-error Package Kubo previously depended on multiple multi-error packages, `github.com/hashicorp/go-multierror` and `go.uber.org/multierr`. These have nearly identical functionality so there was no need to use both. Therefore, `go.uber.org/multierr` was selected as the package to depend on. Any future code needing multi-error functionality should use `go.uber.org/multierr` to avoid introducing unneeded dependencies. #### Fix hanging pinset operations during reprovides The reprovide process can be quite slow. In default settings, the reprovide process will start reading CIDs that belong to the pinset. During this operation, starvation can occur for other operations that need pinset access (see https://github.com/ipfs/kubo/issues/10596). We have now switch to buffering pinset-related cids that are going to be reprovided in memory, so that we can free pinset mutexes as soon as possible so that pinset-writes and subsequent read operations can proceed. The downside is larger pinsets will need some extra memory, with an estimation of ~1GiB of RAM memory-use per 20 million items to be reprovided. Use [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy) to balance announcement prioritization, speed, and memory utilization. #### 📦️ Important dependency updates - update `go-libp2p` to [v0.41.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.41.0) (incl. [v0.40.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.40.0)) - update `go-libp2p-kad-dht` to [v0.30.2](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.30.2) (incl. [v0.29.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.29.0), [v0.29.1](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.29.1), [v0.29.2](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.29.2), [v0.30.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.30.0), [v0.30.1](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.30.1)) - update `boxo` to [v0.29.1](https://github.com/ipfs/boxo/releases/tag/v0.29.1) (incl. [v0.28.0](https://github.com/ipfs/boxo/releases/tag/v0.28.0) [v0.29.0](https://github.com/ipfs/boxo/releases/tag/v0.29.0)) - update `ipfs-webui` to [v4.6.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.6.0) (incl. [v4.5.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.5.0)) - update `p2p-forge/client` to [v0.4.0](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.4.0) - update `go-datastore` to [v0.8.2](https://github.com/ipfs/go-datastore/releases/tag/v0.8.2) (incl. [v0.7.0](https://github.com/ipfs/go-datastore/releases/tag/v0.7.0), [v0.8.0](https://github.com/ipfs/go-datastore/releases/tag/v0.8.0)) ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: v0.34.0 - chore: v0.34.0-rc2 - docs: mention Reprovider.Strategy config - docs: ipns ttl change - feat: ipfs-webui v4.6 (#10756) ([ipfs/kubo#10756](https://github.com/ipfs/kubo/pull/10756)) - docs(readme): update min. requirements + cleanup (#10750) ([ipfs/kubo#10750](https://github.com/ipfs/kubo/pull/10750)) - Upgrade to Boxo v0.29.1 (#10755) ([ipfs/kubo#10755](https://github.com/ipfs/kubo/pull/10755)) - Nonfunctional (#10753) ([ipfs/kubo#10753](https://github.com/ipfs/kubo/pull/10753)) - Update docs/changelogs/v0.34.md - provider: buffer pin providers. - chore: 0.34.0-rc1 - fix(mfs): basic UnixFS sanity checks in `files cp` (#10701) ([ipfs/kubo#10701](https://github.com/ipfs/kubo/pull/10701)) - Upgrade to Boxo v0.29.0 (#10742) ([ipfs/kubo#10742](https://github.com/ipfs/kubo/pull/10742)) - use go-datastore without go-process (#10736) ([ipfs/kubo#10736](https://github.com/ipfs/kubo/pull/10736)) - docs(config): add security considerations for rpc (#10739) ([ipfs/kubo#10739](https://github.com/ipfs/kubo/pull/10739)) - chore: update go-libp2p to v0.41.0 (#10733) ([ipfs/kubo#10733](https://github.com/ipfs/kubo/pull/10733)) - feat: ipfs-webui v4.5.0 (#10735) ([ipfs/kubo#10735](https://github.com/ipfs/kubo/pull/10735)) - Create FUNDING.json (#10734) ([ipfs/kubo#10734](https://github.com/ipfs/kubo/pull/10734)) - feat(AutoTLS): enabled by default with 1h RegistrationDelay (#10724) ([ipfs/kubo#10724](https://github.com/ipfs/kubo/pull/10724)) - Upgrade to Boxo v0.28.0 (#10725) ([ipfs/kubo#10725](https://github.com/ipfs/kubo/pull/10725)) - Upgrade to go1.24 (#10726) ([ipfs/kubo#10726](https://github.com/ipfs/kubo/pull/10726)) - Replace go-random with random-data from go-test package (#10731) ([ipfs/kubo#10731](https://github.com/ipfs/kubo/pull/10731)) - Update to new go-test (#10729) ([ipfs/kubo#10729](https://github.com/ipfs/kubo/pull/10729)) - Update go-test and use new random-files generator (#10728) ([ipfs/kubo#10728](https://github.com/ipfs/kubo/pull/10728)) - docs(readme): update docker section (#10716) ([ipfs/kubo#10716](https://github.com/ipfs/kubo/pull/10716)) - Update go-ds-badger to v0.3.1 (#10722) ([ipfs/kubo#10722](https://github.com/ipfs/kubo/pull/10722)) - Update pebble db to latest format by default (#10720) ([ipfs/kubo#10720](https://github.com/ipfs/kubo/pull/10720)) - fix: switch away from IPFS_LOG_LEVEL (#10694) ([ipfs/kubo#10694](https://github.com/ipfs/kubo/pull/10694)) - Merge release v0.33.2 ([ipfs/kubo#10713](https://github.com/ipfs/kubo/pull/10713)) - Remove unused TimeParts struct (#10708) ([ipfs/kubo#10708](https://github.com/ipfs/kubo/pull/10708)) - fix(rpc): restore and deprecate `bitswap reprovide` (#10699) ([ipfs/kubo#10699](https://github.com/ipfs/kubo/pull/10699)) - docs(release): update RELEASE_CHECKLIST.md after v0.33.1 (#10697) ([ipfs/kubo#10697](https://github.com/ipfs/kubo/pull/10697)) - docs: update min requirements (#10687) ([ipfs/kubo#10687](https://github.com/ipfs/kubo/pull/10687)) - Merge release v0.33.1 ([ipfs/kubo#10698](https://github.com/ipfs/kubo/pull/10698)) - fix: boxo v0.27.4 (#10692) ([ipfs/kubo#10692](https://github.com/ipfs/kubo/pull/10692)) - fix: Issue #9364 JSON config validation (#10679) ([ipfs/kubo#10679](https://github.com/ipfs/kubo/pull/10679)) - docs: RELEASE_CHECKLIST.md update for 0.33 (#10674) ([ipfs/kubo#10674](https://github.com/ipfs/kubo/pull/10674)) - feat: Better self-service commands for DHT providing (#10677) ([ipfs/kubo#10677](https://github.com/ipfs/kubo/pull/10677)) - docs: add webrtc-direct fixes to 0.33 release changelog (#10688) ([ipfs/kubo#10688](https://github.com/ipfs/kubo/pull/10688)) - fix: config help (#10686) ([ipfs/kubo#10686](https://github.com/ipfs/kubo/pull/10686)) - feat: Add CI for Spell Checking (#10637) ([ipfs/kubo#10637](https://github.com/ipfs/kubo/pull/10637)) - Merge release v0.33.0 ([ipfs/kubo#10684](https://github.com/ipfs/kubo/pull/10684)) - test: fix the socat tests after the ubuntu 24.04 upgrade (#10683) ([ipfs/kubo#10683](https://github.com/ipfs/kubo/pull/10683)) - fix: quic-go v0.49.0 (#10673) ([ipfs/kubo#10673](https://github.com/ipfs/kubo/pull/10673)) - Upgrade to Boxo v0.27.2 (#10672) ([ipfs/kubo#10672](https://github.com/ipfs/kubo/pull/10672)) - Upgrade to Boxo v0.27.1 (#10671) ([ipfs/kubo#10671](https://github.com/ipfs/kubo/pull/10671)) - fix(autotls): renewal and AutoTLS.ShortAddrs (#10669) ([ipfs/kubo#10669](https://github.com/ipfs/kubo/pull/10669)) - update changelog for boxo and go-libp2p (#10668) ([ipfs/kubo#10668](https://github.com/ipfs/kubo/pull/10668)) - Upgrade to Boxo v0.27.0 (#10665) ([ipfs/kubo#10665](https://github.com/ipfs/kubo/pull/10665)) - update dependencies (#10664) ([ipfs/kubo#10664](https://github.com/ipfs/kubo/pull/10664)) - docs(readme): add unofficial Fedora COPR (#10660) ([ipfs/kubo#10660](https://github.com/ipfs/kubo/pull/10660)) - fix(dns): update default DNSLink resolvers (#10655) ([ipfs/kubo#10655](https://github.com/ipfs/kubo/pull/10655)) - chore: p2p-forge v0.2.2 + go-libp2p-kad-dht v0.28.2 (#10663) ([ipfs/kubo#10663](https://github.com/ipfs/kubo/pull/10663)) - fix(cli): support HTTPS in ipfs --api (#10659) ([ipfs/kubo#10659](https://github.com/ipfs/kubo/pull/10659)) - chore: fix typos and comment formatting (#10653) ([ipfs/kubo#10653](https://github.com/ipfs/kubo/pull/10653)) - fix/gateway: escape directory redirect url (#10649) ([ipfs/kubo#10649](https://github.com/ipfs/kubo/pull/10649)) - Add example of setting array to config command help ([ipfs/kubo#10650](https://github.com/ipfs/kubo/pull/10650)) - collection of typo fixes (#10647) ([ipfs/kubo#10647](https://github.com/ipfs/kubo/pull/10647)) - chore: bump master to 0.34.0-dev - github.com/ipfs/boxo (v0.27.4 -> v0.29.1): - Release v0.29.1 ([ipfs/boxo#885](https://github.com/ipfs/boxo/pull/885)) - fix(provider): call reprovider throughput callback only if reprovide is enabled (#871) ([ipfs/boxo#871](https://github.com/ipfs/boxo/pull/871)) - bitswap/httpnet: do not follow redirects (#878) ([ipfs/boxo#878](https://github.com/ipfs/boxo/pull/878)) - Refactor(hostname): Skip DNSLink for local IP addresses to avoid DNS queries (#880) ([ipfs/boxo#880](https://github.com/ipfs/boxo/pull/880)) - Nonfunctional (#882) ([ipfs/boxo#882](https://github.com/ipfs/boxo/pull/882)) - fix(bitswap/client): dont set nil for DontHaveTimeoutConfig (#872) ([ipfs/boxo#872](https://github.com/ipfs/boxo/pull/872)) - provider: add a buffered KeyChanFunc. ([ipfs/boxo#870](https://github.com/ipfs/boxo/pull/870)) - Release v0.29.0 (#869) ([ipfs/boxo#869](https://github.com/ipfs/boxo/pull/869)) - Do not use multiple multi-error packages, pick one (#867) ([ipfs/boxo#867](https://github.com/ipfs/boxo/pull/867)) - feat(bitswap/client): MinTimeout for DontHaveTimeoutConfig (#865) ([ipfs/boxo#865](https://github.com/ipfs/boxo/pull/865)) - use go-datastore without go-process (#858) ([ipfs/boxo#858](https://github.com/ipfs/boxo/pull/858)) - minimize peermanager lock scope (#860) ([ipfs/boxo#860](https://github.com/ipfs/boxo/pull/860)) - chore(ipns): lower `DefaultRecordTTL` to 5m (#859) ([ipfs/boxo#859](https://github.com/ipfs/boxo/pull/859)) - httpnet: bitswap network for HTTP block retrieval over trustless gateway endpoints. ([ipfs/boxo#747](https://github.com/ipfs/boxo/pull/747)) - chore: Update FUNDING.json for Optimism RPF (#857) ([ipfs/boxo#857](https://github.com/ipfs/boxo/pull/857)) - Release v0.28.0 (#854) ([ipfs/boxo#854](https://github.com/ipfs/boxo/pull/854)) - Update deps (#852) ([ipfs/boxo#852](https://github.com/ipfs/boxo/pull/852)) - fix: gateway/blocks-backend: GetBlock should not perform IPLD decoding (#845) ([ipfs/boxo#845](https://github.com/ipfs/boxo/pull/845)) - Protobuf pkg name (#850) ([ipfs/boxo#850](https://github.com/ipfs/boxo/pull/850)) - Fix intermittent test failure (#849) ([ipfs/boxo#849](https://github.com/ipfs/boxo/pull/849)) - move `ipld/merkledag` from gogo protobuf (#841) ([ipfs/boxo#841](https://github.com/ipfs/boxo/pull/841)) - move `ipld/unixfs` from gogo protobuf (#840) ([ipfs/boxo#840](https://github.com/ipfs/boxo/pull/840)) - Start moving from gogo protobuf (#839) ([ipfs/boxo#839](https://github.com/ipfs/boxo/pull/839)) - ci: uci/update-go (#848) ([ipfs/boxo#848](https://github.com/ipfs/boxo/pull/848)) - expose DontHaveTimeoutConfig (#846) ([ipfs/boxo#846](https://github.com/ipfs/boxo/pull/846)) - Upgrade go-libp2p to v0.39.1 (#843) ([ipfs/boxo#843](https://github.com/ipfs/boxo/pull/843)) - feat: Prevent multiple instances of "ipfs routing reprovide" running together. (#834) ([ipfs/boxo#834](https://github.com/ipfs/boxo/pull/834)) - Upgrade to go-libp2p v0.39.0 (#837) ([ipfs/boxo#837](https://github.com/ipfs/boxo/pull/837)) - bitswap/client/internal/messagequeue: run tests in parallel (#835) ([ipfs/boxo#835](https://github.com/ipfs/boxo/pull/835)) - github.com/ipfs/go-cid (v0.4.1 -> v0.5.0): - v0.5.0 bump (#172) ([ipfs/go-cid#172](https://github.com/ipfs/go-cid/pull/172)) - move _rsrch/cidiface into an internal package - github.com/ipfs/go-datastore (v0.6.0 -> v0.8.2): - bump version (#231) ([ipfs/go-datastore#231](https://github.com/ipfs/go-datastore/pull/231)) - Results.Close should return error (#230) ([ipfs/go-datastore#230](https://github.com/ipfs/go-datastore/pull/230)) - new version (#229) ([ipfs/go-datastore#229](https://github.com/ipfs/go-datastore/pull/229)) - Update fuzz module dependencies (#228) ([ipfs/go-datastore#228](https://github.com/ipfs/go-datastore/pull/228)) - new version (#225) ([ipfs/go-datastore#225](https://github.com/ipfs/go-datastore/pull/225)) - No goprocess (#223) ([ipfs/go-datastore#223](https://github.com/ipfs/go-datastore/pull/223)) - Release version 0.7.0 (#213) ([ipfs/go-datastore#213](https://github.com/ipfs/go-datastore/pull/213)) - query result ordering does not create additional goroutine (#221) ([ipfs/go-datastore#221](https://github.com/ipfs/go-datastore/pull/221)) - Remove unneeded dependencies (#220) ([ipfs/go-datastore#220](https://github.com/ipfs/go-datastore/pull/220)) - Add traced datastore (#209) ([ipfs/go-datastore#209](https://github.com/ipfs/go-datastore/pull/209)) - Add root namespace method to Key (#208) ([ipfs/go-datastore#208](https://github.com/ipfs/go-datastore/pull/208)) - ci: uci/copy-templates (#207) ([ipfs/go-datastore#207](https://github.com/ipfs/go-datastore/pull/207)) - test: fix fuzz commands - fix fuzz tests by adding the missing context.Context argument (#198) ([ipfs/go-datastore#198](https://github.com/ipfs/go-datastore/pull/198)) - sync: update CI config files (#195) ([ipfs/go-datastore#195](https://github.com/ipfs/go-datastore/pull/195)) - github.com/ipfs/go-ds-badger (v0.3.0 -> v0.3.4): - new version (#137) ([ipfs/go-ds-badger#137](https://github.com/ipfs/go-ds-badger/pull/137)) - new version (#135) ([ipfs/go-ds-badger#135](https://github.com/ipfs/go-ds-badger/pull/135)) - new version (#132) ([ipfs/go-ds-badger#132](https://github.com/ipfs/go-ds-badger/pull/132)) - Update to use go-datastore without go-process (#131) ([ipfs/go-ds-badger#131](https://github.com/ipfs/go-ds-badger/pull/131)) - new version ([ipfs/go-ds-badger#128](https://github.com/ipfs/go-ds-badger/pull/128)) - Update dependencies and minimum go version ([ipfs/go-ds-badger#127](https://github.com/ipfs/go-ds-badger/pull/127)) - ci: uci/update-go ([ipfs/go-ds-badger#123](https://github.com/ipfs/go-ds-badger/pull/123)) - ci: uci/copy-templates ([ipfs/go-ds-badger#122](https://github.com/ipfs/go-ds-badger/pull/122)) - chore: check PersistentDatastore conformance at build time (#120) ([ipfs/go-ds-badger#120](https://github.com/ipfs/go-ds-badger/pull/120)) - github.com/ipfs/go-ds-flatfs (v0.5.1 -> v0.5.5): - bump version (#130) ([ipfs/go-ds-flatfs#130](https://github.com/ipfs/go-ds-flatfs/pull/130)) - new version (#128) ([ipfs/go-ds-flatfs#128](https://github.com/ipfs/go-ds-flatfs/pull/128)) - new version (#126) ([ipfs/go-ds-flatfs#126](https://github.com/ipfs/go-ds-flatfs/pull/126)) - Fix race condition due to concurrent use of rand source (#125) ([ipfs/go-ds-flatfs#125](https://github.com/ipfs/go-ds-flatfs/pull/125)) - new version ([ipfs/go-ds-flatfs#124](https://github.com/ipfs/go-ds-flatfs/pull/124)) - Use go-datastore without go-process ([ipfs/go-ds-flatfs#123](https://github.com/ipfs/go-ds-flatfs/pull/123)) - ci: uci/update-go (#122) ([ipfs/go-ds-flatfs#122](https://github.com/ipfs/go-ds-flatfs/pull/122)) - fix: actually use the size hint in util_windows.go - perf: do not use virtual call when passing os.Rename as rename - chore(logging): update go-log v2 (#117) ([ipfs/go-ds-flatfs#117](https://github.com/ipfs/go-ds-flatfs/pull/117)) - ci: uci/copy-templates ([ipfs/go-ds-flatfs#116](https://github.com/ipfs/go-ds-flatfs/pull/116)) - sync: update CI config files ([ipfs/go-ds-flatfs#111](https://github.com/ipfs/go-ds-flatfs/pull/111)) - possibly fix a bug in renameAndUpdateDiskUsage - add documentation and comment - perf: avoid syncing directories when they already existed (#107) ([ipfs/go-ds-flatfs#107](https://github.com/ipfs/go-ds-flatfs/pull/107)) - test: faster TestNoCluster by batching the 3200 Puts ([ipfs/go-ds-flatfs#108](https://github.com/ipfs/go-ds-flatfs/pull/108)) - query: also teard down on ctx done (#106) ([ipfs/go-ds-flatfs#106](https://github.com/ipfs/go-ds-flatfs/pull/106)) - github.com/ipfs/go-ds-leveldb (v0.5.0 -> v0.5.2): - new version (#75) ([ipfs/go-ds-leveldb#75](https://github.com/ipfs/go-ds-leveldb/pull/75)) - Results close needs to return error (#74) ([ipfs/go-ds-leveldb#74](https://github.com/ipfs/go-ds-leveldb/pull/74)) - new version ([ipfs/go-ds-leveldb#73](https://github.com/ipfs/go-ds-leveldb/pull/73)) - use go-datastore without go-process ([ipfs/go-ds-leveldb#72](https://github.com/ipfs/go-ds-leveldb/pull/72)) - sync: update CI config files (#62) ([ipfs/go-ds-leveldb#62](https://github.com/ipfs/go-ds-leveldb/pull/62)) - chore: add PersistentDatastore and Batching interface checks - github.com/ipfs/go-ds-measure (v0.2.0 -> v0.2.2): - new version ([ipfs/go-ds-measure#54](https://github.com/ipfs/go-ds-measure/pull/54)) - new version ([ipfs/go-ds-measure#52](https://github.com/ipfs/go-ds-measure/pull/52)) - github.com/ipfs/go-ds-pebble (v0.4.2 -> v0.4.4): - new version (#51) ([ipfs/go-ds-pebble#51](https://github.com/ipfs/go-ds-pebble/pull/51)) - new version (#48) ([ipfs/go-ds-pebble#48](https://github.com/ipfs/go-ds-pebble/pull/48)) - Use go-datastore without go-process (#47) ([ipfs/go-ds-pebble#47](https://github.com/ipfs/go-ds-pebble/pull/47)) - github.com/ipfs/go-metrics-interface (v0.0.1 -> v0.3.0): - CounterVec: even more ergonomic ([ipfs/go-metrics-interface#22](https://github.com/ipfs/go-metrics-interface/pull/22)) - Improve CounterVec abstraction ([ipfs/go-metrics-interface#21](https://github.com/ipfs/go-metrics-interface/pull/21)) - v0.1.0 ([ipfs/go-metrics-interface#20](https://github.com/ipfs/go-metrics-interface/pull/20)) - Feat: Add CounterVec type. ([ipfs/go-metrics-interface#19](https://github.com/ipfs/go-metrics-interface/pull/19)) - sync: update CI config files (#10) ([ipfs/go-metrics-interface#10](https://github.com/ipfs/go-metrics-interface/pull/10)) - sync: update CI config files (#8) ([ipfs/go-metrics-interface#8](https://github.com/ipfs/go-metrics-interface/pull/8)) - use a struct as a key for the context ([ipfs/go-metrics-interface#4](https://github.com/ipfs/go-metrics-interface/pull/4)) - github.com/ipfs/go-metrics-prometheus (v0.0.3 -> v0.1.0): - Implement the CounterVec type. ([ipfs/go-metrics-prometheus#26](https://github.com/ipfs/go-metrics-prometheus/pull/26)) - github.com/ipfs/go-test (v0.0.4 -> v0.2.1): - new version (#20) ([ipfs/go-test#20](https://github.com/ipfs/go-test/pull/20)) - No newline at end of random raw data (#19) ([ipfs/go-test#19](https://github.com/ipfs/go-test/pull/19)) - new-version (#18) ([ipfs/go-test#18](https://github.com/ipfs/go-test/pull/18)) - new version (#15) ([ipfs/go-test#15](https://github.com/ipfs/go-test/pull/15)) - refactor: Make go-multiaddr v0.15 forward compatible change (#16) ([ipfs/go-test#16](https://github.com/ipfs/go-test/pull/16)) - Move cli apps (#17) ([ipfs/go-test#17](https://github.com/ipfs/go-test/pull/17)) - Update help text (#14) ([ipfs/go-test#14](https://github.com/ipfs/go-test/pull/14)) - Add package to generate random filesystem hierarchies for testing (#13) ([ipfs/go-test#13](https://github.com/ipfs/go-test/pull/13)) - github.com/ipfs/go-unixfsnode (v1.9.2 -> v1.10.0): - new version ([ipfs/go-unixfsnode#81](https://github.com/ipfs/go-unixfsnode/pull/81)) - upgrade to boxo v0.27.4 ([ipfs/go-unixfsnode#80](https://github.com/ipfs/go-unixfsnode/pull/80)) - github.com/libp2p/go-libp2p (v0.38.3 -> v0.41.0): - Release v0.41.0 (#3210) ([libp2p/go-libp2p#3210](https://github.com/libp2p/go-libp2p/pull/3210)) - fix(libp2phttp): Fix relative to absolute multiaddr URI logic (#3208) ([libp2p/go-libp2p#3208](https://github.com/libp2p/go-libp2p/pull/3208)) - fix(dcutr): Fix end to end tests and add legacy behavior flag (default=true) (#3044) ([libp2p/go-libp2p#3044](https://github.com/libp2p/go-libp2p/pull/3044)) - feat(libp2phttp): More ergonomic auth (#3188) ([libp2p/go-libp2p#3188](https://github.com/libp2p/go-libp2p/pull/3188)) - chore(identify): move log to debug level (#3206) ([libp2p/go-libp2p#3206](https://github.com/libp2p/go-libp2p/pull/3206)) - chore: Update go-multiaddr to v0.15 (#3145) ([libp2p/go-libp2p#3145](https://github.com/libp2p/go-libp2p/pull/3145)) - chore: update quic-go to v0.50.0 (#3204) ([libp2p/go-libp2p#3204](https://github.com/libp2p/go-libp2p/pull/3204)) - chore: move go-nat to internal package - basichost: add certhashes to addrs in place (#3200) ([libp2p/go-libp2p#3200](https://github.com/libp2p/go-libp2p/pull/3200)) - autorelay: send addresses on eventbus; dont wrap address factory (#3071) ([libp2p/go-libp2p#3071](https://github.com/libp2p/go-libp2p/pull/3071)) - chore: update ci for go1.24 (#3195) ([libp2p/go-libp2p#3195](https://github.com/libp2p/go-libp2p/pull/3195)) - Release v0.40.0 (#3192) ([libp2p/go-libp2p#3192](https://github.com/libp2p/go-libp2p/pull/3192)) - chore: bump deps for v0.40.0 (#3191) ([libp2p/go-libp2p#3191](https://github.com/libp2p/go-libp2p/pull/3191)) - autonatv2: allow multiple concurrent requests per peer (#3187) ([libp2p/go-libp2p#3187](https://github.com/libp2p/go-libp2p/pull/3187)) - feat: add AutoTLS example (#3103) ([libp2p/go-libp2p#3103](https://github.com/libp2p/go-libp2p/pull/3103)) - feat(swarm): logging waitForDirectConn return error (#3183) ([libp2p/go-libp2p#3183](https://github.com/libp2p/go-libp2p/pull/3183)) - tcpreuse: fix Scope() for *tls.Conn (#3181) ([libp2p/go-libp2p#3181](https://github.com/libp2p/go-libp2p/pull/3181)) - test(p2p/protocol/identify): fix user agent assertion in Go 1.24 (#3177) ([libp2p/go-libp2p#3177](https://github.com/libp2p/go-libp2p/pull/3177)) - swarm: remove unnecessary error log (#3128) ([libp2p/go-libp2p#3128](https://github.com/libp2p/go-libp2p/pull/3128)) - Implement error codes spec (#2927) ([libp2p/go-libp2p#2927](https://github.com/libp2p/go-libp2p/pull/2927)) - chore: update pion/ice to v4 (#3175) ([libp2p/go-libp2p#3175](https://github.com/libp2p/go-libp2p/pull/3175)) - chore: release v0.39.0 (#3174) ([libp2p/go-libp2p#3174](https://github.com/libp2p/go-libp2p/pull/3174)) - feat(holepunch): add logging when DirectConnect execution fails (#3146) ([libp2p/go-libp2p#3146](https://github.com/libp2p/go-libp2p/pull/3146)) - feat: Implement Custom TCP Dialers (#3166) ([libp2p/go-libp2p#3166](https://github.com/libp2p/go-libp2p/pull/3166)) - Update quic-go to v0.49.0 (#3153) ([libp2p/go-libp2p#3153](https://github.com/libp2p/go-libp2p/pull/3153)) - feat(transport/websocket): support SOCKS proxy with ws(s) (#3137) ([libp2p/go-libp2p#3137](https://github.com/libp2p/go-libp2p/pull/3137)) - tcpreuse: fix rcmgr accounting when tcp metrics are enabled (#3142) ([libp2p/go-libp2p#3142](https://github.com/libp2p/go-libp2p/pull/3142)) - fix(net/nat): data race problem of `extAddr` (#3140) ([libp2p/go-libp2p#3140](https://github.com/libp2p/go-libp2p/pull/3140)) - test: fix failing test (#3141) ([libp2p/go-libp2p#3141](https://github.com/libp2p/go-libp2p/pull/3141)) - quicreuse: make it possible to use an application-constructed quic.Transport (#3122) ([libp2p/go-libp2p#3122](https://github.com/libp2p/go-libp2p/pull/3122)) - nat: ignore mapping if external port is 0 (#3094) ([libp2p/go-libp2p#3094](https://github.com/libp2p/go-libp2p/pull/3094)) - tcpreuse: error on using tcpreuse with pnet (#3129) ([libp2p/go-libp2p#3129](https://github.com/libp2p/go-libp2p/pull/3129)) - chore: Update contribution guidelines (#3134) ([libp2p/go-libp2p#3134](https://github.com/libp2p/go-libp2p/pull/3134)) - tcp: fix metrics test build directive (#3052) ([libp2p/go-libp2p#3052](https://github.com/libp2p/go-libp2p/pull/3052)) - webrtc: upgrade pion/webrtc to v4 (#3098) ([libp2p/go-libp2p#3098](https://github.com/libp2p/go-libp2p/pull/3098)) - webtransport: fix docstring comment for getCurrentBucketStartTime - chore: release v0.38.1 (#3114) ([libp2p/go-libp2p#3114](https://github.com/libp2p/go-libp2p/pull/3114)) - github.com/libp2p/go-libp2p-kad-dht (v0.28.2 -> v0.30.2): - new version (#1059) ([libp2p/go-libp2p-kad-dht#1059](https://github.com/libp2p/go-libp2p-kad-dht/pull/1059)) - do not use multiple multi-error packages, pick one (#1058) ([libp2p/go-libp2p-kad-dht#1058](https://github.com/libp2p/go-libp2p-kad-dht/pull/1058)) - update version (#1057) ([libp2p/go-libp2p-kad-dht#1057](https://github.com/libp2p/go-libp2p-kad-dht/pull/1057)) - chore: release v0.30.0 (#1054) ([libp2p/go-libp2p-kad-dht#1054](https://github.com/libp2p/go-libp2p-kad-dht/pull/1054)) - fix: crawler polluting peerstore (#1053) ([libp2p/go-libp2p-kad-dht#1053](https://github.com/libp2p/go-libp2p-kad-dht/pull/1053)) - new version (#1052) ([libp2p/go-libp2p-kad-dht#1052](https://github.com/libp2p/go-libp2p-kad-dht/pull/1052)) - use go-datastore without go-process (#1051) ([libp2p/go-libp2p-kad-dht#1051](https://github.com/libp2p/go-libp2p-kad-dht/pull/1051)) - feat: use OTEL for metrics (removes opencensus) (#1045) ([libp2p/go-libp2p-kad-dht#1045](https://github.com/libp2p/go-libp2p-kad-dht/pull/1045)) - release v0.29.1 (#1042) ([libp2p/go-libp2p-kad-dht#1042](https://github.com/libp2p/go-libp2p-kad-dht/pull/1042)) - fix: flaky TestInvalidServer (#1049) ([libp2p/go-libp2p-kad-dht#1049](https://github.com/libp2p/go-libp2p-kad-dht/pull/1049)) - chore: update deps (#1048) ([libp2p/go-libp2p-kad-dht#1048](https://github.com/libp2p/go-libp2p-kad-dht/pull/1048)) - fix addrsSoFar comparison (#1046) ([libp2p/go-libp2p-kad-dht#1046](https://github.com/libp2p/go-libp2p-kad-dht/pull/1046)) - fix: flaky TestInvalidServer (#1043) ([libp2p/go-libp2p-kad-dht#1043](https://github.com/libp2p/go-libp2p-kad-dht/pull/1043)) - add verbose to TestFindProviderAsync (dual) (#1040) ([libp2p/go-libp2p-kad-dht#1040](https://github.com/libp2p/go-libp2p-kad-dht/pull/1040)) - test: cover dns addresses in TestAddrFilter (#1041) ([libp2p/go-libp2p-kad-dht#1041](https://github.com/libp2p/go-libp2p-kad-dht/pull/1041)) - fix: flaky TestSearchValue (dual) (#1038) ([libp2p/go-libp2p-kad-dht#1038](https://github.com/libp2p/go-libp2p-kad-dht/pull/1038)) - fix: flaky TestClientModeConnect (#1037) ([libp2p/go-libp2p-kad-dht#1037](https://github.com/libp2p/go-libp2p-kad-dht/pull/1037)) - fix: flaky TestFindPeerQueryMinimal (#1036) ([libp2p/go-libp2p-kad-dht#1036](https://github.com/libp2p/go-libp2p-kad-dht/pull/1036)) - fix: flaky TestInvalidServer (#1032) ([libp2p/go-libp2p-kad-dht#1032](https://github.com/libp2p/go-libp2p-kad-dht/pull/1032)) - fix: flaky TestFindPeerWithQueryFilter (#1034) ([libp2p/go-libp2p-kad-dht#1034](https://github.com/libp2p/go-libp2p-kad-dht/pull/1034)) - fix: Flaky TestInvalidServer (#1029) ([libp2p/go-libp2p-kad-dht#1029](https://github.com/libp2p/go-libp2p-kad-dht/pull/1029)) - fix: flaky TestClientModeConnect (#1028) ([libp2p/go-libp2p-kad-dht#1028](https://github.com/libp2p/go-libp2p-kad-dht/pull/1028)) - fix: increase timeout in TestProvidesMany (#1027) ([libp2p/go-libp2p-kad-dht#1027](https://github.com/libp2p/go-libp2p-kad-dht/pull/1027)) - fix(tests): cleanup of skipped tests (#1025) ([libp2p/go-libp2p-kad-dht#1025](https://github.com/libp2p/go-libp2p-kad-dht/pull/1025)) - fix: don't skip TestProvidesExpire (#1024) ([libp2p/go-libp2p-kad-dht#1024](https://github.com/libp2p/go-libp2p-kad-dht/pull/1024)) - fixing flaky TestFindPeerQueryMinimal (#1020) ([libp2p/go-libp2p-kad-dht#1020](https://github.com/libp2p/go-libp2p-kad-dht/pull/1020)) - fix flaky TestSkipRefreshOnGapCpls (#1021) ([libp2p/go-libp2p-kad-dht#1021](https://github.com/libp2p/go-libp2p-kad-dht/pull/1021)) - fix: don't skip TestContextShutDown (#1022) ([libp2p/go-libp2p-kad-dht#1022](https://github.com/libp2p/go-libp2p-kad-dht/pull/1022)) - comments formatting and typos (#1019) ([libp2p/go-libp2p-kad-dht#1019](https://github.com/libp2p/go-libp2p-kad-dht/pull/1019)) - log peers rejected for diversity (#759) ([libp2p/go-libp2p-kad-dht#759](https://github.com/libp2p/go-libp2p-kad-dht/pull/759)) - docs: update fullrt docs (#768) ([libp2p/go-libp2p-kad-dht#768](https://github.com/libp2p/go-libp2p-kad-dht/pull/768)) - query cleanup (#1017) ([libp2p/go-libp2p-kad-dht#1017](https://github.com/libp2p/go-libp2p-kad-dht/pull/1017)) - better variable names (#787) ([libp2p/go-libp2p-kad-dht#787](https://github.com/libp2p/go-libp2p-kad-dht/pull/787)) - release v0.29.0 (#1014) ([libp2p/go-libp2p-kad-dht#1014](https://github.com/libp2p/go-libp2p-kad-dht/pull/1014)) - Move from gogo protobuf (#975) ([libp2p/go-libp2p-kad-dht#975](https://github.com/libp2p/go-libp2p-kad-dht/pull/975)) - fix: don't copy message to OnRequestHook ([libp2p/go-libp2p-kad-dht#1012](https://github.com/libp2p/go-libp2p-kad-dht/pull/1012)) - chore: remove boxo/util deps ([libp2p/go-libp2p-kad-dht#1013](https://github.com/libp2p/go-libp2p-kad-dht/pull/1013)) - feat: add request callback config option ([libp2p/go-libp2p-kad-dht#1011](https://github.com/libp2p/go-libp2p-kad-dht/pull/1011)) - github.com/libp2p/go-libp2p-kbucket (v0.6.4 -> v0.6.5): - upgrading deps (#137) ([libp2p/go-libp2p-kbucket#137](https://github.com/libp2p/go-libp2p-kbucket/pull/137)) - github.com/libp2p/go-libp2p-pubsub (v0.12.0 -> v0.13.0): - Release v0.13.0 (#593) ([libp2p/go-libp2p-pubsub#593](https://github.com/libp2p/go-libp2p-pubsub/pull/593)) - Allow cancelling IWANT using IDONTWANT (#591) ([libp2p/go-libp2p-pubsub#591](https://github.com/libp2p/go-libp2p-pubsub/pull/591)) - Improve IDONTWANT Flood Protection (#590) ([libp2p/go-libp2p-pubsub#590](https://github.com/libp2p/go-libp2p-pubsub/pull/590)) - Fix the Router's Ability to Prune the Mesh Periodically (#589) ([libp2p/go-libp2p-pubsub#589](https://github.com/libp2p/go-libp2p-pubsub/pull/589)) - Add Function to Enable Application Layer to Send Direct Control Messages (#562) ([libp2p/go-libp2p-pubsub#562](https://github.com/libp2p/go-libp2p-pubsub/pull/562)) - Do not format expensive debug messages in non-debug levels in doDropRPC (#580) ([libp2p/go-libp2p-pubsub#580](https://github.com/libp2p/go-libp2p-pubsub/pull/580)) - github.com/libp2p/go-libp2p-record (v0.2.0 -> v0.3.1): - fix: missing protobuf package (#64) ([libp2p/go-libp2p-record#64](https://github.com/libp2p/go-libp2p-record/pull/64)) - release: v0.3.0 (#63) ([libp2p/go-libp2p-record#63](https://github.com/libp2p/go-libp2p-record/pull/63)) - fix: protobuf namespace conflicts (#62) ([libp2p/go-libp2p-record#62](https://github.com/libp2p/go-libp2p-record/pull/62)) - Remove gogo protobuf (#60) ([libp2p/go-libp2p-record#60](https://github.com/libp2p/go-libp2p-record/pull/60)) - github.com/libp2p/go-libp2p-routing-helpers (v0.7.4 -> v0.7.5): - new version ([libp2p/go-libp2p-routing-helpers#90](https://github.com/libp2p/go-libp2p-routing-helpers/pull/90)) - Consolidate multi-error packages by choosing one ([libp2p/go-libp2p-routing-helpers#88](https://github.com/libp2p/go-libp2p-routing-helpers/pull/88)) - update dependencies ([libp2p/go-libp2p-routing-helpers#89](https://github.com/libp2p/go-libp2p-routing-helpers/pull/89)) - github.com/multiformats/go-multiaddr (v0.14.0 -> v0.15.0): - chore: release v0.15.0 (#266) ([multiformats/go-multiaddr#266](https://github.com/multiformats/go-multiaddr/pull/266)) - refactor: Backwards compatible Encapsulate/Decapsulate/Join/NewComponent (#272) ([multiformats/go-multiaddr#272](https://github.com/multiformats/go-multiaddr/pull/272)) - refactor: keep same api as v0.14.0 for SplitFirst/SplitLast (#271) ([multiformats/go-multiaddr#271](https://github.com/multiformats/go-multiaddr/pull/271)) - refactor: Follows up on #261 (#264) ([multiformats/go-multiaddr#264](https://github.com/multiformats/go-multiaddr/pull/264)) - refactor!: make the API harder to misuse (#261) ([multiformats/go-multiaddr#261](https://github.com/multiformats/go-multiaddr/pull/261))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Hector Sanjuan | 100 | +4777/-1495 | 200 | | Marco Munizaga | 22 | +3482/-1632 | 122 | | Andrew Gillis | 69 | +1628/-1509 | 191 | | sukun | 13 | +1240/-288 | 67 | | Simon Menke | 7 | +766/-97 | 16 | | Guillaume Michel | 33 | +438/-383 | 62 | | Marcin Rataj | 24 | +494/-266 | 47 | | Sergey Gorbunov | 4 | +384/-103 | 20 | | AvyChanna | 1 | +294/-193 | 9 | | gammazero | 22 | +208/-217 | 28 | | Dennis Trautwein | 3 | +425/-0 | 8 | | web3-bot | 18 | +193/-184 | 46 | | Steven Allen | 8 | +204/-82 | 13 | | Marten Seemann | 5 | +215/-63 | 11 | | Daniel Norman | 2 | +225/-0 | 6 | | Abhinav Prakash | 1 | +190/-2 | 4 | | guillaumemichel | 3 | +93/-56 | 15 | | youyyytrok | 1 | +84/-63 | 29 | | Nishant Das | 2 | +111/-1 | 4 | | Pop Chunhapanya | 1 | +109/-0 | 2 | | Michael Muré | 7 | +78/-29 | 15 | | Jorropo | 4 | +53/-20 | 7 | | Ryan Skidmore | 1 | +62/-0 | 2 | | GITSRC | 1 | +44/-0 | 3 | | Russell Dempsey | 1 | +22/-17 | 10 | | Adin Schmahmann | 2 | +29/-8 | 3 | | Gabriel Cruz | 1 | +13/-13 | 1 | | Wlynxg | 3 | +12/-9 | 3 | | Khaled Yakdan | 1 | +11/-10 | 1 | | Yahya Hassanzadeh, Ph.D. | 1 | +17/-0 | 1 | | Can ZHANG | 2 | +15/-2 | 3 | | Pavel Zbitskiy | 1 | +13/-1 | 2 | | Yuttakhan B. | 1 | +6/-6 | 6 | | Hlib Kanunnikov | 2 | +9/-2 | 4 | | Petar Maymounkov | 1 | +7/-2 | 1 | | Prithvi Shahi | 2 | +8/-0 | 2 | | Piotr Galar | 1 | +4/-4 | 2 | | Michael Vorburger | 1 | +6/-0 | 1 | | Gus Eggert | 2 | +6/-0 | 2 | | Raúl Kripalani | 1 | +4/-0 | 1 | | linchizhen | 1 | +1/-1 | 1 | | achingbrain | 1 | +1/-1 | 1 | | Rod Vagg | 1 | +1/-1 | 1 | | Ian Davis | 1 | +1/-1 | 1 | | Fabio Bozzo | 1 | +1/-1 | 1 | ## v0.34.1 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [📦️ Important dependency updates](#-important-dependency-updates) ### Overview ### 🔦 Highlights #### 📦️ Important dependency updates - update `go-libp2p` to [v0.41.1](https://github.com/libp2p/go-libp2p/releases/tag/v0.41.1) - high impact fix from [go-libp2p#3221](https://github.com/libp2p/go-libp2p/pull/3221) improves [hole punching](https://github.com/libp2p/specs/blob/master/relay/DCUtR.md) success rate - update `quic-go` to [v0.50.1](https://github.com/quic-go/quic-go/releases/tag/v0.50.1) ================================================ FILE: docs/changelogs/v0.35.md ================================================ # Kubo changelog v0.35 This release was brought to you by the [Shipyard](http://ipshipyard.com/) team. - [v0.35.0](#v0340) ## v0.35.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [Opt-in HTTP Retrieval client](#opt-in-http-retrieval-client) - [Dedicated `Reprovider.Strategy` for MFS](#dedicated-reproviderstrategy-for-mfs) - [Experimental support for MFS as a FUSE mount point](#experimental-support-for-mfs-as-a-fuse-mount-point) - [Grid view in WebUI](#grid-view-in-webui) - [Enhanced DAG-Shaping Controls](#enhanced-dag-shaping-controls) - [New DAG-Shaping `ipfs add` Options](#new-dag-shaping-ipfs-add-options) - [Persistent DAG-Shaping `Import.*` Configuration](#persistent-dag-shaping-import-configuration) - [Updated DAG-Shaping `Import` Profiles](#updated-dag-shaping-import-profiles) - [`Datastore` Metrics Now Opt-In](#datastore-metrics-now-opt-in) - [Improved performance of data onboarding](#improved-performance-of-data-onboarding) - [Fast `ipfs add` in online mode](#fast-ipfs-add-in-online-mode) - [Optimized, dedicated queue for providing fresh CIDs](#optimized-dedicated-queue-for-providing-fresh-cids) - [Deprecated `ipfs stats provider`](#deprecated-ipfs-stats-provider) - [New `Bitswap` configuration options](#new-bitswap-configuration-options) - [New `Routing` configuration options](#new-routing-configuration-options) - [New Pebble database format config](#new-pebble-database-format-config) - [New environment variables](#new-environment-variables) - [Improved Log Output Setting](#improved-log-output-setting) - [New Repo Lock Optional Wait](#new-repo-lock-optional-wait) - [📦️ Important dependency updates](#-important-dependency-updates) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview This release brings significant UX and performance improvements to data onboarding, provisioning, and retrieval systems. New configuration options let you customize the shape of UnixFS DAGs generated during the data import, control the scope of DAGs announced on the Amino DHT, select which delegated routing endpoints are queried, and choose whether to enable HTTP retrieval alongside Bitswap over Libp2p. Continue reading for more details. ### 🔦 Highlights #### Opt-in HTTP Retrieval client This release adds experimental support for retrieving blocks directly over HTTPS (HTTP/2), complementing the existing Bitswap over Libp2p. The opt-in client enables Kubo to use [delegated routing](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingdelegatedrouters) results with `/tls/http` multiaddrs, connecting to HTTPS servers that support [Trustless HTTP Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway)'s Block Responses (`?format=raw`, `application/vnd.ipld.raw`). Fetching blocks via HTTPS (HTTP/2) simplifies infrastructure and reduces costs for storage providers by leveraging HTTP caching and CDNs. To enable this feature for testing and feedback, set: ```console $ ipfs config --json HTTPRetrieval.Enabled true ``` See [`HTTPRetrieval`](https://github.com/ipfs/kubo/blob/master/docs/config.md#httpretrieval) for more details. #### Dedicated `Reprovider.Strategy` for MFS The [Mutable File System (MFS)](https://docs.ipfs.tech/concepts/glossary/#mfs) in Kubo is a UnixFS filesystem managed with [`ipfs files`](https://docs.ipfs.tech/reference/kubo/cli/#ipfs-files) commands. It supports familiar file operations like cp and mv within a folder-tree structure, automatically updating a MerkleDAG and a "root CID" that reflects the current MFS state. Files in MFS are protected from garbage collection, offering a simpler alternative to `ipfs pin`. This makes it a popular choice for tools like [IPFS Desktop](https://docs.ipfs.tech/install/ipfs-desktop/) and the [WebUI](https://github.com/ipfs/ipfs-webui/#readme). Previously, the `pinned` reprovider strategy required manual pin management: each dataset update meant pinning the new version and unpinning the old one. Now, new strategies—`mfs` and `pinned+mfs`—let users limit announcements to data explicitly placed in MFS. This simplifies updating datasets and announcing only the latest version to the Amino DHT. Users relying on the `pinned` strategy can switch to `pinned+mfs` and use MFS alone to manage updates and announcements, eliminating the need for manual pinning and unpinning. We hope this makes it easier to publish just the data that matters to you. See [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy) for more details. #### Experimental support for MFS as a FUSE mount point The MFS root (filesystem behind the `ipfs files` API) is now available as a read/write FUSE mount point at `Mounts.MFS`. This filesystem is mounted in the same way as `Mounts.IPFS` and `Mounts.IPNS` when running `ipfs mount` or `ipfs daemon --mount`. Note that the operations supported by the MFS FUSE mountpoint are limited, since MFS doesn't store file attributes. See [`Mounts`](https://github.com/ipfs/kubo/blob/master/docs/config.md#mounts) and [`docs/fuse.md`](https://github.com/ipfs/kubo/blob/master/docs/fuse.md) for more details. #### Grid view in WebUI The WebUI, accessible at http://127.0.0.1:5001/webui/, now includes support for the grid view on the _Files_ screen: > ![image](https://github.com/user-attachments/assets/80dcf0d0-8103-426f-ae91-416fb25d32b6) #### Enhanced DAG-Shaping Controls This release advances CIDv1 support by introducing fine-grained control over UnixFS DAG shaping during data ingestion with the `ipfs add` command. Wider DAG trees (more links per node, higher fanout, larger thresholds) are beneficial for large files and directories with many files, reducing tree depth and lookup latency in high-latency networks, but they increase node size, straining memory and CPU on resource-constrained devices. Narrower trees (lower link count, lower fanout, smaller thresholds) are preferable for smaller directories, frequent updates, or low-power clients, minimizing overhead and ensuring compatibility, though they may increase traversal steps for very large datasets. Kubo now allows users to act on these tradeoffs and customize the width of the DAG created by `ipfs add` command. ##### New DAG-Shaping `ipfs add` Options Three new options allow you to override default settings for specific import operations: - `--max-file-links`: Sets the maximum number of child links for a single file chunk. - `--max-directory-links`: Defines the maximum number of child entries in a "basic" (single-chunk) directory. - Note: Directories exceeding this limit or the `Import.UnixFSHAMTDirectorySizeThreshold` are converted to HAMT-based (sharded across multiple blocks) structures. - `--max-hamt-fanout`: Specifies the maximum number of child nodes for HAMT internal structures. ##### Persistent DAG-Shaping `Import.*` Configuration You can set default values for these options using the following configuration settings: - [`Import.UnixFSFileMaxLinks`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfsfilemaxlinks) - [`Import.UnixFSDirectoryMaxLinks`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfsdirectorymaxlinks) - [`Import.UnixFSHAMTDirectoryMaxFanout`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtdirectorymaxfanout) - [`Import.UnixFSHAMTDirectorySizeThreshold`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtdirectorysizethreshold) ##### Updated DAG-Shaping `Import` Profiles The release updated configuration [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles) to incorporate these new `Import.*` settings: - Updated Profile: `test-cid-v1` now includes current defaults as explicit `Import.UnixFSFileMaxLinks=174`, `Import.UnixFSDirectoryMaxLinks=0`, `Import.UnixFSHAMTDirectoryMaxFanout=256` and `Import.UnixFSHAMTDirectorySizeThreshold=256KiB` - New Profile: `test-cid-v1-wide` adopts experimental directory DAG-shaping defaults, increasing the maximum file DAG width from 174 to 1024, HAMT fanout from 256 to 1024, and raising the HAMT directory sharding threshold from 256KiB to 1MiB, aligning with 1MiB file chunks. - Feedback: Try it out and share your thoughts at [discuss.ipfs.tech/t/should-we-profile-cids](https://discuss.ipfs.tech/t/should-we-profile-cids/18507) or [ipfs/specs#499](https://github.com/ipfs/specs/pull/499). > [!TIP] > Apply one of CIDv1 test [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles) with `ipfs config profile apply test-cid-v1[-wide]`. #### `Datastore` Metrics Now Opt-In To reduce overhead in the default configuration, datastore metrics are no longer enabled by default when initializing a Kubo repository with `ipfs init`. Metrics prefixed with `_datastore` (e.g., `flatfs_datastore_...`, `leveldb_datastore_...`) are not exposed unless explicitly enabled. For a complete list of affected default metrics, refer to [`prometheus_metrics_added_by_measure_profile`](https://github.com/ipfs/kubo/blob/master/test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_measure_profile). Convenience opt-in [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles) can be enabled at initialization time with `ipfs init --profile`: `flatfs-measure`, `pebbleds-measure`, `badgerds-measure` It is also possible to manually add the `measure` wrapper. See examples in [`Datastore.Spec`](https://github.com/ipfs/kubo/blob/master/docs/config.md#datastorespec) documentation. #### Improved performance of data onboarding This Kubo release significantly improves both the speed of ingesting data via `ipfs add` and announcing newly produced CIDs to Amino DHT. ##### Fast `ipfs add` in online mode Adding a large directory of data when `ipfs daemon` was running in online mode took a long time. A significant amount of this time was spent writing to and reading from the persisted provider queue. Due to this, many users had to shut down the daemon and perform data import in offline mode. This release fixes this known limitation, significantly improving the speed of `ipfs add`. > [!IMPORTANT] > Performing `ipfs add` of 10GiB file would take about 30 minutes. > Now it takes close to 30 seconds. Kubo v0.34: ```console $ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-100M > /dev/null 100.00 MiB / 100.00 MiB [=====================================================================] 100.00% real 0m6.464s $ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-1G > /dev/null 1000.00 MiB / 1000.00 MiB [===================================================================] 100.00% real 1m10.542s $ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-10G > /dev/null 10.00 GiB / 10.00 GiB [=======================================================================] 100.00% real 24m5.744s ``` Kubo v0.35: ```console $ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-100M > /dev/null 100.00 MiB / 100.00 MiB [=====================================================================] 100.00% real 0m0.326s $ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-1G > /dev/null 1.00 GiB / 1.00 GiB [=========================================================================] 100.00% real 0m2.819s $ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-10G > /dev/null 10.00 GiB / 10.00 GiB [=======================================================================] 100.00% real 0m28.405s ``` ##### Optimized, dedicated queue for providing fresh CIDs From `kubo` [`v0.33.0`](https://github.com/ipfs/kubo/releases/tag/v0.33.0), Bitswap stopped advertising newly added and received blocks to the DHT. Since then `boxo/provider` is responsible for the first time provide and the recurring reprovide logic. Prior to `v0.35.0`, provides and reprovides were handled together in batches, leading to delays in initial advertisements (provides). Provides and Reprovides now have separate queues, allowing for immediate provide of new CIDs and optimised batching of reprovides. ###### New `Provider` configuration options This change introduces a new configuration options: - [`Provider.Enabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providerenabled) is a global flag for disabling both [Provider](https://github.com/ipfs/kubo/blob/master/docs/config.md#provider) and [Reprovider](https://github.com/ipfs/kubo/blob/master/docs/config.md#reprovider) systems (announcing new/old CIDs to amino DHT). - [`Provider.WorkerCount`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providerworkercount) for limiting the number of concurrent provide operations, allows for fine-tuning the trade-off between announcement speed and system load when announcing new CIDs. - Removed `Experimental.StrategicProviding`. Superseded by `Provider.Enabled`, `Reprovider.Interval` and [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy). > [!TIP] > Users who need to provide large volumes of content immediately should consider setting `Routing.AcceleratedDHTClient` to `true`. If that is not enough, consider adjusting `Provider.WorkerCount` to a higher value. ###### Deprecated `ipfs stats provider` Since the `ipfs stats provider` command was displaying statistics for both provides and reprovides, this command isn't relevant anymore after separating the two queues. The successor command is `ipfs stats reprovide`, showing the same statistics, but for reprovides only. > [!NOTE] > `ipfs stats provider` still works, but is marked as deprecated and will be removed in a future release. Be mindful that the command provides only statistics about reprovides (similar to `ipfs stats reprovide`) and not the new provide queue (this will be fixed as a part of wider refactor planned for a future release). #### New `Bitswap` configuration options - [`Bitswap.Libp2pEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#bitswaplibp2penabled) determines whether Kubo will use Bitswap over libp2p (both client and server). - [`Bitswap.ServerEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#bitswapserverenabled) controls whether Kubo functions as a Bitswap server to host and respond to block requests. - [`Internal.Bitswap.ProviderSearchMaxResults`](https://github.com/ipfs/kubo/blob/master/docs/config.md#internalbitswapprovidersearchmaxresults) for adjusting the maximum number of providers bitswap client should aim at before it stops searching for new ones. #### New `Routing` configuration options - [`Routing.IgnoreProviders`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingignoreproviders) allows ignoring specific peer IDs when returned by the content routing system as providers of content. - Simplifies testing `HTTPRetrieval.Enabled` in setups where Bitswap over Libp2p and HTTP retrieval is served under different PeerIDs. - [`Routing.DelegatedRouters`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingdelegatedrouters) allows customizing HTTP routers used by Kubo when `Routing.Type` is set to `auto` or `autoclient`. - Users are now able to adjust the default routing system and directly query custom routers for increased resiliency or when dataset is too big and CIDs are not announced on Amino DHT. > [!TIP] > > For example, to use Pinata's routing endpoint in addition to IPNI at `cid.contact`: > > ```console > $ ipfs config --json Routing.DelegatedRouters '["https://cid.contact","https://indexer.pinata.cloud"]' > ``` #### New Pebble database format config This Kubo release provides node operators with more control over [Pebble's `FormatMajorVersion`](https://github.com/cockroachdb/pebble/tree/master?tab=readme-ov-file#format-major-versions). This allows testing a new Kubo release without automatically migrating Pebble datastores, keeping the ability to switch back to older Kubo. When IPFS is initialized to use the pebbleds datastore (opt-in via `ipfs init --profile=pebbleds`), the latest pebble database format is configured in the pebble datastore config as `"formatMajorVersion"`. Setting this in the datastore config prevents automatically upgrading to the latest available version when Kubo is upgraded. If a later version becomes available, the Kubo daemon prints a startup message to indicate this. The user can them update the config to use the latest format when they are certain a downgrade will not be necessary. Without the `"formatMajorVersion"` in the pebble datastore config, the database format is automatically upgraded to the latest version. If this happens, then it is possible a downgrade back to the previous version of Kubo will not work if new format is not compatible with the pebble datastore in the previous version of Kubo. When installing a new version of Kubo when `"formatMajorVersion"` is configured, automatic repository migration (`ipfs daemon with --migrate=true`) does not upgrade this to the latest available version. This is done because a user may have reasons not to upgrade the pebble database format, and may want to be able to downgrade Kubo if something else is not working in the new version. If the configured pebble database format in the old Kubo is not supported in the new Kubo, then the configured version must be updated and the old Kubo run, before installing the new Kubo. See other caveats and configuration options at [`kubo/docs/datastores.md#pebbleds`](https://github.com/ipfs/kubo/blob/master/docs/datastores.md#pebbleds) #### New environment variables The [`environment-variables.md`](https://github.com/ipfs/kubo/blob/master/docs/environment-variables.md) was extended with two new features: ##### Improved Log Output Setting When stderr and/or stdout options are configured or specified by the `GOLOG_OUTPUT` environ variable, log only to the output(s) specified. For example: - `GOLOG_OUTPUT="stderr"` logs only to stderr - `GOLOG_OUTPUT="stdout"` logs only to stdout - `GOLOG_OUTPUT="stderr+stdout"` logs to both stderr and stdout ##### New Repo Lock Optional Wait The environment variable `IPFS_WAIT_REPO_LOCK` specifies the amount of time to wait for the repo lock. Set the value of this variable to a string that can be [parsed](https://pkg.go.dev/time@go1.24.3#ParseDuration) as a golang `time.Duration`. For example: ``` IPFS_WAIT_REPO_LOCK="15s" ``` If the lock cannot be acquired because someone else has the lock, and `IPFS_WAIT_REPO_LOCK` is set to a valid value, then acquiring the lock is retried every second until the lock is acquired or the specified wait time has elapsed. #### 📦️ Important dependency updates - update `boxo` to [v0.30.0](https://github.com/ipfs/boxo/releases/tag/v0.30.0) - update `ipfs-webui` to [v4.7.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.7.0) - update `go-ds-pebble` to [v0.5.0](https://github.com/ipfs/go-ds-pebble/releases/tag/v0.5.0) - update `pebble` to [v2.0.3](https://github.com/cockroachdb/pebble/releases/tag/v2.0.3) - update `go-libp2p-pubsub` to [v0.13.1](https://github.com:/libp2p/go-libp2p-pubsub/releases/tag/v0.13.1) - update `go-libp2p-kad-dht` to [v0.33.1](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.33.1) (incl. [v0.33.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.33.0), [v0.32.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.32.0), [v0.31.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.31.0)) - update `go-log` to [v2.6.0](https://github.com/ipfs/go-log/releases/tag/v2.6.0) - update `p2p-forge/client` to [v0.5.1](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.5.1) ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore(version): 0.35.0 - fix: go-libp2p-kad-dht v0.33.1 (#10814) ([ipfs/kubo#10814](https://github.com/ipfs/kubo/pull/10814)) - fix: p2p-forge v0.5.1 ignoring /p2p-circuit (#10813) ([ipfs/kubo#10813](https://github.com/ipfs/kubo/pull/10813)) - chore(version): 0.35.0-rc2 - fix(fuse): ipns error handling and friendly errors (#10807) ([ipfs/kubo#10807](https://github.com/ipfs/kubo/pull/10807)) - fix(config): wire up `Provider.Enabled` flag (#10804) ([ipfs/kubo#10804](https://github.com/ipfs/kubo/pull/10804)) - docs(changelog): go-libp2p-kad-dht - chore(version): 0.35.0-rc1 - feat: IPFS_WAIT_REPO_LOCK (#10797) ([ipfs/kubo#10797](https://github.com/ipfs/kubo/pull/10797)) - logging: upgrade to go-log/v2 v2.6.0 (#10798) ([ipfs/kubo#10798](https://github.com/ipfs/kubo/pull/10798)) - chore: ensure /mfs is present in docker - feat(fuse): Expose MFS as FUSE mount point (#10781) ([ipfs/kubo#10781](https://github.com/ipfs/kubo/pull/10781)) - feat: opt-in http retrieval client (#10772) ([ipfs/kubo#10772](https://github.com/ipfs/kubo/pull/10772)) - Update go-libp2p-pubsub to v0.13.1 (#10795) ([ipfs/kubo#10795](https://github.com/ipfs/kubo/pull/10795)) - feat(config): ability to disable Bitswap fully or just server (#10782) ([ipfs/kubo#10782](https://github.com/ipfs/kubo/pull/10782)) - refactor: make datastore metrics opt-in (#10788) ([ipfs/kubo#10788](https://github.com/ipfs/kubo/pull/10788)) - feat(pebble): support pinning `FormatMajorVersion` (#10789) ([ipfs/kubo#10789](https://github.com/ipfs/kubo/pull/10789)) - feat: `Provider.WorkerCount` and `stats reprovide` (#10779) ([ipfs/kubo#10779](https://github.com/ipfs/kubo/pull/10779)) - Upgrade to Boxo v0.30.0 (#10794) ([ipfs/kubo#10794](https://github.com/ipfs/kubo/pull/10794)) - docs: use latest fuse package (#10791) ([ipfs/kubo#10791](https://github.com/ipfs/kubo/pull/10791)) - remove duplicate words (#10790) ([ipfs/kubo#10790](https://github.com/ipfs/kubo/pull/10790)) - feat(config): `ipfs add` and `Import` options for controlling UnixFS DAG Width (#10774) ([ipfs/kubo#10774](https://github.com/ipfs/kubo/pull/10774)) - feat(config): expose ProviderSearchMaxResults (#10773) ([ipfs/kubo#10773](https://github.com/ipfs/kubo/pull/10773)) - feat: ipfs-webui v4.7.0 (#10780) ([ipfs/kubo#10780](https://github.com/ipfs/kubo/pull/10780)) - feat: partial DAG provides with Reprovider.Strategy=mfs|pinned+mfs (#10754) ([ipfs/kubo#10754](https://github.com/ipfs/kubo/pull/10754)) - chore: update url - docs: known issues with file/urlstores (#10768) ([ipfs/kubo#10768](https://github.com/ipfs/kubo/pull/10768)) - fix: Add IPFS & IPNS path details to error (re. #10762) (#10770) ([ipfs/kubo#10770](https://github.com/ipfs/kubo/pull/10770)) - docs: Fix typo in v0.34 changelog (#10771) ([ipfs/kubo#10771](https://github.com/ipfs/kubo/pull/10771)) - Support WithIgnoreProviders() in provider query manager ([ipfs/kubo#10765](https://github.com/ipfs/kubo/pull/10765)) - Merge release v0.34.1 ([ipfs/kubo#10766](https://github.com/ipfs/kubo/pull/10766)) - fix: reprovides warning (#10761) ([ipfs/kubo#10761](https://github.com/ipfs/kubo/pull/10761)) - Merge release v0.34.0 ([ipfs/kubo#10759](https://github.com/ipfs/kubo/pull/10759)) - feat: ipfs-webui v4.6 (#10756) ([ipfs/kubo#10756](https://github.com/ipfs/kubo/pull/10756)) - docs(readme): update min. requirements + cleanup (#10750) ([ipfs/kubo#10750](https://github.com/ipfs/kubo/pull/10750)) - Upgrade to Boxo v0.29.1 (#10755) ([ipfs/kubo#10755](https://github.com/ipfs/kubo/pull/10755)) - Nonfunctional (#10753) ([ipfs/kubo#10753](https://github.com/ipfs/kubo/pull/10753)) - provider: buffer pin providers ([ipfs/kubo#10746](https://github.com/ipfs/kubo/pull/10746)) - chore: 0.35.0-dev - github.com/ipfs/boxo (v0.29.1 -> v0.30.0): - Release v0.30.0 ([ipfs/boxo#915](https://github.com/ipfs/boxo/pull/915)) - feat(bitswap): add option to disable Bitswap server (#911) ([ipfs/boxo#911](https://github.com/ipfs/boxo/pull/911)) - provider: dedicated provide queue (#907) ([ipfs/boxo#907](https://github.com/ipfs/boxo/pull/907)) - provider: deduplicate cids in queue (#910) ([ipfs/boxo#910](https://github.com/ipfs/boxo/pull/910)) - feat(unixfs/mfs): support MaxLinks and MaxHAMTFanout (#906) ([ipfs/boxo#906](https://github.com/ipfs/boxo/pull/906)) - feat(ipld/unixfs): DagModifier: allow specifying MaxLinks per file (#898) ([ipfs/boxo#898](https://github.com/ipfs/boxo/pull/898)) - feat: NewDAGProvider to walk partial DAGs in offline mode (#905) ([ipfs/boxo#905](https://github.com/ipfs/boxo/pull/905)) - gateway: check for UseSubdomains with IP addresses (#903) ([ipfs/boxo#903](https://github.com/ipfs/boxo/pull/903)) - feat(gateway): add cid copy button to directory listings (#899) ([ipfs/boxo#899](https://github.com/ipfs/boxo/pull/899)) - Improve performance of data onboarding (#888) ([ipfs/boxo#888](https://github.com/ipfs/boxo/pull/888)) - bitswap: add requestsInFlight metric ([ipfs/boxo#904](https://github.com/ipfs/boxo/pull/904)) - provider: simplify reprovide (#890) ([ipfs/boxo#890](https://github.com/ipfs/boxo/pull/890)) - Upgrade to go-libp2p v0.41.1 ([ipfs/boxo#896](https://github.com/ipfs/boxo/pull/896)) - Update RELEASE.md ([ipfs/boxo#892](https://github.com/ipfs/boxo/pull/892)) - changelog: document bsnet import path change ([ipfs/boxo#891](https://github.com/ipfs/boxo/pull/891)) - fix(gateway): preserve query parameters on _redirects ([ipfs/boxo#886](https://github.com/ipfs/boxo/pull/886)) - bitswap/httpnet: Add WithDenylist option ([ipfs/boxo#877](https://github.com/ipfs/boxo/pull/877)) - github.com/ipfs/go-block-format (v0.2.0 -> v0.2.1): - Update version (#60) ([ipfs/go-block-format#60](https://github.com/ipfs/go-block-format/pull/60)) - Update go-ipfs-util to use boxo (#52) ([ipfs/go-block-format#52](https://github.com/ipfs/go-block-format/pull/52)) - github.com/ipfs/go-ds-pebble (v0.4.4 -> v0.5.0): - new version (#53) ([ipfs/go-ds-pebble#53](https://github.com/ipfs/go-ds-pebble/pull/53)) - Upgrade to pebble v2.0.3 (#45) ([ipfs/go-ds-pebble#45](https://github.com/ipfs/go-ds-pebble/pull/45)) - github.com/ipfs/go-fs-lock (v0.0.7 -> v0.1.1): - new version (#48) ([ipfs/go-fs-lock#48](https://github.com/ipfs/go-fs-lock/pull/48)) - Return original error when WaitLock times out (#47) ([ipfs/go-fs-lock#47](https://github.com/ipfs/go-fs-lock/pull/47)) - new version (#45) ([ipfs/go-fs-lock#45](https://github.com/ipfs/go-fs-lock/pull/45)) - Add WaitLock function (#44) ([ipfs/go-fs-lock#44](https://github.com/ipfs/go-fs-lock/pull/44)) - sync: update CI config files ([ipfs/go-fs-lock#30](https://github.com/ipfs/go-fs-lock/pull/30)) - sync: update CI config files (#27) ([ipfs/go-fs-lock#27](https://github.com/ipfs/go-fs-lock/pull/27)) - sync: update CI config files ([ipfs/go-fs-lock#25](https://github.com/ipfs/go-fs-lock/pull/25)) - github.com/ipfs/go-log/v2 (v2.5.1 -> v2.6.0): - new version (#155) ([ipfs/go-log#155](https://github.com/ipfs/go-log/pull/155)) - feat: only log to stderr or to stdout or both if configured (#154) ([ipfs/go-log#154](https://github.com/ipfs/go-log/pull/154)) - ci: uci/copy-templates ([ipfs/go-log#145](https://github.com/ipfs/go-log/pull/145)) - sync: update CI config files (#137) ([ipfs/go-log#137](https://github.com/ipfs/go-log/pull/137)) - github.com/libp2p/go-libp2p-kad-dht (v0.30.2 -> v0.33.1): - chore: release v0.33.1 (#1088) ([libp2p/go-libp2p-kad-dht#1088](https://github.com/libp2p/go-libp2p-kad-dht/pull/1088)) - fix(fullrt): mutex cleanup (#1087) ([libp2p/go-libp2p-kad-dht#1087](https://github.com/libp2p/go-libp2p-kad-dht/pull/1087)) - fix: use correct mutex for reading keyToPeerMap (#1086) ([libp2p/go-libp2p-kad-dht#1086](https://github.com/libp2p/go-libp2p-kad-dht/pull/1086)) - fix: fullrt kMapLk unlock (#1085) ([libp2p/go-libp2p-kad-dht#1085](https://github.com/libp2p/go-libp2p-kad-dht/pull/1085)) - chore: release v0.33.0 (#1083) ([libp2p/go-libp2p-kad-dht#1083](https://github.com/libp2p/go-libp2p-kad-dht/pull/1083)) - fix/updates to use context passed in New function for context cancellation (#1081) ([libp2p/go-libp2p-kad-dht#1081](https://github.com/libp2p/go-libp2p-kad-dht/pull/1081)) - chore: release v0.31.1 (#1079) ([libp2p/go-libp2p-kad-dht#1079](https://github.com/libp2p/go-libp2p-kad-dht/pull/1079)) - fix: netsize warning (#1077) ([libp2p/go-libp2p-kad-dht#1077](https://github.com/libp2p/go-libp2p-kad-dht/pull/1077)) - fix: use correct message type attribute in metrics (#1076) ([libp2p/go-libp2p-kad-dht#1076](https://github.com/libp2p/go-libp2p-kad-dht/pull/1076)) - chore: bump go-log to v2 (#1074) ([libp2p/go-libp2p-kad-dht#1074](https://github.com/libp2p/go-libp2p-kad-dht/pull/1074)) - release v0.31.0 (#1072) ([libp2p/go-libp2p-kad-dht#1072](https://github.com/libp2p/go-libp2p-kad-dht/pull/1072)) - query: ip diversity filter (#1070) ([libp2p/go-libp2p-kad-dht#1070](https://github.com/libp2p/go-libp2p-kad-dht/pull/1070)) - tests: fix flaky TestProvidesExpire (#1069) ([libp2p/go-libp2p-kad-dht#1069](https://github.com/libp2p/go-libp2p-kad-dht/pull/1069)) - refactor: replace fmt.Errorf with errors.New when not formatting is required (#1067) ([libp2p/go-libp2p-kad-dht#1067](https://github.com/libp2p/go-libp2p-kad-dht/pull/1067)) - fix: error on no valid provs (#1065) ([libp2p/go-libp2p-kad-dht#1065](https://github.com/libp2p/go-libp2p-kad-dht/pull/1065)) - cleanup: remove deprecated opt package (#1064) ([libp2p/go-libp2p-kad-dht#1064](https://github.com/libp2p/go-libp2p-kad-dht/pull/1064)) - cleanup: fullrt ([libp2p/go-libp2p-kad-dht#1062](https://github.com/libp2p/go-libp2p-kad-dht/pull/1062)) - fix: remove peerstore no-op (#1063) ([libp2p/go-libp2p-kad-dht#1063](https://github.com/libp2p/go-libp2p-kad-dht/pull/1063)) - tests: flaky TestSearchValue (dual) (#1060) ([libp2p/go-libp2p-kad-dht#1060](https://github.com/libp2p/go-libp2p-kad-dht/pull/1060)) - github.com/libp2p/go-libp2p-kbucket (v0.6.5 -> v0.7.0): - chore: release v0.7.0 (#143) ([libp2p/go-libp2p-kbucket#143](https://github.com/libp2p/go-libp2p-kbucket/pull/143)) - peerdiversity: export IPGroupKey (#141) ([libp2p/go-libp2p-kbucket#141](https://github.com/libp2p/go-libp2p-kbucket/pull/141)) - fix: flaky TestUsefulNewPeer (#140) ([libp2p/go-libp2p-kbucket#140](https://github.com/libp2p/go-libp2p-kbucket/pull/140)) - fix: flaky TestTableFindMultipleBuckets (#139) ([libp2p/go-libp2p-kbucket#139](https://github.com/libp2p/go-libp2p-kbucket/pull/139)) - github.com/libp2p/go-libp2p-pubsub (v0.13.0 -> v0.13.1): - feat: WithValidatorData publishing option (#603) ([libp2p/go-libp2p-pubsub#603](https://github.com/libp2p/go-libp2p-pubsub/pull/603)) - feat: avoid repeated checksum calculations (#599) ([libp2p/go-libp2p-pubsub#599](https://github.com/libp2p/go-libp2p-pubsub/pull/599)) - github.com/libp2p/go-yamux/v4 (v4.0.1 -> v4.0.2): - Release v4.0.2 (#124) ([libp2p/go-yamux#124](https://github.com/libp2p/go-yamux/pull/124)) - fix: remove noisy logs (#116) ([libp2p/go-yamux#116](https://github.com/libp2p/go-yamux/pull/116)) - check deadline before sending a message (#114) ([libp2p/go-yamux#114](https://github.com/libp2p/go-yamux/pull/114)) - only check KeepAliveInterval if keep-alive are enabled (#113) ([libp2p/go-yamux#113](https://github.com/libp2p/go-yamux/pull/113)) - ci: uci/copy-templates ([libp2p/go-yamux#109](https://github.com/libp2p/go-yamux/pull/109))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Hector Sanjuan | 16 | +2662/-590 | 71 | | Guillaume Michel | 27 | +1339/-714 | 69 | | Andrew Gillis | 22 | +1056/-377 | 54 | | Sergey Gorbunov | 1 | +962/-42 | 26 | | Marcin Rataj | 19 | +714/-133 | 47 | | IGP | 2 | +419/-35 | 11 | | GITSRC | 1 | +90/-1 | 3 | | guillaumemichel | 1 | +21/-43 | 1 | | blockchainluffy | 1 | +27/-26 | 8 | | web3-bot | 9 | +21/-22 | 13 | | VersaliX | 1 | +31/-2 | 4 | | gammazero | 5 | +18/-5 | 5 | | Hlib Kanunnikov | 1 | +14/-4 | 1 | | diogo464 | 1 | +6/-7 | 1 | | Asutorufa | 2 | +7/-1 | 2 | | Russell Dempsey | 1 | +6/-1 | 1 | | Steven Allen | 1 | +1/-5 | 1 | | Michael Vorburger | 2 | +3/-3 | 2 | | Aayush Rajasekaran | 1 | +2/-2 | 1 | | sukun | 1 | +1/-1 | 1 | ================================================ FILE: docs/changelogs/v0.36.md ================================================ # Kubo changelog v0.36 This release was brought to you by the [Interplanetary Shipyard](https://ipshipyard.com/) team. - [v0.36.0](#v0360) ## v0.36.0 [](https://github.com/user-attachments/assets/0d830631-7b92-48ca-8ce9-b537e1479dfb) - [Overview](#overview) - [🔦 Highlights](#-highlights) - [HTTP Retrieval Client Now Enabled by Default](#http-retrieval-client-now-enabled-by-default) - [Bitswap Broadcast Reduction](#bitswap-broadcast-reduction) - [Update go-log to v2](#update-go-log-to-v2) - [Kubo now uses AutoNATv2 as a client](#kubo-now-uses-autonatv2-as-a-client) - [Smarter AutoTLS registration](#smarter-autotls-registration) - [Overwrite option for files cp command](#overwrite-option-for-files-cp-command) - [Gateway now supports negative HTTP Range requests](#gateway-now-supports-negative-http-range-requests) - [Option for `filestore` command to remove bad blocks](#option-for-filestore-command-to-remove-bad-blocks) - [`ConnMgr.SilencePeriod` configuration setting exposed](#connmgrsilenceperiod-configuration-setting-exposed) - [Fix handling of EDITOR env var](#fix-handling-of-editor-env-var) - [📦️ Important dependency updates](#-important-dependency-updates) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### HTTP Retrieval Client Now Enabled by Default This release promotes the HTTP Retrieval client from an experimental feature to a standard feature that is enabled by default. When possible, Kubo will retrieve blocks over plain HTTPS (HTTP/2) without any extra user configuration. See [`HTTPRetrieval`](https://github.com/ipfs/kubo/blob/master/docs/config.md#httpretrieval) for more details. #### Bitswap Broadcast Reduction The Bitswap client now supports broadcast reduction logic, which is enabled by default. This feature significantly reduces the number of broadcast messages sent to peers, resulting in lower bandwidth usage during load spikes. The overall logic works by sending to non-local peers only if those peers have previously replied that they have data blocks. To minimize impact on existing workloads, by default, broadcasts are still always sent to peers on the local network, or the ones defined in `Peering.Peers`. At Shipyard, we conducted A/B testing on our internal Kubo staging gateway with organic CID requests to `ipfs.io`. While these results may not exactly match your specific workload, the benefits proved significant enough to make this feature default. Here are the key findings: - **Dramatic Resource Usage Reduction:** Internal testing demonstrated a reduction in Bitswap broadcast messages by 80-98% and network bandwidth savings of 50-95%, with the greatest improvements occurring during high traffic and peer spikes. These efficiency gains lower operational costs of running Kubo under high load and improve the IPFS Mainnet (which is >80% Kubo-based) by reducing ambient traffic for all connected peers. - **Improved Memory Stability:** Memory stays stable even during major CID request spikes that increase peer count, preventing the out-of-memory (OOM) issues found in earlier Kubo versions. - **Data Retrieval Performance Remains Strong:** Our tests suggest that Kubo gateway hosts with broadcast reduction enabled achieve similar or better HTTP 200 success rates compared to version 0.35, while maintaining equivalent or higher want-have responses and unique blocks received. For more information about our A/B tests, see [kubo#10825](https://github.com/ipfs/kubo/pull/10825). To revert to the previous behavior for your own A/B testing, set `Internal.Bitswap.BroadcastControl.Enable` to `false` and monitor relevant metrics (`ipfs_bitswap_bcast_skips_total`, `ipfs_bitswap_haves_received`, `ipfs_bitswap_unique_blocks_received`, `ipfs_bitswap_wanthaves_broadcast`, HTTP 200 success rate). For a description of the configuration items, see the documentation of [`Internal.Bitswap.BroadcastControl`](https://github.com/ipfs/kubo/blob/master/docs/config.md#internalbitswapbroadcastcontrol). #### Update go-log to v2 go-log v2 has been out for quite a while now and it's time to deprecate v1. - Replace all use of `go-log` with `go-log/v2` - Makes `/api/v0/log/tail` useful over HTTP - Fixes `ipfs log tail` - Removes support for `ContextWithLoggable` as this is not needed for tracing-like functionality #### Kubo now uses AutoNATv2 as a client This Kubo release starts utilizing [AutoNATv2](https://github.com/libp2p/specs/blob/master/autonat/autonat-v2.md) client functionality. go-libp2p v0.42 supports and depends on both AutoNATv1 and v2, and Autorelay feature continues to use v1. go-libp2p v0.43+ will discontinue internal use of AutoNATv1. We will maintain support for both v1 and v2 until then, though v1 will gradually be deprecated and ultimately removed. ##### Smarter AutoTLS registration This update to libp2p and [AutoTLS](https://github.com/ipfs/kubo/blob/master/docs/config.md#autotls) incorporates AutoNATv2 changes. It aims to reduce false-positive scenarios where AutoTLS certificate registration occurred before a publicly dialable multiaddr was available. This should result in fewer error logs during node start, especially when IPv6 and/or IPv4 NATs with UPnP/PCP/NAT-PMP are at play. #### Overwrite option for files cp command The `ipfs files cp` command has a `--force` option to allow it to overwrite existing files. Attempting to overwrite an existing directory results in an error. #### Gateway now supports negative HTTP Range requests The latest update to `boxo/gateway` adds support for negative HTTP Range requests, achieving [gateway-conformance@v0.8](https://github.com/ipfs/gateway-conformance/releases/tag/v0.8.0) compatibility. This provides greater interoperability with generic HTTP-based tools. For example, [WebRecorder](https://webrecorder.net/archivewebpage/)'s https://replayweb.page/ can now directly load website snapshots from Kubo-backed URLs. #### Option for `filestore` command to remove bad blocks The [experimental `filestore`](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-filestore) command has a new option, `--remove-bad-blocks`, to verify objects in the filestore and remove those that fail verification. #### `ConnMgr.SilencePeriod` configuration setting exposed This connection manager option controls how often connections are swept and potentially terminated. See the [ConnMgr documentation](https://github.com/ipfs/kubo/blob/master/docs/config.md#swarmconnmgrsilenceperiod). #### Fix handling of EDITOR env var The `ipfs config edit` command did not correctly handle the `EDITOR` environment variable when its value contains flags and arguments, i.e. `EDITOR=emacs -nw`. The command was treating the entire value of `$EDITOR` as the name of the editor command. This has been fixed to parse the value of `$EDITOR` into separate args, respecting shell quoting. #### 📦️ Important dependency updates - update `go-libp2p` to [v0.42.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.42.0) - update `go-libp2p-kad-dht` to [v0.33.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.33.0) - update `boxo` to [v0.33.0](https://github.com/ipfs/boxo/releases/tag/v0.33.0) (incl. [v0.32.0](https://github.com/ipfs/boxo/releases/tag/v0.32.0)) - update `gateway-conformance` to [v0.8](https://github.com/ipfs/gateway-conformance/releases/tag/v0.8.0) - update `p2p-forge/client` to [v0.6.0](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.6.0) - update `github.com/cockroachdb/pebble/v2` to [v2.0.6](https://github.com/cockroachdb/pebble/releases/tag/v2.0.6) for Go 1.25 support ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: 0.36.0 - chore: update links in markdown - chore: 0.36.0-rc2 - feat(httpnet): gather metrics for allowlist - chore: changelog - test: TestEditorParsing - fix: handling of EDITOR env var (#10855) ([ipfs/kubo#10855](https://github.com/ipfs/kubo/pull/10855)) - refactor: use slices.Sort where appropriate (#10858) ([ipfs/kubo#10858](https://github.com/ipfs/kubo/pull/10858)) - Upgrade to Boxo v0.33.0 (#10857) ([ipfs/kubo#10857](https://github.com/ipfs/kubo/pull/10857)) - chore: Upgrade github.com/cockroachdb/pebble/v2 to v2.0.6 for Go 1.25 support (#10850) ([ipfs/kubo#10850](https://github.com/ipfs/kubo/pull/10850)) - core:constructor: add a log line about http retrieval - chore: p2p-forge v0.6.0 + go-libp2p 0.42.0 (#10840) ([ipfs/kubo#10840](https://github.com/ipfs/kubo/pull/10840)) - docs: fix minor typos (#10849) ([ipfs/kubo#10849](https://github.com/ipfs/kubo/pull/10849)) - Replace use of go-car v1 with go-car/v2 (#10845) ([ipfs/kubo#10845](https://github.com/ipfs/kubo/pull/10845)) - chore: v0.36.0-rc1 - chore: deduplicate 0.36 changelog - feat(config): connmgr: expose silence period (#10827) ([ipfs/kubo#10827](https://github.com/ipfs/kubo/pull/10827)) - bitswap/client: configurable broadcast reduction (#10825) ([ipfs/kubo#10825](https://github.com/ipfs/kubo/pull/10825)) - Upgrade to Boxo v0.32.0 (#10839) ([ipfs/kubo#10839](https://github.com/ipfs/kubo/pull/10839)) - feat: HTTP retrieval enabled by default (#10836) ([ipfs/kubo#10836](https://github.com/ipfs/kubo/pull/10836)) - feat: AutoTLS with AutoNATv2 client (#10835) ([ipfs/kubo#10835](https://github.com/ipfs/kubo/pull/10835)) - commands: add `--force` option to `files cp` command (#10823) ([ipfs/kubo#10823](https://github.com/ipfs/kubo/pull/10823)) - docs/env variables: Document LIBP2P_SWARM_FD_LIMIT ([ipfs/kubo#10828](https://github.com/ipfs/kubo/pull/10828)) - test: fix "invert" commands in sharness tests (#9652) ([ipfs/kubo#9652](https://github.com/ipfs/kubo/pull/9652)) - Ivan386/filestore fix (#7474) ([ipfs/kubo#7474](https://github.com/ipfs/kubo/pull/7474)) - wrap user-facing mfs.Lookup error (#10821) ([ipfs/kubo#10821](https://github.com/ipfs/kubo/pull/10821)) - Update fuse docs with FreeBSD specifics (#10820) ([ipfs/kubo#10820](https://github.com/ipfs/kubo/pull/10820)) - Minor wording fixes in docs (#10822) ([ipfs/kubo#10822](https://github.com/ipfs/kubo/pull/10822)) - fix(gateway): gateway-conformance v0.8 (#10818) ([ipfs/kubo#10818](https://github.com/ipfs/kubo/pull/10818)) - Upgrade to Boxo v0.31.0 (#10819) ([ipfs/kubo#10819](https://github.com/ipfs/kubo/pull/10819)) - Merge release v0.35.0 ([ipfs/kubo#10815](https://github.com/ipfs/kubo/pull/10815)) - fix: go-libp2p-kad-dht v0.33.1 (#10814) ([ipfs/kubo#10814](https://github.com/ipfs/kubo/pull/10814)) - fix: p2p-forge v0.5.1 ignoring /p2p-circuit (#10813) ([ipfs/kubo#10813](https://github.com/ipfs/kubo/pull/10813)) - Upgrade go-libp2p-kad-dht to v0.33.0 (#10811) ([ipfs/kubo#10811](https://github.com/ipfs/kubo/pull/10811)) - chore: use go-log/v2 (#10801) ([ipfs/kubo#10801](https://github.com/ipfs/kubo/pull/10801)) - fix(fuse): ipns error handling and friendly errors (#10807) ([ipfs/kubo#10807](https://github.com/ipfs/kubo/pull/10807)) - fix(config): wire up `Provider.Enabled` flag (#10804) ([ipfs/kubo#10804](https://github.com/ipfs/kubo/pull/10804)) - chore: bump version to 0.36.0-dev - github.com/ipfs/boxo (v0.30.0 -> v0.33.0): - Release v0.33.0 ([ipfs/boxo#974](https://github.com/ipfs/boxo/pull/974)) - [skip changelog] fix sending empty want from #968 (#975) ([ipfs/boxo#975](https://github.com/ipfs/boxo/pull/975)) - minor typo fixes (#972) ([ipfs/boxo#972](https://github.com/ipfs/boxo/pull/972)) - fix: normalize delegated /routing/v1 urls (#971) ([ipfs/boxo#971](https://github.com/ipfs/boxo/pull/971)) - bitswap/client: Set DontHaveTimeout MinTimeout to 50ms (#965) ([ipfs/boxo#965](https://github.com/ipfs/boxo/pull/965)) - remove unused code (#967) ([ipfs/boxo#967](https://github.com/ipfs/boxo/pull/967)) - Fix sending extra wants (#968) ([ipfs/boxo#968](https://github.com/ipfs/boxo/pull/968)) - Handle Bitswap messages without `Wantlist` (#961) ([ipfs/boxo#961](https://github.com/ipfs/boxo/pull/961)) - bitswap/httpnet: limit metric cardinality ([ipfs/boxo#957](https://github.com/ipfs/boxo/pull/957)) - bitswap/httpnet: Sanitize allow/denylist inputs ([ipfs/boxo#964](https://github.com/ipfs/boxo/pull/964)) - Bitswap: Set DontHaveTimeout/MinTimeout to 200ms. ([ipfs/boxo#959](https://github.com/ipfs/boxo/pull/959)) - upgrade go-libp2p to v0.42.0 (#960) ([ipfs/boxo#960](https://github.com/ipfs/boxo/pull/960)) - refactor: use the built-in max/min to simplify the code [skip changelog] (#941) ([ipfs/boxo#941](https://github.com/ipfs/boxo/pull/941)) - bitswap/httpnet: adjust error logging (#958) ([ipfs/boxo#958](https://github.com/ipfs/boxo/pull/958)) - docs: reprovider metrics name in changelog (#953) ([ipfs/boxo#953](https://github.com/ipfs/boxo/pull/953)) - Release v0.32.0 (#952) ([ipfs/boxo#952](https://github.com/ipfs/boxo/pull/952)) - Remove redundant loop over published blocks (#950) ([ipfs/boxo#950](https://github.com/ipfs/boxo/pull/950)) - Fix links in README.md (#948) ([ipfs/boxo#948](https://github.com/ipfs/boxo/pull/948)) - chore(provider): meaningful info level log (#940) ([ipfs/boxo#940](https://github.com/ipfs/boxo/pull/940)) - feat(provider): reprovide metrics (#944) ([ipfs/boxo#944](https://github.com/ipfs/boxo/pull/944)) - ci: set up golangci lint in boxo (#943) ([ipfs/boxo#943](https://github.com/ipfs/boxo/pull/943)) - Do not return error from notify blocks when bitswap shutdown (#947) ([ipfs/boxo#947](https://github.com/ipfs/boxo/pull/947)) - bitswap/client: broadcast reduction and metrics (#937) ([ipfs/boxo#937](https://github.com/ipfs/boxo/pull/937)) - fix: typo in HAMT error message ([ipfs/boxo#945](https://github.com/ipfs/boxo/pull/945)) - bitswap/httpnet: expose the errors on connect when connection impossible ([ipfs/boxo#939](https://github.com/ipfs/boxo/pull/939)) - fix(unixfs): int check (#936) ([ipfs/boxo#936](https://github.com/ipfs/boxo/pull/936)) - Remove WithPeerLedger option and PeerLedger interface (#938) ([ipfs/boxo#938](https://github.com/ipfs/boxo/pull/938)) - fix(gateway): support suffix range requests (#922) ([ipfs/boxo#922](https://github.com/ipfs/boxo/pull/922)) - Release v0.31.0 ([ipfs/boxo#934](https://github.com/ipfs/boxo/pull/934)) - Revert "Remove an unused timestamp from traceability.Block" (#931) ([ipfs/boxo#931](https://github.com/ipfs/boxo/pull/931)) - update changelog (#930) ([ipfs/boxo#930](https://github.com/ipfs/boxo/pull/930)) - Deprecate WithPeerLedger option for bitswap server (#929) ([ipfs/boxo#929](https://github.com/ipfs/boxo/pull/929)) - refactor: use a more efficient querying method (#921) ([ipfs/boxo#921](https://github.com/ipfs/boxo/pull/921)) - Use go-car/v2 for reading CAR files in gateway backend (#927) ([ipfs/boxo#927](https://github.com/ipfs/boxo/pull/927)) - Upgrade go-libp2p-kad-dht v0.33.1 (#924) ([ipfs/boxo#924](https://github.com/ipfs/boxo/pull/924)) - bitswap/httpnet: Disconnect peers after client errors ([ipfs/boxo#919](https://github.com/ipfs/boxo/pull/919)) - Remove an unused timestamp from traceability.Block (#923) ([ipfs/boxo#923](https://github.com/ipfs/boxo/pull/923)) - fix(bitswap/httpnet): idempotent Stop() (#920) ([ipfs/boxo#920](https://github.com/ipfs/boxo/pull/920)) - Update dependencies (#916) ([ipfs/boxo#916](https://github.com/ipfs/boxo/pull/916)) - github.com/ipfs/go-block-format (v0.2.1 -> v0.2.2): - new version (#62) ([ipfs/go-block-format#62](https://github.com/ipfs/go-block-format/pull/62)) - Use value receivers for `BasicBlock` methods (#61) ([ipfs/go-block-format#61](https://github.com/ipfs/go-block-format/pull/61)) - github.com/ipfs/go-ds-badger4 (v0.1.5 -> v0.1.8): - new version (#7) ([ipfs/go-ds-badger4#7](https://github.com/ipfs/go-ds-badger4/pull/7)) - update version (#5) ([ipfs/go-ds-badger4#5](https://github.com/ipfs/go-ds-badger4/pull/5)) - update dependencies (#4) ([ipfs/go-ds-badger4#4](https://github.com/ipfs/go-ds-badger4/pull/4)) - new version ([ipfs/go-ds-badger4#3](https://github.com/ipfs/go-ds-badger4/pull/3)) - use go-datastore without goprocess ([ipfs/go-ds-badger4#2](https://github.com/ipfs/go-ds-badger4/pull/2)) - github.com/ipfs/go-ds-pebble (v0.5.0 -> v0.5.1): - new version (#55) ([ipfs/go-ds-pebble#55](https://github.com/ipfs/go-ds-pebble/pull/55)) - github.com/ipfs/go-ipfs-cmds (v0.14.1 -> v0.15.0): - new version (#287) ([ipfs/go-ipfs-cmds#287](https://github.com/ipfs/go-ipfs-cmds/pull/287)) - minor document updates (#286) ([ipfs/go-ipfs-cmds#286](https://github.com/ipfs/go-ipfs-cmds/pull/286)) - Update go log v2 (#285) ([ipfs/go-ipfs-cmds#285](https://github.com/ipfs/go-ipfs-cmds/pull/285)) - ci: uci/update-go (#281) ([ipfs/go-ipfs-cmds#281](https://github.com/ipfs/go-ipfs-cmds/pull/281)) - github.com/ipfs/go-ipld-format (v0.6.0 -> v0.6.2): - new version (#96) ([ipfs/go-ipld-format#96](https://github.com/ipfs/go-ipld-format/pull/96)) - bump version (#94) ([ipfs/go-ipld-format#94](https://github.com/ipfs/go-ipld-format/pull/94)) - github.com/ipfs/go-ipld-legacy (v0.2.1 -> v0.2.2): - new version ([ipfs/go-ipld-legacy#25](https://github.com/ipfs/go-ipld-legacy/pull/25)) - github.com/ipfs/go-test (v0.2.1 -> v0.2.2): - new version (#25) ([ipfs/go-test#25](https://github.com/ipfs/go-test/pull/25)) - Update README.md (#24) ([ipfs/go-test#24](https://github.com/ipfs/go-test/pull/24)) - github.com/ipfs/go-unixfsnode (v1.10.0 -> v1.10.1): - new version ([ipfs/go-unixfsnode#84](https://github.com/ipfs/go-unixfsnode/pull/84)) - github.com/ipld/go-car/v2 (v2.14.2 -> v2.14.3): - bump version ([ipld/go-car#579](https://github.com/ipld/go-car/pull/579)) - chore: update to boxo merkledag package - feat: car debug handles the zero length block ([ipld/go-car#569](https://github.com/ipld/go-car/pull/569)) - chore(deps): bump github.com/rogpeppe/go-internal from 1.13.1 to 1.14.1 in /cmd ([ipld/go-car#566](https://github.com/ipld/go-car/pull/566)) - Add a concatenation cli utility ([ipld/go-car#565](https://github.com/ipld/go-car/pull/565)) - github.com/ipld/go-codec-dagpb (v1.6.0 -> v1.7.0): - chore: v1.7.0 bump - github.com/libp2p/go-flow-metrics (v0.2.0 -> v0.3.0): - chore: release v0.3.0 ([libp2p/go-flow-metrics#38](https://github.com/libp2p/go-flow-metrics/pull/38)) - go-clock migration ([libp2p/go-flow-metrics#36](https://github.com/libp2p/go-flow-metrics/pull/36)) - github.com/libp2p/go-libp2p (v0.41.1 -> v0.42.0): - Release v0.42.0 (#3318) ([libp2p/go-libp2p#3318](https://github.com/libp2p/go-libp2p/pull/3318)) - mocknet: notify listeners on listen (#3310) ([libp2p/go-libp2p#3310](https://github.com/libp2p/go-libp2p/pull/3310)) - autonatv2: add metrics (#3308) ([libp2p/go-libp2p#3308](https://github.com/libp2p/go-libp2p/pull/3308)) - chore: fix errors reported by golangci-lint ([libp2p/go-libp2p#3295](https://github.com/libp2p/go-libp2p/pull/3295)) - autonatv2: add Unknown addrs to event (#3305) ([libp2p/go-libp2p#3305](https://github.com/libp2p/go-libp2p/pull/3305)) - transport: rate limit new connections (#3283) ([libp2p/go-libp2p#3283](https://github.com/libp2p/go-libp2p/pull/3283)) - basichost: use autonatv2 to verify reachability (#3231) ([libp2p/go-libp2p#3231](https://github.com/libp2p/go-libp2p/pull/3231)) - chore: Revert "go-clock migration" (#3303) ([libp2p/go-libp2p#3303](https://github.com/libp2p/go-libp2p/pull/3303)) - tcp: ensure tcpGatedMaListener wrapping happens always (#3275) ([libp2p/go-libp2p#3275](https://github.com/libp2p/go-libp2p/pull/3275)) - go-clock migration ([libp2p/go-libp2p#3293](https://github.com/libp2p/go-libp2p/pull/3293)) - swarm_test: support more transports for GenSwarm (#3130) ([libp2p/go-libp2p#3130](https://github.com/libp2p/go-libp2p/pull/3130)) - eventbus: change slow consumer event from error to warn (#3286) ([libp2p/go-libp2p#3286](https://github.com/libp2p/go-libp2p/pull/3286)) - quicreuse: add some documentation for the package (#3279) ([libp2p/go-libp2p#3279](https://github.com/libp2p/go-libp2p/pull/3279)) - identify: rate limit id push protocol (#3266) ([libp2p/go-libp2p#3266](https://github.com/libp2p/go-libp2p/pull/3266)) - fix(pstoreds): add missing log for failed GC record unmarshalling in `purgeStore()` (#3273) ([libp2p/go-libp2p#3273](https://github.com/libp2p/go-libp2p/pull/3273)) - nat: improve port mapping failure logging (#3261) ([libp2p/go-libp2p#3261](https://github.com/libp2p/go-libp2p/pull/3261)) - ci: add golangci-lint for linting (#3269) ([libp2p/go-libp2p#3269](https://github.com/libp2p/go-libp2p/pull/3269)) - build(test_analysis): use `modernc.org/sqlite` directly (#3227) ([libp2p/go-libp2p#3227](https://github.com/libp2p/go-libp2p/pull/3227)) - chore(certificate): update test vectors (#3242) ([libp2p/go-libp2p#3242](https://github.com/libp2p/go-libp2p/pull/3242)) - rcmgr: use netip.Prefix as map key instead of string (#3264) ([libp2p/go-libp2p#3264](https://github.com/libp2p/go-libp2p/pull/3264)) - webrtc: support receiving 256kB messages (#3255) ([libp2p/go-libp2p#3255](https://github.com/libp2p/go-libp2p/pull/3255)) - peerstore: remove leveldb tests (#3260) ([libp2p/go-libp2p#3260](https://github.com/libp2p/go-libp2p/pull/3260)) - identify: reduce timeout to 5 seconds (#3259) ([libp2p/go-libp2p#3259](https://github.com/libp2p/go-libp2p/pull/3259)) - fix(relay): fix data-race in relayFinder (#3258) ([libp2p/go-libp2p#3258](https://github.com/libp2p/go-libp2p/pull/3258)) - chore: update p2p-forge to v0.5.0 for autotls example (#3257) ([libp2p/go-libp2p#3257](https://github.com/libp2p/go-libp2p/pull/3257)) - peerstore: remove unused badger tests (#3252) ([libp2p/go-libp2p#3252](https://github.com/libp2p/go-libp2p/pull/3252)) - chore: using t.TempDir() instead of os.MkdirTemp (#3222) ([libp2p/go-libp2p#3222](https://github.com/libp2p/go-libp2p/pull/3222)) - chore(examples): p2p-forge/client v0.4.0 (#3211) ([libp2p/go-libp2p#3211](https://github.com/libp2p/go-libp2p/pull/3211)) - transport: add GatedMaListener type (#3186) ([libp2p/go-libp2p#3186](https://github.com/libp2p/go-libp2p/pull/3186)) - autonatv2: explicitly handle dns addrs (#3249) ([libp2p/go-libp2p#3249](https://github.com/libp2p/go-libp2p/pull/3249)) - autonatv2: fix server dial data request policy (#3247) ([libp2p/go-libp2p#3247](https://github.com/libp2p/go-libp2p/pull/3247)) - webtransport: wrap underlying transport error on stream resets (#3237) ([libp2p/go-libp2p#3237](https://github.com/libp2p/go-libp2p/pull/3237)) - connmgr: remove WithEmergencyTrim (#3217) ([libp2p/go-libp2p#3217](https://github.com/libp2p/go-libp2p/pull/3217)) - connmgr: fix transport association bug (#3221) ([libp2p/go-libp2p#3221](https://github.com/libp2p/go-libp2p/pull/3221)) - webrtc: fix memory leak with udpmux.muxedConnection context (#3243) ([libp2p/go-libp2p#3243](https://github.com/libp2p/go-libp2p/pull/3243)) - fix(libp2phttp): bound NewStream timeout (#3225) ([libp2p/go-libp2p#3225](https://github.com/libp2p/go-libp2p/pull/3225)) - conngater: fix incorrect err return value (#3219) ([libp2p/go-libp2p#3219](https://github.com/libp2p/go-libp2p/pull/3219)) - addrsmanager: extract out addressing logic from basichost (#3075) ([libp2p/go-libp2p#3075](https://github.com/libp2p/go-libp2p/pull/3075)) - github.com/libp2p/go-socket-activation (v0.1.0 -> v0.1.1): - new version (#35) ([libp2p/go-socket-activation#35](https://github.com/libp2p/go-socket-activation/pull/35)) - Upgrade to go-log/v2 v2.6.0 (#33) ([libp2p/go-socket-activation#33](https://github.com/libp2p/go-socket-activation/pull/33)) - sync: update CI config files (#20) ([libp2p/go-socket-activation#20](https://github.com/libp2p/go-socket-activation/pull/20)) - sync: update CI config files (#18) ([libp2p/go-socket-activation#18](https://github.com/libp2p/go-socket-activation/pull/18)) - sync: update CI config files (#17) ([libp2p/go-socket-activation#17](https://github.com/libp2p/go-socket-activation/pull/17)) - github.com/libp2p/go-yamux/v5 (v5.0.0 -> v5.0.1): - Release v5.0.1 - fix: deadlock on close (#130) ([libp2p/go-yamux#130](https://github.com/libp2p/go-yamux/pull/130)) - github.com/multiformats/go-multiaddr (v0.15.0 -> v0.16.0): - Release v0.16.0 (#279) ([multiformats/go-multiaddr#279](https://github.com/multiformats/go-multiaddr/pull/279)) - Rename CaptureStringVal to CaptureString (#278) ([multiformats/go-multiaddr#278](https://github.com/multiformats/go-multiaddr/pull/278)) - Megular Expressions (#263) ([multiformats/go-multiaddr#263](https://github.com/multiformats/go-multiaddr/pull/263)) - github.com/multiformats/go-multicodec (v0.9.0 -> v0.9.2): - v0.9.2 bump - chore: update submodules and go generate - chore: v0.9.1 bump - chore: update submodules and go generate - ci: uci/update-go (#97) ([multiformats/go-multicodec#97](https://github.com/multiformats/go-multicodec/pull/97)) - chore: update submodules and go generate - chore: update submodules and go generate - chore: update submodules and go generate - chore: update submodules and go generate - github.com/multiformats/go-multistream (v0.6.0 -> v0.6.1): - Release v0.6.1 ([multiformats/go-multistream#121](https://github.com/multiformats/go-multistream/pull/121)) - refactor(lazyClientConn): Use synctest friendly once func ([multiformats/go-multistream#120](https://github.com/multiformats/go-multistream/pull/120))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | sukun | 25 | +7274/-1586 | 140 | | galargh | 13 | +1714/-1680 | 115 | | rvagg | 2 | +1383/-960 | 6 | | Andrew Gillis | 46 | +1226/-564 | 140 | | Marco Munizaga | 6 | +1643/-36 | 24 | | Hector Sanjuan | 20 | +624/-202 | 40 | | Marcin Rataj | 24 | +583/-175 | 49 | | Dennis Trautwein | 1 | +134/-14 | 4 | | Piotr Galar | 1 | +73/-71 | 23 | | Guillaume Michel | 4 | +58/-44 | 23 | | Ivan | 1 | +90/-9 | 3 | | Will Scott | 1 | +97/-0 | 2 | | gammazero | 11 | +47/-30 | 13 | | guillaumemichel | 3 | +40/-35 | 21 | | Adin Schmahmann | 1 | +58/-17 | 8 | | Laurent Senta | 1 | +26/-24 | 4 | | pullmerge | 1 | +20/-16 | 5 | | vladopajic | 1 | +20/-14 | 1 | | Probot | 1 | +18/-4 | 1 | | Dmitry Markin | 1 | +13/-9 | 2 | | overallteach | 1 | +4/-12 | 3 | | web3-bot | 5 | +9/-6 | 7 | | Pavel Zbitskiy | 1 | +14/-1 | 1 | | Rod Vagg | 5 | +7/-7 | 5 | | argentpapa | 1 | +3/-10 | 1 | | GarmashAlex | 1 | +8/-3 | 1 | | huochexizhan | 1 | +3/-3 | 1 | | VolodymyrBg | 1 | +2/-3 | 1 | | levisyin | 1 | +2/-2 | 2 | | b00f | 1 | +3/-0 | 1 | | achingbrain | 1 | +1/-1 | 1 | | Ocenka | 1 | +1/-1 | 1 | | Dreamacro | 1 | +1/-1 | 1 | | Štefan Baebler | 1 | +1/-0 | 1 | ================================================ FILE: docs/changelogs/v0.37.md ================================================ # Kubo changelog v0.37 This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [v0.37.0](#v0370) ## v0.37.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [🚀 Repository migration from v16 to v17 with embedded tooling](#-repository-migration-from-v16-to-v17-with-embedded-tooling) - [🚦 Gateway concurrent request limits and retrieval timeouts](#-gateway-concurrent-request-limits-and-retrieval-timeouts) - [🔧 AutoConf: Complete control over network defaults](#-autoconf-complete-control-over-network-defaults) - [🗑️ Clear provide queue when reprovide strategy changes](#-clear-provide-queue-when-reprovide-strategy-changes) - [🪵 Revamped `ipfs log level` command](#-revamped-ipfs-log-level-command) - [📌 Named pins in `ipfs add` command](#-named-pins-in-ipfs-add-command) - [📝 New IPNS publishing options](#-new-ipns-publishing-options) - [🔢 Custom sequence numbers in `ipfs name publish`](#-custom-sequence-numbers-in-ipfs-name-publish) - [⚙️ `Reprovider.Strategy` is now consistently respected](#-reprovider-strategy-is-now-consistently-respected) - [⚙️ `Reprovider.Strategy=all`: improved memory efficiency](#-reproviderstrategyall-improved-memory-efficiency) - [🧹 Removed unnecessary dependencies](#-removed-unnecessary-dependencies) - [🔍 Improved `ipfs cid`](#-improved-ipfs-cid) - [⚠️ Deprecated `ipfs stats reprovide`](#-deprecated-ipfs-stats-reprovide) - [🔄 AutoRelay now uses all connected peers for relay discovery](#-autorelay-now-uses-all-connected-peers-for-relay-discovery) - [📊 Anonymous telemetry for better feature prioritization](#-anonymous-telemetry-for-better-feature-prioritization) - [📦️ Important dependency updates](#-important-dependency-updates) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview Kubo 0.37.0 introduces embedded repository migrations, gateway resource protection, complete AutoConf control, improved reprovider strategies, and anonymous telemetry for better feature prioritization. This release significantly improves memory efficiency, network configuration flexibility, and operational reliability while maintaining full backward compatibility. ### 🔦 Highlights #### 🚀 Repository migration from v16 to v17 with embedded tooling This release migrates the Kubo repository from version 16 to version 17. Migrations are now built directly into the binary - completing in milliseconds without internet access or external downloads. `ipfs daemon --migrate` performs migrations automatically. Manual migration: `ipfs repo migrate --to=17` (or `--to=16 --allow-downgrade` for compatibility). Embedded migrations apply to v17+; older versions still require external tools. **Legacy migration deprecation**: Support for legacy migrations that download binaries from the internet will be removed in a future version. Only embedded migrations for the last 3 releases will be supported. Users with very old repositories should update in stages rather than skipping multiple versions. #### 🚦 Gateway concurrent request limits and retrieval timeouts New configurable limits protect gateway resources during high load: - **[`Gateway.RetrievalTimeout`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewayretrievaltimeout)** (default: 30s): Maximum duration for content retrieval. Returns 504 Gateway Timeout when exceeded - applies to both initial retrieval (time to first byte) and between subsequent writes. - **[`Gateway.MaxConcurrentRequests`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaymaxconcurrentrequests)** (default: 4096): Limits concurrent HTTP requests. Returns 429 Too Many Requests when exceeded. Protects nodes from traffic spikes and resource exhaustion, especially useful behind reverse proxies without rate-limiting. New Prometheus metrics for monitoring: - `ipfs_http_gw_concurrent_requests`: Current requests being processed - `ipfs_http_gw_responses_total`: HTTP responses by status code - `ipfs_http_gw_retrieval_timeouts_total`: Timeouts by status code and truncation status Tuning tips: - Monitor metrics to understand gateway behavior and adjust based on observations - Watch `ipfs_http_gw_concurrent_requests` for saturation - Track `ipfs_http_gw_retrieval_timeouts_total` vs success rates to identify timeout patterns indicating routing or storage provider issues #### 🔧 AutoConf: Complete control over network defaults Configuration fields now support `["auto"]` placeholders that resolve to network defaults from [`AutoConf.URL`](https://github.com/ipfs/kubo/blob/master/docs/config.md#autoconfurl). These defaults can be inspected, replaced with custom values, or disabled entirely. Previously, empty configuration fields like `Routing.DelegatedRouters: []` would use hardcoded defaults - this system makes those defaults explicit through `"auto"` values. When upgrading to Kubo 0.37, custom configurations remain unchanged. New `--expand-auto` flag shows resolved values for any config field: ```bash ipfs config show --expand-auto # View all resolved endpoints ipfs config Bootstrap --expand-auto # Check specific values ipfs config Routing.DelegatedRouters --expand-auto ipfs config DNS.Resolvers --expand-auto ``` Configuration can be managed via: - Replace `"auto"` with custom endpoints or set `[]` to disable features - Switch modes with `--profile=autoconf-on|autoconf-off` - Configure via `AutoConf.Enabled` and custom manifests via `AutoConf.URL` ```bash # Enable automatic configuration ipfs config profiles apply autoconf-on # Or manually set specific fields ipfs config Bootstrap '["auto"]' ipfs config --json DNS.Resolvers '{".": ["https://dns.example.com/dns-query"], "eth.": ["auto"]}' ``` Organizations can host custom AutoConf manifests for private networks. See [AutoConf documentation](https://github.com/ipfs/kubo/blob/master/docs/config.md#autoconf) and format spec at https://conf.ipfs-mainnet.org/ #### 🗑️ Clear provide queue when reprovide strategy changes Changing [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy) and restarting Kubo now automatically clears the provide queue. Only content matching the new strategy will be announced. Manual queue clearing is also available: - `ipfs provide clear` - clear all queued content announcements > [!NOTE] > Upgrading to Kubo 0.37 will automatically clear any preexisting provide queue. The next time `Reprovider.Interval` hits, `Reprovider.Strategy` will be executed on a clean slate, ensuring consistent behavior with your current configuration. #### 🪵 Revamped `ipfs log level` command The `ipfs log level` command has been completely revamped to support both getting and setting log levels with a unified interface. **New: Getting log levels** - `ipfs log level` - Shows default level only - `ipfs log level all` - Shows log level for every subsystem, including default level - `ipfs log level foo` - Shows log level for a specific subsystem only - Kubo RPC API: `POST /api/v0/log/level?arg=` **Enhanced: Setting log levels** - `ipfs log level foo debug` - Sets "foo" subsystem to "debug" level - `ipfs log level all info` - Sets all subsystems to "info" level (convenient, no escaping) - `ipfs log level '*' info` - Equivalent to above but requires shell escaping - `ipfs log level foo default` - Sets "foo" subsystem to current default level The command now provides full visibility into your current logging configuration while maintaining full backward compatibility. Both `all` and `*` work for specifying all subsystems, with `all` being more convenient since it doesn't require shell escaping. #### 🧷 Named pins in `ipfs add` command Added `--pin-name` flag to `ipfs add` for assigning names to pins. ```console $ ipfs add --pin-name=testname cat.jpg added bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi cat.jpg $ ipfs pin ls --names bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi recursive testname ``` #### 📝 New IPNS publishing options Added support for controlling IPNS record publishing strategies with new command flags and configuration. **New command flags:** ```bash # Publish without network connectivity (local datastore only) ipfs name publish --allow-offline /ipfs/QmHash # Publish without DHT connectivity (uses local datastore and HTTP delegated publishers) ipfs name publish --allow-delegated /ipfs/QmHash ``` **Delegated publishers configuration:** [`Ipns.DelegatedPublishers`](https://github.com/ipfs/kubo/blob/master/docs/config.md#ipnsdelegatedpublishers) configures HTTP endpoints for IPNS publishing. Supports `"auto"` for network defaults or custom HTTP endpoints. The `--allow-delegated` flag enables publishing through these endpoints without requiring DHT connectivity, useful for nodes behind restrictive networks or during testing. #### 🔢 Custom sequence numbers in `ipfs name publish` Added `--sequence` flag to `ipfs name publish` for setting custom sequence numbers in IPNS records. This enables advanced use cases like manually coordinating updates across multiple nodes. See `ipfs name publish --help` for details. #### ⚙️ `Reprovider.Strategy` is now consistently respected Prior to this version, files added, blocks received etc. were "provided" to the network (announced on the DHT) regardless of the ["reproviding strategy" setting](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy). For example: - Strategy set to "pinned" + `ipfs add --pin=false` → file was provided regardless - Strategy set to "roots" + `ipfs pin add` → all blocks (not only the root) were provided Only the periodic "reproviding" action (runs every 22h by default) respected the strategy. This was inefficient as content that should not be provided was getting provided once. Now all operations respect `Reprovider.Strategy`. If set to "roots", no blocks other than pin roots will be provided regardless of what is fetched, added etc. > [!NOTE] > **Behavior change:** The `--offline` flag no longer affects providing behavior. Both `ipfs add` and `ipfs --offline add` now provide blocks according to the reproviding strategy when run against an online daemon (previously `--offline add` did not provide). Since `ipfs add` has been nearly as fast as offline mode [since v0.35](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.35.md#fast-ipfs-add-in-online-mode), `--offline` is rarely needed. To run truly offline operations, use `ipfs --offline daemon`. #### ⚙️ `Reprovider.Strategy=all`: improved memory efficiency The memory cost of `Reprovider.Strategy=all` no longer grows with the number of pins. The strategy now processes blocks directly from the datastore in undefined order, eliminating the memory pressure tied to the number of pins. As part of this improvement, the `flat` reprovider strategy has been renamed to `all` (the default). This cleanup removes the workaround introduced in v0.28 for pin root prioritization. With the introduction of more granular strategies like [`pinned+mfs`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy), we can now optimize the default `all` strategy for lower memory usage without compromising users who need pin root prioritization ([rationale](https://github.com/ipfs/kubo/pull/10928#issuecomment-3211040182)). > [!NOTE] > **Migration guidance:** If you experience undesired announcement delays of root CIDs with the new `all` strategy, switch to `pinned+mfs` for root prioritization. #### 🧹 Removed unnecessary dependencies Kubo has been cleaned up by removing unnecessary dependencies and packages: - Removed `thirdparty/assert` (replaced by `github.com/stretchr/testify/require`) - Removed `thirdparty/dir` (replaced by `misc/fsutil`) - Removed `thirdparty/notifier` (unused) - Removed `goprocess` dependency (replaced with native Go `context` patterns) These changes reduce the dependency footprint while improving code maintainability and following Go best practices. #### 🔍 Improved `ipfs cid` Certain `ipfs cid` commands can now be run without a daemon or repository, and return correct exit code 1 on error, making it easier to perform CID conversion in scripts and CI/CD pipelines. While at it, we also fixed unicode support in `ipfs cid bases --prefix` to correctly show `base256emoji` 🚀 :-) #### ⚠️ Deprecated `ipfs stats reprovide` The `ipfs stats reprovide` command has moved to `ipfs provide stat`. This was done to organize provider commands in one location. > [!NOTE] > `ipfs stats reprovide` still works, but is marked as deprecated and will be removed in a future release. #### 🔄 AutoRelay now uses all connected peers for relay discovery AutoRelay's relay discovery now includes all connected peers as potential relay candidates, not just peers discovered through the DHT. This allows peers connected via HTTP routing and manual `ipfs swarm connect` commands to serve as relays, improving connectivity for nodes using non-DHT routing configurations. #### 📊 Anonymous telemetry for better feature prioritization Per a suggestion from the IPFS Foundation, Kubo now sends optional anonymized telemetry information to Shipyard [maintainers](https://github.com/ipshipyard/roadmaps/issues/20). **Privacy first**: The telemetry system collects only anonymous data - no personally identifiable information, file paths, or content data. A random UUID is generated on first run for anonymous identification. Users are notified before any data is sent and have time to opt-out. **Why**: We want to better understand Kubo usage across the ecosystem so we can better direct funding and work efforts. For example, we have little insights into how many nodes are NAT'ed and rely on AutoNAT for reachability. Some of the information can be inferred by crawling the network or logging `/identify` details in the bootstrappers, but users have no way of opting out from that, so we believe it is more transparent to concentrate this functionality in one place. **What**: Currently, we send the following anonymous metrics:
Click to see telemetry metrics example ``` "uuid": "", "agent_version": "kubo/0.37.0-dev", "private_network": false, "bootstrappers_custom": false, "repo_size_bucket": 1073741824, "uptime_bucket": 86400000000000, "reprovider_strategy": "pinned", "routing_type": "auto", "routing_accelerated_dht_client": false, "routing_delegated_count": 0, "autonat_service_mode": "enabled", "autonat_reachability": "", "autoconf": true, "autoconf_custom": false, "swarm_enable_hole_punching": true, "swarm_circuit_addresses": false, "swarm_ipv4_public_addresses": true, "swarm_ipv6_public_addresses": true, "auto_tls_auto_wss": true, "auto_tls_domain_suffix_custom": false, "discovery_mdns_enabled": true, "platform_os": "linux", "platform_arch": "amd64", "platform_containerized": false, "platform_vm": false ```
The exact data sent for your node can be inspected by setting `GOLOG_LOG_LEVEL="telemetry=debug"`. Users will see an informative message the first time they launch a telemetry-enabled daemon, with time to opt-out before any data is collected. Telemetry data is sent every 24h, with the first collection starting 15 minutes after daemon launch. **User control**: You can opt-out at any time: - Set environment variable `IPFS_TELEMETRY=off` before starting the daemon - Or run `ipfs config Plugins.Plugins.telemetry.Config.Mode off` and restart the daemon The telemetry plugin code lives in `plugin/plugins/telemetry`. Learn more: [`/kubo/docs/telemetry.md`](https://github.com/ipfs/kubo/blob/master/docs/telemetry.md) ### 📦️ Important dependency updates - update `boxo` to [v0.34.0](https://github.com/ipfs/boxo/releases/tag/v0.34.0) (incl. [v0.33.1](https://github.com/ipfs/boxo/releases/tag/v0.33.1)) - update `go-libp2p` to [v0.43.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.43.0) - update `go-libp2p-kad-dht` to [v0.34.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.34.0) - update `go-libp2p-pubsub` to [v0.14.2](https://github.com/libp2p/go-libp2p-pubsub/releases/tag/v0.14.2) (incl. [v0.14.1](https://github.com/libp2p/go-libp2p-pubsub/releases/tag/v0.14.1), [v0.14.0](https://github.com/libp2p/go-libp2p-pubsub/releases/tag/v0.14.0)) - update `ipfs-webui` to [v4.8.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.8.0) - update to [Go 1.25](https://go.dev/doc/go1.25) ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: set version to v0.37.0 - feat(ci): docker linting (#10927) ([ipfs/kubo#10927](https://github.com/ipfs/kubo/pull/10927)) - fix: disable telemetry in test profile (#10931) ([ipfs/kubo#10931](https://github.com/ipfs/kubo/pull/10931)) - fix: harness tests random panic (#10933) ([ipfs/kubo#10933](https://github.com/ipfs/kubo/pull/10933)) - chore: v0.37.0-rc1 - feat: Reprovider.Strategy: rename "flat" to "all" (#10928) ([ipfs/kubo#10928](https://github.com/ipfs/kubo/pull/10928)) - docs: improve `ipfs add --help` (#10926) ([ipfs/kubo#10926](https://github.com/ipfs/kubo/pull/10926)) - feat: optimize docker builds (#10925) ([ipfs/kubo#10925](https://github.com/ipfs/kubo/pull/10925)) - feat(config): AutoConf with "auto" placeholders (#10883) ([ipfs/kubo#10883](https://github.com/ipfs/kubo/pull/10883)) - fix(ci): make NewRandPort thread-safe (#10921) ([ipfs/kubo#10921](https://github.com/ipfs/kubo/pull/10921)) - fix: resolve TestAddMultipleGCLive race condition (#10916) ([ipfs/kubo#10916](https://github.com/ipfs/kubo/pull/10916)) - feat: telemetry plugin (#10866) ([ipfs/kubo#10866](https://github.com/ipfs/kubo/pull/10866)) - fix typos in docs and comments (#10920) ([ipfs/kubo#10920](https://github.com/ipfs/kubo/pull/10920)) - Upgrade to Boxo v0.34.0 (#10917) ([ipfs/kubo#10917](https://github.com/ipfs/kubo/pull/10917)) - test: fix flaky repo verify (#10743) ([ipfs/kubo#10743](https://github.com/ipfs/kubo/pull/10743)) - feat(config): `Gateway.RetrievalTimeout|MaxConcurrentRequests` (#10905) ([ipfs/kubo#10905](https://github.com/ipfs/kubo/pull/10905)) - chore: replace random test utils with equivalents in go-test/random (#10915) ([ipfs/kubo#10915](https://github.com/ipfs/kubo/pull/10915)) - feat: require go1.25 for building kubo (#10913) ([ipfs/kubo#10913](https://github.com/ipfs/kubo/pull/10913)) - feat(ci): reusable spellcheck from unified CI (#10873) ([ipfs/kubo#10873](https://github.com/ipfs/kubo/pull/10873)) - fix(ci): docker build (#10914) ([ipfs/kubo#10914](https://github.com/ipfs/kubo/pull/10914)) - Replace `uber-go/multierr` with `errors.Join` (#10912) ([ipfs/kubo#10912](https://github.com/ipfs/kubo/pull/10912)) - feat(ipns): support passing custom sequence number during publishing (#10851) ([ipfs/kubo#10851](https://github.com/ipfs/kubo/pull/10851)) - fix(relay): feed connected peers to AutoRelay discovery (#10901) ([ipfs/kubo#10901](https://github.com/ipfs/kubo/pull/10901)) - fix(sharness): no blocking on unclean FUSE unmount (#10906) ([ipfs/kubo#10906](https://github.com/ipfs/kubo/pull/10906)) - feat: add query functionality to log level command (#10885) ([ipfs/kubo#10885](https://github.com/ipfs/kubo/pull/10885)) - fix(ci): switch to debian:bookworm-slim - Fix failing FUSE test (#10904) ([ipfs/kubo#10904](https://github.com/ipfs/kubo/pull/10904)) - fix(cmd): exit 1 on error (#10903) ([ipfs/kubo#10903](https://github.com/ipfs/kubo/pull/10903)) - feat: go-libp2p v0.43.0 (#10892) ([ipfs/kubo#10892](https://github.com/ipfs/kubo/pull/10892)) - fix: `ipfs cid` without repo (#10897) ([ipfs/kubo#10897](https://github.com/ipfs/kubo/pull/10897)) - client/rpc: re-enable tests on windows. (#10895) ([ipfs/kubo#10895](https://github.com/ipfs/kubo/pull/10895)) - fix: Provide according to Reprovider.Strategy (#10886) ([ipfs/kubo#10886](https://github.com/ipfs/kubo/pull/10886)) - feat: ipfs-webui v4.8.0 (#10902) ([ipfs/kubo#10902](https://github.com/ipfs/kubo/pull/10902)) - refactor: move `ipfs stat provide/reprovide` to `ipfs provide stat` (#10896) ([ipfs/kubo#10896](https://github.com/ipfs/kubo/pull/10896)) - Bitswap: use a single ConnectEventManager. ([ipfs/kubo#10889](https://github.com/ipfs/kubo/pull/10889)) - feat(add): add support for naming pinned CIDs (#10877) ([ipfs/kubo#10877](https://github.com/ipfs/kubo/pull/10877)) - refactor: remove goprocess (#10872) ([ipfs/kubo#10872](https://github.com/ipfs/kubo/pull/10872)) - feat(daemon): accelerated client startup note (#10859) ([ipfs/kubo#10859](https://github.com/ipfs/kubo/pull/10859)) - docs:added GOLOG_LOG_LEVEL to debug-guide for logging more info (#10894) ([ipfs/kubo#10894](https://github.com/ipfs/kubo/pull/10894)) - core: Add a ContentDiscovery field ([ipfs/kubo#10890](https://github.com/ipfs/kubo/pull/10890)) - chore: update go-libp2p and p2p-forge (#10887) ([ipfs/kubo#10887](https://github.com/ipfs/kubo/pull/10887)) - Upgrade to Boxo v0.33.1 (#10888) ([ipfs/kubo#10888](https://github.com/ipfs/kubo/pull/10888)) - remove unneeded thirdparty packages (#10871) ([ipfs/kubo#10871](https://github.com/ipfs/kubo/pull/10871)) - provider: clear provide queue when reprovide strategy changes (#10863) ([ipfs/kubo#10863](https://github.com/ipfs/kubo/pull/10863)) - chore: merge release v0.36.0 ([ipfs/kubo#10868](https://github.com/ipfs/kubo/pull/10868)) - docs: release checklist fixes from 0.36 (#10861) ([ipfs/kubo#10861](https://github.com/ipfs/kubo/pull/10861)) - docs(config): add network exposure considerations (#10856) ([ipfs/kubo#10856](https://github.com/ipfs/kubo/pull/10856)) - fix: handling of EDITOR env var (#10855) ([ipfs/kubo#10855](https://github.com/ipfs/kubo/pull/10855)) - refactor: use slices.Sort where appropriate (#10858) ([ipfs/kubo#10858](https://github.com/ipfs/kubo/pull/10858)) - Upgrade to Boxo v0.33.0 (#10857) ([ipfs/kubo#10857](https://github.com/ipfs/kubo/pull/10857)) - chore: Upgrade github.com/cockroachdb/pebble/v2 to v2.0.6 for Go 1.25 support (#10850) ([ipfs/kubo#10850](https://github.com/ipfs/kubo/pull/10850)) - core:constructor: add a log line about http retrieval ([ipfs/kubo#10852](https://github.com/ipfs/kubo/pull/10852)) - chore: p2p-forge v0.6.0 + go-libp2p 0.42.0 (#10840) ([ipfs/kubo#10840](https://github.com/ipfs/kubo/pull/10840)) - docs: fix minor typos (#10849) ([ipfs/kubo#10849](https://github.com/ipfs/kubo/pull/10849)) - Replace use of go-car v1 with go-car/v2 (#10845) ([ipfs/kubo#10845](https://github.com/ipfs/kubo/pull/10845)) - chore: 0.37.0-dev - github.com/ipfs/boxo (v0.33.0 -> v0.34.0): - Release v0.34.0 ([ipfs/boxo#1003](https://github.com/ipfs/boxo/pull/1003)) - blockstore: remove HashOnRead ([ipfs/boxo#1001](https://github.com/ipfs/boxo/pull/1001)) - Update go-log to v2.8.1 ([ipfs/boxo#998](https://github.com/ipfs/boxo/pull/998)) - feat: autoconf client library (#997) ([ipfs/boxo#997](https://github.com/ipfs/boxo/pull/997)) - feat(gateway): concurrency and retrieval timeout limits (#994) ([ipfs/boxo#994](https://github.com/ipfs/boxo/pull/994)) - update dependencies ([ipfs/boxo#999](https://github.com/ipfs/boxo/pull/999)) - fix: cidqueue gc must iterate all elements in queue ([ipfs/boxo#1000](https://github.com/ipfs/boxo/pull/1000)) - Replace `uber-go/multierr` with `errors.Join` ([ipfs/boxo#996](https://github.com/ipfs/boxo/pull/996)) - feat(namesys/IPNSPublisher): expose ability to set Sequence (#962) ([ipfs/boxo#962](https://github.com/ipfs/boxo/pull/962)) - upgrade to go-libp2p v0.43.0 ([ipfs/boxo#993](https://github.com/ipfs/boxo/pull/993)) - Remove providing Exchange. Call Provide() from relevant places. ([ipfs/boxo#976](https://github.com/ipfs/boxo/pull/976)) - reprovider: s/initial/initial ([ipfs/boxo#992](https://github.com/ipfs/boxo/pull/992)) - Release v0.33.1 ([ipfs/boxo#991](https://github.com/ipfs/boxo/pull/991)) - fix(bootstrap): filter-out peers behind relays (#987) ([ipfs/boxo#987](https://github.com/ipfs/boxo/pull/987)) - Bitswap: fix double-worker in connectEventManager. Logging improvements. ([ipfs/boxo#986](https://github.com/ipfs/boxo/pull/986)) - upgrade to go-libp2p v0.42.1 (#988) ([ipfs/boxo#988](https://github.com/ipfs/boxo/pull/988)) - bitswap/httpnet: fix sudden stop of http retrieval requests (#984) ([ipfs/boxo#984](https://github.com/ipfs/boxo/pull/984)) - bitswap/client: disable use of traceability block by default (#956) ([ipfs/boxo#956](https://github.com/ipfs/boxo/pull/956)) - test(gateway): fix race in TestCarBackendTar (#985) ([ipfs/boxo#985](https://github.com/ipfs/boxo/pull/985)) - Shutdown the sessionWantSender changes queue when session is shutdown (#983) ([ipfs/boxo#983](https://github.com/ipfs/boxo/pull/983)) - bitswap/httpnet: start pinging before signaling Connected ([ipfs/boxo#982](https://github.com/ipfs/boxo/pull/982)) - Queue all changes in order using non-blocking async queue ([ipfs/boxo#981](https://github.com/ipfs/boxo/pull/981)) - bitswap/httpnet: fix peers silently stopping from doing http requests ([ipfs/boxo#980](https://github.com/ipfs/boxo/pull/980)) - provider: clear provide queue (#978) ([ipfs/boxo#978](https://github.com/ipfs/boxo/pull/978)) - update dependencies ([ipfs/boxo#977](https://github.com/ipfs/boxo/pull/977)) - github.com/ipfs/go-datastore (v0.8.2 -> v0.8.3): - new version (#245) ([ipfs/go-datastore#245](https://github.com/ipfs/go-datastore/pull/245)) - sort using slices.Sort (#243) ([ipfs/go-datastore#243](https://github.com/ipfs/go-datastore/pull/243)) - Replace `uber-go/multierr` with `errors.Join` (#242) ([ipfs/go-datastore#242](https://github.com/ipfs/go-datastore/pull/242)) - replace gopkg.in/check.v1 with github.com/stretchr/testify (#241) ([ipfs/go-datastore#241](https://github.com/ipfs/go-datastore/pull/241)) - github.com/ipfs/go-ipld-cbor (v0.2.0 -> v0.2.1): - new version ([ipfs/go-ipld-cbor#111](https://github.com/ipfs/go-ipld-cbor/pull/111)) - update dependencies ([ipfs/go-ipld-cbor#110](https://github.com/ipfs/go-ipld-cbor/pull/110)) - github.com/ipfs/go-log/v2 (v2.6.0 -> v2.8.1): - new version (#171) ([ipfs/go-log#171](https://github.com/ipfs/go-log/pull/171)) - feat: add LevelEnabled function to check if log level enabled (#170) ([ipfs/go-log#170](https://github.com/ipfs/go-log/pull/170)) - Replace `uber-go/multierr` with `errors.Join` (#168) ([ipfs/go-log#168](https://github.com/ipfs/go-log/pull/168)) - new version (#167) ([ipfs/go-log#167](https://github.com/ipfs/go-log/pull/167)) - Test using testify package (#166) ([ipfs/go-log#166](https://github.com/ipfs/go-log/pull/166)) - Revise the loglevel API to be more golang idiomatic (#165) ([ipfs/go-log#165](https://github.com/ipfs/go-log/pull/165)) - new version (#164) ([ipfs/go-log#164](https://github.com/ipfs/go-log/pull/164)) - feat: add GetLogLevel and GetAllLogLevels (#160) ([ipfs/go-log#160](https://github.com/ipfs/go-log/pull/160)) - github.com/ipfs/go-test (v0.2.2 -> v0.2.3): - new version (#30) ([ipfs/go-test#30](https://github.com/ipfs/go-test/pull/30)) - fix: multihash random generation (#28) ([ipfs/go-test#28](https://github.com/ipfs/go-test/pull/28)) - Add RandomName function to generate random filename (#26) ([ipfs/go-test#26](https://github.com/ipfs/go-test/pull/26)) - github.com/libp2p/go-libp2p (v0.42.0 -> v0.43.0): - Release v0.43 (#3353) ([libp2p/go-libp2p#3353](https://github.com/libp2p/go-libp2p/pull/3353)) - basichost: fix deadlock with addrs_manager (#3348) ([libp2p/go-libp2p#3348](https://github.com/libp2p/go-libp2p/pull/3348)) - basichost: fix Addrs docstring (#3341) ([libp2p/go-libp2p#3341](https://github.com/libp2p/go-libp2p/pull/3341)) - quic: upgrade quic-go to v0.53 (#3323) ([libp2p/go-libp2p#3323](https://github.com/libp2p/go-libp2p/pull/3323)) - github.com/libp2p/go-libp2p-kad-dht (v0.33.1 -> v0.34.0): - chore: release v0.34.0 (#1130) ([libp2p/go-libp2p-kad-dht#1130](https://github.com/libp2p/go-libp2p-kad-dht/pull/1130)) - make crawler protocol messenger configurable (#1128) ([libp2p/go-libp2p-kad-dht#1128](https://github.com/libp2p/go-libp2p-kad-dht/pull/1128)) - fix: move non-error log to warning level (#1119) ([libp2p/go-libp2p-kad-dht#1119](https://github.com/libp2p/go-libp2p-kad-dht/pull/1119)) - migrate providers package (#1094) ([libp2p/go-libp2p-kad-dht#1094](https://github.com/libp2p/go-libp2p-kad-dht/pull/1094)) - github.com/libp2p/go-libp2p-pubsub (v0.13.1 -> v0.14.2): - Release v0.14.2 (#629) ([libp2p/go-libp2p-pubsub#629](https://github.com/libp2p/go-libp2p-pubsub/pull/629)) - Fix test races and enable race tests in CI (#626) ([libp2p/go-libp2p-pubsub#626](https://github.com/libp2p/go-libp2p-pubsub/pull/626)) - Fix race when calling Preprocess and msg ID generator(#627) ([libp2p/go-libp2p-pubsub#627](https://github.com/libp2p/go-libp2p-pubsub/pull/627)) - Release v0.14.1 (#623) ([libp2p/go-libp2p-pubsub#623](https://github.com/libp2p/go-libp2p-pubsub/pull/623)) - fix(BatchPublishing): Make topic.AddToBatch threadsafe (#622) ([libp2p/go-libp2p-pubsub#622](https://github.com/libp2p/go-libp2p-pubsub/pull/622)) - Release v0.14.0 (#614) ([libp2p/go-libp2p-pubsub#614](https://github.com/libp2p/go-libp2p-pubsub/pull/614)) - refactor: 10x faster RPC splitting (#615) ([libp2p/go-libp2p-pubsub#615](https://github.com/libp2p/go-libp2p-pubsub/pull/615)) - test: Fix flaky TestMessageBatchPublish (#616) ([libp2p/go-libp2p-pubsub#616](https://github.com/libp2p/go-libp2p-pubsub/pull/616)) - Send IDONTWANT before first publish (#612) ([libp2p/go-libp2p-pubsub#612](https://github.com/libp2p/go-libp2p-pubsub/pull/612)) - feat(gossipsub): Add MessageBatch (#607) ([libp2p/go-libp2p-pubsub#607](https://github.com/libp2p/go-libp2p-pubsub/pull/607)) - fix(IDONTWANT)!: Do not IDONTWANT your sender (#609) ([libp2p/go-libp2p-pubsub#609](https://github.com/libp2p/go-libp2p-pubsub/pull/609)) - github.com/multiformats/go-multiaddr (v0.16.0 -> v0.16.1): - Release v0.16.1 (#281) ([multiformats/go-multiaddr#281](https://github.com/multiformats/go-multiaddr/pull/281)) - reduce allocations in Bytes() and manet methods (#280) ([multiformats/go-multiaddr#280](https://github.com/multiformats/go-multiaddr/pull/280)) - github.com/whyrusleeping/cbor-gen (v0.1.2 -> v0.3.1): - fix: capture field count early for "optional" length check (#112) ([whyrusleeping/cbor-gen#112](https://github.com/whyrusleeping/cbor-gen/pull/112)) - doc: basic cbor-gen documentation (#110) ([whyrusleeping/cbor-gen#110](https://github.com/whyrusleeping/cbor-gen/pull/110)) - feat: add support for optional fields at the end of tuple structs (#109) ([whyrusleeping/cbor-gen#109](https://github.com/whyrusleeping/cbor-gen/pull/109)) - Regenerate test files ([whyrusleeping/cbor-gen#107](https://github.com/whyrusleeping/cbor-gen/pull/107)) - improve allocations in map serialization ([whyrusleeping/cbor-gen#105](https://github.com/whyrusleeping/cbor-gen/pull/105)) - fixed array in struct instead of heap slice ([whyrusleeping/cbor-gen#104](https://github.com/whyrusleeping/cbor-gen/pull/104)) - optionally sort type names in generated code file ([whyrusleeping/cbor-gen#102](https://github.com/whyrusleeping/cbor-gen/pull/102)) - fix handling of an []*string field ([whyrusleeping/cbor-gen#101](https://github.com/whyrusleeping/cbor-gen/pull/101)) - fix: reject negative big integers ([whyrusleeping/cbor-gen#100](https://github.com/whyrusleeping/cbor-gen/pull/100))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Marcin Rataj | 26 | +16033/-755 | 176 | | Andrew Gillis | 35 | +2656/-1911 | 142 | | Hector Sanjuan | 30 | +2638/-760 | 114 | | Marco Munizaga | 11 | +1244/-362 | 41 | | Russell Dempsey | 2 | +1031/-33 | 7 | | Guillaume Michel | 4 | +899/-65 | 15 | | whyrusleeping | 4 | +448/-177 | 15 | | sukun | 9 | +312/-191 | 31 | | gammazero | 23 | +239/-216 | 45 | | Brian Olson | 5 | +343/-16 | 11 | | Steven Allen | 3 | +294/-7 | 9 | | Sergey Gorbunov | 2 | +247/-11 | 9 | | Kapil Sareen | 1 | +86/-13 | 10 | | Masih H. Derkani | 1 | +72/-24 | 1 | | Piotr Galar | 1 | +40/-55 | 23 | | Rod Vagg | 1 | +13/-11 | 3 | | Ankita Sahu | 1 | +2/-0 | 1 | | Štefan Baebler | 1 | +1/-0 | 1 | ================================================ FILE: docs/changelogs/v0.38.md ================================================ # Kubo changelog v0.38 This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [v0.38.0](#v0380) - [v0.38.1](#v0381) - [v0.38.2](#v0382) ## v0.38.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [🚀 Repository migration: simplified provide configuration](#-repository-migration-simplified-provide-configuration) - [🧹 Experimental Sweeping DHT Provider](#-experimental-sweeping-dht-provider) - [📊 Exposed DHT metrics](#-exposed-dht-metrics) - [🚨 Improved gateway error pages with diagnostic tools](#-improved-gateway-error-pages-with-diagnostic-tools) - [🎨 Updated WebUI](#-updated-webui) - [📌 Pin name improvements](#-pin-name-improvements) - [🛠️ Identity CID size enforcement and `ipfs files write` fixes](#️-identity-cid-size-enforcement-and-ipfs-files-write-fixes) - [📤 Provide Filestore and Urlstore blocks on write](#-provide-filestore-and-urlstore-blocks-on-write) - [🚦 MFS operation limit for --flush=false](#-mfs-operation-limit-for---flush=false) - [📦️ Important dependency updates](#-important-dependency-updates) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview Kubo 0.38.0 simplifies content announcement configuration, introduces an experimental sweeping DHT provider for efficient large-scale operations, and includes various performance improvements. ### 🔦 Highlights #### 🚀 Repository migration: simplified provide configuration This release migrates the repository from version 17 to version 18, simplifying how you configure content announcements. The old `Provider` and `Reprovider` sections are now combined into a single [`Provide`](https://github.com/ipfs/kubo/blob/master/docs/config.md#provide) section. Your existing settings are automatically migrated - no manual changes needed. **Migration happens automatically** when you run `ipfs daemon --migrate`. For manual migration: `ipfs repo migrate --to=18`. Read more about the new system below. #### 🧹 Experimental Sweeping DHT Provider A new experimental DHT provider is available as an alternative to both the default provider and the resource-intensive [accelerated DHT client](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient). Enable it via [`Provide.DHT.SweepEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtsweepenabled). **How it works:** Instead of providing keys one-by-one, the sweep provider systematically explores DHT keyspace regions in batches. > > > > Reprovide Cycle Comparison > > > The diagram shows how sweep mode avoids the hourly traffic spikes of Accelerated DHT while maintaining similar effectiveness. By grouping CIDs into keyspace regions and processing them in batches, sweep mode reduces memory overhead and creates predictable network patterns. **Benefits for large-scale operations:** Handles hundreds of thousands of CIDs with reduced memory and network connections, spreads operations evenly to eliminate resource spikes, maintains state across restarts through persistent keystore, and provides better metrics visibility. **Monitoring and debugging:** Legacy mode (`SweepEnabled=false`) tracks `provider_reprovider_provide_count` and `provider_reprovider_reprovide_count`, while sweep mode (`SweepEnabled=true`) tracks `total_provide_count_total`. Enable debug logging with `GOLOG_LOG_LEVEL=error,provider=debug,dht/provider=debug` to see detailed logs from either system. > [!IMPORTANT] > The metric `total_provide_count_total` was renamed to `provider_provides_total` in Kubo v0.39 to follow OpenTelemetry naming conventions. If you have dashboards or alerts monitoring this metric, update them accordingly. > [!NOTE] > This feature is experimental and opt-in. In the future, it will become the default and replace the legacy system. Some commands like `ipfs stats provide` and `ipfs routing provide` are not yet available with sweep mode. Run `ipfs provide --help` for alternatives. For configuration details, see [`Provide.DHT`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providedht). For metrics documentation, see [Provide metrics](https://github.com/ipfs/kubo/blob/master/docs/metrics.md#provide). #### 📊 Exposed DHT metrics Kubo now exposes DHT metrics from [go-libp2p-kad-dht](https://github.com/libp2p/go-libp2p-kad-dht/), including `total_provide_count_total` for sweep provider operations and RPC metrics prefixed with `rpc_inbound_` and `rpc_outbound_` for DHT message traffic. See [Kubo metrics documentation](https://github.com/ipfs/kubo/blob/master/docs/metrics.md) for details. > [!IMPORTANT] > The metric `total_provide_count_total` was renamed to `provider_provides_total` in Kubo v0.39 to follow OpenTelemetry naming conventions. If you have dashboards or alerts monitoring this metric, update them accordingly. #### 🚨 Improved gateway error pages with diagnostic tools Gateway error pages now provide more actionable information during content retrieval failures. When a 504 Gateway Timeout occurs, users see detailed retrieval state information including which phase failed and a sample of providers that were attempted: > ![Improved gateway error page showing retrieval diagnostics](https://github.com/user-attachments/assets/18432c74-a5e0-4bbf-9815-7c780779dc98) > > - **[`Gateway.DiagnosticServiceURL`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaydiagnosticserviceurl)** (default: `https://check.ipfs.network`): Configures the diagnostic service URL. When set, 504 errors show a "Check CID retrievability" button that links to this service with `?cid=` for external diagnostics. Set to empty string to disable. > - **Enhanced error details**: Timeout errors now display the retrieval phase where failure occurred (e.g., "connecting to providers", "fetching data") and up to 3 peer IDs that were attempted but couldn't deliver the content, making it easier to diagnose network or provider issues. > - **Retry button on all error pages**: Every gateway error page now includes a retry button for quick page refresh without manual URL re-entry. #### 🎨 Updated WebUI The Web UI has been updated to [v4.9](https://github.com/ipfs/ipfs-webui/releases/tag/v4.9.0) with a new **Diagnostics** screen for troubleshooting and system monitoring. Access it at `http://127.0.0.1:5001/webui` when running your local IPFS node. | Diagnostics: Logs | Files: Check Retrieval | Diagnostics: Retrieval Results | |:---:|:---:|:---:| | ![Diagnostics logs](https://github.com/user-attachments/assets/a1560fd2-6f4e-4e4f-9506-85ecb10f96e5) | ![Retrieval check interface](https://github.com/user-attachments/assets/6efa8bf1-705e-4256-8c66-282455daf789) | ![Retrieval check results](https://github.com/user-attachments/assets/970f2de3-94a3-4d48-b0a4-46832f73c2e9) | | Debug issues in real-time by adjusting [log level](https://github.com/ipfs/kubo/blob/master/docs/environment-variables.md#golog_log_level) without restart (global or per-subsystem like bitswap) | Check if content is available to other peers directly from Files screen | Find out why content won't load or who is providing it to the network | | Peers: Agent Versions | Files: Custom Sorting | |:---:|:---:| | ![Peers with Agent Version](https://github.com/user-attachments/assets/4bf95e72-193a-415d-9428-dd222795107a) | ![File sorting options](https://github.com/user-attachments/assets/fd7a1807-c487-4393-ab60-a16ae087e6cd) | | Know what software peers run | Find files faster with new sorting | Additional improvements include a close button in the file viewer, better error handling, and fixed navigation highlighting. #### 📌 Pin name improvements `ipfs pin ls --names` now correctly returns pin names for specific CIDs ([#10649](https://github.com/ipfs/kubo/issues/10649), [boxo#1035](https://github.com/ipfs/boxo/pull/1035)), RPC no longer incorrectly returns names from other pins ([#10966](https://github.com/ipfs/kubo/pull/10966)), and pin names are now limited to 255 bytes for better cross-platform compatibility ([#10981](https://github.com/ipfs/kubo/pull/10981)). #### 🛠️ Identity CID size enforcement and `ipfs files write` fixes **Identity CID size limits are now enforced** This release enforces a maximum of 128 bytes for identity CIDs ([IPIP-512](https://github.com/ipfs/specs/pull/512)) - attempting to exceed this limit will return a clear error message. Identity CIDs use [multihash `0x00`](https://github.com/multiformats/multicodec/blob/master/table.csv#L2) to embed data directly in the CID without hashing. This experimental optimization was designed for tiny data where a CID reference would be larger than the data itself, but without size limits it was easy to misuse and could turn into an anti-pattern that wastes resources and enables abuse. - `ipfs add --inline-limit` and `--hash=identity` now enforce the 128-byte maximum (error when exceeded) - `ipfs files write` prevents creation of oversized identity CIDs **Multiple `ipfs files write` bugs have been fixed** This release resolves several long-standing MFS issues: raw nodes now preserve their codec instead of being forced to dag-pb, append operations on raw nodes work correctly by converting to UnixFS when needed, and identity CIDs properly inherit the full CID prefix from parent directories. #### 📤 Provide Filestore and Urlstore blocks on write Improvements to the providing system in the last release (provide blocks according to the configured [Strategy](https://github.com/ipfs/kubo/blob/master/docs/config.md#providestrategy)) left out [Filestore](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-filestore) and [Urlstore](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-urlstore) blocks when the "all" strategy was used. They would only be reprovided but not provided on write. This is now fixed, and both Filestore blocks (local file references) and Urlstore blocks (HTTP/HTTPS URL references) will be provided correctly shortly after initial add. #### 🚦 MFS operation limit for --flush=false The new [`Internal.MFSNoFlushLimit`](https://github.com/ipfs/kubo/blob/master/docs/config.md#internalmfsnoflushlimit) configuration option prevents unbounded memory growth when using `--flush=false` with `ipfs files` commands. After performing the configured number of operations without flushing (default: 256), further operations will fail with a clear error message instructing users to flush manually. ### 📦️ Important dependency updates - update `boxo` to [v0.35.0](https://github.com/ipfs/boxo/releases/tag/v0.35.0) - update `go-libp2p-kad-dht` to [v0.35.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.35.0) - update `ipfs-webui` to [v4.9.1](https://github.com/ipfs/ipfs-webui/releases/tag/v4.9.1) (incl. [v4.9.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.9.0)) ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: v0.38.0 - chore: bump go-libp2p-kad-dht to v0.35.0 (#11002) ([ipfs/kubo#11002](https://github.com/ipfs/kubo/pull/11002)) - docs: add sweeping provide worker count recommendation (#11001) ([ipfs/kubo#11001](https://github.com/ipfs/kubo/pull/11001)) - Upgrade to Boxo v0.35.0 (#10999) ([ipfs/kubo#10999](https://github.com/ipfs/kubo/pull/10999)) - chore: 0.38.0-rc2 - chore: update boxo and kad-dht dependencies (#10995) ([ipfs/kubo#10995](https://github.com/ipfs/kubo/pull/10995)) - fix: update webui to v4.9.1 (#10994) ([ipfs/kubo#10994](https://github.com/ipfs/kubo/pull/10994)) - fix: provider merge conflicts (#10989) ([ipfs/kubo#10989](https://github.com/ipfs/kubo/pull/10989)) - fix(mfs): add soft limit for `--flush=false` (#10985) ([ipfs/kubo#10985](https://github.com/ipfs/kubo/pull/10985)) - fix: provide Filestore nodes (#10990) ([ipfs/kubo#10990](https://github.com/ipfs/kubo/pull/10990)) - feat: limit pin names to 255 bytes (#10981) ([ipfs/kubo#10981](https://github.com/ipfs/kubo/pull/10981)) - fix: SweepingProvider slow start (#10980) ([ipfs/kubo#10980](https://github.com/ipfs/kubo/pull/10980)) - chore: release v0.38.0-rc1 - fix: SweepingProvider shouldn't error when missing DHT (#10975) ([ipfs/kubo#10975](https://github.com/ipfs/kubo/pull/10975)) - fix: allow custom http provide when libp2p node is offline (#10974) ([ipfs/kubo#10974](https://github.com/ipfs/kubo/pull/10974)) - docs(provide): validation and reprovide cycle visualization (#10977) ([ipfs/kubo#10977](https://github.com/ipfs/kubo/pull/10977)) - refactor(ci): optimize build workflows (#10973) ([ipfs/kubo#10973](https://github.com/ipfs/kubo/pull/10973)) - fix(cmds): cleanup unicode identify strings (#9465) ([ipfs/kubo#9465](https://github.com/ipfs/kubo/pull/9465)) - feat: ipfs-webui v4.9.0 with retrieval diagnostics (#10969) ([ipfs/kubo#10969](https://github.com/ipfs/kubo/pull/10969)) - fix(mfs): unbound cache growth with `flush=false` (#10971) ([ipfs/kubo#10971](https://github.com/ipfs/kubo/pull/10971)) - fix: `ipfs pin ls --names` (#10970) ([ipfs/kubo#10970](https://github.com/ipfs/kubo/pull/10970)) - refactor(config): migration 17-to-18 to unify Provider/Reprovider into Provide.DHT (#10951) ([ipfs/kubo#10951](https://github.com/ipfs/kubo/pull/10951)) - feat: opt-in new Sweep provide system (#10834) ([ipfs/kubo#10834](https://github.com/ipfs/kubo/pull/10834)) - rpc: retrieve pin names when Detailed option provided (#10966) ([ipfs/kubo#10966](https://github.com/ipfs/kubo/pull/10966)) - fix: enforce identity CID size limits (#10949) ([ipfs/kubo#10949](https://github.com/ipfs/kubo/pull/10949)) - docs: kubo logo sources (#10964) ([ipfs/kubo#10964](https://github.com/ipfs/kubo/pull/10964)) - feat(config): validate Import config at daemon startup (#10957) ([ipfs/kubo#10957](https://github.com/ipfs/kubo/pull/10957)) - fix(telemetry): improve vm/container detection (#10944) ([ipfs/kubo#10944](https://github.com/ipfs/kubo/pull/10944)) - feat(gateway): improved error page with retrieval state details (#10950) ([ipfs/kubo#10950](https://github.com/ipfs/kubo/pull/10950)) - close files opened during migration (#10956) ([ipfs/kubo#10956](https://github.com/ipfs/kubo/pull/10956)) - fix ctrl-c prompt during run migrations prompt (#10947) ([ipfs/kubo#10947](https://github.com/ipfs/kubo/pull/10947)) - repo: use config api to get node root path (#10934) ([ipfs/kubo#10934](https://github.com/ipfs/kubo/pull/10934)) - docs: simplify release process (#10870) ([ipfs/kubo#10870](https://github.com/ipfs/kubo/pull/10870)) - Merge release v0.37.0 ([ipfs/kubo#10943](https://github.com/ipfs/kubo/pull/10943)) - feat(ci): docker linting (#10927) ([ipfs/kubo#10927](https://github.com/ipfs/kubo/pull/10927)) - fix: disable telemetry in test profile (#10931) ([ipfs/kubo#10931](https://github.com/ipfs/kubo/pull/10931)) - fix: harness tests random panic (#10933) ([ipfs/kubo#10933](https://github.com/ipfs/kubo/pull/10933)) - chore: 0.38.0-dev - github.com/ipfs/boxo (v0.34.0 -> v0.35.0): - Release v0.35.0 ([ipfs/boxo#1046](https://github.com/ipfs/boxo/pull/1046)) - feat(gateway): add `MaxRangeRequestFileSize` protection (#1043) ([ipfs/boxo#1043](https://github.com/ipfs/boxo/pull/1043)) - revert: remove MFS auto-flush mechanism (#1041) ([ipfs/boxo#1041](https://github.com/ipfs/boxo/pull/1041)) - Filestore: add Provider option to provide filestore blocks. (#1042) ([ipfs/boxo#1042](https://github.com/ipfs/boxo/pull/1042)) - fix(pinner): restore indirect pin detection and add context cancellation (#1039) ([ipfs/boxo#1039](https://github.com/ipfs/boxo/pull/1039)) - fix(mfs): limit cache growth by default (#1037) ([ipfs/boxo#1037](https://github.com/ipfs/boxo/pull/1037)) - update dependencies (#1038) ([ipfs/boxo#1038](https://github.com/ipfs/boxo/pull/1038)) - feat(pinner): add `CheckIfPinnedWithType` for efficient checks with names (#1035) ([ipfs/boxo#1035](https://github.com/ipfs/boxo/pull/1035)) - fix(routing/http): don't cancel batch prematurely (#1036) ([ipfs/boxo#1036](https://github.com/ipfs/boxo/pull/1036)) - refactor: use the new Reprovide Sweep interface (#995) ([ipfs/boxo#995](https://github.com/ipfs/boxo/pull/995)) - Update go-dsqueue to latest (#1034) ([ipfs/boxo#1034](https://github.com/ipfs/boxo/pull/1034)) - feat(routing/http): return 200 for empty results per IPIP-513 (#1032) ([ipfs/boxo#1032](https://github.com/ipfs/boxo/pull/1032)) - replace provider queue with go-dsqueue (#1033) ([ipfs/boxo#1033](https://github.com/ipfs/boxo/pull/1033)) - refactor: use slices package to simplify slice manipulation (#1031) ([ipfs/boxo#1031](https://github.com/ipfs/boxo/pull/1031)) - bitswap/network: fix read/write data race in bitswap network test (#1030) ([ipfs/boxo#1030](https://github.com/ipfs/boxo/pull/1030)) - fix(verifcid): enforce size limit for identity CIDs (#1018) ([ipfs/boxo#1018](https://github.com/ipfs/boxo/pull/1018)) - docs: boxo logo source files (#1028) ([ipfs/boxo#1028](https://github.com/ipfs/boxo/pull/1028)) - feat(gateway): enhance 504 timeout errors with diagnostic UX (#1023) ([ipfs/boxo#1023](https://github.com/ipfs/boxo/pull/1023)) - Use `time.Duration` for rebroadcast delay (#1027) ([ipfs/boxo#1027](https://github.com/ipfs/boxo/pull/1027)) - refactor(bitswap/client/internal): close session with Close method instead of context (#1011) ([ipfs/boxo#1011](https://github.com/ipfs/boxo/pull/1011)) - fix: use %q for logging routing keys with binary data (#1025) ([ipfs/boxo#1025](https://github.com/ipfs/boxo/pull/1025)) - rename `retrieval.RetrievalState` to `retrieval.State` (#1026) ([ipfs/boxo#1026](https://github.com/ipfs/boxo/pull/1026)) - feat(gateway): add retrieval state tracking for timeout diagnostics (#1015) ([ipfs/boxo#1015](https://github.com/ipfs/boxo/pull/1015)) - Nonfunctional changes (#1017) ([ipfs/boxo#1017](https://github.com/ipfs/boxo/pull/1017)) - fix: flaky TestCancelOverridesPendingWants (#1016) ([ipfs/boxo#1016](https://github.com/ipfs/boxo/pull/1016)) - bitswap/client: GetBlocks cancels session when finished (#1007) ([ipfs/boxo#1007](https://github.com/ipfs/boxo/pull/1007)) - Remove unused context ([ipfs/boxo#1006](https://github.com/ipfs/boxo/pull/1006)) - github.com/ipfs/go-block-format (v0.2.2 -> v0.2.3): - new version (#66) ([ipfs/go-block-format#66](https://github.com/ipfs/go-block-format/pull/66)) - Replace CI badge and add GoDoc link in README (#65) ([ipfs/go-block-format#65](https://github.com/ipfs/go-block-format/pull/65)) - github.com/ipfs/go-datastore (v0.8.3 -> v0.9.0): - new version (#255) ([ipfs/go-datastore#255](https://github.com/ipfs/go-datastore/pull/255)) - feat(keytransform): support transaction feature (#239) ([ipfs/go-datastore#239](https://github.com/ipfs/go-datastore/pull/239)) - feat: context datastore (#238) ([ipfs/go-datastore#238](https://github.com/ipfs/go-datastore/pull/238)) - new version (#254) ([ipfs/go-datastore#254](https://github.com/ipfs/go-datastore/pull/254)) - fix comment (#253) ([ipfs/go-datastore#253](https://github.com/ipfs/go-datastore/pull/253)) - feat: query iterator (#244) ([ipfs/go-datastore#244](https://github.com/ipfs/go-datastore/pull/244)) - Update readme links (#246) ([ipfs/go-datastore#246](https://github.com/ipfs/go-datastore/pull/246)) - github.com/ipfs/go-ipld-format (v0.6.2 -> v0.6.3): - new version (#100) ([ipfs/go-ipld-format#100](https://github.com/ipfs/go-ipld-format/pull/100)) - avoid unnecessary slice allocation (#99) ([ipfs/go-ipld-format#99](https://github.com/ipfs/go-ipld-format/pull/99)) - github.com/ipfs/go-unixfsnode (v1.10.1 -> v1.10.2): - new version ([ipfs/go-unixfsnode#88](https://github.com/ipfs/go-unixfsnode/pull/88)) - github.com/ipld/go-car/v2 (v2.14.3 -> v2.15.0): - v2.15.0 bump (#606) ([ipld/go-car#606](https://github.com/ipld/go-car/pull/606)) - feat: add NextReader to BlockReader (#603) ([ipld/go-car#603](https://github.com/ipld/go-car/pull/603)) - Remove `@masih` form CODEOWNERS ([ipld/go-car#605](https://github.com/ipld/go-car/pull/605)) - github.com/libp2p/go-libp2p-kad-dht (v0.34.0 -> v0.35.0): - chore: release v0.35.0 (#1162) ([libp2p/go-libp2p-kad-dht#1162](https://github.com/libp2p/go-libp2p-kad-dht/pull/1162)) - refactor: adjust FIND_NODE response exceptions (#1158) ([libp2p/go-libp2p-kad-dht#1158](https://github.com/libp2p/go-libp2p-kad-dht/pull/1158)) - refactor: remove provider status command (#1157) ([libp2p/go-libp2p-kad-dht#1157](https://github.com/libp2p/go-libp2p-kad-dht/pull/1157)) - refactor(provider): closestPeerToPrefix coverage trie (#1156) ([libp2p/go-libp2p-kad-dht#1156](https://github.com/libp2p/go-libp2p-kad-dht/pull/1156)) - fix: don't empty mapdatastore keystore on close (#1155) ([libp2p/go-libp2p-kad-dht#1155](https://github.com/libp2p/go-libp2p-kad-dht/pull/1155)) - provider: default options (#1153) ([libp2p/go-libp2p-kad-dht#1153](https://github.com/libp2p/go-libp2p-kad-dht/pull/1153)) - fix(keystore): use new batch after commit (#1154) ([libp2p/go-libp2p-kad-dht#1154](https://github.com/libp2p/go-libp2p-kad-dht/pull/1154)) - provider: more minor fixes (#1152) ([libp2p/go-libp2p-kad-dht#1152](https://github.com/libp2p/go-libp2p-kad-dht/pull/1152)) - rename KeyStore -> Keystore (#1151) ([libp2p/go-libp2p-kad-dht#1151](https://github.com/libp2p/go-libp2p-kad-dht/pull/1151)) - provider: minor fixes (#1150) ([libp2p/go-libp2p-kad-dht#1150](https://github.com/libp2p/go-libp2p-kad-dht/pull/1150)) - buffered provider (#1149) ([libp2p/go-libp2p-kad-dht#1149](https://github.com/libp2p/go-libp2p-kad-dht/pull/1149)) - keystore: remove mutex (#1147) ([libp2p/go-libp2p-kad-dht#1147](https://github.com/libp2p/go-libp2p-kad-dht/pull/1147)) - provider: ResettableKeyStore (#1146) ([libp2p/go-libp2p-kad-dht#1146](https://github.com/libp2p/go-libp2p-kad-dht/pull/1146)) - keystore: revamp (#1142) ([libp2p/go-libp2p-kad-dht#1142](https://github.com/libp2p/go-libp2p-kad-dht/pull/1142)) - provider: use synctest for testing time (#1136) ([libp2p/go-libp2p-kad-dht#1136](https://github.com/libp2p/go-libp2p-kad-dht/pull/1136)) - provider: connectivity state machine (#1135) ([libp2p/go-libp2p-kad-dht#1135](https://github.com/libp2p/go-libp2p-kad-dht/pull/1135)) - provider: minor fixes (#1133) ([libp2p/go-libp2p-kad-dht#1133](https://github.com/libp2p/go-libp2p-kad-dht/pull/1133)) - dual: provider (#1132) ([libp2p/go-libp2p-kad-dht#1132](https://github.com/libp2p/go-libp2p-kad-dht/pull/1132)) - provider: refresh schedule (#1131) ([libp2p/go-libp2p-kad-dht#1131](https://github.com/libp2p/go-libp2p-kad-dht/pull/1131)) - provider: integration tests (#1127) ([libp2p/go-libp2p-kad-dht#1127](https://github.com/libp2p/go-libp2p-kad-dht/pull/1127)) - provider: daemon (#1126) ([libp2p/go-libp2p-kad-dht#1126](https://github.com/libp2p/go-libp2p-kad-dht/pull/1126)) - provide: handle reprovide (#1125) ([libp2p/go-libp2p-kad-dht#1125](https://github.com/libp2p/go-libp2p-kad-dht/pull/1125)) - provider: options (#1124) ([libp2p/go-libp2p-kad-dht#1124](https://github.com/libp2p/go-libp2p-kad-dht/pull/1124)) - provider: catchup pending work (#1123) ([libp2p/go-libp2p-kad-dht#1123](https://github.com/libp2p/go-libp2p-kad-dht/pull/1123)) - provider: batch reprovide (#1122) ([libp2p/go-libp2p-kad-dht#1122](https://github.com/libp2p/go-libp2p-kad-dht/pull/1122)) - provider: batch provide (#1121) ([libp2p/go-libp2p-kad-dht#1121](https://github.com/libp2p/go-libp2p-kad-dht/pull/1121)) - provider: swarm exploration (#1120) ([libp2p/go-libp2p-kad-dht#1120](https://github.com/libp2p/go-libp2p-kad-dht/pull/1120)) - provider: handleProvide (#1118) ([libp2p/go-libp2p-kad-dht#1118](https://github.com/libp2p/go-libp2p-kad-dht/pull/1118)) - provider: schedule (#1117) ([libp2p/go-libp2p-kad-dht#1117](https://github.com/libp2p/go-libp2p-kad-dht/pull/1117)) - provider: schedule prefix length (#1116) ([libp2p/go-libp2p-kad-dht#1116](https://github.com/libp2p/go-libp2p-kad-dht/pull/1116)) - provider: ProvideStatus interface (#1110) ([libp2p/go-libp2p-kad-dht#1110](https://github.com/libp2p/go-libp2p-kad-dht/pull/1110)) - provider: network operations (#1115) ([libp2p/go-libp2p-kad-dht#1115](https://github.com/libp2p/go-libp2p-kad-dht/pull/1115)) - provider: adding provide and reprovide queue (#1114) ([libp2p/go-libp2p-kad-dht#1114](https://github.com/libp2p/go-libp2p-kad-dht/pull/1114)) - provider: trie allocation helper (#1108) ([libp2p/go-libp2p-kad-dht#1108](https://github.com/libp2p/go-libp2p-kad-dht/pull/1108)) - add missing ShortestCoveredPrefix ([libp2p/go-libp2p-kad-dht@d0b110d](https://github.com/libp2p/go-libp2p-kad-dht/commit/d0b110d)) - provider: keyspace helpers ([libp2p/go-libp2p-kad-dht@af3ce09](https://github.com/libp2p/go-libp2p-kad-dht/commit/af3ce09)) - provider: helpers package rename (#1111) ([libp2p/go-libp2p-kad-dht#1111](https://github.com/libp2p/go-libp2p-kad-dht/pull/1111)) - provider: trie region helpers (#1109) ([libp2p/go-libp2p-kad-dht#1109](https://github.com/libp2p/go-libp2p-kad-dht/pull/1109)) - provider: PruneSubtrie helper (#1107) ([libp2p/go-libp2p-kad-dht#1107](https://github.com/libp2p/go-libp2p-kad-dht/pull/1107)) - provider: NextNonEmptyLeaf trie helper (#1106) ([libp2p/go-libp2p-kad-dht#1106](https://github.com/libp2p/go-libp2p-kad-dht/pull/1106)) - provider: find subtrie helper (#1105) ([libp2p/go-libp2p-kad-dht#1105](https://github.com/libp2p/go-libp2p-kad-dht/pull/1105)) - provider: helpers trie find prefix (#1104) ([libp2p/go-libp2p-kad-dht#1104](https://github.com/libp2p/go-libp2p-kad-dht/pull/1104)) - provider: trie items listing helpers (#1103) ([libp2p/go-libp2p-kad-dht#1103](https://github.com/libp2p/go-libp2p-kad-dht/pull/1103)) - provider: add ShortestCoveredPrefix helper (#1102) ([libp2p/go-libp2p-kad-dht#1102](https://github.com/libp2p/go-libp2p-kad-dht/pull/1102)) - provider: key helpers (#1101) ([libp2p/go-libp2p-kad-dht#1101](https://github.com/libp2p/go-libp2p-kad-dht/pull/1101)) - provider: Connectivity Checker (#1099) ([libp2p/go-libp2p-kad-dht#1099](https://github.com/libp2p/go-libp2p-kad-dht/pull/1099)) - provider: SweepingProvider interface (#1098) ([libp2p/go-libp2p-kad-dht#1098](https://github.com/libp2p/go-libp2p-kad-dht/pull/1098)) - provider: keystore (#1096) ([libp2p/go-libp2p-kad-dht#1096](https://github.com/libp2p/go-libp2p-kad-dht/pull/1096)) - provider initial commit ([libp2p/go-libp2p-kad-dht@70d21a8](https://github.com/libp2p/go-libp2p-kad-dht/commit/70d21a8)) - test GCP result order (#1097) ([libp2p/go-libp2p-kad-dht#1097](https://github.com/libp2p/go-libp2p-kad-dht/pull/1097)) - refactor: apply suggestions in records (#1113) ([libp2p/go-libp2p-kad-dht#1113](https://github.com/libp2p/go-libp2p-kad-dht/pull/1113)) - github.com/libp2p/go-libp2p-kbucket (v0.7.0 -> v0.8.0): - chore: release v0.8.0 (#147) ([libp2p/go-libp2p-kbucket#147](https://github.com/libp2p/go-libp2p-kbucket/pull/147)) - feat: generic find PeerID with CPL (#145) ([libp2p/go-libp2p-kbucket#145](https://github.com/libp2p/go-libp2p-kbucket/pull/145)) - github.com/multiformats/go-varint (v0.0.7 -> v0.1.0): - v0.1.0 bump (#29) ([multiformats/go-varint#29](https://github.com/multiformats/go-varint/pull/29)) - chore: optimise UvarintSize (#28) ([multiformats/go-varint#28](https://github.com/multiformats/go-varint/pull/28))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Guillaume Michel | 62 | +15401/-5657 | 209 | | Marcin Rataj | 33 | +9540/-1734 | 215 | | Andrew Gillis | 29 | +771/-1093 | 70 | | Hlib Kanunnikov | 2 | +350/-0 | 5 | | Rod Vagg | 3 | +260/-9 | 4 | | Hector Sanjuan | 4 | +188/-33 | 11 | | Jakub Sztandera | 1 | +67/-15 | 3 | | Masih H. Derkani | 1 | +1/-2 | 2 | | Dominic Della Valle | 1 | +2/-1 | 1 | ## v0.38.1 Fixes migration panic on Windows when upgrading from v0.37 to v0.38 ("panic: error can't be dealt with transactionally: Access is denied"). Updates go-ds-pebble to v0.5.3 (pebble v2.1.0). ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: v0.38.1 - fix: migrations for Windows (#11010) ([ipfs/kubo#11010](https://github.com/ipfs/kubo/pull/11010)) - Upgrade go-ds-pebble to v0.5.3 (#11011) ([ipfs/kubo#11011](https://github.com/ipfs/kubo/pull/11011)) - upgrade go-ds-pebble to v0.5.2 (#11000) ([ipfs/kubo#11000](https://github.com/ipfs/kubo/pull/11000)) - github.com/ipfs/go-ds-pebble (v0.5.1 -> v0.5.3): - new version (#62) ([ipfs/go-ds-pebble#62](https://github.com/ipfs/go-ds-pebble/pull/62)) - fix panic when batch is reused after commit (#61) ([ipfs/go-ds-pebble#61](https://github.com/ipfs/go-ds-pebble/pull/61)) - new version (#60) ([ipfs/go-ds-pebble#60](https://github.com/ipfs/go-ds-pebble/pull/60)) - Upgrade to pebble v2.1.0 (#59) ([ipfs/go-ds-pebble#59](https://github.com/ipfs/go-ds-pebble/pull/59)) - update readme (#57) ([ipfs/go-ds-pebble#57](https://github.com/ipfs/go-ds-pebble/pull/57))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Marcin Rataj | 2 | +613/-267 | 15 | | Andrew Gillis | 6 | +148/-22 | 8 | ## v0.38.2 - Updates [boxo v0.35.1](https://github.com/ipfs/boxo/releases/tag/v0.35.1) with bitswap and HTTP retrieval fixes: - Fixed bitswap trace context not being passed to sessions, restoring observability for monitoring tools - Kubo now fetches from HTTP gateways that return errors in legacy IPLD format, improving compatibility with older providers - Better handling of rate-limited HTTP endpoints and clearer timeout error messages - Updates [go-libp2p-kad-dht v0.35.1](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.35.1) with memory optimizations for nodes using `Provide.DHT.SweepEnabled=true` - Updates [quic-go v0.55.0](https://github.com/quic-go/quic-go/releases/tag/v0.55.0) to fix memory pooling where stream frames weren't returned to the pool on cancellation ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - chore: boxo and kad-dht updates - fix: update quic-go to v0.55.0 - github.com/ipfs/boxo (v0.35.0 -> v0.35.1): - Release v0.35.1 ([ipfs/boxo#1063](https://github.com/ipfs/boxo/pull/1063)) - bitswap/httpnet: improve "Connect"/testCid check (#1057) ([ipfs/boxo#1057](https://github.com/ipfs/boxo/pull/1057)) - fix: revert go-libp2p to v0.43.0 (#1061) ([ipfs/boxo#1061](https://github.com/ipfs/boxo/pull/1061)) - bitswap/client: propagate trace state when calling `GetBlocks` ([ipfs/boxo#1060](https://github.com/ipfs/boxo/pull/1060)) - fix(tracing): use context to pass trace and retrieval state to session ([ipfs/boxo#1059](https://github.com/ipfs/boxo/pull/1059)) - bitswap: link traces ([ipfs/boxo#1053](https://github.com/ipfs/boxo/pull/1053)) - fix(gateway): deduplicate peer IDs in retrieval diagnostics (#1058) ([ipfs/boxo#1058](https://github.com/ipfs/boxo/pull/1058)) - update go-dsqueue to v0.1.0 ([ipfs/boxo#1049](https://github.com/ipfs/boxo/pull/1049)) - Update go-libp2p to v0.44 ([ipfs/boxo#1048](https://github.com/ipfs/boxo/pull/1048)) - github.com/ipfs/go-dsqueue (v0.0.5 -> v0.1.0): - new version (#24) ([ipfs/go-dsqueue#24](https://github.com/ipfs/go-dsqueue/pull/24)) - Do not reuse datastore Batch (#23) ([ipfs/go-dsqueue#23](https://github.com/ipfs/go-dsqueue/pull/23)) - github.com/ipfs/go-log/v2 (v2.8.1 -> v2.8.2): - new version (#175) ([ipfs/go-log#175](https://github.com/ipfs/go-log/pull/175)) - fix: revert removal of LevelFromString to avoid breaking change (#174) ([ipfs/go-log#174](https://github.com/ipfs/go-log/pull/174)) - github.com/ipld/go-car/v2 (v2.15.0 -> v2.16.0): - v2.16.0 bump (#625) ([ipld/go-car#625](https://github.com/ipld/go-car/pull/625)) - github.com/ipld/go-ipld-prime/storage/bsadapter (v0.0.0-20230102063945-1a409dc236dd -> v0.0.0-20250821084354-a425e60cd714): - github.com/libp2p/go-libp2p-kad-dht (v0.35.0 -> v0.35.1): - chore: release v0.35.1 (#1165) ([libp2p/go-libp2p-kad-dht#1165](https://github.com/libp2p/go-libp2p-kad-dht/pull/1165)) - feat(provider): use Trie.AddMany (#1164) ([libp2p/go-libp2p-kad-dht#1164](https://github.com/libp2p/go-libp2p-kad-dht/pull/1164)) - fix(provider): memory usage (#1163) ([libp2p/go-libp2p-kad-dht#1163](https://github.com/libp2p/go-libp2p-kad-dht/pull/1163)) - github.com/libp2p/go-netroute (v0.2.2 -> v0.3.0): - release v0.3.0 - remove google/gopacket dependency - Query routes via routesocket ([libp2p/go-netroute#57](https://github.com/libp2p/go-netroute/pull/57)) - ci: uci/update-go (#52) ([libp2p/go-netroute#52](https://github.com/libp2p/go-netroute/pull/52)) - github.com/multiformats/go-multicodec (v0.9.2 -> v0.10.0): - chore: v0.10.0 bump - chore: update submodules and go generate - chore(deps): update stringer to v0.38.0 - ci: uci/update-go ([multiformats/go-multicodec#104](https://github.com/multiformats/go-multicodec/pull/104))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | rvagg | 1 | +537/-481 | 3 | | Carlos Hernandez | 9 | +556/-218 | 11 | | Guillaume Michel | 3 | +139/-105 | 6 | | gammazero | 8 | +101/-97 | 14 | | Hector Sanjuan | 1 | +87/-28 | 5 | | Marcin Rataj | 4 | +57/-9 | 7 | | Marco Munizaga | 2 | +42/-14 | 7 | | Dennis Trautwein | 2 | +19/-7 | 7 | | Andrew Gillis | 3 | +3/-19 | 3 | | Rod Vagg | 4 | +12/-3 | 4 | | web3-bot | 1 | +2/-1 | 1 | | galargh | 1 | +1/-1 | 1 | ================================================ FILE: docs/changelogs/v0.39.md ================================================ # Kubo changelog v0.39 This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [v0.39.0](#v0390) ## v0.39.0 [](https://github.com/user-attachments/assets/427702e8-b6b8-4ac2-8425-18069626c321) - [Overview](#overview) - [🔦 Highlights](#-highlights) - [🎯 DHT Sweep provider is now the default](#-dht-sweep-provider-is-now-the-default) - [⚡ Fast root CID providing for immediate content discovery](#-fast-root-cid-providing-for-immediate-content-discovery) - [⏯️ Provider state persists across restarts](#️-provider-state-persists-across-restarts) - [📊 Detailed statistics with `ipfs provide stat`](#-detailed-statistics-with-ipfs-provide-stat) - [🔔 Slow reprovide warnings](#-slow-reprovide-warnings) - [📊 Metric rename: `provider_provides_total`](#-metric-rename-provider_provides_total) - [🔧 Automatic UPnP recovery after router restarts](#-automatic-upnp-recovery-after-router-restarts) - [🪦 Deprecated `go-ipfs` name no longer published](#-deprecated-go-ipfs-name-no-longer-published) - [🚦 Gateway range request limits for CDN compatibility](#-gateway-range-request-limits-for-cdn-compatibility) - [🖥️ RISC-V support with prebuilt binaries](#️-risc-v-support-with-prebuilt-binaries) - [📦️ Important dependency updates](#-important-dependency-updates) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview Kubo 0.39 makes self-hosting practical on consumer hardware and home networks. The DHT sweep provider (now default) announces your content to the network without traffic spikes that overwhelm residential connections. Automatic UPnP recovery means your node stays reachable after router restarts without manual intervention. New content becomes findable immediately after `ipfs add`. The provider system persists state across restarts, alerts you when falling behind, and exposes detailed stats for monitoring. This release also finalizes the deprecation of the legacy `go-ipfs` name. ### 🔦 Highlights #### 🎯 DHT Sweep provider is now the default The Amino DHT Sweep provider system, introduced as experimental in v0.38, is now enabled by default (`Provide.DHT.SweepEnabled=true`). **What this means:** All nodes now benefit from efficient keyspace-sweeping content announcements that reduce memory overhead and create predictable network patterns, especially for nodes providing large content collections. **Migration:** The transition is automatic on upgrade. Your existing configuration is preserved: - If you explicitly set `Provide.DHT.SweepEnabled=false` in v0.38, you'll continue using the legacy provider - If you were using the default settings, you'll automatically get the sweep provider - To opt out and return to legacy behavior: `ipfs config --json Provide.DHT.SweepEnabled false` - Providers with medium to large datasets may need to adjust defaults; see [Capacity Planning](https://github.com/ipfs/kubo/blob/master/docs/provide-stats.md#capacity-planning) - When `Routing.AcceleratedDHTClient` is enabled, full sweep efficiency may not be available yet; consider disabling the accelerated client as sweep is sufficient for most workloads. See [caveat 4](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient). **New features available with sweep mode:** - Detailed statistics via `ipfs provide stat` ([see below](#-detailed-statistics-with-ipfs-provide-stat)) - Automatic resume after restarts with persistent state ([see below](#️-provider-state-persists-across-restarts)) - Proactive alerts when reproviding falls behind ([see below](#-slow-reprovide-warnings)) - Better metrics for monitoring (`provider_provides_total`) ([see below](#-metric-rename-provider_provides_total)) - Fast optimistic provide of new root CIDs ([see below](#-fast-root-cid-providing-for-immediate-content-discovery)) For background on the sweep provider design and motivations, see [`Provide.DHT.SweepEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtsweepenabled) and Shipyard's blogpost [Provide Sweep: Solving the DHT Provide Bottleneck](https://ipshipyard.com/blog/2025-dht-provide-sweep/). #### ⚡ Fast root CID providing for immediate content discovery When you add content to IPFS, the sweep provider queues it for efficient DHT provides over time. While this is resource-efficient, other peers won't find your content immediately after `ipfs add` or `ipfs dag import` completes. To make sharing faster, `ipfs add` and `ipfs dag import` now do an immediate provide of root CIDs to the DHT in addition to the regular queue (controlled by the new `--fast-provide-root` flag, enabled by default). This complements the sweep provider system: fast-provide handles the urgent case (root CIDs that users share and reference), while the sweep provider efficiently provides all blocks according to `Provide.Strategy` over time. This closes the gap between command completion and content shareability: root CIDs typically become discoverable on the network in under a second (compared to 30+ seconds previously). The feature uses optimistic DHT operations, which are significantly faster with the sweep provider (now enabled by default). By default, this immediate provide runs in the background without blocking the command. For use cases requiring guaranteed discoverability before the command returns (e.g., sharing a link immediately), use `--fast-provide-wait` to block until the provide completes. **Simple examples:** ```bash ipfs add file.txt # Root provided immediately, blocks queued for sweep provider ipfs add file.txt --fast-provide-wait # Wait for root provide to complete ipfs dag import file.car # Same for CAR imports ``` **Configuration:** Set defaults via `Import.FastProvideRoot` (default: `true`) and `Import.FastProvideWait` (default: `false`). See `ipfs add --help` and `ipfs dag import --help` for more details and examples. Fast root CID provide is automatically skipped when DHT routing is unavailable (e.g., `Routing.Type=none` or delegated-only configurations). #### ⏯️ Provider state persists across restarts The Sweep provider now persists the reprovide cycle state and automatically resumes where it left off after a restart. This brings several improvements: - **Persistent progress**: The provider saves its position in the reprovide cycle to the datastore. On restart, it continues from where it stopped instead of starting from scratch. - **Catch-up reproviding**: If the node was offline for an extended period, all CIDs that haven't been reprovided within the configured reprovide interval are immediately queued for reproviding when the node starts up. This ensures content availability is maintained even after downtime. - **Persistent provide queue**: The provide queue is persisted to the datastore on shutdown. When the node restarts, queued CIDs are restored and provided as expected, preventing loss of pending provide operations. - **Resume control**: The resume behavior is controlled via [`Provide.DHT.ResumeEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtresumeenabled) (default: `true`). Set to `false` if you don't want to keep the persisted provider state from a previous run. This feature improves reliability for nodes that experience intermittent connectivity or restarts. #### 📊 Detailed statistics with `ipfs provide stat` The Sweep provider system now exposes detailed statistics through `ipfs provide stat`, helping you monitor provider health and troubleshoot issues. Run `ipfs provide stat` for a quick summary, or use `--all` to see complete metrics including connectivity status, queue sizes, reprovide schedules, network statistics, operation rates, and worker utilization. For real-time monitoring, use `watch ipfs provide stat --all --compact` to observe changes in a 2-column layout. Individual sections can be displayed with flags like `--network`, `--operations`, or `--workers`. For Dual DHT configurations, use `--lan` to view LAN DHT statistics instead of the default WAN DHT stats. For more information, run `ipfs provide stat --help` or see the [Provide Stats documentation](https://github.com/ipfs/kubo/blob/master/docs/provide-stats.md), including [Capacity Planning](https://github.com/ipfs/kubo/blob/master/docs/provide-stats.md#capacity-planning). > [!NOTE] > Legacy provider (when `Provide.DHT.SweepEnabled=false`) shows basic statistics without flag support. #### 🔔 Slow reprovide warnings Kubo now monitors DHT reprovide operations when `Provide.DHT.SweepEnabled=true` and alerts you if your node is falling behind on reprovides. When the reprovide queue consistently grows and all periodic workers are busy, a warning displays with: - Queue size and worker utilization details - Recommended solutions: increase `Provide.DHT.MaxWorkers` or `Provide.DHT.DedicatedPeriodicWorkers` - Command to monitor real-time progress: `watch ipfs provide stat --all --compact` The alert polls every 15 minutes (to avoid alert fatigue while catching persistent issues) and only triggers after sustained growth across multiple intervals. The legacy provider is unaffected by this change. #### 📊 Metric rename: `provider_provides_total` The Amino DHT Sweep provider metric has been renamed from `total_provide_count_total` to `provider_provides_total` to follow OpenTelemetry naming conventions and maintain consistency with other kad-dht metrics (which use dot notation like `rpc.inbound.messages`, `rpc.outbound.requests`, etc.). **Migration:** If you have Prometheus queries, dashboards, or alerts monitoring the old `total_provide_count_total` metric, update them to use `provider_provides_total` instead. This affects all nodes using sweep mode, which is now the default in v0.39 (previously opt-in experimental in v0.38). #### 🔧 Automatic UPnP recovery after router restarts Kubo now automatically recovers UPnP port mappings when routers restart or become temporarily unavailable, fixing a critical connectivity issue that affected self-hosted nodes behind NAT. **Previous behavior:** When a UPnP-enabled router restarted, Kubo would lose its port mapping and fail to re-establish it automatically. Nodes would become unreachable to the network until the daemon was manually restarted, forcing reliance on relay connections which degraded performance. **New behavior:** The upgraded go-libp2p (v0.44.0) includes [Shipyard's fix](https://github.com/libp2p/go-libp2p/pull/3367) for self-healing NAT mappings that automatically rediscover and re-establish port forwarding after router events. Nodes now maintain public connectivity without manual intervention. > [!NOTE] > If your node runs behind a router and you haven't manually configured port > forwarding, make sure [`Swarm.DisableNatPortMap=false`](https://github.com/ipfs/kubo/blob/master/docs/config.md#swarmdisablenatportmap) > so UPnP can automatically handle port mapping (this is the default). This significantly improves reliability for desktop and self-hosted IPFS nodes using UPnP for NAT traversal. #### 🪦 Deprecated `go-ipfs` name no longer published The `go-ipfs` name was deprecated in 2022 and renamed to `kubo`. Starting with this release, the legacy Docker image name has been replaced with a stub that displays an error message directing users to switch to `ipfs/kubo`. **Docker images:** The `ipfs/go-ipfs` image tags now contain only a stub script that exits with an error, instructing users to update their Docker configurations to use [`ipfs/kubo`](https://hub.docker.com/r/ipfs/kubo) instead. This ensures users are aware of the deprecation while allowing existing automation to fail explicitly rather than silently using outdated images. **Distribution binaries:** Download Kubo from or . The legacy `go-ipfs` distribution path should no longer be used. All users should migrate to the `kubo` name in their scripts and configurations. #### 🚦 Gateway range request limits for CDN compatibility The new [`Gateway.MaxRangeRequestFileSize`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaymaxrangerequestfilesize) configuration protects against CDN range request limitations that cause bandwidth overcharges on deserialized responses. Some CDNs convert range requests over large files into full file downloads, causing clients requesting small byte ranges to unknowingly download entire multi-gigabyte files. This only impacts deserialized responses. Clients using verifiable block requests (`application/vnd.ipld.raw`) are not affected. See the [configuration documentation](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaymaxrangerequestfilesize) for details. #### 🖥️ RISC-V support with prebuilt binaries Kubo provides official `linux-riscv64` prebuilt binaries, bringing IPFS to [RISC-V](https://en.wikipedia.org/wiki/RISC-V) open hardware. As RISC-V single-board computers and embedded systems become more accessible, the distributed web is now supported on open hardware architectures - a natural pairing of open technologies. Download from or and look for the `linux-riscv64` archive. ### 📦️ Important dependency updates - update `go-libp2p` to [v0.45.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.45.0) (incl. [v0.44.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.44.0)) with self-healing UPnP port mappings and go-log/slog interop fixes - update `quic-go` to [v0.55.0](https://github.com/quic-go/quic-go/releases/tag/v0.55.0) - update `go-log` to [v2.9.0](https://github.com/ipfs/go-log/releases/tag/v2.9.0) with slog integration for go-libp2p - update `go-ds-pebble` to [v0.5.7](https://github.com/ipfs/go-ds-pebble/releases/tag/v0.5.7) (includes pebble [v2.1.2](https://github.com/cockroachdb/pebble/releases/tag/v2.1.2)) - update `boxo` to [v0.35.2](https://github.com/ipfs/boxo/releases/tag/v0.35.2) (includes boxo [v0.35.1](https://github.com/ipfs/boxo/releases/tag/v0.35.1)) - update `ipfs-webui` to [v4.10.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.10.0) - update `go-libp2p-kad-dht` to [v0.36.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.36.0) ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - docs: mkreleaselog for 0.39 - chore: version 0.39.0 - bin/mkreleaselog: add github handle resolution and deduplication - docs: restructure v0.39 changelog for clarity - upgrade go-libp2p-kad-dht to v0.36.0 (#11079) ([ipfs/kubo#11079](https://github.com/ipfs/kubo/pull/11079)) - fix(docker): include symlinks in scanning for init scripts (#11077) ([ipfs/kubo#11077](https://github.com/ipfs/kubo/pull/11077)) - Update deprecation message for Reprovider fields (#11072) ([ipfs/kubo#11072](https://github.com/ipfs/kubo/pull/11072)) - chore: release v0.39.0-rc1 - test: add regression tests for config secrets protection (#11061) ([ipfs/kubo#11061](https://github.com/ipfs/kubo/pull/11061)) - test: add regression tests for API.Authorizations (#11060) ([ipfs/kubo#11060](https://github.com/ipfs/kubo/pull/11060)) - test: verifyWorkerRun and helptext (#11063) ([ipfs/kubo#11063](https://github.com/ipfs/kubo/pull/11063)) - test(cmdutils): add tests for PathOrCidPath and ValidatePinName (#11062) ([ipfs/kubo#11062](https://github.com/ipfs/kubo/pull/11062)) - fix: return original error in PathOrCidPath fallback (#11059) ([ipfs/kubo#11059](https://github.com/ipfs/kubo/pull/11059)) - feat: fast provide support in `dag import` (#11058) ([ipfs/kubo#11058](https://github.com/ipfs/kubo/pull/11058)) - feat(cli/rpc/add): fast provide of root CID (#11046) ([ipfs/kubo#11046](https://github.com/ipfs/kubo/pull/11046)) - feat(telemetry): collect high level provide DHT sweep settings (#11056) ([ipfs/kubo#11056](https://github.com/ipfs/kubo/pull/11056)) - feat: enable DHT Provide Sweep by default (#10955) ([ipfs/kubo#10955](https://github.com/ipfs/kubo/pull/10955)) - feat(config): optional Gateway.MaxRangeRequestFileSize (#10997) ([ipfs/kubo#10997](https://github.com/ipfs/kubo/pull/10997)) - docs: clarify provide stats metric types and calculations (#11041) ([ipfs/kubo#11041](https://github.com/ipfs/kubo/pull/11041)) - Upgrade to Boxo v0.35.2 (#11050) ([ipfs/kubo#11050](https://github.com/ipfs/kubo/pull/11050)) - fix(go-log@2.9/go-libp2p@0.45): dynamic log level control and tail (#11039) ([ipfs/kubo#11039](https://github.com/ipfs/kubo/pull/11039)) - chore: update webui to v4.10.0 (#11048) ([ipfs/kubo#11048](https://github.com/ipfs/kubo/pull/11048)) - fix(provider/stats): number format (#11045) ([ipfs/kubo#11045](https://github.com/ipfs/kubo/pull/11045)) - provider: protect libp2p connections (#11028) ([ipfs/kubo#11028](https://github.com/ipfs/kubo/pull/11028)) - Merge release v0.38.2 ([ipfs/kubo#11044](https://github.com/ipfs/kubo/pull/11044)) - Upgrade to Boxo v0.35.1 (#11043) ([ipfs/kubo#11043](https://github.com/ipfs/kubo/pull/11043)) - feat(provider): resume cycle (#11031) ([ipfs/kubo#11031](https://github.com/ipfs/kubo/pull/11031)) - chore: upgrade pebble to v2.1.1 (#11040) ([ipfs/kubo#11040](https://github.com/ipfs/kubo/pull/11040)) - fix(cli): provide stat cosmetics (#11034) ([ipfs/kubo#11034](https://github.com/ipfs/kubo/pull/11034)) - fix: go-libp2p v0.44 with self-healing UPnP port mappings (#11032) ([ipfs/kubo#11032](https://github.com/ipfs/kubo/pull/11032)) - feat(provide): slow reprovide alerts when SweepEnabled (#11021) ([ipfs/kubo#11021](https://github.com/ipfs/kubo/pull/11021)) - feat: trace delegated routing http client (#11017) ([ipfs/kubo#11017](https://github.com/ipfs/kubo/pull/11017)) - feat(provide): detailed `ipfs provide stat` (#11019) ([ipfs/kubo#11019](https://github.com/ipfs/kubo/pull/11019)) - config: increase default Provide.DHT.MaxProvideConnsPerWorker (#11016) ([ipfs/kubo#11016](https://github.com/ipfs/kubo/pull/11016)) - docs: update release checklist based on v0.38.0 learnings (#11007) ([ipfs/kubo#11007](https://github.com/ipfs/kubo/pull/11007)) - chore: merge release v0.38.1 ([ipfs/kubo#11020](https://github.com/ipfs/kubo/pull/11020)) - fix: migrations for Windows (#11010) ([ipfs/kubo#11010](https://github.com/ipfs/kubo/pull/11010)) - Upgrade go-ds-pebble to v0.5.3 (#11011) ([ipfs/kubo#11011](https://github.com/ipfs/kubo/pull/11011)) - Merge release v0.38.0 ([ipfs/kubo#11006](https://github.com/ipfs/kubo/pull/11006)) - feat: add docker stub for deprecated ipfs/go-ipfs name (#10998) ([ipfs/kubo#10998](https://github.com/ipfs/kubo/pull/10998)) - docs: add sweeping provide worker count recommendation (#11001) ([ipfs/kubo#11001](https://github.com/ipfs/kubo/pull/11001)) - chore: bump go-libp2p-kad-dht to v0.35.0 (#11002) ([ipfs/kubo#11002](https://github.com/ipfs/kubo/pull/11002)) - upgrade go-ds-pebble to v0.5.2 (#11000) ([ipfs/kubo#11000](https://github.com/ipfs/kubo/pull/11000)) - Upgrade to Boxo v0.35.0 (#10999) ([ipfs/kubo#10999](https://github.com/ipfs/kubo/pull/10999)) - Non-functional changes (#10996) ([ipfs/kubo#10996](https://github.com/ipfs/kubo/pull/10996)) - chore: update boxo and kad-dht dependencies (#10995) ([ipfs/kubo#10995](https://github.com/ipfs/kubo/pull/10995)) - fix: update webui to v4.9.1 (#10994) ([ipfs/kubo#10994](https://github.com/ipfs/kubo/pull/10994)) - fix: provider merge conflicts (#10989) ([ipfs/kubo#10989](https://github.com/ipfs/kubo/pull/10989)) - fix(mfs): add soft limit for `--flush=false` (#10985) ([ipfs/kubo#10985](https://github.com/ipfs/kubo/pull/10985)) - fix: provide Filestore nodes (#10990) ([ipfs/kubo#10990](https://github.com/ipfs/kubo/pull/10990)) - feat: limit pin names to 255 bytes (#10981) ([ipfs/kubo#10981](https://github.com/ipfs/kubo/pull/10981)) - fix: SweepingProvider slow start (#10980) ([ipfs/kubo#10980](https://github.com/ipfs/kubo/pull/10980)) - chore: start v0.39.0 release cycle - github.com/gammazero/deque (v1.1.0 -> v1.2.0): - add slice operation functions (#40) ([gammazero/deque#40](https://github.com/gammazero/deque/pull/40)) - maintain base capacity after IterPop iteration (#44) ([gammazero/deque#44](https://github.com/gammazero/deque/pull/44)) - github.com/ipfs/boxo (v0.35.1 -> v0.35.2): - Release v0.35.2 ([ipfs/boxo#1068](https://github.com/ipfs/boxo/pull/1068)) - fix(logs): upgrade go-libp2p to v0.45.0 and go-log to v2.9.0 ([ipfs/boxo#1066](https://github.com/ipfs/boxo/pull/1066)) - github.com/ipfs/go-cid (v0.5.0 -> v0.6.0): - v0.6.0 bump (#178) ([ipfs/go-cid#178](https://github.com/ipfs/go-cid/pull/178)) - github.com/ipfs/go-ds-pebble (v0.5.3 -> v0.5.7): - new version (#74) ([ipfs/go-ds-pebble#74](https://github.com/ipfs/go-ds-pebble/pull/74)) - do not override logger if logger is provided (#72) ([ipfs/go-ds-pebble#72](https://github.com/ipfs/go-ds-pebble/pull/72)) - new version (#70) ([ipfs/go-ds-pebble#70](https://github.com/ipfs/go-ds-pebble/pull/70)) - new-version (#68) ([ipfs/go-ds-pebble#68](https://github.com/ipfs/go-ds-pebble/pull/68)) - Do not allow batch to be reused after commit (#67) ([ipfs/go-ds-pebble#67](https://github.com/ipfs/go-ds-pebble/pull/67)) - new version (#66) ([ipfs/go-ds-pebble#66](https://github.com/ipfs/go-ds-pebble/pull/66)) - Make pebble write options configurable ([ipfs/go-ds-pebble#63](https://github.com/ipfs/go-ds-pebble/pull/63)) - github.com/ipfs/go-dsqueue (v0.1.0 -> v0.1.1): - new version (#26) ([ipfs/go-dsqueue#26](https://github.com/ipfs/go-dsqueue/pull/26)) - update deque package and add stress test (#25) ([ipfs/go-dsqueue#25](https://github.com/ipfs/go-dsqueue/pull/25)) - github.com/ipfs/go-log/v2 (v2.8.2 -> v2.9.0): - chore: release v2.9.0 (#177) ([ipfs/go-log#177](https://github.com/ipfs/go-log/pull/177)) - fix: go-libp2p and slog interop (#176) ([ipfs/go-log#176](https://github.com/ipfs/go-log/pull/176)) - github.com/libp2p/go-libp2p (v0.43.0 -> v0.45.0): - Release v0.45.0 (#3424) ([libp2p/go-libp2p#3424](https://github.com/libp2p/go-libp2p/pull/3424)) - feat(gologshim): Add SetDefaultHandler (#3418) ([libp2p/go-libp2p#3418](https://github.com/libp2p/go-libp2p/pull/3418)) - Update Drips ownedBy address in FUNDING.json - fix(websocket): use debug level for http.Server errors - chore: release v0.44.0 - autonatv2: fix normalization for websocket addrs - autonatv2: remove dependency on webrtc and webtransport - quicreuse: update libp2p/go-netroute (#3405) ([libp2p/go-libp2p#3405](https://github.com/libp2p/go-libp2p/pull/3405)) - basichost: don't advertise unreachable addrs. (#3357) ([libp2p/go-libp2p#3357](https://github.com/libp2p/go-libp2p/pull/3357)) - basichost: improve autonatv2 reachability logic (#3356) ([libp2p/go-libp2p#3356](https://github.com/libp2p/go-libp2p/pull/3356)) - basichost: fix lint error - basichost: move EvtLocalAddrsChanged to addrs_manager (#3355) ([libp2p/go-libp2p#3355](https://github.com/libp2p/go-libp2p/pull/3355)) - chore: gitignore go.work files - refactor!: move insecure transport outside of core - refactor: drop go-varint dependency - refactor!: move canonicallog package outside of core - fix: assignment to entry in nil map - docs: Update contribute section with mailing list and irc (#3387) ([libp2p/go-libp2p#3387](https://github.com/libp2p/go-libp2p/pull/3387)) - README: remove Drand from notable users section - chore: add help comment - refactor: replace context.WithCancel with t.Context - feat(network): Add Conn.As - Skip mdns tests on macOS in CI - fix: deduplicate NAT port mapping requests - fix: heal NAT mappings after router restart - feat: relay: add option for custom filter function - docs: remove broken link (#3375) ([libp2p/go-libp2p#3375](https://github.com/libp2p/go-libp2p/pull/3375)) - AI tooling must be disclosed for contributions (#3372) ([libp2p/go-libp2p#3372](https://github.com/libp2p/go-libp2p/pull/3372)) - feat: Migrate to log/slog (#3364) ([libp2p/go-libp2p#3364](https://github.com/libp2p/go-libp2p/pull/3364)) - basichost: move observed address manager to basichost (#3332) ([libp2p/go-libp2p#3332](https://github.com/libp2p/go-libp2p/pull/3332)) - chore: support Go 1.24 & 1.25 (#3366) ([libp2p/go-libp2p#3366](https://github.com/libp2p/go-libp2p/pull/3366)) - feat(simlibp2p): Simulated libp2p Networks (#3262) ([libp2p/go-libp2p#3262](https://github.com/libp2p/go-libp2p/pull/3262)) - bandwidthcounter: add Reset and TrimIdle methods to reporter interface (#3343) ([libp2p/go-libp2p#3343](https://github.com/libp2p/go-libp2p/pull/3343)) - network: rename NAT Types (#3331) ([libp2p/go-libp2p#3331](https://github.com/libp2p/go-libp2p/pull/3331)) - refactor(quicreuse): use errors.Join in Close method (#3363) ([libp2p/go-libp2p#3363](https://github.com/libp2p/go-libp2p/pull/3363)) - swarm: move AddCertHashes to swarm (#3330) ([libp2p/go-libp2p#3330](https://github.com/libp2p/go-libp2p/pull/3330)) - quicreuse: clean up associations for closed listeners. (#3306) ([libp2p/go-libp2p#3306](https://github.com/libp2p/go-libp2p/pull/3306)) - github.com/libp2p/go-libp2p-kad-dht (v0.35.1 -> v0.36.0): - new version (#1204) ([libp2p/go-libp2p-kad-dht#1204](https://github.com/libp2p/go-libp2p-kad-dht/pull/1204)) - update dependencies (#1205) ([libp2p/go-libp2p-kad-dht#1205](https://github.com/libp2p/go-libp2p-kad-dht/pull/1205)) - fix(provider): protect `SweepingProvider.wg` (#1200) ([libp2p/go-libp2p-kad-dht#1200](https://github.com/libp2p/go-libp2p-kad-dht/pull/1200)) - fix(ResettableKeystore): race when closing during reset (#1201) ([libp2p/go-libp2p-kad-dht#1201](https://github.com/libp2p/go-libp2p-kad-dht/pull/1201)) - fix(provider): conflict resolution (#1199) ([libp2p/go-libp2p-kad-dht#1199](https://github.com/libp2p/go-libp2p-kad-dht/pull/1199)) - fix(provider): remove from trie by pruning prefix (#1198) ([libp2p/go-libp2p-kad-dht#1198](https://github.com/libp2p/go-libp2p-kad-dht/pull/1198)) - fix(provider): rename metric to follow OpenTelemetry conventions (#1195) ([libp2p/go-libp2p-kad-dht#1195](https://github.com/libp2p/go-libp2p-kad-dht/pull/1195)) - fix(provider): resume cycle from persisted keystore (#1193) ([libp2p/go-libp2p-kad-dht#1193](https://github.com/libp2p/go-libp2p-kad-dht/pull/1193)) - feat(provider): connectivity callbacks (#1194) ([libp2p/go-libp2p-kad-dht#1194](https://github.com/libp2p/go-libp2p-kad-dht/pull/1194)) - feat(provider): trie iterators (#1189) ([libp2p/go-libp2p-kad-dht#1189](https://github.com/libp2p/go-libp2p-kad-dht/pull/1189)) - refactor(provider): optimize memory when allocating keys to peers (#1187) ([libp2p/go-libp2p-kad-dht#1187](https://github.com/libp2p/go-libp2p-kad-dht/pull/1187)) - refactor(keystore): track size (#1181) ([libp2p/go-libp2p-kad-dht#1181](https://github.com/libp2p/go-libp2p-kad-dht/pull/1181)) - Remove go-libp2p-maintainers from codeowners (#1192) ([libp2p/go-libp2p-kad-dht#1192](https://github.com/libp2p/go-libp2p-kad-dht/pull/1192)) - switch to bit256.NewKeyFromArray (#1188) ([libp2p/go-libp2p-kad-dht#1188](https://github.com/libp2p/go-libp2p-kad-dht/pull/1188)) - fix(provider): `RegionsFromPeers` may return multiple regions (#1185) ([libp2p/go-libp2p-kad-dht#1185](https://github.com/libp2p/go-libp2p-kad-dht/pull/1185)) - feat(provider): skip bootstrap reprovide (#1186) ([libp2p/go-libp2p-kad-dht#1186](https://github.com/libp2p/go-libp2p-kad-dht/pull/1186)) - refactor(provider): use adaptive deadline for CycleStats cleanup (#1183) ([libp2p/go-libp2p-kad-dht#1183](https://github.com/libp2p/go-libp2p-kad-dht/pull/1183)) - refactor(provider/stats): use int64 to avoid overflows (#1182) ([libp2p/go-libp2p-kad-dht#1182](https://github.com/libp2p/go-libp2p-kad-dht/pull/1182)) - provider: trigger connectivity check when missing libp2p addresses (#1180) ([libp2p/go-libp2p-kad-dht#1180](https://github.com/libp2p/go-libp2p-kad-dht/pull/1180)) - fix(provider): resume cycle (#1176) ([libp2p/go-libp2p-kad-dht#1176](https://github.com/libp2p/go-libp2p-kad-dht/pull/1176)) - tests: fix flaky TestProvidesExpire (#1179) ([libp2p/go-libp2p-kad-dht#1179](https://github.com/libp2p/go-libp2p-kad-dht/pull/1179)) - tests: fix flaky TestFindPeerWithQueryFilter (#1178) ([libp2p/go-libp2p-kad-dht#1178](https://github.com/libp2p/go-libp2p-kad-dht/pull/1178)) - tests: fix #1175 (#1177) ([libp2p/go-libp2p-kad-dht#1177](https://github.com/libp2p/go-libp2p-kad-dht/pull/1177)) - feat(provider): exit early region exploration if no new peers discovered (#1174) ([libp2p/go-libp2p-kad-dht#1174](https://github.com/libp2p/go-libp2p-kad-dht/pull/1174)) - provider: protect connections (#1172) ([libp2p/go-libp2p-kad-dht#1172](https://github.com/libp2p/go-libp2p-kad-dht/pull/1172)) - feat(provider): resume reprovides (#1170) ([libp2p/go-libp2p-kad-dht#1170](https://github.com/libp2p/go-libp2p-kad-dht/pull/1170)) - fix(provider): custom logger name (#1173) ([libp2p/go-libp2p-kad-dht#1173](https://github.com/libp2p/go-libp2p-kad-dht/pull/1173)) - feat(provider): persist provide queue (#1167) ([libp2p/go-libp2p-kad-dht#1167](https://github.com/libp2p/go-libp2p-kad-dht/pull/1167)) - provider: stats (#1144) ([libp2p/go-libp2p-kad-dht#1144](https://github.com/libp2p/go-libp2p-kad-dht/pull/1144)) - github.com/probe-lab/go-libdht (v0.3.0 -> v0.4.0): - chore: release v0.4.0 (#26) ([probe-lab/go-libdht#26](https://github.com/probe-lab/go-libdht/pull/26)) - feat(key/bit256): memory optimized constructor (#25) ([probe-lab/go-libdht#25](https://github.com/probe-lab/go-libdht/pull/25)) - refactor(trie): AddMany memory optimization (#24) ([probe-lab/go-libdht#24](https://github.com/probe-lab/go-libdht/pull/24))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | [@guillaumemichel](https://github.com/guillaumemichel) | 41 | +9906/-1383 | 170 | | [@lidel](https://github.com/lidel) | 30 | +6652/-694 | 97 | | [@sukunrt](https://github.com/sukunrt) | 9 | +1618/-1524 | 39 | | [@MarcoPolo](https://github.com/MarcoPolo) | 17 | +1665/-1452 | 160 | | [@gammazero](https://github.com/gammazero) | 23 | +514/-53 | 29 | | [@Prabhat1308](https://github.com/Prabhat1308) | 1 | +197/-67 | 4 | | [@peterargue](https://github.com/peterargue) | 3 | +82/-25 | 5 | | [@cargoedit](https://github.com/cargoedit) | 1 | +35/-72 | 14 | | [@hsanjuan](https://github.com/hsanjuan) | 2 | +66/-29 | 5 | | [@shoriwe](https://github.com/shoriwe) | 1 | +68/-21 | 3 | | [@dennis-tra](https://github.com/dennis-tra) | 2 | +27/-2 | 2 | | [@Lil-Duckling-22](https://github.com/Lil-Duckling-22) | 1 | +4/-1 | 1 | | [@crStiv](https://github.com/crStiv) | 1 | +1/-3 | 1 | | [@cpeliciari](https://github.com/cpeliciari) | 1 | +3/-0 | 1 | | [@rvagg](https://github.com/rvagg) | 1 | +1/-1 | 1 | | [@p-shahi](https://github.com/p-shahi) | 1 | +1/-1 | 1 | | [@lbarrettanderson](https://github.com/lbarrettanderson) | 1 | +1/-1 | 1 | | [@filipremb](https://github.com/filipremb) | 1 | +1/-1 | 1 | | [@marten-seemann](https://github.com/marten-seemann) | 1 | +0/-1 | 1 | ================================================ FILE: docs/changelogs/v0.4.md ================================================ # go-ipfs changelog v0.4 ## v0.4.23 2020-01-29 Given the large number of fixes merged since 0.4.22, we've decided to cut another patch release. This release contains critical fixes. Please upgrade ASAP. Importantly, we're strongly considering switching to TLS by default in go-ipfs 0.5.0 and dropping SECIO support. However, the current TLS transport in go-ipfs 0.4.22 has a bug that can cause connections to spontaneously disconnect during the handshake. This release fixes that bug, among many other issues. Users that _don't_ upgrade may experience connectivity issues when the network upgrades to go-ipfs 0.5.0. ### Highlights * Fixes build on go 1.13 * Fixes an issue where we may not connect to providers in bitswap. * Fixes an issue on the TLS transport where we may abort a handshake unintentionally. * Fixes a common panic in the websocket transport. * Adds support for recursively resolving dnsaddrs (makes go-ipfs compatible with the new bootstrappers). * Fixes several potential panics/crashes. * Switches to using pre-defined autorelays instead of trying to find them in the DHT: * Avoids selecting random, potentially poor, relays. * Avoids spamming the DHT with requests trying to find relays. * Reduces the impact of accidentally enabling AutoRelay + RelayHop. I.e., the network won't try to DoS you. * Modifies the connection manager to not count connections in the grace period towards the connection limit. * Pro: New connections don't cause us to close useful, existing connections. * Con: Libp2p will keep more connections. Consider reducing your HighWater after applying this patch. * Improved peer usefulness tracking in bitswap. Frequently used peers will be marked as "important" and the connection manager will avoid closing connections to these peers. * Includes a new version of the WebUI to fix some issues with the peers map. ### Changelog - github.com/ipfs/go-ipfs: - feat: update the webui to fix some performance issues ([ipfs/go-ipfs#6844](https://github.com/ipfs/go-ipfs/pull/6844)) - fix: limit SW registration to content root ([ipfs/go-ipfs#6801](https://github.com/ipfs/go-ipfs/pull/6801)) - fix issue 6760, adding with hash-only, high CPU usage. ([ipfs/go-ipfs#6764](https://github.com/ipfs/go-ipfs/pull/6764)) - fix(coreapi/add): close the fake repo used when adding with hash-only ([ipfs/go-ipfs#6747](https://github.com/ipfs/go-ipfs/pull/6747)) - fix bug 6748 ([ipfs/go-ipfs#6754](https://github.com/ipfs/go-ipfs/pull/6754)) - fix(pin): wait till after fetching to remove direct pin ([ipfs/go-ipfs#6708](https://github.com/ipfs/go-ipfs/pull/6708)) - pin: fix pin update X Y where X==Y ([ipfs/go-ipfs#6669](https://github.com/ipfs/go-ipfs/pull/6669)) - namesys: set the correct cache TTL on publish ([ipfs/go-ipfs#6667](https://github.com/ipfs/go-ipfs/pull/6667)) - build: fix golangci again ([ipfs/go-ipfs#6641](https://github.com/ipfs/go-ipfs/pull/6641)) - make: move all test deps to a separate module ([ipfs/go-ipfs#6637](https://github.com/ipfs/go-ipfs/pull/6637)) - fix: close peerstore on stop ([ipfs/go-ipfs#6629](https://github.com/ipfs/go-ipfs/pull/6629)) - build: fix build when we don't have a full git tree ([ipfs/go-ipfs#6626](https://github.com/ipfs/go-ipfs/pull/6626)) - github.com/ipfs/go-bitswap (v0.0.8-cbb485998356 -> v0.0.8-e37498cf10d6): - fix: wait until we finish connecting before we cancel the context ([ipfs/go-bitswap#226](https://github.com/ipfs/go-bitswap/pull/226)) - engine: tag peers based on usefulness ([ipfs/go-bitswap#191](https://github.com/ipfs/go-bitswap/pull/191)) - github.com/ipfs/go-cid (v0.0.2 -> v0.0.4): - fix parsing issues and nits ([ipfs/go-cid#97](https://github.com/ipfs/go-cid/pull/97)) - Verify that prefix is correct v0 prefix ([ipfs/go-cid#96](https://github.com/ipfs/go-cid/pull/96)) - github.com/multiformats/go-multihash (v0.0.5 -> v0.0.10): - Ensure that length of multihash is properly handled ([multiformats/go-multihash#119](https://github.com/multiformats/go-multihash/pull/119)) - fix murmur3 name ([multiformats/go-multihash#115](https://github.com/multiformats/go-multihash/pull/115)) - rename ID to IDENTITY ([multiformats/go-multihash#113](https://github.com/multiformats/go-multihash/pull/113)) ([multiformats/go-multihash#119](https://github.com/multiformats/go-multihash/pull/119)) - github.com/libp2p/go-flow-metrics (v0.0.1 -> v0.0.3): - fix bug in meter traversal logic ([libp2p/go-flow-metrics#11](https://github.com/libp2p/go-flow-metrics/pull/11)) - github.com/libp2p/go-libp2p (v0.0.28 -> v0.0.32): - options to configure known relays for autorelay ([libp2p/go-libp2p#705](https://github.com/libp2p/go-libp2p/pull/705)) - feat(host): recursively resolve addresses ([libp2p/go-libp2p#764](https://github.com/libp2p/go-libp2p/pull/764)) - mdns: always use interface addresses ([libp2p/go-libp2p#667](https://github.com/libp2p/go-libp2p/pull/667)) - github.com/libp2p/go-libp2p-connmgr (v0.0.6 -> v0.2.1): - don't count connections in the grace period against the limit ([libp2p/go-libp2p-connmgr#50](https://github.com/libp2p/go-libp2p-connmgr/pull/50)) - github.com/libp2p/go-libp2p-kad-dht (v0.0.13 -> v0.0.15): - metrics: fix memory leak ([libp2p/go-libp2p-kad-dht#390](https://github.com/libp2p/go-libp2p-kad-dht/pull/390)) - github.com/libp2p/go-libp2p-tls (v0.0.1 -> v0.0.2): - close the underlying connection when the handshake fails ([libp2p/go-libp2p-tls#39](https://github.com/libp2p/go-libp2p-tls/pull/39)) - make the error check for not receiving a public key more explicit ([libp2p/go-libp2p-tls#34](https://github.com/libp2p/go-libp2p-tls/pull/34)) - Fix: Connection Closed after handshake ([libp2p/go-libp2p-tls#37](https://github.com/libp2p/go-libp2p-tls/pull/37)) - github.com/libp2p/go-libp2p-swarm (v0.0.6 -> v0.0.7): - fix: don't assume that transports implement stringer ([libp2p/go-libp2p-swarm#134](https://github.com/libp2p/go-libp2p-swarm/pull/134)) - github.com/libp2p/go-ws-transport (v0.0.4 -> v0.0.6): - Add mutex for write/close ([libp2p/go-ws-transport#65](https://github.com/libp2p/go-ws-transport/pull/65)) Other: Update bloom filter libraries to remove unsound usage of the `unsafe` package. ### Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Steven Allen | 52 | +1866/-578 | 102 | | vyzo | 12 | +167/-90 | 22 | | whyrusleeping | 5 | +136/-52 | 7 | | Roman Proskuryakov | 7 | +94/-7 | 10 | | Jakub Sztandera | 3 | +58/-13 | 7 | | hcg1314 | 2 | +31/-11 | 2 | | Raúl Kripalani | 2 | +7/-33 | 6 | | Marten Seemann | 3 | +27/-10 | 5 | | Marcin Rataj | 2 | +26/-0 | 5 | | b5 | 1 | +2/-22 | 1 | | Hector Sanjuan | 1 | +11/-0 | 1 | | Yusef Napora | 1 | +4/-0 | 1 | ## v0.4.22 2019-08-06 We're releasing a PATCH release of go-ipfs based on 0.4.21 containing some critical fixes. The IPFS network has scaled to the point where small changes can have a wide-reaching impact on the entire network. To keep this situation from escalating, we've put a hold on releasing new features until we can improve our [release process](https://github.com/ipfs/go-ipfs/blob/master/docs/releases.md) (which we've trialed in this release) and [testing procedures](https://github.com/ipfs/go-ipfs/issues/6483). This release includes fixes for the following regressions: 1. A major bitswap throughput regression introduced in 0.4.21 ([ipfs/go-ipfs#6442](https://github.com/ipfs/go-ipfs/issues/6442)). 2. High bitswap CPU usage when connected to many (e.g. 10,000) peers. See [ipfs/go-bitswap#154](https://github.com/ipfs/go-bitswap/issues/154). 2. The local network discovery service sometimes initializes before the networking module, causing it to announce the wrong addresses and sometimes complain about not being able to determine the IP address ([ipfs/go-ipfs#6415](https://github.com/ipfs/go-ipfs/pull/6415)). It also includes fixes for: 1. Pins not being persisted after `ipfs block add --pin` ([ipfs/go-ipfs#6441](https://github.com/ipfs/go-ipfs/pull/6441)). 2. Panic due to concurrent map access when adding and listing pins at the same time ([ipfs/go-ipfs#6419](https://github.com/ipfs/go-ipfs/pull/6419)). 3. Potential pin-set corruption given a concurrent `ipfs repo gc` and `ipfs pin rm` ([ipfs/go-ipfs#6444](https://github.com/ipfs/go-ipfs/pull/6444)). 4. Build failure due to a deleted git tag in one of our dependencies ([ipfs/go-ds-badger#64](https://github.com/ipfs/go-ds-badger/pull/65)). Thanks to: * [@hannahhoward](https://github.com/hannahhoward) for fixing both bitswap issues. * [@sanderpick](https://github.com/sanderpick) for catching and fixing the local discovery bug. * [@campoy](https://github.com/campoy) for fixing the build issue. ## v0.4.21 2019-05-30 We're happy to announce go-ipfs 0.4.21. This release has some critical bug fixes and a handful of new features so every user should upgrade. Key bug fixes: * Too many open file descriptors/too many peers ([#6237](https://github.com/ipfs/go-ipfs/issues/6237)). * Adding multiple files at the same time doesn't work ([#6254](https://github.com/ipfs/go-ipfs/pull/6255)). * CPU utilization spikes and then holds at 100% ([#5613](https://github.com/ipfs/go-ipfs/issues/5613)). Key features: * Experimental TLS1.3 support (to eventually replace secio). * OpenSSL support for SECIO handshakes (performance improvement). **IMPORTANT:** This release fixes a bug in our security transport that could potentially drop data from the channel. Note: This issue affects neither the privacy nor the integrity of the data with respect to a third-party attacker. Only the peer sending us data could trigger this bug. **ALL USERS MUST UPGRADE.** We intended to introduce a feature this release that, unfortunately, [reliably triggered this bug][secio-bug]. To avoid partitioning the network, we've decided to postpone this feature for a release or two. Specifically, we're going to provide a minimum _one month_ upgrade period. After that, we'll start testing the impact of deploying the proposed changes. If you're running the mainline go-ipfs, please upgrade ASAP. If you're building a separate app or working on a forked go-ipfs, make sure to upgrade github.com/libp2p/go-libp2p-secio to _at least_ v0.0.3. [secio-bug]: https://github.com/libp2p/go-libp2p/issues/644 ### Contributors First off, we'd like to give a shout-out to all contributors that participated in this release (including contributions to ipld, libp2p, and multiformats): | Contributor | Commits | Lines ± | Files Changed | |----------------------------|---------|-------------|---------------| | Steven Allen | 220 | +6078/-4211 | 520 | | Łukasz Magiera | 53 | +5039/-4557 | 274 | | vyzo | 179 | +2929/-1704 | 238 | | Raúl Kripalani | 44 | +757/-1895 | 134 | | hannahhoward | 11 | +755/-1005 | 49 | | Marten Seemann | 16 | +862/-203 | 44 | | keks | 10 | +359/-110 | 12 | | Jan Winkelmann | 8 | +368/-26 | 16 | | Jakub Sztandera | 4 | +361/-8 | 7 | | Adrian Lanzafame | 1 | +287/-18 | 5 | | Erik Ingenito | 4 | +247/-28 | 8 | | Reid 'arrdem' McKenzie | 1 | +220/-20 | 3 | | Yusef Napora | 26 | +98/-130 | 26 | | Michael Avila | 3 | +116/-59 | 8 | | Raghav Gulati | 13 | +145/-26 | 13 | | tg | 1 | +41/-33 | 1 | | Matt Joiner | 6 | +41/-30 | 7 | | Cole Brown | 1 | +37/-25 | 1 | | Dominic Della Valle | 2 | +12/-40 | 4 | | Overbool | 1 | +50/-0 | 2 | | Christopher Buesser | 3 | +29/-16 | 10 | | myself659 | 1 | +38/-5 | 2 | | Alex Browne | 3 | +30/-8 | 3 | | jmank88 | 1 | +27/-4 | 2 | | Vikram | 1 | +25/-1 | 2 | | MollyM | 7 | +17/-9 | 7 | | Marcin Rataj | 1 | +17/-1 | 1 | | requilence | 1 | +11/-4 | 1 | | Teran McKinney | 1 | +8/-2 | 1 | | Oli Evans | 1 | +5/-5 | 1 | | Masashi Salvador Mitsuzawa | 1 | +5/-1 | 1 | | chenminjian | 1 | +4/-0 | 1 | | Edgar Lee | 1 | +3/-1 | 1 | | Dirk McCormick | 1 | +2/-2 | 2 | | ia | 1 | +1/-1 | 1 | | Alan Shaw | 1 | +1/-1 | 1 | ### Bug Fixes And Enhancements This release includes quite a number of critical bug fixes and performance/reliability enhancements. #### Error when adding multiple files The last release broke the simple command `ipfs add file1 file2`. It turns out we simply lacked a test case for this. Both of these issues (the bug and the lack of a test case) have now been fixed. #### SECIO As noted above, we've fixed a bug that could cause data to be dropped from a SECIO connection on read. Specifically, this happens when: 1. The capacity of the read buffer is greater than the length. 2. The remote peer sent more than the length but less than the capacity in a single secio "frame". In this case, we'd fill the read buffer to it's capacity instead of its length. #### Too many open files, too many peers, etc. Go-ipfs automatically closes the least useful connections when it accumulates too many connections. Unfortunately, some relayed connections were blocking in `Close()`, halting the entire process. #### Out of control CPU usage Many users noted out of control CPU usage this release. This turned out to be a long-standing issue with how the DHT handled provider records (records recording which peers have what content): 1. It wasn't removing provider records for content until the set of providers completely emptied. 2. It was loading every provider record into memory whenever we updated the set of providers. Combined, these two issues were trashing the provider record cache, forcing the DHT to repeatedly load and discard provider records. #### More Reliable Connection Management Go-ipfs has a subsystem called the "connection manager" to close the least-useful connections when go-ipfs runs low on resources. Unfortunately, other IPFS subsystems may learn about connections _before_ the connection manager. Previously, if some IPFS subsystem tried to mark a connection as useful before the connection manager learned about it, the connection manager would discard this information. We believe this was causing [#6271](https://github.com/ipfs/go-ipfs/issues/6271). [It no longer does that](https://github.com/libp2p/go-libp2p-connmgr/pull/39). #### Improved Bitswap Connection Management Bitswap now uses the connection manager to mark all peers downloading blocks as important (while downloading). Previously, it only marked peers from which _it_ was downloading blocks. #### Reduced Memory Usage The most noticeable memory reduction in this release comes from fixing connection closing. However, we've made a few additional improvements: * Bitswap's "work queue" no longer remembers every peer it has seen indefinitely. * The peerstore now interns protocol names. * The per-peer goroutine count has been reduced. * The DHT now wastes less memory on idle peers by pooling buffered writers and returning them to the pool when not actively using them. #### Increased File Descriptor Limit The default file descriptor limit has been raised to 8192 (from 2048). Unfortunately, go-ipfs behaves poorly when it runs out of file descriptors and it uses a _lot_ of file descriptors. Luckily, most modern kernels can handle thousands of file descriptors without any difficulty. #### Decreased Connection Handshake Latency Libp2p now shaves off a couple of round trips when initiating connections by beginning the protocol negotiation before the remote peer responds to the initial handshake message. In the optimal case (when the target peer speaks our preferred protocol), this reduces the number of handshake round-trips from 6 to 4 (including the TCP handshake). ### Commands This release brings no new commands but does introduce a few changes, bug fixes, and enhancements. This section is hardly complete but it lists the most noticeable changes. Take note: this release also introduces a few breaking changes. #### [DEPRECATION] The URLStore Command Deprecated The experimental `ipfs urlstore` command is now deprecated. Please use `ipfs add --nocopy URL` instead. #### [BREAKING] The DHT Command Base64 Encodes Values When responding to an `ipfs dht get` command, the daemon now encodes the returned value using base64. The `ipfs` command will automatically decode this value before returning it to the user so this change should only affect those using the HTTP API directly. Unfortunately, this change was necessary as DHT records are arbitrary binary blobs which can't be directly stored in JSON strings. #### [BREAKING] Base32 Encoded v1 CIDs By Default Both js-ipfs and go-ipfs now encode CIDv1 CIDs using base32 by default, instead of base58. Unfortunately, base58 is case-sensitive and doesn't play well with browsers (see [#4143](https://github.com/ipfs/go-ipfs/issues/4143). #### Human Readable Numbers The `ipfs bitswap stat` and `ipfs object stat` commands now support a `--humanize` flag that formats numbers with human-readable units (GiB, MiB, etc.). #### Improved Errors This release improves two types of errors: 1. Commands that take paths/multiaddrs now include the path/multiaddr in the error message when it fails to parse. 2. `ipfs swarm connect` now returns a detailed error describing which addresses were tried and why the dial failed. #### Ping Improvements The ping command has received some small improvements and fixes: 1. It now exits with a non-zero exit status on failure. 2. It no longer succeeds with zero successful pings if we have a zombie but non-functional connection to the peer being pinged ([#6298](https://github.com/ipfs/go-ipfs/issues/6298)). 3. It now prints out the average latency when canceled with `^C` (like the unix `ping` command). #### Improved Help Text Go-ipfs now intelligently wraps help text for easier reading. On an 80 character wide terminal, **Before** ``` USAGE ipfs add ... - Add a file or directory to ipfs. SYNOPSIS ipfs add [--recursive | -r] [--dereference-args] [--stdin-name=] [ --hidden | -H] [--quiet | -q] [--quieter | -Q] [--silent] [--progress | -p] [--t rickle | -t] [--only-hash | -n] [--wrap-with-directory | -w] [--chunker= | -s] [--pin=false] [--raw-leaves] [--nocopy] [--fscache] [--cid-version=] [--hash=] [--inline] [--inline-limit=] [--] ... ARGUMENTS ... - The path to a file to be added to ipfs. OPTIONS -r, --recursive bool - Add directory paths recursive ly. --dereference-args bool - Symlinks supplied in argument s are dereferenced. --stdin-name string - Assign a name if the file sou rce is stdin. -H, --hidden bool - Include files that are hidden . Only takes effect on recursive add. -q, --quiet bool - Write minimal output. -Q, --quieter bool - Write only final hash. --silent bool - Write no output. -p, --progress bool - Stream progress data. -t, --trickle bool - Use trickle-dag format for da g generation. -n, --only-hash bool - Only chunk and hash - do not write to disk. -w, --wrap-with-directory bool - Wrap files with a directory o object. -s, --chunker string - Chunking algorithm, size-[byt es] or rabin-[min]-[avg]-[max]. Default: size-262144. --pin bool - Pin this object when adding. Default: true. --raw-leaves bool - Use raw blocks for leaf nodes . (experimental). --nocopy bool - Add the file using filestore. Implies raw-leaves. (experimental). --fscache bool - Check the filestore for pre-e xisting blocks. (experimental). --cid-version int - CID version. Defaults to 0 un less an option that depends on CIDv1 is passed. (experimental). --hash string - Hash function to use. Implies CIDv1 if not sha2-256. (experimental). Default: sha2-256. --inline bool - Inline small blocks into CIDs . (experimental). --inline-limit int - Maximum block size to inline. (experimental). Default: 32. ``` **After** ``` USAGE ipfs add ... - Add a file or directory to ipfs. SYNOPSIS ipfs add [--recursive | -r] [--dereference-args] [--stdin-name=] [--hidden | -H] [--quiet | -q] [--quieter | -Q] [--silent] [--progress | -p] [--trickle | -t] [--only-hash | -n] [--wrap-with-directory | -w] [--chunker= | -s] [--pin=false] [--raw-leaves] [--nocopy] [--fscache] [--cid-version=] [--hash=] [--inline] [--inline-limit=] [--] ... ARGUMENTS ... - The path to a file to be added to ipfs. OPTIONS -r, --recursive bool - Add directory paths recursively. --dereference-args bool - Symlinks supplied in arguments are dereferenced. --stdin-name string - Assign a name if the file source is stdin. -H, --hidden bool - Include files that are hidden. Only takes effect on recursive add. -q, --quiet bool - Write minimal output. -Q, --quieter bool - Write only final hash. --silent bool - Write no output. -p, --progress bool - Stream progress data. -t, --trickle bool - Use trickle-dag format for dag generation. -n, --only-hash bool - Only chunk and hash - do not write to disk. -w, --wrap-with-directory bool - Wrap files with a directory object. -s, --chunker string - Chunking algorithm, size-[bytes] or rabin-[min]-[avg]-[max]. Default: size-262144. --pin bool - Pin this object when adding. Default: true. --raw-leaves bool - Use raw blocks for leaf nodes. (experimental). --nocopy bool - Add the file using filestore. Implies raw-leaves. (experimental). --fscache bool - Check the filestore for pre-existing blocks. (experimental). --cid-version int - CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. (experimental). --hash string - Hash function to use. Implies CIDv1 if not sha2-256. (experimental). Default: sha2-256. --inline bool - Inline small blocks into CIDs. (experimental). --inline-limit int - Maximum block size to inline. (experimental). Default: 32. ``` ### Features This release is primarily a bug fix release but it still includes two nice features from libp2p. #### Experimental TLS1.3 support Go-ipfs now has experimental TLS1.3 support. Currently, libp2p (IPFS's networking library) uses a custom TLS-like protocol we call SECIO. However, the conventional wisdom concerning custom security transports is "just don't" so we are working on replacing it with TLS1.3 To choose this protocol by default, set the `Experimental.PreferTLS` config variable: ```bash > ipfs config --bool Experimental.PreferTLS true ``` Why TLS1.3 and not X (noise, etc.)? 1. Libp2p allows negotiating transports so there's no reason not to add noise support to libp2p as well. 2. TLS has wide language support which should make implementing libp2p for new languages significantly simpler. #### OpenSSL Support Go-ipfs can now (optionally) be built with OpenSSL support for improved performance when establishing connections. This is primarily useful for nodes receiving multiple inbound connections per second. To enable openssl support, rebuild go-ipfs with: ```bash > make build GOTAGS=openssl ``` ### CoreAPI The CoreAPI refactor is still underway and we've made significant progress towards a usable ipfs-as-a-library constructor. Specifically, we've integrated the [fx](https://go.uber.org/fx) dependency injection system and are now working on cleaning up our initialization logic. This should make it easier to inject new services into a go-ipfs process without messing with the core internals. ### Build: `GOCC` Environment Variable Build system now uses `GOCC` environment variable allowing for use of specific go versions during builds. ### Changelog - github.com/ipfs/go-ipfs: - fix: use http.Error for sending errors ([ipfs/go-ipfs#6379](https://github.com/ipfs/go-ipfs/pull/6379)) - core: call app.Stop once ([ipfs/go-ipfs#6380](https://github.com/ipfs/go-ipfs/pull/6380)) - explain what dhtclient does ([ipfs/go-ipfs#6375](https://github.com/ipfs/go-ipfs/pull/6375)) - ci: actually enable golangci-lint ([ipfs/go-ipfs#6362](https://github.com/ipfs/go-ipfs/pull/6362)) - commands/swarm(fix): handle empty multiaddrs ([ipfs/go-ipfs#6355](https://github.com/ipfs/go-ipfs/pull/6355)) - feat: improve errors when a path fails to parse ([ipfs/go-ipfs#6346](https://github.com/ipfs/go-ipfs/pull/6346)) - fix vendoring dependencies when building the source tarball ([ipfs/go-ipfs#6349](https://github.com/ipfs/go-ipfs/pull/6349)) - core: Use correct default for connmgr lowWater ([ipfs/go-ipfs#6352](https://github.com/ipfs/go-ipfs/pull/6352)) - doc: remove out of date documentation ([ipfs/go-ipfs#6345](https://github.com/ipfs/go-ipfs/pull/6345)) - Add generation of dependency changes to mkreleaselog ([ipfs/go-ipfs#6348](https://github.com/ipfs/go-ipfs/pull/6348)) - readme: remove mention of DCO ([ipfs/go-ipfs#6344](https://github.com/ipfs/go-ipfs/pull/6344)) - Add golangci-lint ([ipfs/go-ipfs#6321](https://github.com/ipfs/go-ipfs/pull/6321)) - docs+mk: update guidance for unsupported platforms ([ipfs/go-ipfs#6338](https://github.com/ipfs/go-ipfs/pull/6338)) - fix formatting in object get ([ipfs/go-ipfs#6340](https://github.com/ipfs/go-ipfs/pull/6340)) - fail start when loading a plugin fails ([ipfs/go-ipfs#6339](https://github.com/ipfs/go-ipfs/pull/6339)) - fix a typo in the issue template ([ipfs/go-ipfs#6335](https://github.com/ipfs/go-ipfs/pull/6335)) - github: turn issue template into a multiple-choice question ([ipfs/go-ipfs#6333](https://github.com/ipfs/go-ipfs/pull/6333)) - object put: Allow empty objects ([ipfs/go-ipfs#6330](https://github.com/ipfs/go-ipfs/pull/6330)) - Update fuse.md ([ipfs/go-ipfs#6332](https://github.com/ipfs/go-ipfs/pull/6332)) - work towards fixing dht commands ([ipfs/go-ipfs#6277](https://github.com/ipfs/go-ipfs/pull/6277)) - fix setting ulimit ([ipfs/go-ipfs#6319](https://github.com/ipfs/go-ipfs/pull/6319)) - switch to base32 by default for CIDv1 ([ipfs/go-ipfs#6300](https://github.com/ipfs/go-ipfs/pull/6300)) - cmdkit -> cmds ([ipfs/go-ipfs#6318](https://github.com/ipfs/go-ipfs/pull/6318)) - raise default fd limit to 8192 ([ipfs/go-ipfs#6266](https://github.com/ipfs/go-ipfs/pull/6266)) - pin: don't walk all pinned blocks when removing a non-existent pin ([ipfs/go-ipfs#6311](https://github.com/ipfs/go-ipfs/pull/6311)) - ping: fix a bunch of issues ([ipfs/go-ipfs#6312](https://github.com/ipfs/go-ipfs/pull/6312)) - test(coreapi): use a thread-safe datastore everywhere ([ipfs/go-ipfs#6222](https://github.com/ipfs/go-ipfs/pull/6222)) - fix(Dockerfile): Allow ipfs mount in Docker container ([ipfs/go-ipfs#5560](https://github.com/ipfs/go-ipfs/pull/5560)) - docs: fix Routing section ([ipfs/go-ipfs#6309](https://github.com/ipfs/go-ipfs/pull/6309)) - License update to dual MIT and Apache 2 ([ipfs/go-ipfs#6301](https://github.com/ipfs/go-ipfs/pull/6301)) - Go test fix ([ipfs/go-ipfs#6293](https://github.com/ipfs/go-ipfs/pull/6293)) - commands(pin update): return resolved CIDs instead of paths ([ipfs/go-ipfs#6275](https://github.com/ipfs/go-ipfs/pull/6275)) - core: fix autonat construction ([ipfs/go-ipfs#6289](https://github.com/ipfs/go-ipfs/pull/6289)) - Test and fix GC/pin bug ([ipfs/go-ipfs#6288](https://github.com/ipfs/go-ipfs/pull/6288)) - GOCC implementation & fix in make & build scripts ([ipfs/go-ipfs#6282](https://github.com/ipfs/go-ipfs/pull/6282)) - gc: cancel context ([ipfs/go-ipfs#6281](https://github.com/ipfs/go-ipfs/pull/6281)) - fix: windows friendly daemon help ([ipfs/go-ipfs#6278](https://github.com/ipfs/go-ipfs/pull/6278)) - Invert constructor config handling ([ipfs/go-ipfs#6276](https://github.com/ipfs/go-ipfs/pull/6276)) - docs: document environment variables ([ipfs/go-ipfs#6268](https://github.com/ipfs/go-ipfs/pull/6268)) - add: Return error from iterator ([ipfs/go-ipfs#6272](https://github.com/ipfs/go-ipfs/pull/6272)) - commands(feat): use the coreapi in the urlstore command ([ipfs/go-ipfs#6259](https://github.com/ipfs/go-ipfs/pull/6259)) - humanize for ipfs bitswap stat ([ipfs/go-ipfs#6258](https://github.com/ipfs/go-ipfs/pull/6258)) - Revert "raise default fd limit to 8192" ([ipfs/go-ipfs#6265](https://github.com/ipfs/go-ipfs/pull/6265)) - raise default fd limit to 8192 ([ipfs/go-ipfs#6261](https://github.com/ipfs/go-ipfs/pull/6261)) - Fix AutoNAT service for private network ([ipfs/go-ipfs#6251](https://github.com/ipfs/go-ipfs/pull/6251)) - add: Fix adding multiple files ([ipfs/go-ipfs#6255](https://github.com/ipfs/go-ipfs/pull/6255)) - reprovider: Use goprocess ([ipfs/go-ipfs#6248](https://github.com/ipfs/go-ipfs/pull/6248)) - core/corehttp/gateway_handler: pass a request ctx instead of the node ([ipfs/go-ipfs#6244](https://github.com/ipfs/go-ipfs/pull/6244)) - constructor: cleanup some things ([ipfs/go-ipfs#6246](https://github.com/ipfs/go-ipfs/pull/6246)) - Support --human flag in cmd/object-stat ([ipfs/go-ipfs#6241](https://github.com/ipfs/go-ipfs/pull/6241)) - build: fix macos build with fuse ([ipfs/go-ipfs#6235](https://github.com/ipfs/go-ipfs/pull/6235)) - add an experiment to prefer TLS 1.3 over secio ([ipfs/go-ipfs#6229](https://github.com/ipfs/go-ipfs/pull/6229)) - fix two small nits in the go-ipfs constructor ([ipfs/go-ipfs#6234](https://github.com/ipfs/go-ipfs/pull/6234)) - DI-based core.NewNode ([ipfs/go-ipfs#6162](https://github.com/ipfs/go-ipfs/pull/6162)) - coreapi: Drop error from ParsePath ([ipfs/go-ipfs#6122](https://github.com/ipfs/go-ipfs/pull/6122)) - fix the wrong path configuration in root redirection ([ipfs/go-ipfs#6215](https://github.com/ipfs/go-ipfs/pull/6215)) - github.com/ipfs/go-bitswap (v0.0.4 -> v0.0.7): - feat(engine): tag peers with requests ([ipfs/go-bitswap#128](https://github.com/ipfs/go-bitswap/pull/128)) - fix(network): add mutex to avoid data race ([ipfs/go-bitswap#127](https://github.com/ipfs/go-bitswap/pull/127)) - Change bitswap provide toggle to not be static ([ipfs/go-bitswap#124](https://github.com/ipfs/go-bitswap/pull/124)) - Use shared peer task queue with Graphsync ([ipfs/go-bitswap#119](https://github.com/ipfs/go-bitswap/pull/119)) - Add missing godoc comments, refactor to avoid confusion ([ipfs/go-bitswap#117](https://github.com/ipfs/go-bitswap/pull/117)) - fix(decision): cleanup request queues ([ipfs/go-bitswap#116](https://github.com/ipfs/go-bitswap/pull/116)) - Control provider workers with experiment flag ([ipfs/go-bitswap#110](https://github.com/ipfs/go-bitswap/pull/110)) - connmgr: give peers more weight when actively participating in a session ([ipfs/go-bitswap#111](https://github.com/ipfs/go-bitswap/pull/111)) - make the WantlistManager own the PeerHandler ([ipfs/go-bitswap#78](https://github.com/ipfs/go-bitswap/pull/78)) - remove IPFS_LOW_MEM flag support ([ipfs/go-bitswap#115](https://github.com/ipfs/go-bitswap/pull/115)) - github.com/ipfs/go-cid (v0.0.1 -> v0.0.2): - default cidv1 to base32 ([ipfs/go-cid#85](https://github.com/ipfs/go-cid/pull/85)) - github.com/ipfs/go-cidutil (v0.0.1 -> v0.0.2): - default cidv1 to base32 ([ipfs/go-cidutil#13](https://github.com/ipfs/go-cidutil/pull/13)) - github.com/ipfs/go-datastore (v0.0.3 -> v0.0.5): - MapDatastore: obey KeysOnly ([ipfs/go-datastore#130](https://github.com/ipfs/go-datastore/pull/130)) - fix the keytransform datastore's query implementation ([ipfs/go-datastore#127](https://github.com/ipfs/go-datastore/pull/127)) - sync: apply entire query while locked ([ipfs/go-datastore#129](https://github.com/ipfs/go-datastore/pull/129)) - filter: values are now always bytes ([ipfs/go-datastore#126](https://github.com/ipfs/go-datastore/pull/126)) - autobatch: batch deletes ([ipfs/go-datastore#128](https://github.com/ipfs/go-datastore/pull/128)) - github.com/ipfs/go-ipfs-cmds (v0.0.5 -> v0.0.8): - fix: use golang's http.Error to send errors ([ipfs/go-ipfs-cmds#167](https://github.com/ipfs/go-ipfs-cmds/pull/167)) - improve help text on narrow terminals ([ipfs/go-ipfs-cmds#140](https://github.com/ipfs/go-ipfs-cmds/pull/140)) - chore: remove an old hack ([ipfs/go-ipfs-cmds#165](https://github.com/ipfs/go-ipfs-cmds/pull/165)) - http: use the request context ([ipfs/go-ipfs-cmds#163](https://github.com/ipfs/go-ipfs-cmds/pull/163)) - merge in go-ipfs-cmdkit ([ipfs/go-ipfs-cmds#164](https://github.com/ipfs/go-ipfs-cmds/pull/164)) - fix: return the correct error ([ipfs/go-ipfs-cmds#162](https://github.com/ipfs/go-ipfs-cmds/pull/162)) - github.com/ipfs/go-ipfs-config (v0.0.1 -> v0.0.3): - Closes: #6284 Add appropriate IPv6 ranges to defaultServerFilters ([ipfs/go-ipfs-config#34](https://github.com/ipfs/go-ipfs-config/pull/34)) - add an experiment to prefer TLS 1.3 over secio ([ipfs/go-ipfs-config#32](https://github.com/ipfs/go-ipfs-config/pull/32)) - github.com/ipfs/go-ipfs-files (v0.0.2 -> v0.0.3): - webfile: make Size() work before Read ([ipfs/go-ipfs-files#18](https://github.com/ipfs/go-ipfs-files/pull/18)) - check http status code during WebFile reads and return error for non-2XX ([ipfs/go-ipfs-files#17](https://github.com/ipfs/go-ipfs-files/pull/17)) - github.com/ipfs/go-ipld-cbor (v0.0.1 -> v0.0.2): - switch to base32 by default ([ipfs/go-ipld-cbor#62](https://github.com/ipfs/go-ipld-cbor/pull/62)) - github.com/ipfs/go-ipld-git (v0.0.1 -> v0.0.2): - switch to base32 by default ([ipfs/go-ipld-git#40](https://github.com/ipfs/go-ipld-git/pull/40)) - github.com/ipfs/go-mfs (v0.0.4 -> v0.0.7): - Fix directory mv and add tests ([ipfs/go-mfs#76](https://github.com/ipfs/go-mfs/pull/76)) - fix: not remove file by mistakes ([ipfs/go-mfs#73](https://github.com/ipfs/go-mfs/pull/73)) - github.com/ipfs/go-path (v0.0.3 -> v0.0.4): - include the path in path errors ([ipfs/go-path#28](https://github.com/ipfs/go-path/pull/28)) - github.com/ipfs/go-unixfs (v0.0.4 -> v0.0.6): - chore: remove URL field ([ipfs/go-unixfs#72](https://github.com/ipfs/go-unixfs/pull/72)) - github.com/ipfs/interface-go-ipfs-core (v0.0.6 -> v0.0.8): - switch to base32 cidv1 by default ([ipfs/interface-go-ipfs-core#29](https://github.com/ipfs/interface-go-ipfs-core/pull/29)) - path: drop error from ParsePath ([ipfs/interface-go-ipfs-core#22](https://github.com/ipfs/interface-go-ipfs-core/pull/22)) - tests: fix a bunch of small test lints/issues ([ipfs/interface-go-ipfs-core#28](https://github.com/ipfs/interface-go-ipfs-core/pull/28)) - Update Pin.RmRecursive docs to clarify shared indirect pins are not removed ([ipfs/interface-go-ipfs-core#26](https://github.com/ipfs/interface-go-ipfs-core/pull/26)) - github.com/libp2p/go-buffer-pool (v0.0.1 -> v0.0.2): - feat: add buffered writer ([libp2p/go-buffer-pool#9](https://github.com/libp2p/go-buffer-pool/pull/9)) - github.com/libp2p/go-conn-security-multistream (v0.0.1 -> v0.0.2): - block while writing ([libp2p/go-conn-security-multistream#10](https://github.com/libp2p/go-conn-security-multistream/pull/10)) - github.com/libp2p/go-libp2p (v0.0.12 -> v0.0.28): - Close the connection manager ([libp2p/go-libp2p#639](https://github.com/libp2p/go-libp2p/pull/639)) - Frequent Relay Advertisements ([libp2p/go-libp2p#637](https://github.com/libp2p/go-libp2p/pull/637)) - ping: return a stream of results ([libp2p/go-libp2p#626](https://github.com/libp2p/go-libp2p/pull/626)) - Use cancelable background context in identify ([libp2p/go-libp2p#624](https://github.com/libp2p/go-libp2p/pull/624)) - avoid intermediate allocation in relayAddrs ([libp2p/go-libp2p#609](https://github.com/libp2p/go-libp2p/pull/609)) - cache relayAddrs for a short period of time ([libp2p/go-libp2p#608](https://github.com/libp2p/go-libp2p/pull/608)) - autorelay: break findRelays into multiple functions and avoid the goto ([libp2p/go-libp2p#606](https://github.com/libp2p/go-libp2p/pull/606)) - autorelay: curtail addrsplosion ([libp2p/go-libp2p#598](https://github.com/libp2p/go-libp2p/pull/598)) - Periodically schedule identify push if the address set has changed ([libp2p/go-libp2p#597](https://github.com/libp2p/go-libp2p/pull/597)) - Replace peer addresses in identify ([libp2p/go-libp2p#599](https://github.com/libp2p/go-libp2p/pull/599)) - github.com/libp2p/go-libp2p-circuit (v0.0.4 -> v0.0.8): - call Stream.Reset instead of Stream.Close ([libp2p/go-libp2p-circuit#76](https://github.com/libp2p/go-libp2p-circuit/pull/76)) - Tag the hop relay when creating stop streams ([libp2p/go-libp2p-circuit#77](https://github.com/libp2p/go-libp2p-circuit/pull/77)) - Tag peers with live hop streams ([libp2p/go-libp2p-circuit#75](https://github.com/libp2p/go-libp2p-circuit/pull/75)) - Hard Limit the number of hop stream goroutines ([libp2p/go-libp2p-circuit#74](https://github.com/libp2p/go-libp2p-circuit/pull/74)) - set deadline for stop handshake ([libp2p/go-libp2p-circuit#73](https://github.com/libp2p/go-libp2p-circuit/pull/73)) - github.com/libp2p/go-libp2p-connmgr (v0.0.1 -> v0.0.6): - Background trimming ([libp2p/go-libp2p-connmgr#43](https://github.com/libp2p/go-libp2p-connmgr/pull/43)) - Implement UpsertTag ([libp2p/go-libp2p-connmgr#38](https://github.com/libp2p/go-libp2p-connmgr/pull/38)) - Add peer protection capability (implementation) ([libp2p/go-libp2p-connmgr#36](https://github.com/libp2p/go-libp2p-connmgr/pull/36)) - github.com/libp2p/go-libp2p-crypto (v0.0.1 -> v0.0.2): - add openssl support ([libp2p/go-libp2p-crypto#61](https://github.com/libp2p/go-libp2p-crypto/pull/61)) - github.com/libp2p/go-libp2p-discovery (v0.0.1 -> v0.0.4): - More consistent use of options ([libp2p/go-libp2p-discovery#25](https://github.com/libp2p/go-libp2p-discovery/pull/25)) - Use 3hrs as routing advertisement ttl ([libp2p/go-libp2p-discovery#23](https://github.com/libp2p/go-libp2p-discovery/pull/23)) - github.com/libp2p/go-libp2p-interface-connmgr (v0.0.1 -> v0.0.5): - Add Close method to the ConnManager interface ([libp2p/go-libp2p-interface-connmgr#18](https://github.com/libp2p/go-libp2p-interface-connmgr/pull/18)) - Add UpsertTag to the interface ([libp2p/go-libp2p-interface-connmgr#17](https://github.com/libp2p/go-libp2p-interface-connmgr/pull/17)) - Fix NullConnMgr to respect ConnManager interface ([libp2p/go-libp2p-interface-connmgr#15](https://github.com/libp2p/go-libp2p-interface-connmgr/pull/15)) - Add peer protection capability ([libp2p/go-libp2p-interface-connmgr#14](https://github.com/libp2p/go-libp2p-interface-connmgr/pull/14)) - github.com/libp2p/go-libp2p-kad-dht (v0.0.7 -> v0.0.13): - fix: reduce memory used by buffered writers ([libp2p/go-libp2p-kad-dht#332](https://github.com/libp2p/go-libp2p-kad-dht/pull/332)) - query: fix a goroutine leak when the routing table is empty ([libp2p/go-libp2p-kad-dht#329](https://github.com/libp2p/go-libp2p-kad-dht/pull/329)) - query: fix error "leak" ([libp2p/go-libp2p-kad-dht#328](https://github.com/libp2p/go-libp2p-kad-dht/pull/328)) - providers: run datastore GC concurrently ([libp2p/go-libp2p-kad-dht#326](https://github.com/libp2p/go-libp2p-kad-dht/pull/326)) - fix(providers): gc ([libp2p/go-libp2p-kad-dht#325](https://github.com/libp2p/go-libp2p-kad-dht/pull/325)) - Remove the old protocol from the defaults ([libp2p/go-libp2p-kad-dht#320](https://github.com/libp2p/go-libp2p-kad-dht/pull/320)) - Fix some provider subsystem performance issues ([libp2p/go-libp2p-kad-dht#319](https://github.com/libp2p/go-libp2p-kad-dht/pull/319)) - github.com/libp2p/go-libp2p-peerstore (v0.0.2 -> v0.0.6): - segment the memory peerstore + granular locks ([libp2p/go-libp2p-peerstore#78](https://github.com/libp2p/go-libp2p-peerstore/pull/78)) - don't delete under the read lock ([libp2p/go-libp2p-peerstore#76](https://github.com/libp2p/go-libp2p-peerstore/pull/76)) - Read/Write locking ([libp2p/go-libp2p-peerstore#74](https://github.com/libp2p/go-libp2p-peerstore/pull/74)) - optimize peerstore memory ([libp2p/go-libp2p-peerstore#71](https://github.com/libp2p/go-libp2p-peerstore/pull/71)) - fix unmarshalling of peer IDs ([libp2p/go-libp2p-peerstore#72](https://github.com/libp2p/go-libp2p-peerstore/pull/72)) - fix error handling in UpdateAddrs: return on error ([libp2p/go-libp2p-peerstore#70](https://github.com/libp2p/go-libp2p-peerstore/pull/70)) - github.com/libp2p/go-libp2p-pubsub (v0.0.1 -> v0.0.3): - rework validator pipeline ([libp2p/go-libp2p-pubsub#176](https://github.com/libp2p/go-libp2p-pubsub/pull/176)) - Test adversarial signing ([libp2p/go-libp2p-pubsub#181](https://github.com/libp2p/go-libp2p-pubsub/pull/181)) - Strict message signing by default ([libp2p/go-libp2p-pubsub#180](https://github.com/libp2p/go-libp2p-pubsub/pull/180)) - github.com/libp2p/go-libp2p-secio (v0.0.1 -> v0.0.3): - fix buffer size check ([libp2p/go-libp2p-secio#44](https://github.com/libp2p/go-libp2p-secio/pull/44)) - github.com/libp2p/go-libp2p-swarm (v0.0.2 -> v0.0.6): - dial: return a nice custom dial error ([libp2p/go-libp2p-swarm#121](https://github.com/libp2p/go-libp2p-swarm/pull/121)) - github.com/libp2p/go-libp2p-tls (null -> v0.0.1): - implement the new handshake ([libp2p/go-libp2p-tls#20](https://github.com/libp2p/go-libp2p-tls/pull/20)) - use a prefix when signing the public key ([libp2p/go-libp2p-tls#26](https://github.com/libp2p/go-libp2p-tls/pull/26)) - use ChaCha if one of the peers doesn't have AES hardware support ([libp2p/go-libp2p-tls#23](https://github.com/libp2p/go-libp2p-tls/pull/23)) - improve peer verification ([libp2p/go-libp2p-tls#17](https://github.com/libp2p/go-libp2p-tls/pull/17)) - add an example (mainly for development) ([libp2p/go-libp2p-tls#14](https://github.com/libp2p/go-libp2p-tls/pull/14)) - github.com/libp2p/go-libp2p-transport-upgrader (v0.0.1 -> v0.0.4): - improve correctness of closing connections on failure ([libp2p/go-libp2p-transport-upgrader#19](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/19)) - github.com/libp2p/go-maddr-filter (v0.0.1 -> v0.0.4): - fix filter listing ([libp2p/go-maddr-filter#13](https://github.com/libp2p/go-maddr-filter/pull/13)) - Reinstate deprecated Remove() method to reverse breakage ([libp2p/go-maddr-filter#12](https://github.com/libp2p/go-maddr-filter/pull/12)) - Implement support for whitelists, default-deny/allow ([libp2p/go-maddr-filter#8](https://github.com/libp2p/go-maddr-filter/pull/8)) - github.com/libp2p/go-mplex (v0.0.1 -> v0.0.4): - disable write coalescing ([libp2p/go-mplex#61](https://github.com/libp2p/go-mplex/pull/61)) - fix SetDeadline error conditions ([libp2p/go-mplex#59](https://github.com/libp2p/go-mplex/pull/59)) - don't use contexts for deadlines ([libp2p/go-mplex#58](https://github.com/libp2p/go-mplex/pull/58)) - don't reset on pathologies, just ignore the data ([libp2p/go-mplex#57](https://github.com/libp2p/go-mplex/pull/57)) - coalesce writes ([libp2p/go-mplex#54](https://github.com/libp2p/go-mplex/pull/54)) - read as much as we can in one go ([libp2p/go-mplex#53](https://github.com/libp2p/go-mplex/pull/53)) - use timeouts when sending messages for stream open, close, and reset. ([libp2p/go-mplex#52](https://github.com/libp2p/go-mplex/pull/52)) - fix: reset a stream even if closed remotely ([libp2p/go-mplex#50](https://github.com/libp2p/go-mplex/pull/50)) - downgrade Error log to Warning ([libp2p/go-mplex#46](https://github.com/libp2p/go-mplex/pull/46)) - Fix race condition by adding a mutex for deadline access ([libp2p/go-mplex#41](https://github.com/libp2p/go-mplex/pull/41)) - github.com/libp2p/go-msgio (v0.0.1 -> v0.0.2): - fix: never claim to read more than read ([libp2p/go-msgio#12](https://github.com/libp2p/go-msgio/pull/12)) - github.com/libp2p/go-ws-transport (v0.0.2 -> v0.0.4): - dep: import go-smux-* into the libp2p org ([libp2p/go-ws-transport#43](https://github.com/libp2p/go-ws-transport/pull/43)) - replace gx instructions with note about gomod ([libp2p/go-ws-transport#42](https://github.com/libp2p/go-ws-transport/pull/42)) ## v0.4.20 2019-04-16 We're happy to release go-ipfs 0.4.20. This release includes some critical performance and stability fixes so all users should upgrade ASAP. This is also the first release to use go modules instead of GX. While GX has been a great way to dogfood an IPFS-based package manager, building and maintaining a custom package manager is a _lot_ of work and we haven't been able to dedicate enough time to bring the user experience of gx to an acceptable level. You can read [#5850](https://github.com/ipfs/go-ipfs/issues/5850) for some discussion on this matter. ### Docker As of this release, it's now much easier to run arbitrary IPFS commands within the docker container: ```bash > docker run --name my-ipfs ipfs/go-ipfs:v0.4.20 config profile apply server # apply the server profile > docker start my-ipfs # start the daemon ``` This release also [reverts](https://github.com/ipfs/go-ipfs/pull/6040) a change that caused some significant trouble in 0.4.19. If you've been running into Docker permission errors in 0.4.19, please upgrade. ### WebUI This release contains a major [WebUI](https://github.com/ipfs-shipyard/ipfs-webui) release with some significant improvements to the file browser and new opt-in, privately hosted, anonymous usage analytics. ### Commands As usual, we've made several changes and improvements to our commands. The most notable changes are listed in this section. #### New: `ipfs version deps` This release includes a new command, `ipfs version deps`, to list all dependencies (with versions) of the current go-ipfs build. This should make it easy to tell exactly how go-ipfs was built when tracking down issues. #### New: `ipfs add URL` The `ipfs add` command has gained support for URLs. This means you can: 1. Add files with `ipfs add URL` instead of downloading the file first. 2. Replace all uses of the `ipfs urlstore` command with a call to `ipfs add --nocopy`. The `ipfs urlstore` command will be deprecated in a future release. #### Changed: `ipfs swarm connect` The `ipfs swarm connect` command has a few new features: It now marks the newly created connection as "important". This should ensure that the connection manager won't come along later and close the connection if it doesn't think it's being used. It can now resolve `/dnsaddr` addresses that _don't_ end in a peer ID. For example, you can now run `ipfs swarm connect /dnsaddr/bootstrap.libp2p.io` to connect to one of the bootstrap peers at random. NOTE: This could connect you to an _arbitrary_ peer as DNS is not secure (by default). Please do not rely on this except for testing or unless you know what you're doing. Finally, `ipfs swarm connect` now returns _all_ errors on failure. This should make it much easier to debug connectivity issues. For example, one might see an error like: ``` Error: connect QmYou failure: dial attempt failed: 6 errors occurred: * --> (/ip4/127.0.0.1/tcp/4001) dial attempt failed: dial tcp4 127.0.0.1:4001: connect: connection refused * --> (/ip6/::1/tcp/4001) dial attempt failed: dial tcp6 [::1]:4001: connect: connection refused * --> (/ip6/2604::1/tcp/4001) dial attempt failed: dial tcp6 [2604::1]:4001: connect: network is unreachable * --> (/ip6/2602::1/tcp/4001) dial attempt failed: dial tcp6 [2602::1]:4001: connect: network is unreachable * --> (/ip4/150.0.1.2/tcp/4001) dial attempt failed: dial tcp4 0.0.0.0:4001->150.0.1.2:4001: i/o timeout * --> (/ip4/200.0.1.2/tcp/4001) dial attempt failed: dial tcp4 0.0.0.0:4001->200.0.1.2:4001: i/o timeout ``` #### Changed: `ipfs bitswap stat` `ipfs bitswap stat` no longer lists bitswap partners unless the `-v` flag is passed. That is, it will now return: ``` > ipfs bitswap stat bitswap status provides buffer: 0 / 256 blocks received: 0 blocks sent: 79 data received: 0 data sent: 672706 dup blocks received: 0 dup data received: 0 B wantlist [0 keys] partners [197] ``` Instead of: ``` > ipfs bitswap stat -v bitswap status provides buffer: 0 / 256 blocks received: 0 blocks sent: 79 data received: 0 data sent: 672706 dup blocks received: 0 dup data received: 0 B wantlist [0 keys] partners [203] QmNQTTTRCDpCYCiiu6TYWCqEa7ShAUo9jrZJvWngfSu1mL QmNWaxbqERvdcgoWpqAhDMrbK2gKi3SMGk3LUEvfcqZcf4 QmNgSVpgZVEd41pBX6DyCaHRof8UmUJLqQ3XH2qNL9xLvN ... omitting 200 lines ... ``` #### Changed: `ipfs repo stat --human` The `--human` flag in the `ipfs repo stat` command now intelligently picks a size unit instead of always using MiB. #### Changed: `ipfs resolve` (`ipfs dns`, `ipfs name resolve`) All of the resolve commands now: 1. Resolve _recursively_ (up to 32 steps) by default to better match user expectations (these commands used to be non-recursive by default). To turn recursion off, pass `-r false`. 2. When resolving non-recursively, these commands no longer fail when partially resolving a name. Instead, they simply return the intermediate result. #### Changed: `ipfs files flush` The `ipfs files flush` command now returns the CID of the flushed file. ### Performance And Reliability This release has the usual collection of performance and reliability improvements. #### Badger Memory Usage Those of you using the badger datastore should notice reduced memory usage in this release due to some upstream changes. Badger still uses significantly more memory than the default datastore configuration but this will hopefully continue to improve. #### Bitswap We fixed some critical CPU utilization regressions in bitswap for this release. If you've been noticing CPU _regressions_ in go-ipfs 0.4.19, especially when running a public gateway, upgrading to 0.4.20 will likely fix them. #### Relays After AutoRelay was introduced in go-ipfs 0.4.19, the number of peers connecting through relays skyrocketed to over 120K concurrent peers. This highlighted some performance issues that we've now fixed in this release. Specifically: * We've significantly reduced the amount of memory allocated per-peer. * We've fixed a bug where relays might, in rare cases, try to actively dial a peer to relay traffic. By default, relays only forward traffic between peers already connected to the relay. * We've fixed quite a number of performance issues that only show up when rapidly forming new connections. This will actually help _all_ nodes but will especially help relays. If you've enabled relay _hop_ (`Swarm.EnableRelayHop`) in go-ipfs 0.4.19 and it hasn't burned down your machine yet, this release should improve things significantly. However, relays are still under heavy load so running an open relay will continue to be resource intensive. We're continuing to investigate this issue and have a few more patches on the way that, unfortunately, won't make it into this release. #### Panics We've fixed two notable panics in this release: * We've fixed a frequent panic in the DHT. * We've fixed an occasional panic in the experimental QUIC transport. ### Content Routing IPFS announces and finds content by sending and retrieving content routing ("provider") records to and from the DHT. Unfortunately, sending out these records can be quite resource intensive. This release has two changes to alleviate this: a reduced number of initial provide workers and a persistent provider queue. We've reduced the number of parallel initial provide workers (workers that send out provider records when content is initially added to go-ipfs) from 512 to 6. Each provide request (currently, due to some issues in our DHT) tries to establish hundreds of connections, significantly impacting the performance of go-ipfs and [crashing some routers](https://github.com/ipfs/go-ipfs/issues/3320). We've introduced a new persistent provider queue for files added via `ipfs add` and `ipfs pin add`. When new directory trees are added to go-ipfs, go-ipfs will add the root/final CID to this queue. Then, in the background, go-ipfs will walk the queue, sequentially sending out provider records for each CID. This ensures that root CIDs are sent out as soon as possible and are sent even when files are added when the go-ipfs daemon isn't running. By example, let's add a directory tree to go-ipfs: ```bash > # We're going to do this in "online" mode first so let's start the daemon. > ipfs daemon & ... Daemon is ready > # Now, we're going to create a directory to add. > mkdir foo > for i in {0..1000}; do echo do echo $i > foo/$i; done > # finally, we're going to add it. > ipfs add -r foo added QmUQcSjQx2bg4cSe2rUZyQi6F8QtJFJb74fWL7D784UWf9 foo/0 ... added QmQac2chFyJ24yfG2Dfuqg1P5gipLcgUDuiuYkQ5ExwGap foo/990 added QmQWwz9haeQ5T2QmQeXzqspKdowzYELShBCLzLJjVa2DuV foo/991 added QmQ5D4MtHUN4LTS4n7mgyHyaUukieMMyCfvnzXQAAbgTJm foo/992 added QmZq4n4KRNq3k1ovzxJ4qdQXZSrarfJjnoLYPR3ztHd7EY foo/993 added QmdtrsuVf8Nf1s1MaSjLAd54iNqrn1KN9VoFNgKGnLgjbt foo/994 added QmbstvU9mnW2hsE94WFmw5WbrXdLTu2Sf9kWWSozrSDscL foo/995 added QmXFd7f35gAnmisjfFmfYKkjA3F3TSpvUYB9SXr6tLsdg8 foo/996 added QmV5BxS1YQ9V227Np2Cq124cRrFDAyBXNMqHHa6kpJ9cr6 foo/997 added QmcXsccUtwKeQ1SuYC3YgyFUeYmAR9CXwGGnT3LPeCg5Tx foo/998 added Qmc4mcQcpaNzyDQxQj5SyxwFg9ZYz5XBEeEZAuH4cQirj9 foo/999 added QmXpXzUhcS9edmFBuVafV5wFXKjfXkCQcjAUZsTs7qFf3G foo ``` In 0.4.19, we would have sent out provider records for files `foo/{0..1000}` _before_ sending out a provider record for `foo`. If you were ask a friend to download /ipfs/QmUQcSjQx2bg4cSe2rUZyQi6F8QtJFJb74fWL7D784UWf9, they would (baring other issues) be able to find it pretty quickly as this is the first CID you'll have announced to the network. However, if you ask your friend to download /ipfs/QmXpXzUhcS9edmFBuVafV5wFXKjfXkCQcjAUZsTs7qFf3G/0, they'll have to wait for you to finish telling the network about every file in `foo` first. In 0.4.20, we _immediately_ tell the network about `QmXpXzUhcS9edmFBuVafV5wFXKjfXkCQcjAUZsTs7qFf3G` (the `foo` directory) as soon as we finish adding the directory to go-ipfs _without_ waiting to finish announcing `foo/{0..1000}`. This is especially important in this release because we've drastically reduced the number of provide workers. The second benefit is that this queue is persistent. That means go-ipfs won't forget to send out this record, even if it was offline when the content was initially added. NOTE: go-ipfs _does_ continuously _re_-send provider records in the background twice a day, it just might be a while before it gets around to sending one out any specific one. ### Bitswap Bitswap now periodically re-sends its wantlist to connected peers. This should help work around some race conditions we've seen in bitswap where one node wants a block but the other doesn't know for some reason. You can track this issue here: https://github.com/ipfs/go-ipfs/issues/5183. ### Improved NAT Traversal While NATs are still p2p enemy #1, this release includes slightly improved support for traversing them. Specifically, this release now: 1. Better detects the "gateway" NAT, even when multiple devices on the network _claim_ to be NATs. 2. Better guesses the external IP address when port mapping, even when the gateway lies. ### Reduced AutoRelay Boot Time The experimental AutoRelay feature can now detect NATs _much_ faster as we've reduced initial NAT detection delay to 15 seconds. There's still room for improvement but this should make nodes that have enabled this feature dialable earlier on start. ### Changelogs - github.com/ipfs/go-ipfs: - gitattributes: avoid normalizing known binary files ([ipfs/go-ipfs#6209](https://github.com/ipfs/go-ipfs/pull/6209)) - gitattributes: default to LF ([ipfs/go-ipfs#6198](https://github.com/ipfs/go-ipfs/pull/6198)) - Fix level db panic ([ipfs/go-ipfs#6186](https://github.com/ipfs/go-ipfs/pull/6186)) - Dockerfile: Remove 2 year old deprecation warning ([ipfs/go-ipfs#6188](https://github.com/ipfs/go-ipfs/pull/6188)) - align output for the command ipfs object stat ([ipfs/go-ipfs#6189](https://github.com/ipfs/go-ipfs/pull/6189)) - provider queue: don't repeatedly retry the same item if we fail ([ipfs/go-ipfs#6187](https://github.com/ipfs/go-ipfs/pull/6187)) - test: remove version/deps from ro commands test ([ipfs/go-ipfs#6185](https://github.com/ipfs/go-ipfs/pull/6185)) - feat: add version deps command [modversion] ([ipfs/go-ipfs#6115](https://github.com/ipfs/go-ipfs/pull/6115)) - readme: update for go modules ([ipfs/go-ipfs#6180](https://github.com/ipfs/go-ipfs/pull/6180)) - Switch to Go 1.12 ([ipfs/go-ipfs#6144](https://github.com/ipfs/go-ipfs/pull/6144)) - ci: avoid interleaving output from different sharness tests ([ipfs/go-ipfs#6175](https://github.com/ipfs/go-ipfs/pull/6175)) - fix two bugs where the repo may not properly be closed ([ipfs/go-ipfs#6176](https://github.com/ipfs/go-ipfs/pull/6176)) - fix error check in swarm connect ([ipfs/go-ipfs#6174](https://github.com/ipfs/go-ipfs/pull/6174)) - feat(coreapi): tag all explicit connect requests in the connection manager ([ipfs/go-ipfs#6171](https://github.com/ipfs/go-ipfs/pull/6171)) - chore: remove CODEOWNERS ([ipfs/go-ipfs#6172](https://github.com/ipfs/go-ipfs/pull/6172)) - feat: update to IPFS Web UI 2.4.4 ([ipfs/go-ipfs#6169](https://github.com/ipfs/go-ipfs/pull/6169)) - fix add error handling ([ipfs/go-ipfs#6156](https://github.com/ipfs/go-ipfs/pull/6156)) - chore: remove waffle ([ipfs/go-ipfs#6157](https://github.com/ipfs/go-ipfs/pull/6157)) - chore: fix a bunch of issues caught by golangci-lint ([ipfs/go-ipfs#6140](https://github.com/ipfs/go-ipfs/pull/6140)) - docs/experimental-features.md: link to ipfs-ds-convert ([ipfs/go-ipfs#6154](https://github.com/ipfs/go-ipfs/pull/6154)) - interrupt: fix send on closed ([ipfs/go-ipfs#6147](https://github.com/ipfs/go-ipfs/pull/6147)) - docs: document Gateway.Writable not Gateway.Writeable ([ipfs/go-ipfs#6151](https://github.com/ipfs/go-ipfs/pull/6151)) - Fuse fixes ([ipfs/go-ipfs#6135](https://github.com/ipfs/go-ipfs/pull/6135)) - Remove duplicate blockstore from the package list ([ipfs/go-ipfs#6138](https://github.com/ipfs/go-ipfs/pull/6138)) - Query for provider head/tail ([ipfs/go-ipfs#6125](https://github.com/ipfs/go-ipfs/pull/6125)) - Remove dead link from ISSUE_TEMPLATE.md ([ipfs/go-ipfs#6128](https://github.com/ipfs/go-ipfs/pull/6128)) - coreapi: remove Unixfs.Wrap ([ipfs/go-ipfs#6123](https://github.com/ipfs/go-ipfs/pull/6123)) - coreapi unixfs: change Wrap logic to make more sense ([ipfs/go-ipfs#6019](https://github.com/ipfs/go-ipfs/pull/6019)) - deps: switch back to jbenet go-is-domain ([ipfs/go-ipfs#6119](https://github.com/ipfs/go-ipfs/pull/6119)) - command repo stat: add human flag tests to t0080-repo.sh ([ipfs/go-ipfs#6116](https://github.com/ipfs/go-ipfs/pull/6116)) - gc: fix a potential deadlock ([ipfs/go-ipfs#6112](https://github.com/ipfs/go-ipfs/pull/6112)) - fix config options in osxfuse error messages ([ipfs/go-ipfs#6105](https://github.com/ipfs/go-ipfs/pull/6105)) - Command repo stat: improve human flag behavior ([ipfs/go-ipfs#6106](https://github.com/ipfs/go-ipfs/pull/6106)) - Provide root node immediately on add and pin add ([ipfs/go-ipfs#6068](https://github.com/ipfs/go-ipfs/pull/6068)) - gomod: Update Dockerfile, remove Dockerfile.fast ([ipfs/go-ipfs#6100](https://github.com/ipfs/go-ipfs/pull/6100)) - Return CID from 'ipfs files flush' ([ipfs/go-ipfs#6102](https://github.com/ipfs/go-ipfs/pull/6102)) - resolve: fix recursion ([ipfs/go-ipfs#6087](https://github.com/ipfs/go-ipfs/pull/6087)) - fix(swarm): add dnsaddr support in swarm connect ([ipfs/go-ipfs#5535](https://github.com/ipfs/go-ipfs/pull/5535)) - make in-memory datastore thread-safe ([ipfs/go-ipfs#6085](https://github.com/ipfs/go-ipfs/pull/6085)) - Update package table to remove broken jenkins links ([ipfs/go-ipfs#6084](https://github.com/ipfs/go-ipfs/pull/6084)) - mk: fix maketarball to work with gomod ([ipfs/go-ipfs#6078](https://github.com/ipfs/go-ipfs/pull/6078)) - fix ls command to use the new coreinterface types ([ipfs/go-ipfs#6051](https://github.com/ipfs/go-ipfs/pull/6051)) - mk: remove install_unsupported, leave a note ([ipfs/go-ipfs#6063](https://github.com/ipfs/go-ipfs/pull/6063)) - mk: change git-hash command to include information about modifications ([ipfs/go-ipfs#6060](https://github.com/ipfs/go-ipfs/pull/6060)) - mk: fix make install by not setting GOBIN ([ipfs/go-ipfs#6059](https://github.com/ipfs/go-ipfs/pull/6059)) - go: require Golang 1.11.4 ([ipfs/go-ipfs#6057](https://github.com/ipfs/go-ipfs/pull/6057)) - yamux: increase yamux window size to 8MiB. ([ipfs/go-ipfs#6049](https://github.com/ipfs/go-ipfs/pull/6049)) - Introduce go modules [yey] ([ipfs/go-ipfs#6038](https://github.com/ipfs/go-ipfs/pull/6038)) - cleanup daemon online logic ([ipfs/go-ipfs#6050](https://github.com/ipfs/go-ipfs/pull/6050)) - ci: test on 32bit os ([ipfs/go-ipfs#5429](https://github.com/ipfs/go-ipfs/pull/5429)) - feat/cmds: hide peers info default in bitswap stat ([ipfs/go-ipfs#5820](https://github.com/ipfs/go-ipfs/pull/5820)) - Improve CLI help pages ([ipfs/go-ipfs#6013](https://github.com/ipfs/go-ipfs/pull/6013)) - Close #6044 ([ipfs/go-ipfs#6045](https://github.com/ipfs/go-ipfs/pull/6045)) - commands(dht): return final error ([ipfs/go-ipfs#6034](https://github.com/ipfs/go-ipfs/pull/6034)) - Revert "Really run as non-root user in docker container" ([ipfs/go-ipfs#6040](https://github.com/ipfs/go-ipfs/pull/6040)) - github.com/ipfs/go-bitswap: - feat(messagequeue): rebroadcast wantlist ([ipfs/go-bitswap#106](https://github.com/ipfs/go-bitswap/pull/106)) - reduce provide workers to 6 ([ipfs/go-bitswap#93](https://github.com/ipfs/go-bitswap/pull/93)) - Reduce memory allocation ([ipfs/go-bitswap#103](https://github.com/ipfs/go-bitswap/pull/103)) - refactor(messagequeue): remove dead code ([ipfs/go-bitswap#98](https://github.com/ipfs/go-bitswap/pull/98)) - fix: limit use of custom context type ([ipfs/go-bitswap#89](https://github.com/ipfs/go-bitswap/pull/89)) - fix: remove non-error log message ([ipfs/go-bitswap#91](https://github.com/ipfs/go-bitswap/pull/91)) - fix(messagequeue): Remove second run loop ([ipfs/go-bitswap#94](https://github.com/ipfs/go-bitswap/pull/94)) - github.com/ipfs/go-blockservice: - Revert "Remove verifcid as it is handled in go-cid" ([ipfs/go-blockservice#25](https://github.com/ipfs/go-blockservice/pull/25)) - Remove verifcid as it is handled in go-cid ([ipfs/go-blockservice#23](https://github.com/ipfs/go-blockservice/pull/23)) - github.com/ipfs/go-datastore: - cleanup and optimize naive query filters ([ipfs/go-datastore#125](https://github.com/ipfs/go-datastore/pull/125)) - Fix – sorted limited offset mount queries ([ipfs/go-datastore#124](https://github.com/ipfs/go-datastore/pull/124)) - Fix function comments based on best practices from Effective Go ([ipfs/go-datastore#122](https://github.com/ipfs/go-datastore/pull/122)) - remove ThreadSafeDatastore ([ipfs/go-datastore#120](https://github.com/ipfs/go-datastore/pull/120)) - Splinter TTLDatastore interface into TTL + Datastore ([ipfs/go-datastore#118](https://github.com/ipfs/go-datastore/pull/118)) - github.com/ipfs/go-ds-badger: - tweak the default options ([ipfs/go-ds-badger#52](https://github.com/ipfs/go-ds-badger/pull/52)) - remove thread-safe assertion ([ipfs/go-ds-badger#55](https://github.com/ipfs/go-ds-badger/pull/55)) - make memory-safe against concurrent closure/operations ([ipfs/go-ds-badger#53](https://github.com/ipfs/go-ds-badger/pull/53)) - make badger use our logging framework ([ipfs/go-ds-badger#50](https://github.com/ipfs/go-ds-badger/pull/50)) - github.com/ipfs/go-ds-flatfs: - remove thread-safe assertion ([ipfs/go-ds-flatfs#53](https://github.com/ipfs/go-ds-flatfs/pull/53)) - github.com/ipfs/go-ds-leveldb: - Fast reverse query ([ipfs/go-ds-leveldb#28](https://github.com/ipfs/go-ds-leveldb/pull/28)) - remove thread-safe assertion ([ipfs/go-ds-leveldb#27](https://github.com/ipfs/go-ds-leveldb/pull/27)) - github.com/ipfs/go-ipfs-cmdkit: - Extract files package ([ipfs/go-ipfs-cmdkit#31](https://github.com/ipfs/go-ipfs-cmdkit/pull/31)) - github.com/ipfs/go-ipfs-cmds: - sync: add yet another sync error ([ipfs/go-ipfs-cmds#161](https://github.com/ipfs/go-ipfs-cmds/pull/161)) - Removed broken link from readme ([ipfs/go-ipfs-cmds#159](https://github.com/ipfs/go-ipfs-cmds/pull/159)) - Fix broken link in readme ([ipfs/go-ipfs-cmds#160](https://github.com/ipfs/go-ipfs-cmds/pull/160)) - set WebFile fpath to URL base ([ipfs/go-ipfs-cmds#158](https://github.com/ipfs/go-ipfs-cmds/pull/158)) - Handle stdin name in cli/parse ([ipfs/go-ipfs-cmds#157](https://github.com/ipfs/go-ipfs-cmds/pull/157)) - support url paths as files.WebFile ([ipfs/go-ipfs-cmds#154](https://github.com/ipfs/go-ipfs-cmds/pull/154)) - typed encoder: improve pointer reflection ([ipfs/go-ipfs-cmds#155](https://github.com/ipfs/go-ipfs-cmds/pull/155)) - cli: don't sync output to NUL on Windows ([ipfs/go-ipfs-cmds#153](https://github.com/ipfs/go-ipfs-cmds/pull/153)) - github.com/ipfs/go-ipfs-files: - return url as AbsPath from WebFile to implement FileInfo ([ipfs/go-ipfs-files#13](https://github.com/ipfs/go-ipfs-files/pull/13)) - fix the content disposition header ([ipfs/go-ipfs-files#14](https://github.com/ipfs/go-ipfs-files/pull/14)) - go format ([ipfs/go-ipfs-files#15](https://github.com/ipfs/go-ipfs-files/pull/15)) - simplify content type checking ([ipfs/go-ipfs-files#9](https://github.com/ipfs/go-ipfs-files/pull/9)) - remove extra webfile test code ([ipfs/go-ipfs-files#12](https://github.com/ipfs/go-ipfs-files/pull/12)) - github.com/ipfs/go-merkledag: - add function to marshal raw nodes to json ([ipfs/go-merkledag#36](https://github.com/ipfs/go-merkledag/pull/36)) - fix some performance regressions when reading protobuf nodes ([ipfs/go-merkledag#34](https://github.com/ipfs/go-merkledag/pull/34)) - github.com/ipfs/go-metrics-interface: - update the counter interface to match Prometheus ([ipfs/go-metrics-interface#2](https://github.com/ipfs/go-metrics-interface/pull/2)) - github.com/ipfs/go-mfs: - Return node from FlushPath ([ipfs/go-mfs#72](https://github.com/ipfs/go-mfs/pull/72)) - Wire up context to FlushPath ([ipfs/go-mfs#70](https://github.com/ipfs/go-mfs/pull/70)) - github.com/ipfs/interface-go-ipfs-core: - don't close the top-level addr ([ipfs/interface-go-ipfs-core#25](https://github.com/ipfs/interface-go-ipfs-core/pull/25)) - fix a bunch of small test "bugs" ([ipfs/interface-go-ipfs-core#24](https://github.com/ipfs/interface-go-ipfs-core/pull/24)) - remove Wrap ([ipfs/interface-go-ipfs-core#21](https://github.com/ipfs/interface-go-ipfs-core/pull/21)) - Unixfs.Wrap Fixes ([ipfs/interface-go-ipfs-core#10](https://github.com/ipfs/interface-go-ipfs-core/pull/10)) - tweak the Ls interface ([ipfs/interface-go-ipfs-core#14](https://github.com/ipfs/interface-go-ipfs-core/pull/14)) - github.com/libp2p/go-buffer-pool: - Enable tests ([libp2p/go-buffer-pool#6](https://github.com/libp2p/go-buffer-pool/pull/6)) - github.com/libp2p/go-flow-metrics: - Just repair spelling mistake ([libp2p/go-flow-metrics#3](https://github.com/libp2p/go-flow-metrics/pull/3)) - github.com/libp2p/go-libp2p: - Deprecate gx in readme & link to workspace repo ([libp2p/go-libp2p#591](https://github.com/libp2p/go-libp2p/pull/591)) - Respect nodial option in routed host ([libp2p/go-libp2p#590](https://github.com/libp2p/go-libp2p/pull/590)) - fix panic in observed address activation check ([libp2p/go-libp2p#586](https://github.com/libp2p/go-libp2p/pull/586)) - Improve observed address handling ([libp2p/go-libp2p#585](https://github.com/libp2p/go-libp2p/pull/585)) - identify: avoid parsing/printing multiaddrs ([libp2p/go-libp2p#583](https://github.com/libp2p/go-libp2p/pull/583)) - move things outside of the lock in obsaddr ([libp2p/go-libp2p#582](https://github.com/libp2p/go-libp2p/pull/582)) - identify: be more careful about the addresses we store ([libp2p/go-libp2p#577](https://github.com/libp2p/go-libp2p/pull/577)) - relay: turn autorelay into a service and always filter out relay addresses ([libp2p/go-libp2p#578](https://github.com/libp2p/go-libp2p/pull/578)) - chore: fail in the libp2p constructor if we fail to store the key ([libp2p/go-libp2p#576](https://github.com/libp2p/go-libp2p/pull/576)) - Fix broken link in README.md ([libp2p/go-libp2p#580](https://github.com/libp2p/go-libp2p/pull/580)) - Link to docs & discuss in readme ([libp2p/go-libp2p#571](https://github.com/libp2p/go-libp2p/pull/571)) - Reduce autorelay boot delay and correctly handle private->public transition ([libp2p/go-libp2p#570](https://github.com/libp2p/go-libp2p/pull/570)) - reduce nat error level ([libp2p/go-libp2p#568](https://github.com/libp2p/go-libp2p/pull/568)) - relay: simplify declaration of multiaddr var ([libp2p/go-libp2p#563](https://github.com/libp2p/go-libp2p/pull/563)) - Fix UDP listen on a Unspecified Address and Dial from the Unspecified Address ([libp2p/go-libp2p#561](https://github.com/libp2p/go-libp2p/pull/561)) - Remove jenkins column from package table ([libp2p/go-libp2p#562](https://github.com/libp2p/go-libp2p/pull/562)) - Fix typos in p2p/net/README.md ([libp2p/go-libp2p#555](https://github.com/libp2p/go-libp2p/pull/555)) - better nat mapping ([libp2p/go-libp2p#549](https://github.com/libp2p/go-libp2p/pull/549)) - github.com/libp2p/go-libp2p-autonat: - fully close the autonat client stream ([libp2p/go-libp2p-autonat#21](https://github.com/libp2p/go-libp2p-autonat/pull/21)) - parallelize dialbacks ([libp2p/go-libp2p-autonat#20](https://github.com/libp2p/go-libp2p-autonat/pull/20)) - Pacify the race detector ([libp2p/go-libp2p-autonat#17](https://github.com/libp2p/go-libp2p-autonat/pull/17)) - github.com/libp2p/go-libp2p-autonat-svc: - full close the autonat stream ([libp2p/go-libp2p-autonat-svc#20](https://github.com/libp2p/go-libp2p-autonat-svc/pull/20)) - reduce dialback timeout to 15s ([libp2p/go-libp2p-autonat-svc#17](https://github.com/libp2p/go-libp2p-autonat-svc/pull/17)) - github.com/libp2p/go-libp2p-circuit: - use buffer pool in newDelimitedReader ([libp2p/go-libp2p-circuit#71](https://github.com/libp2p/go-libp2p-circuit/pull/71)) - Use NoDial option when opening hop streams for non-active relays ([libp2p/go-libp2p-circuit#70](https://github.com/libp2p/go-libp2p-circuit/pull/70)) - use io.CopyBuffer with explicitly allocated buffers ([libp2p/go-libp2p-circuit#69](https://github.com/libp2p/go-libp2p-circuit/pull/69)) - docs and nits ([libp2p/go-libp2p-circuit#66](https://github.com/libp2p/go-libp2p-circuit/pull/66)) - github.com/libp2p/go-libp2p-kad-dht: - dialQueue: start the control loop later ([libp2p/go-libp2p-kad-dht#312](https://github.com/libp2p/go-libp2p-kad-dht/pull/312)) - make it work in wasm ([libp2p/go-libp2p-kad-dht#310](https://github.com/libp2p/go-libp2p-kad-dht/pull/310)) - Revert "GoModules: Checksum mismatch:" ([libp2p/go-libp2p-kad-dht#309](https://github.com/libp2p/go-libp2p-kad-dht/pull/309)) - defer dialqueue action until initial peers have been added ([libp2p/go-libp2p-kad-dht#301](https://github.com/libp2p/go-libp2p-kad-dht/pull/301)) - github.com/libp2p/go-libp2p-nat: - switch to libp2p's go-nat fork ([libp2p/go-libp2p-nat#16](https://github.com/libp2p/go-libp2p-nat/pull/16)) - remove all uses of multiaddrs ([libp2p/go-libp2p-nat#14](https://github.com/libp2p/go-libp2p-nat/pull/14)) - github.com/libp2p/go-libp2p-net: - fix WithNoDial to return the context ([libp2p/go-libp2p-net#43](https://github.com/libp2p/go-libp2p-net/pull/43)) - NoDial context option ([libp2p/go-libp2p-net#42](https://github.com/libp2p/go-libp2p-net/pull/42)) - github.com/libp2p/go-libp2p-peer: - Let ID implement encoding.Binary[Un]Marshaler and encoding.Text[Un]Marshaler ([libp2p/go-libp2p-peer#44](https://github.com/libp2p/go-libp2p-peer/pull/44)) - github.com/libp2p/go-libp2p-peerstore: - keep temp addresses for 2 minutes ([libp2p/go-libp2p-peerstore#67](https://github.com/libp2p/go-libp2p-peerstore/pull/67)) - migrate to multiformats/go-base32 ([libp2p/go-libp2p-peerstore#61](https://github.com/libp2p/go-libp2p-peerstore/pull/61)) - github.com/libp2p/go-libp2p-protocol: - update readme ([libp2p/go-libp2p-protocol#6](https://github.com/libp2p/go-libp2p-protocol/pull/6)) - Enable standard Travis CI tests. ([libp2p/go-libp2p-protocol#5](https://github.com/libp2p/go-libp2p-protocol/pull/5)) - Fix go get address. ([libp2p/go-libp2p-protocol#4](https://github.com/libp2p/go-libp2p-protocol/pull/4)) - Add MIT license ([libp2p/go-libp2p-protocol#3](https://github.com/libp2p/go-libp2p-protocol/pull/3)) - Standardized Readme ([libp2p/go-libp2p-protocol#2](https://github.com/libp2p/go-libp2p-protocol/pull/2)) - github.com/libp2p/go-libp2p-pubsub-router: - gx publish 0.5.17 ([libp2p/go-libp2p-pubsub-router#26](https://github.com/libp2p/go-libp2p-pubsub-router/pull/26)) - github.com/libp2p/go-libp2p-quic-transport: - update quic-go to v0.11.0 ([libp2p/go-libp2p-quic-transport#54](https://github.com/libp2p/go-libp2p-quic-transport/pull/54)) - github.com/libp2p/go-libp2p-routing-helpers: - fix(put): fail if any router fails ([libp2p/go-libp2p-routing-helpers#19](https://github.com/libp2p/go-libp2p-routing-helpers/pull/19)) - github.com/libp2p/go-libp2p-swarm: - Add context option to disable dialing when opening a new stream ([libp2p/go-libp2p-swarm#116](https://github.com/libp2p/go-libp2p-swarm/pull/116)) - return all dial errors if dial has failed ([libp2p/go-libp2p-swarm#115](https://github.com/libp2p/go-libp2p-swarm/pull/115)) - Differentiate no addresses error from no good addresses ([libp2p/go-libp2p-swarm#113](https://github.com/libp2p/go-libp2p-swarm/pull/113)) - github.com/libp2p/go-libp2p-transport: - tests: constrain concurrency with race detector. ([libp2p/go-libp2p-transport#47](https://github.com/libp2p/go-libp2p-transport/pull/47)) - pick test timeout from env var if available. ([libp2p/go-libp2p-transport#46](https://github.com/libp2p/go-libp2p-transport/pull/46)) - increase test timeout. ([libp2p/go-libp2p-transport#45](https://github.com/libp2p/go-libp2p-transport/pull/45)) - github.com/libp2p/go-msgio: - Improve test coverage ([libp2p/go-msgio#10](https://github.com/libp2p/go-msgio/pull/10)) - github.com/libp2p/go-reuseport: - fix: add wasm build tag to wasm module ([libp2p/go-reuseport#70](https://github.com/libp2p/go-reuseport/pull/70)) - github.com/libp2p/go-reuseport-transport: - don't set linger to 0 ([libp2p/go-reuseport-transport#14](https://github.com/libp2p/go-reuseport-transport/pull/14)) - github.com/libp2p/go-tcp-transport: - set linger to 0 for both inbound and outbound connections ([libp2p/go-tcp-transport#36](https://github.com/libp2p/go-tcp-transport/pull/36)) - github.com/libp2p/go-ws-transport: - modernize request handling ([libp2p/go-ws-transport#41](https://github.com/libp2p/go-ws-transport/pull/41)) ## v0.4.19 2019-03-01 We're happy to announce go 0.4.19. This release contains a bunch of important fixes and a slew of new and improved features. Get pumped and upgrade ASAP to benefit from all the new goodies! 🎁 ### Features #### 🔌 Initializing With Random Ports Go-ipfs can now be configured to listen on a random but _stable_ port (across restarts) using the new `randomports` configuration profile. This should be helpful when testing and/or running multiple go-ipfs instances on a single machine. To initialize a go-ipfs instance with a randomly chosen port, run: ```bash > ipfs init --profile=randomports ``` #### 👂 Gateway Directory Listing IPNS (and/or DNSLink) directory listings on the gateway, e.g. https://ipfs.io/ipns/dist.ipfs.tech/go-ipfs/, will now display the _ipfs_ hash of the current directory. This way users can more easily create permanent links to otherwise mutable data. #### 📡 AutoRelay and AutoNAT This release introduces two new experimental features (courtesy of libp2p): AutoRelay and AutoNAT. AutoRelay is a new service that automatically chooses a public relay when it detects that the go-ipfs node is behind a NAT. While relaying connections through a third-party node isn't the most efficient way to route around NATs, it's a reliable fallback. To enable AutoRelay, set the `Swarm.EnableAutoRelay` option in the config. AutoNAT is the service AutoRelay uses to detect if the node is behind a NAT. You don't have to set any special config flags to enable it. In this same config section, you may also notice options like `EnableRelayHop`, `EnableAutoNATService`, etc. You _do not_ need to enable these: * `EnableRelayHop` -- Allow _other_ nodes to use _your_ node as a relay (disabled by default). * `EnableAutoNATService` -- Help _other_ nodes detect if they're behind a NAT (disabled by default). #### 📵 Offline Operation There are two new "offline" features in this release: a global `--offline` flag and an option to configure the gateway to not fetch files. Most go-ipfs commands now support the `--offline` flag. This causes IPFS to avoid network operations when performing the requested operation. If you've ever used the `--local` flag, the `--offline` flag is the (almost) universally supported replacement. For example: * If the daemon is started with `ipfs daemon --offline`, it won't even _connect_ to the network. (note: this feature isn't new, just an example). * `ipfs add --offline some_file` won't send out provider records. * `ipfs cat --offline Qm...` won't fetch any blocks from the network. * `ipfs block stat --offline Qm...` is a great way to tell if a block is locally available. Note: It doesn't _yet_ work with the `refs`, `urlstore`, or `tar` commands ([#6002](https://github.com/ipfs/go-ipfs/issues/6002)). On to the gateway, there's a new `Gateway.NoFetch` option to configure the gateway to only serve locally present files. This makes it possible to run an IPFS node as a gateway to serve content of _your_ choosing without acting like a public proxy. 🤫 #### 📍 Adding And Pinning Content There's a new `--pin` flag for both `ipfs block put` and `ipfs urlstore add` to match the `--pin` flag in `ipfs add`. This allows one to atomically add and pin content with these APIs. **NOTE 1:** For `ipfs urlstore add`, `--pin` has been enabled _by default_ to match the behavior in `ipfs add`. However, `ipfs block put` _does not_ pin by default to match the _current_ behavior. **NOTE 2:** If you had previously used the urlstore and _weren't_ explicitly pinning content after adding it, it isn't pinned and running the garbage collector will delete it. While technically documented in the `ipfs urlstore add` helptext, this behavior was non-obvious and bears mentioning. #### 🗂 File Listing The `ipfs ls` command has two significant changes this release: it reports _file_ sizes instead of _dag_ sizes and has gained a new `--stream` flag. First up, `ipfs ls` now reports _file_ sizes instead of _dag_ sizes. Previously, for historical reasons, `ipfs ls` would report the size of a file/directory as seen by IPFS _including_ all the filesystem datastructures and metadata. However, this meant that `ls -l` and `ipfs ls` would print _different_ sizes: ```bash > ipfs ls /ipfs/QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv QmZTR5bcpQD7cFgTorqxZDYaew1Wqgfbd2ud9QqGPAkK2V 1688 about QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y 200 contact QmY5heUM5qgRubMDD1og9fhCPA6QdkMp3QCwd4s7gJsyE7 322 help QmejvEPop4D7YUadeGqYWmZxHhLc4JBUCzJJHWMzdcMe2y 12 ping QmXgqKTbzdh83pQtKFb19SpMCpDDcKR2ujqk3pKph9aCNF 1692 quick-start QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB 1102 readme QmQ5vhrL7uv6tuoN9KeVBwd4PwfQkXdVVmDLUZuTNxqgvm 1173 security-notes > ipfs get /ipfs/QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv Saving file(s) to QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv 6.39 KiB / 6.39 KiB [================================] 100.00% 0s > ls -l QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv total 28 -rw------- 1 user group 1677 Feb 14 17:03 about -rw------- 1 user group 189 Feb 14 17:03 contact -rw------- 1 user group 311 Feb 14 17:03 help -rw------- 1 user group 4 Feb 14 17:03 ping -rw------- 1 user group 1681 Feb 14 17:03 quick-start -rw------- 1 user group 1091 Feb 14 17:03 readme -rw------- 1 user group 1162 Feb 14 17:03 security-notes ``` This is now no longer the case. `ipfs ls` and `ls -l` now return the _same_ sizes. 🙌 Second up, `ipfs ls` now has a new `--stream` flag. In IPFS, very large directories (e.g., Wikipedia) are split up into multiple chunks (shards) as there are too many entries to fit in a single block. Unfortunately, `ipfs ls` buffers the _entire_ file list in memory and then sorts it. This means that `ipfs ls /ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco/wiki` (Wikipedia) will take a _very_ long time to return anything (it'll also use quite a bit of memory). However, the new `--stream` flag makes it possible to stream a directory listing as new chunks are fetched from the network. To test this, you can run `ipfs ls --stream --size=false --resolve-type=false /ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco/wiki`. You probably won't want to wait for that command to finish, Wikipedia has a _lot_ of entries. 😉 #### 🔁 HTTP Proxy This release sees a new (experimental) feature contributed by our friends at [Peergos](https://peergos.org): HTTP proxy over libp2p. When enabled, the local gateway can act as an HTTP proxy and forward HTTP requests to libp2p peers. When combined with the `ipfs p2p` command, users can use this to expose HTTP services to other go-ipfs nodes via their gateways. For details, check out the [documentation](https://github.com/ipfs/go-ipfs/blob/master/docs/experimental-features.md#p2p-http-proxy). ### Performance And Reliability This release introduces quite a few performance/reliability improvements and, as usual, fixes several memory leaks. Below is a non-exhaustive list of noticeable changes. #### 📞 DHT This release includes an important DHT fix that should significantly: 1. Reduce dialing. 2. Speed up DHT queries. 3. Improve performance of the gateways. Basically, in the worst case, a DHT query would turn into a random walk of the entire IPFS network. Yikes! Relevant PR: https://github.com/libp2p/go-libp2p-kad-dht/pull/237 #### 🕸 Bitswap Bitswap sessions have improved and are now used for _all_ requests. Sessions allow us to group related content and ask peers most likely to _have_ the content instead of broadcasting the request to all connected peers. This gives us two significant benefits: 1. Less wasted upload bandwidth. Instead of broadcasting which blocks we want to everyone, we can ask fewer peers thus reducing the number of requests we send out. 2. Less wasted download bandwidth. Because we _know_ which peers likely have content, we can ask an individual peer for a block and expect to get an answer. In the past, we'd ask every peer at the same time to optimize for latency at the expense of bandwidth (getting the same block from multiple peers). We had to do this because we had to assume that _most_ peers didn't have the requested block. #### ‼️ Pubsub This release includes some significant reliability improvements in pubsub subscription handling. If you've previously had issues with connected pubsub peers _not_ seeing each-other's messages, please upgrade ASAP. #### ♻️ Reuseport In this release, we've rewritten our previously error-prone `go-reuseport` library to _not_ duplicate a significant portion of Go's low-level networking code. This was made possible by Go's new `Control` [`net.Dialer`](https://golang.org/pkg/net/#Dialer) option. In the past, our first suggestion to anyone experiencing weird resource or connectivity issues was to disable `REUSEPORT` (set `IPFS_REUSEPORT` to false). This should no longer be necessary. #### 🐺 Badger Datastore [Badger has reached 1.0][badger-release]. This release brings an audit and numerous reliability fixes. We are now reasonably confident that badger will become the default datastore in a future release. 👍 [badger-release]: https://blog.dgraph.io/post/releasing-v1.0/ This release also adds a new `Truncate` configuration option for the badger datastore (enabled by default for new IPFS nodes). When enabled, badger will _delete_ any un-synced data on start instead of simply refusing to start. This should be safe on all filesystems where the `sync` operation is safe and removes the need for manual intervention when restarting an IPFS node after a crash. Assuming you initialized your badger repo with `ipfs init --profile=badgerds`, you can enable truncate on an existing repo by running: `ipfs config --json "Datastore.Spec.child.truncate" true`. ### Refactors and Endeavors #### 🕹 Commands Library The legacy commands library shim has now been completely removed. This won't mean much for many users but the go-ipfs team is happy to have this behind them. #### 🌐 Base32 CIDs This release can now encode CIDs in responses in bases other than base58. This is primarily useful for web-browser integration as it allows us to (a) encode CIDs in a lower-case base (e.g., base32) and then use them in the _origin_ part of URLs. The take away is: this release brings us a step closer to better browser integration. Specifically, this release adds two flags: 1. `--cid-base`: When specified, the IPFS CLI will encode all CIDv1 CIDs using the requested base. 2. `--upgrade-cidv0-in-output`: When specified, the IPFS CLI will _upgrade_ CIDv0 CIDs to CIDv1 CIDs when returning them to the user. This upgrade is necessary because CIDv0 doesn't support multibase however, it's off by default as it changes the _binary_ representation of the CIDs (which could have unintended consequences). #### 🎛 CoreAPI The work on the CoreAPI refactor ([ipfs/go-ipfs#4498][]) has progressed leaps and bounds this release. The CoreAPI is a comprehensive programmatic interface designed to allow go-ipfs be used as a daemon or a library interchangeably. As of this release, go-ipfs now has: * External interface definitions in [ipfs/interface-go-ipfs-core][]. * A work-in-progress implementation ([ipfs/go-ipfs-http-client][]) of these interfaces that uses the IPFS HTTP API. This will replace the ([ipfs/go-ipfs-api][]) library. * A new plugin type ["Daemon"][daemon-plugin]. Daemon plugins are started and stopped along with the go-ipfs daemon and are instantiated with a copy of the CoreAPI. This allows them to control and extend the go-ipfs daemon from within the daemon itself. The next steps are: 1. Finishing the remaining API surface area. At the moment, the two key missing parts are: 1. Config manipulation. 2. The `ipfs files` API. 1. Finalizing the [ipfs/go-ipfs-http-client][] implementation. 2. Creating a simple way to construct and initialize a go-ipfs node when using go-ipfs as a library. [ipfs/go-ipfs#4498]: https://github.com/ipfs/go-ipfs/issues/4498 [ipfs/interface-go-ipfs-core]: https://github.com/ipfs/interface-go-ipfs-core [ipfs/go-ipfs-http-client]: https://github.com/ipfs/go-ipfs-http-client [ipfs/go-ipfs-api]: https://github.com/ipfs/go-ipfs-http-client [daemon-plugin]: https://github.com/ipfs/go-ipfs/blob/master/docs/plugins.md#daemon ### Changelogs - github.com/ipfs/go-ipfs: - fix: show interactive output from install.sh ([ipfs/go-ipfs#6024](https://github.com/ipfs/go-ipfs/pull/6024)) - fix: return the shortest, completely resolved path in the resolve command ([ipfs/go-ipfs#5704](https://github.com/ipfs/go-ipfs/pull/5704)) - fix a few interop test issues ([ipfs/go-ipfs#6004](https://github.com/ipfs/go-ipfs/pull/6004)) - fix HAMT bookmark ln ([ipfs/go-ipfs#6005](https://github.com/ipfs/go-ipfs/pull/6005)) - docs: document Gateway.NoFetch ([ipfs/go-ipfs#5999](https://github.com/ipfs/go-ipfs/pull/5999)) - Improve "name publish" ttl option documentation ([ipfs/go-ipfs#5979](https://github.com/ipfs/go-ipfs/pull/5979)) - fix(cmd/mv): dst filename error ([ipfs/go-ipfs#5964](https://github.com/ipfs/go-ipfs/pull/5964)) - coreapi: extract interface ([ipfs/go-ipfs#5978](https://github.com/ipfs/go-ipfs/pull/5978)) - coreapi: cleanup non-gx references ([ipfs/go-ipfs#5976](https://github.com/ipfs/go-ipfs/pull/5976)) - coreapi: fix seek test on http impl ([ipfs/go-ipfs#5971](https://github.com/ipfs/go-ipfs/pull/5971)) - block put --pin ([ipfs/go-ipfs#5969](https://github.com/ipfs/go-ipfs/pull/5969)) - Port `ipfs ls` to CoreAPI ([ipfs/go-ipfs#5962](https://github.com/ipfs/go-ipfs/pull/5962)) - docs: duplicate default helptext in `name publish` ([ipfs/go-ipfs#5960](https://github.com/ipfs/go-ipfs/pull/5960)) - plugin: add a daemon plugin with access to the CoreAPI ([ipfs/go-ipfs#5955](https://github.com/ipfs/go-ipfs/pull/5955)) - coreapi: add some seeker tests ([ipfs/go-ipfs#5934](https://github.com/ipfs/go-ipfs/pull/5934)) - Refactor ipfs get to use CoreAPI ([ipfs/go-ipfs#5943](https://github.com/ipfs/go-ipfs/pull/5943)) - refact(cmd/init): change string option to const ([ipfs/go-ipfs#5949](https://github.com/ipfs/go-ipfs/pull/5949)) - cmds/pin: use coreapi/pin ([ipfs/go-ipfs#5843](https://github.com/ipfs/go-ipfs/pull/5843)) - Only perform DNSLink lookups on fully qualified domain names (FQDN) ([ipfs/go-ipfs#5950](https://github.com/ipfs/go-ipfs/pull/5950)) - Fix DontCheckOSXFUSE config command example ([ipfs/go-ipfs#5951](https://github.com/ipfs/go-ipfs/pull/5951)) - refact(cmd/config): change string option to const ([ipfs/go-ipfs#5948](https://github.com/ipfs/go-ipfs/pull/5948)) - clarification the document of --resolve flag in name.publish ([ipfs/go-ipfs#5651](https://github.com/ipfs/go-ipfs/pull/5651)) - Drop some coreunix code ([ipfs/go-ipfs#5938](https://github.com/ipfs/go-ipfs/pull/5938)) - commands: fix verbose flag ([ipfs/go-ipfs#5940](https://github.com/ipfs/go-ipfs/pull/5940)) - Fixes #4558 ([ipfs/go-ipfs#5937](https://github.com/ipfs/go-ipfs/pull/5937)) - Port dag commansds to CoreAPI ([ipfs/go-ipfs#5939](https://github.com/ipfs/go-ipfs/pull/5939)) - mfs: make sure to flush after mv and chcid ([ipfs/go-ipfs#5936](https://github.com/ipfs/go-ipfs/pull/5936)) - docs/code-flow : Add code flow documentation for add cmd. ([ipfs/go-ipfs#5864](https://github.com/ipfs/go-ipfs/pull/5864)) - coreapi: few more error check fixes ([ipfs/go-ipfs#5935](https://github.com/ipfs/go-ipfs/pull/5935)) - Fixed and cleaned up TestIpfsStressRead ([ipfs/go-ipfs#5920](https://github.com/ipfs/go-ipfs/pull/5920)) - Clarify that chunker sizes are in bytes ([ipfs/go-ipfs#5923](https://github.com/ipfs/go-ipfs/pull/5923)) - refact(cmd/patch): change string to const ([ipfs/go-ipfs#5931](https://github.com/ipfs/go-ipfs/pull/5931)) - refact(cmd/object): change option string to const ([ipfs/go-ipfs#5932](https://github.com/ipfs/go-ipfs/pull/5932)) - coreapi: replace coreiface.DagAPI with ipld.DAGService ([ipfs/go-ipfs#5922](https://github.com/ipfs/go-ipfs/pull/5922)) - Add global option to specify the multibase encoding (server side) ([ipfs/go-ipfs#5789](https://github.com/ipfs/go-ipfs/pull/5789)) - coreapi: Adjust some tests for go-ipfs-http-api ([ipfs/go-ipfs#5926](https://github.com/ipfs/go-ipfs/pull/5926)) - chore: update to Web UI v2.3.3 ([ipfs/go-ipfs#5928](https://github.com/ipfs/go-ipfs/pull/5928)) - ls: Report real file size ([ipfs/go-ipfs#5906](https://github.com/ipfs/go-ipfs/pull/5906)) - Improve the Filestore document ([ipfs/go-ipfs#5927](https://github.com/ipfs/go-ipfs/pull/5927)) - [CORS] Bubble go-ipfs-cmds 2.0.10 - Updates CORS library ([ipfs/go-ipfs#5919](https://github.com/ipfs/go-ipfs/pull/5919)) - reduce verbosity of daemon start ([ipfs/go-ipfs#5904](https://github.com/ipfs/go-ipfs/pull/5904)) - feat: update to Web UI v2.3.2 ([ipfs/go-ipfs#5899](https://github.com/ipfs/go-ipfs/pull/5899)) - CoreAPI: Don't panic when testing incomplete implementations ([ipfs/go-ipfs#5900](https://github.com/ipfs/go-ipfs/pull/5900)) - gateway: fix CORs headers ([ipfs/go-ipfs#5893](https://github.com/ipfs/go-ipfs/pull/5893)) - Local Gateway option ([ipfs/go-ipfs#5649](https://github.com/ipfs/go-ipfs/pull/5649)) - Show hash on gateway ([ipfs/go-ipfs#5830](https://github.com/ipfs/go-ipfs/pull/5830)) - fix: ulimit docs mistake ([ipfs/go-ipfs#5894](https://github.com/ipfs/go-ipfs/pull/5894)) - Move coreapi tests to the interface ([ipfs/go-ipfs#5865](https://github.com/ipfs/go-ipfs/pull/5865)) - Move checkHelptextRecursive forward a bit ([ipfs/go-ipfs#5889](https://github.com/ipfs/go-ipfs/pull/5889)) - coreapi/unixfs: Use path instead of raw hash in AddEvent ([ipfs/go-ipfs#5854](https://github.com/ipfs/go-ipfs/pull/5854)) - Fix name resolve --offline ([ipfs/go-ipfs#5885](https://github.com/ipfs/go-ipfs/pull/5885)) - testing: slow down republisher sharness test ([ipfs/go-ipfs#5856](https://github.com/ipfs/go-ipfs/pull/5856)) - docs: flesh out plugin documentation ([ipfs/go-ipfs#5876](https://github.com/ipfs/go-ipfs/pull/5876)) - main: move InterruptHandler to util ([ipfs/go-ipfs#5872](https://github.com/ipfs/go-ipfs/pull/5872)) - make: fix building source tarball on macos ([ipfs/go-ipfs#5860](https://github.com/ipfs/go-ipfs/pull/5860)) - fix config data race ([ipfs/go-ipfs#5634](https://github.com/ipfs/go-ipfs/pull/5634)) - CoreAPI: Global offline option ([ipfs/go-ipfs#5825](https://github.com/ipfs/go-ipfs/pull/5825)) - Update for go-ipfs-files refactor ([ipfs/go-ipfs#5661](https://github.com/ipfs/go-ipfs/pull/5661)) - feat: update Web UI to v2.3.0 ([ipfs/go-ipfs#5855](https://github.com/ipfs/go-ipfs/pull/5855)) - Stateful plugin loading ([ipfs/go-ipfs#4806](https://github.com/ipfs/go-ipfs/pull/4806)) - startup: always load the private key ([ipfs/go-ipfs#5844](https://github.com/ipfs/go-ipfs/pull/5844)) - add --dereference-args parameter ([ipfs/go-ipfs#5801](https://github.com/ipfs/go-ipfs/pull/5801)) - config: document the connection manager ([ipfs/go-ipfs#5839](https://github.com/ipfs/go-ipfs/pull/5839)) - add pinning support to the urlstore ([ipfs/go-ipfs#5834](https://github.com/ipfs/go-ipfs/pull/5834)) - refact(cmd/cat): remove useless code ([ipfs/go-ipfs#5836](https://github.com/ipfs/go-ipfs/pull/5836)) - Really run as non-root user in docker container ([ipfs/go-ipfs#5048](https://github.com/ipfs/go-ipfs/pull/5048)) - README: document guix package ([ipfs/go-ipfs#5832](https://github.com/ipfs/go-ipfs/pull/5832)) - docs: Improve config documentation ([ipfs/go-ipfs#5829](https://github.com/ipfs/go-ipfs/pull/5829)) - block: rm extra output ([ipfs/go-ipfs#5751](https://github.com/ipfs/go-ipfs/pull/5751)) - merge github-issue-guide with the issue template ([ipfs/go-ipfs#4636](https://github.com/ipfs/go-ipfs/pull/4636)) - docs: fix inconsistent capitalization of "API". ([ipfs/go-ipfs#5824](https://github.com/ipfs/go-ipfs/pull/5824)) - Update README.md ([ipfs/go-ipfs#5818](https://github.com/ipfs/go-ipfs/pull/5818)) - CONTRIBUTING.md link ([ipfs/go-ipfs#5811](https://github.com/ipfs/go-ipfs/pull/5811)) - README: Update required Go version ([ipfs/go-ipfs#5813](https://github.com/ipfs/go-ipfs/pull/5813)) - p2p: report-peer-id option for listen ([ipfs/go-ipfs#5771](https://github.com/ipfs/go-ipfs/pull/5771)) - really fix netcat race ([ipfs/go-ipfs#5803](https://github.com/ipfs/go-ipfs/pull/5803)) - [http_proxy_over_p2p] ([ipfs/go-ipfs#5526](https://github.com/ipfs/go-ipfs/pull/5526)) - coreapi/pin: Use CID's directly in maps instead of converting to string ([ipfs/go-ipfs#5809](https://github.com/ipfs/go-ipfs/pull/5809)) - Gx update go-merkledag and related deps. ([ipfs/go-ipfs#5802](https://github.com/ipfs/go-ipfs/pull/5802)) - cmds: rm old lib ([ipfs/go-ipfs#5786](https://github.com/ipfs/go-ipfs/pull/5786)) - badger: add truncate flag ([ipfs/go-ipfs#5625](https://github.com/ipfs/go-ipfs/pull/5625)) - docker: allow IPFS_PROFILE to choose the profile for `ipfs init` ([ipfs/go-ipfs#5473](https://github.com/ipfs/go-ipfs/pull/5473)) - Add --stream option to `ls` command ([ipfs/go-ipfs#5611](https://github.com/ipfs/go-ipfs/pull/5611)) - Switch to using request.Context() ([ipfs/go-ipfs#5782](https://github.com/ipfs/go-ipfs/pull/5782)) - Update go-ipfs-delay and assoc deps ([ipfs/go-ipfs#5762](https://github.com/ipfs/go-ipfs/pull/5762)) - Suppress bootstrap error ([ipfs/go-ipfs#5769](https://github.com/ipfs/go-ipfs/pull/5769)) - ISSUE_TEMPLATE: move the support question comment to the very top ([ipfs/go-ipfs#5770](https://github.com/ipfs/go-ipfs/pull/5770)) - cmds: use MakeTypedEncoder ([ipfs/go-ipfs#5760](https://github.com/ipfs/go-ipfs/pull/5760)) - cmds/bitswap: sort wantlist ([ipfs/go-ipfs#5759](https://github.com/ipfs/go-ipfs/pull/5759)) - cmds/update: use new cmds lib ([ipfs/go-ipfs#5730](https://github.com/ipfs/go-ipfs/pull/5730)) - cmds/file: use new cmds lib ([ipfs/go-ipfs#5756](https://github.com/ipfs/go-ipfs/pull/5756)) - cmds: remove redundant func ([ipfs/go-ipfs#5750](https://github.com/ipfs/go-ipfs/pull/5750)) - commands/refs: use new cmds ([ipfs/go-ipfs#5679](https://github.com/ipfs/go-ipfs/pull/5679)) - commands/pin: use new cmds lib ([ipfs/go-ipfs#5674](https://github.com/ipfs/go-ipfs/pull/5674)) - commands/bootstrap: use new cmds ([ipfs/go-ipfs#5678](https://github.com/ipfs/go-ipfs/pull/5678)) - fix(cmd/add): progressbar output error when input is read from stdin ([ipfs/go-ipfs#5743](https://github.com/ipfs/go-ipfs/pull/5743)) - unexport GOFLAGS ([ipfs/go-ipfs#5747](https://github.com/ipfs/go-ipfs/pull/5747)) - refactor(cmds): use new cmds ([ipfs/go-ipfs#5659](https://github.com/ipfs/go-ipfs/pull/5659)) - commands/filestore: use new cmds lib ([ipfs/go-ipfs#5673](https://github.com/ipfs/go-ipfs/pull/5673)) - Fix broken links ([ipfs/go-ipfs#5721](https://github.com/ipfs/go-ipfs/pull/5721)) - fix `ipfs help` bug #5557 ([ipfs/go-ipfs#5573](https://github.com/ipfs/go-ipfs/pull/5573)) - commands/bitswap: use new cmds lib ([ipfs/go-ipfs#5676](https://github.com/ipfs/go-ipfs/pull/5676)) - refact(cmd/repo): repo's sub cmds uses new cmd lib ([ipfs/go-ipfs#5677](https://github.com/ipfs/go-ipfs/pull/5677)) - fix the maketarball script ([ipfs/go-ipfs#5718](https://github.com/ipfs/go-ipfs/pull/5718)) - output link to WebUI on daemon startup ([ipfs/go-ipfs#5729](https://github.com/ipfs/go-ipfs/pull/5729)) - Move persistent datastores to plugins ([ipfs/go-ipfs#5695](https://github.com/ipfs/go-ipfs/pull/5695)) - Update IPTB test ([ipfs/go-ipfs#5636](https://github.com/ipfs/go-ipfs/pull/5636)) - enhance(cmd/verify): add goroutine count to improve verify speed ([ipfs/go-ipfs#5710](https://github.com/ipfs/go-ipfs/pull/5710)) - Update go-mfs and go-unixfs ([ipfs/go-ipfs#5714](https://github.com/ipfs/go-ipfs/pull/5714)) - fix(flag/version): flag `all` should have a higher priority ([ipfs/go-ipfs#5719](https://github.com/ipfs/go-ipfs/pull/5719)) - commands/p2p: use new cmds lib ([ipfs/go-ipfs#5672](https://github.com/ipfs/go-ipfs/pull/5672)) - commands/dht: use new cmds lib ([ipfs/go-ipfs#5671](https://github.com/ipfs/go-ipfs/pull/5671)) - commands/object: use new cmds ([ipfs/go-ipfs#5666](https://github.com/ipfs/go-ipfs/pull/5666)) - commands/files: use new cmds ([ipfs/go-ipfs#5665](https://github.com/ipfs/go-ipfs/pull/5665)) - cmds/env: add a config path helper ([ipfs/go-ipfs#5712](https://github.com/ipfs/go-ipfs/pull/5712)) - github.com/ipfs/dir-index-html: - show hash if given ([ipfs/dir-index-html#21](https://github.com/ipfs/dir-index-html/pull/21)) - Add "jpeg" as an alias to "jpg". ([ipfs/dir-index-html#16](https://github.com/ipfs/dir-index-html/pull/16)) - github.com/libp2p/go-addr-util: - Improve test coverage ([libp2p/go-addr-util#14](https://github.com/libp2p/go-addr-util/pull/14)) - github.com/ipfs/go-bitswap: - fix(prq): fix a bunch of goroutine leaks and deadlocks ([ipfs/go-bitswap#87](https://github.com/ipfs/go-bitswap/pull/87)) - remove allocations round two ([ipfs/go-bitswap#84](https://github.com/ipfs/go-bitswap/pull/84)) - fix(bitswap): remove CancelWants function ([ipfs/go-bitswap#80](https://github.com/ipfs/go-bitswap/pull/80)) - Avoid allocating for wantlist entries ([ipfs/go-bitswap#79](https://github.com/ipfs/go-bitswap/pull/79)) - ci(Jenkins): remove Jenkinsfile ([ipfs/go-bitswap#83](https://github.com/ipfs/go-bitswap/pull/83)) - More specific wantlists ([ipfs/go-bitswap#74](https://github.com/ipfs/go-bitswap/pull/74)) - fix(wantlist): remove races on setup ([ipfs/go-bitswap#72](https://github.com/ipfs/go-bitswap/pull/72)) - fix multiple data races ([ipfs/go-bitswap#76](https://github.com/ipfs/go-bitswap/pull/76)) - ci: add travis ([ipfs/go-bitswap#75](https://github.com/ipfs/go-bitswap/pull/75)) - providers: don't add every connected node as a provider ([ipfs/go-bitswap#59](https://github.com/ipfs/go-bitswap/pull/59)) - refactor(GetBlocks): Merge session/non-session ([ipfs/go-bitswap#64](https://github.com/ipfs/go-bitswap/pull/64)) - Feat: A more robust provider finder for sessions (for now) and soon for all bitswap ([ipfs/go-bitswap#60](https://github.com/ipfs/go-bitswap/pull/60)) - fix(tests): stabilize session tests ([ipfs/go-bitswap#63](https://github.com/ipfs/go-bitswap/pull/63)) - contexts: make sure to abort when a context is canceled ([ipfs/go-bitswap#58](https://github.com/ipfs/go-bitswap/pull/58)) - fix(sessions): explicitly connect found peers ([ipfs/go-bitswap#56](https://github.com/ipfs/go-bitswap/pull/56)) - Speed up sessions Round #1 ([ipfs/go-bitswap#27](https://github.com/ipfs/go-bitswap/pull/27)) - Fix debug log formatting issues ([ipfs/go-bitswap#37](https://github.com/ipfs/go-bitswap/pull/37)) - Feat/bandwidth limited tests ([ipfs/go-bitswap#42](https://github.com/ipfs/go-bitswap/pull/42)) - fix(tests): stabilize unreliable session tests ([ipfs/go-bitswap#44](https://github.com/ipfs/go-bitswap/pull/44)) - Bitswap Refactor #4: Extract session peer manager from sessions ([ipfs/go-bitswap#26](https://github.com/ipfs/go-bitswap/pull/26)) - Bitswap Refactor #3: Extract sessions to package ([ipfs/go-bitswap#30](https://github.com/ipfs/go-bitswap/pull/30)) - docs(comments): end comment sentences to have full-stop ([ipfs/go-bitswap#33](https://github.com/ipfs/go-bitswap/pull/33)) - Bitswap Refactor #2: Extract PeerManager From Want Manager + Unit Test ([ipfs/go-bitswap#29](https://github.com/ipfs/go-bitswap/pull/29)) - Bitswap Refactor #1: Session Manager & Extract Want Manager ([ipfs/go-bitswap#28](https://github.com/ipfs/go-bitswap/pull/28)) - fix(Receiver): Ignore unwanted blocks ([ipfs/go-bitswap#24](https://github.com/ipfs/go-bitswap/pull/24)) - feat(Benchmarks): Add real world dup blocks test ([ipfs/go-bitswap#25](https://github.com/ipfs/go-bitswap/pull/25)) - Feat/bitswap pr improvements ([ipfs/go-bitswap#19](https://github.com/ipfs/go-bitswap/pull/19)) - github.com/ipfs/go-blockservice: - Don't return errors on closed exchange ([ipfs/go-blockservice#15](https://github.com/ipfs/go-blockservice/pull/15)) - github.com/ipfs/go-cid: - fix inline CIDs generated by Prefix.Sum ([ipfs/go-cid#84](https://github.com/ipfs/go-cid/pull/84)) - Let Cid implement Binary[Un]Marshaler and Text[Un]Marshaler interfaces. ([ipfs/go-cid#81](https://github.com/ipfs/go-cid/pull/81)) - fix typo in comment ([ipfs/go-cid#80](https://github.com/ipfs/go-cid/pull/80)) - add codecs for Dash blocks, tx ([ipfs/go-cid#78](https://github.com/ipfs/go-cid/pull/78)) - github.com/ipfs/go-cidutil: - Fix Travis CI to run all tests. ([ipfs/go-cidutil#11](https://github.com/ipfs/go-cidutil/pull/11)) - Changes needed for `--cid-base` option in go-ipfs (simplified version) ([ipfs/go-cidutil#10](https://github.com/ipfs/go-cidutil/pull/10)) - add a utility method for sorting CID slices ([ipfs/go-cidutil#5](https://github.com/ipfs/go-cidutil/pull/5)) - github.com/libp2p/go-conn-security: - fix link to usage example in README ([libp2p/go-conn-security#4](https://github.com/libp2p/go-conn-security/pull/4)) - github.com/ipfs/go-datastore: - interfaces: make GetBacked* take a Read instead of a Datastore ([ipfs/go-datastore#115](https://github.com/ipfs/go-datastore/pull/115)) - remove closer type assertions ([ipfs/go-datastore#112](https://github.com/ipfs/go-datastore/pull/112)) - remove io.Closer from the transaction interface ([ipfs/go-datastore#113](https://github.com/ipfs/go-datastore/pull/113)) - feat(datastore): expose datastore Close() ([ipfs/go-datastore#111](https://github.com/ipfs/go-datastore/pull/111)) - query: make datastore ordering act like a user would expect ([ipfs/go-datastore#110](https://github.com/ipfs/go-datastore/pull/110)) - delayed: implement io.Closer and export datastore type. ([ipfs/go-datastore#108](https://github.com/ipfs/go-datastore/pull/108)) - split the datastore into a read and a write interface ([ipfs/go-datastore#107](https://github.com/ipfs/go-datastore/pull/107)) - Describe behavior of Batching datastores ([ipfs/go-datastore#105](https://github.com/ipfs/go-datastore/pull/105)) - handle concurrent puts/deletes in BasicBatch ([ipfs/go-datastore#103](https://github.com/ipfs/go-datastore/pull/103)) - add a GetSize method ([ipfs/go-datastore#99](https://github.com/ipfs/go-datastore/pull/99)) - github.com/ipfs/go-ds-badger: - removed additional/wasteful Prefix conversion ([ipfs/go-ds-badger#45](https://github.com/ipfs/go-ds-badger/pull/45)) - Enable Jenkins ([ipfs/go-ds-badger#35](https://github.com/ipfs/go-ds-badger/pull/35)) - fix application or ordering for interface change ([ipfs/go-ds-badger#44](https://github.com/ipfs/go-ds-badger/pull/44)) - Update badger ([ipfs/go-ds-badger#40](https://github.com/ipfs/go-ds-badger/pull/40)) - github.com/ipfs/go-ds-flatfs: - fix a goroutine leak killing the gateways ([ipfs/go-ds-flatfs#51](https://github.com/ipfs/go-ds-flatfs/pull/51)) - github.com/ipfs/go-ds-leveldb: - Expose Datastore type ([ipfs/go-ds-leveldb#20](https://github.com/ipfs/go-ds-leveldb/pull/20)) - fix application or ordering for interface change ([ipfs/go-ds-leveldb#23](https://github.com/ipfs/go-ds-leveldb/pull/23)) - github.com/ipfs/go-ipfs-cmds: - fix sync error with go1.12 on darwin ([ipfs/go-ipfs-cmds#147](https://github.com/ipfs/go-ipfs-cmds/pull/147)) - cli: fix ignoring std{out,err} sync errors on windows ([ipfs/go-ipfs-cmds#146](https://github.com/ipfs/go-ipfs-cmds/pull/146)) - roundup of cleanup fixes ([ipfs/go-ipfs-cmds#144](https://github.com/ipfs/go-ipfs-cmds/pull/144)) - Update cors library ([ipfs/go-ipfs-cmds#139](https://github.com/ipfs/go-ipfs-cmds/pull/139)) - expand on the api error ([ipfs/go-ipfs-cmds#138](https://github.com/ipfs/go-ipfs-cmds/pull/138)) - set the connection close header if we have a body to read ([ipfs/go-ipfs-cmds#116](https://github.com/ipfs/go-ipfs-cmds/pull/116)) - print a nicer error on timeout/cancel ([ipfs/go-ipfs-cmds#137](https://github.com/ipfs/go-ipfs-cmds/pull/137)) - Add link traversal option ([ipfs/go-ipfs-cmds#96](https://github.com/ipfs/go-ipfs-cmds/pull/96)) - Don't skip stdin test on Windows ([ipfs/go-ipfs-cmds#136](https://github.com/ipfs/go-ipfs-cmds/pull/136)) - MakeTypedEncoder: accept results by pointer or value ([ipfs/go-ipfs-cmds#134](https://github.com/ipfs/go-ipfs-cmds/pull/134)) - github.com/ipfs/go-ipfs-config: - Gateway.NoFetch ([ipfs/go-ipfs-config#19](https://github.com/ipfs/go-ipfs-config/pull/19)) - add a Clone function ([ipfs/go-ipfs-config#16](https://github.com/ipfs/go-ipfs-config/pull/16)) - randomports: give user ability to init ipfs using random port for swarm. ([ipfs/go-ipfs-config#17](https://github.com/ipfs/go-ipfs-config/pull/17)) - Allow the use of the User-Agent header ([ipfs/go-ipfs-config#15](https://github.com/ipfs/go-ipfs-config/pull/15)) - autorelay options ([ipfs/go-ipfs-config#21](https://github.com/ipfs/go-ipfs-config/pull/21)) - profile: add badger truncate option ([ipfs/go-ipfs-config#20](https://github.com/ipfs/go-ipfs-config/pull/20)) - github.com/ipfs/go-ipfs-delay: - Feat/refactor wait time ([ipfs/go-ipfs-delay#1](https://github.com/ipfs/go-ipfs-delay/pull/1)) - github.com/ipfs/go-ipfs-files: - multipart: fix handling of common prefixes ([ipfs/go-ipfs-files#7](https://github.com/ipfs/go-ipfs-files/pull/7)) - create implicit directories from multipart requests ([ipfs/go-ipfs-files#6](https://github.com/ipfs/go-ipfs-files/pull/6)) - TarWriter ([ipfs/go-ipfs-files#4](https://github.com/ipfs/go-ipfs-files/pull/4)) - Refactor filename - file relation ([ipfs/go-ipfs-files#2](https://github.com/ipfs/go-ipfs-files/pull/2)) - github.com/ipfs/go-ipld-cbor: - cbor: decode undefined as null ([ipfs/go-ipld-cbor#54](https://github.com/ipfs/go-ipld-cbor/pull/54)) - error when trying to encode an empty link ([ipfs/go-ipld-cbor#52](https://github.com/ipfs/go-ipld-cbor/pull/52)) - test for struct with both a cid and a bigint ([ipfs/go-ipld-cbor#51](https://github.com/ipfs/go-ipld-cbor/pull/51)) - github.com/ipfs/go-ipld-format: - Add a DAG walker with support for IPLD `Node`s ([ipfs/go-ipld-format#39](https://github.com/ipfs/go-ipld-format/pull/39)) - Add BufferedDAG wrapping Batch as a DAGService. ([ipfs/go-ipld-format#48](https://github.com/ipfs/go-ipld-format/pull/48)) - github.com/ipfs/go-ipld-git: - Fix blob marshalling ([ipfs/go-ipld-git#37](https://github.com/ipfs/go-ipld-git/pull/37)) - Re-enable assertion on commit size -- it is correct after #31 ([ipfs/go-ipld-git#33](https://github.com/ipfs/go-ipld-git/pull/33)) - Use OS path separator in testing, fixes #30 ([ipfs/go-ipld-git#34](https://github.com/ipfs/go-ipld-git/pull/34)) - Use rawdata length for size, fixes #7 ([ipfs/go-ipld-git#31](https://github.com/ipfs/go-ipld-git/pull/31)) - Cache RawData for Commit, Tag, & Tree, fixes #6 ([ipfs/go-ipld-git#28](https://github.com/ipfs/go-ipld-git/pull/28)) - Precompute Blob CID, fixes #21 ([ipfs/go-ipld-git#27](https://github.com/ipfs/go-ipld-git/pull/27)) - Enable Jenkins ([ipfs/go-ipld-git#29](https://github.com/ipfs/go-ipld-git/pull/29)) - github.com/ipfs/go-ipns: - fix community/CONTRIBUTING.md link in README.md ([ipfs/go-ipns#20](https://github.com/ipfs/go-ipns/pull/20)) - fix typo in README.md ([ipfs/go-ipns#21](https://github.com/ipfs/go-ipns/pull/21)) - testing: disable inline peer ID test ([ipfs/go-ipns#19](https://github.com/ipfs/go-ipns/pull/19)) - github.com/libp2p/go-libp2p: - Fixed race conditions in mock package mock_stream and mock_conn ([libp2p/go-libp2p#535](https://github.com/libp2p/go-libp2p/pull/535)) - increase initial relay advertisement delay to 30s ([libp2p/go-libp2p#534](https://github.com/libp2p/go-libp2p/pull/534)) - Use PeerRouting in autorelay to find relay peer addresses ([libp2p/go-libp2p#531](https://github.com/libp2p/go-libp2p/pull/531)) - docs: update broken links in NEWS.md ([libp2p/go-libp2p#517](https://github.com/libp2p/go-libp2p/pull/517)) - don't advertise the raw public address in autorelay ([libp2p/go-libp2p#511](https://github.com/libp2p/go-libp2p/pull/511)) - mock: export ratelimiter as RateLimiter ([libp2p/go-libp2p#507](https://github.com/libp2p/go-libp2p/pull/507)) - readme: remove duplicate repo entries in README and package-list.json ([libp2p/go-libp2p#506](https://github.com/libp2p/go-libp2p/pull/506)) - explicit option to enable autorelay ([libp2p/go-libp2p#500](https://github.com/libp2p/go-libp2p/pull/500)) - Add delay in initial relay advertisement to allow the dht time to bootstrap ([libp2p/go-libp2p#495](https://github.com/libp2p/go-libp2p/pull/495)) - suppressing error msg for NoSecurity option ([libp2p/go-libp2p#498](https://github.com/libp2p/go-libp2p/pull/498)) - pulling updates ([libp2p/go-libp2p#4](https://github.com/libp2p/go-libp2p/pull/4)) - fix contributing link in README ([libp2p/go-libp2p#494](https://github.com/libp2p/go-libp2p/pull/494)) - Fix badges and links on README.md ([libp2p/go-libp2p#485](https://github.com/libp2p/go-libp2p/pull/485)) - mocknet: fix NewStream and self dials ([libp2p/go-libp2p#480](https://github.com/libp2p/go-libp2p/pull/480)) - deflake identify test ([libp2p/go-libp2p#479](https://github.com/libp2p/go-libp2p/pull/479)) - mocknet: use peer ID in peer address ([libp2p/go-libp2p#476](https://github.com/libp2p/go-libp2p/pull/476)) - autorelay ([libp2p/go-libp2p#454](https://github.com/libp2p/go-libp2p/pull/454)) - Getting updates ([libp2p/go-libp2p#3](https://github.com/libp2p/go-libp2p/pull/3)) - github.com/libp2p/go-libp2p-autonat: - track autonat peer addresses ([libp2p/go-libp2p-autonat#7](https://github.com/libp2p/go-libp2p-autonat/pull/7)) - github.com/libp2p/go-libp2p-circuit: - Don't log raw binary ([libp2p/go-libp2p-circuit#53](https://github.com/libp2p/go-libp2p-circuit/pull/53)) - github.com/libp2p/go-libp2p-connmgr: - Fix concurrency and silence period not being honoured ([libp2p/go-libp2p-connmgr#26](https://github.com/libp2p/go-libp2p-connmgr/pull/26)) - github.com/libp2p/go-libp2p-crypto: - Fix: Remove redundant Ed25519 public key (#36). ([libp2p/go-libp2p-crypto#54](https://github.com/libp2p/go-libp2p-crypto/pull/54)) - libp2p badges, remove IPFS ([libp2p/go-libp2p-crypto#52](https://github.com/libp2p/go-libp2p-crypto/pull/52)) - Fix broken contribute link in README ([libp2p/go-libp2p-crypto#46](https://github.com/libp2p/go-libp2p-crypto/pull/46)) - forbid RSA keys smaller than 512 bits ([libp2p/go-libp2p-crypto#43](https://github.com/libp2p/go-libp2p-crypto/pull/43)) - Added ECDSA; Added RSA tests; Fixed linting errors; Handling all un-handled errors ([libp2p/go-libp2p-crypto#35](https://github.com/libp2p/go-libp2p-crypto/pull/35)) - switch to the go-crypto ed25519 implementation ([libp2p/go-libp2p-crypto#38](https://github.com/libp2p/go-libp2p-crypto/pull/38)) - update gogo protobuf ([libp2p/go-libp2p-crypto#37](https://github.com/libp2p/go-libp2p-crypto/pull/37)) - github.com/libp2p/go-libp2p-discovery: - add a timeout to Provide in routing.Advertise ([libp2p/go-libp2p-discovery#12](https://github.com/libp2p/go-libp2p-discovery/pull/12)) - correctly encode ns to CID ([libp2p/go-libp2p-discovery#11](https://github.com/libp2p/go-libp2p-discovery/pull/11)) - use 6hrs as ttl for routing based advertisements ([libp2p/go-libp2p-discovery#8](https://github.com/libp2p/go-libp2p-discovery/pull/8)) - github.com/libp2p/go-libp2p-host: - Helper to get PeerInfo from Host ([libp2p/go-libp2p-host#20](https://github.com/libp2p/go-libp2p-host/pull/20)) - github.com/libp2p/go-libp2p-kad-dht: - fix(dialQueue): account for failed dials ([libp2p/go-libp2p-kad-dht#277](https://github.com/libp2p/go-libp2p-kad-dht/pull/277)) - Fix Bootstrap sub-queries ([libp2p/go-libp2p-kad-dht#264](https://github.com/libp2p/go-libp2p-kad-dht/pull/264)) - dial queue: fix possible goroutine leak ([libp2p/go-libp2p-kad-dht#262](https://github.com/libp2p/go-libp2p-kad-dht/pull/262)) - Alter some logging ([libp2p/go-libp2p-kad-dht#269](https://github.com/libp2p/go-libp2p-kad-dht/pull/269)) - Revert #236: Test go mod in travis and use major versioning in import paths ([libp2p/go-libp2p-kad-dht#259](https://github.com/libp2p/go-libp2p-kad-dht/pull/259)) - fix tests on freebsd ([libp2p/go-libp2p-kad-dht#255](https://github.com/libp2p/go-libp2p-kad-dht/pull/255)) - Fix "no protocol with name dnsaddr" error ([libp2p/go-libp2p-kad-dht#247](https://github.com/libp2p/go-libp2p-kad-dht/pull/247)) - Fix a race in dial queue ([libp2p/go-libp2p-kad-dht#248](https://github.com/libp2p/go-libp2p-kad-dht/pull/248)) - Fix races with DialQueue variables ([libp2p/go-libp2p-kad-dht#241](https://github.com/libp2p/go-libp2p-kad-dht/pull/241)) - Fix CircleCI ([libp2p/go-libp2p-kad-dht#238](https://github.com/libp2p/go-libp2p-kad-dht/pull/238)) - Adaptive queue for staging dials ([libp2p/go-libp2p-kad-dht#237](https://github.com/libp2p/go-libp2p-kad-dht/pull/237)) - Add the full libp2p default bootstrap peer list ([libp2p/go-libp2p-kad-dht#226](https://github.com/libp2p/go-libp2p-kad-dht/pull/226)) - Revert "Tidy up bootstrapping" ([libp2p/go-libp2p-kad-dht#232](https://github.com/libp2p/go-libp2p-kad-dht/pull/232)) - Tidy up bootstrapping ([libp2p/go-libp2p-kad-dht#225](https://github.com/libp2p/go-libp2p-kad-dht/pull/225)) - Revert "Remove signal bootstrapping" ([libp2p/go-libp2p-kad-dht#227](https://github.com/libp2p/go-libp2p-kad-dht/pull/227)) - Remove signal bootstrapping ([libp2p/go-libp2p-kad-dht#224](https://github.com/libp2p/go-libp2p-kad-dht/pull/224)) - fix a potential DHT query hang ([libp2p/go-libp2p-kad-dht#219](https://github.com/libp2p/go-libp2p-kad-dht/pull/219)) - docs: duplicate pkg documentation ([libp2p/go-libp2p-kad-dht#218](https://github.com/libp2p/go-libp2p-kad-dht/pull/218)) - tests: skip key inlining test ([libp2p/go-libp2p-kad-dht#212](https://github.com/libp2p/go-libp2p-kad-dht/pull/212)) - Rephrase "betterPeersToQuery" method comment to be less cryptic ([libp2p/go-libp2p-kad-dht#206](https://github.com/libp2p/go-libp2p-kad-dht/pull/206)) - github.com/libp2p/go-libp2p-loggables: - test: add unit tests ([libp2p/go-libp2p-loggables#21](https://github.com/libp2p/go-libp2p-loggables/pull/21)) - github.com/libp2p/go-libp2p-netutil: - Add tests ([libp2p/go-libp2p-netutil#28](https://github.com/libp2p/go-libp2p-netutil/pull/28)) - github.com/libp2p/go-libp2p-peer: - fix: re-enable peer ID inlining but make it configurable ([libp2p/go-libp2p-peer#42](https://github.com/libp2p/go-libp2p-peer/pull/42)) - Protobuf and JSON (un-)marshalling methods for peer.ID ([libp2p/go-libp2p-peer#41](https://github.com/libp2p/go-libp2p-peer/pull/41)) - disable key inlining ([libp2p/go-libp2p-peer#40](https://github.com/libp2p/go-libp2p-peer/pull/40)) - github.com/libp2p/go-libp2p-peerstore: - Add unit test to verify AddAddr doesn't shorten TTL ([libp2p/go-libp2p-peerstore#52](https://github.com/libp2p/go-libp2p-peerstore/pull/52)) - disable inline-peer id test ([libp2p/go-libp2p-peerstore#49](https://github.com/libp2p/go-libp2p-peerstore/pull/49)) - README: Update contributing guideline linkrot. ([libp2p/go-libp2p-peerstore#48](https://github.com/libp2p/go-libp2p-peerstore/pull/48)) - Deterministic benchmark order; Keybook interface benchmarks ([libp2p/go-libp2p-peerstore#43](https://github.com/libp2p/go-libp2p-peerstore/pull/43)) - PeerInfo UnMarshal Error #393 ([libp2p/go-libp2p-peerstore#45](https://github.com/libp2p/go-libp2p-peerstore/pull/45)) - fix the inline key test ([libp2p/go-libp2p-peerstore#44](https://github.com/libp2p/go-libp2p-peerstore/pull/44)) - github.com/libp2p/go-libp2p-pubsub: - move timecache check/update after validation ([libp2p/go-libp2p-pubsub#156](https://github.com/libp2p/go-libp2p-pubsub/pull/156)) - fix nonsensical check ([libp2p/go-libp2p-pubsub#154](https://github.com/libp2p/go-libp2p-pubsub/pull/154)) - Extend validator interface to include message source ([libp2p/go-libp2p-pubsub#151](https://github.com/libp2p/go-libp2p-pubsub/pull/151)) - Implement peer blacklist ([libp2p/go-libp2p-pubsub#149](https://github.com/libp2p/go-libp2p-pubsub/pull/149)) - make timecache duration configurable ([libp2p/go-libp2p-pubsub#148](https://github.com/libp2p/go-libp2p-pubsub/pull/148)) - godoc is not html either ([libp2p/go-libp2p-pubsub#147](https://github.com/libp2p/go-libp2p-pubsub/pull/147)) - godoc documentation is not markdown ([libp2p/go-libp2p-pubsub#146](https://github.com/libp2p/go-libp2p-pubsub/pull/146)) - Add documentation for subscribe's non-instantaneous semantics ([libp2p/go-libp2p-pubsub#145](https://github.com/libp2p/go-libp2p-pubsub/pull/145)) - Some documentation ([libp2p/go-libp2p-pubsub#140](https://github.com/libp2p/go-libp2p-pubsub/pull/140)) - rework peer tracking logic to handle multiple connections ([libp2p/go-libp2p-pubsub#132](https://github.com/libp2p/go-libp2p-pubsub/pull/132)) - github.com/libp2p/go-libp2p-pubsub-router: - encode record-store keys in pubsub ([libp2p/go-libp2p-pubsub-router#17](https://github.com/libp2p/go-libp2p-pubsub-router/pull/17)) - github.com/libp2p/go-libp2p-quic-transport: - fix badges in README ([libp2p/go-libp2p-quic-transport#39](https://github.com/libp2p/go-libp2p-quic-transport/pull/39)) - Fix missing transport parameter in dialed connection ([libp2p/go-libp2p-quic-transport#38](https://github.com/libp2p/go-libp2p-quic-transport/pull/38)) - github.com/libp2p/go-libp2p-routing: - Update the comment on IpfsRouting.Bootstrap ([libp2p/go-libp2p-routing#36](https://github.com/libp2p/go-libp2p-routing/pull/36)) - github.com/libp2p/go-libp2p-swarm: - Make FD limits configurable by environment property ([libp2p/go-libp2p-swarm#102](https://github.com/libp2p/go-libp2p-swarm/pull/102)) - Fix logging race ([libp2p/go-libp2p-swarm#100](https://github.com/libp2p/go-libp2p-swarm/pull/100)) - Add CircleCI config ([libp2p/go-libp2p-swarm#99](https://github.com/libp2p/go-libp2p-swarm/pull/99)) - Enhance debug logging in dial limiter ([libp2p/go-libp2p-swarm#98](https://github.com/libp2p/go-libp2p-swarm/pull/98)) - dialer: handle dial cancel and/or completion before trying new addresses ([libp2p/go-libp2p-swarm#96](https://github.com/libp2p/go-libp2p-swarm/pull/96)) - avoid spawning goroutines for canceled dials ([libp2p/go-libp2p-swarm#95](https://github.com/libp2p/go-libp2p-swarm/pull/95)) - warn when we encounter a useless transport ([libp2p/go-libp2p-swarm#90](https://github.com/libp2p/go-libp2p-swarm/pull/90)) - github.com/libp2p/go-libp2p-transport: - fix transport tests for quic ([libp2p/go-libp2p-transport#39](https://github.com/libp2p/go-libp2p-transport/pull/39)) - fix: fully close streams before returning ([libp2p/go-libp2p-transport#37](https://github.com/libp2p/go-libp2p-transport/pull/37)) - fix typo in README ([libp2p/go-libp2p-transport#36](https://github.com/libp2p/go-libp2p-transport/pull/36)) - github.com/libp2p/go-libp2p-transport-upgrader: - annotate errors ([libp2p/go-libp2p-transport-upgrader#11](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/11)) - github.com/ipfs/go-log: - uglify the (event) logs ([ipfs/go-log#53](https://github.com/ipfs/go-log/pull/53)) - add environment variable for writing tracing information to a file ([ipfs/go-log#52](https://github.com/ipfs/go-log/pull/52)) - correctly display the line number when FinishWithErr fails ([ipfs/go-log#51](https://github.com/ipfs/go-log/pull/51)) - github.com/libp2p/go-maddr-filter: - test: extend test to improve coverage ([libp2p/go-maddr-filter#7](https://github.com/libp2p/go-maddr-filter/pull/7)) - github.com/ipfs/go-merkledag: - Increase FetchGraphConcurrency to 32 ([ipfs/go-merkledag#29](https://github.com/ipfs/go-merkledag/pull/29)) - Enable CI ([ipfs/go-merkledag#9](https://github.com/ipfs/go-merkledag/pull/9)) - fix a fetch deadlock on error ([ipfs/go-merkledag#21](https://github.com/ipfs/go-merkledag/pull/21)) - Wait for all go routines to finish before function returns ([ipfs/go-merkledag#19](https://github.com/ipfs/go-merkledag/pull/19)) - github.com/ipfs/go-metrics-prometheus: - use Prometheus instead of gxed ([ipfs/go-metrics-prometheus#3](https://github.com/ipfs/go-metrics-prometheus/pull/3)) - github.com/ipfs/go-mfs: - fix(mv): dst filename error ([ipfs/go-mfs#62](https://github.com/ipfs/go-mfs/pull/62)) - fix over-wait in WaitPub ([ipfs/go-mfs#53](https://github.com/ipfs/go-mfs/pull/53)) - Fix/32/pr ports from go-ipfs to go-mfs ([ipfs/go-mfs#49](https://github.com/ipfs/go-mfs/pull/49)) - remove the `fullSync` option from `updateChildEntry` ([ipfs/go-mfs#45](https://github.com/ipfs/go-mfs/pull/45)) - Various refactorings ([ipfs/go-mfs#36](https://github.com/ipfs/go-mfs/pull/36)) - use RW lock for the `File`'s lock ([ipfs/go-mfs#43](https://github.com/ipfs/go-mfs/pull/43)) - add documentation links in README ([ipfs/go-mfs#41](https://github.com/ipfs/go-mfs/pull/41)) - [WIP] documentation notes ([ipfs/go-mfs#27](https://github.com/ipfs/go-mfs/pull/27)) - feat(inode): add inode struct ([ipfs/go-mfs#12](https://github.com/ipfs/go-mfs/pull/12)) - github.com/libp2p/go-mplex: - fix deadlock ([libp2p/go-mplex#39](https://github.com/libp2p/go-mplex/pull/39)) - When a stream is closed, cancel pending writes ([libp2p/go-mplex#35](https://github.com/libp2p/go-mplex/pull/35)) - make sure to but the buffer back in the pool ([libp2p/go-mplex#34](https://github.com/libp2p/go-mplex/pull/34)) - reduce the packet count ([libp2p/go-mplex#29](https://github.com/libp2p/go-mplex/pull/29)) - github.com/ipfs/go-path: - fix: no components error ([ipfs/go-path#18](https://github.com/ipfs/go-path/pull/18)) - nit: validate CIDs in IPLD paths ([ipfs/go-path#16](https://github.com/ipfs/go-path/pull/16)) - github.com/libp2p/go-reuseport: - Fix build on wasm ([libp2p/go-reuseport#59](https://github.com/libp2p/go-reuseport/pull/59)) - Use Go Control API ([libp2p/go-reuseport#56](https://github.com/libp2p/go-reuseport/pull/56)) - Support WASM ([libp2p/go-reuseport#54](https://github.com/libp2p/go-reuseport/pull/54)) - github.com/libp2p/go-reuseport-transport: - Update to go-reuseport 0.2.0 ([libp2p/go-reuseport-transport#6](https://github.com/libp2p/go-reuseport-transport/pull/6)) - github.com/libp2p/go-stream-muxer: - add standard reset error ([libp2p/go-stream-muxer#23](https://github.com/libp2p/go-stream-muxer/pull/23)) - ci: fix ([libp2p/go-stream-muxer#24](https://github.com/libp2p/go-stream-muxer/pull/24)) - Document Reset versus Close ([libp2p/go-stream-muxer#18](https://github.com/libp2p/go-stream-muxer/pull/18)) - WIP document Conn.Close ([libp2p/go-stream-muxer#19](https://github.com/libp2p/go-stream-muxer/pull/19)) - github.com/libp2p/go-tcp-transport: - Deprecate IPFS_REUSEPORT, use LIBP2P_TCP_REUSEPORT ([libp2p/go-tcp-transport#27](https://github.com/libp2p/go-tcp-transport/pull/27)) - github.com/ipfs/go-unixfs: - unixfile: precalc dir size ([ipfs/go-unixfs#61](https://github.com/ipfs/go-unixfs/pull/61)) - Archive refactor ([ipfs/go-unixfs#59](https://github.com/ipfs/go-unixfs/pull/59)) - decouple the DAG traversal logic from the DAG reader (local branch) ([ipfs/go-unixfs#60](https://github.com/ipfs/go-unixfs/pull/60)) - Unixfs: enforce refs on files when using nocopy ([ipfs/go-unixfs#56](https://github.com/ipfs/go-unixfs/pull/56)) - Fix/handle overflow ([ipfs/go-unixfs#53](https://github.com/ipfs/go-unixfs/pull/53)) - feat(Directory): Add EnumLinksAsync method ([ipfs/go-unixfs#39](https://github.com/ipfs/go-unixfs/pull/39)) ## v0.4.18 2018-10-26 This is probably one of the largest go-ipfs releases in recent history, 3 months in the making. ### Features The headline features this release are experimental QUIC support, the gossipsub pubsub routing algorithm, pubsub message signing, and a refactored `ipfs p2p` command. However, that's just scratching the surface. #### QUIC First up, on the networking front, this release has also introduced experimental support for the QUIC protocol. QUIC is a new UDP-based network transport that solves many of the long standing issues with TCP. For us, this means (eventually): * **Fewer local resources.** TCP requires a file-descriptor per connection while QUIC (and most UDP based transports) can share a single file descriptor between all connections. This should allow us to dial faster and keep more connections open. * **Faster connection establishment.** When client authentication is included, QUIC has a three-way handshake like TCP. However, unlike TCP, this handshake brings us from all the way from 0 to a fully encrypted, authenticated, and multiplexed connection. In theory (not yet in practice), this should significantly reduce the latency of DHT queries. * **Behaves better on lossy networks.** When multiplexing multiple requests over a single TCP connection, a single dropped packet will bring the entire connection to a halt while the packet is re-transmitted. However, because QUIC handles multiplexing internally, dropping a single packets affects only the related stream. * **Better NAT traversal.** TL;DR: NAT hole-punching is significantly easier and, in many cases, more reliable with UDP than with TCP. However, we still have a long way to go. While we encourage users to test this, the IETF QUIC protocol is still being actively developed and *will* change. You can find instructions for enabling it [here](https://github.com/ipfs/go-ipfs/blob/master/docs/experimental-features.md#QUIC). #### Pubsub In terms of pubsub, go-ipfs now supports the gossipsub routing algorithm and message signing. The gossipsub routing algorithm is *significantly* more efficient than the current floodsub routing algorithm. Even better, it's fully backwards compatible so you can enable it and still talk to nodes using the floodsub algorithm. You can find instructions to enable gossipsub in go-ipfs [here](https://github.com/ipfs/go-ipfs/blob/master/docs/experimental-features.md#gossipsub). Messages are now signed by their authors. While signing has now been enabled by default, strict signature verification has not been and will not be for at least one release (probably multiple) to avoid breaking existing applications. You can read about how to configure this feature [here](https://github.com/ipfs/go-ipfs/blob/master/docs/experimental-features.md#message-signing). #### Commands In terms of new toys, this release introduces a new `ipfs cid` subcommand for working with CIDs, a completely refactored `ipfs p2p` command, streaming name resolution, and complete inline block support. The new `ipfs cid` command allows users to both inspect CIDs and convert them between various formats and versions. For example: ```sh # Print out the CID metadata (prefix) > ipfs cid format -f %P QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o cidv0-protobuf-sha2-256-32 # Get the hex sha256 hash from the CID. > ipfs cid format -b base16 -f '0x%D' QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o 0x46d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e # Convert a base58 v0 CID to a base32 v1 CID. > ipfs cid base32 QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o bafybeicg2rebjoofv4kbyovkw7af3rpiitvnl6i7ckcywaq6xjcxnc2mby ``` The refactored `ipfs p2p` command allows forwarding TCP streams through two IPFS nodes from one host to another. It's `ssh -L` but for IPFS. You can find documentation [here](https://github.com/ipfs/go-ipfs/blob/master/docs/experimental-features.md#ipfs-p2p). It's still experimental but we don't expect too many breaking changes at this point (it will very likely be stabilized in the next release). Quick summary of breaking changes: * We don't stop listening for local (forwarded) connections after accepting a single connection. * `ipfs p2p stream ls` output now returns more useful output, first address is always the initiator address. * `ipfs p2p listener ls` is renamed to `ipfs p2p ls` * `ipfs p2p listener close` is renamed to `ipfs p2p close` * Protocol names have to be prefixed with `/x/` and are now just passed to libp2p as handler name. Previous version did this 'under the hood' and with `/p2p/` prefix. There is a `--allow-custom-protocol` flag which allows you to use any libp2p handler name. * `ipfs p2p listener open` and `ipfs p2p stream dial` got renamed: * `ipfs p2p listener open p2p-test /ip4/127.0.0.1/tcp/10101` new becomes `ipfs p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10101` * `ipfs p2p stream dial $NODE_A_PEERID p2p-test /ip4/127.0.0.1/tcp/10102` is now `ipfs p2p forward /x/p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/$NODE_A_PEERID` There is now a new flag for `ipfs name resolve` - `--stream`. When the command is invoked with the flag set, it will start returning results as soon as they are discovered in the DHT and other routing mechanisms. This enables certain applications to start prefetching/displaying data while the discovery is still running. Note that this command will likely return many outdated records before it finding and returning the latest. However, it will always return *valid* records (even if a bit stale). Finally, in the previous release, we added support for extracting blocks inlined into CIDs. In this release, we've added support for creating these CIDs. You can now run `ipfs add` with the `--inline` flag to inline blocks less than or equal to 32 bytes in length into a CID, instead of writing an actual block. This should significantly reduce the size of filesystem trees with many empty directories and tiny files. #### IPNS You can now publish and resolve paths with namespaces *other* than `/ipns` and `/ipfs` through IPNS. Critically, IPNS can now be used with IPLD paths (paths starting with `/ipld`). #### WebUI Finally, this release includes the shiny [updated webui](https://github.com/ipfs-shipyard/ipfs-webui). You can view it by installing go-ipfs and visiting http://localhost:5001/webui. ### Performance This release includes some significant performance improvements, both in terms of resource utilization and speed. This section will go into some technical details so feel free to skip it if you're just looking for shiny new features. #### Resource Utilization In this release, we've (a) fixed a slow memory leak in libp2p and (b) significantly reduced the allocation load. Together, these should improve both memory and CPU usage. ##### Datastructures We've changed two of our most frequently used datastructures, CIDs and Multiaddrs, to reduce allocation load. First, we now store CIDs *encode* as strings, instead of decoded in structs (behind pointers). In addition to being more compact, our `Cid` type is now a valid `map` key so we no longer have to encode CIDs every time we want to use them in a map/set. Allocations when inserting CIDs into maps/sets was showing up as a significant source of allocations under heavy load so this change should improve memory usage. Second, we've changed many of our multiaddr parsing/processing/formatting functions to allocate less. Much of our DHT related-work includes processing multiaddrs so this should reduce CPU utilization when heavily using the DHT. ##### Streams and Yamux Streams have always plagued us in terms of memory utilization. This was partially solved by introducing the connection manager, keeping our maximum connection count to a reasonable number but they're still a major memory sink. This release sees two improvements on this front: 1. A memory [leak in identify](https://github.com/libp2p/go-libp2p/issues/419) has been fixed. This was slowly causing us to leak connections (locking up the memory used by the connections' streams). 2. Yamux streams now use a buffer-pool backed, auto shrinking read buffer. Before, this read buffer would grow to its maximum size (a few megabytes) and never shrink but these buffers now shrink as they're emptied. #### Bitswap Performance Bitswap will now pack *multiple* small blocks into a single message thanks [ipfs/go-bitswap#5](https://github.com/ipfs/go-bitswap/pull/5). While this won't help when transferring large files (with large blocks), this should help when transferring many tiny files. ### Refactors and Endeavors This release saw yet another commands-library refactor, work towards the CoreAPI, and the first step towards reliable base32 CID support. #### Commands Lib We've completely refactored our commands library (again). While it still needs quite a bit of work, it now requires significantly less boilerplate and should be significantly more robust. The refactor immediately found two broken tests and probably fixed quite a few bugs around properly returning and handling errors. #### CoreAPI CoreAPI is a new way to interact with IPFS from Go. While it's still not final, most things you can do via the CLI or HTTP interfaces, can now be done through the new API. Currently there is only one implementation, backed by go-ipfs node, and there are plans to start http-api backed one soon. We are also looking into creating RPC interface using this API, which could help performance in some use cases. You can track progress in https://github.com/ipfs/go-ipfs/issues/4498 #### IPLD paths We introduced new path type which introduces distinction between IPLD and IPFS (unixfs) paths. From now on paths prefixed with `/ipld/` will always use IPLD link traversal and `/ipfs/` will use unixfs path resolver, which takes things like sharding into account. Note that this is only initial support and there likely are some bugs in how the paths are handled internally, so consider this feature experimental for now. #### CIDv1/Base32 Migration Currently, IPFS is usually used in browsers by browsing to `https://SOME_GATEWAY/ipfs/CID/...`. There are two significant drawbacks to this approach: 1. From a browser security standpoint, all IPFS "sites" will live under the same origin (SOME_GATEWAY). 2. From a UX standpoint, this doesn't feel very "native" (even if the gateway is a local IPFS node). To fix the security issue, we intend to switch IPFS gateway links `https://ipfs.io/ipfs/CID` to `https://CID.ipfs.dweb.link`. This way, the CID will be a part of the ["origin"](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Origin) so each IPFS website will get a separate security origin. To fix the UX issue, we've been working on adding support for `ipfs://CID/...` to web browsers through our [ipfs-companion](https://github.com/ipfs/ipfs-companion/) add-on and some new, experimental extension APIs from Mozilla. This has the same effect of putting the CID in the URL origin but has the added benefit of looking "native". Unfortunately, origins must be *case insensitive*. Currently, most CIDs users see are *CIDv0* CIDs (those starting with `Qm`) which are *always* base58 encoded and are therefore case-sensitive. Fortunately, CIDv1 (the latest CID format) supports arbitrary bases using the [multibase](https://github.com/multiformats/multibase/) standard. Unfortunately, IPFS has always treated equivalent CIDv0 and CIDv1 CIDs as distinct. This means that files added with CIDv0 CIDs (the default) can't be looked up using the equivalent CIDv1. This release makes some significant progress towards solving this issue by introducing two features: (1) The previous mentioned `ipfs cid base32` command for converting CID to a case intensive encoding required by domain names. This command converts a CID to version 1 and encodes it using base32. (2) A hack to allow locally looking up blocks associated with a CIDv0 CID using the equivalent CIDv1 CID (or the reverse). This hack will eventually be replaced with a multihash indexed blockstore, which is agnostic to both the CID version and multicodec content type. ### go-ipfs changelog Features (i.e., users take heed): - gossipsub ([ipfs/go-ipfs#5373](https://github.com/ipfs/go-ipfs/pull/5373)) - support /ipfs/CID in `ipfs dht findprovs` ([ipfs/go-ipfs#5329](https://github.com/ipfs/go-ipfs/pull/5329)) - return a json object from config show ([ipfs/go-ipfs#5345](https://github.com/ipfs/go-ipfs/pull/5345)) - Set filename in Content-Disposition if filename=x is passed in URI query ([ipfs/go-ipfs#4177](https://github.com/ipfs/go-ipfs/pull/4177)) - Allow mfs files.write command to create parent directories ([ipfs/go-ipfs#5359](https://github.com/ipfs/go-ipfs/pull/5359)) - Run DNS lookup for --api endpoint provided in CLI ([ipfs/go-ipfs#5372](https://github.com/ipfs/go-ipfs/pull/5372)) - Add support for inlinling blocks into CIDs the id-hash ([ipfs/go-ipfs#5281](https://github.com/ipfs/go-ipfs/pull/5281)) - depth limited refs -r ([ipfs/go-ipfs#5337](https://github.com/ipfs/go-ipfs/pull/5337)) - remove bitswap unwant ([ipfs/go-ipfs#5308](https://github.com/ipfs/go-ipfs/pull/5308)) - add experimental QUIC support ([ipfs/go-ipfs#5350](https://github.com/ipfs/go-ipfs/pull/5350)) - add a --stdin-name flag for naming files from stdin ([ipfs/go-ipfs#5399](https://github.com/ipfs/go-ipfs/pull/5399)) - Refactor `ipfs p2p` ([ipfs/go-ipfs#4929](https://github.com/ipfs/go-ipfs/pull/4929)) - add dns support in`ipfs p2p forward` and refactor code ([ipfs/go-ipfs#5533](https://github.com/ipfs/go-ipfs/pull/5533)) - feat(command): expose connection direction ([ipfs/go-ipfs#5457](https://github.com/ipfs/go-ipfs/pull/5457)) - error when publishing ipns records without a running daemon ([ipfs/go-ipfs#5477](https://github.com/ipfs/go-ipfs/pull/5477)) - feat(daemon): print version on start ([ipfs/go-ipfs#5503](https://github.com/ipfs/go-ipfs/pull/5503)) - add quieter option to name publish ([ipfs/go-ipfs#5494](https://github.com/ipfs/go-ipfs/pull/5494)) - Provide new "cid" sub-command. ([ipfs/go-ipfs#5385](https://github.com/ipfs/go-ipfs/pull/5385)) - feat(command): add force flag for files rm ([ipfs/go-ipfs#5555](https://github.com/ipfs/go-ipfs/pull/5555)) - Add support for datastore plugins ([ipfs/go-ipfs#5187](https://github.com/ipfs/go-ipfs/pull/5187)) - files ls: append slash to directory names ([ipfs/go-ipfs#5605](https://github.com/ipfs/go-ipfs/pull/5605)) - ipfs name resolve --stream ([ipfs/go-ipfs#5404](https://github.com/ipfs/go-ipfs/pull/5404)) - update webui to 2.1.0 ([ipfs/go-ipfs#5627](https://github.com/ipfs/go-ipfs/pull/5627)) - feat: add dry-run flag for config profile apply command ([ipfs/go-ipfs#5455](https://github.com/ipfs/go-ipfs/pull/5455)) - configurable pubsub signing ([ipfs/go-ipfs#5647](https://github.com/ipfs/go-ipfs/pull/5647)) Fixes (i.e., users take note): - pin update fixes ([ipfs/go-ipfs#5265](https://github.com/ipfs/go-ipfs/pull/5265)) - Fix inability to pin two things at once ([ipfs/go-ipfs#5512](https://github.com/ipfs/go-ipfs/pull/5512)) - wait for all connections to close before exiting on shutdown. ([ipfs/go-ipfs#5322](https://github.com/ipfs/go-ipfs/pull/5322)) - Fixed ipns address resolution in fuse unix mount ([ipfs/go-ipfs#5384](https://github.com/ipfs/go-ipfs/pull/5384)) - core/commands/ls: wrap `NewDirectoryFromNode` error ([ipfs/go-ipfs#5166](https://github.com/ipfs/go-ipfs/pull/5166)) - fix goroutine leaks in filestore.go ([ipfs/go-ipfs#5427](https://github.com/ipfs/go-ipfs/pull/5427)) - move VersionOption after GatewayOption to fix #5422 ([ipfs/go-ipfs#5424](https://github.com/ipfs/go-ipfs/pull/5424)) - fix(commands): fix filestore.go goroutine leak ([ipfs/go-ipfs#5439](https://github.com/ipfs/go-ipfs/pull/5439)) - fix(commands): goroutine leaks in ping.go ([ipfs/go-ipfs#5444](https://github.com/ipfs/go-ipfs/pull/5444)) - fix output of object command ([ipfs/go-ipfs#5459](https://github.com/ipfs/go-ipfs/pull/5459)) - add warning when no bootstrap in config ([ipfs/go-ipfs#5445](https://github.com/ipfs/go-ipfs/pull/5445)) - fix behaviour of key rename to same name ([ipfs/go-ipfs#5465](https://github.com/ipfs/go-ipfs/pull/5465)) - fix(object): print object diff error ([ipfs/go-ipfs#5469](https://github.com/ipfs/go-ipfs/pull/5469)) - fix(pin): goroutine leaks ([ipfs/go-ipfs#5453](https://github.com/ipfs/go-ipfs/pull/5453)) - fix offline id bug ([ipfs/go-ipfs#5486](https://github.com/ipfs/go-ipfs/pull/5486)) - files cp: improve flush error message ([ipfs/go-ipfs#5485](https://github.com/ipfs/go-ipfs/pull/5485)) - resolve: fix unixfs resolution through sharded directories ([ipfs/go-ipfs#5484](https://github.com/ipfs/go-ipfs/pull/5484)) - Switch name publish/resolve to coreapi ([ipfs/go-ipfs#5563](https://github.com/ipfs/go-ipfs/pull/5563)) - use CoreAPI resolver everywhere (fixes sharded directory resolution) ([ipfs/go-ipfs#5492](https://github.com/ipfs/go-ipfs/pull/5492)) - add pin lock in AddallPin function ([ipfs/go-ipfs#5506](https://github.com/ipfs/go-ipfs/pull/5506)) - take the pinlock when updating pins ([ipfs/go-ipfs#5550](https://github.com/ipfs/go-ipfs/pull/5550)) - fix(object): add support for raw leaves in object diff ([ipfs/go-ipfs#5472](https://github.com/ipfs/go-ipfs/pull/5472)) - don't use the domain name as a filename in /ipns/a.com ([ipfs/go-ipfs#5564](https://github.com/ipfs/go-ipfs/pull/5564)) - refactor(command): modify int to int64 ([ipfs/go-ipfs#5612](https://github.com/ipfs/go-ipfs/pull/5612)) - fix(core): ipns config RecordLifetime panic ([ipfs/go-ipfs#5648](https://github.com/ipfs/go-ipfs/pull/5648)) - simplify dag put and correctly take pin lock ([ipfs/go-ipfs#5667](https://github.com/ipfs/go-ipfs/pull/5667)) - fix Prometheus concurrent map write bug ([ipfs/go-ipfs#5706](https://github.com/ipfs/go-ipfs/pull/5706)) Regressions Fixes (fixes for bugs introduced since the last release): - namesys: properly attach path in name.Resolve ([ipfs/go-ipfs#5660](https://github.com/ipfs/go-ipfs/pull/5660)) - fix(p2p): issue #5523 ([ipfs/go-ipfs#5529](https://github.com/ipfs/go-ipfs/pull/5529)) - fix infinite loop in `stats bw` ([ipfs/go-ipfs#5598](https://github.com/ipfs/go-ipfs/pull/5598)) - make warnings on no bootstrap peers less noisy ([ipfs/go-ipfs#5466](https://github.com/ipfs/go-ipfs/pull/5466)) - fix two transport related bugs ([ipfs/go-ipfs#5417](https://github.com/ipfs/go-ipfs/pull/5417)) - Fix pin ls output when hash is specified ([ipfs/go-ipfs#5699](https://github.com/ipfs/go-ipfs/pull/5699)) - ping: switch to the ping service enabled in the libp2p constructor ([ipfs/go-ipfs#5698](https://github.com/ipfs/go-ipfs/pull/5698)) - commands: fix a bunch of tiny commands-lib issues ([ipfs/go-ipfs#5697](https://github.com/ipfs/go-ipfs/pull/5697)) - cleanup the ping command ([ipfs/go-ipfs#5680](https://github.com/ipfs/go-ipfs/pull/5680)) - fix gossipsub goroutine explosion ([ipfs/go-ipfs#5688](https://github.com/ipfs/go-ipfs/pull/5688)) - fix(cmd/gc): Run func does not return error when Emit func returns error ([ipfs/go-ipfs#5687](https://github.com/ipfs/go-ipfs/pull/5687)) Extractions: - Extract bitswap to go-bitswap ([ipfs/go-ipfs#5294](https://github.com/ipfs/go-ipfs/pull/5294)) - Extract blockservice and verifcid ([ipfs/go-ipfs#5296](https://github.com/ipfs/go-ipfs/pull/5296)) - Extract merkledag package, move dagutils to top level ([ipfs/go-ipfs#5298](https://github.com/ipfs/go-ipfs/pull/5298)) - Extract path and resolver ([ipfs/go-ipfs#5306](https://github.com/ipfs/go-ipfs/pull/5306)) - Extract config package ([ipfs/go-ipfs#5277](https://github.com/ipfs/go-ipfs/pull/5277)) - Extract unixfs and importers to go-unixfs ([ipfs/go-ipfs#5316](https://github.com/ipfs/go-ipfs/pull/5316)) - delete unixfs code... ([ipfs/go-ipfs#5319](https://github.com/ipfs/go-ipfs/pull/5319)) - Extract /mfs to github.com/ipfs/go-mfs ([ipfs/go-ipfs#5391](https://github.com/ipfs/go-ipfs/pull/5391)) - re-format log output as ndjson ([ipfs/go-ipfs#5708](https://github.com/ipfs/go-ipfs/pull/5708)) - error on resolving non-terminal paths ([ipfs/go-ipfs#5705](https://github.com/ipfs/go-ipfs/pull/5705)) Documentation: - document the fact that we now publish releases on GitHub ([ipfs/go-ipfs#5301](https://github.com/ipfs/go-ipfs/pull/5301)) - docs: add url to dev weekly sync to the README ([ipfs/go-ipfs#5371](https://github.com/ipfs/go-ipfs/pull/5371)) - docs: README refresh, add cli-http-api-core diagram ([ipfs/go-ipfs#5396](https://github.com/ipfs/go-ipfs/pull/5396)) - add some basic gateway documentation ([ipfs/go-ipfs#5393](https://github.com/ipfs/go-ipfs/pull/5393)) - fix the default gateway port ([ipfs/go-ipfs#5419](https://github.com/ipfs/go-ipfs/pull/5419)) - fix order of events in the release process ([ipfs/go-ipfs#5434](https://github.com/ipfs/go-ipfs/pull/5434)) - docs: add some minimal read-only API documentation ([ipfs/go-ipfs#5437](https://github.com/ipfs/go-ipfs/pull/5437)) - feat: use package-table ([ipfs/go-ipfs#5395](https://github.com/ipfs/go-ipfs/pull/5395)) - link to go-{libp2p,ipld} package tables ([ipfs/go-ipfs#5446](https://github.com/ipfs/go-ipfs/pull/5446)) - api: fix outdated HTTPHeaders config documentation ([ipfs/go-ipfs#5451](https://github.com/ipfs/go-ipfs/pull/5451)) - add version, usage, and planning info for urlstore ([ipfs/go-ipfs#5552](https://github.com/ipfs/go-ipfs/pull/5552)) - debug-guide.md added memory statistics command ([ipfs/go-ipfs#5546](https://github.com/ipfs/go-ipfs/pull/5546)) - Change to point to combined go contributing guidelines ([ipfs/go-ipfs#5607](https://github.com/ipfs/go-ipfs/pull/5607)) - docs: Update link format ([ipfs/go-ipfs#5617](https://github.com/ipfs/go-ipfs/pull/5617)) - Fix link in readme ([ipfs/go-ipfs#5632](https://github.com/ipfs/go-ipfs/pull/5632)) - docs: add a note for dns command ([ipfs/go-ipfs#5629](https://github.com/ipfs/go-ipfs/pull/5629)) - Dockerfile: Specifies comments on exposed ports ([ipfs/go-ipfs#5615](https://github.com/ipfs/go-ipfs/pull/5615)) - document pubsub message signing ([ipfs/go-ipfs#5669](https://github.com/ipfs/go-ipfs/pull/5669)) Testing: - Include cid-fmt binary in test/bin. ([ipfs/go-ipfs#5297](https://github.com/ipfs/go-ipfs/pull/5297)) - wait for the nodes to fully stop ([ipfs/go-ipfs#5315](https://github.com/ipfs/go-ipfs/pull/5315)) - apply timeout for build steps after getting node ([ipfs/go-ipfs#5313](https://github.com/ipfs/go-ipfs/pull/5313)) - ci: check for gx deps dupes ([ipfs/go-ipfs#5338](https://github.com/ipfs/go-ipfs/pull/5338)) - ci: call cleanWs after each step ([ipfs/go-ipfs#5374](https://github.com/ipfs/go-ipfs/pull/5374)) - add correct test for GC completeness ([ipfs/go-ipfs#5364](https://github.com/ipfs/go-ipfs/pull/5364)) - fix the urlstore tests ([ipfs/go-ipfs#5397](https://github.com/ipfs/go-ipfs/pull/5397)) - improve gateway options test ([ipfs/go-ipfs#5433](https://github.com/ipfs/go-ipfs/pull/5433)) - coreapi name: Increase test swarm size ([ipfs/go-ipfs#5481](https://github.com/ipfs/go-ipfs/pull/5481)) - fix fuse unmount test ([ipfs/go-ipfs#5476](https://github.com/ipfs/go-ipfs/pull/5476)) - test(add): add test for issue \#5456 ([ipfs/go-ipfs#5493](https://github.com/ipfs/go-ipfs/pull/5493)) - fixed tests of raised fd limits ([ipfs/go-ipfs#5496](https://github.com/ipfs/go-ipfs/pull/5496)) - pprof: create HTTP endpoint for setting MutexProfileFraction ([ipfs/go-ipfs#5527](https://github.com/ipfs/go-ipfs/pull/5527)) - fix(command):update `add --chunker` test ([ipfs/go-ipfs#5571](https://github.com/ipfs/go-ipfs/pull/5571)) - switch to go 1.11 ([ipfs/go-ipfs#5483](https://github.com/ipfs/go-ipfs/pull/5483)) - fix: sharness race in directory_size if file is removed ([ipfs/go-ipfs#5586](https://github.com/ipfs/go-ipfs/pull/5586)) - Bump Go versions and use '.x' to always get latest minor versions ([ipfs/go-ipfs#5682](https://github.com/ipfs/go-ipfs/pull/5682)) - add rabin min error test ([ipfs/go-ipfs#5449](https://github.com/ipfs/go-ipfs/pull/5449)) - Use CircleCI 2.0 ([ipfs/go-ipfs#5691](https://github.com/ipfs/go-ipfs/pull/5691)) Internal: - Add ability to retrieve blocks even if given using a different CID version ([ipfs/go-ipfs#5285](https://github.com/ipfs/go-ipfs/pull/5285)) - update gogo-protobuf ([ipfs/go-ipfs#5355](https://github.com/ipfs/go-ipfs/pull/5355)) - update protobuf files in go-ipfs ([ipfs/go-ipfs#5356](https://github.com/ipfs/go-ipfs/pull/5356)) - string-backed CIDs ([ipfs/go-ipfs#5441](https://github.com/ipfs/go-ipfs/pull/5441)) - commands: switch object to CoreAPI ([ipfs/go-ipfs#4643](https://github.com/ipfs/go-ipfs/pull/4643)) - coreapi: dag: Batching interface ([ipfs/go-ipfs#5340](https://github.com/ipfs/go-ipfs/pull/5340)) - key cmd: Refactor to use coreapi ([ipfs/go-ipfs#5339](https://github.com/ipfs/go-ipfs/pull/5339)) - coreapi: DHT API ([ipfs/go-ipfs#4804](https://github.com/ipfs/go-ipfs/pull/4804)) - block cmd: Use coreapi ([ipfs/go-ipfs#5331](https://github.com/ipfs/go-ipfs/pull/5331)) - mk: embed CurrentCommit in the right place ([ipfs/go-ipfs#5507](https://github.com/ipfs/go-ipfs/pull/5507)) - added binary executable files to .dockerignore ([ipfs/go-ipfs#5544](https://github.com/ipfs/go-ipfs/pull/5544)) - Add sessions when fetching MerkleDAG in LS ([ipfs/go-ipfs#5509](https://github.com/ipfs/go-ipfs/pull/5509)) - coreapi: Swarm API ([ipfs/go-ipfs#4803](https://github.com/ipfs/go-ipfs/pull/4803)) - coreapi swarm: unify impl type with other apis ([ipfs/go-ipfs#5551](https://github.com/ipfs/go-ipfs/pull/5551)) - Refactor UnixFS CoreAPI ([ipfs/go-ipfs#5501](https://github.com/ipfs/go-ipfs/pull/5501)) - coreapi: PubSub API ([ipfs/go-ipfs#4805](https://github.com/ipfs/go-ipfs/pull/4805)) - fix: maketarball.sh for OSX ([ipfs/go-ipfs#5575](https://github.com/ipfs/go-ipfs/pull/5575)) - test the correct return value when checking directory size ([ipfs/go-ipfs#5580](https://github.com/ipfs/go-ipfs/pull/5580)) - coreapi unixfs: remove Cat ([ipfs/go-ipfs#5574](https://github.com/ipfs/go-ipfs/pull/5574)) - Explicitly use BufferedDAG after removing Batch from importers ([ipfs/go-ipfs#5626](https://github.com/ipfs/go-ipfs/pull/5626)) Cleanup: - Fix some weird code in core/coreunix/add.go ([ipfs/go-ipfs#5354](https://github.com/ipfs/go-ipfs/pull/5354)) - name cmd: move subcommands to subdirectory ([ipfs/go-ipfs#5392](https://github.com/ipfs/go-ipfs/pull/5392)) - directly parse peer IDs as peer IDs ([ipfs/go-ipfs#5409](https://github.com/ipfs/go-ipfs/pull/5409)) - don't bother caching if we're using a nil repo ([ipfs/go-ipfs#5414](https://github.com/ipfs/go-ipfs/pull/5414)) - object:refactor data encode error ([ipfs/go-ipfs#5426](https://github.com/ipfs/go-ipfs/pull/5426)) - remove Godeps ([ipfs/go-ipfs#5440](https://github.com/ipfs/go-ipfs/pull/5440)) - update for the go-ipfs-cmds refactor ([ipfs/go-ipfs#5035](https://github.com/ipfs/go-ipfs/pull/5035)) - fix(unixfs): issue #5217 (Avoid use of `pb.Data`) ([ipfs/go-ipfs#5505](https://github.com/ipfs/go-ipfs/pull/5505)) - fix(unixfs): issue #5055 ([ipfs/go-ipfs#5525](https://github.com/ipfs/go-ipfs/pull/5525)) - add offline id test #4978 and refactor command code ([ipfs/go-ipfs#5562](https://github.com/ipfs/go-ipfs/pull/5562)) - refact(command): replace option name with const string ([ipfs/go-ipfs#5642](https://github.com/ipfs/go-ipfs/pull/5642)) - remove p2p-circuit addr hack in ipfs swarm peers ([ipfs/go-ipfs#5645](https://github.com/ipfs/go-ipfs/pull/5645)) - refactor(commands/id): use new command ([ipfs/go-ipfs#5646](https://github.com/ipfs/go-ipfs/pull/5646)) - object patch rm-link: change arg from 'link' to 'name' ([ipfs/go-ipfs#5638](https://github.com/ipfs/go-ipfs/pull/5638)) - refactor(cmds): use new cmds lib in version, tar and dns ([ipfs/go-ipfs#5650](https://github.com/ipfs/go-ipfs/pull/5650)) - cmds/dag: use new cmds lib ([ipfs/go-ipfs#5662](https://github.com/ipfs/go-ipfs/pull/5662)) - commands/ping: use new cmds lib ([ipfs/go-ipfs#5675](https://github.com/ipfs/go-ipfs/pull/5675)) ### related changelogs Changes to sub-packages go-ipfs depends on. This *does not* include libp2p or multiformats. github.com/ipfs/go-log - update gogo protobuf ([ipfs/go-log#39](https://github.com/ipfs/go-log/pull/39)) - rename the protobuf to loggabletracer ([ipfs/go-log#41](https://github.com/ipfs/go-log/pull/41)) - protect loggers with rwmutex ([ipfs/go-log#44](https://github.com/ipfs/go-log/pull/44)) - make logging prettier ([ipfs/go-log#45](https://github.com/ipfs/go-log/pull/45)) - add env vars for logging to file and syslog ([ipfs/go-log#46](https://github.com/ipfs/go-log/pull/46)) - remove syslogger ([ipfs/go-log#47](https://github.com/ipfs/go-log/pull/47)) github.com/ipfs/go-datastore - implement DiskUsage for the rest of the datastores ([ipfs/go-datastore#86](https://github.com/ipfs/go-datastore/pull/86)) - switch to google's uuid library ([ipfs/go-datastore#89](https://github.com/ipfs/go-datastore/pull/89)) - return ErrNotFound from the NullDatastore instead of nil, nil ([ipfs/go-datastore#92](https://github.com/ipfs/go-datastore/pull/92)) - Add TTL and Transactional interfaces ([ipfs/go-datastore#91](https://github.com/ipfs/go-datastore/pull/91)) - improve testing ([ipfs/go-datastore#93](https://github.com/ipfs/go-datastore/pull/93)) - Add support for querying entry expiration ([ipfs/go-datastore#96](https://github.com/ipfs/go-datastore/pull/96)) - Allow ds.NewTransaction() to return an error ([ipfs/go-datastore#98](https://github.com/ipfs/go-datastore/pull/98)) - add a GetSize method ([ipfs/go-datastore#99](https://github.com/ipfs/go-datastore/pull/99)) github.com/ipfs/go-cid - Add tests for Set type ([ipfs/go-cid#63](https://github.com/ipfs/go-cid/pull/63)) - Create new Builder interface for creating CIDs. ([ipfs/go-cid#53](https://github.com/ipfs/go-cid/pull/53)) - cid-fmt enhancements ([ipfs/go-cid#61](https://github.com/ipfs/go-cid/pull/61)) - add String benchmark ([ipfs/go-cid#44](https://github.com/ipfs/go-cid/pull/44)) - add a streaming CID set ([ipfs/go-cid#67](https://github.com/ipfs/go-cid/pull/67)) - Extract non-core functionality from go-cid into go-cidutil ([ipfs/go-cid#69](https://github.com/ipfs/go-cid/pull/69)) - cid implementation research ([ipfs/go-cid#70](https://github.com/ipfs/go-cid/pull/70)) - cid implementation variations++ ([ipfs/go-cid#72](https://github.com/ipfs/go-cid/pull/72)) - Create a new Encode method that is like StringOfBase but never errors ([ipfs/go-cid#60](https://github.com/ipfs/go-cid/pull/60)) - add codecs for Dash blocks, tx ([ipfs/go-cid#78](https://github.com/ipfs/go-cid/pull/78)) github.com/ipfs/go-ds-flatfs - check error before defer-removing disk usage file ([ipfs/go-ds-flatfs#47](https://github.com/ipfs/go-ds-flatfs/pull/47)) - add GetSize function ([ipfs/go-ds-flatfs#48](https://github.com/ipfs/go-ds-flatfs/pull/48)) github.com/ipfs/go-ds-measure - ([ipfs/go-ds-measure#](https://github.com/ipfs/go-ds-measure/pull/)) github.com/ipfs/go-ds-leveldb - recover datastore on corruption ([ipfs/go-ds-leveldb#15](https://github.com/ipfs/go-ds-leveldb/pull/15)) - Add transactional support to leveldb datastore. ([ipfs/go-ds-leveldb#17](https://github.com/ipfs/go-ds-leveldb/pull/17)) - implement GetSize ([ipfs/go-ds-leveldb#18](https://github.com/ipfs/go-ds-leveldb/pull/18)) github.com/ipfs/go-metrics-prometheus - use an existing metric when it has already been registered ([ipfs/go-metrics-prometheus#1](https://github.com/ipfs/go-metrics-prometheus/pull/1)) github.com/ipfs/go-metrics-interface - update the counter interface to match Prometheus ([ipfs/go-metrics-interface#2](https://github.com/ipfs/go-metrics-interface/pull/2)) github.com/ipfs/go-ipld-format - add copy dagservice function ([ipfs/go-ipld-format#41](https://github.com/ipfs/go-ipld-format/pull/41)) github.com/ipfs/go-ipld-cbor - Refactor to refmt ([ipfs/go-ipld-cbor#30](https://github.com/ipfs/go-ipld-cbor/pull/30)) - import changes from the filecoin branch ([ipfs/go-ipld-cbor#41](https://github.com/ipfs/go-ipld-cbor/pull/41)) - register the BitIntAtlasEntry for the tests ([ipfs/go-ipld-cbor#43](https://github.com/ipfs/go-ipld-cbor/pull/43)) - attempt to allocate a bit less ([ipfs/go-ipld-cbor#45](https://github.com/ipfs/go-ipld-cbor/pull/45)) github.com/ipfs/go-ipfs-cmds - check if we can decode an error before trying ([ipfs/go-ipfs-cmds#108](https://github.com/ipfs/go-ipfs-cmds/pull/108)) - fix(option): print error message for error timeout option ([ipfs/go-ipfs-cmds#118](https://github.com/ipfs/go-ipfs-cmds/pull/118)) - Create Jenkinsfile ([ipfs/go-ipfs-cmds#89](https://github.com/ipfs/go-ipfs-cmds/pull/89)) - fix(add): refer to ipfs issue #5456 ([ipfs/go-ipfs-cmds#121](https://github.com/ipfs/go-ipfs-cmds/pull/121)) - commands refactor 2.0 ([ipfs/go-ipfs-cmds#112](https://github.com/ipfs/go-ipfs-cmds/pull/112)) - always assign keks to review new PRs ([ipfs/go-ipfs-cmds#123](https://github.com/ipfs/go-ipfs-cmds/pull/123)) - extract go-ipfs-files ([ipfs/go-ipfs-cmds#125](https://github.com/ipfs/go-ipfs-cmds/pull/125)) - split the value encoder and the error encoder ([ipfs/go-ipfs-cmds#128](https://github.com/ipfs/go-ipfs-cmds/pull/128)) github.com/ipfs/go-ipfs-cmdkit - all: gofmt ([ipfs/go-ipfs-cmdkit#22](https://github.com/ipfs/go-ipfs-cmdkit/pull/22)) - add standard ci scripts ([ipfs/go-ipfs-cmdkit#23](https://github.com/ipfs/go-ipfs-cmdkit/pull/23)) - only count size for regular files ([ipfs/go-ipfs-cmdkit#25](https://github.com/ipfs/go-ipfs-cmdkit/pull/25)) - Create Jenkinsfile ([ipfs/go-ipfs-cmdkit#16](https://github.com/ipfs/go-ipfs-cmdkit/pull/16)) - Feat: add WebFile File implementation. ([ipfs/go-ipfs-cmdkit#26](https://github.com/ipfs/go-ipfs-cmdkit/pull/26)) - feat(type): fix issue #28 ([ipfs/go-ipfs-cmdkit#29](https://github.com/ipfs/go-ipfs-cmdkit/pull/29)) - Extract files package ([ipfs/go-ipfs-cmdkit#31](https://github.com/ipfs/go-ipfs-cmdkit/pull/31)) github.com/ipfs/go-ds-badger - update protobuf ([ipfs/go-ds-badger#26](https://github.com/ipfs/go-ds-badger/pull/26)) - exported type datastore => Datastore ([ipfs/go-ds-badger#1](https://github.com/ipfs/go-ds-badger/pull/1)) - using exported Datastore type ([ipfs/go-ds-badger#2](https://github.com/ipfs/go-ds-badger/pull/2)) - exported type datastore => Datastore ([ipfs/go-ds-badger#28](https://github.com/ipfs/go-ds-badger/pull/28)) - Implement new TxDatastore and Txn interfaces ([ipfs/go-ds-badger#27](https://github.com/ipfs/go-ds-badger/pull/27)) - Avoid discarding transaction too early in queries ([ipfs/go-ds-badger#31](https://github.com/ipfs/go-ds-badger/pull/31)) - Ability to get entry expirations ([ipfs/go-ds-badger#32](https://github.com/ipfs/go-ds-badger/pull/32)) - Update badger to 2.8.0 ([ipfs/go-ds-badger#33](https://github.com/ipfs/go-ds-badger/pull/33)) - ds.NewTransaction() now returns an error parameter ([ipfs/go-ds-badger#36](https://github.com/ipfs/go-ds-badger/pull/36)) - make has faster ([ipfs/go-ds-badger#37](https://github.com/ipfs/go-ds-badger/pull/37)) - Implement GetSize and update badger ([ipfs/go-ds-badger#38](https://github.com/ipfs/go-ds-badger/pull/38)) github.com/ipfs/go-ipfs-addr - Remove dependency on libp2p-circuit ([ipfs/go-ipfs-addr#7](https://github.com/ipfs/go-ipfs-addr/pull/7)) github.com/ipfs/go-ipfs-chunker - return err when rabin min less than 16 ([ipfs/go-ipfs-chunker#3](https://github.com/ipfs/go-ipfs-chunker/pull/3)) - switch to go-buffer-pool ([ipfs/go-ipfs-chunker#8](https://github.com/ipfs/go-ipfs-chunker/pull/8)) - fix size-0 chunker bug ([ipfs/go-ipfs-chunker#9](https://github.com/ipfs/go-ipfs-chunker/pull/9)) github.com/ipfs/go-ipfs-routing - update protobuf ([ipfs/go-ipfs-routing#8](https://github.com/ipfs/go-ipfs-routing/pull/8)) - Implement SearchValue ([ipfs/go-ipfs-routing#12](https://github.com/ipfs/go-ipfs-routing/pull/12)) github.com/ipfs/go-ipfs-blockstore - blockstore: Adding Stat method to map from Cid to BlockSize ([ipfs/go-ipfs-blockstore#5](https://github.com/ipfs/go-ipfs-blockstore/pull/5)) - correctly convert the datastore not found errors ([ipfs/go-ipfs-blockstore#10](https://github.com/ipfs/go-ipfs-blockstore/pull/10)) - Fix typo: Change 'should not' to 'should' ([ipfs/go-ipfs-blockstore#14](https://github.com/ipfs/go-ipfs-blockstore/pull/14)) - fix test race condition ([ipfs/go-ipfs-blockstore#9](https://github.com/ipfs/go-ipfs-blockstore/pull/9)) - make arccache.GetSize return ErrNotFound when not found ([ipfs/go-ipfs-blockstore#16](https://github.com/ipfs/go-ipfs-blockstore/pull/16)) - use datastore.GetSize ([ipfs/go-ipfs-blockstore#17](https://github.com/ipfs/go-ipfs-blockstore/pull/17)) github.com/ipfs/go-ipns - update gogo protobuf ([ipfs/go-ipns#16](https://github.com/ipfs/go-ipns/pull/16)) - use new ExtractPublicKey signature ([ipfs/go-ipns#17](https://github.com/ipfs/go-ipns/pull/17)) github.com/ipfs/go-bitswap - update gogo protobuf ([ipfs/go-bitswap#2](https://github.com/ipfs/go-bitswap/pull/2)) - ci: add jenkins ([ipfs/go-bitswap#9](https://github.com/ipfs/go-bitswap/pull/9)) - bitswap: Bitswap now sends multiple blocks per message ([ipfs/go-bitswap#5](https://github.com/ipfs/go-bitswap/pull/5)) - reduce allocations ([ipfs/go-bitswap#12](https://github.com/ipfs/go-bitswap/pull/12)) - buffer writes ([ipfs/go-bitswap#15](https://github.com/ipfs/go-bitswap/pull/15)) - delay finding providers ([ipfs/go-bitswap#17](https://github.com/ipfs/go-bitswap/pull/17)) github.com/ipfs/go-blockservice - Avoid allocating a session unless we need it ([ipfs/go-blockservice#6](https://github.com/ipfs/go-blockservice/pull/6)) github.com/ipfs/go-cidutil - add a utility method for sorting CID slices ([ipfs/go-cidutil#5](https://github.com/ipfs/go-cidutil/pull/5)) github.com/ipfs/go-ipfs-config - Add pubsub configuration options ([ipfs/go-ipfs-config#3](https://github.com/ipfs/go-ipfs-config/pull/3)) - add QUIC experiment ([ipfs/go-ipfs-config#4](https://github.com/ipfs/go-ipfs-config/pull/4)) - Add Gateway.APICommands for /api allowlists ([ipfs/go-ipfs-config#10](https://github.com/ipfs/go-ipfs-config/pull/10)) - allow multiple API/Gateway addresses ([ipfs/go-ipfs-config#11](https://github.com/ipfs/go-ipfs-config/pull/11)) - Fix handling of null strings ([ipfs/go-ipfs-config#12](https://github.com/ipfs/go-ipfs-config/pull/12)) - add experiment for p2p http proxy ([ipfs/go-ipfs-config#13](https://github.com/ipfs/go-ipfs-config/pull/13)) - add message signing config options ([ipfs/go-ipfs-config#18](https://github.com/ipfs/go-ipfs-config/pull/18)) github.com/ipfs/go-merkledag - Add FetchGraphWithDepthLimit to specify depth-limited graph fetching. ([ipfs/go-merkledag#2](https://github.com/ipfs/go-merkledag/pull/2)) - update gogo protobuf ([ipfs/go-merkledag#4](https://github.com/ipfs/go-merkledag/pull/4)) - Update to use new Builder interface for creating CIDs. ([ipfs/go-merkledag#6](https://github.com/ipfs/go-merkledag/pull/6)) - perf: avoid allocations when filtering nodes ([ipfs/go-merkledag#11](https://github.com/ipfs/go-merkledag/pull/11)) github.com/ipfs/go-mfs - fix(unixfs): issue #6 ([ipfs/go-mfs#7](https://github.com/ipfs/go-mfs/pull/7)) - fix(type): issue #13 ([ipfs/go-mfs#14](https://github.com/ipfs/go-mfs/pull/14)) github.com/ipfs/go-path - fix: don't dag.Get in ResolveToLastNode when not needed ([ipfs/go-path#1](https://github.com/ipfs/go-path/pull/1)) github.com/ipfs/go-unixfs - update gogo protobuf ([ipfs/go-unixfs#6](https://github.com/ipfs/go-unixfs/pull/6)) - Update to use new Builder interface for creating CIDs. ([ipfs/go-unixfs#7](https://github.com/ipfs/go-unixfs/pull/7)) - nit: make dagTruncate a method on DagModifier ([ipfs/go-unixfs#13](https://github.com/ipfs/go-unixfs/pull/13)) - fix(fsnode): issue #17 ([ipfs/go-unixfs#18](https://github.com/ipfs/go-unixfs/pull/18)) - Use EnumerateChildrenAsync in for enumerating HAMT links ([ipfs/go-unixfs#19](https://github.com/ipfs/go-unixfs/pull/19)) ## v0.4.17 2018-07-27 Ipfs 0.4.17 is a quick release to fix a major performance regression in bitswap (mostly affecting go-ipfs -> js-ipfs transfers). However, while motivated by this fix, this release contains a few other goodies that will excite some users. The headline feature in this release is [urlstore][] support. Urlstore is a generalization of the filestore backend that can fetch file blocks from remote URLs on-demand instead of storing them in the local datastore. Additionally, we've added support for extracting inline blocks from CIDs (blocks inlined into CIDs using the identity hash function). However, go-ipfs won't yet *create* such CIDs so you're unlikely to see any in the wild. [urlstore]: https://github.com/ipfs/go-ipfs/blob/master/docs/experimental-features.md#ipfs-urlstore Features: * URLStore ([ipfs/go-ipfs#4896](https://github.com/ipfs/go-ipfs/pull/4896)) * Add trickle-dag support to the urlstore ([ipfs/go-ipfs#5245](https://github.com/ipfs/go-ipfs/pull/5245)). * Allow specifying how the data field in the `object get` is encoded ([ipfs/go-ipfs#5139](https://github.com/ipfs/go-ipfs/pull/5139)) * Add a `-U` flag to `files ls` to disable sorting ([ipfs/go-ipfs#5219](https://github.com/ipfs/go-ipfs/pull/5219)) * Add an efficient `--size-only` flag to the `repo stat` ([ipfs/go-ipfs#5010](https://github.com/ipfs/go-ipfs/pull/5010)) * Inline blocks in CIDs ([ipfs/go-ipfs#5117](https://github.com/ipfs/go-ipfs/pull/5117)) Changes/Fixes: * Make `ipfs files ls -l` correctly report the hash and size of files ([ipfs/go-ipfs#5045](https://github.com/ipfs/go-ipfs/pull/5045)) * Fix sorting of `files ls` ([ipfs/go-ipfs#5219](https://github.com/ipfs/go-ipfs/pull/5219)) * Improve prefetching in `ipfs cat` and related commands ([ipfs/go-ipfs#5162](https://github.com/ipfs/go-ipfs/pull/5162)) * Better error message when `ipfs cp` fails ([ipfs/go-ipfs#5218](https://github.com/ipfs/go-ipfs/pull/5218)) * Don't wait for the peer to close it's end of a bitswap stream before considering the block "sent" ([ipfs/go-ipfs#5258](https://github.com/ipfs/go-ipfs/pull/5258)) * Fix resolving links in sharded directories via the gateway ([ipfs/go-ipfs#5271](https://github.com/ipfs/go-ipfs/pull/5271)) * Fix building when there's a space in the current directory ([ipfs/go-ipfs#5261](https://github.com/ipfs/go-ipfs/pull/5261)) Documentation: * Improve documentation about the bloomfilter config options ([ipfs/go-ipfs#4924](https://github.com/ipfs/go-ipfs/pull/4924)) General refactorings and internal bug fixes: * Remove the `Offset()` method from the DAGReader ([ipfs/go-ipfs#5190](https://github.com/ipfs/go-ipfs/pull/5190)) * Fix TestLargeWriteChunks seek behavior ([ipfs/go-ipfs#5276](https://github.com/ipfs/go-ipfs/pull/5276)) * Add a build tag to disable dynamic plugins ([ipfs/go-ipfs#5274](https://github.com/ipfs/go-ipfs/pull/5274)) * Use FSNode instead of the Protobuf structure in PBDagReader ([ipfs/go-ipfs#5189](https://github.com/ipfs/go-ipfs/pull/5189)) * Remove support for non-directory MFS roots ([ipfs/go-ipfs#5170](https://github.com/ipfs/go-ipfs/pull/5170)) * Remove `UnixfsNode` from the balanced builder ([ipfs/go-ipfs#5118](https://github.com/ipfs/go-ipfs/pull/5118)) * Fix truncating files (internal) when already at the correct size ([ipfs/go-ipfs#5253](https://github.com/ipfs/go-ipfs/pull/5253)) * Fix `dagTruncate` (internal) to preserve the node type ([ipfs/go-ipfs#5216](https://github.com/ipfs/go-ipfs/pull/5216)) * Add an internal interface for unixfs directories ([ipfs/go-ipfs#5160](https://github.com/ipfs/go-ipfs/pull/5160)) * Refactor the CoreAPI path types and interfaces ([ipfs/go-ipfs#4672](https://github.com/ipfs/go-ipfs/pull/4672)) * Refactor `precalcNextBuf` in the dag reader ([ipfs/go-ipfs#5237](https://github.com/ipfs/go-ipfs/pull/5237)) * Update a bunch of dependencies that haven't been updated for a while ([ipfs/go-ipfs#5268](https://github.com/ipfs/go-ipfs/pull/5268)) ## v0.4.16 2018-07-13 Ipfs 0.4.16 is a fairly small release in terms of changes to the ipfs codebase, but it contains a huge amount of changes and improvements from the libraries we depend on, notably libp2p. This release includes small a repo migration to account for some changes to the DHT. It should only take a second to run but, depending on your configuration, you may need to run it manually. You can run a migration by either: 1. Selecting "Yes" when the daemon prompts you to migrate. 2. Running the daemon with the `--migrate=true` flag. 3. Manually [running](https://github.com/ipfs/fs-repo-migrations/blob/master/run.md#running-repo-migrations) the migration. ### Libp2p This version of ipfs contains the changes made in libp2p from v5.0.14 through v6.0.5. In that time, we have made significant changes to the codebase to allow for easier integration of future transports and modules along with the usual performance and reliability improvements. You can find many of these improvements in the libp2p 6.0 [release blog post](https://ipfs.io/blog/39-go-libp2p-6-0-0/). The primary motivation for this refactor was adding support for network transports like QUIC that have built-in support for encryption, authentication, and stream multiplexing. It will also allow us to plug-in new security transports (like TLS) without hard-coding them. For example, our [QUIC transport](https://github.com/libp2p/go-libp2p-quic-transport) currently works, and can be plugged into libp2p manually (though note that it is still experimental, as the upstream spec is still in flux). Further work is needed to make enabling this inside ipfs easy and not require recompilation. On the user-visible side of things, we've improved our dialing logic and timeouts. We now abort dials to local subnets after 5 seconds and abort all dials if the TCP handshake takes longer than 5 seconds. This should significantly improve performance in some cases as we limit the number of concurrent dials and slow dials to non-responsive peers have been known to clog the dialer, blocking dials to reachable peers. Importantly, this should improve DHT performance as it tends to spend a disproportional amount of time connecting to peers. We have also made a few noticeable changes to the DHT: we've significantly improved the chances of finding a value on the DHT, tightened up some of our validation logic, and fixed some issues that should reduce traffic to nodes running in dhtclient mode over time. Of these, the first one will likely see the most impact. In the past, when putting a value (e.g., an IPNS entry) into the DHT, we'd try to put the value to K peers (where K for us is 20). However, we'd often fail to connect to many of these peers so we'd end up putting the value to significantly fewer than K peers. We now try to put the value to the K peers we can actually connect to. Finally, we've fixed JavaScript interoperability in go-multiplex, the one stream muxer that both go-libp2p and js-libp2p implement. This should significantly improve go-libp2p and js-libp2p interoperability. ### Multiformats We are also changing the way that people write 'ipfs' multiaddrs. Currently, ipfs multiaddrs look something like `/ip4/104.131.131.82/tcp/4001/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ`. However, calling them 'ipfs' multiaddrs is a bit misleading, as this is actually the multiaddr of a libp2p peer that happens to run ipfs. Other protocols built on libp2p right now still have to use multiaddrs that say 'ipfs', even if they have nothing to do with ipfs. Therefore, we are renaming them to 'p2p' multiaddrs. Moving forward, these addresses will be written as: `/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ`. This release adds support for *parsing* both types of addresses (`.../ipfs/...` and `.../p2p/...`) into the same network format, and the network format is remaining exactly the same. A future release will have the ipfs daemon switch to *printing* out addresses this way once a large enough portion of the network has upgraded. N.B., these addresses are *not* related to IPFS *file* names (`/ipfs/Qm...`). Disambiguating the two was yet another motivation to switch the protocol name to `/p2p/`. ### IPFS On the ipfs side of things, we've started embedding public keys inside IPNS records and have enabled the Git plugin by default. Embedding public keys inside IPNS records allows lookups to be faster as we only need to fetch the record itself (and not the public key separately). It also fixes an issue where DHT peers wouldn't store a record for a peer if they didn't have their public key already. Combined with some of the DHT and dialing fixes, this should improve the performance of IPNS (once a majority of the network updates). Second, our public builds now include the Git plugin (in past builds, you could add it yourself, but doing so was not easy). With this, ipfs can ingest and operate over Git repositories and commit graphs directly. For more information on this, see [the go-ipld-git repo](https://github.com/ipfs/go-ipld-git). Finally, we've included many smaller bug fixes, refactorings, improved documentation, and a good bit more. For the full details, see the changelog below. ## v0.4.16-rc3 2018-07-09 - Bug fixes - Fix dht commands when ipns over pubsub is enabled ([ipfs/go-ipfs#5200](https://github.com/ipfs/go-ipfs/pull/5200)) - Fix content routing when ipns over pubsub is enabled ([ipfs/go-ipfs#5200](https://github.com/ipfs/go-ipfs/pull/5200)) - Correctly handle multi-hop dnslink resolution ([ipfs/go-ipfs#5202](https://github.com/ipfs/go-ipfs/pull/5202)) ## v0.4.16-rc2 2018-07-05 - Bug fixes - Fix usage of file name vs path name in adder ([ipfs/go-ipfs#5167](https://github.com/ipfs/go-ipfs/pull/5167)) - Fix `ipfs update` working with migrations ([ipfs/go-ipfs#5194](https://github.com/ipfs/go-ipfs/pull/5194)) - Documentation - Grammar fix in fuse docs ([ipfs/go-ipfs#5164](https://github.com/ipfs/go-ipfs/pull/5164)) ## v0.4.16-rc1 2018-06-27 - Features - Embed public keys inside ipns records, use for validation ([ipfs/go-ipfs#5079](https://github.com/ipfs/go-ipfs/pull/5079)) - Preload git plugin by default ([ipfs/go-ipfs#4991](https://github.com/ipfs/go-ipfs/pull/4991)) - Improvements - Only resolve dnslinks once in the gateway ([ipfs/go-ipfs#4977](https://github.com/ipfs/go-ipfs/pull/4977)) - Libp2p transport refactor update ([ipfs/go-ipfs#4817](https://github.com/ipfs/go-ipfs/pull/4817)) - Improve swarm connect/disconnect commands ([ipfs/go-ipfs#5107](https://github.com/ipfs/go-ipfs/pull/5107)) - Documentation - Fix typo of sudo install command ([ipfs/go-ipfs#5001](https://github.com/ipfs/go-ipfs/pull/5001)) - Fix experimental features Table of Contents ([ipfs/go-ipfs#4976](https://github.com/ipfs/go-ipfs/pull/4976)) - Fix link to systemd init scripts in the README ([ipfs/go-ipfs#4968](https://github.com/ipfs/go-ipfs/pull/4968)) - Add package overview comments to coreapi ([ipfs/go-ipfs#5108](https://github.com/ipfs/go-ipfs/pull/5108)) - Add README to docs folder ([ipfs/go-ipfs#5095](https://github.com/ipfs/go-ipfs/pull/5095)) - Add system requirements to README ([ipfs/go-ipfs#5137](https://github.com/ipfs/go-ipfs/pull/5137)) - Bug fixes - Fix goroutine leak in pin verify ([ipfs/go-ipfs#5011](https://github.com/ipfs/go-ipfs/pull/5011)) - Fix commit string in version ([ipfs/go-ipfs#4982](https://github.com/ipfs/go-ipfs/pull/4982)) - Fix `key rename` command output error ([ipfs/go-ipfs#4962](https://github.com/ipfs/go-ipfs/pull/4962)) - Report error source when failing to construct private network ([ipfs/go-ipfs#4952](https://github.com/ipfs/go-ipfs/pull/4952)) - Fix build on DragonFlyBSD ([ipfs/go-ipfs#5031](https://github.com/ipfs/go-ipfs/pull/5031)) - Fix goroutine leak in dag put ([ipfs/go-ipfs#5016](https://github.com/ipfs/go-ipfs/pull/5016)) - Fix goroutine leaks in refs.go ([ipfs/go-ipfs#5018](https://github.com/ipfs/go-ipfs/pull/5018)) - Fix panic, Don't handle errors with fallthrough ([ipfs/go-ipfs#5072](https://github.com/ipfs/go-ipfs/pull/5072)) - Fix how filestore is hooked up with caching ([ipfs/go-ipfs#5122](https://github.com/ipfs/go-ipfs/pull/5122)) - Add record validation to offline routing ([ipfs/go-ipfs#5116](https://github.com/ipfs/go-ipfs/pull/5116)) - Fix `ipfs update` working with migrations ([ipfs/go-ipfs#5194](https://github.com/ipfs/go-ipfs/pull/5194)) - General Changes and Refactorings - Remove leftover bits of dead code ([ipfs/go-ipfs#5022](https://github.com/ipfs/go-ipfs/pull/5022)) - Remove fuse platform build constraints ([ipfs/go-ipfs#5033](https://github.com/ipfs/go-ipfs/pull/5033)) - Warning when legacy NoSync setting is set ([ipfs/go-ipfs#5036](https://github.com/ipfs/go-ipfs/pull/5036)) - Clean up and refactor namesys module ([ipfs/go-ipfs#5007](https://github.com/ipfs/go-ipfs/pull/5007)) - When raw-leaves are used for empty files use 'Raw' nodes ([ipfs/go-ipfs#4693](https://github.com/ipfs/go-ipfs/pull/4693)) - Update dist_root in build scripts ([ipfs/go-ipfs#5093](https://github.com/ipfs/go-ipfs/pull/5093)) - Integrate `pb.Data` into `FSNode` to avoid duplicating fields ([ipfs/go-ipfs#5098](https://github.com/ipfs/go-ipfs/pull/5098)) - Reduce log level when we can't republish ([ipfs/go-ipfs#5091](https://github.com/ipfs/go-ipfs/pull/5091)) - Extract ipns record logic to go-ipns ([ipfs/go-ipfs#5124](https://github.com/ipfs/go-ipfs/pull/5124)) - Testing - Collect test times for sharness ([ipfs/go-ipfs#4959](https://github.com/ipfs/go-ipfs/pull/4959)) - Fix sharness iptb connect timeout ([ipfs/go-ipfs#4966](https://github.com/ipfs/go-ipfs/pull/4966)) - Add more timeouts to the jenkins pipeline ([ipfs/go-ipfs#4958](https://github.com/ipfs/go-ipfs/pull/4958)) - Use go 1.10 on jenkins ([ipfs/go-ipfs#5009](https://github.com/ipfs/go-ipfs/pull/5009)) - Speed up multinode sharness test ([ipfs/go-ipfs#4967](https://github.com/ipfs/go-ipfs/pull/4967)) - Print out iptb logs on iptb test failure (for debugging CI) ([ipfs/go-ipfs#5069](https://github.com/ipfs/go-ipfs/pull/5069)) - Disable the MacOS tests in jenkins ([ipfs/go-ipfs#5119](https://github.com/ipfs/go-ipfs/pull/5119)) - Make republisher test robust against timing issues ([ipfs/go-ipfs#5125](https://github.com/ipfs/go-ipfs/pull/5125)) - Archive sharness trash dirs in jenkins ([ipfs/go-ipfs#5071](https://github.com/ipfs/go-ipfs/pull/5071)) - Fix up DHT sharness tests ([ipfs/go-ipfs#5114](https://github.com/ipfs/go-ipfs/pull/5114)) - Dependencies - Update go-ipld-git to fix mergetag resolving ([ipfs/go-ipfs#4988](https://github.com/ipfs/go-ipfs/pull/4988)) - Fix duplicate /x/sys imports ([ipfs/go-ipfs#5068](https://github.com/ipfs/go-ipfs/pull/5068)) - Update stream multiplexers ([ipfs/go-ipfs#5075](https://github.com/ipfs/go-ipfs/pull/5075)) - Update dependencies: go-log, sys, go-crypto ([ipfs/go-ipfs#5100](https://github.com/ipfs/go-ipfs/pull/5100)) - Explicitly import go-multiaddr-dns in config/bootstrap_peers ([ipfs/go-ipfs#5144](https://github.com/ipfs/go-ipfs/pull/5144)) - Gx update with dht and dialing improvements ([ipfs/go-ipfs#5158](https://github.com/ipfs/go-ipfs/pull/5158)) ## v0.4.15 2018-05-09 This release is significantly smaller than the last as much of the work on improving our datastores, and other libraries libp2p has yet to be merged. However, it still includes many welcome improvements. As with 0.4.12 and 0.4.14 (0.4.13 was a patch), this release has a negative diff-stat. Unfortunately, much of this code isn't actually going away but at least it's being moved out into separate repositories. Much of the work that made it into this release is under the hood. We've cleaned up some code, extracted several packages into their own repositories, and made some long neglected optimizations (e.g., handling of sharded directories). Additionally, this release includes a bunch of tests for our CLI commands that should help us avoid some of the issues we've seen in the past few releases. More visibly, thanks to @djdv's efforts, this release includes some significant Windows improvements (with more on the way). Specifically, this release includes better handling of repo lockfiles (no more `ipfs repo fsck`), stdin command-line support, and, last but not least, IPFS no longer writes random files with scary garbage in the drive root. To read more about future windows improvements, take a look at this [blog post](https://blog.ipfs.io/36-a-look-at-windows/). To better support low-power devices, we've added a low-power config profile. This can be enabled when initializing a repo by running `ipfs init` with the `--profile=lowpower` flag or later by running `ipfs config profile apply lowpower`. Finally, with this release we have begun distributing self-contained source archives of go-ipfs and its dependencies. This should be a welcome improvement for both packagers and those living in countries with harmonized internet access. - Features - Add options for record count and timeout for resolving DHT paths ([ipfs/go-ipfs#4733](https://github.com/ipfs/go-ipfs/pull/4733)) - Add low power init profile ([ipfs/go-ipfs#4154](https://github.com/ipfs/go-ipfs/pull/4154)) - Add Opentracing plugin support ([ipfs/go-ipfs#4506](https://github.com/ipfs/go-ipfs/pull/4506)) - Add make target to build source tarballs ([ipfs/go-ipfs#4920](https://github.com/ipfs/go-ipfs/pull/4920)) - Improvements - Add BlockedFetched/Added/Removed events to Blockservice ([ipfs/go-ipfs#4649](https://github.com/ipfs/go-ipfs/pull/4649)) - Improve performance of HAMT code ([ipfs/go-ipfs#4889](https://github.com/ipfs/go-ipfs/pull/4889)) - Avoid unnecessarily resolving child nodes when listing a sharded directory ([ipfs/go-ipfs#4884](https://github.com/ipfs/go-ipfs/pull/4884)) - Tar writer now supports sharded ipfs directories ([ipfs/go-ipfs#4873](https://github.com/ipfs/go-ipfs/pull/4873)) - Infer type from CID when possible in `ipfs ls` ([ipfs/go-ipfs#4890](https://github.com/ipfs/go-ipfs/pull/4890)) - Deduplicate keys in GetMany ([ipfs/go-ipfs#4888](https://github.com/ipfs/go-ipfs/pull/4888)) - Documentation - Fix spelling of retrieval ([ipfs/go-ipfs#4819](https://github.com/ipfs/go-ipfs/pull/4819)) - Update broken links ([ipfs/go-ipfs#4798](https://github.com/ipfs/go-ipfs/pull/4798)) - Remove roadmap.md ([ipfs/go-ipfs#4834](https://github.com/ipfs/go-ipfs/pull/4834)) - Remove link to IPFS paper in contribute.md ([ipfs/go-ipfs#4812](https://github.com/ipfs/go-ipfs/pull/4812)) - Fix broken todo link in readme.md ([ipfs/go-ipfs#4865](https://github.com/ipfs/go-ipfs/pull/4865)) - Document ipns pubsub ([ipfs/go-ipfs#4903](https://github.com/ipfs/go-ipfs/pull/4903)) - Fix missing profile docs ([ipfs/go-ipfs#4846](https://github.com/ipfs/go-ipfs/pull/4846)) - Fix a few typos ([ipfs/go-ipfs#4835](https://github.com/ipfs/go-ipfs/pull/4835)) - Fix typo in fsrepo error message ([ipfs/go-ipfs#4933](https://github.com/ipfs/go-ipfs/pull/4933)) - Remove go-ipfs version from issue template ([ipfs/go-ipfs#4943](https://github.com/ipfs/go-ipfs/pull/4943)) - Add docs for --profile=lowpower ([ipfs/go-ipfs#4970](https://github.com/ipfs/go-ipfs/pull/4970)) - Improve Windows build documentation ([ipfs/go-ipfs#4691](https://github.com/ipfs/go-ipfs/pull/4691)) - Bug fixes - Check CIDs in base case when diffing nodes ([ipfs/go-ipfs#4767](https://github.com/ipfs/go-ipfs/pull/4767)) - Support for CIDv1 with custom mhtype in `ipfs block put` ([ipfs/go-ipfs#4563](https://github.com/ipfs/go-ipfs/pull/4563)) - Clean path in DagArchive ([ipfs/go-ipfs#4743](https://github.com/ipfs/go-ipfs/pull/4743)) - Set the prefix for MFS root in `ipfs add --hash-only` ([ipfs/go-ipfs#4755](https://github.com/ipfs/go-ipfs/pull/4755)) - Fix get output path ([ipfs/go-ipfs#4809](https://github.com/ipfs/go-ipfs/pull/4809)) - Fix incorrect Read calls ([ipfs/go-ipfs#4792](https://github.com/ipfs/go-ipfs/pull/4792)) - Use prefix in bootstrapWritePeers ([ipfs/go-ipfs#4832](https://github.com/ipfs/go-ipfs/pull/4832)) - Fix mfs Directory.Path not working ([ipfs/go-ipfs#4844](https://github.com/ipfs/go-ipfs/pull/4844)) - Remove header in `ipfs stats bw` if not polling ([ipfs/go-ipfs#4856](https://github.com/ipfs/go-ipfs/pull/4856)) - Match Go's GOPATH defaults behaviour in build scripts ([ipfs/go-ipfs#4678](https://github.com/ipfs/go-ipfs/pull/4678)) - Fix default-net profile not reverting bootstrap config ([ipfs/go-ipfs#4845](https://github.com/ipfs/go-ipfs/pull/4845)) - Fix excess goroutines in bitswap caused by insecure CIDs ([ipfs/go-ipfs#4946](https://github.com/ipfs/go-ipfs/pull/4946)) - General Changes and Refactorings - Refactor trickle DAG builder ([ipfs/go-ipfs#4730](https://github.com/ipfs/go-ipfs/pull/4730)) - Split the coreapi interface into multiple files ([ipfs/go-ipfs#4802](https://github.com/ipfs/go-ipfs/pull/4802)) - Make `ipfs init` command use new cmds lib ([ipfs/go-ipfs#4732](https://github.com/ipfs/go-ipfs/pull/4732)) - Extract thirdparty/tar package ([ipfs/go-ipfs#4857](https://github.com/ipfs/go-ipfs/pull/4857)) - Reduce log level when for disconnected peers to info ([ipfs/go-ipfs#4811](https://github.com/ipfs/go-ipfs/pull/4811)) - Only visit nodes in EnumerateChildrenAsync when asked ([ipfs/go-ipfs#4885](https://github.com/ipfs/go-ipfs/pull/4885)) - Refactor coreapi options ([ipfs/go-ipfs#4807](https://github.com/ipfs/go-ipfs/pull/4807)) - Fix error style for most errors ([ipfs/go-ipfs#4829](https://github.com/ipfs/go-ipfs/pull/4829)) - Ensure `--help` always works, even with /dev/null stdin ([ipfs/go-ipfs#4849](https://github.com/ipfs/go-ipfs/pull/4849)) - Deduplicate AddNodeLinkClean into AddNodeLink ([ipfs/go-ipfs#4940](https://github.com/ipfs/go-ipfs/pull/4940)) - Remove some dead code ([ipfs/go-ipfs#4833](https://github.com/ipfs/go-ipfs/pull/4833)) - Remove unused imports ([ipfs/go-ipfs#4955](https://github.com/ipfs/go-ipfs/pull/4955)) - Fix go vet warnings ([ipfs/go-ipfs#4859](https://github.com/ipfs/go-ipfs/pull/4859)) - Testing - Generate JUnit test reports for sharness tests ([ipfs/go-ipfs#4530](https://github.com/ipfs/go-ipfs/pull/4530)) - Fix t0063-daemon-init.sh by adding test profile to daemon ([ipfs/go-ipfs#4816](https://github.com/ipfs/go-ipfs/pull/4816)) - Remove circular dependencies in merkledag package tests ([ipfs/go-ipfs#4704](https://github.com/ipfs/go-ipfs/pull/4704)) - Check that all the commands fail when passed a bad flag ([ipfs/go-ipfs#4848](https://github.com/ipfs/go-ipfs/pull/4848)) - Allow for some small margin of code coverage dropping on commit ([ipfs/go-ipfs#4867](https://github.com/ipfs/go-ipfs/pull/4867)) - Add confirmation to archive-branches script ([ipfs/go-ipfs#4797](https://github.com/ipfs/go-ipfs/pull/4797)) - Dependencies - Update lock package ([ipfs/go-ipfs#4855](https://github.com/ipfs/go-ipfs/pull/4855)) - Update to latest go-datastore. Remove thirdparty/datastore2 ([ipfs/go-ipfs#4742](https://github.com/ipfs/go-ipfs/pull/4742)) - Extract fs lock into go-fs-lock ([ipfs/go-ipfs#4631](https://github.com/ipfs/go-ipfs/pull/4631)) - Extract: exchange/interface.go, blocks/blocksutil, exchange/offline ([ipfs/go-ipfs#4912](https://github.com/ipfs/go-ipfs/pull/4912)) - Remove unused lock dep ([ipfs/go-ipfs#4971](https://github.com/ipfs/go-ipfs/pull/4971)) - Update iptb ([ipfs/go-ipfs#4965](https://github.com/ipfs/go-ipfs/pull/4965)) - Update go-ipfs-cmds to fix stdin on windows ([ipfs/go-ipfs#4975](https://github.com/ipfs/go-ipfs/pull/4975)) - Update go-ds-flatfs to fix windows corruption issue ([ipfs/go-ipfs#4872](https://github.com/ipfs/go-ipfs/pull/4872)) ## v0.4.14 2018-03-22 Ipfs 0.4.14 is a big release with a large number of improvements and bug fixes. It is also the first release of 2018, and our first release in over three months. The release took longer than expected due to our refactoring and extracting of our commands library. This refactor had two stages. The first round of the refactor disentangled the commands code from core ipfs code, allowing us to move it out into a [separate repository](https://github.com/ipfs/go-ipfs-cmds). The code was previously very entangled with the go-ipfs codebase and not usable for other projects. The second round of the refactor had the goal of fixing several major issues around streaming outputs, progress bars, and error handling. It also paved the way for us to more easily provide an API over other transports, such as websockets and unix domain sockets. It took a while to flush out all the kinks on such a massive change. We're pretty sure we've got most of them, but if you notice anything weird, please let us know. Beyond that, we've added a new experimental way to use IPNS. With the new pubsub IPNS resolver and publisher, you can subscribe to updates of an IPNS entry, and the owner can publish out changes in real time. With this, IPNS can become nearly instantaneous. To make use of this, simply start your ipfs daemon with the `--enable-namesys-pubsub` option, and all IPNS resolution and publishing will use pubsub. Note that resolving an IPNS name via pubsub without someone publishing it via pubsub will result in a fallback to using the DHT. Please give this a try and let us know how it goes! Memory and CPU usage should see a noticeable improvement in this release. We have spent considerable time fixing excess memory usage throughout the codebase and down into libp2p. Fixes in peer tracking, bitswap allocation, pinning, and many other places have brought down both peak and average memory usage. An upgraded hashing library, base58 encoding library, and improved allocation patterns all contribute to overall lower CPU usage across the board. See the full changelist below for more memory and CPU usage improvements. This release also brings the beginning of the ipfs 'Core API'. Once finished, the Core API will be the primary way to interact with go-ipfs using go. Both embedded nodes and nodes accessed over the http API will have the same interface. Stay tuned for future updates and documentation. These are only a sampling of the changes that made it into this release, the full list (almost 100 PRs!) is below. Finally, I'd like to thank everyone who contributed to this release, whether you're just contributing a typo fix or driving new features. We are really grateful to everyone who has spent their their time pushing ipfs forward. SECURITY NOTE: This release of ipfs disallows the usage of insecure hash functions and lengths. Ipfs does not create these insecure objects for any purpose, but it did allow manually creating them and fetching them from other peers. If you currently have objects using insecure hashes in your local ipfs repo, please remove them before updating. #### Changes from rc2 to rc3 - Fix bug in stdin argument parsing ([ipfs/go-ipfs#4827](https://github.com/ipfs/go-ipfs/pull/4827)) - Revert commands back to sending a single response ([ipfs/go-ipfs#4822](https://github.com/ipfs/go-ipfs/pull/4822)) #### Changes from rc1 to rc2 - Fix issue in ipfs get caused by go1.10 changes ([ipfs/go-ipfs#4790](https://github.com/ipfs/go-ipfs/pull/4790)) - Features - Pubsub IPNS Publisher and Resolver (experimental) ([ipfs/go-ipfs#4047](https://github.com/ipfs/go-ipfs/pull/4047)) - Implement coreapi Dag interface ([ipfs/go-ipfs#4471](https://github.com/ipfs/go-ipfs/pull/4471)) - Add --offset flag to ipfs cat ([ipfs/go-ipfs#4538](https://github.com/ipfs/go-ipfs/pull/4538)) - Command to apply config profile after init ([ipfs/go-ipfs#4195](https://github.com/ipfs/go-ipfs/pull/4195)) - Implement coreapi Name and Key interfaces ([ipfs/go-ipfs#4477](https://github.com/ipfs/go-ipfs/pull/4477)) - Add --length flag to ipfs cat ([ipfs/go-ipfs#4553](https://github.com/ipfs/go-ipfs/pull/4553)) - Implement coreapi Object interface ([ipfs/go-ipfs#4492](https://github.com/ipfs/go-ipfs/pull/4492)) - Implement coreapi Block interface ([ipfs/go-ipfs#4548](https://github.com/ipfs/go-ipfs/pull/4548)) - Implement coreapi Pin interface ([ipfs/go-ipfs#4575](https://github.com/ipfs/go-ipfs/pull/4575)) - Add a --with-local flag to ipfs files stat ([ipfs/go-ipfs#4638](https://github.com/ipfs/go-ipfs/pull/4638)) - Disallow usage of blocks with insecure hashes ([ipfs/go-ipfs#4751](https://github.com/ipfs/go-ipfs/pull/4751)) - Improvements - Add uuid to event logs ([ipfs/go-ipfs#4392](https://github.com/ipfs/go-ipfs/pull/4392)) - Add --quiet flag to object put ([ipfs/go-ipfs#4411](https://github.com/ipfs/go-ipfs/pull/4411)) - Pinning memory improvements and fixes ([ipfs/go-ipfs#4451](https://github.com/ipfs/go-ipfs/pull/4451)) - Update WebUI version ([ipfs/go-ipfs#4449](https://github.com/ipfs/go-ipfs/pull/4449)) - Check strong and weak ETag validator ([ipfs/go-ipfs#3983](https://github.com/ipfs/go-ipfs/pull/3983)) - Improve and refactor FD limit handling ([ipfs/go-ipfs#3801](https://github.com/ipfs/go-ipfs/pull/3801)) - Support linking to non-dagpb objects in ipfs object patch ([ipfs/go-ipfs#4460](https://github.com/ipfs/go-ipfs/pull/4460)) - Improve allocation patterns of slices in bitswap ([ipfs/go-ipfs#4458](https://github.com/ipfs/go-ipfs/pull/4458)) - Secio handshake now happens synchronously ([libp2p/go-libp2p-secio#25](https://github.com/libp2p/go-libp2p-secio/pull/25)) - Don't block closing connections on pending writes ([libp2p/go-msgio#7](https://github.com/libp2p/go-msgio/pull/7)) - Improve memory usage of multiaddr parsing ([multiformats/go-multiaddr#56](https://github.com/multiformats/go-multiaddr/pull/56)) - Don't lock up 256KiB buffers when adding small files ([ipfs/go-ipfs#4508](https://github.com/ipfs/go-ipfs/pull/4508)) - Clear out memory after reads from the dagreader ([ipfs/go-ipfs#4525](https://github.com/ipfs/go-ipfs/pull/4525)) - Improve error handling in ipfs ping ([ipfs/go-ipfs#4546](https://github.com/ipfs/go-ipfs/pull/4546)) - Allow install.sh to be run without being the script dir ([ipfs/go-ipfs#4547](https://github.com/ipfs/go-ipfs/pull/4547)) - Much faster base58 encoding ([libp2p/go-libp2p-peer#24](https://github.com/libp2p/go-libp2p-peer/pull/24)) - Use faster sha256 and blake2b libs ([multiformats/go-multihash#63](https://github.com/multiformats/go-multihash/pull/63)) - Greatly improve peerstore memory usage ([libp2p/go-libp2p-peerstore#22](https://github.com/libp2p/go-libp2p-peerstore/pull/22)) - Improve dht memory usage and peer tracking ([libp2p/go-libp2p-kad-dht#111](https://github.com/libp2p/go-libp2p-kad-dht/pull/111)) - New libp2p metrics lib with lower overhead ([libp2p/go-libp2p-metrics#8](https://github.com/libp2p/go-libp2p-metrics/pull/8)) - Fix memory leak that occurred when dialing many peers ([libp2p/go-libp2p-swarm#51](https://github.com/libp2p/go-libp2p-swarm/pull/51)) - Wire up new dag interfaces to make sessions easier ([ipfs/go-ipfs#4641](https://github.com/ipfs/go-ipfs/pull/4641)) - Documentation - Correct StorageMax config description ([ipfs/go-ipfs#4388](https://github.com/ipfs/go-ipfs/pull/4388)) - Add how to download IPFS with IPFS doc ([ipfs/go-ipfs#4390](https://github.com/ipfs/go-ipfs/pull/4390)) - Document gx release checklist item ([ipfs/go-ipfs#4480](https://github.com/ipfs/go-ipfs/pull/4480)) - Add some documentation to CoreAPI ([ipfs/go-ipfs#4493](https://github.com/ipfs/go-ipfs/pull/4493)) - Add interop tests to the release checklist ([ipfs/go-ipfs#4501](https://github.com/ipfs/go-ipfs/pull/4501)) - Add badgerds to experimental-features ToC ([ipfs/go-ipfs#4537](https://github.com/ipfs/go-ipfs/pull/4537)) - Fix typos and inconsistencies in commands documentation ([ipfs/go-ipfs#4552](https://github.com/ipfs/go-ipfs/pull/4552)) - Add a document to help troubleshoot data transfers ([ipfs/go-ipfs#4332](https://github.com/ipfs/go-ipfs/pull/4332)) - Add a bunch of documentation on public interfaces ([ipfs/go-ipfs#4599](https://github.com/ipfs/go-ipfs/pull/4599)) - Expand the issue template and remove the severity field ([ipfs/go-ipfs#4624](https://github.com/ipfs/go-ipfs/pull/4624)) - Add godocs for importers module ([ipfs/go-ipfs#4640](https://github.com/ipfs/go-ipfs/pull/4640)) - Document make targets ([ipfs/go-ipfs#4653](https://github.com/ipfs/go-ipfs/pull/4653)) - Add godocs for merkledag module ([ipfs/go-ipfs#4665](https://github.com/ipfs/go-ipfs/pull/4665)) - Add godocs for unixfs module ([ipfs/go-ipfs#4664](https://github.com/ipfs/go-ipfs/pull/4664)) - Add sharding to experimental features list ([ipfs/go-ipfs#4569](https://github.com/ipfs/go-ipfs/pull/4569)) - Add godocs for routing module ([ipfs/go-ipfs#4676](https://github.com/ipfs/go-ipfs/pull/4676)) - Add godocs for path module ([ipfs/go-ipfs#4689](https://github.com/ipfs/go-ipfs/pull/4689)) - Add godocs for pin module ([ipfs/go-ipfs#4696](https://github.com/ipfs/go-ipfs/pull/4696)) - Update link to filestore experimental status ([ipfs/go-ipfs#4557](https://github.com/ipfs/go-ipfs/pull/4557)) - Bug fixes - Remove trailing slash in ipfs get paths, fixes #3729 ([ipfs/go-ipfs#4365](https://github.com/ipfs/go-ipfs/pull/4365)) - fix deadlock in bitswap sessions ([ipfs/go-ipfs#4407](https://github.com/ipfs/go-ipfs/pull/4407)) - Fix two race conditions (and possibly go routine leaks) in commands ([ipfs/go-ipfs#4406](https://github.com/ipfs/go-ipfs/pull/4406)) - Fix output delay in ipfs pubsub sub ([ipfs/go-ipfs#4402](https://github.com/ipfs/go-ipfs/pull/4402)) - Use correct context in AddWithContext ([ipfs/go-ipfs#4433](https://github.com/ipfs/go-ipfs/pull/4433)) - Fix various IPNS republisher issues ([ipfs/go-ipfs#4440](https://github.com/ipfs/go-ipfs/pull/4440)) - Fix error handling in commands add and get ([ipfs/go-ipfs#4454](https://github.com/ipfs/go-ipfs/pull/4454)) - Fix hamt (sharding) delete issue ([ipfs/go-ipfs#4398](https://github.com/ipfs/go-ipfs/pull/4398)) - More correctly check for reuseport support ([libp2p/go-reuseport#40](https://github.com/libp2p/go-reuseport/pull/40)) - Fix goroutine leak in websockets transport ([libp2p/go-ws-transport#21](https://github.com/libp2p/go-ws-transport/pull/21)) - Update badgerds to fix i386 windows build ([ipfs/go-ipfs#4464](https://github.com/ipfs/go-ipfs/pull/4464)) - Only construct bitswap event loggable if necessary ([ipfs/go-ipfs#4533](https://github.com/ipfs/go-ipfs/pull/4533)) - Ensure that flush on the mfs root flushes its directory ([ipfs/go-ipfs#4509](https://github.com/ipfs/go-ipfs/pull/4509)) - Fix deferred unlock of pin lock in AddR ([ipfs/go-ipfs#4562](https://github.com/ipfs/go-ipfs/pull/4562)) - Fix iOS builds ([ipfs/go-ipfs#4610](https://github.com/ipfs/go-ipfs/pull/4610)) - Calling repo gc now frees up space with badgerds ([ipfs/go-ipfs#4578](https://github.com/ipfs/go-ipfs/pull/4578)) - Fix leak in bitswap sessions shutdown ([ipfs/go-ipfs#4658](https://github.com/ipfs/go-ipfs/pull/4658)) - Fix make on windows ([ipfs/go-ipfs#4682](https://github.com/ipfs/go-ipfs/pull/4682)) - Ignore invalid key files in keystore directory ([ipfs/go-ipfs#4700](https://github.com/ipfs/go-ipfs/pull/4700)) - General Changes and Refactorings - Extract and refactor commands library ([ipfs/go-ipfs#3856](https://github.com/ipfs/go-ipfs/pull/3856)) - Remove all instances of `Default(false)` ([ipfs/go-ipfs#4042](https://github.com/ipfs/go-ipfs/pull/4042)) - Build for all supported platforms when testing ([ipfs/go-ipfs#4445](https://github.com/ipfs/go-ipfs/pull/4445)) - Refine gateway and namesys logging ([ipfs/go-ipfs#4428](https://github.com/ipfs/go-ipfs/pull/4428)) - Demote bitswap error to an info ([ipfs/go-ipfs#4472](https://github.com/ipfs/go-ipfs/pull/4472)) - Extract posinfo package to github.com/ipfs/go-ipfs-posinfo ([ipfs/go-ipfs#4669](https://github.com/ipfs/go-ipfs/pull/4669)) - Move signature verification to ipns validator ([ipfs/go-ipfs#4628](https://github.com/ipfs/go-ipfs/pull/4628)) - Extract importers/chunk module as go-ipfs-chunker ([ipfs/go-ipfs#4661](https://github.com/ipfs/go-ipfs/pull/4661)) - Extract go-detect-race from Godeps ([ipfs/go-ipfs#4686](https://github.com/ipfs/go-ipfs/pull/4686)) - Extract flags, delay, ds-help ([ipfs/go-ipfs#4685](https://github.com/ipfs/go-ipfs/pull/4685)) - Extract routing package to go-ipfs-routing ([ipfs/go-ipfs#4703](https://github.com/ipfs/go-ipfs/pull/4703)) - Extract blocks/blockstore package to go-ipfs-blockstore ([ipfs/go-ipfs#4707](https://github.com/ipfs/go-ipfs/pull/4707)) - Add exchange.SessionExchange interface for exchanges that support sessions ([ipfs/go-ipfs#4709](https://github.com/ipfs/go-ipfs/pull/4709)) - Extract thirdparty/pq to go-ipfs-pq ([ipfs/go-ipfs#4711](https://github.com/ipfs/go-ipfs/pull/4711)) - Separate "path" from "path/resolver" ([ipfs/go-ipfs#4713](https://github.com/ipfs/go-ipfs/pull/4713)) - Testing - Increase verbosity of t0088-repo-stat-symlink.sh test ([ipfs/go-ipfs#4434](https://github.com/ipfs/go-ipfs/pull/4434)) - Make repo size test pass deterministically ([ipfs/go-ipfs#4443](https://github.com/ipfs/go-ipfs/pull/4443)) - Always set IPFS_PATH in test-lib.sh ([ipfs/go-ipfs#4469](https://github.com/ipfs/go-ipfs/pull/4469)) - Fix sharness docker ([ipfs/go-ipfs#4489](https://github.com/ipfs/go-ipfs/pull/4489)) - Fix loops in sharness tests to fail the test if the inner command fails ([ipfs/go-ipfs#4482](https://github.com/ipfs/go-ipfs/pull/4482)) - Improve bitswap tests, fix race conditions ([ipfs/go-ipfs#4499](https://github.com/ipfs/go-ipfs/pull/4499)) - Fix circleci cache directory list ([ipfs/go-ipfs#4564](https://github.com/ipfs/go-ipfs/pull/4564)) - Only run the build test on test_go_expensive ([ipfs/go-ipfs#4645](https://github.com/ipfs/go-ipfs/pull/4645)) - Fix go test on Windows ([ipfs/go-ipfs#4632](https://github.com/ipfs/go-ipfs/pull/4632)) - Fix some tests on FreeBSD ([ipfs/go-ipfs#4662](https://github.com/ipfs/go-ipfs/pull/4662)) ## v0.4.13 2017-11-16 Ipfs 0.4.13 is a patch release that fixes two high priority issues that were discovered in the 0.4.12 release. Bug fixes: - Fix periodic bitswap deadlock ([ipfs/go-ipfs#4386](https://github.com/ipfs/go-ipfs/pull/4386)) - Fix badgerds crash on startup ([ipfs/go-ipfs#4384](https://github.com/ipfs/go-ipfs/pull/4384)) ## v0.4.12 2017-11-09 Ipfs 0.4.12 brings with it many important fixes for the huge spike in network size we've seen this past month. These changes include the Connection Manager, faster batching in `ipfs add`, libp2p fixes that reduce CPU usage, and a bunch of new documentation. The most critical change is the 'Connection Manager': it allows an ipfs node to maintain a limited set of connections to other peers in the network. By default (and with no config changes required by the user), ipfs nodes will now try to maintain between 600 and 900 open connections. These limits are still likely higher than needed, and future releases may lower the default recommendation, but for now we want to make changes gradually. The rationale for this selection of numbers is as follows: - The DHT routing table for a large network may rise to around 400 peers - Bitswap connections tend to be separate from the DHT - PubSub connections also generally are another distinct set of peers (including js-ipfs nodes) Because of this, we selected 600 as a 'LowWater' number, and 900 as a 'HighWater' number to avoid having to clear out connections too frequently. You can configure different numbers as you see fit via the `Swarm.ConnMgr` field in your ipfs config file. See [here](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#connmgr) for more details. Disk utilization during `ipfs add` has been optimized for large files by doing batch writes in parallel. Previously, when adding a large file, users might have noticed that the add progressed by about 8MB at a time, with brief pauses in between. This was caused by quickly filling up the batch, then blocking while it was writing to disk. We now write to disk in the background while continuing to add the remainder of the file. Other changes in this release have noticeably reduced memory consumption and CPU usage. This was done by optimising some frequently called functions in libp2p that were expensive in terms of both CPU usage and memory allocations. We also lowered the yamux accept buffer sizes which were raised over a year ago to combat a separate bug that has since been fixed. And finally, thank you to everyone who filed bugs, tested out the release candidates, filed pull requests, and contributed in any other way to this release! - Features - Implement Connection Manager ([ipfs/go-ipfs#4288](https://github.com/ipfs/go-ipfs/pull/4288)) - Support multiple files in dag put ([ipfs/go-ipfs#4254](https://github.com/ipfs/go-ipfs/pull/4254)) - Add 'raw' support to the dag put command ([ipfs/go-ipfs#4285](https://github.com/ipfs/go-ipfs/pull/4285)) - Improvements - Parallelize dag batch flushing ([ipfs/go-ipfs#4296](https://github.com/ipfs/go-ipfs/pull/4296)) - Update go-peerstream to improve CPU usage ([ipfs/go-ipfs#4323](https://github.com/ipfs/go-ipfs/pull/4323)) - Add full support for CidV1 in Files API and Dag Modifier ([ipfs/go-ipfs#4026](https://github.com/ipfs/go-ipfs/pull/4026)) - Lower yamux accept buffer size ([ipfs/go-ipfs#4326](https://github.com/ipfs/go-ipfs/pull/4326)) - Optimise `ipfs pin update` command ([ipfs/go-ipfs#4348](https://github.com/ipfs/go-ipfs/pull/4348)) - Documentation - Add some docs on plugins ([ipfs/go-ipfs#4255](https://github.com/ipfs/go-ipfs/pull/4255)) - Add more info about private network bootstrap ([ipfs/go-ipfs#4270](https://github.com/ipfs/go-ipfs/pull/4270)) - Add more info about `ipfs add` chunker option ([ipfs/go-ipfs#4306](https://github.com/ipfs/go-ipfs/pull/4306)) - Remove cruft in readme and mention discourse forum ([ipfs/go-ipfs#4345](https://github.com/ipfs/go-ipfs/pull/4345)) - Add note about updating before reporting issues ([ipfs/go-ipfs#4361](https://github.com/ipfs/go-ipfs/pull/4361)) - Bug fixes - Fix FreeBSD build issues ([ipfs/go-ipfs#4275](https://github.com/ipfs/go-ipfs/pull/4275)) - Don't crash when Datastore.StorageMax is not defined ([ipfs/go-ipfs#4246](https://github.com/ipfs/go-ipfs/pull/4246)) - Do not call 'Connect' on NewStream in bitswap ([ipfs/go-ipfs#4317](https://github.com/ipfs/go-ipfs/pull/4317)) - Filter out "" from active peers in bitswap sessions ([ipfs/go-ipfs#4316](https://github.com/ipfs/go-ipfs/pull/4316)) - Fix "seeker can't seek" on specific files ([ipfs/go-ipfs#4320](https://github.com/ipfs/go-ipfs/pull/4320)) - Do not set "gecos" field in Dockerfile ([ipfs/go-ipfs#4331](https://github.com/ipfs/go-ipfs/pull/4331)) - Handle sym links in when calculating repo size ([ipfs/go-ipfs#4305](https://github.com/ipfs/go-ipfs/pull/4305)) - General Changes and Refactorings - Fix indent in sharness tests ([ipfs/go-ipfs#4212](https://github.com/ipfs/go-ipfs/pull/4212)) - Remove supernode routing ([ipfs/go-ipfs#4302](https://github.com/ipfs/go-ipfs/pull/4302)) - Extract go-ipfs-addr ([ipfs/go-ipfs#4340](https://github.com/ipfs/go-ipfs/pull/4340)) - Remove dead code and config files ([ipfs/go-ipfs#4357](https://github.com/ipfs/go-ipfs/pull/4357)) - Update badgerds to 1.0 ([ipfs/go-ipfs#4327](https://github.com/ipfs/go-ipfs/pull/4327)) - Wrap help descriptions under 80 chars ([ipfs/go-ipfs#4121](https://github.com/ipfs/go-ipfs/pull/4121)) - Testing - Make sharness t0180-p2p less racy ([ipfs/go-ipfs#4310](https://github.com/ipfs/go-ipfs/pull/4310)) ### v0.4.11 2017-09-14 Ipfs 0.4.11 is a larger release that brings many long-awaited features and performance improvements. These include new datastore options, more efficient bitswap transfers, greatly improved resource consumption, circuit relay support, ipld plugins, and more! Take a look at the full changelog below for a detailed list of every change. The ipfs datastore has, until now, been a combination of leveldb and a custom git-like storage backend called 'flatfs'. This works well enough for the average user, but different ipfs usecases demand different backend configurations. To address this, we have changed the configuration file format for datastores to be a modular way of specifying exactly how you want the datastore to be structured. You will now be able to configure ipfs to use flatfs, leveldb, badger, an in-memory datastore, and more to suit your needs. See the new [datastore documentation](https://github.com/ipfs/go-ipfs/blob/master/docs/datastores.md) for more information. Bitswap received some much needed attention during this release cycle. The concept of 'Bitswap Sessions' allows bitswap to associate requests for different blocks to the same underlying session, and from that infer better ways of requesting that data. In more concrete terms, parts of the ipfs codebase that take advantage of sessions (currently, only `ipfs pin add`) will cause much less extra traffic than before. This is done by making optimistic guesses about which nodes might be providing given blocks and not sending wantlist updates to every connected bitswap partner, as well as searching the DHT for providers less frequently. In future releases we will migrate over more ipfs commands to take advantage of bitswap sessions. As nodes update to this and future versions, expect to see idle bandwidth usage on the ipfs network go down noticeably. The never ending effort to reduce resource consumption had a few important updates this release. First, the bitswap sessions changes discussed above will help with improving bandwidth usage. Aside from that there are two important libp2p updates that improved things significantly. The first was a fix to a bug in the dial limiter code that was causing it to not limit outgoing dials correctly. This resulted in ipfs running out of file descriptors very frequently (as well as incurring a decent amount of excess outgoing bandwidth), this has now been fixed. Users who previously received "too many open files" errors should see this much less often in 0.4.11. The second change was a memory leak in the DHT that was identified and fixed. Streams being tracked in a map in the DHT weren't being cleaned up after the peer disconnected leading to the multiplexer session not being cleaned up properly. This issue has been resolved, and now memory usage appears to be stable over time. There is still a lot of work to be done improving memory usage, but we feel this is a solid victory. It is often said that NAT traversal is the hardest problem in peer to peer technology, we tend to agree with this. In an effort to provide a more ubiquitous p2p mesh, we have implemented a relay mechanism that allows willing peers to relay traffic for other peers who might not otherwise be able to communicate with each other. This feature is still pretty early, and currently users have to manually connect through a relay. The next step in this endeavour is automatic relaying, and research for this is currently in progress. We expect that when it lands, it will improve the perceived performance of ipfs by spending less time attempting connections to hard to reach nodes. A short guide on using the circuit relay feature can be found [here](https://github.com/ipfs/go-ipfs/blob/master/docs/experimental-features.md#circuit-relay). The last feature we want to highlight (but by no means the last feature in this release) is our new plugin system. There are many different workflows and usecases that ipfs should be able to support, but not everyone wants to be able to use every feature. We could simply merge in all these features, but that causes problems for several reasons: first off, the size of the ipfs binary starts to get very large very quickly. Second, each of these different pieces needs to be maintained and updated independently, which would cause significant churn in the codebase. To address this, we have come up with a system that allows users to install plugins to the vanilla ipfs daemon that augment its capabilities. The first of these plugins are a [git plugin](https://github.com/ipfs/go-ipfs/blob/master/plugin/plugins/git/git.go) that allows ipfs to natively address git objects and an [ethereum plugin](https://github.com/ipfs/go-ipld-eth) that lets ipfs ingest and operate on all ethereum blockchain data. Soon to come are plugins for the bitcoin and zcash data formats. In the future, we will be adding plugins for other things like datastore backends and specialized libp2p network transports. You can read more on this topic in [Plugin docs](docs/plugins.md) In order to simplify its integration with fs-repo-migrations, we've switched the ipfs/go-ipfs docker image from a musl base to a glibc base. For most users this will not be noticeable, but if you've been building your own images based off this image, you'll have to update your dockerfile. We recommend a multi-stage dockerfile, where the build stage is based off of a regular Debian or other glibc-based image, and the assembly stage is based off of the ipfs/go-ipfs image, and you copy build artifacts from the build stage to the assembly stage. Note, if you are using the docker image and see a deprecation message, please update your usage. We will stop supporting the old method of starting the dockerfile in the next release. Finally, I would like to thank all of our contributors, users, supporters, and friends for helping us along the way. Ipfs would not be where it is without you. - Features - Add `--pin` option to `ipfs dag put` ([ipfs/go-ipfs#4004](https://github.com/ipfs/go-ipfs/pull/4004)) - Add `--pin` option to `ipfs object put` ([ipfs/go-ipfs#4095](https://github.com/ipfs/go-ipfs/pull/4095)) - Implement `--profile` option on `ipfs init` ([ipfs/go-ipfs#4001](https://github.com/ipfs/go-ipfs/pull/4001)) - Add CID Codecs to `ipfs block put` ([ipfs/go-ipfs#4022](https://github.com/ipfs/go-ipfs/pull/4022)) - Bitswap sessions ([ipfs/go-ipfs#3867](https://github.com/ipfs/go-ipfs/pull/3867)) - Create plugin API and loader, add ipld-git plugin ([ipfs/go-ipfs#4033](https://github.com/ipfs/go-ipfs/pull/4033)) - Make announced swarm addresses configurable ([ipfs/go-ipfs#3948](https://github.com/ipfs/go-ipfs/pull/3948)) - Reprovider strategies ([ipfs/go-ipfs#4113](https://github.com/ipfs/go-ipfs/pull/4113)) - Circuit Relay integration ([ipfs/go-ipfs#4091](https://github.com/ipfs/go-ipfs/pull/4091)) - More configurable datastore configs ([ipfs/go-ipfs#3575](https://github.com/ipfs/go-ipfs/pull/3575)) - Add experimental support for badger datastore ([ipfs/go-ipfs#4007](https://github.com/ipfs/go-ipfs/pull/4007)) - Improvements - Add better support for Raw Nodes in MFS and elsewhere ([ipfs/go-ipfs#3996](https://github.com/ipfs/go-ipfs/pull/3996)) - Added file size to response of `ipfs add` command ([ipfs/go-ipfs#4082](https://github.com/ipfs/go-ipfs/pull/4082)) - Add /dnsaddr bootstrap nodes ([ipfs/go-ipfs#4127](https://github.com/ipfs/go-ipfs/pull/4127)) - Do not publish public keys extractable from ID ([ipfs/go-ipfs#4020](https://github.com/ipfs/go-ipfs/pull/4020)) - Documentation - Adding documentation that PubSub Sub can be encoded. ([ipfs/go-ipfs#3909](https://github.com/ipfs/go-ipfs/pull/3909)) - Add Comms items from js-ipfs, including blog ([ipfs/go-ipfs#3936](https://github.com/ipfs/go-ipfs/pull/3936)) - Add Developer Certificate of Origin ([ipfs/go-ipfs#4006](https://github.com/ipfs/go-ipfs/pull/4006)) - Add `transports.md` document ([ipfs/go-ipfs#4034](https://github.com/ipfs/go-ipfs/pull/4034)) - Add `experimental-features.md` document ([ipfs/go-ipfs#4036](https://github.com/ipfs/go-ipfs/pull/4036)) - Update release docs ([ipfs/go-ipfs#4165](https://github.com/ipfs/go-ipfs/pull/4165)) - Add documentation for datastore configs ([ipfs/go-ipfs#4223](https://github.com/ipfs/go-ipfs/pull/4223)) - General update and clean-up of docs ([ipfs/go-ipfs#4222](https://github.com/ipfs/go-ipfs/pull/4222)) - Bug fixes - Fix shutdown check in t0023 ([ipfs/go-ipfs#3969](https://github.com/ipfs/go-ipfs/pull/3969)) - Fix pinning of unixfs sharded directories ([ipfs/go-ipfs#3975](https://github.com/ipfs/go-ipfs/pull/3975)) - Show escaped url in gateway 404 message ([ipfs/go-ipfs#4005](https://github.com/ipfs/go-ipfs/pull/4005)) - Fix early opening of bitswap message sender ([ipfs/go-ipfs#4069](https://github.com/ipfs/go-ipfs/pull/4069)) - Fix determination of 'root' node in dag put ([ipfs/go-ipfs#4072](https://github.com/ipfs/go-ipfs/pull/4072)) - Fix bad multipart message panic in gateway ([ipfs/go-ipfs#4053](https://github.com/ipfs/go-ipfs/pull/4053)) - Add blocks to the blockstore before returning them from blockservice sessions ([ipfs/go-ipfs#4169](https://github.com/ipfs/go-ipfs/pull/4169)) - Various fixes for /ipfs fuse code ([ipfs/go-ipfs#4194](https://github.com/ipfs/go-ipfs/pull/4194)) - Fix memory leak in dht stream tracking ([ipfs/go-ipfs#4251](https://github.com/ipfs/go-ipfs/pull/4251)) - General Changes and Refactorings - Require go 1.8 ([ipfs/go-ipfs#4044](https://github.com/ipfs/go-ipfs/pull/4044)) - Change IPFS to use the new pluggable Block to IPLD decoding framework. ([ipfs/go-ipfs#4060](https://github.com/ipfs/go-ipfs/pull/4060)) - Remove tour command from ipfs ([ipfs/go-ipfs#4123](https://github.com/ipfs/go-ipfs/pull/4123)) - Add support for Go 1.9 ([ipfs/go-ipfs#4156](https://github.com/ipfs/go-ipfs/pull/4156)) - Remove some dead code ([ipfs/go-ipfs#4204](https://github.com/ipfs/go-ipfs/pull/4204)) - Switch docker image from musl to glibc ([ipfs/go-ipfs#4219](https://github.com/ipfs/go-ipfs/pull/4219)) ### v0.4.10 - 2017-06-27 Ipfs 0.4.10 is a patch release that contains several exciting new features, bug fixes and general improvements. Including new commands, easier corruption recovery, and a generally cleaner codebase. The `ipfs pin` command has two new subcommands, `verify` and `update`. `ipfs pin verify` is used to scan the repo for pinned object graphs and check their integrity. Any issues are reported back with helpful error text to make error recovery simpler. This subcommand was added to help recover from datastore corruptions, particularly if using the experimental filestore and accidentally deleting tracked files. `ipfs pin update` was added to make the task of keeping a large, frequently changing object graph pinned. Previously users had to call `ipfs pin rm` on the old pin, and `ipfs pin add` on the new one. The 'new' `ipfs pin add` call would be very expensive as it would need to verify the entirety of the graph again. The `ipfs pin update` command takes shortcuts, portions of the graph that were covered under the old pin are assumed to be fine, and the command skips checking them. Next up, we have finally implemented an `ipfs shutdown` command so users can shut down their ipfs daemons via the API. This is especially useful on platforms that make it difficult to control processes (Android, for example), and is also useful when needing to shut down a node remotely and you do not have access to the machine itself. `ipfs add` has gained a new flag; the `--hash` flag allows you to select which hash function to use and we have given it the ability to select `blake2b-256`. This pushes us one step closer to shifting over to using blake2b as the default. Blake2b is significantly faster than sha2-256, and also is conjectured to provide superior security. We have also finally implemented a very early (and experimental) `ipfs p2p`. This command and its subcommands will allow you to open up arbitrary streams to other ipfs peers through libp2p. The interfaces are a little bit clunky right now, but shouldn't get in the way of anyone wanting to try building a fully peer to peer application on top of ipfs and libp2p. For more info on this command, to ask questions, or to provide feedback, head over to the [feedback issue](https://github.com/ipfs/go-ipfs/issues/3994) for the command. A few other subcommands and flags were added around the API, as well as many other requested improvements. See below for the full list of changes. - Features - Add support for specifying the hash function in `ipfs add` ([ipfs/go-ipfs#3919](https://github.com/ipfs/go-ipfs/pull/3919)) - Implement `ipfs key {rm, rename}` ([ipfs/go-ipfs#3892](https://github.com/ipfs/go-ipfs/pull/3892)) - Implement `ipfs shutdown` command ([ipfs/go-ipfs#3884](https://github.com/ipfs/go-ipfs/pull/3884)) - Implement `ipfs pin update` ([ipfs/go-ipfs#3846](https://github.com/ipfs/go-ipfs/pull/3846)) - Implement `ipfs pin verify` ([ipfs/go-ipfs#3843](https://github.com/ipfs/go-ipfs/pull/3843)) - Implemented experimental p2p commands ([ipfs/go-ipfs#3943](https://github.com/ipfs/go-ipfs/pull/3943)) - Improvements - Add MaxStorage field to output of "repo stat" ([ipfs/go-ipfs#3915](https://github.com/ipfs/go-ipfs/pull/3915)) - Add Suborigin header to gateway responses ([ipfs/go-ipfs#3914](https://github.com/ipfs/go-ipfs/pull/3914)) - Add "--file-order" option to "filestore ls" and "verify" ([ipfs/go-ipfs#3938](https://github.com/ipfs/go-ipfs/pull/3938)) - Allow selecting ipns keys by Peer ID ([ipfs/go-ipfs#3882](https://github.com/ipfs/go-ipfs/pull/3882)) - Don't redirect to trailing slash in gateway for `go get` ([ipfs/go-ipfs#3963](https://github.com/ipfs/go-ipfs/pull/3963)) - Add 'ipfs dht findprovs --num-providers' to allow choosing number of providers to find ([ipfs/go-ipfs#3966](https://github.com/ipfs/go-ipfs/pull/3966)) - Make sure all keystore keys get republished ([ipfs/go-ipfs#3951](https://github.com/ipfs/go-ipfs/pull/3951)) - Documentation - Adding documentation on PubSub encodings ([ipfs/go-ipfs#3909](https://github.com/ipfs/go-ipfs/pull/3909)) - Change 'necessary' to 'necessary' ([ipfs/go-ipfs#3941](https://github.com/ipfs/go-ipfs/pull/3941)) - README.md: add Nix to the linux package managers ([ipfs/go-ipfs#3939](https://github.com/ipfs/go-ipfs/pull/3939)) - More verbose errors in filestore ([ipfs/go-ipfs#3964](https://github.com/ipfs/go-ipfs/pull/3964)) - Bug fixes - Fix typo in message when file size check fails ([ipfs/go-ipfs#3895](https://github.com/ipfs/go-ipfs/pull/3895)) - Clean up bitswap ledgers when disconnecting ([ipfs/go-ipfs#3437](https://github.com/ipfs/go-ipfs/pull/3437)) - Make odds of 'process added after close' panic less likely ([ipfs/go-ipfs#3940](https://github.com/ipfs/go-ipfs/pull/3940)) - General Changes and Refactorings - Remove 'ipfs diag net' from codebase ([ipfs/go-ipfs#3916](https://github.com/ipfs/go-ipfs/pull/3916)) - Update to dht code with provide announce option ([ipfs/go-ipfs#3928](https://github.com/ipfs/go-ipfs/pull/3928)) - Apply the megacheck code vetting tool ([ipfs/go-ipfs#3949](https://github.com/ipfs/go-ipfs/pull/3949)) - Expose port 8081 in docker container for /ws listener ([ipfs/go-ipfs#3954](https://github.com/ipfs/go-ipfs/pull/3954)) ### v0.4.9 - 2017-04-30 Ipfs 0.4.9 is a maintenance release that contains several useful bug fixes and improvements. Notably, `ipfs add` has gained the ability to select which CID version will be output. The common ipfs hash that looks like this: `QmRjNgF2mRLDT8AzCPsQbw1EYF2hDTFgfUmJokJPhCApYP` is a multihash. Multihashes allow us to specify the hashing algorithm that was used to verify the data, but it doesn't give us any indication of what format that data might be. To address that issue, we are adding another couple of bytes to the prefix that will allow us to indicate the format of the data referenced by the hash. This new format is called a Content ID, or CID for short. The previous bare multihashes will still be fully supported throughout the entire application as CID version 0. The new format with the type information will be CID version 1. To give an example, the content referenced by the hash above is "Hello Ipfs!". That same content, in the same format (dag-protobuf) using CIDv1 is `zb2rhkgXZVkT2xvDiuUsJENPSbWJy7fdYnsboLBzzEjjZMRoG`. CIDv1 hashes are supported in ipfs versions back to 0.4.5. Nodes running 0.4.4 and older will not be able to load content via CIDv1 and we recommend that they update to a newer version. There are many other use cases for CIDs. Plugins can be written to allow ipfs to natively address content from any other merkletree based system, such as git, bitcoin, zcash and ethereum -- a few systems we've already started work on. Aside from the CID flag, there were many other changes as noted below: - Features - Add support for using CidV1 in 'ipfs add' ([ipfs/go-ipfs#3743](https://github.com/ipfs/go-ipfs/pull/3743)) - Improvements - Use CID as an ETag strong validator ([ipfs/go-ipfs#3869](https://github.com/ipfs/go-ipfs/pull/3869)) - Update go-multihash with keccak and bitcoin hashes ([ipfs/go-ipfs#3833](https://github.com/ipfs/go-ipfs/pull/3833)) - Update go-is-domain to contain new gTLD ([ipfs/go-ipfs#3873](https://github.com/ipfs/go-ipfs/pull/3873)) - Periodically flush cached directories during ipfs add ([ipfs/go-ipfs#3888](https://github.com/ipfs/go-ipfs/pull/3888)) - improved gateway directory listing for sharded nodes ([ipfs/go-ipfs#3897](https://github.com/ipfs/go-ipfs/pull/3897)) - Documentation - Change issue template to use Severity instead of Priority ([ipfs/go-ipfs#3834](https://github.com/ipfs/go-ipfs/pull/3834)) - Fix link to commit hook script in contribute.md ([ipfs/go-ipfs#3863](https://github.com/ipfs/go-ipfs/pull/3863)) - Fix install_unsupported for openbsd, add docs ([ipfs/go-ipfs#3880](https://github.com/ipfs/go-ipfs/pull/3880)) - Bug fixes - Fix wantlist typo in Prometheus metric name ([ipfs/go-ipfs#3841](https://github.com/ipfs/go-ipfs/pull/3841)) - Fix `make install` not using ldflags for git hash ([ipfs/go-ipfs#3838](https://github.com/ipfs/go-ipfs/pull/3838)) - Fix `make install` not installing dependencies ([ipfs/go-ipfs#3848](https://github.com/ipfs/go-ipfs/pull/3848)) - Fix erroneous Cache-Control: immutable on dir listings ([ipfs/go-ipfs#3870](https://github.com/ipfs/go-ipfs/pull/3870)) - Fix bitswap accounting of 'BytesSent' in ledger ([ipfs/go-ipfs#3876](https://github.com/ipfs/go-ipfs/pull/3876)) - Fix gateway handling of sharded directories ([ipfs/go-ipfs#3889](https://github.com/ipfs/go-ipfs/pull/3889)) - Fix sharding memory growth, and fix resolver for unixfs paths ([ipfs/go-ipfs#3890](https://github.com/ipfs/go-ipfs/pull/3890)) - General Changes and Refactorings - Use ctx var consistently in daemon.go ([ipfs/go-ipfs#3864](https://github.com/ipfs/go-ipfs/pull/3864)) - Handle 404 correctly in dist_get tool ([ipfs/go-ipfs#3879](https://github.com/ipfs/go-ipfs/pull/3879)) - Testing - Fix go fuse tests ([ipfs/go-ipfs#3840](https://github.com/ipfs/go-ipfs/pull/3840)) ### v0.4.8 - 2017-03-29 Ipfs 0.4.8 brings with it several improvements, bug fixes, documentation improvements, and the long awaited directory sharding code. Currently, when too many items are added into a unixfs directory, the object gets too large and you may experience issues. To prevent this problem, and generally make working really large directories more efficient, we have implemented a HAMT structure for unixfs. To enable this feature, run: ``` ipfs config --json Experimental.ShardingEnabled true ``` And restart your daemon if it was running. Note: With this setting enabled, the hashes of any newly added directories will be different than they previously were, as the new code will use the sharded HAMT structure for all directories. Also, nodes running ipfs 0.4.7 and earlier will not be able to access directories created with this option. That said, please do give it a try, let us know how it goes, and then take a look at all the other cool things added in 0.4.8 below. - Features - Implement unixfs directory sharding ([ipfs/go-ipfs#3042](https://github.com/ipfs/go-ipfs/pull/3042)) - Add DisableNatPortMap option ([ipfs/go-ipfs#3798](https://github.com/ipfs/go-ipfs/pull/3798)) - Basic Filestore utility commands ([ipfs/go-ipfs#3653](https://github.com/ipfs/go-ipfs/pull/3653)) - Improvements - More Robust GC ([ipfs/go-ipfs#3712](https://github.com/ipfs/go-ipfs/pull/3712)) - Automatically fix permissions for docker volumes ([ipfs/go-ipfs#3744](https://github.com/ipfs/go-ipfs/pull/3744)) - Core API refinements and efficiency improvements ([ipfs/go-ipfs#3493](https://github.com/ipfs/go-ipfs/pull/3493)) - Improve IsPinned() lookups for indirect pins ([ipfs/go-ipfs#3809](https://github.com/ipfs/go-ipfs/pull/3809)) - Documentation - Improve 'name' and 'key' helptexts ([ipfs/go-ipfs#3806](https://github.com/ipfs/go-ipfs/pull/3806)) - Update link to paper in dev.md ([ipfs/go-ipfs#3812](https://github.com/ipfs/go-ipfs/pull/3812)) - Add test to enforce helptext on commands ([ipfs/go-ipfs#2648](https://github.com/ipfs/go-ipfs/pull/2648)) - Bug fixes - Remove bloom filter check on Put call in blockstore ([ipfs/go-ipfs#3782](https://github.com/ipfs/go-ipfs/pull/3782)) - Re-add the GOPATH checking functionality ([ipfs/go-ipfs#3787](https://github.com/ipfs/go-ipfs/pull/3787)) - Use fsrepo.IsInitialized to test for initialization ([ipfs/go-ipfs#3805](https://github.com/ipfs/go-ipfs/pull/3805)) - Return 404 Not Found for failed path resolutions ([ipfs/go-ipfs#3777](https://github.com/ipfs/go-ipfs/pull/3777)) - Fix 'dist\_get' failing without failing ([ipfs/go-ipfs#3818](https://github.com/ipfs/go-ipfs/pull/3818)) - Update iptb with fix for t0130 hanging issue ([ipfs/go-ipfs#3823](https://github.com/ipfs/go-ipfs/pull/3823)) - fix hidden file detection on windows ([ipfs/go-ipfs#3829](https://github.com/ipfs/go-ipfs/pull/3829)) - General Changes and Refactorings - Fix multiple govet warnings ([ipfs/go-ipfs#3824](https://github.com/ipfs/go-ipfs/pull/3824)) - Make Golint happy in the blocks submodule ([ipfs/go-ipfs#3827](https://github.com/ipfs/go-ipfs/pull/3827)) - Testing - Enable codeclimate for automated linting and vetting ([ipfs/go-ipfs#3821](https://github.com/ipfs/go-ipfs/pull/3821)) - Fix EOF test failure with Multipart.Read ([ipfs/go-ipfs#3804](https://github.com/ipfs/go-ipfs/pull/3804)) ### v0.4.7 - 2017-03-15 Ipfs 0.4.7 contains several exciting new features! First off, The long awaited filestore feature has been merged, allowing users the option to not have ipfs store chunked copies of added files in the blockstore, pushing to burden of ensuring those files are not changed to the user. The filestore feature is currently still experimental, and must be enabled in your config with: ``` ipfs config --json Experimental.FilestoreEnabled true ``` before it can be used. Please see [this issue](https://github.com/ipfs/go-ipfs/issues/3397#issuecomment-284337564) for more details. Next up, We have merged initial support for ipfs 'Private Networks'. This feature allows users to run ipfs in a mode that will only connect to other peers in the private network. This feature, like the filestore is being released experimentally, but if you're interested please try it out. Instructions for setting it up can be found [here](https://github.com/ipfs/go-ipfs/issues/3397#issuecomment-284341649). This release also enables support for the 'mplex' stream muxer by default. This stream multiplexing protocol was available previously via the `--enable-mplex-experiment` daemon flag, but has now graduated to being 'less experimental' and no longer requires the flag to use it. Aside from those, we have a good number of bug fixes, perf improvements and new tests. Heres a list of highlights: - Features - Implement basic filestore 'no-copy' functionality ([ipfs/go-ipfs#3629](https://github.com/ipfs/go-ipfs/pull/3629)) - Add support for private ipfs networks ([ipfs/go-ipfs#3697](https://github.com/ipfs/go-ipfs/pull/3697)) - Enable 'mplex' stream muxer by default ([ipfs/go-ipfs#3725](https://github.com/ipfs/go-ipfs/pull/3725)) - Add `--quieter` option to `ipfs add` ([ipfs/go-ipfs#3770](https://github.com/ipfs/go-ipfs/pull/3770)) - Report progress during `pin add` via `--progress` ([ipfs/go-ipfs#3671](https://github.com/ipfs/go-ipfs/pull/3671)) - Improvements - Allow `ipfs get` to handle content added with raw leaves option ([ipfs/go-ipfs#3757](https://github.com/ipfs/go-ipfs/pull/3757)) - Fix accuracy of progress bar on `ipfs get` ([ipfs/go-ipfs#3758](https://github.com/ipfs/go-ipfs/pull/3758)) - Limit number of objects in batches to prevent too many fds issue ([ipfs/go-ipfs#3756](https://github.com/ipfs/go-ipfs/pull/3756)) - Add more info to bitswap stat ([ipfs/go-ipfs#3635](https://github.com/ipfs/go-ipfs/pull/3635)) - Add multiple performance metrics ([ipfs/go-ipfs#3615](https://github.com/ipfs/go-ipfs/pull/3615)) - Make `dist_get` fall back to other downloaders if one fails ([ipfs/go-ipfs#3692](https://github.com/ipfs/go-ipfs/pull/3692)) - Documentation - Add Arch Linux install instructions to readme ([ipfs/go-ipfs#3742](https://github.com/ipfs/go-ipfs/pull/3742)) - Improve release checklist document ([ipfs/go-ipfs#3717](https://github.com/ipfs/go-ipfs/pull/3717)) - Bug fixes - Fix drive root parsing on windows ([ipfs/go-ipfs#3328](https://github.com/ipfs/go-ipfs/pull/3328)) - Fix panic in ipfs get when passing no parameters to API ([ipfs/go-ipfs#3768](https://github.com/ipfs/go-ipfs/pull/3768)) - Fix breakage of `ipfs pin add` api output ([ipfs/go-ipfs#3760](https://github.com/ipfs/go-ipfs/pull/3760)) - Fix issue in DHT queries that was causing poor record replication ([ipfs/go-ipfs#3748](https://github.com/ipfs/go-ipfs/pull/3748)) - Fix `ipfs mount` crashing if no name was published before ([ipfs/go-ipfs#3728](https://github.com/ipfs/go-ipfs/pull/3728)) - Add `self` key to the `ipfs key list` listing ([ipfs/go-ipfs#3734](https://github.com/ipfs/go-ipfs/pull/3734)) - Fix panic when shutting down `ipfs daemon` pre gateway setup ([ipfs/go-ipfs#3723](https://github.com/ipfs/go-ipfs/pull/3723)) - General Changes and Refactorings - Refactor `EnumerateChildren` to avoid need for bestEffort parameter ([ipfs/go-ipfs#3700](https://github.com/ipfs/go-ipfs/pull/3700)) - Update fuse dependency, fixing several issues ([ipfs/go-ipfs#3727](https://github.com/ipfs/go-ipfs/pull/3727)) - Add `install_unsupported` makefile target for 'exotic' systems ([ipfs/go-ipfs#3719](https://github.com/ipfs/go-ipfs/pull/3719)) - Deprecate implicit daemon argument in Dockerfile ([ipfs/go-ipfs#3685](https://github.com/ipfs/go-ipfs/pull/3685)) - Testing - Add test to ensure helptext is under 80 columns wide ([ipfs/go-ipfs#3774](https://github.com/ipfs/go-ipfs/pull/3774)) - Add unit tests for auto migration code ([ipfs/go-ipfs#3618](https://github.com/ipfs/go-ipfs/pull/3618)) - Fix iptb stop issue in sharness tests ([ipfs/go-ipfs#3714](https://github.com/ipfs/go-ipfs/pull/3714)) ### v0.4.6 - 2017-02-21 Ipfs 0.4.6 contains several bug fixes related to migrations and also contains a few other improvements to other parts of the codebase. Notably: - The default config will now contain some ipv6 addresses for bootstrap nodes. - `ipfs pin add` should be faster and consume less memory. - Pinning thousands of files no longer causes superlinear usage of storage space. - Improvements - Make pinset sharding deterministic ([ipfs/go-ipfs#3640](https://github.com/ipfs/go-ipfs/pull/3640)) - Update to go-multihash with blake2 ([ipfs/go-ipfs#3649](https://github.com/ipfs/go-ipfs/pull/3649)) - Pass cids instead of nodes around in EnumerateChildrenAsync ([ipfs/go-ipfs#3598](https://github.com/ipfs/go-ipfs/pull/3598)) - Add /ip6 bootstrap nodes ([ipfs/go-ipfs#3523](https://github.com/ipfs/go-ipfs/pull/3523)) - Add sub-object support to `dag get` command ([ipfs/go-ipfs#3687](https://github.com/ipfs/go-ipfs/pull/3687)) - Add half-closed streams support to multiplex experiment ([ipfs/go-ipfs#3695](https://github.com/ipfs/go-ipfs/pull/3695)) - Documentation - Add the snap installation instructions ([ipfs/go-ipfs#3663](https://github.com/ipfs/go-ipfs/pull/3663)) - Add closed PRs, Issues throughput ([ipfs/go-ipfs#3602](https://github.com/ipfs/go-ipfs/pull/3602)) - Bug fixes - Fix auto-migration on docker nodes ([ipfs/go-ipfs#3698](https://github.com/ipfs/go-ipfs/pull/3698)) - Update flatfs to v1.1.2, fixing directory fd issue ([ipfs/go-ipfs#3711](https://github.com/ipfs/go-ipfs/pull/3711)) - General Changes and Refactorings - Remove `FindProviders` from routing mocks ([ipfs/go-ipfs#3617](https://github.com/ipfs/go-ipfs/pull/3617)) - Use Marshalers instead of PostRun to process `block rm` output ([ipfs/go-ipfs#3708](https://github.com/ipfs/go-ipfs/pull/3708)) - Testing - Makefile rework and sharness test coverage ([ipfs/go-ipfs#3504](https://github.com/ipfs/go-ipfs/pull/3504)) - Print out all daemon stderr files when iptb stop fails ([ipfs/go-ipfs#3701](https://github.com/ipfs/go-ipfs/pull/3701)) - Add tests for recursively pinning a dag ([ipfs/go-ipfs#3691](https://github.com/ipfs/go-ipfs/pull/3691)) - Fix lack of commit hash during build ([ipfs/go-ipfs#3705](https://github.com/ipfs/go-ipfs/pull/3705)) ### v0.4.5 - 2017-02-11 #### Changes from rc3 to rc4 - Update to fixed webui. ([ipfs/go-ipfs#3669](https://github.com/ipfs/go-ipfs/pull/3669)) #### Changes from rc2 to rc3 - Fix handling of null arrays in cbor ipld objects. ([ipfs/go-ipfs#3666](https://github.com/ipfs/go-ipfs/pull/3666)) - Add env var to enable yamux debug logging. ([ipfs/go-ipfs#3668](https://github.com/ipfs/go-ipfs/pull/3668)) - Fix libc check during auto-migrations. ([ipfs/go-ipfs#3665](https://github.com/ipfs/go-ipfs/pull/3665)) #### Changes from rc1 to rc2 - Fixed json output of ipld objects in `ipfs dag get` ([ipfs/go-ipfs#3655](https://github.com/ipfs/go-ipfs/pull/3655)) #### Changes since 0.4.4 - Notable changes - IPLD and CIDs - Rework go-ipfs to use Content IDs ([ipfs/go-ipfs#3187](https://github.com/ipfs/go-ipfs/pull/3187)) ([ipfs/go-ipfs#3290](https://github.com/ipfs/go-ipfs/pull/3290)) - Turn merkledag.Node into an interface ([ipfs/go-ipfs#3301](https://github.com/ipfs/go-ipfs/pull/3301)) - Implement cbor ipld nodes ([ipfs/go-ipfs#3325](https://github.com/ipfs/go-ipfs/pull/3325)) - Allow cid format selection in block put command ([ipfs/go-ipfs#3324](https://github.com/ipfs/go-ipfs/pull/3324)) ([ipfs/go-ipfs#3483](https://github.com/ipfs/go-ipfs/pull/3483)) - Bitswap protocol extension to handle cids ([ipfs/go-ipfs#3297](https://github.com/ipfs/go-ipfs/pull/3297)) - Add dag get to read-only api ([ipfs/go-ipfs#3499](https://github.com/ipfs/go-ipfs/pull/3499)) - Raw Nodes - Implement 'Raw Node' node type for addressing raw data ([ipfs/go-ipfs#3307](https://github.com/ipfs/go-ipfs/pull/3307)) - Optimize DagService GetLinks for Raw Nodes. ([ipfs/go-ipfs#3351](https://github.com/ipfs/go-ipfs/pull/3351)) - Experimental PubSub - Added a very basic pubsub implementation ([ipfs/go-ipfs#3202](https://github.com/ipfs/go-ipfs/pull/3202)) - Core API - gateway: use core api for serving GET/HEAD/POST ([ipfs/go-ipfs#3244](https://github.com/ipfs/go-ipfs/pull/3244)) - Improvements - Disable auto-gc check in 'ipfs cat' ([ipfs/go-ipfs#3100](https://github.com/ipfs/go-ipfs/pull/3100)) - Add `bitswap ledger` command ([ipfs/go-ipfs#2852](https://github.com/ipfs/go-ipfs/pull/2852)) - Add `ipfs block rm` command. ([ipfs/go-ipfs#2962](https://github.com/ipfs/go-ipfs/pull/2962)) - Add config option to disable bandwidth metrics ([ipfs/go-ipfs#3381](https://github.com/ipfs/go-ipfs/pull/3381)) - Add experimental dht 'client mode' flag ([ipfs/go-ipfs#3269](https://github.com/ipfs/go-ipfs/pull/3269)) - Add config option to set reprovider interval ([ipfs/go-ipfs#3101](https://github.com/ipfs/go-ipfs/pull/3101)) - Add `ipfs dht provide` command ([ipfs/go-ipfs#3106](https://github.com/ipfs/go-ipfs/pull/3106)) - Add stream info to `ipfs swarm peers -v` ([ipfs/go-ipfs#3352](https://github.com/ipfs/go-ipfs/pull/3352)) - Add option to enable go-multiplex experiment ([ipfs/go-ipfs#3447](https://github.com/ipfs/go-ipfs/pull/3447)) - Basic Keystore implementation ([ipfs/go-ipfs#3472](https://github.com/ipfs/go-ipfs/pull/3472)) - Make `ipfs add --local` not send providers messages ([ipfs/go-ipfs#3102](https://github.com/ipfs/go-ipfs/pull/3102)) - Fix bug in `ipfs tar add` that buffered input in memory ([ipfs/go-ipfs#3334](https://github.com/ipfs/go-ipfs/pull/3334)) - Make blockstore retry operations on temporary errors ([ipfs/go-ipfs#3091](https://github.com/ipfs/go-ipfs/pull/3091)) - Don't hold the PinLock in adder when not pinning. ([ipfs/go-ipfs#3222](https://github.com/ipfs/go-ipfs/pull/3222)) - Validate repo/api file and improve error message ([ipfs/go-ipfs#3219](https://github.com/ipfs/go-ipfs/pull/3219)) - no longer hard code gomaxprocs ([ipfs/go-ipfs#3357](https://github.com/ipfs/go-ipfs/pull/3357)) - Updated Bash complete script ([ipfs/go-ipfs#3377](https://github.com/ipfs/go-ipfs/pull/3377)) - Remove expensive debug statement in blockstore AllKeysChan ([ipfs/go-ipfs#3384](https://github.com/ipfs/go-ipfs/pull/3384)) - Remove GC timeout, fix GC tests ([ipfs/go-ipfs#3494](https://github.com/ipfs/go-ipfs/pull/3494)) - Fix `ipfs pin add` resource consumption ([ipfs/go-ipfs#3495](https://github.com/ipfs/go-ipfs/pull/3495)) ([ipfs/go-ipfs#3571](https://github.com/ipfs/go-ipfs/pull/3571)) - Add IPNS entry to DHT cache after publish ([ipfs/go-ipfs#3501](https://github.com/ipfs/go-ipfs/pull/3501)) - Add in `--routing=none` daemon option ([ipfs/go-ipfs#3605](https://github.com/ipfs/go-ipfs/pull/3605)) - Bitswap - Don't re-provide blocks we've provided very recently ([ipfs/go-ipfs#3105](https://github.com/ipfs/go-ipfs/pull/3105)) - Add a deadline to sendmsg calls ([ipfs/go-ipfs#3445](https://github.com/ipfs/go-ipfs/pull/3445)) - cleanup bitswap and handle message send failure slightly better ([ipfs/go-ipfs#3408](https://github.com/ipfs/go-ipfs/pull/3408)) - Increase wantlist resend delay to one minute ([ipfs/go-ipfs#3448](https://github.com/ipfs/go-ipfs/pull/3448)) - Fix issue where wantlist fullness wasn't included in messages ([ipfs/go-ipfs#3461](https://github.com/ipfs/go-ipfs/pull/3461)) - Only pass keys down newBlocks chan in bitswap ([ipfs/go-ipfs#3271](https://github.com/ipfs/go-ipfs/pull/3271)) - Bug fixes - gateway: fix --writable flag ([ipfs/go-ipfs#3206](https://github.com/ipfs/go-ipfs/pull/3206)) - Fix relative seek in unixfs not expanding file properly ([ipfs/go-ipfs#3095](https://github.com/ipfs/go-ipfs/pull/3095)) - Update multicodec service names for ipfs services ([ipfs/go-ipfs#3132](https://github.com/ipfs/go-ipfs/pull/3132)) - dht: add missing protocol ID to newStream call ([ipfs/go-ipfs#3203](https://github.com/ipfs/go-ipfs/pull/3203)) - Return immediately on namesys error ([ipfs/go-ipfs#3345](https://github.com/ipfs/go-ipfs/pull/3345)) - Improve osxfuse handling ([ipfs/go-ipfs#3098](https://github.com/ipfs/go-ipfs/pull/3098)) ([ipfs/go-ipfs#3413](https://github.com/ipfs/go-ipfs/pull/3413)) - commands: fix opt.Description panic when desc was empty ([ipfs/go-ipfs#3521](https://github.com/ipfs/go-ipfs/pull/3521)) - Fixes #3133: Properly handle release candidates in version comparison ([ipfs/go-ipfs#3136](https://github.com/ipfs/go-ipfs/pull/3136)) - Don't drop error in readStreamedJson. ([ipfs/go-ipfs#3276](https://github.com/ipfs/go-ipfs/pull/3276)) - Error out on invalid `--routing` option ([ipfs/go-ipfs#3482](https://github.com/ipfs/go-ipfs/pull/3482)) - Respect contexts when returning diagnostics responses ([ipfs/go-ipfs#3353](https://github.com/ipfs/go-ipfs/pull/3353)) - Fix json marshalling of pbnode ([ipfs/go-ipfs#3507](https://github.com/ipfs/go-ipfs/pull/3507)) - General changes and refactorings - Disable Suborigins the spec changed and our impl conflicts ([ipfs/go-ipfs#3519](https://github.com/ipfs/go-ipfs/pull/3519)) - Avoid sending provide messages for pinsets ([ipfs/go-ipfs#3103](https://github.com/ipfs/go-ipfs/pull/3103)) - Refactor cli handling to expose argument parsing functionality ([ipfs/go-ipfs#3308](https://github.com/ipfs/go-ipfs/pull/3308)) - Create a FilestoreNode object to carry PosInfo ([ipfs/go-ipfs#3314](https://github.com/ipfs/go-ipfs/pull/3314)) - Print 'n/a' instead of zero latency in `ipfs swarm peers` ([ipfs/go-ipfs#3491](https://github.com/ipfs/go-ipfs/pull/3491)) - Add DAGService.GetLinks() method to optimize traversals. ([ipfs/go-ipfs#3255](https://github.com/ipfs/go-ipfs/pull/3255)) - Make path resolver no longer require whole IpfsNode for construction ([ipfs/go-ipfs#3321](https://github.com/ipfs/go-ipfs/pull/3321)) - Distinguish between Offline and Local Modes of daemon operation. ([ipfs/go-ipfs#3259](https://github.com/ipfs/go-ipfs/pull/3259)) - Separate out the GC Locking from the Blockstore interface. ([ipfs/go-ipfs#3348](https://github.com/ipfs/go-ipfs/pull/3348)) - Avoid unnecessary allocs in datastore key handling ([ipfs/go-ipfs#3407](https://github.com/ipfs/go-ipfs/pull/3407)) - Use NextSync method for datastore queries ([ipfs/go-ipfs#3386](https://github.com/ipfs/go-ipfs/pull/3386)) - Switch unixfs.Metadata.MimeType to optional ([ipfs/go-ipfs#3458](https://github.com/ipfs/go-ipfs/pull/3458)) - Fix path parsing in `ipfs name publish` ([ipfs/go-ipfs#3592](https://github.com/ipfs/go-ipfs/pull/3592)) - Fix inconsistent `ipfs stats bw` formatting ([ipfs/go-ipfs#3554](https://github.com/ipfs/go-ipfs/pull/3554)) - Set the libp2p agent version based on version string ([ipfs/go-ipfs#3569](https://github.com/ipfs/go-ipfs/pull/3569)) - Cross Platform Changes - Fix 'dist_get' script on BSDs. ([ipfs/go-ipfs#3264](https://github.com/ipfs/go-ipfs/pull/3264)) - ulimit: Tune resource limits on BSDs ([ipfs/go-ipfs#3374](https://github.com/ipfs/go-ipfs/pull/3374)) - Metrics - Introduce go-metrics-interface ([ipfs/go-ipfs#3189](https://github.com/ipfs/go-ipfs/pull/3189)) - Fix metrics injection ([ipfs/go-ipfs#3315](https://github.com/ipfs/go-ipfs/pull/3315)) - Misc - Bump Go requirement to 1.7 ([ipfs/go-ipfs#3111](https://github.com/ipfs/go-ipfs/pull/3111)) - Merge 0.4.3 release candidate changes back into master ([ipfs/go-ipfs#3248](https://github.com/ipfs/go-ipfs/pull/3248)) - Add security@ipfs.io GPG key to assets ([ipfs/go-ipfs#2997](https://github.com/ipfs/go-ipfs/pull/2997)) - Improve makefiles ([ipfs/go-ipfs#2999](https://github.com/ipfs/go-ipfs/pull/2999)) ([ipfs/go-ipfs#3265](https://github.com/ipfs/go-ipfs/pull/3265)) - Refactor install.sh script ([ipfs/go-ipfs#3194](https://github.com/ipfs/go-ipfs/pull/3194)) - Add test check for go code formatting ([ipfs/go-ipfs#3421](https://github.com/ipfs/go-ipfs/pull/3421)) - bin: dist_get script: prevents get_go_vars() returns same values twice ([ipfs/go-ipfs#3079](https://github.com/ipfs/go-ipfs/pull/3079)) - Dependencies - Update libp2p to have fixed spdystream dep ([ipfs/go-ipfs#3210](https://github.com/ipfs/go-ipfs/pull/3210)) - Update libp2p and dht packages ([ipfs/go-ipfs#3263](https://github.com/ipfs/go-ipfs/pull/3263)) - Update to libp2p 4.0.1 and propagate other changes ([ipfs/go-ipfs#3284](https://github.com/ipfs/go-ipfs/pull/3284)) - Update to libp2p 4.0.4 ([ipfs/go-ipfs#3361](https://github.com/ipfs/go-ipfs/pull/3361)) - Update go-libp2p across codebase ([ipfs/go-ipfs#3406](https://github.com/ipfs/go-ipfs/pull/3406)) - Update to go-libp2p 4.1.0 ([ipfs/go-ipfs#3373](https://github.com/ipfs/go-ipfs/pull/3373)) - Update deps for libp2p 3.4.0 ([ipfs/go-ipfs#3110](https://github.com/ipfs/go-ipfs/pull/3110)) - Update go-libp2p-swarm with deadlock fixes ([ipfs/go-ipfs#3339](https://github.com/ipfs/go-ipfs/pull/3339)) - Update to new cid and ipld node packages ([ipfs/go-ipfs#3326](https://github.com/ipfs/go-ipfs/pull/3326)) - Update to newer ipld node interface with Copy and better Tree ([ipfs/go-ipfs#3391](https://github.com/ipfs/go-ipfs/pull/3391)) - Update experimental go-multiplex to 0.2.6 ([ipfs/go-ipfs#3475](https://github.com/ipfs/go-ipfs/pull/3475)) - Rework routing interfaces to make separation easier ([ipfs/go-ipfs#3107](https://github.com/ipfs/go-ipfs/pull/3107)) - Update to dht code with fixed GetClosestPeers ([ipfs/go-ipfs#3346](https://github.com/ipfs/go-ipfs/pull/3346)) - Move go-is-domain to gx ([ipfs/go-ipfs#3077](https://github.com/ipfs/go-ipfs/pull/3077)) - Extract thirdparty/loggables and thirdparty/peerset ([ipfs/go-ipfs#3204](https://github.com/ipfs/go-ipfs/pull/3204)) - Completely remove go-key dep ([ipfs/go-ipfs#3439](https://github.com/ipfs/go-ipfs/pull/3439)) - Remove randbo dep, its no longer needed ([ipfs/go-ipfs#3118](https://github.com/ipfs/go-ipfs/pull/3118)) - Update libp2p for identify configuration updates ([ipfs/go-ipfs#3539](https://github.com/ipfs/go-ipfs/pull/3539)) - Use newer flatfs sharding scheme ([ipfs/go-ipfs#3608](https://github.com/ipfs/go-ipfs/pull/3608)) - Testing - fix test_fsh arg quoting in ipfs-test-lib ([ipfs/go-ipfs#3085](https://github.com/ipfs/go-ipfs/pull/3085)) - 100% coverage for blocks/blocksutil ([ipfs/go-ipfs#3090](https://github.com/ipfs/go-ipfs/pull/3090)) - 100% coverage on blocks/set ([ipfs/go-ipfs#3084](https://github.com/ipfs/go-ipfs/pull/3084)) - 81% coverage on blockstore ([ipfs/go-ipfs#3074](https://github.com/ipfs/go-ipfs/pull/3074)) - 80% coverage of unixfs/mod ([ipfs/go-ipfs#3096](https://github.com/ipfs/go-ipfs/pull/3096)) - 82% coverage on blocks ([ipfs/go-ipfs#3086](https://github.com/ipfs/go-ipfs/pull/3086)) - 87% coverage on unixfs ([ipfs/go-ipfs#3492](https://github.com/ipfs/go-ipfs/pull/3492)) - Improve coverage on routing/offline ([ipfs/go-ipfs#3516](https://github.com/ipfs/go-ipfs/pull/3516)) - Add test for flags package ([ipfs/go-ipfs#3449](https://github.com/ipfs/go-ipfs/pull/3449)) - improve test coverage on merkledag package ([ipfs/go-ipfs#3113](https://github.com/ipfs/go-ipfs/pull/3113)) - 80% coverage of unixfs/io ([ipfs/go-ipfs#3097](https://github.com/ipfs/go-ipfs/pull/3097)) - Accept more than one digit in repo version tests ([ipfs/go-ipfs#3130](https://github.com/ipfs/go-ipfs/pull/3130)) - Fix typo in hash in t0050 ([ipfs/go-ipfs#3170](https://github.com/ipfs/go-ipfs/pull/3170)) - fix bug in pinsets and add a stress test for the scenario ([ipfs/go-ipfs#3273](https://github.com/ipfs/go-ipfs/pull/3273)) ([ipfs/go-ipfs#3302](https://github.com/ipfs/go-ipfs/pull/3302)) - Report coverage to codecov ([ipfs/go-ipfs#3473](https://github.com/ipfs/go-ipfs/pull/3473)) - Add test for 'ipfs config replace' ([ipfs/go-ipfs#3073](https://github.com/ipfs/go-ipfs/pull/3073)) - Fix netcat on macOS not closing socket when the stdin sends EOF ([ipfs/go-ipfs#3515](https://github.com/ipfs/go-ipfs/pull/3515)) - Documentation - Update dns help with a correct domain name ([ipfs/go-ipfs#3087](https://github.com/ipfs/go-ipfs/pull/3087)) - Add period to `ipfs pin rm` ([ipfs/go-ipfs#3088](https://github.com/ipfs/go-ipfs/pull/3088)) - Make all Taglines use imperative mood ([ipfs/go-ipfs#3041](https://github.com/ipfs/go-ipfs/pull/3041)) - Document listing commands better ([ipfs/go-ipfs#3083](https://github.com/ipfs/go-ipfs/pull/3083)) - Add notes to readme on building for uncommon systems ([ipfs/go-ipfs#3051](https://github.com/ipfs/go-ipfs/pull/3051)) - Add branch naming conventions doc ([ipfs/go-ipfs#3035](https://github.com/ipfs/go-ipfs/pull/3035)) - Replace keyword with <> ([ipfs/go-ipfs#3129](https://github.com/ipfs/go-ipfs/pull/3129)) - Fix Add() docs regarding pinning ([ipfs/go-ipfs#3513](https://github.com/ipfs/go-ipfs/pull/3513)) - Add sudo to install commands. ([ipfs/go-ipfs#3201](https://github.com/ipfs/go-ipfs/pull/3201)) - Add docs for `"commands".Command.Run` ([ipfs/go-ipfs#3382](https://github.com/ipfs/go-ipfs/pull/3382)) - Put config keys in proper case ([ipfs/go-ipfs#3365](https://github.com/ipfs/go-ipfs/pull/3365)) - Fix link in `ipfs stats bw` help message ([ipfs/go-ipfs#3620](https://github.com/ipfs/go-ipfs/pull/3620)) ## v0.4.4 - 2016-10-11 This release contains an important hotfix for a bug we discovered in how pinning works. If you had a large number of pins, new pins would overwrite existing pins. Apart from the hotfix, this release is equal to the previous release 0.4.3. - Fix bug in pinsets fanout, and add stress test. (@whyrusleeping, [ipfs/go-ipfs#3273](https://github.com/ipfs/go-ipfs/pull/3273)) We published a [detailed account of the bug and fix in a blog post](https://ipfs.io/blog/21-go-ipfs-0-4-4-released/). ## v0.4.3 - 2016-09-20 There have been no changes since the last release candidate 0.4.3-rc4. \o/ ## v0.4.3-rc4 - 2016-09-09 This release candidate fixes issues in Bitswap and the `ipfs add` command, and improves testing. We plan for this to be the last release candidate before the release of go-ipfs v0.4.3. With this release candidate, we're also moving go-ipfs to Go 1.7, which we expect will yield improvements in runtime performance, memory usage, build time and size of the release binaries. - Require Go 1.7. (@whyrusleeping, @Kubuxu, @lgierth, [ipfs/go-ipfs#3163](https://github.com/ipfs/go-ipfs/pull/3163)) - For this purpose, switch Docker image from Alpine 3.4 to Alpine Edge. - Fix cancellation of Bitswap `wantlist` entries. (@whyrusleeping, [ipfs/go-ipfs#3182](https://github.com/ipfs/go-ipfs/pull/3182)) - Fix clearing of `active` state of Bitswap provider queries. (@whyrusleeping, [ipfs/go-ipfs#3169](https://github.com/ipfs/go-ipfs/pull/3169)) - Fix a panic in the DHT code. (@Kubuxu, [ipfs/go-ipfs#3200](https://github.com/ipfs/go-ipfs/pull/3200)) - Improve handling of `Identity` field in `ipfs config` command. (@Kubuxu, @whyrusleeping, [ipfs/go-ipfs#3141](https://github.com/ipfs/go-ipfs/pull/3141)) - Fix explicit adding of symlinked files and directories. (@kevina, [ipfs/go-ipfs#3135](https://github.com/ipfs/go-ipfs/pull/3135)) - Fix bash auto-completion of `ipfs daemon --unrestricted-api` option. (@lgierth, [ipfs/go-ipfs#3159](https://github.com/ipfs/go-ipfs/pull/3159)) - Introduce a new timeout tool for tests to avoid licensing issues. (@Kubuxu, [ipfs/go-ipfs#3152](https://github.com/ipfs/go-ipfs/pull/3152)) - Improve output for migrations of fs-repo. (@lgierth, [ipfs/go-ipfs#3158](https://github.com/ipfs/go-ipfs/pull/3158)) - Fix info notice of commands taking input from stdin. (@Kubuxu, [ipfs/go-ipfs#3134](https://github.com/ipfs/go-ipfs/pull/3134)) - Bring back a few tests for stdin handling of `ipfs cat` and `ipfs add`. (@Kubuxu, [ipfs/go-ipfs#3144](https://github.com/ipfs/go-ipfs/pull/3144)) - Improve sharness tests for `ipfs repo verify` command. (@whyrusleeping, [ipfs/go-ipfs#3148](https://github.com/ipfs/go-ipfs/pull/3148)) - Improve sharness tests for CORS headers on the gateway. (@Kubuxu, [ipfs/go-ipfs#3142](https://github.com/ipfs/go-ipfs/pull/3142)) - Improve tests for pinning within `ipfs files`. (@kevina, [ipfs/go-ipfs#3151](https://github.com/ipfs/go-ipfs/pull/3151)) - Improve tests for the automatic raising of file descriptor limits. (@whyrusleeping, [ipfs/go-ipfs#3149](https://github.com/ipfs/go-ipfs/pull/3149)) ## v0.4.3-rc3 - 2016-08-11 This release candidate fixes a panic that occurs when input from stdin was expected, but none was given: [ipfs/go-ipfs#3050](https://github.com/ipfs/go-ipfs/pull/3050) ## v0.4.3-rc2 - 2016-08-04 This release includes bug fixes and fixes for regressions that were introduced between 0.4.2 and 0.4.3-rc1. - Regressions - Fix daemon panic when there is no multipart input provided over the HTTP API. (@whyrusleeping, [ipfs/go-ipfs#2989](https://github.com/ipfs/go-ipfs/pull/2989)) - Fix `ipfs refs --edges` not printing edges. (@Kubuxu, [ipfs/go-ipfs#3007](https://github.com/ipfs/go-ipfs/pull/3007)) - Fix progress option for `ipfs add` defaulting to true on the HTTP API. (@whyrusleeping, [ipfs/go-ipfs#3025](https://github.com/ipfs/go-ipfs/pull/3025)) - Fix erroneous printing of stdin reading message. (@whyrusleeping, [ipfs/go-ipfs#3033](https://github.com/ipfs/go-ipfs/pull/3033)) - Fix panic caused by passing `--mount` and `--offline` flags to `ipfs daemon`. (@Kubuxu, [ipfs/go-ipfs#3022](https://github.com/ipfs/go-ipfs/pull/3022)) - Fix symlink path resolution on windows. (@Kubuxu, [ipfs/go-ipfs#3023](https://github.com/ipfs/go-ipfs/pull/3023)) - Add in code to prevent issue 3032 from crashing the daemon. (@whyrusleeping, [ipfs/go-ipfs#3037](https://github.com/ipfs/go-ipfs/pull/3037)) ## v0.4.3-rc1 - 2016-07-23 This is a maintenance release which comes with a couple of nice enhancements, and improves the performance of Storage, Bitswap, as well as Content and Peer Routing. It also introduces a handful of new commands and options, and fixes a good bunch of bugs. This is the first Release Candidate. Unless there are vulnerabilities or regressions discovered, the final 0.4.3 release will happen about one week from now. - Security Vulnerability - The `master` branch if go-ipfs suffered from a vulnerability for about 3 weeks. It allowed an attacker to use an iframe to request malicious HTML and JS from the API of a local go-ipfs node. The attacker could then gain unrestricted access to the node's API, and e.g. extract the private key. We fixed this issue by reintroducing restrictions on which particular objects can be loaded through the API (@lgierth, [ipfs/go-ipfs#2949](https://github.com/ipfs/go-ipfs/pull/2949)), and by completely excluding the private key from the API (@Kubuxu, [ipfs/go-ipfs#2957](https://github.com/ipfs/go-ipfs/pull/2957)). We will also work on more hardening of the API in the next release. - **The previous release 0.4.2 is not vulnerable. That means if you're using official binaries from [dist.ipfs.tech](https://dist.ipfs.tech) you're not affected.** If you're running go-ipfs built from the `master` branch between June 17th ([ipfs/go-ipfs@1afebc21](https://github.com/ipfs/go-ipfs/commit/1afebc21f324982141ca8a29710da0d6f83ca804)) and July 7th ([ipfs/go-ipfs@39bef0d5](https://github.com/ipfs/go-ipfs/commit/39bef0d5b01f70abf679fca2c4d078a2d55620e2)), please update to v0.4.3-rc1 immediately. - We are grateful to the group of independent researchers who made us aware of this vulnerability. We wanna use this opportunity to reiterate that we're very happy about any additional review of pull requests and releases. You can contact us any time at security@ipfs.io (GPG [4B9665FB 92636D17 7C7A86D3 50AAE8A9 59B13AF3](https://pgp.mit.edu/pks/lookup?op=get&search=0x50AAE8A959B13AF3)). - Notable changes - Improve Bitswap performance. (@whyrusleeping, [ipfs/go-ipfs#2727](https://github.com/ipfs/go-ipfs/pull/2727), [ipfs/go-ipfs#2798](https://github.com/ipfs/go-ipfs/pull/2798)) - Improve Content Routing and Peer Routing performance. (@whyrusleeping, [ipfs/go-ipfs#2817](https://github.com/ipfs/go-ipfs/pull/2817), [ipfs/go-ipfs#2841](https://github.com/ipfs/go-ipfs/pull/2841)) - Improve datastore, blockstore, and dagstore performance. (@kevina, @Kubuxu, @whyrusleeping [ipfs/go-datastore#43](https://github.com/ipfs/go-datastore/pull/43), [ipfs/go-ipfs#2885](https://github.com/ipfs/go-ipfs/pull/2885), [ipfs/go-ipfs#2961](https://github.com/ipfs/go-ipfs/pull/2961), [ipfs/go-ipfs#2953](https://github.com/ipfs/go-ipfs/pull/2953), [ipfs/go-ipfs#2960](https://github.com/ipfs/go-ipfs/pull/2960)) - Content Providers are now stored on disk to gain savings on process memory. (@whyrusleeping, [ipfs/go-ipfs#2804](https://github.com/ipfs/go-ipfs/pull/2804), [ipfs/go-ipfs#2860](https://github.com/ipfs/go-ipfs/pull/2860)) - Migrations of the fs-repo (usually stored at `~/.ipfs`) now run automatically. If there's a TTY available, you'll get prompted when running `ipfs daemon`, and in addition you can use the `--migrate=true` or `--migrate=false` options to avoid the prompt. (@whyrusleeping, @lgierth, [ipfs/go-ipfs#2939](https://github.com/ipfs/go-ipfs/pull/2939)) - The internal naming of blocks in the blockstore has changed, which requires a migration of the fs-repo, from version 3 to 4. (@whyrusleeping, [ipfs/go-ipfs#2903](https://github.com/ipfs/go-ipfs/pull/2903)) - We now automatically raise the file descriptor limit to 1024 if necessary. (@whyrusleeping, [ipfs/go-ipfs#2884](https://github.com/ipfs/go-ipfs/pull/2884), [ipfs/go-ipfs#2891](https://github.com/ipfs/go-ipfs/pull/2891)) - After a long struggle with deadlocks and hanging connections, we've decided to disable the uTP transport by default for now. (@whyrusleeping, [ipfs/go-ipfs#2840](https://github.com/ipfs/go-ipfs/pull/2840), [ipfs/go-libp2p-transport@88244000](https://github.com/ipfs/go-libp2p-transport/commit/88244000f0ce8851ffcfbac746ebc0794b71d2a4)) - There is now documentation for the configuration options in `docs/config.md`. (@whyrusleeping, [ipfs/go-ipfs#2974](https://github.com/ipfs/go-ipfs/pull/2974)) - All commands now sanely handle the combination of stdin and optional flags in certain edge cases. (@lgierth, [ipfs/go-ipfs#2952](https://github.com/ipfs/go-ipfs/pull/2952)) - New Features - Add `--offline` option to `ipfs daemon` command, which disables all swarm networking. (@Kubuxu, [ipfs/go-ipfs#2696](https://github.com/ipfs/go-ipfs/pull/2696), [ipfs/go-ipfs#2867](https://github.com/ipfs/go-ipfs/pull/2867)) - Add `Datastore.HashOnRead` option for verifying block hashes on read access. (@Kubuxu, [ipfs/go-ipfs#2904](https://github.com/ipfs/go-ipfs/pull/2904)) - Add `Datastore.BloomFilterSize` option for tuning the blockstore's new lookup bloom filter. (@Kubuxu, [ipfs/go-ipfs#2973](https://github.com/ipfs/go-ipfs/pull/2973)) - Bug fixes - Fix publishing of local IPNS entries, and more. (@whyrusleeping, [ipfs/go-ipfs#2943](https://github.com/ipfs/go-ipfs/pull/2943)) - Fix progress bars in `ipfs add` and `ipfs get`. (@whyrusleeping, [ipfs/go-ipfs#2893](https://github.com/ipfs/go-ipfs/pull/2893), [ipfs/go-ipfs#2948](https://github.com/ipfs/go-ipfs/pull/2948)) - Make sure files added through `ipfs files` are pinned and don't get GC'd. (@kevina, [ipfs/go-ipfs#2872](https://github.com/ipfs/go-ipfs/pull/2872)) - Fix copying into directory using `ipfs files cp`. (@whyrusleeping, [ipfs/go-ipfs#2977](https://github.com/ipfs/go-ipfs/pull/2977)) - Fix `ipfs version --commit` with Docker containers. (@lgierth, [ipfs/go-ipfs#2734](https://github.com/ipfs/go-ipfs/pull/2734)) - Run `ipfs diag` commands in the daemon instead of the CLI. (@Kubuxu, [ipfs/go-ipfs#2761](https://github.com/ipfs/go-ipfs/pull/2761)) - Fix protobuf encoding on the API and in commands. (@stebalien, [ipfs/go-ipfs#2516](https://github.com/ipfs/go-ipfs/pull/2516)) - Fix goroutine leak in `/ipfs/ping` protocol handler. (@whyrusleeping, [ipfs/go-libp2p#58](https://github.com/ipfs/go-libp2p/pull/58)) - Fix `--flags` option on `ipfs commands`. (@Kubuxu, [ipfs/go-ipfs#2773](https://github.com/ipfs/go-ipfs/pull/2773)) - Fix the error channels in `namesys`. (@whyrusleeping, [ipfs/go-ipfs#2788](https://github.com/ipfs/go-ipfs/pull/2788)) - Fix consumptions of observed swarm addresses. (@whyrusleeping, [ipfs/go-libp2p#63](https://github.com/ipfs/go-libp2p/pull/63), [ipfs/go-ipfs#2771](https://github.com/ipfs/go-ipfs/issues/2771)) - Fix a rare DHT panic. (@whyrusleeping, [ipfs/go-ipfs#2856](https://github.com/ipfs/go-ipfs/pull/2856)) - Fix go-ipfs/js-ipfs interoperability issues in SPDY. (@whyrusleeping, [whyrusleeping/go-smux-spdystream@fae17783](https://github.com/whyrusleeping/go-smux-spdystream/commit/fae1778302a9e029bb308cf71cf33f857f2d89e8)) - Fix a logging race condition during shutdown. (@Kubuxu, [ipfs/go-log#3](https://github.com/ipfs/go-log/pull/3)) - Prevent DHT connection hangs. (@whyrusleeping, [ipfs/go-ipfs#2826](https://github.com/ipfs/go-ipfs/pull/2826), [ipfs/go-ipfs#2863](https://github.com/ipfs/go-ipfs/pull/2863)) - Fix NDJSON output of `ipfs refs local`. (@Kubuxu, [ipfs/go-ipfs#2812](https://github.com/ipfs/go-ipfs/pull/2812)) - Fix race condition in NAT detection. (@whyrusleeping, [ipfs/go-libp2p#69](https://github.com/ipfs/go-libp2p/pull/69)) - Fix error messages. (@whyrusleeping, @Kubuxu, [ipfs/go-ipfs#2905](https://github.com/ipfs/go-ipfs/pull/2905), [ipfs/go-ipfs#2928](https://github.com/ipfs/go-ipfs/pull/2928)) - Enhancements - Increase maximum object size on `ipfs put` from 1 MiB to 2 MiB. The maximum object size on the wire including all framing is 4 MiB. (@kpcyrd, [ipfs/go-ipfs#2980](https://github.com/ipfs/go-ipfs/pull/2980)) - Add CORS headers to the Gateway's default config. (@Kubuxu, [ipfs/go-ipfs#2778](https://github.com/ipfs/go-ipfs/pull/2778)) - Clear the dial backoff for a peer when using `ipfs swarm connect`. (@whyrusleeping, [ipfs/go-ipfs#2941](https://github.com/ipfs/go-ipfs/pull/2941)) - Allow passing options to daemon in Docker container. (@lgierth, [ipfs/go-ipfs#2955](https://github.com/ipfs/go-ipfs/pull/2955)) - Add `-v/--verbose` to `ìpfs swarm peers` command. (@csasarak, [ipfs/go-ipfs#2713](https://github.com/ipfs/go-ipfs/pull/2713)) - Add `--format`, `--hash`, and `--size` options to `ipfs files stat` command. (@Kubuxu, [ipfs/go-ipfs#2706](https://github.com/ipfs/go-ipfs/pull/2706)) - Add `--all` option to `ipfs version` command. (@Kubuxu, [ipfs/go-ipfs#2790](https://github.com/ipfs/go-ipfs/pull/2790)) - Add `ipfs repo version` command. (@pfista, [ipfs/go-ipfs#2598](https://github.com/ipfs/go-ipfs/pull/2598)) - Add `ipfs repo verify` command. (@whyrusleeping, [ipfs/go-ipfs#2924](https://github.com/ipfs/go-ipfs/pull/2924), [ipfs/go-ipfs#2951](https://github.com/ipfs/go-ipfs/pull/2951)) - Add `ipfs stats repo` and `ipfs stats bitswap` command aliases. (@pfista, [ipfs/go-ipfs#2810](https://github.com/ipfs/go-ipfs/pull/2810)) - Add success indication to responses of `ipfs ping` command. (@Kubuxu, [ipfs/go-ipfs#2813](https://github.com/ipfs/go-ipfs/pull/2813)) - Save changes made via `ipfs swarm filter` to the config file. (@yuvallanger, [ipfs/go-ipfs#2880](https://github.com/ipfs/go-ipfs/pull/2880)) - Expand `ipfs_p2p_peers` metric to include libp2p transport. (@lgierth, [ipfs/go-ipfs#2728](https://github.com/ipfs/go-ipfs/pull/2728)) - Rework `ipfs files add` internals to avoid caching and prevent memory leaks. (@whyrusleeping, [ipfs/go-ipfs#2795](https://github.com/ipfs/go-ipfs/pull/2795)) - Support `GOPATH` with multiple path components. (@karalabe, @lgierth, @djdv, [ipfs/go-ipfs#2808](https://github.com/ipfs/go-ipfs/pull/2808), [ipfs/go-ipfs#2862](https://github.com/ipfs/go-ipfs/pull/2862), [ipfs/go-ipfs#2975](https://github.com/ipfs/go-ipfs/pull/2975)) - General Codebase - Take steps towards the `filestore` datastore. (@kevina, [ipfs/go-ipfs#2792](https://github.com/ipfs/go-ipfs/pull/2792), [ipfs/go-ipfs#2634](https://github.com/ipfs/go-ipfs/pull/2634)) - Update recommended Golang version to 1.6.2 (@Kubuxu, [ipfs/go-ipfs#2724](https://github.com/ipfs/go-ipfs/pull/2724)) - Update to Gx 0.8.0 and Gx-Go 1.2.1, which is faster and less noisy. (@whyrusleeping, [ipfs/go-ipfs#2979](https://github.com/ipfs/go-ipfs/pull/2979)) - Use `go4.org/lock` instead of `camlistore/lock` for locking. (@whyrusleeping, [ipfs/go-ipfs#2887](https://github.com/ipfs/go-ipfs/pull/2887)) - Manage `go.uuid`, `hamming`, `backoff`, `proquint`, `pb`, `go-context`, `cors`, `go-datastore` packages with Gx. (@Kubuxu, [ipfs/go-ipfs#2733](https://github.com/ipfs/go-ipfs/pull/2733), [ipfs/go-ipfs#2736](https://github.com/ipfs/go-ipfs/pull/2736), [ipfs/go-ipfs#2757](https://github.com/ipfs/go-ipfs/pull/2757), [ipfs/go-ipfs#2825](https://github.com/ipfs/go-ipfs/pull/2825), [ipfs/go-ipfs#2838](https://github.com/ipfs/go-ipfs/pull/2838)) - Clean up the gateway's surface. (@lgierth, [ipfs/go-ipfs#2874](https://github.com/ipfs/go-ipfs/pull/2874)) - Simplify the API gateway's access restrictions. (@lgierth, [ipfs/go-ipfs#2949](https://github.com/ipfs/go-ipfs/pull/2949), [ipfs/go-ipfs#2956](https://github.com/ipfs/go-ipfs/pull/2956)) - Update docker image to Alpine Linux 3.4 and remove Go version constraint. (@lgierth, [ipfs/go-ipfs#2901](https://github.com/ipfs/go-ipfs/pull/2901), [ipfs/go-ipfs#2929](https://github.com/ipfs/go-ipfs/pull/2929)) - Clarify `Dockerfile` and `Dockerfile.fast`. (@lgierth, [ipfs/go-ipfs#2796](https://github.com/ipfs/go-ipfs/pull/2796)) - Simplify resolution of Git commit refs in Dockerfiles. (@lgierth, [ipfs/go-ipfs#2754](https://github.com/ipfs/go-ipfs/pull/2754)) - Consolidate `--verbose` description across commands. (@Kubuxu, [ipfs/go-ipfs#2746](https://github.com/ipfs/go-ipfs/pull/2746)) - Allow setting position of default values in command option descriptions. (@Kubuxu, [ipfs/go-ipfs#2744](https://github.com/ipfs/go-ipfs/pull/2744)) - Set explicit default values for boolean command options. (@RichardLitt, [ipfs/go-ipfs#2657](https://github.com/ipfs/go-ipfs/pull/2657)) - Autogenerate command synopses. (@Kubuxu, [ipfs/go-ipfs#2785](https://github.com/ipfs/go-ipfs/pull/2785)) - Fix and improve lots of documentation. (@RichardLitt, [ipfs/go-ipfs#2741](https://github.com/ipfs/go-ipfs/pull/2741), [ipfs/go-ipfs#2781](https://github.com/ipfs/go-ipfs/pull/2781)) - Improve command descriptions to fit a width of 78 characters. (@RichardLitt, [ipfs/go-ipfs#2779](https://github.com/ipfs/go-ipfs/pull/2779), [ipfs/go-ipfs#2780](https://github.com/ipfs/go-ipfs/pull/2780), [ipfs/go-ipfs#2782](https://github.com/ipfs/go-ipfs/pull/2782)) - Fix filename conflict in the debugging guide. (@Kubuxu, [ipfs/go-ipfs#2752](https://github.com/ipfs/go-ipfs/pull/2752)) - Decapitalize log messages, according to Golang style guides. (@RichardLitt, [ipfs/go-ipfs#2853](https://github.com/ipfs/go-ipfs/pull/2853)) - Add GitHub Issues HowTo guide. (@RichardLitt, @chriscool, [ipfs/go-ipfs#2889](https://github.com/ipfs/go-ipfs/pull/2889), [ipfs/go-ipfs#2895](https://github.com/ipfs/go-ipfs/pull/2895)) - Add GitHub Issue template. (@chriscool, [ipfs/go-ipfs#2786](https://github.com/ipfs/go-ipfs/pull/2786)) - Apply standard-readme to the README file. (@RichardLitt, [ipfs/go-ipfs#2883](https://github.com/ipfs/go-ipfs/pull/2883)) - Fix issues pointed out by `govet`. (@Kubuxu, [ipfs/go-ipfs#2854](https://github.com/ipfs/go-ipfs/pull/2854)) - Clarify `ipfs get` error message. (@whyrusleeping, [ipfs/go-ipfs#2886](https://github.com/ipfs/go-ipfs/pull/2886)) - Remove dead code. (@whyrusleeping, [ipfs/go-ipfs#2819](https://github.com/ipfs/go-ipfs/pull/2819)) - Add changelog for v0.4.3. (@lgierth, [ipfs/go-ipfs#2984](https://github.com/ipfs/go-ipfs/pull/2984)) - Tests & CI - Fix flaky `ipfs mount` sharness test by using the `iptb` tool. (@noffle, [ipfs/go-ipfs#2707](https://github.com/ipfs/go-ipfs/pull/2707)) - Fix flaky IP port selection in tests. (@Kubuxu, [ipfs/go-ipfs#2855](https://github.com/ipfs/go-ipfs/pull/2855)) - Fix CLI tests on OSX by resolving /tmp symlink. (@Kubuxu, [ipfs/go-ipfs#2926](https://github.com/ipfs/go-ipfs/pull/2926)) - Fix flaky GC test by running the daemon in offline mode. (@Kubuxu, [ipfs/go-ipfs#2908](https://github.com/ipfs/go-ipfs/pull/2908)) - Add tests for `ipfs add` with hidden files. (@Kubuxu, [ipfs/go-ipfs#2756](https://github.com/ipfs/go-ipfs/pull/2756)) - Add test to make sure the body of HEAD responses is empty. (@Kubuxu, [ipfs/go-ipfs#2775](https://github.com/ipfs/go-ipfs/pull/2775)) - Add test to catch misdials. (@Kubuxu, [ipfs/go-ipfs#2831](https://github.com/ipfs/go-ipfs/pull/2831)) - Mark flaky tests for `ipfs dht query` as known failure. (@noffle, [ipfs/go-ipfs#2720](https://github.com/ipfs/go-ipfs/pull/2720)) - Remove failing blockstore-without-context test. (@Kubuxu, [ipfs/go-ipfs#2857](https://github.com/ipfs/go-ipfs/pull/2857)) - Fix `--version` tests for versions with a suffix like `-dev` or `-rc1`. (@lgierth, [ipfs/go-ipfs#2937](https://github.com/ipfs/go-ipfs/pull/2937)) - Make sharness tests work in cases where go-ipfs is symlinked into GOPATH. (@lgierth, [ipfs/go-ipfs#2937](https://github.com/ipfs/go-ipfs/pull/2937)) - Add variable delays to blockstore mocks. (@rikonor, [ipfs/go-ipfs#2871](https://github.com/ipfs/go-ipfs/pull/2871)) - Disable Travis CI email notifications. (@Kubuxu, [ipfs/go-ipfs#2896](https://github.com/ipfs/go-ipfs/pull/2896)) ## v0.4.2 - 2016-05-17 This is a patch release which fixes performance and networking bugs in go-libp2p, You should see improvements in CPU and RAM usage, as well as speed of object lookups. There are also a few other nice improvements. * Notable Fixes * Set a deadline for dialing attempts. This prevents a node from accumulating failed connections. (@whyrusleeping) * Avoid unnecessary string/byte conversions in go-multihash. (@whyrusleeping) * Fix a deadlock around the yamux stream muxer. (@whyrusleeping) * Fix a bug that left channels open, causing hangs. (@whyrusleeping) * Fix a bug around yamux which caused connection hangs. (@whyrusleeping) * Fix a crash caused by nil multiaddrs. (@whyrusleeping) * Enhancements * Add NetBSD support. (@erde74) * Set Cache-Control: immutable on /ipfs responses. (@kpcyrd) * Have `ipfs init` optionally accept a default configuration from stdin. (@sivachandran) * Add `ipfs log ls` command for listing logging subsystems. (@hsanjuan) * Allow bitswap to read multiple messages per stream. (@whyrusleeping) * Remove `make toolkit_upgrade` step. (@chriscool) * Documentation * Add a debug-guidelines document. (@richardlitt) * Update the contribute document. (@richardlitt) * Fix documentation of many `ipfs` commands. (@richardlitt) * Fall back to ShortDesc if LongDesc is missing. (@Kubuxu) * Removals * Remove -f option from `ipfs init` command. (@whyrusleeping) * Bug fixes * Fix `ipfs object patch` argument handling and validation. (@jbenet) * Fix `ipfs config edit` command by running it client-side. (@Kubuxu) * Set default value for `ipfs refs` arguments. (@richardlitt) * Fix parsing of incorrect command and argument permutations. (@thomas-gardner) * Update Dockerfile to latest go1.5.4-r0. (@chriscool) * Allow passing IPFS_LOGGING to Docker image. (@lgierth) * Fix dot path parsing on Windows. (@djdv) * Fix formatting of `ipfs log ls` output. (@richardlitt) * General Codebase * Refactor Makefile. (@kevina) * Wire context into bitswap requests more deeply. (@whyrusleeping) * Use gx for iptb. (@chriscool) * Update gx and gx-go. (@chriscool) * Make blocks.Block an interface. (@kevina) * Silence check for Docker existence. (@chriscool) * Add dist_get script for fetching tools from dist.ipfs.tech. (@whyrusleeping) * Add proper defaults to all `ipfs` commands. (@richardlitt) * Remove dead `count` option from `ipfs pin ls`. (@richardlitt) * Initialize pin mode strings only once. (@chriscool) * Add changelog for v0.4.2. (@lgierth) * Specify a dist.ipfs.tech hash for tool downloads instead of trusting DNS. (@lgierth) * CI * Fix t0170-dht sharness test. (@chriscool) * Increase timeout in t0060-daemon sharness test. (@Kubuxu) * Have CircleCI use `make deps` instead of `gx` directly. (@whyrusleeping) ## v0.4.1 - 2016-04-25 This is a patch release that fixes a few bugs, and adds a few small (but not insignificant) features. The primary reason for this release is the listener hang bugfix that was shipped in the 0.4.0 release. * Features * implemented ipfs object diff (@whyrusleeping) * allow promises (used in get, refs) to fail (@whyrusleeping) * Tool changes * Adds 'toolkit_upgrade' to the makefile help target (@achin) * General Codebase * Use extracted go-libp2p-crypto, -secio, -peer packages (@lgierth) * Update go-libp2p (@lgierth) * Fix package manifest fields (@lgierth) * remove incfusever dead-code (@whyrusleeping) * remove a ton of unused godeps (@whyrusleeping) * metrics: add Prometheus back (@lgierth) * clean up dead code and config fields (@whyrusleeping) * Add log events when blocks are added/removed from the blockstore (@michealmure) * repo: don't create logs directory, not used any longer (@lgierth) * Bug fixes * fixed ipfs name resolve --local multihash error (@pfista) * ipfs patch commands won't return null links field anymore (@whyrusleeping) * Make non recursive resolve print the result (@Kubuxu) * Output dirs on ipfs add -rn (@noffle) * update libp2p dep to fix hanging listeners problem (@whyrusleeping) * Fix Swarm.AddrFilters config setting with regard to `/ip6` addresses (@lgierth) * fix dht command key escaping (@whyrusleeping) * Testing * Adds tests to make sure 'object patch' writes. (@noffle) * small sharness test for promise failure checking (@whyrusleeping) * sharness/Makefile: clean all BINS when cleaning (@chriscool) * Documentation * Fix disconnect argument description (@richardlitt) * Added a note about swarm disconnect (@richardlitt) * Also fixed syntax for comment (@richardlitt) * Alphabetized swarm subcmds (@richardlitt) * Added note to ipfs stats bw interval option (@richardlitt) * Small syntax changes to repo stat man (@richardlitt) * update log command help text (@pfista) * Added a long description to add (@richardlitt) * Edited object patch set-data doc (@richardlitt) * add roadmap.md (@Jeromy) * Adds files api cmd to helptext (@noffle) ## v0.4.0 - 2016-04-05 This is a major release with plenty of new features and bug fixes. It also includes breaking changes which make it incompatible with v0.3.x on the networking layer. * Major Changes * Multistream * The addition of multistream is a breaking change on the networking layer, but gives IPFS implementations the ability to mix and match different stream multiplexers, e.g. yamux, spdystream, or muxado. This adds a ton of flexibility on one of the lower layers of the protocol, and will help us avoid further breaking protocol changes in the future. * Files API * The new `files` command and API allow a program to interact with IPFS using familiar filesystem operations, namely: creating directories, reading, writing, and deleting files, listing out different directories, and so on. This feature enables any other application that uses a filesystem-like backend for storage, to use IPFS as its storage driver without having change the application logic at all. * Gx * go-ipfs now uses [gx](https://github.com/whyrusleeping/gx) to manage its dependencies. This means that under the hood, go-ipfs's dependencies are backed by IPFS itself! It also means that go-ipfs is no longer installed using `go get`. Use `make install` instead. * New Features * Web UI * Update to new version which is compatible with 0.4.0. (@dignifiedquire) * Networking * Implement uTP transport. (@whyrusleeping) * Allow multiple addresses per configured bootstrap node. (@whyrusleeping) * IPNS * Improve IPNS resolution performance. (@whyrusleeping) * Have dnslink prefer `TXT _dnslink.example.com`, allows usage of CNAME records. (@Kubuxu) * Prevent `ipfs name publish` when `/ipns` is mounted. (@noffle) * Repo * Improve performance of `ipfs add`. (@whyrusleeping) * Add `Datastore.NoSync` config option for flatfs. (@rht) * Implement mark-and-sweep GC. (@whyrusleeping) * Allow for GC during `ipfs add`. (@whyrusleeping) * Add `ipfs repo stat` command. (@tmg, @diasdavid) * General * Add support for HTTP OPTIONS requests. (@lidel) * Add `ipfs diag cmds` to view active API requests (@whyrusleeping) * Add an `IPFS_LOW_MEM` environment variable which relaxes Bitswap's memory usage. (@whyrusleeping) * The Docker image now lives at `ipfs/go-ipfs` and has been completely reworked. (@lgierth) * Security fixes * The gateway path prefix added in v0.3.10 was vulnerable to cross-site scripting attacks. This release introduces a configurable list of allowed path prefixes. It's called `Gateway.PathPrefixes` and takes a list of strings, e.g. `["/blog", "/foo/bar"]`. The v0.3.x line will not receive any further updates, so please update to v0.4.0 as soon as possible. (@lgierth) * Incompatible Changes * Install using `make install` instead of `go get` (@whyrusleeping) * Rewrite pinning to store pins in IPFS objects. (@tv42) * Bump fs-repo version to 3. (@whyrusleeping) * Use multistream muxer (@whyrusleeping) * The default for `--type` in `ipfs pin ls` is now `all`. (@chriscool) * Bug Fixes * Remove msgio double wrap. (@jbenet) * Buffer msgio. (@whyrusleeping) * Perform various fixes to the FUSE code. (@tv42) * Compute `ipfs add` size in background to not stall add operation. (@whyrusleeping) * Add option to have `ipfs add` include top-level hidden files. (@noffle) * Fix CORS checks on the API. (@rht) * Fix `ipfs update` error message. (@tomgg) * Resolve paths in `ipfs pin rm` without network lookup. (@noffle) * Detect FUSE unmounts and track mount state. (@noffle) * Fix go1.6rc2 panic caused by CloseNotify being called from wrong goroutine. (@rwcarlsen) * Bump DHT kvalue from 10 to 20. (@whyrusleeping) * Put public key and IPNS entry to DHT in parallel. (@whyrusleeping) * Fix panic in CLI argument parsing. (@whyrusleeping) * Fix range error by using larger-than-zero-length buffer. (@noffle) * Fix yamux hanging issue by increasing AcceptBacklog. (@whyrusleeping) * Fix double Transport-Encoding header bug. (@whyrusleeping) * Fix uTP panic and file descriptor leak. (@whyrusleeping) * Tool Changes * Add `--pin` option to `ipfs add`, which defaults to `true` and allows `--pin=false`. (@eminence) * Add arguments to `ipfs pin ls`. (@chriscool) * Add `dns` and `resolve` commands to read-only API. (@Kubuxu) * Add option to display headers for `ipfs object links`. (@palkeo) * General Codebase Changes * Check Golang version in Makefile. (@chriscool) * Improve Makefile. (@tomgg) * Remove dead Jenkins CI code. (@lgierth) * Add locking interface to blockstore. (@whyrusleeping) * Add Merkledag FetchGraph and EnumerateChildren. (@whyrusleeping) * Rename Lock/RLock to GCLock/PinLock. (@jbenet) * Implement pluggable datastore types. (@tv42) * Record datastore metrics for non-default datastores. (@tv42) * Allow multistream to have zero-rtt stream opening. (@whyrusleeping) * Refactor `ipnsfs` into a more generic and well tested `mfs`. (@whyrusleeping) * Grab more peers if bucket doesn't contain enough. (@whyrusleeping) * Use CloseNotify in gateway. (@whyrusleeping) * Flatten multipart file transfers. (@whyrusleeping) * Send updated DHT record fixes to peers who sent outdated records. (@whyrusleeping) * Replace go-psutil with go-sysinfo. (@whyrusleeping) * Use ServeContent for index.html. (@AtnNn) * Refactor `object patch` API to not store data in URL. (@whyrusleeping) * Use mfs for `ipfs add`. (@whyrusleeping) * Add `Server` header to API responses. (@Kubuxu) * Wire context directly into HTTP requests. (@rht) * Wire context directly into GetDAG operations within GC. (@rht) * Vendor libp2p using gx. (@whyrusleeping) * Use gx vendored packages instead of Godeps. (@whyrusleeping) * Simplify merkledag package interface to ease IPLD inclusion. (@mildred) * Add default option value support to commands lib. (@whyrusleeping) * Refactor merkledag fetching methods. (@whyrusleeping) * Use net/url to escape paths within Web UI. (@noffle) * Deprecated key.Pretty(). (@MichealMure) * Documentation * Fix and update help text for **every** `ipfs` command. (@RichardLitt) * Change sample API origin settings from wildcard (`*`) to `example.com`. (@Kubuxu) * Improve documentation of installation process in README. (@whyrusleeping) * Improve windows.md. (@chriscool) * Clarify instructions for installing from source. (@noffle) * Make version checking more robust. (@jedahan) * Assert the source code is located within GOPATH. (@whyrusleeping) * Remove mentions of `/dns` from `ipfs dns` command docs. (@lgierth) * Testing * Refactor iptb tests. (@chriscool) * Improve t0240 sharness test. (@chriscool) * Make bitswap tests less flaky. (@whyrusleeping) * Use TCP port zero for ipfs daemon in sharness tests. (@whyrusleeping) * Improve sharness tests on AppVeyor. (@chriscool) * Add a pause to fix timing on t0065. (@whyrusleeping) * Add support for arbitrary TCP ports to t0060-daemon.sh. (@noffle) * Make t0060 sharness test use TCP port zero. (@whyrusleeping) * Randomized ipfs stress testing via randor (@dignifiedquire) * Stress test pinning and migrations (@whyrusleeping) ================================================ FILE: docs/changelogs/v0.40.md ================================================ # Kubo changelog v0.40 This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [v0.40.0](#v0400) - [v0.40.1](#v0401) ## v0.40.0 [](https://github.com/user-attachments/assets/c065d5e5-2a8a-4651-8142-d7baf3106623) - [Overview](#overview) - [🔦 Highlights](#-highlights) - [🔢 IPIP-499: UnixFS CID Profiles](#-ipip-499-unixfs-cid-profiles) - [🧹 Automatic cleanup of interrupted imports](#-automatic-cleanup-of-interrupted-imports) - [🌍 Light clients can now use your node for delegated routing](#-light-clients-can-now-use-your-node-for-delegated-routing) - [📊 See total size when pinning](#-see-total-size-when-pinning) - [🔀 IPIP-523: `?format=` takes precedence over `Accept` header](#-ipip-523-format-takes-precedence-over-accept-header) - [🚫 IPIP-524: Gateway codec conversion disabled by default](#-ipip-524-gateway-codec-conversion-disabled-by-default) - [✅ More reliable IPNS over PubSub](#-more-reliable-ipns-over-pubsub) - [🗄️ New `ipfs diag datastore` commands](#️-new-ipfs-diag-datastore-commands) - [🔍 New `ipfs swarm addrs autonat` command](#-new-ipfs-swarm-addrs-autonat-command) - [🚇 Improved `ipfs p2p` tunnels with foreground mode](#-improved-ipfs-p2p-tunnels-with-foreground-mode) - [📊 Friendlier `ipfs dag stat` output](#-friendlier-ipfs-dag-stat-output) - [🔑 `ipfs key` improvements](#-ipfs-key-improvements) - [🤝 More reliable content providing after startup](#-more-reliable-content-providing-after-startup) - [🌐 No unnecessary DNS lookups for AutoTLS addresses](#-no-unnecessary-dns-lookups-for-autotls-addresses) - [⏱️ Configurable gateway request duration limit](#️-configurable-gateway-request-duration-limit) - [🔧 Recovery from corrupted MFS root](#-recovery-from-corrupted-mfs-root) - [📡 RPC `Content-Type` headers for binary responses](#-rpc-content-type-headers-for-binary-responses) - [🔖 New `ipfs name get|put` commands](#-new-ipfs-name-getput-commands) - [📋 Long listing format for `ipfs ls`](#-long-listing-format-for-ipfs-ls) - [🖥️ WebUI Improvements](#-webui-improvements) - [📉 Fixed Prometheus metrics bloat on popular subdomain gateways](#-fixed-prometheus-metrics-bloat-on-popular-subdomain-gateways) - [📢 libp2p announces all interface addresses](#-libp2p-announces-all-interface-addresses) - [🗑️ Badger v1 datastore slated for removal this year](#-badger-v1-datastore-slated-for-removal-this-year) - [🐹 Go 1.26](#-go-126) - [📦️ Dependency updates](#-dependency-updates) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview This release brings reproducible file imports (CID Profiles), automatic cleanup of interrupted operations, better connectivity diagnostics, and improved gateway behavior. It also ships with Go 1.26, lowering memory usage and GC overhead across the board. ### 🔦 Highlights #### 🔢 IPIP-499: UnixFS CID Profiles CID Profiles are presets that pin down how files get split into blocks and organized into directories, so you get the same CID for the same data across different software or versions. Defined in [IPIP-499](https://specs.ipfs.tech/ipips/ipip-0499/). **New configuration [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles)** - `unixfs-v1-2025`: modern CIDv1 profile with improved defaults - `unixfs-v0-2015` (alias `legacy-cid-v0`): best-effort legacy CIDv0 behavior Apply with: `ipfs config profile apply unixfs-v1-2025` The `test-cid-v1` and `test-cid-v1-wide` profiles have been removed. Use `unixfs-v1-2025` or manually set specific `Import.*` settings instead. **New [`Import.*`](https://github.com/ipfs/kubo/blob/master/docs/config.md#import) options** - `Import.UnixFSHAMTDirectorySizeEstimation`: estimation mode (`links`, `block`, or `disabled`) - `Import.UnixFSDAGLayout`: DAG layout (`balanced` or `trickle`) **New [`ipfs add`](https://docs.ipfs.tech/reference/kubo/cli/#ipfs-add) CLI flags** - `--dereference-symlinks` resolves all symlinks to their target content, replacing the deprecated `--dereference-args` which only resolved CLI argument symlinks - `--empty-dirs` / `-E` controls inclusion of empty directories (default: true) - `--hidden` / `-H` includes hidden files (default: false) - `--trickle` implicit default can be adjusted via `Import.UnixFSDAGLayout` **`ipfs files write` fix for CIDv1 directories** When writing to MFS directories that use CIDv1 (via `--cid-version=1` or `ipfs files chcid`), single-block files now produce raw block CIDs (like `bafkrei...`), matching the behavior of `ipfs add --raw-leaves`. Previously, MFS would wrap single-block files in dag-pb even when raw leaves were enabled. CIDv0 directories continue to use dag-pb. **Block size limit raised to 2MiB** `ipfs block put`, `ipfs dag put`, and `ipfs dag import` now accept blocks up to 2MiB without `--allow-big-block`, matching the [bitswap spec](https://specs.ipfs.tech/bitswap-protocol/#block-sizes). The previous 1MiB limit was too restrictive and broke `ipfs dag import` of 1MiB-chunked non-raw-leaf data (protobuf wrapping pushes blocks slightly over 1MiB). The max `--chunker` value for `ipfs add` is `2MiB - 256 bytes` to leave room for protobuf framing. IPIP-499 profiles use lower chunk sizes (256KiB and 1MiB) and are not affected. **HAMT Threshold Fix** HAMT directory sharding threshold changed from `>=` to `>` to match the Go docs and JS implementation ([ipfs/boxo@6707376](https://github.com/ipfs/boxo/commit/6707376002a3d4ba64895749ce9be2e00d265ed5)). A directory exactly at 256 KiB now stays as a basic directory instead of converting to HAMT. This is a theoretical breaking change, but unlikely to impact real-world users as it requires a directory to be exactly at the threshold boundary. If you depend on the old behavior, adjust [`Import.UnixFSHAMTShardingSize`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtshardingsize) to be 1 byte lower. #### 🧹 Automatic cleanup of interrupted imports If you cancel `ipfs add` or `ipfs dag import` mid-operation, Kubo now automatically cleans up incomplete data on the next daemon start. Previously, interrupted imports would leave orphan blocks in your repository that were difficult to identify and remove without pins and running explicit garbage collection. Batch operations also use less memory now. Block data is written to disk immediately rather than held in RAM until the batch commits. Under the hood, the block storage layer (flatfs) was rewritten to use atomic batch operations via a temporary staging directory. See [go-ds-flatfs#142](https://github.com/ipfs/go-ds-flatfs/pull/142) for details. #### 🌍 Light clients can now use your node for delegated routing The [Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) is now exposed by default at `http://127.0.0.1:8080/routing/v1`. This allows light clients in browsers to use Kubo Gateway as a delegated routing backend instead of running a full DHT client. Support for [IPIP-476: Delegated Routing DHT Closest Peers API](https://specs.ipfs.tech/ipips/ipip-0476/) is included. Can be disabled via [`Gateway.ExposeRoutingAPI`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewayexposeroutingapi). #### 📊 See total size when pinning `ipfs pin add --progress` now shows the total size of the pinned DAG as it fetches blocks. Example output: ``` Fetched/Processed 336 nodes (83 MB) ``` #### 🔀 IPIP-523: `?format=` takes precedence over `Accept` header The `?format=` URL query parameter now always wins over the `Accept` header ([IPIP-523](https://specs.ipfs.tech/ipips/ipip-0523/)), giving you deterministic HTTP caching and protecting against CDN cache-key collisions. Browsers can also use `?format=` reliably even when they send `Accept` headers with specific content types. The only breaking change is for edge cases where a client sends both a specific `Accept` header and a different `?format=` value for an explicitly supported format (`tar`, `raw`, `car`, `dag-json`, `dag-cbor`, etc.). Previously `Accept` would win. Now `?format=` always wins. #### 🚫 IPIP-524: Gateway codec conversion disabled by default Gateways no longer convert between codecs by default ([IPIP-524](https://specs.ipfs.tech/ipips/ipip-0524/)). This removes gateways from a gatekeeping role: clients can adopt new codecs immediately without waiting for gateway operator updates. Requests for a format that differs from the block's codec now return `406 Not Acceptable`. **Migration**: Clients should fetch raw blocks (`?format=raw` or `Accept: application/vnd.ipld.raw`) and convert client-side using libraries like [@helia/verified-fetch](https://www.npmjs.com/package/@helia/verified-fetch). Set [`Gateway.AllowCodecConversion`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewayallowcodecconversion) to `true` to restore previous behavior. #### ✅ More reliable IPNS over PubSub [IPNS over PubSub](https://specs.ipfs.tech/ipns/ipns-pubsub-router/) implementation in Kubo is now more reliable. Duplicate messages are rejected even in large networks where messages may cycle back after the in-memory cache expires. Kubo now persists the maximum seen sequence number per peer to the datastore ([go-libp2p-pubsub#BasicSeqnoValidator](https://pkg.go.dev/github.com/libp2p/go-libp2p-pubsub#BasicSeqnoValidator)), providing stronger duplicate detection that survives node restarts. This addresses message flooding issues reported in [#9665](https://github.com/ipfs/kubo/issues/9665). IPNS over PubSub is opt-in via [`Ipns.UsePubsub`](https://github.com/ipfs/kubo/blob/master/docs/config.md#ipnsusepubsub). Kubo's pubsub is optimized for IPNS use case. For custom pubsub applications requiring different validation logic, use [go-libp2p-pubsub](https://github.com/libp2p/go-libp2p-pubsub) directly in a dedicated binary. #### 🗄️ New `ipfs diag datastore` commands New experimental commands for low-level datastore inspection: - `ipfs diag datastore get ` - Read raw value at a datastore key (use `--hex` for hex dump) - `ipfs diag datastore count ` - Count entries matching a datastore prefix The daemon must not be running when using these commands. Run `ipfs diag datastore --help` for usage examples. #### 🔍 New `ipfs swarm addrs autonat` command The new `ipfs swarm addrs autonat` command shows the network reachability status of your node's addresses as verified by AutoNAT V2. AutoNAT V2 leverages other nodes in the IPFS network to test your node's external public reachability, providing a self-service way to debug connectivity. Public reachability is important for: - **Direct data fetching**: Other nodes can fetch data directly from your node without NAT hole punching. - **Browser access**: Web browsers can connect to your node directly for content retrieval. - **DHT participation**: Your node can act as a DHT server, helping to maintain the distributed hash table and making content routing more robust. The command displays: - Overall reachability status (public, private, or unknown) - Per-address reachability showing which specific addresses are reachable, unreachable, or unknown Example output: ``` AutoNAT V2 Status: Reachability: public Per-Address Reachability: Reachable: /ip4/203.0.113.42/tcp/4001 /ip4/203.0.113.42/udp/4001/quic-v1 Unreachable: /ip6/2001:db8::1/tcp/4001 Unknown: /ip4/203.0.113.42/udp/4001/webrtc-direct ``` This helps diagnose connectivity issues and understand if your node is publicly reachable. See the [AutoNAT V2 spec](https://github.com/libp2p/specs/blob/master/autonat/autonat-v2.md) for more details. #### 🚇 Improved `ipfs p2p` tunnels with foreground mode P2P tunnels can now run like SSH port forwarding: start a tunnel, use it, and it cleans up automatically when you're done. The new `--foreground` (`-f`) flag for `ipfs p2p listen` and `ipfs p2p forward` keeps the command running until interrupted. When you Ctrl+C, send SIGTERM, or stop the service, the tunnel is removed automatically: ```console $ ipfs p2p listen /x/ssh /ip4/127.0.0.1/tcp/22 --foreground Listening on /x/ssh, forwarding to /ip4/127.0.0.1/tcp/22, waiting for interrupt... ^C Received interrupt, removing listener for /x/ssh ``` Without `--foreground`, commands return immediately and tunnels persist until explicitly closed (existing behavior). See [docs/p2p-tunnels.md](https://github.com/ipfs/kubo/blob/master/docs/p2p-tunnels.md) for usage examples. #### 📊 Friendlier `ipfs dag stat` output The `ipfs dag stat` command has been improved for better terminal UX: - Progress output now uses a single line with carriage return, avoiding terminal flooding - Progress is auto-detected: shown only in interactive terminals by default - Human-readable sizes are now displayed alongside raw byte counts Example progress (interactive terminal): ``` Fetched/Processed 84 blocks, 2097152 bytes (2.1 MB) ``` Example summary output: ``` Summary Total Size: 2097152 (2.1 MB) Unique Blocks: 42 Shared Size: 1048576 (1.0 MB) Ratio: 1.500000 ``` Use `--progress=true` to force progress even when piped, or `--progress=false` to disable it. #### 🔑 `ipfs key` improvements `ipfs key ls` is now the canonical command for listing keys, matching `ipfs pin ls` and `ipfs files ls`. The old `ipfs key list` still works but is deprecated. Listing also became more resilient: bad keys are now skipped with an error log instead of failing the entire operation. #### 🤝 More reliable content providing after startup Previously, provide operations could start before the Accelerated DHT Client discovered enough peers, causing sweep mode to lose its efficiency benefits. Now, providing waits for the initial network crawl (about 10 minutes). Your content will be properly distributed across DHT regions after initial DHT map is created. Check `ipfs provide stat` to see when providing begins. #### 🌐 No unnecessary DNS lookups for AutoTLS addresses Kubo no longer makes DNS queries for [AutoTLS](https://web.archive.org/web/20260112031855/https://blog.libp2p.io/autotls/) addresses like `1-2-3-4.peerid.libp2p.direct`. Since the IP is encoded in the hostname (`1-2-3-4` means `1.2.3.4`), Kubo extracts it locally. This reduces load on the public good DNS servers at `libp2p.direct` run by [Shipyard](https://ipshipyard.com), reserving them for web browsers which lack direct DNS access and must rely on the browser's resolver. To disable, set [`AutoTLS.SkipDNSLookup`](https://github.com/ipfs/kubo/blob/master/docs/config.md#autotlsskipdnslookup) to `false`. #### ⏱️ Configurable gateway request duration limit [`Gateway.MaxRequestDuration`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaymaxrequestduration) sets an absolute deadline for gateway requests. Unlike `RetrievalTimeout` (which resets on each data write and catches stalled transfers), this is a hard limit on the total time a request can take. The default 1 hour limit (previously hardcoded) can now be adjusted to fit your deployment needs. This is a fallback that prevents requests from hanging indefinitely when subsystem timeouts are misconfigured or fail to trigger. Returns 504 Gateway Timeout when exceeded. #### 🔧 Recovery from corrupted MFS root If your daemon fails to start because the MFS root is not a directory (due to misconfiguration, operational error, or disk corruption), you can now recover without deleting and recreating your repository in a new `IPFS_PATH`. The new `ipfs files chroot` command lets you reset the MFS (Mutable File System) root or restore it to a known valid CID: ```console # Reset MFS to an empty directory $ ipfs files chroot --confirm # Or restore from a previously saved directory CID $ ipfs files chroot --confirm QmYourBackupCID ``` See `ipfs files chroot --help` for details. #### 📡 RPC `Content-Type` headers for binary responses HTTP RPC endpoints that return binary data now set appropriate `Content-Type` headers, making it easier to integrate with HTTP clients and tooling that rely on MIME types. On CLI these commands behave the same as before, but over HTTP RPC you now get proper headers: | Endpoint | Content-Type | |------------------------|-------------------------------------------| | `/api/v0/get` | `application/x-tar` or `application/gzip` | | `/api/v0/dag/export` | `application/vnd.ipld.car` | | `/api/v0/block/get` | `application/vnd.ipld.raw` | | `/api/v0/name/get` | `application/vnd.ipfs.ipns-record` | | `/api/v0/diag/profile` | `application/zip` | #### 🔖 New `ipfs name get|put` commands You can now backup, restore, and share IPNS records without needing the private key. ```console $ ipfs name get /ipns/k51... > record.bin $ ipfs name get /ipns/k51... | ipfs name inspect $ ipfs name put k51... record.bin ``` These are low-level tools primarily for debugging and testing IPNS. The `put` command validates records by default. Use `--force` to skip validation and test how routing systems handle malformed or outdated records. Note that `--force` only bypasses this command's checks; the routing system may still reject invalid records. #### 📋 Long listing format for `ipfs ls` The `ipfs ls` command now supports `--long` (`-l`) flag for displaying Unix-style file permissions and modification times. This works with files added using `--preserve-mode` and `--preserve-mtime`. See `ipfs ls --help` for format details and examples. #### 🖥️ WebUI Improvements IPFS Web UI has been updated to [v4.11.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.11.0). ##### Search and filter files You can now search and filter files directly in the Files screen. Type a name, CID, or file extension and the list narrows down in real time. Works in both list and grid view. > ![Search and filter files](https://github.com/user-attachments/assets/cc266dbc-8424-4a8a-a6b7-a80a5a25683b) ##### DHT Provide diagnostic screen New screen under Diagnostics that shows the health of DHT Provide operations. You can see reprovide cycle progress, worker utilization, queue status, and network throughput at a glance, without having to use the [`ipfs provide stat`](https://docs.ipfs.tech/reference/kubo/cli/#ipfs-provide-stat) CLI. > ![DHT Provide diagnostic screen](https://github.com/user-attachments/assets/c577309a-2249-46f8-87d9-f0da42955f32) ##### Better path handling in Files The Inspect button now resolves `/ipfs/` and `/ipns/` paths to their final CID before opening the IPLD Explorer. The Explore form also accepts `ipfs://` and `ipns://` protocol URLs. Previously, these would show a blank screen or an infinite spinner. Path resolution errors now also show better error pages: > ![Better path handling in Files](https://github.com/user-attachments/assets/3494835b-0b93-4990-9971-078273671928) #### 📉 Fixed Prometheus metrics bloat on popular subdomain gateways Most Kubo users are unaffected by this change. It matters if you run Kubo as a public subdomain gateway (with [`Gateway.PublicGateways`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaypublicgateways) and `UseSubdomains: true`), where the `otelhttp` instrumentation was including the raw `Host` header as the `server_address` metric label. Every unique hostname (e.g., each `CID.ipfs.dweb.link`) created a separate time series, resulting in millions of metric lines, multi-gigabyte `/debug/metrics/prometheus` responses, and Prometheus scrape timeouts. **What changed:** - `http_server_*` metrics replace the unbounded `server_address` label with a new `server_domain` label that groups requests by gateway domain: - Gateway: matched [`Gateway.PublicGateways`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaypublicgateways) suffix (e.g., `dweb.link`, `ipfs.io`), or `localhost`, `loopback`, `other` - RPC API: `api` / Libp2p Gateway: `libp2p` - Prometheus exemplars are disabled to prevent log noise from long subdomain hostnames. Tracing spans are unaffected. If you use [Rainbow](https://github.com/ipfs/rainbow) for your public gateway (recommended), this issue never applied to you -- Rainbow uses its own low-cardinality HTTP metrics. #### 📢 libp2p announces all interface addresses go-libp2p [v0.47.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.47.0) includes a rewritten routing library ([`go-netroute`](https://github.com/libp2p/go-netroute/pull/64)) that fixes interop with VPN and WireGuard/Tailscale setups. A side effect: when listening on `0.0.0.0`, libp2p now returns addresses from all network interfaces instead of just the primary one ([go-libp2p#3460](https://github.com/libp2p/go-libp2p/issues/3460)). This means easier connectivity and less manual configuration for most desktop, VPN, and self-hosted users. However, if you don't run with the [`server` profile](https://github.com/ipfs/kubo/blob/master/docs/config.md#server-profile) and have an empty [`Addresses.NoAnnounce`](https://github.com/ipfs/kubo/blob/master/docs/config.md#addressesnoannounce), your node may now announce internal addresses (e.g. Docker bridge `172.17.0.0/16` or Tailscale `100.64.0.0/10`) to the DHT. In the default setup, AutoNAT will probe and mark unreachable ones as offline and they won't be listed, but you can also filter them out explicitly. To check what your node announces and filter out unwanted ranges: ```console $ ipfs swarm addrs local $ ipfs config --json Addresses.NoAnnounce '["/ip4/172.17.0.0/ipcidr/16"]' ``` The [`server` profile](https://github.com/ipfs/kubo/blob/master/docs/config.md#server-profile) already [filters common private ranges](https://github.com/ipfs/kubo/blob/master/config/profile.go#L24-L43) via `Addresses.NoAnnounce`. #### 🗑️ Badger v1 datastore slated for removal this year The `badgerds` datastore (based on badger 1.x) is slated for removal. Badger v1 has not been maintained by its upstream maintainers for years and has known bugs including startup timeouts, shutdown hangs, and file descriptor exhaustion. Starting with this release, every daemon start with a badger-based repository prints a loud deprecation error on stderr. See the [`badgerds` profile documentation](https://github.com/ipfs/kubo/blob/master/docs/config.md#badgerds-profile) for migration guidance, and [#11186](https://github.com/ipfs/kubo/issues/11186) for background. #### 🐹 Go 1.26 This release is built with [Go 1.26](https://go.dev/doc/go1.26). You should see lower memory usage and reduced GC pauses thanks to the new Green Tea garbage collector (10-40% less GC overhead). Reading block data and API responses is faster due to `io.ReadAll` improvements (~2x faster, ~50% less memory). On 64-bit platforms, heap base address randomization adds a layer of security hardening. > **Note:** [v0.40.1](#v0401) downgrades to Go 1.25 due to a Windows stability issue. If you run Kubo on Linux or macOS, staying on v0.40.0 is fine and you benefit from Go 1.26's GC improvements. #### 📦️ Dependency updates - update `go-libp2p` to [v0.47.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.47.0) (incl. [v0.46.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.46.0)) - Reduced WebRTC log noise by using debug level for pion errors ([go-libp2p#3426](https://github.com/libp2p/go-libp2p/pull/3426)). - Fixed mDNS discovery on Windows and macOS by filtering addresses to reduce packet size ([go-libp2p#3434](https://github.com/libp2p/go-libp2p/pull/3434)). - AutoTLS addresses no longer get marked unreachable when peers lack WebSockets support. Swarm heals over time ([go-libp2p#3435](https://github.com/libp2p/go-libp2p/pull/3435)). - Fixed `stream.Close()` blocking indefinitely on unresponsive peers ([go-libp2p#3448](https://github.com/libp2p/go-libp2p/pull/3448)). - update `quic-go` to [v0.59.0](https://github.com/quic-go/quic-go/releases/tag/v0.59.0) (incl. [v0.58.0](https://github.com/quic-go/quic-go/releases/tag/v0.58.0) + [v0.57.0](https://github.com/quic-go/quic-go/releases/tag/v0.57.0)) - update `p2p-forge` to [v0.7.0](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.7.0) - update `go-ds-pebble` to [v0.5.9](https://github.com/ipfs/go-ds-pebble/releases/tag/v0.5.9) - updates `github.com/cockroachdb/pebble` to [v2.1.4](https://github.com/cockroachdb/pebble/releases/tag/v2.1.4) to enable Go 1.26 support - update `go-libp2p-pubsub` to [v0.15.0](https://github.com/libp2p/go-libp2p-pubsub/releases/tag/v0.15.0) - update `go-ipld-prime` to [v0.22.0](https://github.com/ipld/go-ipld-prime/releases/tag/v0.22.0) - update `boxo` to [v0.37.0](https://github.com/ipfs/boxo/releases/tag/v0.37.0) (incl. [v0.36.0](https://github.com/ipfs/boxo/releases/tag/v0.36.0)) - update `go-libp2p-kad-dht` to [v0.38.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.38.0) (includes [v0.37.1](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.37.1), [v0.37.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.37.0)) - update `ipfs-webui` to [v4.11.1](https://github.com/ipfs/ipfs-webui/releases/tag/v4.11.1) (incl. [v4.11.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.11.0)) - update `gateway-conformance` tests to [v0.10](https://github.com/ipfs/gateway-conformance/releases/tag/v0.10.0) (incl. [v0.9](https://github.com/ipfs/gateway-conformance/releases/tag/v0.9.0)) ### 📝 Changelog
Full Changelog - github.com/ipfs/kubo: - fix(metrics): disable otel exemplars to prevent rune overflow (#11211) ([ipfs/kubo#11211](https://github.com/ipfs/kubo/pull/11211)) - fix: drop high-cardinality server.address from http_server metrics (#11208) ([ipfs/kubo#11208](https://github.com/ipfs/kubo/pull/11208)) - chore: set version to 0.40.0-rc2 - fix(version): produce shorter user agent for tagged release builds - chore: update webui to v4.11.1 (#11204) ([ipfs/kubo#11204](https://github.com/ipfs/kubo/pull/11204)) - fix: improve `ipfs name put` for IPNS record republishing (#11199) ([ipfs/kubo#11199](https://github.com/ipfs/kubo/pull/11199)) - Upgrade to Boxo v0.37.0 (#11201) ([ipfs/kubo#11201](https://github.com/ipfs/kubo/pull/11201)) - chore: set version to v0.40.0-rc1 - refactor: apply go fix modernizers from Go 1.26 (#11190) ([ipfs/kubo#11190](https://github.com/ipfs/kubo/pull/11190)) - feat: update to Go 1.26 (#11189) ([ipfs/kubo#11189](https://github.com/ipfs/kubo/pull/11189)) - docs: clarify LevelDB compaction limitations and StorageMax scope (#11188) ([ipfs/kubo#11188](https://github.com/ipfs/kubo/pull/11188)) - docs: loud deprecation of badger v1 datastore (#11187) ([ipfs/kubo#11187](https://github.com/ipfs/kubo/pull/11187)) - docs(changelog): add highlight for libp2p AllAddrs behavior change (#11183) ([ipfs/kubo#11183](https://github.com/ipfs/kubo/pull/11183)) - fix: allow dag import of 1MiB chunks wrapped in dag-pb (#11185) ([ipfs/kubo#11185](https://github.com/ipfs/kubo/pull/11185)) - feat: `swarm addrs autonat` command (#11184) ([ipfs/kubo#11184](https://github.com/ipfs/kubo/pull/11184)) - feat(gateway): IPIP-0524 Gateway.AllowCodecConversion config option (#11090) ([ipfs/kubo#11090](https://github.com/ipfs/kubo/pull/11090)) - feat: update ipfs-webui to v4.11.0 (#11182) ([ipfs/kubo#11182](https://github.com/ipfs/kubo/pull/11182)) - feat(config): add Import.* for CID Profiles from IPIP-499 (#11148) ([ipfs/kubo#11148](https://github.com/ipfs/kubo/pull/11148)) - chore: replace libp2p.io URL with Internet Archive (#11181) ([ipfs/kubo#11181](https://github.com/ipfs/kubo/pull/11181)) - test: IPIP-523 format query precedence over Accept header (#11086) ([ipfs/kubo#11086](https://github.com/ipfs/kubo/pull/11086)) - docs: update go-libp2p changelog entry to v0.47.0 - feat(rpc): Content-Type headers and IPNS record get/put (#11067) ([ipfs/kubo#11067](https://github.com/ipfs/kubo/pull/11067)) - feat(key): add 'ipfs key ls' as alias for 'ipfs key list' (#11147) ([ipfs/kubo#11147](https://github.com/ipfs/kubo/pull/11147)) - docs: cleanup broken links and outdated content (#11100) ([ipfs/kubo#11100](https://github.com/ipfs/kubo/pull/11100)) - feat(dns): skip DNS lookups for AutoTLS hostnames (#11140) ([ipfs/kubo#11140](https://github.com/ipfs/kubo/pull/11140)) - Upgrade to Boxo v0.36.0 (#11175) ([ipfs/kubo#11175](https://github.com/ipfs/kubo/pull/11175)) - chore: upgrade go-ds-pebble to v0.5.9 (#11170) ([ipfs/kubo#11170](https://github.com/ipfs/kubo/pull/11170)) - fix(commands/reprovide): update manual reprovide error message (#11151) ([ipfs/kubo#11151](https://github.com/ipfs/kubo/pull/11151)) - feat(cli): ls --long (#11103) ([ipfs/kubo#11103](https://github.com/ipfs/kubo/pull/11103)) - feat(pubsub): persistent validation and diagnostic commands (#11110) ([ipfs/kubo#11110](https://github.com/ipfs/kubo/pull/11110)) - feat(config): add Gateway.MaxRequestDuration option (#11138) ([ipfs/kubo#11138](https://github.com/ipfs/kubo/pull/11138)) - feat(provider): info log AcceleratedDHTClient crawl (#11143) ([ipfs/kubo#11143](https://github.com/ipfs/kubo/pull/11143)) - ipfswatch: fix panic on broken link (#11145) ([ipfs/kubo#11145](https://github.com/ipfs/kubo/pull/11145)) - upgrade go-libp2p-pubsub to v0.15.0 (#11144) ([ipfs/kubo#11144](https://github.com/ipfs/kubo/pull/11144)) - feat(mfs): chroot command to change the root (#8648) ([ipfs/kubo#8648](https://github.com/ipfs/kubo/pull/8648)) - feat: improved go-ds-flatfs (#11092) ([ipfs/kubo#11092](https://github.com/ipfs/kubo/pull/11092)) - test: fix flaky ipfswatch test (#11142) ([ipfs/kubo#11142](https://github.com/ipfs/kubo/pull/11142)) - docs: clarify Routing.Type=custom as experimental (#11111) ([ipfs/kubo#11111](https://github.com/ipfs/kubo/pull/11111)) - fix(routing): defensive clone of AddrInfo from provider channel (#11120) ([ipfs/kubo#11120](https://github.com/ipfs/kubo/pull/11120)) - fix(provider): wait for fullrt crawl completion before providing (#11137) ([ipfs/kubo#11137](https://github.com/ipfs/kubo/pull/11137)) - fix(provide): do not output keystore error on shutdown (#11130) ([ipfs/kubo#11130](https://github.com/ipfs/kubo/pull/11130)) - feat(p2p): add --foreground flag to listen and forward commands (#11099) ([ipfs/kubo#11099](https://github.com/ipfs/kubo/pull/11099)) - feat(cli): improve ipfs dag stat output UX (#11097) ([ipfs/kubo#11097](https://github.com/ipfs/kubo/pull/11097)) - docs: add production deployment guidance for gateway (#11117) ([ipfs/kubo#11117](https://github.com/ipfs/kubo/pull/11117)) - fix(routing): use LegacyProvider for HTTP-only custom routing (#11112) ([ipfs/kubo#11112](https://github.com/ipfs/kubo/pull/11112)) - shutdown daemon after test (#11135) ([ipfs/kubo#11135](https://github.com/ipfs/kubo/pull/11135)) - fix(ci): parallelize gotest, cleanup output, flakiness (#11113) ([ipfs/kubo#11113](https://github.com/ipfs/kubo/pull/11113)) - test: replace `go-clock` with `testing/synctest` (#11131) ([ipfs/kubo#11131](https://github.com/ipfs/kubo/pull/11131)) - docs: improve README for first-time users (#11133) ([ipfs/kubo#11133](https://github.com/ipfs/kubo/pull/11133)) - keys: skip bad keys when listing (#11115) ([ipfs/kubo#11115](https://github.com/ipfs/kubo/pull/11115)) - docs: add developer guide for local development workflow (#11128) ([ipfs/kubo#11128](https://github.com/ipfs/kubo/pull/11128)) - datastore: upgrade go-ds-pebble to v0.5.8 (#11129) ([ipfs/kubo#11129](https://github.com/ipfs/kubo/pull/11129)) - output stdout and stderr on example test failure (#11119) ([ipfs/kubo#11119](https://github.com/ipfs/kubo/pull/11119)) - chore: update go-libp2p 0.46 (#11105) ([ipfs/kubo#11105](https://github.com/ipfs/kubo/pull/11105)) - fix(ipfswatch): loading datastore plugins (#11078) ([ipfs/kubo#11078](https://github.com/ipfs/kubo/pull/11078)) - Add bytes progress tracker for ipfs pin add (#11074) ([ipfs/kubo#11074](https://github.com/ipfs/kubo/pull/11074)) - docs: link sweep blogpost in Provide.DHT.SweepEnabled - docs: note sweep+accelerated DHT client limitation (#11084) ([ipfs/kubo#11084](https://github.com/ipfs/kubo/pull/11084)) - refactor: replace context.WithCancel with t.Context (#11083) ([ipfs/kubo#11083](https://github.com/ipfs/kubo/pull/11083)) - chore: remove deprecated go-ipfs Docker image publishing (#11081) ([ipfs/kubo#11081](https://github.com/ipfs/kubo/pull/11081)) - Merge release v0.39.0 ([ipfs/kubo#11080](https://github.com/ipfs/kubo/pull/11080)) - docs: move IPIP-476 feature to v0.40 changelog - upgrade go-libp2p-kad-dht to v0.36.0 (#11079) ([ipfs/kubo#11079](https://github.com/ipfs/kubo/pull/11079)) - fix(docker): include symlinks in scanning for init scripts (#11077) ([ipfs/kubo#11077](https://github.com/ipfs/kubo/pull/11077)) - Update deprecation message for Reprovider fields (#11072) ([ipfs/kubo#11072](https://github.com/ipfs/kubo/pull/11072)) - fix doc string (#11068) ([ipfs/kubo#11068](https://github.com/ipfs/kubo/pull/11068)) - feat: support GetClosesPeers (IPIP-476) and ExposeRoutingAPI by default (#10954) ([ipfs/kubo#10954](https://github.com/ipfs/kubo/pull/10954)) - chore: start v0.40.0 release cycle - github.com/gammazero/chanqueue (v1.1.1 -> v1.1.2): - require go1.24 or later (#9) ([gammazero/chanqueue#9](https://github.com/gammazero/chanqueue/pull/9)) - update workflow (#7) ([gammazero/chanqueue#7](https://github.com/gammazero/chanqueue/pull/7)) - prefer range loops (#6) ([gammazero/chanqueue#6](https://github.com/gammazero/chanqueue/pull/6)) - github.com/gammazero/deque (v1.2.0 -> v1.2.1): - fix panic after IterPopX leaves buffer exactly full (#51) ([gammazero/deque#51](https://github.com/gammazero/deque/pull/51)) - fix panic if copying in exactly the buffer size (#50) ([gammazero/deque#50](https://github.com/gammazero/deque/pull/50)) - refactor: prefer range loops (#49) ([gammazero/deque#49](https://github.com/gammazero/deque/pull/49)) - github.com/ipfs/boxo (v0.35.2 -> v0.37.0): - Release v0.37.0 ([ipfs/boxo#1109](https://github.com/ipfs/boxo/pull/1109)) - update dependencies (#1107) ([ipfs/boxo#1107](https://github.com/ipfs/boxo/pull/1107)) - refactor: modernize code (#1105) ([ipfs/boxo#1105](https://github.com/ipfs/boxo/pull/1105)) - ensure http response body is closed (#1103) ([ipfs/boxo#1103](https://github.com/ipfs/boxo/pull/1103)) - update multiaddr dns and otel (#1102) ([ipfs/boxo#1102](https://github.com/ipfs/boxo/pull/1102)) - fix: raise block size limits from 1MiB to 2MiB (#1101) ([ipfs/boxo#1101](https://github.com/ipfs/boxo/pull/1101)) - test(gateway): add dag-pb to dag-json codec conversion tests - feat(gateway): IPIP-0524 + AllowCodecConversion config option (#1077) ([ipfs/boxo#1077](https://github.com/ipfs/boxo/pull/1077)) - feat(unixfs): configurable CID Profiles from IPIP-499 (#1088) ([ipfs/boxo#1088](https://github.com/ipfs/boxo/pull/1088)) - feat(gateway): IPIP-523 format query over Accept header (#1074) ([ipfs/boxo#1074](https://github.com/ipfs/boxo/pull/1074)) - Release v0.36.0 ([ipfs/boxo#1099](https://github.com/ipfs/boxo/pull/1099)) - upgrade go-libp2p-kad-dht to v0.37.1 (#1097) ([ipfs/boxo#1097](https://github.com/ipfs/boxo/pull/1097)) - fix(routing): defensive nil checks for multiaddr handling (#1081) ([ipfs/boxo#1081](https://github.com/ipfs/boxo/pull/1081)) - fix: flaky TestSessionBetweenPeers with shuffle enabled (#1022) ([ipfs/boxo#1022](https://github.com/ipfs/boxo/pull/1022)) - upgrade to go-libp2p v0.47.0 (#1095) ([ipfs/boxo#1095](https://github.com/ipfs/boxo/pull/1095)) - update dependencies (#1091) ([ipfs/boxo#1091](https://github.com/ipfs/boxo/pull/1091)) - refactor: rewrite some flaky tests to testing/synctest (#1087) ([ipfs/boxo#1087](https://github.com/ipfs/boxo/pull/1087)) - fix(routing): fix unknown record bytes unmarshalling (#1090) ([ipfs/boxo#1090](https://github.com/ipfs/boxo/pull/1090)) - refactor: replace `go-clock` with `synctest` (#1082) ([ipfs/boxo#1082](https://github.com/ipfs/boxo/pull/1082)) - chore: fix gofumpt formatting in dagreader.go - cosmetic fixes (#1086) ([ipfs/boxo#1086](https://github.com/ipfs/boxo/pull/1086)) - feat(gateway): configurable fallback timeout for MaxRequestDuration (#1079) ([ipfs/boxo#1079](https://github.com/ipfs/boxo/pull/1079)) - fix(bitswap/network): `stream.Close()` blocks indefinitely on unresponsive peers (#1083) ([ipfs/boxo#1083](https://github.com/ipfs/boxo/pull/1083)) - test: cleanup goroutines at end of test (#1084) ([ipfs/boxo#1084](https://github.com/ipfs/boxo/pull/1084)) - keystore: improve error messages and include key file name (#1080) ([ipfs/boxo#1080](https://github.com/ipfs/boxo/pull/1080)) - docs: update deprecation comments to reference IPIP-526 (#1076) ([ipfs/boxo#1076](https://github.com/ipfs/boxo/pull/1076)) - feat(ipld/merkledag): add total size of visited nodes in progress tracker ([ipfs/boxo#1071](https://github.com/ipfs/boxo/pull/1071)) - upgrade to go-libp2p-kad-dht v0.36.0 ([ipfs/boxo#1072](https://github.com/ipfs/boxo/pull/1072)) - routing/http: add support for GetClosestPeers (IPIP-476) (#1021) ([ipfs/boxo#1021](https://github.com/ipfs/boxo/pull/1021)) - tar: fix name filter on windows ([ipfs/boxo#1047](https://github.com/ipfs/boxo/pull/1047)) - github.com/ipfs/go-cidutil (v0.1.0 -> v0.1.1): - new version (#55) ([ipfs/go-cidutil#55](https://github.com/ipfs/go-cidutil/pull/55)) - update dependencies (#53) ([ipfs/go-cidutil#53](https://github.com/ipfs/go-cidutil/pull/53)) - ci: uci/copy-templates ([ipfs/go-cidutil#43](https://github.com/ipfs/go-cidutil/pull/43)) - ci: uci/copy-templates ([ipfs/go-cidutil#42](https://github.com/ipfs/go-cidutil/pull/42)) - github.com/ipfs/go-datastore (v0.9.0 -> v0.9.1): - new version (#266) ([ipfs/go-datastore#266](https://github.com/ipfs/go-datastore/pull/266)) - update opentelemetry to v1.40.0 (#265) ([ipfs/go-datastore#265](https://github.com/ipfs/go-datastore/pull/265)) - refactor: modernize code (#264) ([ipfs/go-datastore#264](https://github.com/ipfs/go-datastore/pull/264)) - test suite: use a non-cancelled context to delete all keys (#259) ([ipfs/go-datastore#259](https://github.com/ipfs/go-datastore/pull/259)) - Document not to reuse batch (#258) ([ipfs/go-datastore#258](https://github.com/ipfs/go-datastore/pull/258)) - Revert "Test that a second batch commit does not error (#256)" (#257) ([ipfs/go-datastore#257](https://github.com/ipfs/go-datastore/pull/257)) - Test that a second batch commit does not error (#256) ([ipfs/go-datastore#256](https://github.com/ipfs/go-datastore/pull/256)) - github.com/ipfs/go-ds-flatfs (v0.5.5 -> v0.6.0): - new version (#144) ([ipfs/go-ds-flatfs#144](https://github.com/ipfs/go-ds-flatfs/pull/144)) - refactor: rewrite batch mode to use temp directory (#142) ([ipfs/go-ds-flatfs#142](https://github.com/ipfs/go-ds-flatfs/pull/142)) - Clarify the usage of RLock and why RUnlock is missing when applying ops (#141) ([ipfs/go-ds-flatfs#141](https://github.com/ipfs/go-ds-flatfs/pull/141)) - update dependencies (#134) ([ipfs/go-ds-flatfs#134](https://github.com/ipfs/go-ds-flatfs/pull/134)) - github.com/ipfs/go-ds-pebble (v0.5.7 -> v0.5.9): - new version (#79) ([ipfs/go-ds-pebble#79](https://github.com/ipfs/go-ds-pebble/pull/79)) - update error checks to use errors.Is (#78) ([ipfs/go-ds-pebble#78](https://github.com/ipfs/go-ds-pebble/pull/78)) - new version (#76) ([ipfs/go-ds-pebble#76](https://github.com/ipfs/go-ds-pebble/pull/76)) - github.com/ipfs/go-dsqueue (v0.1.1 -> v0.2.0): - release v0.2.0 (#33) ([ipfs/go-dsqueue#33](https://github.com/ipfs/go-dsqueue/pull/33)) - update dependencies and required go version (#32) ([ipfs/go-dsqueue#32](https://github.com/ipfs/go-dsqueue/pull/32)) - new version (#31) ([ipfs/go-dsqueue#31](https://github.com/ipfs/go-dsqueue/pull/31)) - refactor: put queued item decoding logic into function (#30) ([ipfs/go-dsqueue#30](https://github.com/ipfs/go-dsqueue/pull/30)) - use testing/synctest for artificial clock (#29) ([ipfs/go-dsqueue#29](https://github.com/ipfs/go-dsqueue/pull/29)) - github.com/ipfs/go-ipfs-cmds (v0.15.0 -> v0.16.0): - new version (#325) ([ipfs/go-ipfs-cmds#325](https://github.com/ipfs/go-ipfs-cmds/pull/325)) - update to use WaitGroup.Go (#324) ([ipfs/go-ipfs-cmds#324](https://github.com/ipfs/go-ipfs-cmds/pull/324)) - refactor: modernize code (#322) ([ipfs/go-ipfs-cmds#322](https://github.com/ipfs/go-ipfs-cmds/pull/322)) - feat: add --dereference-symlinks flag for recursive symlink resolution (#315) ([ipfs/go-ipfs-cmds#315](https://github.com/ipfs/go-ipfs-cmds/pull/315)) - fix: add remaining binary content types to MIMEEncodings - fix: recognize content-type application/x-tar (#320) ([ipfs/go-ipfs-cmds#320](https://github.com/ipfs/go-ipfs-cmds/pull/320)) - feat(http): ability to control Content-Type of binary responses (#311) ([ipfs/go-ipfs-cmds#311](https://github.com/ipfs/go-ipfs-cmds/pull/311)) - update dependencies and fix test (#318) ([ipfs/go-ipfs-cmds#318](https://github.com/ipfs/go-ipfs-cmds/pull/318)) - update dependencies (#296) ([ipfs/go-ipfs-cmds#296](https://github.com/ipfs/go-ipfs-cmds/pull/296)) - github.com/ipfs/go-ipfs-pq (v0.0.3 -> v0.0.4): - new version (#23) ([ipfs/go-ipfs-pq#23](https://github.com/ipfs/go-ipfs-pq/pull/23)) - fix broken test (#22) ([ipfs/go-ipfs-pq#22](https://github.com/ipfs/go-ipfs-pq/pull/22)) - Update readme links (#21) ([ipfs/go-ipfs-pq#21](https://github.com/ipfs/go-ipfs-pq/pull/21)) - github.com/ipfs/go-log/v2 (v2.9.0 -> v2.9.1): - new version (#179) ([ipfs/go-log#179](https://github.com/ipfs/go-log/pull/179)) - github.com/ipfs/go-peertaskqueue (v0.8.2 -> v0.8.3): - new version ([ipfs/go-peertaskqueue#51](https://github.com/ipfs/go-peertaskqueue/pull/51)) - update dependencies ([ipfs/go-peertaskqueue#50](https://github.com/ipfs/go-peertaskqueue/pull/50)) - Add README.md ([ipfs/go-peertaskqueue#49](https://github.com/ipfs/go-peertaskqueue/pull/49)) - replace go-clock with synctest ([ipfs/go-peertaskqueue#47](https://github.com/ipfs/go-peertaskqueue/pull/47)) - github.com/ipfs/go-unixfsnode (v1.10.2 -> v1.10.3): - new version ([ipfs/go-unixfsnode#92](https://github.com/ipfs/go-unixfsnode/pull/92)) - refactor: modernize code ([ipfs/go-unixfsnode#91](https://github.com/ipfs/go-unixfsnode/pull/91)) - github.com/ipld/go-ipld-prime (v0.21.0 -> v0.22.0): failed to fetch repo - github.com/ipshipyard/p2p-forge (v0.6.1 -> v0.7.0): - chore: release v0.7.0 (#81) ([ipshipyard/p2p-forge#81](https://github.com/ipshipyard/p2p-forge/pull/81)) - feat: add IP denylist plugin for abuse prevention (#79) ([ipshipyard/p2p-forge#79](https://github.com/ipshipyard/p2p-forge/pull/79)) - refactor/test/fix: ipparser hardening (#75) ([ipshipyard/p2p-forge#75](https://github.com/ipshipyard/p2p-forge/pull/75)) - github.com/libp2p/go-libp2p (v0.45.0 -> v0.47.0): - Release v0.47.0 (#3454) ([libp2p/go-libp2p#3454](https://github.com/libp2p/go-libp2p/pull/3454)) - rcmgr: expose resource limits to Prometheus (#3433) ([libp2p/go-libp2p#3433](https://github.com/libp2p/go-libp2p/pull/3433)) - update webtransport-go to v0.10.0 and quic-go to v0.59.0 (#3452)f - fix: handle error from mh.Sum in IDFromPublicKey - fix(basic_host): set read deadline before multistream Close to prevent blocking - update simnet dependency - rename simconlibp2p package to simlibp2p - simlibp2p: add GetBasicHostPair helper - run synctest with Go 1.25 - fix(autonatv2): secondary addrs inherit reachability from primary (#3435) ([libp2p/go-libp2p#3435](https://github.com/libp2p/go-libp2p/pull/3435)) - Release v0.46.0 - chore: update quic-go to v0.57.1 (#3439) ([libp2p/go-libp2p#3439](https://github.com/libp2p/go-libp2p/pull/3439)) - fix(mdns): filter addresses to reduce packet size (#3434) ([libp2p/go-libp2p#3434](https://github.com/libp2p/go-libp2p/pull/3434)) - chore: update quic-go to v0.56.0 (#3425) ([libp2p/go-libp2p#3425](https://github.com/libp2p/go-libp2p/pull/3425)) - fix(webrtc): use debug level for pion errors (#3426) ([libp2p/go-libp2p#3426](https://github.com/libp2p/go-libp2p/pull/3426)) - github.com/libp2p/go-libp2p-kad-dht (v0.36.0 -> v0.38.0): - Release v0.38.0 (#1236) ([libp2p/go-libp2p-kad-dht#1236](https://github.com/libp2p/go-libp2p-kad-dht/pull/1236)) - fix(provider/keystore): protect keystore size during reset (#1227) ([libp2p/go-libp2p-kad-dht#1227](https://github.com/libp2p/go-libp2p-kad-dht/pull/1227)) - update dependencies and minimum go version (#1230) ([libp2p/go-libp2p-kad-dht#1230](https://github.com/libp2p/go-libp2p-kad-dht/pull/1230)) - refactor: apply go fix modernizers from Go 1.26 (#1231) ([libp2p/go-libp2p-kad-dht#1231](https://github.com/libp2p/go-libp2p-kad-dht/pull/1231)) - chore: go-libdht org transfer (#1229) ([libp2p/go-libp2p-kad-dht#1229](https://github.com/libp2p/go-libp2p-kad-dht/pull/1229)) - fix(provider): close datastore results (#1226) ([libp2p/go-libp2p-kad-dht#1226](https://github.com/libp2p/go-libp2p-kad-dht/pull/1226)) - new version (#1225) ([libp2p/go-libp2p-kad-dht#1225](https://github.com/libp2p/go-libp2p-kad-dht/pull/1225)) - replace multierr with errors.Join (#1224) ([libp2p/go-libp2p-kad-dht#1224](https://github.com/libp2p/go-libp2p-kad-dht/pull/1224)) - fix(routing): add per-peer timeouts for PutValue and Provide (#1222) ([libp2p/go-libp2p-kad-dht#1222](https://github.com/libp2p/go-libp2p-kad-dht/pull/1222)) - chore: release v0.37.0 (#1221) ([libp2p/go-libp2p-kad-dht#1221](https://github.com/libp2p/go-libp2p-kad-dht/pull/1221)) - fix(provider): keyspace exploration should succeed with a single peer (#1220) ([libp2p/go-libp2p-kad-dht#1220](https://github.com/libp2p/go-libp2p-kad-dht/pull/1220)) - fix(provider): hold scheduleLk when reading schedule.Size() in test (#1219) ([libp2p/go-libp2p-kad-dht#1219](https://github.com/libp2p/go-libp2p-kad-dht/pull/1219)) - fix(provider): close worker pool before wg.Wait() (#1218) ([libp2p/go-libp2p-kad-dht#1218](https://github.com/libp2p/go-libp2p-kad-dht/pull/1218)) - chore: remove deprecated providers pkg (#1211) ([libp2p/go-libp2p-kad-dht#1211](https://github.com/libp2p/go-libp2p-kad-dht/pull/1211)) - fix(provider): don't discard peers if they all share CPL during exploration (#1216) ([libp2p/go-libp2p-kad-dht#1216](https://github.com/libp2p/go-libp2p-kad-dht/pull/1216)) - fix(records): clone addresses received from peerstore (#1210) ([libp2p/go-libp2p-kad-dht#1210](https://github.com/libp2p/go-libp2p-kad-dht/pull/1210)) - tests: fix flaky TestOptimisticProvide (#1213) ([libp2p/go-libp2p-kad-dht#1213](https://github.com/libp2p/go-libp2p-kad-dht/pull/1213)) - tests: fix flaky TestHandleRemotePeerProtocolChanges (#1212) ([libp2p/go-libp2p-kad-dht#1212](https://github.com/libp2p/go-libp2p-kad-dht/pull/1212)) - chore: bump go-libp2p to v0.46 (#1209) ([libp2p/go-libp2p-kad-dht#1209](https://github.com/libp2p/go-libp2p-kad-dht/pull/1209)) - github.com/libp2p/go-libp2p-pubsub (v0.14.2 -> v0.15.0): - release v0.15.0 - Fix data race in test - Skip flaky floodsub test - pubsub: remove redundant sends of hello packet - gossipsub: implement extensions - test: add skeleton gossipsub to drive a gossipsub peer - gossipsub: add plumbing for Gossipsub v1.3 support - pubsub: AddPeer now accepts reference to hello packet - pb: add extensions protobufs - feat: add px peer record reducer + pub addrs filter - Migrate to `log/slog` - chore: add params.Dscore validation - Release v0.14.3 - Unexport Params.Validate to maintain patch release semantics - Merge pull request #642 for GossipSub Params validation - fix: Select ctx.Done() when preprocessing to avoid blocking on cancel (#635) ([libp2p/go-libp2p-pubsub#635](https://github.com/libp2p/go-libp2p-pubsub/pull/635)) - github.com/libp2p/go-netroute (v0.3.0 -> v0.4.0): - v0.4.0 - gofmt ([libp2p/go-netroute#65](https://github.com/libp2p/go-netroute/pull/65)) - linux: use rtnetlink directly ([libp2p/go-netroute#64](https://github.com/libp2p/go-netroute/pull/64)) - github.com/multiformats/go-multiaddr-dns (v0.4.1 -> v0.5.0): - Release v0.5.0 - refactor: remove miekg/dns dep (#72) ([multiformats/go-multiaddr-dns#72](https://github.com/multiformats/go-multiaddr-dns/pull/72))
### 👨‍👩‍👧‍👦 Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | [@lidel](https://github.com/lidel) | 62 | +18446/-3513 | 406 | | [@gammazero](https://github.com/gammazero) | 84 | +5719/-2815 | 374 | | [@MarcoPolo](https://github.com/MarcoPolo) | 24 | +1275/-311 | 58 | | [@guillaumemichel](https://github.com/guillaumemichel) | 14 | +392/-967 | 41 | | [@hsanjuan](https://github.com/hsanjuan) | 4 | +1093/-43 | 15 | | [@sneaxhuh](https://github.com/sneaxhuh) | 2 | +840/-19 | 7 | | [@cortze](https://github.com/cortze) | 1 | +367/-35 | 2 | | [@schomatis](https://github.com/schomatis) | 1 | +288/-17 | 5 | | [@rifeplight](https://github.com/rifeplight) | 1 | +92/-195 | 18 | | [@sukunrt](https://github.com/sukunrt) | 3 | +22/-211 | 7 | | [@2color](https://github.com/2color) | 1 | +207/-2 | 5 | | [@djdv](https://github.com/djdv) | 8 | +96/-65 | 10 | | [@vlerdman](https://github.com/vlerdman) | 4 | +90/-38 | 8 | | [@v1rtl](https://github.com/v1rtl) | 1 | +71/-3 | 2 | | [@VedantMadane](https://github.com/VedantMadane) | 1 | +23/-8 | 4 | | [@dozyio](https://github.com/dozyio) | 1 | +26/-4 | 2 | | [@infrmtcs](https://github.com/infrmtcs) | 1 | +24/-1 | 2 | | [@marten-seemann](https://github.com/marten-seemann) | 1 | +5/-1 | 2 | | [@web3-bot](https://github.com/web3-bot) | 2 | +3/-2 | 2 | | [@MozirDmitriy](https://github.com/MozirDmitriy) | 1 | +4/-1 | 1 | | [@aschmahmann](https://github.com/aschmahmann) | 1 | +2/-1 | 1 | | [@willscott](https://github.com/willscott) | 1 | +1/-1 | 1 | | [@lbarrettanderson](https://github.com/lbarrettanderson) | 1 | +1/-1 | 1 | | [@filipremb](https://github.com/filipremb) | 1 | +1/-1 | 1 | ## v0.40.1 ### 🔦 Highlights #### Windows stability fix If you run Kubo on Windows, v0.40.0 can crash after running for a while. The daemon starts fine and works normally at first, but eventually hits a memory corruption in Go's network I/O layer and dies. This is likely caused by an upstream Go 1.26 regression in overlapped I/O handling that has known issues ([go#77142](https://github.com/golang/go/issues/77142), [#11214](https://github.com/ipfs/kubo/issues/11214)). This patch release downgrades the Go toolchain from 1.26 to 1.25, which does not have this bug. If you are running Kubo on Windows, upgrade to v0.40.1. We will switch back to Go 1.26.x once the upstream fix lands. ### 📝 Changelog
Full Changelog v0.40.1 - github.com/ipfs/kubo: - chore: downgrade to Go 1.25 to fix Windows crash ([ipfs/kubo#11215](https://github.com/ipfs/kubo/pull/11215))
================================================ FILE: docs/changelogs/v0.41.md ================================================ # Kubo changelog v0.41 This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [v0.41.0](#v0410) ## v0.41.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [🗑️ Faster Provide Queue Disk Reclamation](#-faster-provide-queue-disk-reclamation) - [✨ New `ipfs cid inspect` command](#-new-ipfs-cid-inspect-command) - [🖥️ WebUI Improvements](#-webui-improvements) - [🔧 Correct provider addresses for custom HTTP routing](#-correct-provider-addresses-for-custom-http-routing) - [📦️ Dependency updates](#-dependency-updates) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights #### 🗑️ Faster Provide Queue Disk Reclamation Nodes with significant amount of data and DHT provide sweep enabled (`Provide.DHT.SweepEnabled`, the default since Kubo 0.39) could see their `datastore/` directory grow continuously. Each reprovide cycle rewrote the provider keystore inside the shared repo datastore, generating tombstones faster than the storage engine could compact them, and in default configuration Kubo was slow to reclaim this space. The provider keystore now lives in a dedicated datastore under `$IPFS_PATH/provider-keystore/`. After each reprovide cycle the old datastore is removed from disk entirely, so space is reclaimed immediately regardless of storage backend. On first start after upgrading, stale keystore data is cleaned up from the shared datastore automatically. To learn more, see [kubo#11096](https://github.com/ipfs/kubo/issues/11096), [kubo#11198](https://github.com/ipfs/kubo/pull/11198), and [go-libp2p-kad-dht#1233](https://github.com/libp2p/go-libp2p-kad-dht/pull/1233). #### ✨ New `ipfs cid inspect` command New subcommand for breaking down a CID into its components. Works offline, supports `--enc=json`. ```console $ ipfs cid inspect bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi CID: bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi Version: 1 Multibase: base32 (b) Multicodec: dag-pb (0x70) Multihash: sha2-256 (0x12) Length: 32 bytes Digest: c3c4733ec8affd06cf9e9ff50ffc6bcd2ec85a6170004bb709669c31de94391a CIDv0: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR CIDv1: bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi ``` See `ipfs cid --help` for all CID-related commands. #### 🖥️ WebUI Improvements IPFS Web UI has been updated to [v4.12.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.12.0). ##### IPv6 peer geolocation and Peers screen optimizations The Peers screen now resolves IPv6 addresses to geographic locations, and the geolocation database has been updated to `GeoLite2-City-CSV_20260220`. ([ipfs-geoip v9.3.0](https://github.com/ipfs-shipyard/ipfs-geoip/releases/tag/v9.3.0)) Peer locations load faster thanks to UX optimizations in the underlying ipfs-geoip library. #### 🔧 Correct provider addresses for custom HTTP routing Nodes using custom routing (`Routing.Type=custom`) with [IPIP-526](https://github.com/ipfs/specs/pull/526) could end up publishing unresolved `0.0.0.0` addresses in provider records. Addresses are now resolved at provide-time, and when AutoNAT V2 has confirmed publicly reachable addresses, those are preferred automatically. See [#11213](https://github.com/ipfs/kubo/issues/11213). #### 📦️ Dependency updates - update `ipfs-webui` to [v4.12.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.12.0) - update `gateway-conformance` tests to [v0.11](https://github.com/ipfs/gateway-conformance/releases/tag/v0.11.0) ### 📝 Changelog ### 👨‍👩‍👧‍👦 Contributors ================================================ FILE: docs/changelogs/v0.42.md ================================================ # Kubo changelog v0.42 This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [v0.42.0](#v0420) ## v0.42.0 - [Overview](#overview) - [🔦 Highlights](#-highlights) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) ### Overview ### 🔦 Highlights ### 📝 Changelog ### 👨‍👩‍👧‍👦 Contributors ================================================ FILE: docs/changelogs/v0.5.md ================================================ # go-ipfs changelog v0.5 ## v0.5.1 2020-05-08 Hot on the heels of 0.5.0 is 0.5.1 with some important but small bug fixes. This release: 1. Removes the 1 minute timeout for IPNS publishes (fixes #7244). 2. Backport a DHT fix to reduce CPU usage for canceled requests. 3. Fixes some timer leaks in the QUIC transport ([ipfs/go-ipfs#2515](https://github.com/lucas-clemente/quic-go/issues/2515)). ### Changelog - github.com/ipfs/go-ipfs: - IPNS timeout patch from master ([ipfs/go-ipfs#7276](https://github.com/ipfs/go-ipfs/pull/7276)) - github.com/libp2p/go-libp2p-core (v0.5.2 -> v0.5.3): - feat: add a function to tell if a context subscribes to query events ([libp2p/go-libp2p-core#147](https://github.com/libp2p/go-libp2p-core/pull/147)) - github.com/libp2p/go-libp2p-kad-dht (v0.7.10 -> v0.7.11): - fix: optimize for the case where we're not subscribing to query events ([libp2p/go-libp2p-kad-dht#624](https://github.com/libp2p/go-libp2p-kad-dht/pull/624)) - fix: don't spin when the event channel is closed ([libp2p/go-libp2p-kad-dht#622](https://github.com/libp2p/go-libp2p-kad-dht/pull/622)) - github.com/libp2p/go-libp2p-routing-helpers (v0.2.2 -> v0.2.3): - fix: avoid subscribing to query events unless necessary ([libp2p/go-libp2p-routing-helpers#43](https://github.com/libp2p/go-libp2p-routing-helpers/pull/43)) - github.com/lucas-clemente/quic-go (v0.15.5 -> v0.15.7): - reset the PTO when dropping a packet number space - move deadlineTimer declaration out of the Read loop - stop the deadline timer in Stream.Read and Write - fix buffer use after it was released when sending an INVALID_TOKEN error - create the session timer at the beginning of the run loop - stop the timer when the session's run loop returns ### Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------------------|---------|---------|---------------| | Marten Seemann | 10 | +81/-62 | 19 | | Steven Allen | 5 | +42/-18 | 10 | | Adin Schmahmann | 1 | +2/-8 | 1 | | dependabot-preview[bot] | 2 | +6/-2 | 4 | ## v0.5.0 2020-04-28 We're excited to announce go-ipfs 0.5.0! This is by far the largest go-ipfs release with ~2500 commits, 98 contributors, and over 650 PRs across ipfs, libp2p, and multiformats. ### Highlights #### Content Routing The primary focus of this release was on improving content routing. That is, advertising and finding content. To that end, this release heavily focuses on improving the DHT. ##### Improved DHT The distributed hash table (DHT) is how IPFS nodes keep track of who has what data. The DHT implementation has been almost completely rewritten in this release. Providing, finding content, and resolving IPNS records are now all much faster. However, there are risks involved with this update due to the significant amount of changes that have gone into this feature. The current DHT suffers from three core issues addressed in this release: - Most peers in the DHT cannot be dialed (e.g., due to firewalls and NATs). Much of a DHT query time is wasted trying to connect to peers that cannot be reached. - The DHT query logic doesn't properly terminate when it hits the end of the query and, instead, aggressively keeps on searching. - The routing tables are poorly maintained. This can cause search performance to slow down linearly with network size, instead of logarithmically as expected. ###### Reachability We have addressed the problem of undialable nodes by having nodes wait to join the DHT as _server_ nodes until they've confirmed that they are reachable from the public internet. To ensure that nodes which are not publicly reachable (ex behind VPNs, offline LANs, etc.) can still coordinate and share data, go-ipfs 0.5 will run two DHTs: one for private networks and one for the public internet. Every node will participate in a LAN DHT and a public WAN DHT. See [Dual DHT](#dual-dht) for more details. ###### Dual DHT All IPFS nodes will now run two DHTs: one for the public internet WAN, and one for their local network LAN. 1. When connected to the public internet, IPFS will use both DHTs for finding peers, content, and IPNS records. Nodes only publish provider and IPNS records to the WAN DHT to avoid flooding the local network. 2. When not connected to the public internet, nodes publish provider and IPNS records to the LAN DHT. The WAN DHT includes all peers with at least one public IP address. This release will only consider an IPv6 address public if it is in the [public internet range `2000::/3`](https://www.iana.org/assignments/ipv6-address-space/ipv6-address-space.xhtml). This feature should not have any noticeable impact on go-ipfs, performance, or otherwise. Everything should continue to work in all the currently supported network configurations: VPNs, disconnected LANs, public internet, etc. ###### Query Logic We've improved the DHT query logic to more closely follow Kademlia. This should significantly speed up: - Publishing IPNS & provider records. - Resolving IPNS addresses. Previously, nodes would continue searching until they timed out or ran out of peers before stopping (putting or returning data found). Now, nodes will now stop as soon as they find the closest peers. ###### Routing Tables Finally, we've addressed the poorly maintained routing tables by: - Reducing the likelihood that the connection manager will kill connections to peers in the routing table. - Keeping peers in the routing table, even if we get disconnected from them. - Actively and frequently querying the DHT to keep our routing table full. - Prioritizing useful peers that respond to queries quickly. ##### Testing The DHT rewrite was made possible by [Testground](https://github.com/ipfs/testground/), our new testing framework. Testground allows us to spin up multi-thousand node tests with simulated real-world network conditions. By combining Testground and some custom analysis tools, we were able to gain confidence that the new DHT implementation behaves correctly. ##### Provider Record Changes When you add content to your IPFS node, you advertise this content to the network by announcing it in the DHT. We call this _providing_. However, go-ipfs has multiple ways to address the same underlying bytes. Specifically, we address content by content ID (CID) and the same underlying bytes can be addressed using (a) two different versions of CIDs (CIDv0 and CIDv1) and (b) with different _codecs_ depending on how we're interpreting the data. Prior to go-ipfs 0.5.0, we used the content id (CID) in the DHT when sending out provider records for content. Unfortunately, this meant that users trying to find data announced using one CID wouldn't find nodes providing the content under a different CID. In go-ipfs 0.5.0, we're announcing data by _multihash_, not _CID_. This way, regardless of the CID version used by the peer adding the content, the peer trying to download the content should still be able to find it. **Warning:** as part of the network, this could impact finding content added with CIDv1. Because go-ipfs 0.5.0 will announce and search for content using the bare multihash (equivalent to the v0 CID), go-ipfs 0.5.0 will be unable to find CIDv1 content published by nodes prior to go-ipfs 0.5.0 and vice-versa. As CIDv1 is _not_ enabled by default so we believe this will have minimal impact. However, users are _strongly_ encouraged to upgrade as soon as possible. #### Content Transfer A secondary focus in this release was improving content _transfer_, our data exchange protocols. ##### Refactored Bitswap This release includes a major [Bitswap refactor](https://blog.ipfs.io/2020-02-14-improved-bitswap-for-container-distribution/), running a new and backward compatible Bitswap protocol. We expect these changes to improve performance significantly. With the refactored Bitswap, we expect: - Few to no duplicate blocks when fetching data from other nodes speaking the _new_ protocol. - Better parallelism when fetching from multiple peers. The new Bitswap won't magically make downloading content any faster until both seeds and leaches have updated. If you're one of the first to upgrade to `0.5.0` and try downloading from peers that haven't upgraded, you're unlikely to see much of a performance improvement. [bitswap-refactor]: https://blog.ipfs.io/2020-02-14-improved-bitswap-for-container-distribution/ ##### Server-Side Graphsync Support (Experimental) Graphsync is a new exchange protocol that operates at the IPLD Graph layer instead of the Block layer like bitswap. For example, to download "/ipfs/QmExample/index.html": * Bitswap would download QmFoo, lookup "index.html" in the directory named by QmFoo, resolving it to a CID QmIndex. Finally, bitswap would download QmIndex. * Graphsync would ask peers for "/ipfs/QmFoo/index.html". Specifically, it would ask for the child named "index.html" of the object named by "QmFoo". This saves us round-trips in exchange for some extra protocol complexity. Moreover, this protocol allows specifying more powerful queries like "give me everything under QmFoo". This can be used to quickly download a large amount of data with few round-trips. At the moment, go-ipfs cannot use this protocol to download content from other peers. However, if enabled, go-ipfs can _serve_ content to other peers over this protocol. This may be useful for pinning services that wish to quickly replicate client data. To enable, run: ```bash > ipfs config --json Experimental.GraphsyncEnabled true ``` #### Datastores Continuing with the of improving our core data handling subsystems, both of the datastores used in go-ipfs, Badger and flatfs, have received important updates in this release: ##### Badger Badger has been in go-ipfs for over a year as an experimental feature, and we're promoting it to stable (but not default). For this release, we've switched from writing to disk synchronously to explicitly syncing where appropriate, significantly increasing write throughput. The current and default datastore used by go-ipfs is [FlatFS](https://github.com/ipfs/go-ds-flatfs). FlatFS essentially stores blocks of data as individual files on your file system. However, there are lots of optimizations a specialized database can do that a standard file system can not. The benefit of Badger is that adding/fetching data to/from Badger is significantly faster than adding/fetching data to/from the default datastore, FlatFS. In some tests, adding data to Badger is 32x faster than FlatFS (in this release). ###### Enable Badger In this release, we're marking the badger datastore as stable. However, we're not yet enabling it by default. You can enable it at initialization by running: `ipfs init --profile=badgerds` ###### Issues with Badger While Badger is a great solution, there are some issues you should consider before enabling it. Badger is complicated. FlatFS pushes all the complexity down into the filesystem itself. That means that FlatFS is only likely to lose your data if your underlying filesystem gets corrupted while there are more opportunities for Badger itself to get corrupted. Badger can use a lot of memory. In this release, we've tuned Badger to use `~20MB` of memory by default. However, it can still produce spikes as large as [`1GiB` of data](https://github.com/dgraph-io/badger/issues/1292) in memory usage when garbage collecting. Finally, Badger isn't very aggressive when it comes to garbage collection, and we're still investigating ways to get it to more aggressively clean up after itself. We suggest you use Badger if: - Performance is your main requirement. - You rarely delete anything. - You have some memory to spare. ##### Flatfs In the flatfs datastore, we've fixed an issue where temporary files could be left behind in some cases. While this release will avoid leaving behind temporary files, you may want to remove any left behind by previous releases: ```bash > rm ~/.ipfs/blocks/*/put-* > rm ~/.ipfs/blocks/du-* ``` We've also hardened several edge-cases in flatfs to reduce the impact of file descriptor limits, spurious crashes, etc. #### Libp2p Many improvements and bug fixes were made to libp2p over the course of this release. These release notes only include the most important and those most relevant to the content routing improvements. ##### Improved Backoff Logic When we fail to connect to a peer, we "backoff" and refuse to re-connect to that peer for a short period of time. This prevents us from wasting resources repeatedly failing to connect to the same unreachable peer. Unfortunately, the old backoff logic was flawed: if we failed to connect to a peer and entered the "backoff" state, we wouldn't try to re-connect to that peer even if we had learned new and potentially working addresses for the peer. We've fixed this by applying backoff to each _address_ instead of to the peer as a whole. This achieves the same result as we'll stop repeatedly trying to connect to the peer at known-bad addresses, but it allows us to reach the peer if we later learn about a good address. ##### AutoNAT This release uses Automatic NAT Detection (AutoNAT) - determining if the node is _reachable_ from the public internet - to make decisions about how to participate in IPFS. This subsystem is used to determine if the node should store some of the public DHT, and if it needs to use relays to be reached by others. In short: 1. An AutoNAT client asks a node running an AutoNAT service if it can be reached at one of a set of guessed addresses. 2. The AutoNAT service attempts to _dial back_ those addresses, with some restrictions. We won't dial back to a different IP address, for example. 3. If the AutoNAT service succeeds, it reports back the address it successfully dialed, and the AutoNAT client knows that it is reachable from the public internet. All nodes act as AutoNAT clients to determine if they should switch into DHT server mode. As of this release, nodes will by default run the service side of AutoNAT - verifying connectivity - for up to 30 peers every minute. This service should have minimal overhead and will be disabled for nodes in the `lowpower` configuration profile, and those which believe they are not publicly reachable. In addition to enabling the AutoNAT service by default, this release changes the AutoNAT config options: 1. The `Swarm.EnableAutoNATService` option has been removed. 2. A new AutoNAT section has been added to the config. This section is empty by default. ##### IPFS/Libp2p Address Format If you've ever run a command like `ipfs swarm peers`, you've likely seen paths that look like `/ip4/193.45.1.24/tcp/4001/ipfs/QmSomePeerID`. These paths are _not_ file paths, they're multiaddrs; addresses of peers on the network. Unfortunately, `/ipfs/Qm...` is _also_ the same path format we use for files. This release, changes the multiaddr format from /ip4/193.45.1.24/tcp/4001/ipfs/QmSomePeerID to /ip4/193.45.1.24/tcp/4001/p2p/QmSomePeerID to make the distinction clear. What this means for users: * Old-style multiaddrs will still be accepted as inputs to IPFS. * If you were using a multiaddr library (go, js, etc.) to name _files_ because `/ipfs/QmSomePeerID` looks like `/ipfs/QmSomeFile`, your tool may break if you upgrade this library. * If you're manually parsing multiaddrs and are searching for the string `/ipfs/`..., you'll need to search for `/p2p/...`. ##### Minimum RSA Key Size Previously, IPFS did not enforce a minimum RSA key size. In this release, we've introduced a minimum 2048 bit RSA key size. IPFS generates 2048 bit RSA keys by default so this shouldn't be an issue for anyone in practice. However, users who explicitly chose a smaller key size will not be able to communicate with new nodes. Unfortunately, some of the bootstrap peers _did_ intentionally generate 1024 bit RSA keys so they'd have vanity peer addresses (starting with QmSoL for "solar net"). All IPFS nodes should _also_ have peers with >= 2048 bit RSA keys in their bootstrap list, but we've introduced a migration to ensure this. We implemented this change to follow security best practices and to remove a potential foot-gun. However, in practice, the security impact of allowing insecure RSA keys should have been next to none because IPFS doesn't trust other peers on the network anyways. ##### TLS By Default In this release, we're switching TLS to be the _default_ transport. This means we'll try to encrypt the connection with TLS before re-trying with SECIO. Contrary to the announcement in the go-ipfs 0.4.23 release notes, this release does not remove SECIO support to maintain compatibility with js-ipfs. Note: The `Experimental.PreferTLS` configuration option is now ignored. ##### SECIO Deprecation Notice SECIO should be considered to be well on the way to deprecation and will be completely disabled in either the next release (0.6.0, ~mid May) or the one following that (0.7.0, ~end of June). Before SECIO is disabled, support will be added for the NOISE transport for compatibility with other IPFS implementations. ##### QUIC Upgrade If you've been using the experimental QUIC support, this release upgrades to a new and _incompatible_ version of the QUIC protocol (draft 27). Old and new go-ipfs nodes will still interoperate, but not over the QUIC transport. We intend to standardize on this draft of the QUIC protocol and enable QUIC by default in the next release if all goes well. NOTE: QUIC does not yet support [private networks](./docs/experimental-features.md#private-networks). #### Gateway In addition to a bunch of bug fixes, we've made two improvements to the gateway. You can play with both of these features by visiting: > http://bafybeia6po64b6tfqq73lckadrhpihg2oubaxgqaoushquhcek46y3zumm.ipfs.localhost:8080 ##### Subdomain Gateway First up, we've changed how URLs in the IPFS gateway work for better browser security. The gateway will now redirect from `http://localhost:8080/ipfs/CID/...` to `http://CID.ipfs.localhost:8080/...` by default. This: * Ensures that every dapp gets its own browser origin. * Makes it easier to write websites that "just work" with IPFS because absolute paths will now work (though you should still use relative links because they're better). Paths addressing the gateway by IP address (`http://127.0.0.1:5001/ipfs/CID`) will not be altered as IP addresses can't have subdomains. Note: cURL doesn't follow redirects by default. To avoid breaking cURL and other clients that don't support redirects, go-ipfs will return the requested file along with the redirect. Browsers will follow the redirect and abort the download while cURL will ignore the redirect and finish the download. ##### Directory Listing The second feature is a face-lift to the directory listing theme and color palette. > http://bafybeia6po64b6tfqq73lckadrhpihg2oubaxgqaoushquhcek46y3zumm.ipfs.localhost:8080 #### IPNS This release includes several new IPNS and IPNS-related features. ##### ENS IPFS now resolves [ENS](https://ens.domains/) names (e.g., `/ipns/ipfs.eth`) via DNSLink provided by https://eth.link service. ##### IPNS over PubSub IPFS has had experimental support for resolving IPNS over pubsub for a while. However, in the past, this feature was passive. When resolving an IPNS name, one would join a pubsub topic for the IPNS name and subscribe to _future_ updates. Unfortunately, this wouldn't speed-up initial IPNS lookups. In this release, we've introduced a new "record fetch" protocol to speedup the initial lookup. Now, after subscribing to the pubsub topic for the IPNS key, nodes will use this new protocol to "fetch" the last-seen IPNS record from all peers subscribed to the topic. This feature will be enabled by default in 0.6.0. ##### IPNS with base32 PIDs IPNS names can now be expressed as special multibase CIDs. E.g., > /ipns/bafzbeibxfjp4gaxc4cdn57257cyvc7jfa4rlp4e5min6geg44m57g6nx7e Importantly, this allows IPNS names to appear in subdomains in the new [subdomain gateway](#subdomain-gateway) feature. #### PubSub We have made two major changes to the pubsub subsystem in this release: 1. Pubsub now more aggressively finds and connects to other peers subscribing to the same topic. 2. Go-ipfs has switched its default pubsub router from "floodsub", an inefficient but simple "flooding" pubsub implementation, to "gossipsub". PubSub will be stabilized in go-ipfs 0.6.0. #### CLI & API The IPFS CLI and API have a couple of new features and changes. ##### POST Only IPFS has two HTTP APIs: * Port 5001: http://localhost:5001/api/v0/... - the API * Port 8080: http://localhost:8080/api/v0/... - a read-only subset of the API, accessible via the gateway As of this release, the main IPFS API (port 5001) will only accept POST requests. This change is necessary to tighten cross origin security in browsers. If you're using the go-ipfs API in your application, you may need to change GET calls to POST calls or upgrade your libraries and tools. * go - go-ipfs-api - v0.0.3 * js-ipfs-http-api - v0.41.1 * orbit-db - v0.24.0 (unreleased) ##### RIP "Error: api not running" If you've ever seen [the error](https://github.com/ipfs/go-ipfs/issues/5784): > Error: api not running when trying to run a command without the daemon running, we have good news! You should never see this error again. The `ipfs` command now correctly detects that the daemon is not, in fact, running, and directly opens the IPFS repo. ##### RIP `ipfs repo fsck` The `ipfs repo fsck` now does nothing but print an error message. Previously, it was used to cleanup some lock files: the "api" file that caused the aforementioned "api not running" error and the repo lock. However, this is no longer necessary. ##### Init with config It's now possible to initialize an IPFS node with an existing IPFS config by running: ```bash > ipfs init /path/to/existing/config ``` This will reuse the existing configuration in it's entirety (including the private key) and can be useful when: * Migrating a node's identity between machines without keeping the data. * Resetting the datastore. ##### Ignoring Files Files can now be ignored on add by passing the `--ignore` and/or `--ignore-rules-path` flags. * `--ignore=PATTERN` will ignore all files matching the gitignore rule PATTERN. * `--ignore-rules-path=FILENAME` will apply the gitignore rules from the specified file. For example, to add a git repo while ignoring all files git would ignore, you could run: ```bash > cd path/to/some/repo > ipfs add -r --hidden=false --ignore=.git --ignore-rules-path=.gitignore . ``` ##### Named Pipes It's now possible to add data directly from a named pipe: ```bash > mkfifo foo > echo -n "hello " > foo & > echo -n "world" > bar & > ipfs add foo bar ``` This can be useful when adding data from multiple streaming sources. NOTE: To avoid surprising users, IPFS will only add data from FIFOs _directly_ named on the command line, not FIFOs in a recursively added directory. Otherwise, `ipfs add` would halt whenever it encountered a FIFO with no data to be read leading to difficult to debug stalls. ##### DAG import/export (.car) IPFS now allows rapid reading and writing of blocks in [`.car` format](https://github.com/ipld/specs/blob/master/block-layer/content-addressable-archives.md#readme). The functionality is accessible via the experimental `dag import` and `dag export` commands: ``` ~$ ipfs dag export QmQPeNsJPyVWPFDVHb77w8G42Fvo15z4bG2X8D2GhfbSXc \ | xz > welcome_to_ipfs.car.xz 0s 6.73 KiB / ? [-------=-------------------------------------] 5.16 MiB/s 0s ``` Then on another `ipfs` instance, not even connected to the network: ``` ~$ xz -dc welcome_to_ipfs.car.xz | ipfs dag import Pinned root QmQPeNsJPyVWPFDVHb77w8G42Fvo15z4bG2X8D2GhfbSXc success ``` ##### Pins We've made two minor changes to the pinning subsystem: 1. `ipfs pin ls --stream` allows streaming a pin listing. 2. `ipfs pin update` no longer holds the global pin lock while fetching files from the network. This should hopefully make it significantly more useful. #### Daemon ##### Zap Logging The go-ipfs daemon has switched to using [Uber's Zap](https://go.uber.org/zap). Unlike our previous logging system, Zap supports _structured_ logging which can make parsing, filtering, and analyzing go-ipfs logs much simpler. To enable structured logging, set the `IPFS_LOGGING_FMT` environment variable to "json". Note: while we've switched to using Zap as the logging backend, most of go-ipfs still logs strings. ##### Systemd Support For Linux users, this release includes support for two systemd features: socket activation and startup/shutdown notifications. This makes it possible to: * Start IPFS on demand on first use. * Wait for IPFS to finish starting before starting services that depend on it. You can find the new systemd units in the go-ipfs repo under misc/systemd. ##### IPFS API Over Unix Domain Sockets This release supports exposing the IPFS API over a unix domain socket in the filesystem. You use this feature, run: ```bash > ipfs config Addresses.API "/unix/path/to/socket/location" ``` ##### Docker We've made a few improvements to our docker image in this release: * It can now be cross-built for multiple architectures. * It now builds go-ipfs with OpenSSL support by default for faster libp2p handshakes. * A private-network "swarm" key can now be passed in to a docker image via either the `IPFS_SWARM_KEY=` or `IPFS_SWARM_KEY_FILE=` docker variables. Check out the Docker section of the README for more information. #### Plugins go-ipfs plugins allow users to extend go-ipfs without modifying the original source-code. This release includes a few important changes. See [docs/plugins.md](./docs/plugins.md) for details. ##### MacOS Support Plugins are now supported on MacOS, in addition to Linux. Unfortunately, Go still doesn't [support plugins on Windows](https://github.com/golang/go/issues/19282). ##### New Plugin Type: `InternalPlugin` This release introduces a new `InternalPlugin` plugin type. When started, this plugin will be passed a raw `*IpfsNode` object, giving it access to all go-ipfs internals. This plugin interface is permanently unstable as it has access to internals that can change frequently. However, it should allow power-users to develop deeply integrated extensions to go-ipfs, out-of-tree. ##### Plugin Config **BREAKING** Plugins can now be configured and/or disabled via the [ipfs config file](./docs/plugins.md#configuration). To make this possible, the plugin interface has changed. The `Init` function now takes an `*Environment` object. Specifically, the plugin signature has changed from: ```go type Plugin interface { Name() string Version() string Init() error } ``` to ```go type Environment struct { // Path to the IPFS repo. Repo string // The plugin's config, if specified. Config interface{} } type Plugin interface { Name() string Version() string Init(env *Environment) error } ``` #### Repo Migrations IPFS uses repo migrations to make structural changes to the "repo" (the config, data storage, etc.) on upgrade. This release includes two very simple repo migrations: a config migration to ensure that the config contains working bootstrap nodes and a keystore migration to base32 encode all key filenames. In general, migrations should not require significant manual intervention. However, you should be aware of migrations and plan for them. * If you update go-ipfs with `ipfs update`, `ipfs update` will run the migration for you. Note: `ipfs update` will refuse to run the migrations while ipfs itself is running. * If you start the ipfs daemon with `ipfs daemon --migrate`, ipfs will migrate your repo for you on start. Otherwise, if you want more control over the repo migration process, you can manually install and run the [repo migration tool](http://dist.ipfs.tech/#fs-repo-migrations). ##### Bootstrap Peer Changes **AUTOMATIC MIGRATION REQUIRED** The first migration will update the bootstrap peer list to: 1. Replace the old bootstrap nodes (ones with peer IDs starting with QmSoL), with new bootstrap nodes (ones with addresses that start with `/dnsaddr/bootstrap.libp2p.io`). 2. Rewrite the address format from `/ipfs/QmPeerID` to `/p2p/QmPeerID`. We're migrating addresses for a few reasons: 1. We're using DNS to address the new bootstrap nodes so we can change the underlying IP addresses as necessary. 2. The new bootstrap nodes use 2048 bit keys while the old bootstrap nodes use 1024 bit keys. 3. We're normalizing the address format to `/p2p/Qm...`. Note: This migration won't _add_ the new bootstrap peers to your config if you've explicitly removed the old bootstrap peers. It will also leave custom entries in the list alone. In other words, if you've customized your bootstrap list, this migration won't clobber your changes. ##### Keystore Changes **AUTOMATIC MIGRATION REQUIRED** go-ipfs stores additional keys (i.e., all keys other than the "identity" key) in the keystore. You can list these keys with `ipfs key`. Currently, the keystore stores keys as regular files, named after the key itself. Unfortunately, filename restrictions and case-insensitivity are platform specific. To avoid platform specific issues, we're base32 encoding all key names and renaming all keys on-disk. #### Windows As usual, this release contains several Windows specific fixes and improvements: * Double-clicking `ipfs.exe` will now start the daemon inside a console window. * `ipfs add -r` now correctly recognizes and ignores hidden files on Windows. * The default datastore, flatfs, now takes extra precautions to avoid "file in use" errors caused by both go-ipfs and external programs like anti-viruses. If you've ever seen go-ipfs print out an "access denied" or "file in use" error on Windows, this issue was likely the cause. ### Changelog - github.com/ipfs/go-ipfs: - fix: non-blocking peerlog logging ([ipfs/go-ipfs#7232](https://github.com/ipfs/go-ipfs/pull/7232)) - doc: update go-ipfs docs for 0.5.0 release ([ipfs/go-ipfs#7229](https://github.com/ipfs/go-ipfs/pull/7229)) - Add additional documentation links to the new issue screen ([ipfs/go-ipfs#7226](https://github.com/ipfs/go-ipfs/pull/7226)) - docs: note that ShardingEnabled is a global flag ([ipfs/go-ipfs#7218](https://github.com/ipfs/go-ipfs/pull/7218)) - update log helptext to match actual levels ([ipfs/go-ipfs#7199](https://github.com/ipfs/go-ipfs/pull/7199)) - Chore/harden car test a bit harder ([ipfs/go-ipfs#7209](https://github.com/ipfs/go-ipfs/pull/7209)) - fix: fix duplicate block issue in bitswap ([ipfs/go-ipfs#7202](https://github.com/ipfs/go-ipfs/pull/7202)) - feat: update docker image ([ipfs/go-ipfs#7191](https://github.com/ipfs/go-ipfs/pull/7191)) - feat: update dir index ([ipfs/go-ipfs#7192](https://github.com/ipfs/go-ipfs/pull/7192)) - fix: update the dht to fix yggdrasil ([ipfs/go-ipfs#7186](https://github.com/ipfs/go-ipfs/pull/7186)) - Choose architecture when download tini into docker container ([ipfs/go-ipfs#7187](https://github.com/ipfs/go-ipfs/pull/7187)) - Fix typos and cleanup ([ipfs/go-ipfs#7181](https://github.com/ipfs/go-ipfs/pull/7181)) - Fix typos ([ipfs/go-ipfs#7180](https://github.com/ipfs/go-ipfs/pull/7180)) - feat: webui 2.7.5 ([ipfs/go-ipfs#7176](https://github.com/ipfs/go-ipfs/pull/7176)) - integration test for the dual dht ([ipfs/go-ipfs#7151](https://github.com/ipfs/go-ipfs/pull/7151)) - fix: subdomain redirect for dir CIDs ([ipfs/go-ipfs#7165](https://github.com/ipfs/go-ipfs/pull/7165)) - add autonat config options ([ipfs/go-ipfs#7162](https://github.com/ipfs/go-ipfs/pull/7162)) - docs: fix link to version.go ([ipfs/go-ipfs#7157](https://github.com/ipfs/go-ipfs/pull/7157)) - feat: webui v2.7.4 ([ipfs/go-ipfs#7159](https://github.com/ipfs/go-ipfs/pull/7159)) - fix the typo in the serveHTTPApi ([ipfs/go-ipfs#7156](https://github.com/ipfs/go-ipfs/pull/7156)) - test(sharness): improve CAR tests to remove some potential races ([ipfs/go-ipfs#7154](https://github.com/ipfs/go-ipfs/pull/7154)) - feat: introduce the dual WAN/LAN DHT ([ipfs/go-ipfs#7127](https://github.com/ipfs/go-ipfs/pull/7127)) - fix: invalidate cache on failed publish ([ipfs/go-ipfs#7152](https://github.com/ipfs/go-ipfs/pull/7152)) - Temporarily disable gc-race test ([ipfs/go-ipfs#7148](https://github.com/ipfs/go-ipfs/pull/7148)) - Beef up and harden import/export tests ([ipfs/go-ipfs#7140](https://github.com/ipfs/go-ipfs/pull/7140)) - Filter dials to blocked subnets, even when using DNS. ([ipfs/go-ipfs#6996](https://github.com/ipfs/go-ipfs/pull/6996)) - Dag export command, complete ([ipfs/go-ipfs#7036](https://github.com/ipfs/go-ipfs/pull/7036)) - Adding Fission to IPFS early testers page ([ipfs/go-ipfs#7119](https://github.com/ipfs/go-ipfs/pull/7119)) - feat: bump version ([ipfs/go-ipfs#7110](https://github.com/ipfs/go-ipfs/pull/7110)) - feat: initial update to the changelog for 0.5.0 ([ipfs/go-ipfs#6977](https://github.com/ipfs/go-ipfs/pull/6977)) - feat(dht): update to cypress DHT in backwards compatibility mode ([ipfs/go-ipfs#7103](https://github.com/ipfs/go-ipfs/pull/7103)) - update bash completion for `ipfs add` ([ipfs/go-ipfs#7102](https://github.com/ipfs/go-ipfs/pull/7102)) - HTTP API: Only allow POST requests (plus OPTIONS) ([ipfs/go-ipfs#7097](https://github.com/ipfs/go-ipfs/pull/7097)) - Revert last change (the default is now printed twice) ([ipfs/go-ipfs#7098](https://github.com/ipfs/go-ipfs/pull/7098)) - Fix #4996: Improve help text for "ipfs files cp" ([ipfs/go-ipfs#7069](https://github.com/ipfs/go-ipfs/pull/7069)) - changed brew to brew cask ([ipfs/go-ipfs#7072](https://github.com/ipfs/go-ipfs/pull/7072)) - fix: remove internal relay discovery ([ipfs/go-ipfs#7064](https://github.com/ipfs/go-ipfs/pull/7064)) - docs/experimental-features.md: typo ([ipfs/go-ipfs#7062](https://github.com/ipfs/go-ipfs/pull/7062)) - fix: get rid of shutdown errors ([ipfs/go-ipfs#7058](https://github.com/ipfs/go-ipfs/pull/7058)) - feat: tls by default ([ipfs/go-ipfs#7055](https://github.com/ipfs/go-ipfs/pull/7055)) - fix: downgrade to go 1.13 ([ipfs/go-ipfs#7054](https://github.com/ipfs/go-ipfs/pull/7054)) - Keystore: minor maintenance ([ipfs/go-ipfs#7043](https://github.com/ipfs/go-ipfs/pull/7043)) - fix(keystore): avoid racy filesystem access ([ipfs/go-ipfs#6999](https://github.com/ipfs/go-ipfs/pull/6999)) - Forgotten go-fmt ([ipfs/go-ipfs#7030](https://github.com/ipfs/go-ipfs/pull/7030)) - feat: update go-libp2p & go-bitswap ([ipfs/go-ipfs#7028](https://github.com/ipfs/go-ipfs/pull/7028)) - Introducing EncodedFSKeystore with base32 encoding (#5947) ([ipfs/go-ipfs#6955](https://github.com/ipfs/go-ipfs/pull/6955)) - feat: improve key lookup ([ipfs/go-ipfs#7023](https://github.com/ipfs/go-ipfs/pull/7023)) - feat(file-ignore): add ignore opts to add cmd ([ipfs/go-ipfs#7017](https://github.com/ipfs/go-ipfs/pull/7017)) - feat: gateway subdomains + http proxy mode ([ipfs/go-ipfs#6096](https://github.com/ipfs/go-ipfs/pull/6096)) - Chore/sharness fixes 2019 03 16 ([ipfs/go-ipfs#6997](https://github.com/ipfs/go-ipfs/pull/6997)) - Support pipes when named on the cli explicitly ([ipfs/go-ipfs#6998](https://github.com/ipfs/go-ipfs/pull/6998)) - Fix a typo ([ipfs/go-ipfs#7000](https://github.com/ipfs/go-ipfs/pull/7000)) - fix: revert changes to the user agent ([ipfs/go-ipfs#6993](https://github.com/ipfs/go-ipfs/pull/6993)) - feat(peerlog): log protocols/versions ([ipfs/go-ipfs#6972](https://github.com/ipfs/go-ipfs/pull/6972)) - feat: docker build and tag from ci ([ipfs/go-ipfs#6949](https://github.com/ipfs/go-ipfs/pull/6949)) - cmd: ipfs handle GUI environment on Windows ([ipfs/go-ipfs#6646](https://github.com/ipfs/go-ipfs/pull/6646)) - Chore/macos sharness fixes ([ipfs/go-ipfs#6988](https://github.com/ipfs/go-ipfs/pull/6988)) - Update to go-libp2p 0.6.0 ([ipfs/go-ipfs#6914](https://github.com/ipfs/go-ipfs/pull/6914)) - mount: switch over to the CoreAPI ([ipfs/go-ipfs#6602](https://github.com/ipfs/go-ipfs/pull/6602)) - doc(commands): document that `dht put` takes a file ([ipfs/go-ipfs#6960](https://github.com/ipfs/go-ipfs/pull/6960)) - docs: update licence info in README ([ipfs/go-ipfs#6942](https://github.com/ipfs/go-ipfs/pull/6942)) - docs: fix example for files.write ([ipfs/go-ipfs#6943](https://github.com/ipfs/go-ipfs/pull/6943)) - feat(graphsync): mount the graphsync libp2p protocol ([ipfs/go-ipfs#6892](https://github.com/ipfs/go-ipfs/pull/6892)) - feat: update go in docker container ([ipfs/go-ipfs#6933](https://github.com/ipfs/go-ipfs/pull/6933)) - remove expired GPG key from README ([ipfs/go-ipfs#6931](https://github.com/ipfs/go-ipfs/pull/6931)) - test(sharness): test our tests ([ipfs/go-ipfs#6908](https://github.com/ipfs/go-ipfs/pull/6908)) - fix: broken interop tests ([ipfs/go-ipfs#6899](https://github.com/ipfs/go-ipfs/pull/6899)) - feat: pass IPFS_PLUGINS to docker build ([ipfs/go-ipfs#6898](https://github.com/ipfs/go-ipfs/pull/6898)) - doc(add): document hash stability ([ipfs/go-ipfs#6891](https://github.com/ipfs/go-ipfs/pull/6891)) - feat: add peerlog plugin ([ipfs/go-ipfs#6887](https://github.com/ipfs/go-ipfs/pull/6887)) - doc(plugin): document internal plugins ([ipfs/go-ipfs#6888](https://github.com/ipfs/go-ipfs/pull/6888)) - Fix #6878: Improve MFS Cli documentation ([ipfs/go-ipfs#6882](https://github.com/ipfs/go-ipfs/pull/6882)) - Update the license distributed with dist builds to the dual one ([ipfs/go-ipfs#6879](https://github.com/ipfs/go-ipfs/pull/6879)) - doc: add license URLs so go's doc service can detect our license ([ipfs/go-ipfs#6874](https://github.com/ipfs/go-ipfs/pull/6874)) - doc: rename COPYRIGHT to LICENSE ([ipfs/go-ipfs#6873](https://github.com/ipfs/go-ipfs/pull/6873)) - fix: fix id addr format ([ipfs/go-ipfs#6872](https://github.com/ipfs/go-ipfs/pull/6872)) - Help text update for 'ipfs key gen' ([ipfs/go-ipfs#6867](https://github.com/ipfs/go-ipfs/pull/6867)) - fix: make rsa the default key type ([ipfs/go-ipfs#6864](https://github.com/ipfs/go-ipfs/pull/6864)) - doc(config): cleanup ([ipfs/go-ipfs#6855](https://github.com/ipfs/go-ipfs/pull/6855)) - Allow building non-amd64 Docker images ([ipfs/go-ipfs#6854](https://github.com/ipfs/go-ipfs/pull/6854)) - doc(release): add Charity Engine to the early testers programme ([ipfs/go-ipfs#6850](https://github.com/ipfs/go-ipfs/pull/6850)) - fix: fix a potential out of bounds issue in fuse ([ipfs/go-ipfs#6847](https://github.com/ipfs/go-ipfs/pull/6847)) - fix(build): instruct users to use GOTAGS, not GOFLAGS ([ipfs/go-ipfs#6843](https://github.com/ipfs/go-ipfs/pull/6843)) - doc(release): document how RCs should be communicated ([ipfs/go-ipfs#6845](https://github.com/ipfs/go-ipfs/pull/6845)) - doc(release): move WebUI from manual tests to automated tests section ([ipfs/go-ipfs#6838](https://github.com/ipfs/go-ipfs/pull/6838)) - test(sharness): fix typo ([ipfs/go-ipfs#6835](https://github.com/ipfs/go-ipfs/pull/6835)) - test: E2E tests against ipfs-webui HEAD ([ipfs/go-ipfs#6825](https://github.com/ipfs/go-ipfs/pull/6825)) - mkreleaslog: improve edge-cases ([ipfs/go-ipfs#6833](https://github.com/ipfs/go-ipfs/pull/6833)) - fix: don't fail to collect profiles if no ipfs bin ([ipfs/go-ipfs#6829](https://github.com/ipfs/go-ipfs/pull/6829)) - update dockerfile and use openssl ([ipfs/go-ipfs#6828](https://github.com/ipfs/go-ipfs/pull/6828)) - docs: define Gateway.PathPrefixes ([ipfs/go-ipfs#6826](https://github.com/ipfs/go-ipfs/pull/6826)) - fix(badgerds): turn off sync writes by default ([ipfs/go-ipfs#6819](https://github.com/ipfs/go-ipfs/pull/6819)) - gateway cleanups ([ipfs/go-ipfs#6820](https://github.com/ipfs/go-ipfs/pull/6820)) - make it possible to change the codec with the `ipfs cid` subcommand ([ipfs/go-ipfs#6817](https://github.com/ipfs/go-ipfs/pull/6817)) - improve gateway symlink handling ([ipfs/go-ipfs#6680](https://github.com/ipfs/go-ipfs/pull/6680)) - Inclusion of the presence of the go-ipfs package in Solus ([ipfs/go-ipfs#6809](https://github.com/ipfs/go-ipfs/pull/6809)) - Fix Typos ([ipfs/go-ipfs#6807](https://github.com/ipfs/go-ipfs/pull/6807)) - Sharness macos no brainer fixes ([ipfs/go-ipfs#6805](https://github.com/ipfs/go-ipfs/pull/6805)) - Support Asynchronous Datastores ([ipfs/go-ipfs#6785](https://github.com/ipfs/go-ipfs/pull/6785)) - update documentation for /ipfs -> /p2p multiaddr switch ([ipfs/go-ipfs#6538](https://github.com/ipfs/go-ipfs/pull/6538)) - IPNS over PubSub as an Independent Transport ([ipfs/go-ipfs#6758](https://github.com/ipfs/go-ipfs/pull/6758)) - docs: add information on how to enable experiments ([ipfs/go-ipfs#6792](https://github.com/ipfs/go-ipfs/pull/6792)) - Change Reporter to BandwidthCounter in IpfsNode ([ipfs/go-ipfs#6793](https://github.com/ipfs/go-ipfs/pull/6793)) - update go-datastore ([ipfs/go-ipfs#6791](https://github.com/ipfs/go-ipfs/pull/6791)) - go fmt: go-ipfs-as-a-library ([ipfs/go-ipfs#6784](https://github.com/ipfs/go-ipfs/pull/6784)) - feat: web ui 2.7.2 ([ipfs/go-ipfs#6778](https://github.com/ipfs/go-ipfs/pull/6778)) - extract the pinner to go-ipfs-pinner and dagutils into go-merkledag ([ipfs/go-ipfs#6771](https://github.com/ipfs/go-ipfs/pull/6771)) - fix #2203: omit the charset attribute when Content-Type is text/html ([ipfs/go-ipfs#6743](https://github.com/ipfs/go-ipfs/pull/6743)) - Pin ls traverses all indirect pins ([ipfs/go-ipfs#6705](https://github.com/ipfs/go-ipfs/pull/6705)) - fix: ignore nonexistent when force rm ([ipfs/go-ipfs#6773](https://github.com/ipfs/go-ipfs/pull/6773)) - introduce IpfsNode Plugin ([ipfs/go-ipfs#6719](https://github.com/ipfs/go-ipfs/pull/6719)) - improve documentation and fix dht put bug ([ipfs/go-ipfs#6750](https://github.com/ipfs/go-ipfs/pull/6750)) - Adding alias for `ipfs repo stat`. ([ipfs/go-ipfs#6769](https://github.com/ipfs/go-ipfs/pull/6769)) - doc(gateway): document dnslink ([ipfs/go-ipfs#6767](https://github.com/ipfs/go-ipfs/pull/6767)) - pin: add context and error return to most of the Pinner functions ([ipfs/go-ipfs#6715](https://github.com/ipfs/go-ipfs/pull/6715)) - feat: web ui 2.7.1 ([ipfs/go-ipfs#6762](https://github.com/ipfs/go-ipfs/pull/6762)) - doc(README): document requirements for cross-compiling with OpenSSL support ([ipfs/go-ipfs#6738](https://github.com/ipfs/go-ipfs/pull/6738)) - feat: web ui 2.6.0 ([ipfs/go-ipfs#6740](https://github.com/ipfs/go-ipfs/pull/6740)) - Add high-level go-ipfs architecture diagram ([ipfs/go-ipfs#6727](https://github.com/ipfs/go-ipfs/pull/6727)) - docs: remove extra ) on the example README ([ipfs/go-ipfs#6733](https://github.com/ipfs/go-ipfs/pull/6733)) - update maintainer label ([ipfs/go-ipfs#6735](https://github.com/ipfs/go-ipfs/pull/6735)) - ipfs namespace is now being provided to Prometheus ([ipfs/go-ipfs#6643](https://github.com/ipfs/go-ipfs/pull/6643)) - feat: web ui 2.5.8 ([ipfs/go-ipfs#6718](https://github.com/ipfs/go-ipfs/pull/6718)) - docs: add connmgr to config.md toc ([ipfs/go-ipfs#6712](https://github.com/ipfs/go-ipfs/pull/6712)) - feat: web ui 2.5.7 ([ipfs/go-ipfs#6707](https://github.com/ipfs/go-ipfs/pull/6707)) - README: improve build documentation ([ipfs/go-ipfs#6706](https://github.com/ipfs/go-ipfs/pull/6706)) - Introduce buzhash chunker ([ipfs/go-ipfs#6701](https://github.com/ipfs/go-ipfs/pull/6701)) - Pinning interop: Pin ls returns appropriate zero value ([ipfs/go-ipfs#6685](https://github.com/ipfs/go-ipfs/pull/6685)) - fix(resolve): correctly handle .eth domains ([ipfs/go-ipfs#6700](https://github.com/ipfs/go-ipfs/pull/6700)) - Update README.md ([ipfs/go-ipfs#6697](https://github.com/ipfs/go-ipfs/pull/6697)) - daemon: support unix domain sockets for the API/gateway ([ipfs/go-ipfs#6678](https://github.com/ipfs/go-ipfs/pull/6678)) - docs: guide users to the right locations for questions ([ipfs/go-ipfs#6691](https://github.com/ipfs/go-ipfs/pull/6691)) - docs: readme improvements ([ipfs/go-ipfs#6693](https://github.com/ipfs/go-ipfs/pull/6693)) - docs: link remaining docs available, guide people to the right locations ([ipfs/go-ipfs#6694](https://github.com/ipfs/go-ipfs/pull/6694)) - docs: fix broken url ([ipfs/go-ipfs#6692](https://github.com/ipfs/go-ipfs/pull/6692)) - add systemd support ([ipfs/go-ipfs#6675](https://github.com/ipfs/go-ipfs/pull/6675)) - feat: add ipfs version info to prometheus metrics ([ipfs/go-ipfs#6688](https://github.com/ipfs/go-ipfs/pull/6688)) - Fix typo ([ipfs/go-ipfs#6686](https://github.com/ipfs/go-ipfs/pull/6686)) - github: migrate actions ([ipfs/go-ipfs#6681](https://github.com/ipfs/go-ipfs/pull/6681)) - Add bridged chats ([ipfs/go-ipfs#6653](https://github.com/ipfs/go-ipfs/pull/6653)) - doc(config): improve DisableNatPortMap documentation ([ipfs/go-ipfs#6655](https://github.com/ipfs/go-ipfs/pull/6655)) - plugins: support Close() for Tracer plugins as well ([ipfs/go-ipfs#6672](https://github.com/ipfs/go-ipfs/pull/6672)) - fix: make collect-profiles.sh work on mac ([ipfs/go-ipfs#6673](https://github.com/ipfs/go-ipfs/pull/6673)) - namesys(test): test TTL on publish ([ipfs/go-ipfs#6671](https://github.com/ipfs/go-ipfs/pull/6671)) - discovery: improve mdns warnings ([ipfs/go-ipfs#6665](https://github.com/ipfs/go-ipfs/pull/6665)) - feat: web ui 2.5.4 ([ipfs/go-ipfs#6664](https://github.com/ipfs/go-ipfs/pull/6664)) - cmds(help): fix swarm filter add/rm help text ([ipfs/go-ipfs#6654](https://github.com/ipfs/go-ipfs/pull/6654)) - feat: webui 2.5.3 ([ipfs/go-ipfs#6638](https://github.com/ipfs/go-ipfs/pull/6638)) - feat: web ui 2.5.1 ([ipfs/go-ipfs#6630](https://github.com/ipfs/go-ipfs/pull/6630)) - docs: add multiple gateway and api addrs ([ipfs/go-ipfs#6631](https://github.com/ipfs/go-ipfs/pull/6631)) - doc: add post-release checklist ([ipfs/go-ipfs#6625](https://github.com/ipfs/go-ipfs/pull/6625)) - docs: add ship date and next release issue opening time ([ipfs/go-ipfs#6620](https://github.com/ipfs/go-ipfs/pull/6620)) - docker: libdl dependency ([ipfs/go-ipfs#6624](https://github.com/ipfs/go-ipfs/pull/6624)) - docs: improvements to the release doc ([ipfs/go-ipfs#6616](https://github.com/ipfs/go-ipfs/pull/6616)) - plugins: add support for plugin configs ([ipfs/go-ipfs#6613](https://github.com/ipfs/go-ipfs/pull/6613)) - Update README.md ([ipfs/go-ipfs#6615](https://github.com/ipfs/go-ipfs/pull/6615)) - doc: remove gmake instructions ([ipfs/go-ipfs#6614](https://github.com/ipfs/go-ipfs/pull/6614)) - feat: add ability to use existing config during init ([ipfs/go-ipfs#6489](https://github.com/ipfs/go-ipfs/pull/6489)) - doc: expand and cleanup badger documentation ([ipfs/go-ipfs#6611](https://github.com/ipfs/go-ipfs/pull/6611)) - feat: improve plugin preload logic ([ipfs/go-ipfs#6576](https://github.com/ipfs/go-ipfs/pull/6576)) - version: don't print 'VERSION-' if no commit is specified ([ipfs/go-ipfs#6609](https://github.com/ipfs/go-ipfs/pull/6609)) - Update go-libp2p, fix tests with weak RSA keys ([ipfs/go-ipfs#6555](https://github.com/ipfs/go-ipfs/pull/6555)) - cmds/refs: fix ipfs refs for sharded directories ([ipfs/go-ipfs#6601](https://github.com/ipfs/go-ipfs/pull/6601)) - fix: spammy mock when testing ([ipfs/go-ipfs#6583](https://github.com/ipfs/go-ipfs/pull/6583)) - docker: update the docker image ([ipfs/go-ipfs#6582](https://github.com/ipfs/go-ipfs/pull/6582)) - add release process graphic ([ipfs/go-ipfs#6568](https://github.com/ipfs/go-ipfs/pull/6568)) - feat: web ui 2.5.0 ([ipfs/go-ipfs#6566](https://github.com/ipfs/go-ipfs/pull/6566)) - Add swarm key variables to container daemon ([ipfs/go-ipfs#6554](https://github.com/ipfs/go-ipfs/pull/6554)) - doc: update the release template ([ipfs/go-ipfs#6561](https://github.com/ipfs/go-ipfs/pull/6561)) - merge changelog and bump version ([ipfs/go-ipfs#6559](https://github.com/ipfs/go-ipfs/pull/6559)) - require GNU make ([ipfs/go-ipfs#6551](https://github.com/ipfs/go-ipfs/pull/6551)) - tweak the release process ([ipfs/go-ipfs#6553](https://github.com/ipfs/go-ipfs/pull/6553)) - Allow resolution of .eth names via .eth.link ([ipfs/go-ipfs#6448](https://github.com/ipfs/go-ipfs/pull/6448)) - README: update minimum system requirements and recommend OpenSSL ([ipfs/go-ipfs#6543](https://github.com/ipfs/go-ipfs/pull/6543)) - fix and improve the writable gateway ([ipfs/go-ipfs#6539](https://github.com/ipfs/go-ipfs/pull/6539)) - feat: add install instructions for external commands ([ipfs/go-ipfs#6541](https://github.com/ipfs/go-ipfs/pull/6541)) - fix: slightly faster gc ([ipfs/go-ipfs#6505](https://github.com/ipfs/go-ipfs/pull/6505)) - fix {net,open}bsd build by disabling fuse on openbsd ([ipfs/go-ipfs#6535](https://github.com/ipfs/go-ipfs/pull/6535)) - mk: handle stripping paths when GOPATH contains whitespace ([ipfs/go-ipfs#6536](https://github.com/ipfs/go-ipfs/pull/6536)) - make gossipsub the default routing protocol for pubsub ([ipfs/go-ipfs#6512](https://github.com/ipfs/go-ipfs/pull/6512)) - doc: align the early testers program description with its goal ([ipfs/go-ipfs#6529](https://github.com/ipfs/go-ipfs/pull/6529)) - feat: add --long as alias for -l in files.ls ([ipfs/go-ipfs#6528](https://github.com/ipfs/go-ipfs/pull/6528)) - switch to new merkledag walk functions ([ipfs/go-ipfs#6499](https://github.com/ipfs/go-ipfs/pull/6499)) - readme: fix CI badge ([ipfs/go-ipfs#6521](https://github.com/ipfs/go-ipfs/pull/6521)) - Adds Siderus in early testers ([ipfs/go-ipfs#6517](https://github.com/ipfs/go-ipfs/pull/6517)) - Extract Filestore ([ipfs/go-ipfs#6511](https://github.com/ipfs/go-ipfs/pull/6511)) - readme: fix scoop bucket command error ([ipfs/go-ipfs#6510](https://github.com/ipfs/go-ipfs/pull/6510)) - sharness: test pin ls stream ([ipfs/go-ipfs#6504](https://github.com/ipfs/go-ipfs/pull/6504)) - Improve pin/update description ([ipfs/go-ipfs#6501](https://github.com/ipfs/go-ipfs/pull/6501)) - pin cmd: stream recursive pins ([ipfs/go-ipfs#6493](https://github.com/ipfs/go-ipfs/pull/6493)) - Document the AddrFilters option ([ipfs/go-ipfs#6459](https://github.com/ipfs/go-ipfs/pull/6459)) - feat: make it easier to load custom plugins ([ipfs/go-ipfs#6474](https://github.com/ipfs/go-ipfs/pull/6474)) - document the debug script ([ipfs/go-ipfs#6486](https://github.com/ipfs/go-ipfs/pull/6486)) - Extract provider module to `go-ipfs-provider` ([ipfs/go-ipfs#6421](https://github.com/ipfs/go-ipfs/pull/6421)) - ignore stale API files and deprecate ipfs repo fsck ([ipfs/go-ipfs#6478](https://github.com/ipfs/go-ipfs/pull/6478)) - Fix node construction queue error ([ipfs/go-ipfs#6480](https://github.com/ipfs/go-ipfs/pull/6480)) - Update the required go version in the README ([ipfs/go-ipfs#6462](https://github.com/ipfs/go-ipfs/pull/6462)) - gitmodules: use https so we don't need an ssh key ([ipfs/go-ipfs#6450](https://github.com/ipfs/go-ipfs/pull/6450)) - doc: add another Windows package to README ([ipfs/go-ipfs#6440](https://github.com/ipfs/go-ipfs/pull/6440)) - Close started plugins when one of them fails to start. ([ipfs/go-ipfs#6438](https://github.com/ipfs/go-ipfs/pull/6438)) - Load plugins on darwin/macOS ([ipfs/go-ipfs#6439](https://github.com/ipfs/go-ipfs/pull/6439)) - assets: move away from gx ([ipfs/go-ipfs#6414](https://github.com/ipfs/go-ipfs/pull/6414)) - Fix a typo ([ipfs/go-ipfs#6432](https://github.com/ipfs/go-ipfs/pull/6432)) - docs: fix install guide link ([ipfs/go-ipfs#6423](https://github.com/ipfs/go-ipfs/pull/6423)) - Deps: update go-libp2p-http to its new libp2p location ([ipfs/go-ipfs#6422](https://github.com/ipfs/go-ipfs/pull/6422)) - install.sh: Fix wrong destination path for ipfs binary ([ipfs/go-ipfs#6424](https://github.com/ipfs/go-ipfs/pull/6424)) - build: strip GOPATH from build paths ([ipfs/go-ipfs#6412](https://github.com/ipfs/go-ipfs/pull/6412)) - libp2p: moves discovery after host listen ([ipfs/go-ipfs#6415](https://github.com/ipfs/go-ipfs/pull/6415)) - remove mentions of gx from windows build docs ([ipfs/go-ipfs#6413](https://github.com/ipfs/go-ipfs/pull/6413)) - build: use protoc-gen-* from gomod ([ipfs/go-ipfs#6411](https://github.com/ipfs/go-ipfs/pull/6411)) - add unixfs get metric ([ipfs/go-ipfs#6406](https://github.com/ipfs/go-ipfs/pull/6406)) - Run JS interop in CircleCI ([ipfs/go-ipfs#6409](https://github.com/ipfs/go-ipfs/pull/6409)) - Usage of context helper in Blockstore provider ([ipfs/go-ipfs#6399](https://github.com/ipfs/go-ipfs/pull/6399)) - docs: default value for HashOnRead is false ([ipfs/go-ipfs#6401](https://github.com/ipfs/go-ipfs/pull/6401)) - block cmd: allow adding multiple blocks at once ([ipfs/go-ipfs#6331](https://github.com/ipfs/go-ipfs/pull/6331)) - Remove Repo from routing fx provider parameter ([ipfs/go-ipfs#6395](https://github.com/ipfs/go-ipfs/pull/6395)) - migrate to go-libp2p-core. ([ipfs/go-ipfs#6384](https://github.com/ipfs/go-ipfs/pull/6384)) - feat: update Web UI to v2.4.6 ([ipfs/go-ipfs#6392](https://github.com/ipfs/go-ipfs/pull/6392)) - Introduce first strategic provider: do nothing ([ipfs/go-ipfs#6292](https://github.com/ipfs/go-ipfs/pull/6292)) - github.com/ipfs/go-bitswap (v0.0.8-e37498cf10d6 -> v0.2.13): - refactor: remove WantManager ([ipfs/go-bitswap#374](https://github.com/ipfs/go-bitswap/pull/374)) - Send CANCELs when session context is canceled ([ipfs/go-bitswap#375](https://github.com/ipfs/go-bitswap/pull/375)) - refactor: remove unused code ([ipfs/go-bitswap#373](https://github.com/ipfs/go-bitswap/pull/373)) - Change timing for DONT_HAVE timeouts to be more conservative ([ipfs/go-bitswap#371](https://github.com/ipfs/go-bitswap/pull/371)) - fix: avoid calling ctx.SetDeadline() every time we send a message ([ipfs/go-bitswap#369](https://github.com/ipfs/go-bitswap/pull/369)) - feat: optimize entry sorting in MessageQueue ([ipfs/go-bitswap#356](https://github.com/ipfs/go-bitswap/pull/356)) - Move connection management into networking layer ([ipfs/go-bitswap#351](https://github.com/ipfs/go-bitswap/pull/351)) - refactor: simplify messageQueue onSent ([ipfs/go-bitswap#349](https://github.com/ipfs/go-bitswap/pull/349)) - feat: prioritize more important wants ([ipfs/go-bitswap#346](https://github.com/ipfs/go-bitswap/pull/346)) - fix: in message queue only send cancel if want was sent ([ipfs/go-bitswap#345](https://github.com/ipfs/go-bitswap/pull/345)) - fix: ensure wantlist gauge gets decremented on disconnect ([ipfs/go-bitswap#332](https://github.com/ipfs/go-bitswap/pull/332)) - avoid copying messages and improve logging ([ipfs/go-bitswap#326](https://github.com/ipfs/go-bitswap/pull/326)) - fix: log unexpected condition in peerWantManager.prepareSendWants() ([ipfs/go-bitswap#325](https://github.com/ipfs/go-bitswap/pull/325)) - wait for sessionWantSender to shutdown before completing session shutdown ([ipfs/go-bitswap#317](https://github.com/ipfs/go-bitswap/pull/317)) - Perf/message queue ([ipfs/go-bitswap#307](https://github.com/ipfs/go-bitswap/pull/307)) - feat: add a custom CID type ([ipfs/go-bitswap#308](https://github.com/ipfs/go-bitswap/pull/308)) - feat: expose the full wantlist through GetWantlist ([ipfs/go-bitswap#300](https://github.com/ipfs/go-bitswap/pull/300)) - Clean up logs ([ipfs/go-bitswap#299](https://github.com/ipfs/go-bitswap/pull/299)) - Fix order of session broadcast wants ([ipfs/go-bitswap#291](https://github.com/ipfs/go-bitswap/pull/291)) - fix flaky TestRateLimitingRequests ([ipfs/go-bitswap#296](https://github.com/ipfs/go-bitswap/pull/296)) - fix flaky TestDontHaveTimeoutMgrTimeout ([ipfs/go-bitswap#293](https://github.com/ipfs/go-bitswap/pull/293)) - fix: re-export testinstance/testnet ([ipfs/go-bitswap#289](https://github.com/ipfs/go-bitswap/pull/289)) - Simulate DONT_HAVE when peer doesn't respond to want-block (new peers) ([ipfs/go-bitswap#284](https://github.com/ipfs/go-bitswap/pull/284)) - Be less aggressive when pruning peers from session ([ipfs/go-bitswap#276](https://github.com/ipfs/go-bitswap/pull/276)) - fix: races in tests ([ipfs/go-bitswap#279](https://github.com/ipfs/go-bitswap/pull/279)) - Refactor: simplify session peer management ([ipfs/go-bitswap#275](https://github.com/ipfs/go-bitswap/pull/275)) - Prune peers that send too many consecutive DONT_HAVEs ([ipfs/go-bitswap#261](https://github.com/ipfs/go-bitswap/pull/261)) - feat: debounce wants manually ([ipfs/go-bitswap#255](https://github.com/ipfs/go-bitswap/pull/255)) - Fix bug with signaling peer availability to sessions ([ipfs/go-bitswap#247](https://github.com/ipfs/go-bitswap/pull/247)) - feat: move internals to an internal package ([ipfs/go-bitswap#242](https://github.com/ipfs/go-bitswap/pull/242)) - PoC of Bitswap protocol extensions implementation ([ipfs/go-bitswap#189](https://github.com/ipfs/go-bitswap/pull/189)) - fix: abort when the context is canceled while getting blocks ([ipfs/go-bitswap#240](https://github.com/ipfs/go-bitswap/pull/240)) - Add bridged chats ([ipfs/go-bitswap#198](https://github.com/ipfs/go-bitswap/pull/198)) - reduce session contention ([ipfs/go-bitswap#188](https://github.com/ipfs/go-bitswap/pull/188)) - Fix: don't ignore received blocks for pending wants ([ipfs/go-bitswap#174](https://github.com/ipfs/go-bitswap/pull/174)) - Test: fix flakey session peer manager tests ([ipfs/go-bitswap#185](https://github.com/ipfs/go-bitswap/pull/185)) - Refactor: use global pubsub notifier ([ipfs/go-bitswap#177](https://github.com/ipfs/go-bitswap/pull/177)) - network: Allow specifying protocol prefix ([ipfs/go-bitswap#171](https://github.com/ipfs/go-bitswap/pull/171)) - fix: memory leak in latency tracker on timeout after cancel ([ipfs/go-bitswap#164](https://github.com/ipfs/go-bitswap/pull/164)) - Fix typo ([ipfs/go-bitswap#158](https://github.com/ipfs/go-bitswap/pull/158)) - Feat: Track Session Peer Latency More Accurately ([ipfs/go-bitswap#149](https://github.com/ipfs/go-bitswap/pull/149)) - ci(circleci): add benchmark comparisons ([ipfs/go-bitswap#147](https://github.com/ipfs/go-bitswap/pull/147)) - aggressively free memory ([ipfs/go-bitswap#143](https://github.com/ipfs/go-bitswap/pull/143)) - Enhanced logging for bitswap ([ipfs/go-bitswap#137](https://github.com/ipfs/go-bitswap/pull/137)) - fix: rand.Intn(0) panics ([ipfs/go-bitswap#144](https://github.com/ipfs/go-bitswap/pull/144)) - fix some naming nits and broadcast on search ([ipfs/go-bitswap#139](https://github.com/ipfs/go-bitswap/pull/139)) - feat(sessions): add rebroadcasting, search backoff ([ipfs/go-bitswap#133](https://github.com/ipfs/go-bitswap/pull/133)) - testutil: fix block generator ([ipfs/go-bitswap#135](https://github.com/ipfs/go-bitswap/pull/135)) - migrate to go-libp2p-core. ([ipfs/go-bitswap#132](https://github.com/ipfs/go-bitswap/pull/132)) - github.com/ipfs/go-blockservice (v0.0.3 -> v0.1.3): - fix ci badge and lints ([ipfs/go-blockservice#52](https://github.com/ipfs/go-blockservice/pull/52)) - demote warning to debug log ([ipfs/go-blockservice#30](https://github.com/ipfs/go-blockservice/pull/30)) - nil exchange is okay ([ipfs/go-blockservice#29](https://github.com/ipfs/go-blockservice/pull/29)) - set the session context ([ipfs/go-blockservice#28](https://github.com/ipfs/go-blockservice/pull/28)) - make blockservice AddBlocks return more quickly ([ipfs/go-blockservice#10](https://github.com/ipfs/go-blockservice/pull/10)) - feat(session): instantiated sessions lazily ([ipfs/go-blockservice#27](https://github.com/ipfs/go-blockservice/pull/27)) - github.com/ipfs/go-cid (v0.0.4 -> v0.0.5): - fix: enforce minimal encoding ([ipfs/go-cid#99](https://github.com/ipfs/go-cid/pull/99)) - github.com/ipfs/go-datastore (v0.0.5 -> v0.4.4): - Fix test log message about number of values put ([ipfs/go-datastore#150](https://github.com/ipfs/go-datastore/pull/150)) - test suite: Add ElemCount to control how many elements are added. ([ipfs/go-datastore#151](https://github.com/ipfs/go-datastore/pull/151)) - fix: avoid filtering by prefix unless necessary ([ipfs/go-datastore#147](https://github.com/ipfs/go-datastore/pull/147)) - feat: add upper-case keys at a known prefix ([ipfs/go-datastore#148](https://github.com/ipfs/go-datastore/pull/148)) - test(suite): add a bunch of prefix tests for the new behavior ([ipfs/go-datastore#145](https://github.com/ipfs/go-datastore/pull/145)) - Only count a key as an ancestor if there is a separator ([ipfs/go-datastore#141](https://github.com/ipfs/go-datastore/pull/141)) - fix go-check path to use "gopkg.in/check.v1" ([ipfs/go-datastore#144](https://github.com/ipfs/go-datastore/pull/144)) - LogDatastore fulfills the Datastore interface again ([ipfs/go-datastore#142](https://github.com/ipfs/go-datastore/pull/142)) - Support Asynchronous Writing Datastores ([ipfs/go-datastore#140](https://github.com/ipfs/go-datastore/pull/140)) - add a Size field to Query's Result ([ipfs/go-datastore#134](https://github.com/ipfs/go-datastore/pull/134)) - Add clarifying comments on Query#String() ([ipfs/go-datastore#138](https://github.com/ipfs/go-datastore/pull/138)) - Add a large test suite ([ipfs/go-datastore#136](https://github.com/ipfs/go-datastore/pull/136)) - doc: add a lead maintainer ([ipfs/go-datastore#135](https://github.com/ipfs/go-datastore/pull/135)) - feat: make not-found errors discoverable ([ipfs/go-datastore#133](https://github.com/ipfs/go-datastore/pull/133)) - feat: make delete idempotent ([ipfs/go-datastore#132](https://github.com/ipfs/go-datastore/pull/132)) - Misc Typo Fixes ([ipfs/go-datastore#131](https://github.com/ipfs/go-datastore/pull/131)) - github.com/ipfs/go-ds-badger (v0.0.5 -> v0.2.4): - fix: verify that the datastore is still open when querying ([ipfs/go-ds-badger#87](https://github.com/ipfs/go-ds-badger/pull/87)) - feat: switch to file io and shrink tables ([ipfs/go-ds-badger#83](https://github.com/ipfs/go-ds-badger/pull/83)) - fix: update go-datastore ([ipfs/go-ds-badger#80](https://github.com/ipfs/go-ds-badger/pull/80)) - update datastore Interface ([ipfs/go-ds-badger#77](https://github.com/ipfs/go-ds-badger/pull/77)) - query: always return the size ([ipfs/go-ds-badger#78](https://github.com/ipfs/go-ds-badger/pull/78)) - feat(gc): make it possible to disable GC ([ipfs/go-ds-badger#74](https://github.com/ipfs/go-ds-badger/pull/74)) - feat(gc): improve periodic GC logic ([ipfs/go-ds-badger#73](https://github.com/ipfs/go-ds-badger/pull/73)) - periodic GC for badger datastore ([ipfs/go-ds-badger#72](https://github.com/ipfs/go-ds-badger/pull/72)) - Fix combining query filters, offsets, and limits ([ipfs/go-ds-badger#71](https://github.com/ipfs/go-ds-badger/pull/71)) - doc: add lead maintainer ([ipfs/go-ds-badger#67](https://github.com/ipfs/go-ds-badger/pull/67)) - github.com/ipfs/go-ds-flatfs (v0.0.2 -> v0.4.4): - move retries lower and retry rename ops ([ipfs/go-ds-flatfs#82](https://github.com/ipfs/go-ds-flatfs/pull/82)) - cleanup putMany implementation ([ipfs/go-ds-flatfs#80](https://github.com/ipfs/go-ds-flatfs/pull/80)) - feat: read harder ([ipfs/go-ds-flatfs#78](https://github.com/ipfs/go-ds-flatfs/pull/78)) - fix: remove temporary files when multiple write operations conflict ([ipfs/go-ds-flatfs#76](https://github.com/ipfs/go-ds-flatfs/pull/76)) - Windows CI + Fixes ([ipfs/go-ds-flatfs#73](https://github.com/ipfs/go-ds-flatfs/pull/73)) - fix: close query when finished moving ([ipfs/go-ds-flatfs#74](https://github.com/ipfs/go-ds-flatfs/pull/74)) - fix: ensure that we close the diskusage file, even if we fail to rename it ([ipfs/go-ds-flatfs#72](https://github.com/ipfs/go-ds-flatfs/pull/72)) - feat: put all temporary files in the same directory and clean them up ([ipfs/go-ds-flatfs#69](https://github.com/ipfs/go-ds-flatfs/pull/69)) - fix: only log when we find a file we don't expect ([ipfs/go-ds-flatfs#68](https://github.com/ipfs/go-ds-flatfs/pull/68)) - Make flatfs robust ([ipfs/go-ds-flatfs#64](https://github.com/ipfs/go-ds-flatfs/pull/64)) - Update Datastore Interface ([ipfs/go-ds-flatfs#60](https://github.com/ipfs/go-ds-flatfs/pull/60)) - query: deny ReturnsSizes and ReturnExpirations instead of returning wrong result ([ipfs/go-ds-flatfs#59](https://github.com/ipfs/go-ds-flatfs/pull/59)) - doc: add a lead maintainer ([ipfs/go-ds-flatfs#55](https://github.com/ipfs/go-ds-flatfs/pull/55)) - make delete idempotent ([ipfs/go-ds-flatfs#54](https://github.com/ipfs/go-ds-flatfs/pull/54)) - github.com/ipfs/go-ds-leveldb (v0.0.2 -> v0.4.2): - prevent closing concurrently with other operations. ([ipfs/go-ds-leveldb#42](https://github.com/ipfs/go-ds-leveldb/pull/42)) - feat: update go-datastore ([ipfs/go-ds-leveldb#40](https://github.com/ipfs/go-ds-leveldb/pull/40)) - update datastore Interface ([ipfs/go-ds-leveldb#36](https://github.com/ipfs/go-ds-leveldb/pull/36)) - query: always return the size ([ipfs/go-ds-leveldb#35](https://github.com/ipfs/go-ds-leveldb/pull/35)) - doc: add a lead maintainer ([ipfs/go-ds-leveldb#31](https://github.com/ipfs/go-ds-leveldb/pull/31)) - make delete idempotent ([ipfs/go-ds-leveldb#30](https://github.com/ipfs/go-ds-leveldb/pull/30)) - github.com/ipfs/go-ds-measure (v0.0.1 -> v0.1.0): - update datastore Interface ([ipfs/go-ds-measure#23](https://github.com/ipfs/go-ds-measure/pull/23)) - Add Datastore Tests ([ipfs/go-ds-measure#24](https://github.com/ipfs/go-ds-measure/pull/24)) - fix GetSize calls reported as Has ([ipfs/go-ds-measure#20](https://github.com/ipfs/go-ds-measure/pull/20)) - github.com/ipfs/go-fs-lock (v0.0.1 -> v0.0.4): - fix: revert small breaking change ([ipfs/go-fs-lock#10](https://github.com/ipfs/go-fs-lock/pull/10)) - Enh/improve error handling ([ipfs/go-fs-lock#9](https://github.com/ipfs/go-fs-lock/pull/9)) - Use path/filepath instead of path ([ipfs/go-fs-lock#8](https://github.com/ipfs/go-fs-lock/pull/8)) - github.com/ipfs/go-ipfs-blockstore (v0.0.1 -> v0.1.4): - return the correct size when only "has" is cached ([ipfs/go-ipfs-blockstore#36](https://github.com/ipfs/go-ipfs-blockstore/pull/36)) - cache: switch to 2q ([ipfs/go-ipfs-blockstore#20](https://github.com/ipfs/go-ipfs-blockstore/pull/20)) - github.com/ipfs/go-ipfs-chunker (v0.0.1 -> v0.0.5): - fix: don't return an empty block at the end ([ipfs/go-ipfs-chunker#22](https://github.com/ipfs/go-ipfs-chunker/pull/22)) - Rigorous sizing checks ([ipfs/go-ipfs-chunker#21](https://github.com/ipfs/go-ipfs-chunker/pull/21)) - Improve performance of buzhash ([ipfs/go-ipfs-chunker#17](https://github.com/ipfs/go-ipfs-chunker/pull/17)) - Implement buzhash ([ipfs/go-ipfs-chunker#16](https://github.com/ipfs/go-ipfs-chunker/pull/16)) - Add benchmarks ([ipfs/go-ipfs-chunker#15](https://github.com/ipfs/go-ipfs-chunker/pull/15)) - github.com/ipfs/go-ipfs-cmds (v0.0.8 -> v0.2.2): - Fix: disallow POST without Origin nor Referer from specific user agents ([ipfs/go-ipfs-cmds#193](https://github.com/ipfs/go-ipfs-cmds/pull/193)) - doc: document command fields ([ipfs/go-ipfs-cmds#192](https://github.com/ipfs/go-ipfs-cmds/pull/192)) - change HandledMethods to AllowGet and cleanup method handling ([ipfs/go-ipfs-cmds#191](https://github.com/ipfs/go-ipfs-cmds/pull/191)) - remove deprecated log.Warning(f) ([ipfs/go-ipfs-cmds#180](https://github.com/ipfs/go-ipfs-cmds/pull/180)) - http: configurable allowed request methods for the API. ([ipfs/go-ipfs-cmds#190](https://github.com/ipfs/go-ipfs-cmds/pull/190)) - #183 refactored the request options conversion code per the ticket requirements ([ipfs/go-ipfs-cmds#187](https://github.com/ipfs/go-ipfs-cmds/pull/187)) - fix typo ([ipfs/go-ipfs-cmds#188](https://github.com/ipfs/go-ipfs-cmds/pull/188)) - ([ipfs/go-ipfs-cmds#183](https://github.com/ipfs/go-ipfs-cmds/pull/183)) - fix: normalize options when parsing them ([ipfs/go-ipfs-cmds#186](https://github.com/ipfs/go-ipfs-cmds/pull/186)) - feat:add strings option; re-implement file ignore ([ipfs/go-ipfs-cmds#181](https://github.com/ipfs/go-ipfs-cmds/pull/181)) - Special-case accepting explicitly supplied named pipes ([ipfs/go-ipfs-cmds#184](https://github.com/ipfs/go-ipfs-cmds/pull/184)) - Chore/remove gx ([ipfs/go-ipfs-cmds#182](https://github.com/ipfs/go-ipfs-cmds/pull/182)) - http: allow specifying a custom http client ([ipfs/go-ipfs-cmds#175](https://github.com/ipfs/go-ipfs-cmds/pull/175)) - http: cleanup http related errors ([ipfs/go-ipfs-cmds#173](https://github.com/ipfs/go-ipfs-cmds/pull/173)) - fix: too many arguments error text ([ipfs/go-ipfs-cmds#172](https://github.com/ipfs/go-ipfs-cmds/pull/172)) - fallback executor support ([ipfs/go-ipfs-cmds#171](https://github.com/ipfs/go-ipfs-cmds/pull/171)) - make ErrorType a valid error and implement Unwrap on Error ([ipfs/go-ipfs-cmds#170](https://github.com/ipfs/go-ipfs-cmds/pull/170)) - feat: improve error codes ([ipfs/go-ipfs-cmds#168](https://github.com/ipfs/go-ipfs-cmds/pull/168)) - Fix a typo ([ipfs/go-ipfs-cmds#169](https://github.com/ipfs/go-ipfs-cmds/pull/169)) - github.com/ipfs/go-ipfs-config (v0.0.3 -> v0.5.3): - fix: correct the default-datastore config profile ([ipfs/go-ipfs-config#80](https://github.com/ipfs/go-ipfs-config/pull/80)) - feat: disable autonat service when in lowpower mode ([ipfs/go-ipfs-config#77](https://github.com/ipfs/go-ipfs-config/pull/77)) - feat: add and use a duration helper type ([ipfs/go-ipfs-config#76](https://github.com/ipfs/go-ipfs-config/pull/76)) - feat: add an autonat config section ([ipfs/go-ipfs-config#75](https://github.com/ipfs/go-ipfs-config/pull/75)) - feat: remove Routing.PrivateType ([ipfs/go-ipfs-config#74](https://github.com/ipfs/go-ipfs-config/pull/74)) - feat: add private routing config field ([ipfs/go-ipfs-config#73](https://github.com/ipfs/go-ipfs-config/pull/73)) - feat: mark badger as stable ([ipfs/go-ipfs-config#70](https://github.com/ipfs/go-ipfs-config/pull/70)) - feat: remove PreferTLS experiment ([ipfs/go-ipfs-config#71](https://github.com/ipfs/go-ipfs-config/pull/71)) - feat: remove old bootstrap peers ([ipfs/go-ipfs-config#67](https://github.com/ipfs/go-ipfs-config/pull/67)) - add config options for proxy/subdomain ([ipfs/go-ipfs-config#30](https://github.com/ipfs/go-ipfs-config/pull/30)) - feat: add graphsync option ([ipfs/go-ipfs-config#62](https://github.com/ipfs/go-ipfs-config/pull/62)) - profile: badger profile now defaults to asynchronous writes ([ipfs/go-ipfs-config#60](https://github.com/ipfs/go-ipfs-config/pull/60)) - migrate multiaddrs from /ipfs -> /p2p ([ipfs/go-ipfs-config#39](https://github.com/ipfs/go-ipfs-config/pull/39)) - use key size constraints defined in libp2p ([ipfs/go-ipfs-config#57](https://github.com/ipfs/go-ipfs-config/pull/57)) - plugins: don't omit empty config values ([ipfs/go-ipfs-config#46](https://github.com/ipfs/go-ipfs-config/pull/46)) - make it easier to detect an uninitialized repo ([ipfs/go-ipfs-config#45](https://github.com/ipfs/go-ipfs-config/pull/45)) - nit: omit empty plugin values ([ipfs/go-ipfs-config#44](https://github.com/ipfs/go-ipfs-config/pull/44)) - add plugins config section ([ipfs/go-ipfs-config#43](https://github.com/ipfs/go-ipfs-config/pull/43)) - Add very basic (possibly temporary) Provider configs ([ipfs/go-ipfs-config#38](https://github.com/ipfs/go-ipfs-config/pull/38)) - fix string formatting of bootstrap peers ([ipfs/go-ipfs-config#37](https://github.com/ipfs/go-ipfs-config/pull/37)) - migrate to the consolidated libp2p ([ipfs/go-ipfs-config#36](https://github.com/ipfs/go-ipfs-config/pull/36)) - Add strategic provider system experiment flag ([ipfs/go-ipfs-config#33](https://github.com/ipfs/go-ipfs-config/pull/33)) - github.com/ipfs/go-ipfs-files (v0.0.3 -> v0.0.8): - skip ignored files when calculating size ([ipfs/go-ipfs-files#30](https://github.com/ipfs/go-ipfs-files/pull/30)) - Feat/add ignore rules ([ipfs/go-ipfs-files#26](https://github.com/ipfs/go-ipfs-files/pull/26)) - revert(symlink): keep stat argument ([ipfs/go-ipfs-files#23](https://github.com/ipfs/go-ipfs-files/pull/23)) - feat: correctly report the size of symlinks ([ipfs/go-ipfs-files#22](https://github.com/ipfs/go-ipfs-files/pull/22)) - serialfile: fix handling of hidden paths on windows ([ipfs/go-ipfs-files#21](https://github.com/ipfs/go-ipfs-files/pull/21)) - feat: add WriteTo function ([ipfs/go-ipfs-files#20](https://github.com/ipfs/go-ipfs-files/pull/20)) - doc: fix formdata documentation ([ipfs/go-ipfs-files#19](https://github.com/ipfs/go-ipfs-files/pull/19)) - github.com/ipfs/go-ipfs-pinner (v0.0.1 -> v0.0.4): - fix: don't hold the pin lock while updating pins ([ipfs/go-ipfs-pinner#2](https://github.com/ipfs/go-ipfs-pinner/pull/2)) - github.com/ipfs/go-ipfs-pq (v0.0.1 -> v0.0.2): - Remove() ([ipfs/go-ipfs-pq#5](https://github.com/ipfs/go-ipfs-pq/pull/5)) - Fix Peek() test ([ipfs/go-ipfs-pq#4](https://github.com/ipfs/go-ipfs-pq/pull/4)) - add Peek() method ([ipfs/go-ipfs-pq#3](https://github.com/ipfs/go-ipfs-pq/pull/3)) - add gomod support // tag v0.0.1. ([ipfs/go-ipfs-pq#1](https://github.com/ipfs/go-ipfs-pq/pull/1)) - github.com/ipfs/go-ipfs-routing (v0.0.1 -> v0.1.0): - migrate to go-libp2p-core ([ipfs/go-ipfs-routing#22](https://github.com/ipfs/go-ipfs-routing/pull/22)) - github.com/ipfs/go-ipld-cbor (v0.0.2 -> v0.0.4): - doc: add a lead maintainer ([ipfs/go-ipld-cbor#65](https://github.com/ipfs/go-ipld-cbor/pull/65)) - fastpath CBOR ([ipfs/go-ipld-cbor#64](https://github.com/ipfs/go-ipld-cbor/pull/64)) - github.com/ipfs/go-ipld-format (v0.0.2 -> v0.2.0): - fix: change the batch size to avoid buffering too much ([ipfs/go-ipld-format#56](https://github.com/ipfs/go-ipld-format/pull/56)) - doc: add a lead maintainer ([ipfs/go-ipld-format#54](https://github.com/ipfs/go-ipld-format/pull/54)) - github.com/ipfs/go-ipld-git (v0.0.2 -> v0.0.3): - Use RFC3339 to format dates, fixes #16 ([ipfs/go-ipld-git#32](https://github.com/ipfs/go-ipld-git/pull/32)) - doc: add a lead maintainer ([ipfs/go-ipld-git#41](https://github.com/ipfs/go-ipld-git/pull/41)) - github.com/ipfs/go-ipns (v0.0.1 -> v0.0.2): - readme: add a lead maintainer ([ipfs/go-ipns#25](https://github.com/ipfs/go-ipns/pull/25)) - github.com/ipfs/go-log (v0.0.1 -> v1.0.4): - add IPFS_* env vars back for transitionary release of go-log ([ipfs/go-log#67](https://github.com/ipfs/go-log/pull/67)) - Experimental: zap backend for go-log ([ipfs/go-log#61](https://github.com/ipfs/go-log/pull/61)) - Spelling fix ([ipfs/go-log#63](https://github.com/ipfs/go-log/pull/63)) - Deprecate EventLogging and Warning* functions ([ipfs/go-log#62](https://github.com/ipfs/go-log/pull/62)) - github.com/ipfs/go-merkledag (v0.0.3 -> v0.3.2): - fix: correctly construct sessions ([ipfs/go-merkledag#56](https://github.com/ipfs/go-merkledag/pull/56)) - Migrate dagutils from go-ipfs ([ipfs/go-merkledag#50](https://github.com/ipfs/go-merkledag/pull/50)) - Make getPBNode Public ([ipfs/go-merkledag#49](https://github.com/ipfs/go-merkledag/pull/49)) - Pull In Upstream Changes ([ipfs/go-merkledag#1](https://github.com/ipfs/go-merkledag/pull/1)) - fix: slightly reduce memory usage when walking large directory trees ([ipfs/go-merkledag#45](https://github.com/ipfs/go-merkledag/pull/45)) - fix: return ErrLinkNotFound when the _link_ isn't found ([ipfs/go-merkledag#44](https://github.com/ipfs/go-merkledag/pull/44)) - fix: include root in searches by default ([ipfs/go-merkledag#43](https://github.com/ipfs/go-merkledag/pull/43)) - rework the graph walking functions with functional options ([ipfs/go-merkledag#42](https://github.com/ipfs/go-merkledag/pull/42)) - fix inconsistent EnumerateChildrenAsync behavior ([ipfs/go-merkledag#41](https://github.com/ipfs/go-merkledag/pull/41)) - github.com/ipfs/go-mfs (v0.0.7 -> v0.1.1): - migrate to go-libp2p-core ([ipfs/go-mfs#77](https://github.com/ipfs/go-mfs/pull/77)) - github.com/ipfs/go-peertaskqueue (v0.0.5-f09820a0a5b6 -> v0.2.0): - Extend peer task queue to work with want-have / want-block ([ipfs/go-peertaskqueue#8](https://github.com/ipfs/go-peertaskqueue/pull/8)) - migrate to go-libp2p-core ([ipfs/go-peertaskqueue#4](https://github.com/ipfs/go-peertaskqueue/pull/4)) - github.com/ipfs/go-unixfs (v0.0.6 -> v0.2.4): - fix: fix a panic when deleting ([ipfs/go-unixfs#81](https://github.com/ipfs/go-unixfs/pull/81)) - fix(dagreader): remove a buggy workaround for a gateway issue ([ipfs/go-unixfs#80](https://github.com/ipfs/go-unixfs/pull/80)) - fix: correctly handle symlink file sizes ([ipfs/go-unixfs#78](https://github.com/ipfs/go-unixfs/pull/78)) - fix: return the correct error from RemoveChild ([ipfs/go-unixfs#76](https://github.com/ipfs/go-unixfs/pull/76)) - update the last go-merkledag ([ipfs/go-unixfs#75](https://github.com/ipfs/go-unixfs/pull/75)) - fix: enumerate children ([ipfs/go-unixfs#74](https://github.com/ipfs/go-unixfs/pull/74)) - github.com/ipfs/interface-go-ipfs-core (v0.0.8 -> v0.2.7): - Add pin ls tests for indirect pin traversal and pin type precedence ([ipfs/interface-go-ipfs-core#47](https://github.com/ipfs/interface-go-ipfs-core/pull/47)) - fix(test): fix a flaky pubsub test ([ipfs/interface-go-ipfs-core#45](https://github.com/ipfs/interface-go-ipfs-core/pull/45)) - README: stub ([ipfs/interface-go-ipfs-core#44](https://github.com/ipfs/interface-go-ipfs-core/pull/44)) - test: test ReadAt if implemented ([ipfs/interface-go-ipfs-core#43](https://github.com/ipfs/interface-go-ipfs-core/pull/43)) - test: fix put with hash test ([ipfs/interface-go-ipfs-core#41](https://github.com/ipfs/interface-go-ipfs-core/pull/41)) - Bump go-libp2p-core, up test key size to 2048 ([ipfs/interface-go-ipfs-core#39](https://github.com/ipfs/interface-go-ipfs-core/pull/39)) - migrate to go-libp2p-core. ([ipfs/interface-go-ipfs-core#35](https://github.com/ipfs/interface-go-ipfs-core/pull/35)) - tests: expose TestSuite ([ipfs/interface-go-ipfs-core#34](https://github.com/ipfs/interface-go-ipfs-core/pull/34)) - github.com/libp2p/go-libp2p (v0.0.32 -> v0.8.2): - fix: keep observed addrs alive as long as their associated connections are alive ([libp2p/go-libp2p#899](https://github.com/libp2p/go-libp2p/pull/899)) - fix: refactor logic for identifying connections ([libp2p/go-libp2p#898](https://github.com/libp2p/go-libp2p/pull/898)) - fix: reduce log level of a noisy log line ([libp2p/go-libp2p#889](https://github.com/libp2p/go-libp2p/pull/889)) - [discovery] missing defer .Stop on ticker ([libp2p/go-libp2p#888](https://github.com/libp2p/go-libp2p/pull/888)) - deprioritize unspecified addresses in mock connections ([libp2p/go-libp2p#887](https://github.com/libp2p/go-libp2p/pull/887)) - feat: support TLS by default ([libp2p/go-libp2p#884](https://github.com/libp2p/go-libp2p/pull/884)) - Expose option for setting autonat throttling ([libp2p/go-libp2p#882](https://github.com/libp2p/go-libp2p/pull/882)) - Clearer naming of nat override options ([libp2p/go-libp2p#878](https://github.com/libp2p/go-libp2p/pull/878)) - fix: set the private key when constructing the autonat service ([libp2p/go-libp2p#853](https://github.com/libp2p/go-libp2p/pull/853)) - Signal address change ([libp2p/go-libp2p#851](https://github.com/libp2p/go-libp2p/pull/851)) - fix multiple issues in the mock tests ([libp2p/go-libp2p#850](https://github.com/libp2p/go-libp2p/pull/850)) - fix: minimal autonat dialer ([libp2p/go-libp2p#849](https://github.com/libp2p/go-libp2p/pull/849)) - Trigger Autorelay on NAT events ([libp2p/go-libp2p#807](https://github.com/libp2p/go-libp2p/pull/807)) - Local addr updated event ([libp2p/go-libp2p#847](https://github.com/libp2p/go-libp2p/pull/847)) - feat(mock): reliable notifications ([libp2p/go-libp2p#836](https://github.com/libp2p/go-libp2p/pull/836)) - doc(options): fix autorelay documentation ([libp2p/go-libp2p#835](https://github.com/libp2p/go-libp2p/pull/835)) - change PrivateNetwork to accept a PSK, update constructor magic ([libp2p/go-libp2p#796](https://github.com/libp2p/go-libp2p/pull/796)) - docs: Update the README ([libp2p/go-libp2p#827](https://github.com/libp2p/go-libp2p/pull/827)) - fix: remove an unnecessary goroutine ([libp2p/go-libp2p#820](https://github.com/libp2p/go-libp2p/pull/820)) - EnableAutoRelay should work without ContentRouting if there are StaticRelays defined ([libp2p/go-libp2p#810](https://github.com/libp2p/go-libp2p/pull/810)) - Use of mux.ErrReset in mocknet ([libp2p/go-libp2p#815](https://github.com/libp2p/go-libp2p/pull/815)) - docs: uniform comment sentences ([libp2p/go-libp2p#826](https://github.com/libp2p/go-libp2p/pull/826)) - enable non-public address port mapping announcement ([libp2p/go-libp2p#771](https://github.com/libp2p/go-libp2p/pull/771)) - fix: demote stream deadline errors to debug logs ([libp2p/go-libp2p#768](https://github.com/libp2p/go-libp2p/pull/768)) - small grammar fixes and updates to readme ([libp2p/go-libp2p#743](https://github.com/libp2p/go-libp2p/pull/743)) - Identify: Make activation threshold configurable ([libp2p/go-libp2p#740](https://github.com/libp2p/go-libp2p/pull/740)) - better user-agent handling ([libp2p/go-libp2p#702](https://github.com/libp2p/go-libp2p/pull/702)) - Update deps, mocknet tests ([libp2p/go-libp2p#697](https://github.com/libp2p/go-libp2p/pull/697)) - autorelay: ensure candidate relays can hop ([libp2p/go-libp2p#696](https://github.com/libp2p/go-libp2p/pull/696)) - We don't use `cs` here, drop it. ([libp2p/go-libp2p#682](https://github.com/libp2p/go-libp2p/pull/682)) - Fix racy and failing test cases. ([libp2p/go-libp2p#674](https://github.com/libp2p/go-libp2p/pull/674)) - fix: use the goprocess for closing ([libp2p/go-libp2p#669](https://github.com/libp2p/go-libp2p/pull/669)) - update package table after -core refactor ([libp2p/go-libp2p#661](https://github.com/libp2p/go-libp2p/pull/661)) - basic_host: ensure we close correctly when the context is canceled ([libp2p/go-libp2p#656](https://github.com/libp2p/go-libp2p/pull/656)) - Add go-libp2p-gostream and go-libp2p-http to readme ([libp2p/go-libp2p#655](https://github.com/libp2p/go-libp2p/pull/655)) - github.com/libp2p/go-libp2p-autonat (v0.0.6 -> v0.2.2): - Run Autonat Service while in unknown connectivity mode ([libp2p/go-libp2p-autonat#75](https://github.com/libp2p/go-libp2p-autonat/pull/75)) - Add option to force nat into a specified reachability state ([libp2p/go-libp2p-autonat#55](https://github.com/libp2p/go-libp2p-autonat/pull/55)) - Merge Autonat-svc ([libp2p/go-libp2p-autonat#54](https://github.com/libp2p/go-libp2p-autonat/pull/54)) - change autonat interface to use functional options ([libp2p/go-libp2p-autonat#53](https://github.com/libp2p/go-libp2p-autonat/pull/53)) - Limiting autonat service responses/startup ([libp2p/go-libp2p-autonat#45](https://github.com/libp2p/go-libp2p-autonat/pull/45)) - Emit events when NAT status changes ([libp2p/go-libp2p-autonat#37](https://github.com/libp2p/go-libp2p-autonat/pull/37)) - Take eventbus events to completion ([libp2p/go-libp2p-autonat#38](https://github.com/libp2p/go-libp2p-autonat/pull/38)) - Add missing syntax to autonat.proto ([libp2p/go-libp2p-autonat#26](https://github.com/libp2p/go-libp2p-autonat/pull/26)) - full close the autonat stream ([libp2p/go-libp2p-autonat#20](https://github.com/libp2p/go-libp2p-autonat/pull/20)) - reduce dialback timeout to 15s ([libp2p/go-libp2p-autonat#17](https://github.com/libp2p/go-libp2p-autonat/pull/17)) - Extract service implementation from go-libp2p-autonat ([libp2p/go-libp2p-autonat#1](https://github.com/libp2p/go-libp2p-autonat/pull/1)) - github.com/libp2p/go-libp2p-circuit (v0.0.9 -> v0.2.2): - fix: don't abort accept when accepting a single connection fails ([libp2p/go-libp2p-circuit#107](https://github.com/libp2p/go-libp2p-circuit/pull/107)) - Revert "feat: functional options" ([libp2p/go-libp2p-circuit#103](https://github.com/libp2p/go-libp2p-circuit/pull/103)) - feat: remove relay discovery and unspecified relay dialing ([libp2p/go-libp2p-circuit#101](https://github.com/libp2p/go-libp2p-circuit/pull/101)) - move protocol definitions to go-multiaddr ([libp2p/go-libp2p-circuit#81](https://github.com/libp2p/go-libp2p-circuit/pull/81)) - return the full address from conn.RemoteMultiaddr ([libp2p/go-libp2p-circuit#80](https://github.com/libp2p/go-libp2p-circuit/pull/80)) - expose CanHop as a module function ([libp2p/go-libp2p-circuit#79](https://github.com/libp2p/go-libp2p-circuit/pull/79)) - github.com/libp2p/go-libp2p-discovery (v0.0.5 -> v0.4.0): - Fix race with reuse of randomness ([libp2p/go-libp2p-discovery#54](https://github.com/libp2p/go-libp2p-discovery/pull/54)) - Add Backoff Cache Discovery ([libp2p/go-libp2p-discovery#26](https://github.com/libp2p/go-libp2p-discovery/pull/26)) - Discovery based Content Routing ([libp2p/go-libp2p-discovery#27](https://github.com/libp2p/go-libp2p-discovery/pull/27)) - github.com/libp2p/go-libp2p-kad-dht (v0.0.15 -> v0.7.10): - fix: avoid blocking when bootstrapping ([libp2p/go-libp2p-kad-dht#610](https://github.com/libp2p/go-libp2p-kad-dht/pull/610)) - fix: re-validate peers whenever their state changes ([libp2p/go-libp2p-kad-dht#607](https://github.com/libp2p/go-libp2p-kad-dht/pull/607)) - intercept failing query events when finding providers ([libp2p/go-libp2p-kad-dht#603](https://github.com/libp2p/go-libp2p-kad-dht/pull/603)) - feat: set provider manager options ([libp2p/go-libp2p-kad-dht#593](https://github.com/libp2p/go-libp2p-kad-dht/pull/593)) - fix: optimize debug logging a bit ([libp2p/go-libp2p-kad-dht#598](https://github.com/libp2p/go-libp2p-kad-dht/pull/598)) - stricter definition of public for DHT ([libp2p/go-libp2p-kad-dht#596](https://github.com/libp2p/go-libp2p-kad-dht/pull/596)) - feat: reduce allocations ([libp2p/go-libp2p-kad-dht#588](https://github.com/libp2p/go-libp2p-kad-dht/pull/588)) - query.go: Remove shuffle comment ([libp2p/go-libp2p-kad-dht#586](https://github.com/libp2p/go-libp2p-kad-dht/pull/586)) - fix: optimize isRelay ([libp2p/go-libp2p-kad-dht#585](https://github.com/libp2p/go-libp2p-kad-dht/pull/585)) - feat: expose WANActive ([libp2p/go-libp2p-kad-dht#580](https://github.com/libp2p/go-libp2p-kad-dht/pull/580)) - fix: improve error handling in dual dht ([libp2p/go-libp2p-kad-dht#582](https://github.com/libp2p/go-libp2p-kad-dht/pull/582)) - fix: deduplicate addresses ([libp2p/go-libp2p-kad-dht#581](https://github.com/libp2p/go-libp2p-kad-dht/pull/581)) - Fix bug in periodic peer pinging ([libp2p/go-libp2p-kad-dht#579](https://github.com/libp2p/go-libp2p-kad-dht/pull/579)) - Dual DHT scaffold ([libp2p/go-libp2p-kad-dht#570](https://github.com/libp2p/go-libp2p-kad-dht/pull/570)) - fix: linting fixes ([libp2p/go-libp2p-kad-dht#578](https://github.com/libp2p/go-libp2p-kad-dht/pull/578)) - fix: remove local provider check ([libp2p/go-libp2p-kad-dht#577](https://github.com/libp2p/go-libp2p-kad-dht/pull/577)) - fix: use the routing table filter ([libp2p/go-libp2p-kad-dht#576](https://github.com/libp2p/go-libp2p-kad-dht/pull/576)) - fix: handle empty keys ([libp2p/go-libp2p-kad-dht#562](https://github.com/libp2p/go-libp2p-kad-dht/pull/562)) - Set record handlers for the default protocol prefix ([libp2p/go-libp2p-kad-dht#560](https://github.com/libp2p/go-libp2p-kad-dht/pull/560)) - fix incorrect error handling during provider record lookups ([libp2p/go-libp2p-kad-dht#554](https://github.com/libp2p/go-libp2p-kad-dht/pull/554)) - Proposed DHTv2 Changes ([libp2p/go-libp2p-kad-dht#473](https://github.com/libp2p/go-libp2p-kad-dht/pull/473)) - fix: obey the context when sending messages to peers ([libp2p/go-libp2p-kad-dht#462](https://github.com/libp2p/go-libp2p-kad-dht/pull/462)) - Close context correctly ([libp2p/go-libp2p-kad-dht#477](https://github.com/libp2p/go-libp2p-kad-dht/pull/477)) - add benchmark for handleFindPeer ([libp2p/go-libp2p-kad-dht#475](https://github.com/libp2p/go-libp2p-kad-dht/pull/475)) - give views names again ([libp2p/go-libp2p-kad-dht#474](https://github.com/libp2p/go-libp2p-kad-dht/pull/474)) - metrics: record message/request event even in case of error ([libp2p/go-libp2p-kad-dht#464](https://github.com/libp2p/go-libp2p-kad-dht/pull/464)) - fix(dialqueue): fix a timer leak ([libp2p/go-libp2p-kad-dht#466](https://github.com/libp2p/go-libp2p-kad-dht/pull/466)) - fix(query): cancel the context when the query finishes ([libp2p/go-libp2p-kad-dht#467](https://github.com/libp2p/go-libp2p-kad-dht/pull/467)) - fix(providers): upgrade warnings to errors ([libp2p/go-libp2p-kad-dht#455](https://github.com/libp2p/go-libp2p-kad-dht/pull/455)) - Make the Routing Table's latency tolerance configurable. ([libp2p/go-libp2p-kad-dht#454](https://github.com/libp2p/go-libp2p-kad-dht/pull/454)) - Adjust cluster level while encoding as well ([libp2p/go-libp2p-kad-dht#445](https://github.com/libp2p/go-libp2p-kad-dht/pull/445)) - Remove incorrect doc ([libp2p/go-libp2p-kad-dht#443](https://github.com/libp2p/go-libp2p-kad-dht/pull/443)) - feat: reduce stream idle timeout to 1m ([libp2p/go-libp2p-kad-dht#441](https://github.com/libp2p/go-libp2p-kad-dht/pull/441)) - Provider records use multihashes instead of CIDs ([libp2p/go-libp2p-kad-dht#422](https://github.com/libp2p/go-libp2p-kad-dht/pull/422)) - Fix flaky TestEmptyTableTest ([libp2p/go-libp2p-kad-dht#433](https://github.com/libp2p/go-libp2p-kad-dht/pull/433)) - Refresh cpl's in dht ([libp2p/go-libp2p-kad-dht#428](https://github.com/libp2p/go-libp2p-kad-dht/pull/428)) - fix: always send the result channel when triggering a refresh ([libp2p/go-libp2p-kad-dht#425](https://github.com/libp2p/go-libp2p-kad-dht/pull/425)) - feat: allow disabling value and provider storage/messages ([libp2p/go-libp2p-kad-dht#400](https://github.com/libp2p/go-libp2p-kad-dht/pull/400)) - fix: prioritize closer peers ([libp2p/go-libp2p-kad-dht#424](https://github.com/libp2p/go-libp2p-kad-dht/pull/424)) - fix: try to re-add existing peers when the routing table is empty ([libp2p/go-libp2p-kad-dht#420](https://github.com/libp2p/go-libp2p-kad-dht/pull/420)) - feat: refresh and wait ([libp2p/go-libp2p-kad-dht#418](https://github.com/libp2p/go-libp2p-kad-dht/pull/418)) - Make max record age configurable ([libp2p/go-libp2p-kad-dht#410](https://github.com/libp2p/go-libp2p-kad-dht/pull/410)) - fix and simplify some bootstrapping logic ([libp2p/go-libp2p-kad-dht#405](https://github.com/libp2p/go-libp2p-kad-dht/pull/405)) - feat(bootstrap): take autobootstrap to completion ([libp2p/go-libp2p-kad-dht#403](https://github.com/libp2p/go-libp2p-kad-dht/pull/403)) - Feature/correct bootstrapping ([libp2p/go-libp2p-kad-dht#384](https://github.com/libp2p/go-libp2p-kad-dht/pull/384)) - Update tests to use Ed25519 when acceptable. ([libp2p/go-libp2p-kad-dht#380](https://github.com/libp2p/go-libp2p-kad-dht/pull/380)) - Add timeout ([libp2p/go-libp2p-kad-dht#351](https://github.com/libp2p/go-libp2p-kad-dht/pull/351)) - Feat/message size ([libp2p/go-libp2p-kad-dht#353](https://github.com/libp2p/go-libp2p-kad-dht/pull/353)) - reduce background goroutines ([libp2p/go-libp2p-kad-dht#340](https://github.com/libp2p/go-libp2p-kad-dht/pull/340)) - github.com/libp2p/go-libp2p-kbucket (v0.1.1 -> v0.4.1): - fix: use time.Duration for time, not floats ([libp2p/go-libp2p-kbucket#76](https://github.com/libp2p/go-libp2p-kbucket/pull/76)) - Add LastUsefulAt and LastSuccessfulQueryAt for each peer ([libp2p/go-libp2p-kbucket#75](https://github.com/libp2p/go-libp2p-kbucket/pull/75)) - fix: correctly track CPLs of never refreshed buckets ([libp2p/go-libp2p-kbucket#71](https://github.com/libp2p/go-libp2p-kbucket/pull/71)) - Get Peer Infos ([libp2p/go-libp2p-kbucket#69](https://github.com/libp2p/go-libp2p-kbucket/pull/69)) - fix: use accurate bucket logic ([libp2p/go-libp2p-kbucket#64](https://github.com/libp2p/go-libp2p-kbucket/pull/64)) - Replace dead peers & increase replacement cache size ([libp2p/go-libp2p-kbucket#59](https://github.com/libp2p/go-libp2p-kbucket/pull/59)) - Kbucket refactoring for Content Routing ([libp2p/go-libp2p-kbucket#54](https://github.com/libp2p/go-libp2p-kbucket/pull/54)) - Disassociate RT membership from connectivity ([libp2p/go-libp2p-kbucket#50](https://github.com/libp2p/go-libp2p-kbucket/pull/50)) - Unit Test for the util.Closer function ([libp2p/go-libp2p-kbucket#48](https://github.com/libp2p/go-libp2p-kbucket/pull/48)) - Refresh Cpl's, not buckets ([libp2p/go-libp2p-kbucket#46](https://github.com/libp2p/go-libp2p-kbucket/pull/46)) - Fix NearestPeers Doc ([libp2p/go-libp2p-kbucket#45](https://github.com/libp2p/go-libp2p-kbucket/pull/45)) - fix: when the target bucket is empty or low, pull from all other buckets ([libp2p/go-libp2p-kbucket#43](https://github.com/libp2p/go-libp2p-kbucket/pull/43)) - readme: replace IPFS contrib links with libp2p ([libp2p/go-libp2p-kbucket#34](https://github.com/libp2p/go-libp2p-kbucket/pull/34)) - k-bucket support for peoper kad bootstrapping ([libp2p/go-libp2p-kbucket#38](https://github.com/libp2p/go-libp2p-kbucket/pull/38)) - Fix bootstrapping id generation logic ([libp2p/go-libp2p-kbucket#1](https://github.com/libp2p/go-libp2p-kbucket/pull/1)) - fix: avoid hashing under a lock ([libp2p/go-libp2p-kbucket#31](https://github.com/libp2p/go-libp2p-kbucket/pull/31)) - dep: use a faster sha256 library ([libp2p/go-libp2p-kbucket#32](https://github.com/libp2p/go-libp2p-kbucket/pull/32)) - Remove a lot of allocations, and fix some ambiguous naming ([libp2p/go-libp2p-kbucket#30](https://github.com/libp2p/go-libp2p-kbucket/pull/30)) - github.com/libp2p/go-libp2p-mplex (v0.1.1 -> v0.2.3): - Respect mux.ErrReset ([libp2p/go-libp2p-mplex#9](https://github.com/libp2p/go-libp2p-mplex/pull/9)) - github.com/libp2p/go-libp2p-nat (v0.0.4 -> v0.0.6): - typo and changed deprecated method ([libp2p/go-libp2p-nat#26](https://github.com/libp2p/go-libp2p-nat/pull/26)) - nit: fix log format ([libp2p/go-libp2p-nat#19](https://github.com/libp2p/go-libp2p-nat/pull/19)) - fix: remove notifier ([libp2p/go-libp2p-nat#18](https://github.com/libp2p/go-libp2p-nat/pull/18)) - github.com/libp2p/go-libp2p-peerstore (v0.0.6 -> v0.2.3): - fix: handle nil peer IDs ([libp2p/go-libp2p-peerstore#88](https://github.com/libp2p/go-libp2p-peerstore/pull/88)) - Fix memory store signed peer record bug ([libp2p/go-libp2p-peerstore#133](https://github.com/libp2p/go-libp2p-peerstore/pull/133)) - fix: make closing the in-memory peerstore actually close it ([libp2p/go-libp2p-peerstore#131](https://github.com/libp2p/go-libp2p-peerstore/pull/131)) - Correct path to peer.AddrInfo in deprecation ([libp2p/go-libp2p-peerstore#124](https://github.com/libp2p/go-libp2p-peerstore/pull/124)) - fix multiple TTL bugs ([libp2p/go-libp2p-peerstore#92](https://github.com/libp2p/go-libp2p-peerstore/pull/92)) - reduce allocations when adding addrs ([libp2p/go-libp2p-peerstore#86](https://github.com/libp2p/go-libp2p-peerstore/pull/86)) - test: add metadata test ([libp2p/go-libp2p-peerstore#82](https://github.com/libp2p/go-libp2p-peerstore/pull/82)) - set map in constructor ([libp2p/go-libp2p-peerstore#81](https://github.com/libp2p/go-libp2p-peerstore/pull/81)) - improve interning ([libp2p/go-libp2p-peerstore#79](https://github.com/libp2p/go-libp2p-peerstore/pull/79)) - github.com/libp2p/go-libp2p-pnet (v0.0.1 -> v0.2.0): - remove key serialization, construct conn from ipnet.PSK ([libp2p/go-libp2p-pnet#32](https://github.com/libp2p/go-libp2p-pnet/pull/32)) - remove dependency on go-multicodec ([libp2p/go-libp2p-pnet#26](https://github.com/libp2p/go-libp2p-pnet/pull/26)) - github.com/libp2p/go-libp2p-pubsub (v0.0.3 -> v0.2.7): - Replace LRU cache blacklist implementation with a time cache ([libp2p/go-libp2p-pubsub#258](https://github.com/libp2p/go-libp2p-pubsub/pull/258)) - Configurable size of validate queue ([libp2p/go-libp2p-pubsub#255](https://github.com/libp2p/go-libp2p-pubsub/pull/255)) - Rename VaidatorData to ValidatorData ([libp2p/go-libp2p-pubsub#251](https://github.com/libp2p/go-libp2p-pubsub/pull/251)) - Configurable message id function ([libp2p/go-libp2p-pubsub#248](https://github.com/libp2p/go-libp2p-pubsub/pull/248)) - tracing support ([libp2p/go-libp2p-pubsub#227](https://github.com/libp2p/go-libp2p-pubsub/pull/227)) - add ValidatorData field to Message ([libp2p/go-libp2p-pubsub#231](https://github.com/libp2p/go-libp2p-pubsub/pull/231)) - Configurable outbound peer queue sizes ([libp2p/go-libp2p-pubsub#230](https://github.com/libp2p/go-libp2p-pubsub/pull/230)) - Topic handler bug fixes ([libp2p/go-libp2p-pubsub#225](https://github.com/libp2p/go-libp2p-pubsub/pull/225)) - Add Discovery ([libp2p/go-libp2p-pubsub#184](https://github.com/libp2p/go-libp2p-pubsub/pull/184)) - Expose the peer that propagates a message to the recipient ([libp2p/go-libp2p-pubsub#218](https://github.com/libp2p/go-libp2p-pubsub/pull/218)) - gossip methods: renames and predicate adjustment ([libp2p/go-libp2p-pubsub#204](https://github.com/libp2p/go-libp2p-pubsub/pull/204)) - godocs: clarify config params of MessageCache. ([libp2p/go-libp2p-pubsub#205](https://github.com/libp2p/go-libp2p-pubsub/pull/205)) - minor bug fix: on join, source peers from gossip[topic] if insufficient peers in fanout[topic] ([libp2p/go-libp2p-pubsub#196](https://github.com/libp2p/go-libp2p-pubsub/pull/196)) - add PubSub's context to Subscription ([libp2p/go-libp2p-pubsub#201](https://github.com/libp2p/go-libp2p-pubsub/pull/201)) - Add the ability to handle newly subscribed peers ([libp2p/go-libp2p-pubsub#190](https://github.com/libp2p/go-libp2p-pubsub/pull/190)) - Fix gossipsub race condition for heartbeat ([libp2p/go-libp2p-pubsub#188](https://github.com/libp2p/go-libp2p-pubsub/pull/188)) - github.com/libp2p/go-libp2p-pubsub-router (v0.0.3 -> v0.2.1): - fix: ignore bad peers when fetching the latest value ([libp2p/go-libp2p-pubsub-router#54](https://github.com/libp2p/go-libp2p-pubsub-router/pull/54)) - fix: rename MinimalPubsub -> Pubsub interface and improve docs ([libp2p/go-libp2p-pubsub-router#52](https://github.com/libp2p/go-libp2p-pubsub-router/pull/52)) - Use Minimal PubSub Interface Instead Of Full PubSub Router ([libp2p/go-libp2p-pubsub-router#51](https://github.com/libp2p/go-libp2p-pubsub-router/pull/51)) - Remove bootstrapping code ([libp2p/go-libp2p-pubsub-router#37](https://github.com/libp2p/go-libp2p-pubsub-router/pull/37)) - readme: replace IPFS contrib links with libp2p ([libp2p/go-libp2p-pubsub-router#34](https://github.com/libp2p/go-libp2p-pubsub-router/pull/34)) - Add Persistence Layer on top of PubSub ([libp2p/go-libp2p-pubsub-router#33](https://github.com/libp2p/go-libp2p-pubsub-router/pull/33)) - Subscribe to PubSub topic before Publishing ([libp2p/go-libp2p-pubsub-router#30](https://github.com/libp2p/go-libp2p-pubsub-router/pull/30)) - PutValue not blocked by Provide during bootstrapping ([libp2p/go-libp2p-pubsub-router#29](https://github.com/libp2p/go-libp2p-pubsub-router/pull/29)) - github.com/libp2p/go-libp2p-quic-transport (v0.0.3 -> v0.3.5): - add command line client and server ([libp2p/go-libp2p-quic-transport#139](https://github.com/libp2p/go-libp2p-quic-transport/pull/139)) - write qlogs to a temporary file first, then rename them when done ([libp2p/go-libp2p-quic-transport#136](https://github.com/libp2p/go-libp2p-quic-transport/pull/136)) - export qlogs when the QLOGDIR env variable is set ([libp2p/go-libp2p-quic-transport#129](https://github.com/libp2p/go-libp2p-quic-transport/pull/129)) - fix: avoid dialing/listening on dns addresses ([libp2p/go-libp2p-quic-transport#131](https://github.com/libp2p/go-libp2p-quic-transport/pull/131)) - use a stateless reset key derived from the private key ([libp2p/go-libp2p-quic-transport#122](https://github.com/libp2p/go-libp2p-quic-transport/pull/122)) - add support for multiaddr filtering ([libp2p/go-libp2p-quic-transport#125](https://github.com/libp2p/go-libp2p-quic-transport/pull/125)) - use the resolved address for RemoteMultiaddr() ([libp2p/go-libp2p-quic-transport#127](https://github.com/libp2p/go-libp2p-quic-transport/pull/127)) - accept a PSK in the transport constructor (and reject it) ([libp2p/go-libp2p-quic-transport#111](https://github.com/libp2p/go-libp2p-quic-transport/pull/111)) - update quic-go to v0.15.0 ([libp2p/go-libp2p-quic-transport#114](https://github.com/libp2p/go-libp2p-quic-transport/pull/114)) - increase the stream and connection receive windows ([libp2p/go-libp2p-quic-transport#108](https://github.com/libp2p/go-libp2p-quic-transport/pull/108)) - fix key comparisons in tests ([libp2p/go-libp2p-quic-transport#110](https://github.com/libp2p/go-libp2p-quic-transport/pull/110)) - make reuse work on Windows ([libp2p/go-libp2p-quic-transport#83](https://github.com/libp2p/go-libp2p-quic-transport/pull/83)) - add a LICENSE ([libp2p/go-libp2p-quic-transport#78](https://github.com/libp2p/go-libp2p-quic-transport/pull/78)) - Use specific netlink families for android ([libp2p/go-libp2p-quic-transport#75](https://github.com/libp2p/go-libp2p-quic-transport/pull/75)) - implement a garbage-collector for unused reuse connections ([libp2p/go-libp2p-quic-transport#73](https://github.com/libp2p/go-libp2p-quic-transport/pull/73)) - implement connection reuse ([libp2p/go-libp2p-quic-transport#63](https://github.com/libp2p/go-libp2p-quic-transport/pull/63)) - update the README ([libp2p/go-libp2p-quic-transport#69](https://github.com/libp2p/go-libp2p-quic-transport/pull/69)) - use the handshake logic from go-libp2p-tls ([libp2p/go-libp2p-quic-transport#67](https://github.com/libp2p/go-libp2p-quic-transport/pull/67)) - update quic-go to v0.12.0 (supporting QUIC draft-22) ([libp2p/go-libp2p-quic-transport#68](https://github.com/libp2p/go-libp2p-quic-transport/pull/68)) - when ListenUDP fails once, try again next time ([libp2p/go-libp2p-quic-transport#59](https://github.com/libp2p/go-libp2p-quic-transport/pull/59)) - github.com/libp2p/go-libp2p-record (v0.0.1 -> v0.1.2): - readme: replace IPFS contrib links with libp2p ([libp2p/go-libp2p-record#25](https://github.com/libp2p/go-libp2p-record/pull/25)) - Use peer ID utilities to go from pubkey to peer ID ([libp2p/go-libp2p-record#26](https://github.com/libp2p/go-libp2p-record/pull/26)) - github.com/libp2p/go-libp2p-routing-helpers (v0.0.2 -> v0.2.2): - doc: document all types ([libp2p/go-libp2p-routing-helpers#40](https://github.com/libp2p/go-libp2p-routing-helpers/pull/40)) - fix: fetch all providers when count is 0 ([libp2p/go-libp2p-routing-helpers#39](https://github.com/libp2p/go-libp2p-routing-helpers/pull/39)) - feat: implement io.Closer ([libp2p/go-libp2p-routing-helpers#37](https://github.com/libp2p/go-libp2p-routing-helpers/pull/37)) - readme: replace IPFS contrib links with libp2p ([libp2p/go-libp2p-routing-helpers#21](https://github.com/libp2p/go-libp2p-routing-helpers/pull/21)) - github.com/libp2p/go-libp2p-secio (v0.0.3 -> v0.2.2): - feat: remove sha1 hmac ([libp2p/go-libp2p-secio#64](https://github.com/libp2p/go-libp2p-secio/pull/64)) - readme: add context and links ([libp2p/go-libp2p-secio#55](https://github.com/libp2p/go-libp2p-secio/pull/55)) - Update to latest go-libp2p-core, update tests ([libp2p/go-libp2p-secio#54](https://github.com/libp2p/go-libp2p-secio/pull/54)) - Remove support for blowfish ([libp2p/go-libp2p-secio#52](https://github.com/libp2p/go-libp2p-secio/pull/52)) - fix: wait for handshake to complete before returning ([libp2p/go-libp2p-secio#50](https://github.com/libp2p/go-libp2p-secio/pull/50)) - avoid holding the message writer longer than necessary ([libp2p/go-libp2p-secio#49](https://github.com/libp2p/go-libp2p-secio/pull/49)) - github.com/libp2p/go-libp2p-swarm (v0.0.7 -> v0.2.3): - don't expire backoffs until 2x backoff period ([libp2p/go-libp2p-swarm#193](https://github.com/libp2p/go-libp2p-swarm/pull/193)) - fix: slightly simplify backoff logic ([libp2p/go-libp2p-swarm#192](https://github.com/libp2p/go-libp2p-swarm/pull/192)) - change backoffs to per-address ([libp2p/go-libp2p-swarm#191](https://github.com/libp2p/go-libp2p-swarm/pull/191)) - fix: set teardown after storing the context ([libp2p/go-libp2p-swarm#190](https://github.com/libp2p/go-libp2p-swarm/pull/190)) - feat: handle no addresses ([libp2p/go-libp2p-swarm#185](https://github.com/libp2p/go-libp2p-swarm/pull/185)) - fix: make sure to include peer in dial error ([libp2p/go-libp2p-swarm#180](https://github.com/libp2p/go-libp2p-swarm/pull/180)) - Don't drop connections when simultaneous dialing occurs ([libp2p/go-libp2p-swarm#174](https://github.com/libp2p/go-libp2p-swarm/pull/174)) - fix: fire a listen close event when closing the listener ([libp2p/go-libp2p-swarm#164](https://github.com/libp2p/go-libp2p-swarm/pull/164)) - Link to godocs for Host instead of deprecated repo ([libp2p/go-libp2p-swarm#137](https://github.com/libp2p/go-libp2p-swarm/pull/137)) - improve dial errors ([libp2p/go-libp2p-swarm#145](https://github.com/libp2p/go-libp2p-swarm/pull/145)) - Minor Docstring correction ([libp2p/go-libp2p-swarm#143](https://github.com/libp2p/go-libp2p-swarm/pull/143)) - test: close peerstore when closing the test swarm ([libp2p/go-libp2p-swarm#139](https://github.com/libp2p/go-libp2p-swarm/pull/139)) - fix listen addrs race ([libp2p/go-libp2p-swarm#136](https://github.com/libp2p/go-libp2p-swarm/pull/136)) - logging: make the swarm less noisy ([libp2p/go-libp2p-swarm#131](https://github.com/libp2p/go-libp2p-swarm/pull/131)) - feat: cache interface addresses for 1 minute ([libp2p/go-libp2p-swarm#129](https://github.com/libp2p/go-libp2p-swarm/pull/129)) - github.com/libp2p/go-libp2p-tls (v0.0.2 -> v0.1.3): - Readme: link to the libp2p-core docs ([libp2p/go-libp2p-tls#36](https://github.com/libp2p/go-libp2p-tls/pull/36)) - expose the function to derive the peer's public key from the cert chain ([libp2p/go-libp2p-tls#33](https://github.com/libp2p/go-libp2p-tls/pull/33)) - set an ALPN value in the tls.Config ([libp2p/go-libp2p-tls#32](https://github.com/libp2p/go-libp2p-tls/pull/32)) - github.com/libp2p/go-libp2p-transport-upgrader (v0.0.4 -> v0.2.0): - use the ipnet.PSK instead of the ipnet.Protector for private networks ([libp2p/go-libp2p-transport-upgrader#45](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/45)) - readme: add context & fix example code ([libp2p/go-libp2p-transport-upgrader#26](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/26)) - fix an incorrect error message ([libp2p/go-libp2p-transport-upgrader#27](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/27)) - Consolidate abstractions and core types into go-libp2p-core (#28) ([libp2p/go-libp2p-transport-upgrader#22](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/22)) - github.com/libp2p/go-libp2p-yamux (v0.1.3 -> v0.2.7): - Respect mux.ErrReset ([libp2p/go-libp2p-yamux#10](https://github.com/libp2p/go-libp2p-yamux/pull/10)) - github.com/libp2p/go-maddr-filter (v0.0.4 -> v0.0.5): - fix: check for blocked addrs without allocating ([libp2p/go-maddr-filter#14](https://github.com/libp2p/go-maddr-filter/pull/14)) - github.com/libp2p/go-mplex (v0.0.4 -> v0.1.2): - remove deprecated log.Warning(f) ([libp2p/go-mplex#65](https://github.com/libp2p/go-mplex/pull/65)) - Remove dependency on go-libp2p-core and introduce new errors. ([libp2p/go-mplex#72](https://github.com/libp2p/go-mplex/pull/72)) - Bump lodash from 4.17.5 to 4.17.15 in /interop/js ([libp2p/go-mplex#66](https://github.com/libp2p/go-mplex/pull/66)) - add test for deadlines ([libp2p/go-mplex#60](https://github.com/libp2p/go-mplex/pull/60)) - github.com/libp2p/go-msgio (v0.0.2 -> v0.0.4): - make the maximum message size configurable ([libp2p/go-msgio#15](https://github.com/libp2p/go-msgio/pull/15)) - combine writes and avoid a few more allocations ([libp2p/go-msgio#14](https://github.com/libp2p/go-msgio/pull/14)) - avoid allocating unless we need to ([libp2p/go-msgio#13](https://github.com/libp2p/go-msgio/pull/13)) - github.com/libp2p/go-nat (v0.0.3 -> v0.0.5): - feat: switch to go-netroute ([libp2p/go-nat#19](https://github.com/libp2p/go-nat/pull/19)) - fix: really obey the context ([libp2p/go-nat#13](https://github.com/libp2p/go-nat/pull/13)) - don't mask context ([libp2p/go-nat#10](https://github.com/libp2p/go-nat/pull/10)) - github.com/libp2p/go-reuseport-transport (v0.0.2 -> v0.0.3): - fix: less confusing log message ([libp2p/go-reuseport-transport#22](https://github.com/libp2p/go-reuseport-transport/pull/22)) - readme: replace IPFS contrib links with libp2p ([libp2p/go-reuseport-transport#16](https://github.com/libp2p/go-reuseport-transport/pull/16)) - replace gx instructions with note about gomod ([libp2p/go-reuseport-transport#15](https://github.com/libp2p/go-reuseport-transport/pull/15)) - github.com/libp2p/go-tcp-transport (v0.0.4 -> v0.2.0): - fix: don't allow dialing DNS addresses ([libp2p/go-tcp-transport#61](https://github.com/libp2p/go-tcp-transport/pull/61)) - Use new constructor for insecure transport in tests ([libp2p/go-tcp-transport#42](https://github.com/libp2p/go-tcp-transport/pull/42)) - readme: add install, usage & addressing info ([libp2p/go-tcp-transport#41](https://github.com/libp2p/go-tcp-transport/pull/41)) - github.com/libp2p/go-ws-transport (v0.0.6 -> v0.3.1): - fix: add read/write locks ([libp2p/go-ws-transport#85](https://github.com/libp2p/go-ws-transport/pull/85)) - fix: restrict dials to IP + TCP ([libp2p/go-ws-transport#84](https://github.com/libp2p/go-ws-transport/pull/84)) - Revert "add mutex for write/close" ([libp2p/go-ws-transport#73](https://github.com/libp2p/go-ws-transport/pull/73)) - feat: faster copy in wasm ([libp2p/go-ws-transport#68](https://github.com/libp2p/go-ws-transport/pull/68)) - Add WebAssembly support and the ability to Dial from browsers ([libp2p/go-ws-transport#55](https://github.com/libp2p/go-ws-transport/pull/55)) - fix: close gracefully ([libp2p/go-ws-transport#54](https://github.com/libp2p/go-ws-transport/pull/54)) - move multiaddr protocol definitions to go-multiaddr ([libp2p/go-ws-transport#52](https://github.com/libp2p/go-ws-transport/pull/52)) - Add install, usage & addressing info to README ([libp2p/go-ws-transport#49](https://github.com/libp2p/go-ws-transport/pull/49)) - github.com/libp2p/go-yamux (v1.2.3 -> v1.3.5): - fix: synchronize when resetting the keepalive timer ([libp2p/go-yamux#21](https://github.com/libp2p/go-yamux/pull/21)) - fix: don't keepalive when the connection is busy ([libp2p/go-yamux#16](https://github.com/libp2p/go-yamux/pull/16)) - Rename errors ([libp2p/go-yamux#14](https://github.com/libp2p/go-yamux/pull/14)) - fix(stream): set writeDeadline when cleanup and forceClose ([libp2p/go-yamux#12](https://github.com/libp2p/go-yamux/pull/12)) - fixes a stream deadlock multiple ways ([libp2p/go-yamux#8](https://github.com/libp2p/go-yamux/pull/8)) ### Contributors | Contributor | Commits | Lines ± | Files Changed | |----------------------------|---------|---------------|---------------| | Steven Allen | 858 | +27833/-15919 | 1906 | | Dirk McCormick | 134 | +18058/-8347 | 282 | | Aarsh Shah | 83 | +13458/-11883 | 241 | | Adin Schmahmann | 144 | +11878/-6236 | 397 | | Raúl Kripalani | 94 | +6894/-10214 | 598 | | vyzo | 60 | +8923/-1160 | 102 | | Will Scott | 79 | +3776/-1467 | 175 | | Michael Muré | 29 | +1734/-3290 | 104 | | dependabot[bot] | 365 | +3419/-361 | 728 | | Hector Sanjuan | 64 | +2053/-1321 | 132 | | Marten Seemann | 52 | +1922/-1268 | 147 | | Michael Avila | 29 | +828/-1733 | 70 | | Peter Rabbitson | 53 | +1073/-1197 | 100 | | Yusef Napora | 36 | +1610/-378 | 57 | | hannahhoward | 16 | +1342/-559 | 61 | | Łukasz Magiera | 9 | +277/-1623 | 41 | | Marcin Rataj | 9 | +1686/-99 | 32 | | Will | 7 | +936/-709 | 34 | | Alex Browne | 27 | +1019/-503 | 46 | | David Dias | 30 | +987/-431 | 43 | | Jakub Sztandera | 43 | +912/-436 | 77 | | Cole Brown | 21 | +646/-398 | 57 | | Oli Evans | 29 | +488/-466 | 43 | | Cornelius Toole | 3 | +827/-60 | 20 | | Hlib | 15 | +331/-185 | 28 | | Adrian Lanzafame | 9 | +123/-334 | 18 | | Petar Maymounkov | 1 | +385/-48 | 5 | | Alan Shaw | 18 | +262/-146 | 35 | | lnykww | 1 | +303/-52 | 6 | | Hannah Howard | 1 | +198/-27 | 3 | | Dominic Della Valle | 9 | +163/-52 | 14 | | Adam Uhlir | 1 | +211/-2 | 3 | | Dimitris Apostolou | 1 | +105/-105 | 64 | | Frrist | 1 | +186/-18 | 5 | | Henrique Dias | 22 | +119/-28 | 22 | | Gergely Tabiczky | 5 | +74/-60 | 7 | | Matt Joiner | 2 | +63/-62 | 4 | | @RubenKelevra | 12 | +46/-55 | 12 | | whyrusleeping | 6 | +87/-11 | 7 | | deepakgarg | 4 | +42/-43 | 4 | | protolambda | 2 | +49/-17 | 9 | | hucg | 2 | +47/-11 | 3 | | Arber Avdullahu | 3 | +31/-27 | 3 | | Sameer Puri | 1 | +46/-4 | 2 | | Hucg | 3 | +17/-33 | 3 | | Guilhem Fanton | 2 | +29/-10 | 7 | | Christian Muehlhaeuser | 6 | +20/-19 | 14 | | Djalil Dreamski | 3 | +27/-9 | 3 | | Caian | 2 | +36/-0 | 2 | | Topper Bowers | 2 | +31/-4 | 4 | | flowed | 1 | +16/-16 | 11 | | Vibhav Pant | 4 | +21/-10 | 5 | | frrist | 1 | +26/-4 | 1 | | Hlib Kanunnikov | 1 | +25/-3 | 1 | | george xie | 3 | +12/-15 | 11 | | optman | 1 | +13/-9 | 1 | | Roman Proskuryakov | 1 | +11/-11 | 2 | | Vasco Santos | 1 | +10/-10 | 5 | | Pretty Please Mark Darkly | 2 | +16/-2 | 2 | | Piotr Dyraga | 2 | +15/-2 | 2 | | Andrew Nesbitt | 1 | +5/-11 | 5 | | postables | 4 | +19/-8 | 4 | | Jim McDonald | 2 | +13/-1 | 2 | | PoorPockets McNewHold | 1 | +12/-0 | 1 | | Henri S | 1 | +6/-6 | 1 | | Igor Velkov | 1 | +8/-3 | 1 | | swedneck | 4 | +7/-3 | 4 | | Devin | 2 | +5/-5 | 4 | | iulianpascalau | 1 | +5/-3 | 2 | | MollyM | 3 | +7/-1 | 3 | | Jorropo | 2 | +5/-3 | 3 | | lukesolo | 1 | +6/-1 | 2 | | Wes Morgan | 1 | +3/-3 | 1 | | Kishan Mohanbhai Sagathiya | 1 | +3/-3 | 2 | | songjiayang | 1 | +4/-0 | 1 | | Terry Ding | 1 | +2/-2 | 1 | | Preston Van Loon | 2 | +3/-1 | 2 | | Jim Pick | 2 | +2/-2 | 2 | | Jakub Kaczmarzyk | 1 | +2/-2 | 1 | | Simon Menke | 2 | +2/-1 | 2 | | Jessica Schilling | 2 | +1/-2 | 2 | | Edgar Aroutiounian | 1 | +2/-1 | 1 | | hikerpig | 1 | +1/-1 | 1 | | ZenGround0 | 1 | +1/-1 | 1 | | Thomas Preindl | 1 | +1/-1 | 1 | | Sander Pick | 1 | +1/-1 | 1 | | Ronsor | 1 | +1/-1 | 1 | | Roman Khafizianov | 1 | +1/-1 | 1 | | Rod Vagg | 1 | +1/-1 | 1 | | Max Inden | 1 | +1/-1 | 1 | | Leo Arias | 1 | +1/-1 | 1 | | Kuro1 | 1 | +1/-1 | 1 | | Kirill Goncharov | 1 | +1/-1 | 1 | | John B Nelson | 1 | +1/-1 | 1 | | George Masgras | 1 | +1/-1 | 1 | | Aliabbas Merchant | 1 | +1/-1 | 1 | | Lorenzo Setale | 1 | +1/-0 | 1 | | Boris Mann | 1 | +1/-0 | 1 | ================================================ FILE: docs/changelogs/v0.6.md ================================================ # go-ipfs changelog v0.6 ## v0.6.0 2020-06-19 This is a relatively small release in terms of code changes, but it contains some significant changes to the IPFS protocol. ### Highlights The highlights in this release include: * The QUIC transport is enabled by default. Furthermore, go-ipfs will automatically run a migration to listen on the QUIC transport (on the same address/port as the TCP transport) to make this upgrade process seamless. * The new NOISE security transport is now supported but won't be selected by default. This transport will replace SECIO as the default cross-language interoperability security transport. TLS 1.3 will still remain the default security transport between go-ipfs nodes for now. **MIGRATION:** This release contains a small config migration to enable listening on the QUIC transport in addition the TCP transport. This migration will: * Normalize multiaddrs in the bootstrap list to use the `/p2p/Qm...` syntax for multiaddrs instead of the `/ipfs/Qm...` syntax. * Add QUIC addresses for the default bootstrappers, as necessary. If you've removed the default bootstrappers from your bootstrap config, the migration won't add them back. * Add a QUIC listener address to mirror any TCP addresses present in your config. For example, if you're listening on `/ip4/0.0.0.0/tcp/1234`, this migration will add a listen address for `/ip4/0.0.0.0/udp/1234/quic`. #### QUIC by default This release enables the QUIC transport (draft 28) by default for both inbound and outbound connections. When connecting to new peers, libp2p will continue to dial all advertised addresses (tcp + quic) in parallel so if the QUIC connection fails for some reason, the connection should still succeed. The QUIC transport has several key benefits over the current TCP based transports: * It takes fewer round-trips to establish a connection. With the QUIC transport, the IPFS handshake takes two round trips (one to establish the QUIC connection, one for the libp2p handshake). In the future, we should be able to reduce this to one round trip for the initial connection, and zero round trips for subsequent connections to a previously seen peer. This is especially important for DHT requests that contact many new peers. * Because it's UDP based instead of TCP based, it uses fewer file descriptors. The QUIC transport will open one UDP socket per listen address instead of one socket per connection. This should, in the future, allow us to keep more connections open. * Because QUIC connections don't consume file descriptors, we're able to remove the rate limit on outbound QUIC connections, further speeding up DHT queries. Unfortunately, this change isn't without drawbacks: the QUIC transport may not be able to max out some links (usually due to [poorly tuned kernel parameters](https://github.com/lucas-clemente/quic-go/issues/2586#issuecomment-639247615)). On the other hand, it may also be _faster_ in some cases If you hit this performance issue on Linux, you should tune the `net.core.rmem_default` and `net.core.rmem_max` sysctl parameters to increase your UDP receive buffer sizes. If necessary, you can disable the QUIC transport by running: ```bash > ipfs config --json Swarm.Transports.Network.QUIC false ``` **NOTE:** The QUIC transport included in this release is backwards incompatible with the experimental QUIC transport included in previous releases. Unfortunately, the QUIC protocol underwent some significant breaking changes and supporting multiple versions wasn't an option. In practice this degrades gracefully as go-ipfs will simply fall back on the TCP transport when dialing nodes with incompatible QUIC versions. #### Noise Transport This go-ipfs release introduces a new security transport: [libp2p Noise](https://github.com/libp2p/specs/tree/master/noise) (built from the [Noise Protocol Framework](http://www.noiseprotocol.org/)). While TLS1.3 remains the default go-ipfs security transport, Noise is simpler to implement from scratch and will be the standard cross-platform libp2p security transport going forward. This brings us one step closer to deprecating and removing support for SECIO. While enabled by default, Noise won't actually be _used_ by default it's negotiated. Given that TLS1.3 is still the default security transport for go-ipfs, this usually won't happen. If you'd like to prefer Noise over other security transports, you can change its priority in the [config](./docs/config.md) (`Swarm.Transports.Security.Noise`). #### Gateway This release brings two gateway-relevant features: custom 404 pages and base36 support. ##### Custom 404 You can now customize `404 Not Found` error pages by including an `ipfs-404.html` file somewhere in the request path. When a requested file isn't found, go-ipfs will look for an `ipfs-404.html` in the same directory as the requested file, and in each ancestor directory. If found, this file will be returned (with a 404 status code) instead of the usual error message. ##### Support for Base36 This release adds support for a new multibase encoding: base36. Base36 is an optimally efficient case-insensitive alphanumeric encoding. Case-insensitive alphanumeric encodings are important for the subdomain gateway as domain names are case insensitive. While base32 (the current default encoding used in subdomains) is simpler than base36, it's not optimally efficient and base36 Ed25519 IPNS keys are 2 characters too big to fit into the 63 character subdomain length limit. The extra efficiency from base36 brings us under this limit and allows Ed25519 IPNS keys to work with the subdomain gateway. This release adds support for base36 but won't use it by default. If you'd like to re-encode an Ed25519 IPNS key into base36, you can use the `ipfs cid format` command: ```sh $ ipfs cid format -v 1 --codec libp2p-key -b base36 bafzaajaiaejca4syrpdu6gdx4wsdnokxkprgzxf4wrstuc34gxw5k5jrag2so5gk k51qzi5uqu5dj16qyiq0tajolkojyl9qdkr254920wxv7ghtuwcz593tp69z9m ``` #### Gossipsub Upgrade This release brings a new gossipsub protocol version: 1.1. You can read about it in the [blog post](https://blog.ipfs.io/2020-05-20-gossipsub-v1.1/). #### Connectivity This release introduces a new ["peering"](./docs/config.md#peering) feature. The peering subsystem configures go-ipfs to connect to, remain connected to, and reconnect to a set of nodes. Nodes should use this subsystem to create "sticky" links between frequently useful peers to improve reliability. Use-cases: * An IPFS gateway connected to an IPFS cluster should peer to ensure that the gateway can always fetch content from the cluster. * A dapp may peer embedded go-ipfs nodes with a set of pinning services or textile cafes/hubs. * A set of friends may peer to ensure that they can always fetch each other's content. ### Changelog - github.com/ipfs/go-ipfs: - fix 3 bugs responsible for a goroutine leak (plus one other bug) ([ipfs/go-ipfs#7491](https://github.com/ipfs/go-ipfs/pull/7491)) - docs(config): update toc ([ipfs/go-ipfs#7483](https://github.com/ipfs/go-ipfs/pull/7483)) - feat: transport config ([ipfs/go-ipfs#7479](https://github.com/ipfs/go-ipfs/pull/7479)) - fix the minimal go version under 'Build from Source' ([ipfs/go-ipfs#7459](https://github.com/ipfs/go-ipfs/pull/7459)) - fix(migration): migrate /ipfs/ bootstrappers to /p2p/ - fix(migration): correctly migrate quic addresses - chore: add migration to listen on QUIC by default - backport fixes ([ipfs/go-ipfs#7405](https://github.com/ipfs/go-ipfs/pull/7405)) - Use bitswap sessions for `ipfs refs`. - Update to webui 2.9.0 - feat: add noise support ([ipfs/go-ipfs#7365](https://github.com/ipfs/go-ipfs/pull/7365)) - feat: implement peering service ([ipfs/go-ipfs#7362](https://github.com/ipfs/go-ipfs/pull/7362)) - Include the git blob id of the dir-index bundle in the ETag ([ipfs/go-ipfs#7360](https://github.com/ipfs/go-ipfs/pull/7360)) - feat: bootstrap in dht when the routing table is empty ([ipfs/go-ipfs#7340](https://github.com/ipfs/go-ipfs/pull/7340)) - quic: remove experimental status and add it to the default config ([ipfs/go-ipfs#7349](https://github.com/ipfs/go-ipfs/pull/7349)) - fix: support directory listings even if a 404 page is present ([ipfs/go-ipfs#7339](https://github.com/ipfs/go-ipfs/pull/7339)) - doc(plugin): document plugin config ([ipfs/go-ipfs#7309](https://github.com/ipfs/go-ipfs/pull/7309)) - test(sharness): fix fuse tests ([ipfs/go-ipfs#7320](https://github.com/ipfs/go-ipfs/pull/7320)) - docs: update experimental-features doc with IPNS over pubsub changes. ([ipfs/go-ipfs#7334](https://github.com/ipfs/go-ipfs/pull/7334)) - docs: cleanup config formatting ([ipfs/go-ipfs#7336](https://github.com/ipfs/go-ipfs/pull/7336)) - fix(gateway): ensure directory listings have Content-Type text/html ([ipfs/go-ipfs#7330](https://github.com/ipfs/go-ipfs/pull/7330)) - test(sharness): test the local symlink ([ipfs/go-ipfs#7332](https://github.com/ipfs/go-ipfs/pull/7332)) - misc config/experimental-features doc fixes ([ipfs/go-ipfs#7333](https://github.com/ipfs/go-ipfs/pull/7333)) - fix: correctly trim resolved IPNS addresses ([ipfs/go-ipfs#7331](https://github.com/ipfs/go-ipfs/pull/7331)) - Gateway renders pretty 404 pages if available ([ipfs/go-ipfs#4233](https://github.com/ipfs/go-ipfs/pull/4233)) - feat: add a dht stat command ([ipfs/go-ipfs#7221](https://github.com/ipfs/go-ipfs/pull/7221)) - fix: update dists url for OpenBSD support ([ipfs/go-ipfs#7311](https://github.com/ipfs/go-ipfs/pull/7311)) - docs: X-Forwarded-Proto: https ([ipfs/go-ipfs#7306](https://github.com/ipfs/go-ipfs/pull/7306)) - fix(mkreleaselog): make robust against running in different working directories ([ipfs/go-ipfs#7310](https://github.com/ipfs/go-ipfs/pull/7310)) - fix(mkreleasenotes): include commits directly to master ([ipfs/go-ipfs#7296](https://github.com/ipfs/go-ipfs/pull/7296)) - write api file automatically ([ipfs/go-ipfs#7282](https://github.com/ipfs/go-ipfs/pull/7282)) - systemd: disable swap-usage for ipfs ([ipfs/go-ipfs#7299](https://github.com/ipfs/go-ipfs/pull/7299)) - systemd: add helptext ([ipfs/go-ipfs#7265](https://github.com/ipfs/go-ipfs/pull/7265)) - systemd: add the link to the docs ([ipfs/go-ipfs#7287](https://github.com/ipfs/go-ipfs/pull/7287)) - systemd: add state directory setting ([ipfs/go-ipfs#7288](https://github.com/ipfs/go-ipfs/pull/7288)) - Update go version required to build ([ipfs/go-ipfs#7289](https://github.com/ipfs/go-ipfs/pull/7289)) - pin: implement pin/ls with only CoreApi ([ipfs/go-ipfs#6774](https://github.com/ipfs/go-ipfs/pull/6774)) - update go-libp2p-quic-transport to v0.3.7 ([ipfs/go-ipfs#7278](https://github.com/ipfs/go-ipfs/pull/7278)) - Docs: Delete section headers for removed features ([ipfs/go-ipfs#7277](https://github.com/ipfs/go-ipfs/pull/7277)) - README.md: typo ([ipfs/go-ipfs#7061](https://github.com/ipfs/go-ipfs/pull/7061)) - PR autocomment: Only comment for first-time contributors ([ipfs/go-ipfs#7270](https://github.com/ipfs/go-ipfs/pull/7270)) - Fixed typo in config.md ([ipfs/go-ipfs#7267](https://github.com/ipfs/go-ipfs/pull/7267)) - Fixes #7252 - Uses gabriel-vasile/mimetype to support additional content types ([ipfs/go-ipfs#7262](https://github.com/ipfs/go-ipfs/pull/7262)) - update go-libp2p-quic-transport to v0.3.6 ([ipfs/go-ipfs#7266](https://github.com/ipfs/go-ipfs/pull/7266)) - Updates bash completions to be compatible with zsh ([ipfs/go-ipfs#7261](https://github.com/ipfs/go-ipfs/pull/7261)) - systemd service enhancements + run as system user ([ipfs/go-ipfs#7259](https://github.com/ipfs/go-ipfs/pull/7259)) - upgrade to go 1.14.2 ([ipfs/go-ipfs#7130](https://github.com/ipfs/go-ipfs/pull/7130)) - Add module files for go-ipfs-as-a-library example ([ipfs/go-ipfs#7146](https://github.com/ipfs/go-ipfs/pull/7146)) - feat(gateway): show the absolute path and CID every time ([ipfs/go-ipfs#7219](https://github.com/ipfs/go-ipfs/pull/7219)) - fix: do not use hard coded IPNS Publish maximum timeout duration ([ipfs/go-ipfs#7256](https://github.com/ipfs/go-ipfs/pull/7256)) - Auto-comment on submitted PRs ([ipfs/go-ipfs#7248](https://github.com/ipfs/go-ipfs/pull/7248)) - Fixes GitHub link. ([ipfs/go-ipfs#7239](https://github.com/ipfs/go-ipfs/pull/7239)) - docs: fix subdomain examples in CHANGELOG ([ipfs/go-ipfs#7240](https://github.com/ipfs/go-ipfs/pull/7240)) - doc: add snap to the release checklist ([ipfs/go-ipfs#7253](https://github.com/ipfs/go-ipfs/pull/7253)) - Welcome message for users opening their first issue ([ipfs/go-ipfs#7247](https://github.com/ipfs/go-ipfs/pull/7247)) - feat: bump to 0.6.0-dev ([ipfs/go-ipfs#7249](https://github.com/ipfs/go-ipfs/pull/7249)) - github.com/ipfs/go-bitswap (v0.2.13 -> v0.2.19): - fix want gauge calculation ([ipfs/go-bitswap#416](https://github.com/ipfs/go-bitswap/pull/416)) - Fix PeerManager signalAvailabiity() race ([ipfs/go-bitswap#417](https://github.com/ipfs/go-bitswap/pull/417)) - fix: avoid taking accessing the peerQueues without taking the lock ([ipfs/go-bitswap#412](https://github.com/ipfs/go-bitswap/pull/412)) - fix: update circleci ci-go ([ipfs/go-bitswap#396](https://github.com/ipfs/go-bitswap/pull/396)) - fix: only track useful received data in the ledger (#411) ([ipfs/go-bitswap#411](https://github.com/ipfs/go-bitswap/pull/411)) - If peer is first to send a block to session, protect connection ([ipfs/go-bitswap#406](https://github.com/ipfs/go-bitswap/pull/406)) - Ensure sessions register with PeerManager ([ipfs/go-bitswap#405](https://github.com/ipfs/go-bitswap/pull/405)) - Total wants gauge (#402) ([ipfs/go-bitswap#402](https://github.com/ipfs/go-bitswap/pull/402)) - Improve peer manager performance ([ipfs/go-bitswap#395](https://github.com/ipfs/go-bitswap/pull/395)) - fix: return wants from engine.WantlistForPeer() ([ipfs/go-bitswap#390](https://github.com/ipfs/go-bitswap/pull/390)) - Add autocomment configuration - calculate message latency ([ipfs/go-bitswap#386](https://github.com/ipfs/go-bitswap/pull/386)) - fix: use one less go-routine per session (#377) ([ipfs/go-bitswap#377](https://github.com/ipfs/go-bitswap/pull/377)) - Add standard issue template - github.com/ipfs/go-cid (v0.0.5 -> v0.0.6): - feat: add Filecoin multicodecs ([ipfs/go-cid#104](https://github.com/ipfs/go-cid/pull/104)) - Add autocomment configuration - avoid calling the method WriteTo if we don't satisfy its contract ([ipfs/go-cid#103](https://github.com/ipfs/go-cid/pull/103)) - add a couple useful methods ([ipfs/go-cid#102](https://github.com/ipfs/go-cid/pull/102)) - Add standard issue template - github.com/ipfs/go-fs-lock (v0.0.4 -> v0.0.5): - chore: remove xerrors ([ipfs/go-fs-lock#15](https://github.com/ipfs/go-fs-lock/pull/15)) - Add autocomment configuration - Add standard issue template - github.com/ipfs/go-ipfs-cmds (v0.2.2 -> v0.2.9): - build(deps): bump github.com/ipfs/go-log from 1.0.3 to 1.0.4 ([ipfs/go-ipfs-cmds#194](https://github.com/ipfs/go-ipfs-cmds/pull/194)) - Fix go-ipfs#7242: Remove "HEAD" from Allow methods ([ipfs/go-ipfs-cmds#195](https://github.com/ipfs/go-ipfs-cmds/pull/195)) - Staticcheck fixes (#196) ([ipfs/go-ipfs-cmds#196](https://github.com/ipfs/go-ipfs-cmds/pull/196)) - doc: update docs for interface changes ([ipfs/go-ipfs-cmds#197](https://github.com/ipfs/go-ipfs-cmds/pull/197)) - Add standard issue template - github.com/ipfs/go-ipfs-config (v0.5.3 -> v0.8.0): - feat: add a transports section for enabling/disabling transports ([ipfs/go-ipfs-config#102](https://github.com/ipfs/go-ipfs-config/pull/102)) - feat: add an option for security transport experiments ([ipfs/go-ipfs-config#97](https://github.com/ipfs/go-ipfs-config/pull/97)) - feat: add peering service config section ([ipfs/go-ipfs-config#96](https://github.com/ipfs/go-ipfs-config/pull/96)) - fix: include key size in key init method ([ipfs/go-ipfs-config#95](https://github.com/ipfs/go-ipfs-config/pull/95)) - QUIC: remove experimental config option ([ipfs/go-ipfs-config#93](https://github.com/ipfs/go-ipfs-config/pull/93)) - fix bootstrap peers ([ipfs/go-ipfs-config#94](https://github.com/ipfs/go-ipfs-config/pull/94)) - default config: add QUIC listening ports + quic to mars.i.ipfs.io ([ipfs/go-ipfs-config#91](https://github.com/ipfs/go-ipfs-config/pull/91)) - feat: remove strict signing pubsub option. ([ipfs/go-ipfs-config#90](https://github.com/ipfs/go-ipfs-config/pull/90)) - Add autocomment configuration - Add Init Alternative allowing specification of ED25519 key ([ipfs/go-ipfs-config#78](https://github.com/ipfs/go-ipfs-config/pull/78)) - github.com/ipfs/go-mfs (v0.1.1 -> v0.1.2): - Fix incorrect mutex unlock call in File.Open ([ipfs/go-mfs#82](https://github.com/ipfs/go-mfs/pull/82)) - Add autocomment configuration - Add standard issue template - test: add Directory.ListNames test ([ipfs/go-mfs#81](https://github.com/ipfs/go-mfs/pull/81)) - doc: add a lead maintainer - Update README.md with newer travis badge ([ipfs/go-mfs#78](https://github.com/ipfs/go-mfs/pull/78)) - github.com/ipfs/interface-go-ipfs-core (v0.2.7 -> v0.3.0): - add Pin.IsPinned(..) ([ipfs/interface-go-ipfs-core#50](https://github.com/ipfs/interface-go-ipfs-core/pull/50)) - Add autocomment configuration - Add standard issue template - extra time for dht spin-up ([ipfs/interface-go-ipfs-core#61](https://github.com/ipfs/interface-go-ipfs-core/pull/61)) - feat: make the CoreAPI expose a streaming pin interface ([ipfs/interface-go-ipfs-core#49](https://github.com/ipfs/interface-go-ipfs-core/pull/49)) - test: fail early on err to avoid an unrelated panic ([ipfs/interface-go-ipfs-core#57](https://github.com/ipfs/interface-go-ipfs-core/pull/57)) - github.com/jbenet/go-is-domain (v1.0.3 -> v1.0.5): - Add OpenNIC domains to extended TLDs. ([jbenet/go-is-domain#15](https://github.com/jbenet/go-is-domain/pull/15)) - feat: add .crypto and .zil from UnstoppableDomains ([jbenet/go-is-domain#17](https://github.com/jbenet/go-is-domain/pull/17)) - chore: update IANA TLDs to version 2020051300 ([jbenet/go-is-domain#18](https://github.com/jbenet/go-is-domain/pull/18)) - github.com/libp2p/go-addr-util (v0.0.1 -> v0.0.2): - fix discuss badge - add discuss link to readme - fix: fdcostly should take only the prefix into account ([libp2p/go-addr-util#5](https://github.com/libp2p/go-addr-util/pull/5)) - add gomod support // tag v0.0.1 ([libp2p/go-addr-util#17](https://github.com/libp2p/go-addr-util/pull/17)) - github.com/libp2p/go-libp2p (v0.8.3 -> v0.9.6): - fix(nat): use the right addresses when nat port mapping ([libp2p/go-libp2p#966](https://github.com/libp2p/go-libp2p/pull/966)) - chore: update deps ([libp2p/go-libp2p#967](https://github.com/libp2p/go-libp2p/pull/967)) - Fix peer handler race ([libp2p/go-libp2p#965](https://github.com/libp2p/go-libp2p/pull/965)) - optimize numInbound count ([libp2p/go-libp2p#960](https://github.com/libp2p/go-libp2p/pull/960)) - update go-libp2p-circuit ([libp2p/go-libp2p#962](https://github.com/libp2p/go-libp2p/pull/962)) - Chunking large Identify responses with Signed Records ([libp2p/go-libp2p#958](https://github.com/libp2p/go-libp2p/pull/958)) - gomod: update dependencies ([libp2p/go-libp2p#959](https://github.com/libp2p/go-libp2p/pull/959)) - fixed compilation error (#956) ([libp2p/go-libp2p#956](https://github.com/libp2p/go-libp2p/pull/956)) - Filter Interface Addresses (#936) ([libp2p/go-libp2p#936](https://github.com/libp2p/go-libp2p/pull/936)) - fix: remove old addresses in identify immediately ([libp2p/go-libp2p#953](https://github.com/libp2p/go-libp2p/pull/953)) - fix flaky test (#952) ([libp2p/go-libp2p#952](https://github.com/libp2p/go-libp2p/pull/952)) - fix: group observations by zeroing port ([libp2p/go-libp2p#949](https://github.com/libp2p/go-libp2p/pull/949)) - fix: fix connection gater in transport constructor ([libp2p/go-libp2p#948](https://github.com/libp2p/go-libp2p/pull/948)) - Fix potential flakiness in TestIDService ([libp2p/go-libp2p#945](https://github.com/libp2p/go-libp2p/pull/945)) - make the {F=>f}iltersConnectionGater private. (#946) ([libp2p/go-libp2p#946](https://github.com/libp2p/go-libp2p/pull/946)) - Filter observed addresses (#917) ([libp2p/go-libp2p#917](https://github.com/libp2p/go-libp2p/pull/917)) - fix: don't try to marshal a nil record ([libp2p/go-libp2p#943](https://github.com/libp2p/go-libp2p/pull/943)) - add test to demo missing peer records after listen ([libp2p/go-libp2p#941](https://github.com/libp2p/go-libp2p/pull/941)) - fix: don't leak a goroutine if a peer connects and immediately disconnects ([libp2p/go-libp2p#942](https://github.com/libp2p/go-libp2p/pull/942)) - no signed peer records for mocknets (#934) ([libp2p/go-libp2p#934](https://github.com/libp2p/go-libp2p/pull/934)) - implement connection gating at the top level (#881) ([libp2p/go-libp2p#881](https://github.com/libp2p/go-libp2p/pull/881)) - various identify fixes and nits (#922) ([libp2p/go-libp2p#922](https://github.com/libp2p/go-libp2p/pull/922)) - Remove race between ID, Push & Delta (#907) ([libp2p/go-libp2p#907](https://github.com/libp2p/go-libp2p/pull/907)) - fix a compilation error introduced in 077a818. (#919) ([libp2p/go-libp2p#919](https://github.com/libp2p/go-libp2p/pull/919)) - exchange signed routing records in identify (#747) ([libp2p/go-libp2p#747](https://github.com/libp2p/go-libp2p/pull/747)) - github.com/libp2p/go-libp2p-autonat (v0.2.2 -> v0.2.3): - react to incoming events ([libp2p/go-libp2p-autonat#65](https://github.com/libp2p/go-libp2p-autonat/pull/65)) - github.com/libp2p/go-libp2p-blankhost (v0.1.4 -> v0.1.6): - subscribe connmgr to net notifications ([libp2p/go-libp2p-blankhost#45](https://github.com/libp2p/go-libp2p-blankhost/pull/45)) - add WithConnectionManager option to blankhost ([libp2p/go-libp2p-blankhost#44](https://github.com/libp2p/go-libp2p-blankhost/pull/44)) - Blank host should support signed records ([libp2p/go-libp2p-blankhost#42](https://github.com/libp2p/go-libp2p-blankhost/pull/42)) - github.com/libp2p/go-libp2p-circuit (v0.2.2 -> v0.2.3): - Use a fixed connection manager weight for peers with relay connections ([libp2p/go-libp2p-circuit#119](https://github.com/libp2p/go-libp2p-circuit/pull/119)) - github.com/libp2p/go-libp2p-connmgr (v0.2.1 -> v0.2.4): - Implement IsProtected interface ([libp2p/go-libp2p-connmgr#76](https://github.com/libp2p/go-libp2p-connmgr/pull/76)) - decaying tags: support removal and closure. (#72) ([libp2p/go-libp2p-connmgr#72](https://github.com/libp2p/go-libp2p-connmgr/pull/72)) - implement decaying tags. (#61) ([libp2p/go-libp2p-connmgr#61](https://github.com/libp2p/go-libp2p-connmgr/pull/61)) - github.com/libp2p/go-libp2p-core (v0.5.3 -> v0.5.7): - connmgr: add IsProtected interface (#158) ([libp2p/go-libp2p-core#158](https://github.com/libp2p/go-libp2p-core/pull/158)) - eventbus: add wildcard subscription type; getter to enumerate known types (#153) ([libp2p/go-libp2p-core#153](https://github.com/libp2p/go-libp2p-core/pull/153)) - events: add a generic DHT event. (#154) ([libp2p/go-libp2p-core#154](https://github.com/libp2p/go-libp2p-core/pull/154)) - decaying tags: support removal and closure. (#151) ([libp2p/go-libp2p-core#151](https://github.com/libp2p/go-libp2p-core/pull/151)) - implement Stringer for network.{Direction,Connectedness,Reachability}. (#150) ([libp2p/go-libp2p-core#150](https://github.com/libp2p/go-libp2p-core/pull/150)) - connmgr: introduce abstractions and functions for decaying tags. (#104) ([libp2p/go-libp2p-core#104](https://github.com/libp2p/go-libp2p-core/pull/104)) - Interface to verify if a peer supports a protocol without making allocations. ([libp2p/go-libp2p-core#148](https://github.com/libp2p/go-libp2p-core/pull/148)) - add connection gating interfaces and types. (#139) ([libp2p/go-libp2p-core#139](https://github.com/libp2p/go-libp2p-core/pull/139)) - github.com/libp2p/go-libp2p-kad-dht (v0.7.11 -> v0.8.2): - feat: protect all peers in low buckets, tag everyone else with 5 - fix: lookup context cancellation race condition ([libp2p/go-libp2p-kad-dht#656](https://github.com/libp2p/go-libp2p-kad-dht/pull/656)) - fix: protect useful peers in low buckets ([libp2p/go-libp2p-kad-dht#634](https://github.com/libp2p/go-libp2p-kad-dht/pull/634)) - Double the usefulness interval for peers in the Routing Table (#651) ([libp2p/go-libp2p-kad-dht#651](https://github.com/libp2p/go-libp2p-kad-dht/pull/651)) - enhancement/remove-unused-variable ([libp2p/go-libp2p-kad-dht#633](https://github.com/libp2p/go-libp2p-kad-dht/pull/633)) - Put back TestSelfWalkOnAddressChange ([libp2p/go-libp2p-kad-dht#648](https://github.com/libp2p/go-libp2p-kad-dht/pull/648)) - Routing Table Refresh manager (#601) ([libp2p/go-libp2p-kad-dht#601](https://github.com/libp2p/go-libp2p-kad-dht/pull/601)) - bootstrap empty RT and Optimize allocs when we discover new peers (#631) ([libp2p/go-libp2p-kad-dht#631](https://github.com/libp2p/go-libp2p-kad-dht/pull/631)) - fix all flaky tests ([libp2p/go-libp2p-kad-dht#628](https://github.com/libp2p/go-libp2p-kad-dht/pull/628)) - Update default concurrency parameter ([libp2p/go-libp2p-kad-dht#605](https://github.com/libp2p/go-libp2p-kad-dht/pull/605)) - clean up a channel that was dangling ([libp2p/go-libp2p-kad-dht#620](https://github.com/libp2p/go-libp2p-kad-dht/pull/620)) - github.com/libp2p/go-libp2p-kbucket (v0.4.1 -> v0.4.2): - Reduce allocs in AddPeer (#81) ([libp2p/go-libp2p-kbucket#81](https://github.com/libp2p/go-libp2p-kbucket/pull/81)) - NPeersForCpl and collapse empty buckets (#77) ([libp2p/go-libp2p-kbucket#77](https://github.com/libp2p/go-libp2p-kbucket/pull/77)) - github.com/libp2p/go-libp2p-peerstore (v0.2.3 -> v0.2.6): - fix two bugs in signed address handling ([libp2p/go-libp2p-peerstore#155](https://github.com/libp2p/go-libp2p-peerstore/pull/155)) - addrbook: fix races ([libp2p/go-libp2p-peerstore#154](https://github.com/libp2p/go-libp2p-peerstore/pull/154)) - Implement the FirstSupportedProtocol API. ([libp2p/go-libp2p-peerstore#147](https://github.com/libp2p/go-libp2p-peerstore/pull/147)) - github.com/libp2p/go-libp2p-pubsub (v0.2.7 -> v0.3.1): - fix outbound constraint satisfaction in oversubscription pruning - Gossipsub v0.3.0 - set sendTo to remote peer id in trace events ([libp2p/go-libp2p-pubsub#268](https://github.com/libp2p/go-libp2p-pubsub/pull/268)) - make wire protocol message size configurable. (#261) ([libp2p/go-libp2p-pubsub#261](https://github.com/libp2p/go-libp2p-pubsub/pull/261)) - github.com/libp2p/go-libp2p-pubsub-router (v0.2.1 -> v0.3.0): - feat: update pubsub ([libp2p/go-libp2p-pubsub-router#76](https://github.com/libp2p/go-libp2p-pubsub-router/pull/76)) - github.com/libp2p/go-libp2p-quic-transport (v0.3.7 -> v0.5.1): - close the connection when it is refused by InterceptSecured ([libp2p/go-libp2p-quic-transport#157](https://github.com/libp2p/go-libp2p-quic-transport/pull/157)) - gate QUIC connections via new ConnectionGater (#152) ([libp2p/go-libp2p-quic-transport#152](https://github.com/libp2p/go-libp2p-quic-transport/pull/152)) - github.com/libp2p/go-libp2p-record (v0.1.2 -> v0.1.3): - feat: add a better record error ([libp2p/go-libp2p-record#39](https://github.com/libp2p/go-libp2p-record/pull/39)) - github.com/libp2p/go-libp2p-swarm (v0.2.3 -> v0.2.6): - Configure private key for test swarm ([libp2p/go-libp2p-swarm#223](https://github.com/libp2p/go-libp2p-swarm/pull/223)) - Rank Dial addresses (#212) ([libp2p/go-libp2p-swarm#212](https://github.com/libp2p/go-libp2p-swarm/pull/212)) - implement connection gating support: intercept peer, address dials, upgraded conns (#201) ([libp2p/go-libp2p-swarm#201](https://github.com/libp2p/go-libp2p-swarm/pull/201)) - fix: avoid calling AddChild after the process may shutdown. ([libp2p/go-libp2p-swarm#207](https://github.com/libp2p/go-libp2p-swarm/pull/207)) - github.com/libp2p/go-libp2p-transport-upgrader (v0.2.0 -> v0.3.0): - call the connection gater when accepting connections and after crypto handshake (#55) ([libp2p/go-libp2p-transport-upgrader#55](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/55)) - github.com/libp2p/go-openssl (v0.0.4 -> v0.0.5): - add binding for OBJ_create ([libp2p/go-openssl#5](https://github.com/libp2p/go-openssl/pull/5)) - github.com/libp2p/go-yamux (v1.3.5 -> v1.3.7): - tighten lock around appending new chunks of read data in stream ([libp2p/go-yamux#28](https://github.com/libp2p/go-yamux/pull/28)) - fix: unlock recvLock in all cases. ([libp2p/go-yamux#25](https://github.com/libp2p/go-yamux/pull/25)) - github.com/lucas-clemente/quic-go (v0.15.7 -> v0.16.2): - make it possible to use the transport with both draft-28 and draft-29 - update the ALPN for draft-29 ([lucas-clemente/quic-go#2600](https://github.com/lucas-clemente/quic-go/pull/2600)) - update initial salts and test vectors for draft-29 ([lucas-clemente/quic-go#2587](https://github.com/lucas-clemente/quic-go/pull/2587)) - rename the SERVER_BUSY error to CONNECTION_REFUSED ([lucas-clemente/quic-go#2596](https://github.com/lucas-clemente/quic-go/pull/2596)) - reduce calls to time.Now() from the flow controller ([lucas-clemente/quic-go#2591](https://github.com/lucas-clemente/quic-go/pull/2591)) - remove redundant parenthesis and type conversion in flow controller ([lucas-clemente/quic-go#2592](https://github.com/lucas-clemente/quic-go/pull/2592)) - use the receipt of a Retry packet to get a first RTT estimate ([lucas-clemente/quic-go#2588](https://github.com/lucas-clemente/quic-go/pull/2588)) - fix debug message when returning an early session ([lucas-clemente/quic-go#2594](https://github.com/lucas-clemente/quic-go/pull/2594)) - fix closing of the http.Request.Body ([lucas-clemente/quic-go#2584](https://github.com/lucas-clemente/quic-go/pull/2584)) - split PTO calculation into a separate function ([lucas-clemente/quic-go#2576](https://github.com/lucas-clemente/quic-go/pull/2576)) - add a unit test using the ChaCha20 test vector from the draft ([lucas-clemente/quic-go#2585](https://github.com/lucas-clemente/quic-go/pull/2585)) - fix seed generation in frame sorter tests ([lucas-clemente/quic-go#2583](https://github.com/lucas-clemente/quic-go/pull/2583)) - make sure that ACK frames are bundled with data ([lucas-clemente/quic-go#2543](https://github.com/lucas-clemente/quic-go/pull/2543)) - add a Changelog for v0.16 ([lucas-clemente/quic-go#2582](https://github.com/lucas-clemente/quic-go/pull/2582)) - authenticate connection IDs ([lucas-clemente/quic-go#2567](https://github.com/lucas-clemente/quic-go/pull/2567)) - don't switch to PTO mode after using early loss detection ([lucas-clemente/quic-go#2581](https://github.com/lucas-clemente/quic-go/pull/2581)) - only create a single session for duplicate Initials ([lucas-clemente/quic-go#2580](https://github.com/lucas-clemente/quic-go/pull/2580)) - fix broken unit test in ackhandler - update the ALPN tokens to draft-28 ([lucas-clemente/quic-go#2570](https://github.com/lucas-clemente/quic-go/pull/2570)) - drop duplicate packets ([lucas-clemente/quic-go#2569](https://github.com/lucas-clemente/quic-go/pull/2569)) - remove noisy log statement in frame sorter test ([lucas-clemente/quic-go#2571](https://github.com/lucas-clemente/quic-go/pull/2571)) - fix flaky qlog unit tests ([lucas-clemente/quic-go#2572](https://github.com/lucas-clemente/quic-go/pull/2572)) - implement the 3x amplification limit ([lucas-clemente/quic-go#2536](https://github.com/lucas-clemente/quic-go/pull/2536)) - rewrite the frame sorter ([lucas-clemente/quic-go#2561](https://github.com/lucas-clemente/quic-go/pull/2561)) - retire conn IDs with sequence numbers smaller than the currently active ([lucas-clemente/quic-go#2563](https://github.com/lucas-clemente/quic-go/pull/2563)) - remove unused readOffset member variable in receiveStream ([lucas-clemente/quic-go#2559](https://github.com/lucas-clemente/quic-go/pull/2559)) - fix int overflow when parsing the transport parameters ([lucas-clemente/quic-go#2564](https://github.com/lucas-clemente/quic-go/pull/2564)) - use struct{} instead of bool in window update queue ([lucas-clemente/quic-go#2555](https://github.com/lucas-clemente/quic-go/pull/2555)) - update the protobuf library to google.golang.org/protobuf/proto ([lucas-clemente/quic-go#2554](https://github.com/lucas-clemente/quic-go/pull/2554)) - use the correct error code for crypto stream errors ([lucas-clemente/quic-go#2546](https://github.com/lucas-clemente/quic-go/pull/2546)) - bundle small writes on streams ([lucas-clemente/quic-go#2538](https://github.com/lucas-clemente/quic-go/pull/2538)) - reduce the length of the unprocessed packet chan in the session ([lucas-clemente/quic-go#2534](https://github.com/lucas-clemente/quic-go/pull/2534)) - fix flaky session unit test ([lucas-clemente/quic-go#2537](https://github.com/lucas-clemente/quic-go/pull/2537)) - add a send stream test that randomly acknowledges and loses data ([lucas-clemente/quic-go#2535](https://github.com/lucas-clemente/quic-go/pull/2535)) - fix size calculation for version negotiation packets ([lucas-clemente/quic-go#2542](https://github.com/lucas-clemente/quic-go/pull/2542)) - run all unit tests with race detector ([lucas-clemente/quic-go#2528](https://github.com/lucas-clemente/quic-go/pull/2528)) - add support for the ChaCha20 interop test case ([lucas-clemente/quic-go#2517](https://github.com/lucas-clemente/quic-go/pull/2517)) - fix buffer use after it was released when sending an INVALID_TOKEN error ([lucas-clemente/quic-go#2524](https://github.com/lucas-clemente/quic-go/pull/2524)) - run the internal and http3 tests with race detector on Travis ([lucas-clemente/quic-go#2385](https://github.com/lucas-clemente/quic-go/pull/2385)) - reset the PTO when dropping a packet number space ([lucas-clemente/quic-go#2527](https://github.com/lucas-clemente/quic-go/pull/2527)) - stop the deadline timer in Stream.Read and Write ([lucas-clemente/quic-go#2519](https://github.com/lucas-clemente/quic-go/pull/2519)) - don't reset pto_count on Initial ACKs ([lucas-clemente/quic-go#2513](https://github.com/lucas-clemente/quic-go/pull/2513)) - fix all race conditions in the session tests ([lucas-clemente/quic-go#2525](https://github.com/lucas-clemente/quic-go/pull/2525)) - make sure that the server's run loop returned when closing ([lucas-clemente/quic-go#2526](https://github.com/lucas-clemente/quic-go/pull/2526)) - fix flaky proxy test ([lucas-clemente/quic-go#2522](https://github.com/lucas-clemente/quic-go/pull/2522)) - stop the timer when the session's run loop returns ([lucas-clemente/quic-go#2516](https://github.com/lucas-clemente/quic-go/pull/2516)) - make it more likely that a STREAM frame is bundled with the FIN ([lucas-clemente/quic-go#2504](https://github.com/lucas-clemente/quic-go/pull/2504)) - github.com/multiformats/go-multiaddr (v0.2.1 -> v0.2.2): - absorb go-maddr-filter; rm stale Makefile targets; upgrade deps (#124) ([multiformats/go-multiaddr#124](https://github.com/multiformats/go-multiaddr/pull/124)) - github.com/multiformats/go-multibase (v0.0.2 -> v0.0.3): - Base36 implementation ([multiformats/go-multibase#36](https://github.com/multiformats/go-multibase/pull/36)) - Even more tests/benchmarks, less repetition in-code ([multiformats/go-multibase#34](https://github.com/multiformats/go-multibase/pull/34)) - Beef up tests before adding new codec ([multiformats/go-multibase#32](https://github.com/multiformats/go-multibase/pull/32)) - Remove GX, bump spec submodule, fix tests ([multiformats/go-multibase#31](https://github.com/multiformats/go-multibase/pull/31)) ### Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------------------|---------|-------------|---------------| | vyzo | 224 | +8016/-2810 | 304 | | Marten Seemann | 87 | +6081/-2607 | 215 | | Steven Allen | 157 | +4763/-1628 | 266 | | Aarsh Shah | 33 | +4619/-1634 | 128 | | Dirk McCormick | 26 | +3596/-1156 | 69 | | Yusef Napora | 66 | +2622/-785 | 98 | | Raúl Kripalani | 24 | +2424/-782 | 61 | | Hector Sanjuan | 30 | +999/-177 | 61 | | Louis Thibault | 2 | +1111/-4 | 4 | | Will Scott | 15 | +717/-219 | 31 | | dependabot-preview[bot] | 53 | +640/-64 | 106 | | Michael Muré | 7 | +456/-213 | 17 | | David Dias | 11 | +426/-88 | 15 | | Peter Rabbitson | 11 | +254/-189 | 31 | | Lukasz Zimnoch | 9 | +361/-49 | 13 | | Jakub Sztandera | 4 | +157/-104 | 9 | | Rod Vagg | 1 | +91/-83 | 2 | | RubenKelevra | 13 | +84/-84 | 30 | | JP Hastings-Spital | 1 | +145/-0 | 2 | | Adin Schmahmann | 11 | +67/-37 | 15 | | Marcin Rataj | 11 | +41/-43 | 11 | | Tiger | 5 | +53/-8 | 6 | | Akira | 2 | +35/-19 | 2 | | Casey Chance | 2 | +31/-22 | 2 | | Alan Shaw | 1 | +44/-0 | 2 | | Jessica Schilling | 4 | +20/-19 | 7 | | Gowtham G | 4 | +22/-14 | 6 | | Jeromy Johnson | 3 | +24/-6 | 3 | | Edgar Aroutiounian | 3 | +16/-8 | 3 | | Peter Wu | 2 | +12/-9 | 2 | | Sawood Alam | 2 | +7/-7 | 2 | | Command | 1 | +12/-0 | 1 | | Eric Myhre | 1 | +9/-2 | 1 | | mawei | 2 | +5/-5 | 2 | | decanus | 1 | +5/-5 | 1 | | Ignacio Hagopian | 2 | +7/-2 | 2 | | Alfonso Montero | 1 | +1/-5 | 1 | | Volker Mische | 1 | +2/-2 | 1 | | Shotaro Yamada | 1 | +2/-1 | 1 | | Mark Gaiser | 1 | +1/-1 | 1 | | Johnny | 1 | +1/-1 | 1 | | Ganesh Prasad Kumble | 1 | +1/-1 | 1 | | Dominic Della Valle | 1 | +1/-1 | 1 | | Corbin Page | 1 | +1/-1 | 1 | | Bryan Stenson | 1 | +1/-1 | 1 | | Bernhard M. Wiedemann | 1 | +1/-1 | 1 | ================================================ FILE: docs/changelogs/v0.7.md ================================================ # go-ipfs changelog v0.7 ## v0.7.0 2020-09-22 ### Highlights #### Secio is now disabled by default As part of deprecating and removing support for the Secio security transport, we have disabled it by default. TLS1.3 will remain the default security transport with fallback to Noise. You can read more about the deprecation in the blog post, https://blog.ipfs.io/2020-08-07-deprecating-secio/. If you're running IPFS older than 0.5, this may start to impact your performance on the public network. #### Ed25519 keys are now used by default Previously go-ipfs generated 2048 bit RSA keys for new nodes, but it will now use ed25519 keys by default. This will not affect any existing keys, but newly created keys will be ed25519 by default. The main benefit of using ed25519 keys over RSA is that ed25519 keys have an inline public key. This means that someone only needs your PeerId to verify things you've signed, which means we don't have to worry about storing those bulky RSA public keys. ##### Rotating keys Along with switching the default, we've added support for rotating keys. If you would like to change the key type of your IPFS node, you can now do so with the rotate command. **NOTE: This will affect your Peer Id, so be sure you want to do this!** Your existing identity key will be backed up in the Keystore. ```bash ipfs key rotate -o my-old-key -t ed25519 ``` #### Key export/import We've added commands to allow you to export and import keys from the IPFS Keystore to a local .key file. This does not apply to the IPFS identity key, `self`. ```bash ipfs key gen mykey ipfs key export -o mykey.key mykey # ./.key is the default path ipfs key import mykey mykey.key # on another node ``` #### IPNS paths now encode the key name as a base36 CIDv1 by default Previously go-ipfs encoded the key names for IPNS paths as base58btc multihashes (e.g. Qmabc...). We now encode them as base36 encoded CIDv1s as defined in the [peerID spec](https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#string-representation) (e.g. k51xyz...) which also deals with encoding of public keys. This is nice because it means that IPNS keys will by default be case-insensitive and that they will fit into DNS labels (e.g. k51xyz...ipns.localhost) and therefore that subdomain gateway redirections (e.g. from localhost:8080/ipns/{key} to {key}.ipns.localhost) will look better to users in the default case. Many commands will accept a `--ipns-base` option that allows changing command outputs to use a particular encoding (i.e. base58btc multihash, or CIDv1 encoded in any supported base) #### Multiaddresses now accept PeerIDs encoded as CIDv1 In preparation for eventually changing the default PeerID representation multiaddresses can now contain strings like `/p2p/k51xyz...` in addition to the default `/p2p/Qmabc...`. There is a corresponding `--peerid-base` option to many functions that output peerIDs. #### `dag stat` Initial support has been added for the `ipfs dag stat` command. Running this command will traverse the DAG for the given root CID and report statistics. By default, progress will be shown as the DAG is traversed. Supported statistics currently include DAG size and number of blocks. ```bash ipfs dag stat bafybeihpetclqvwb4qnmumvcn7nh4pxrtugrlpw4jgjpqicdxsv7opdm6e # the IPFS webui Size: 30362191, NumBlocks: 346 ``` #### Plugin build changes We have changed the build flags used by the official binary distributions on dist.ipfs.tech (or `/ipns/dist.ipfs.tech`) to use the simpler and more reliable `-trimpath` flag instead of the more complicated and brittle `-asmflags=all=-trimpath="$(GOPATH)" -gcflags=all=-trimpath="$(GOPATH)"` flags, however the build flags used by default in go-ipfs remain the same. The scripts in https://github.com/ipfs/go-ipfs-example-plugin have been updated to reflect this change. This is a breaking change to how people have been building plugins against the dist.ipfs.tech binary of go-ipfs and plugins should update their build processes accordingly see https://github.com/ipfs/go-ipfs-example-plugin/pull/9 for details. ### Changelog - github.com/ipfs/go-ipfs: - chore: bump webui version - fix: remove the (empty) alias for --peerid-base - Release v0.7.0-rc2 - fix: use override GOFLAGS changes from 480defab689610550ee3d346e31441a2bb881fcb but keep trimpath usage as is - Revert "fix: override GOFLAGS" - fix: remove the (empty) alias for --ipns-base - refactor: put all --ipns-base options in one place - docs: update config to indicate SECIO deprecation - fix: ipfs dht put/get commands now work on keys encoded as peerIDs and fail early for namespaces other than /pk or /ipns - Release v0.7.0-rc1 - chore: cleanup ([ipfs/go-ipfs#7628](https://github.com/ipfs/go-ipfs/pull/7628)) - namesys: fixed IPNS republisher to not overwrite IPNS record lifetimes ([ipfs/go-ipfs#7627](https://github.com/ipfs/go-ipfs/pull/7627)) - Fix #7624: Do not fetch dag nodes when checking if a pin exists ([ipfs/go-ipfs#7625](https://github.com/ipfs/go-ipfs/pull/7625)) - chore: update dependencies ([ipfs/go-ipfs#7610](https://github.com/ipfs/go-ipfs/pull/7610)) - use t.Cleanup() to reduce the need to clean up servers in tests ([ipfs/go-ipfs#7550](https://github.com/ipfs/go-ipfs/pull/7550)) - fix: ipfs pin ls - ignore pins that have errors ([ipfs/go-ipfs#7612](https://github.com/ipfs/go-ipfs/pull/7612)) - docs(config): fix Peering header ([ipfs/go-ipfs#7623](https://github.com/ipfs/go-ipfs/pull/7623)) - sharness: use dnsaddr example in ipfs p2p command tests ([ipfs/go-ipfs#7620](https://github.com/ipfs/go-ipfs/pull/7620)) - fix(key): don't allow backup key to be named 'self' ([ipfs/go-ipfs#7615](https://github.com/ipfs/go-ipfs/pull/7615)) - [BOUNTY] Directory page UI improvements ([ipfs/go-ipfs#7536](https://github.com/ipfs/go-ipfs/pull/7536)) - fix: make assets deterministic ([ipfs/go-ipfs#7609](https://github.com/ipfs/go-ipfs/pull/7609)) - use ed25519 keys by default ([ipfs/go-ipfs#7579](https://github.com/ipfs/go-ipfs/pull/7579)) - feat: wildcard support for public gateways ([ipfs/go-ipfs#7319](https://github.com/ipfs/go-ipfs/pull/7319)) - fix: fix go-bindata import path ([ipfs/go-ipfs#7605](https://github.com/ipfs/go-ipfs/pull/7605)) - Upgrade graphsync deps ([ipfs/go-ipfs#7598](https://github.com/ipfs/go-ipfs/pull/7598)) - Add --peerid-base to ipfs id command ([ipfs/go-ipfs#7591](https://github.com/ipfs/go-ipfs/pull/7591)) - use b36 keys by default for keys and IPNS ([ipfs/go-ipfs#7582](https://github.com/ipfs/go-ipfs/pull/7582)) - add ipfs dag stat command (#7553) ([ipfs/go-ipfs#7553](https://github.com/ipfs/go-ipfs/pull/7553)) - Move key rotation command to ipfs key rotate ([ipfs/go-ipfs#7599](https://github.com/ipfs/go-ipfs/pull/7599)) - Disable secio by default ([ipfs/go-ipfs#7600](https://github.com/ipfs/go-ipfs/pull/7600)) - Stop searching for public keys before doing an IPNS Get (#7549) ([ipfs/go-ipfs#7549](https://github.com/ipfs/go-ipfs/pull/7549)) - feat: return supported protocols in id output ([ipfs/go-ipfs#7409](https://github.com/ipfs/go-ipfs/pull/7409)) - docs: fix typo in default swarm addrs config docs ([ipfs/go-ipfs#7585](https://github.com/ipfs/go-ipfs/pull/7585)) - feat: nice errors when failing to load plugins ([ipfs/go-ipfs#7429](https://github.com/ipfs/go-ipfs/pull/7429)) - doc: document reverse proxy bug ([ipfs/go-ipfs#7478](https://github.com/ipfs/go-ipfs/pull/7478)) - fix: ipfs name resolve --dht-record-count flag uses correct type and now works - refactor: get rid of cmdDetails awkwardness - IPNS format keys in b36cid ([ipfs/go-ipfs#7554](https://github.com/ipfs/go-ipfs/pull/7554)) - Key import and export cli commands ([ipfs/go-ipfs#7546](https://github.com/ipfs/go-ipfs/pull/7546)) - feat: add snap package configuration ([ipfs/go-ipfs#7529](https://github.com/ipfs/go-ipfs/pull/7529)) - chore: bump webui version - repeat gateway subdomain test for all key types (#7542) ([ipfs/go-ipfs#7542](https://github.com/ipfs/go-ipfs/pull/7542)) - fix: override GOFLAGS - update QUIC, enable the RetireBugBackwardsCompatibilityMode - Document add behavior when the daemon is not running ([ipfs/go-ipfs#7514](https://github.com/ipfs/go-ipfs/pull/7514)) - ([ipfs/go-ipfs#7515](https://github.com/ipfs/go-ipfs/pull/7515)) - Choose Key type at initialization ([ipfs/go-ipfs#7251](https://github.com/ipfs/go-ipfs/pull/7251)) - feat: add flag to ipfs key and list to output keys in b36/CIDv1 (#7531) ([ipfs/go-ipfs#7531](https://github.com/ipfs/go-ipfs/pull/7531)) - feat: support ED25519 libp2p-key in subdomains - chore: fix a typo - docs: document X-Forwarded-Host - feat: support X-Forwarded-Host when doing gateway redirect - chore: update test deps for graphsync - chore: bump test dependencies ([ipfs/go-ipfs#7524](https://github.com/ipfs/go-ipfs/pull/7524)) - fix: use static binaries in docker container ([ipfs/go-ipfs#7505](https://github.com/ipfs/go-ipfs/pull/7505)) - chore:bump webui version to 2.10.1 ([ipfs/go-ipfs#7504](https://github.com/ipfs/go-ipfs/pull/7504)) - chore: bump webui version ([ipfs/go-ipfs#7501](https://github.com/ipfs/go-ipfs/pull/7501)) - update version to 0.7.0-dev - Merge branch 'release' into master - systemd: specify repo path, to avoid unnecessary subdirectory ([ipfs/go-ipfs#7472](https://github.com/ipfs/go-ipfs/pull/7472)) - doc(prod): start documenting production stuff ([ipfs/go-ipfs#7469](https://github.com/ipfs/go-ipfs/pull/7469)) - Readme: Update link about init systems (and import old readme) ([ipfs/go-ipfs#7473](https://github.com/ipfs/go-ipfs/pull/7473)) - doc(config): expand peering docs ([ipfs/go-ipfs#7466](https://github.com/ipfs/go-ipfs/pull/7466)) - fix: Use the -p option in Dockerfile to make parents as needed ([ipfs/go-ipfs#7464](https://github.com/ipfs/go-ipfs/pull/7464)) - systemd: enable systemd hardening features ([ipfs/go-ipfs#7286](https://github.com/ipfs/go-ipfs/pull/7286)) - fix(migration): migrate /ipfs/ bootstrappers to /p2p/ ([ipfs/go-ipfs#7450](https://github.com/ipfs/go-ipfs/pull/7450)) - readme: update go-version ([ipfs/go-ipfs#7447](https://github.com/ipfs/go-ipfs/pull/7447)) - fix(migration): correctly migrate quic addresses ([ipfs/go-ipfs#7446](https://github.com/ipfs/go-ipfs/pull/7446)) - chore: add migration to listen on QUIC by default ([ipfs/go-ipfs#7443](https://github.com/ipfs/go-ipfs/pull/7443)) - go: bump minimal dependency to 1.14.4 ([ipfs/go-ipfs#7419](https://github.com/ipfs/go-ipfs/pull/7419)) - fix: use bitswap sessions for ipfs refs ([ipfs/go-ipfs#7389](https://github.com/ipfs/go-ipfs/pull/7389)) - fix(commands): print consistent addresses in ipfs id ([ipfs/go-ipfs#7397](https://github.com/ipfs/go-ipfs/pull/7397)) - fix two pubsub issues. ([ipfs/go-ipfs#7394](https://github.com/ipfs/go-ipfs/pull/7394)) - docs: add pacman.store (@RubenKelevra) to the early testers ([ipfs/go-ipfs#7368](https://github.com/ipfs/go-ipfs/pull/7368)) - Update docs-beta links to final URLs ([ipfs/go-ipfs#7386](https://github.com/ipfs/go-ipfs/pull/7386)) - feat: webui v2.9.0 ([ipfs/go-ipfs#7387](https://github.com/ipfs/go-ipfs/pull/7387)) - chore: update WebUI to 2.8.0 ([ipfs/go-ipfs#7380](https://github.com/ipfs/go-ipfs/pull/7380)) - mailmap support ([ipfs/go-ipfs#7375](https://github.com/ipfs/go-ipfs/pull/7375)) - doc: update the release template for git flow changes ([ipfs/go-ipfs#7370](https://github.com/ipfs/go-ipfs/pull/7370)) - chore: update deps ([ipfs/go-ipfs#7369](https://github.com/ipfs/go-ipfs/pull/7369)) - github.com/ipfs/go-bitswap (v0.2.19 -> v0.2.20): - fix: don't say we're sending a full wantlist unless we are (#429) ([ipfs/go-bitswap#429](https://github.com/ipfs/go-bitswap/pull/429)) - github.com/ipfs/go-cid (v0.0.6 -> v0.0.7): - feat: optimize cid.Prefix ([ipfs/go-cid#109](https://github.com/ipfs/go-cid/pull/109)) - github.com/ipfs/go-datastore (v0.4.4 -> v0.4.5): - Add test to ensure that Delete returns no error for missing keys ([ipfs/go-datastore#162](https://github.com/ipfs/go-datastore/pull/162)) - Fix typo in sync/sync.go ([ipfs/go-datastore#159](https://github.com/ipfs/go-datastore/pull/159)) - Add the generated flatfs stub, since it cannot be auto-generated ([ipfs/go-datastore#158](https://github.com/ipfs/go-datastore/pull/158)) - support flatfs fuzzing ([ipfs/go-datastore#157](https://github.com/ipfs/go-datastore/pull/157)) - fuzzing harness (#153) ([ipfs/go-datastore#153](https://github.com/ipfs/go-datastore/pull/153)) - feat(mount): don't give up on error ([ipfs/go-datastore#146](https://github.com/ipfs/go-datastore/pull/146)) - /test: fix bad ElemCount/10 length (should not be divided) ([ipfs/go-datastore#152](https://github.com/ipfs/go-datastore/pull/152)) - github.com/ipfs/go-ds-flatfs (v0.4.4 -> v0.4.5): - Add os.Rename wrapper for Plan 9 (#87) ([ipfs/go-ds-flatfs#87](https://github.com/ipfs/go-ds-flatfs/pull/87)) - github.com/ipfs/go-fs-lock (v0.0.5 -> v0.0.6): - Fix build on Plan 9 ([ipfs/go-fs-lock#17](https://github.com/ipfs/go-fs-lock/pull/17)) - github.com/ipfs/go-graphsync (v0.0.5 -> v0.1.1): - docs(CHANGELOG): update for v0.1.1 - docs(CHANGELOG): update for v0.1.0 release ([ipfs/go-graphsync#84](https://github.com/ipfs/go-graphsync/pull/84)) - Deduplicate by key extension (#83) ([ipfs/go-graphsync#83](https://github.com/ipfs/go-graphsync/pull/83)) - Release infrastructure (#81) ([ipfs/go-graphsync#81](https://github.com/ipfs/go-graphsync/pull/81)) - feat(persistenceoptions): add unregister ability (#80) ([ipfs/go-graphsync#80](https://github.com/ipfs/go-graphsync/pull/80)) - fix(message): regen protobuf code (#79) ([ipfs/go-graphsync#79](https://github.com/ipfs/go-graphsync/pull/79)) - feat(requestmanager): run response hooks on completed requests (#77) ([ipfs/go-graphsync#77](https://github.com/ipfs/go-graphsync/pull/77)) - Revert "add extensions on complete (#76)" - add extensions on complete (#76) ([ipfs/go-graphsync#76](https://github.com/ipfs/go-graphsync/pull/76)) - All changes to date including pause requests & start paused, along with new adds for cleanups and checking of execution (#75) ([ipfs/go-graphsync#75](https://github.com/ipfs/go-graphsync/pull/75)) - More fine grained response controls (#71) ([ipfs/go-graphsync#71](https://github.com/ipfs/go-graphsync/pull/71)) - Refactor request execution and use IPLD SkipMe functionality for proper partial results on a request (#70) ([ipfs/go-graphsync#70](https://github.com/ipfs/go-graphsync/pull/70)) - feat(graphsync): implement do-no-send-cids extension (#69) ([ipfs/go-graphsync#69](https://github.com/ipfs/go-graphsync/pull/69)) - Incoming Block Hooks (#68) ([ipfs/go-graphsync#68](https://github.com/ipfs/go-graphsync/pull/68)) - fix(responsemanager): add nil check (#67) ([ipfs/go-graphsync#67](https://github.com/ipfs/go-graphsync/pull/67)) - refactor(hooks): use external pubsub (#65) ([ipfs/go-graphsync#65](https://github.com/ipfs/go-graphsync/pull/65)) - Update of IPLD Prime (#66) ([ipfs/go-graphsync#66](https://github.com/ipfs/go-graphsync/pull/66)) - feat(responsemanager): add listener for completed responses (#64) ([ipfs/go-graphsync#64](https://github.com/ipfs/go-graphsync/pull/64)) - Update Requests (#63) ([ipfs/go-graphsync#63](https://github.com/ipfs/go-graphsync/pull/63)) - Add pausing and unpausing of requests (#62) ([ipfs/go-graphsync#62](https://github.com/ipfs/go-graphsync/pull/62)) - Outgoing Request Hooks, swapping persistence layers (#61) ([ipfs/go-graphsync#61](https://github.com/ipfs/go-graphsync/pull/61)) - Feat/request hook loader chooser (#60) ([ipfs/go-graphsync#60](https://github.com/ipfs/go-graphsync/pull/60)) - Option to Reject requests by default (#58) ([ipfs/go-graphsync#58](https://github.com/ipfs/go-graphsync/pull/58)) - Testify refactor (#56) ([ipfs/go-graphsync#56](https://github.com/ipfs/go-graphsync/pull/56)) - Switch To Circle CI (#57) ([ipfs/go-graphsync#57](https://github.com/ipfs/go-graphsync/pull/57)) - fix(deps): go mod tidy - docs(README): remove ipldbridge reference - Tech Debt: Remove IPLD Bridge ([ipfs/go-graphsync#55](https://github.com/ipfs/go-graphsync/pull/55)) - github.com/ipfs/go-ipfs-cmds (v0.2.9 -> v0.4.0): - fix: allow requests from electron renderer (#201) ([ipfs/go-ipfs-cmds#201](https://github.com/ipfs/go-ipfs-cmds/pull/201)) - refactor: move external command checks into commands lib (#198) ([ipfs/go-ipfs-cmds#198](https://github.com/ipfs/go-ipfs-cmds/pull/198)) - Fix build on Plan 9 ([ipfs/go-ipfs-cmds#199](https://github.com/ipfs/go-ipfs-cmds/pull/199)) - github.com/ipfs/go-ipfs-config (v0.8.0 -> v0.9.0): - error if bit size specified with ed25519 keys (#105) ([ipfs/go-ipfs-config#105](https://github.com/ipfs/go-ipfs-config/pull/105)) - github.com/ipfs/go-log/v2 (v2.0.8 -> v2.1.1): failed to fetch repo - github.com/ipfs/go-path (v0.0.7 -> v0.0.8): - ResolveToLastNode no longer fetches nodes it does not need ([ipfs/go-path#30](https://github.com/ipfs/go-path/pull/30)) - doc: add a lead maintainer - github.com/ipfs/interface-go-ipfs-core (v0.3.0 -> v0.4.0): - Add ID formatting functions, used by various IPFS cli commands ([ipfs/interface-go-ipfs-core#65](https://github.com/ipfs/interface-go-ipfs-core/pull/65)) - github.com/ipld/go-car (v0.1.0 -> v0.1.1-0.20200429200904-c222d793c339): - Update go-ipld-prime to the era of NodeAssembler. ([ipld/go-car#31](https://github.com/ipld/go-car/pull/31)) - fix: update the cli tool's car dep ([ipld/go-car#30](https://github.com/ipld/go-car/pull/30)) - github.com/ipld/go-ipld-prime (v0.0.2-0.20191108012745-28a82f04c785 -> v0.0.2-0.20200428162820-8b59dc292b8e): - Add two basic examples of usage, as go tests. - Fix marshalling error ([ipld/go-ipld-prime#53](https://github.com/ipld/go-ipld-prime/pull/53)) - Add more test specs for list and map nesting. - traversal.SkipMe feature ([ipld/go-ipld-prime#51](https://github.com/ipld/go-ipld-prime/pull/51)) - Improvements to traversal docs. - Drop code coverage bot config. ([ipld/go-ipld-prime#50](https://github.com/ipld/go-ipld-prime/pull/50)) - Promote NodeAssembler/NodeStyle interface rework to core, and use improved basicnode implementation. ([ipld/go-ipld-prime#49](https://github.com/ipld/go-ipld-prime/pull/49)) - Merge branch 'traversal-benchmarks' - Merge branch 'cycle-breaking-and-traversal-benchmarks' - Merge branch 'assembler-upgrade-to-codecs' - Path clarifications ([ipld/go-ipld-prime#47](https://github.com/ipld/go-ipld-prime/pull/47)) - Merge branch 'research-admissions' - Add a typed link node to allow traversal with code generated builders across links ([ipld/go-ipld-prime#41](https://github.com/ipld/go-ipld-prime/pull/41)) - Merge branch 'research-admissions' - Library updates. - Feat/add code gen disclaimer ([ipld/go-ipld-prime#39](https://github.com/ipld/go-ipld-prime/pull/39)) - Readme and key Node interface docs improvements. - fix(schema/gen): return value not reference ([ipld/go-ipld-prime#38](https://github.com/ipld/go-ipld-prime/pull/38)) - github.com/ipld/go-ipld-prime-proto (v0.0.0-20191113031812-e32bd156a1e5 -> v0.0.0-20200428191222-c1ffdadc01e1): - feat(deps): upgrade to new IPLD prime ([ipld/go-ipld-prime-proto#1](https://github.com/ipld/go-ipld-prime-proto/pull/1)) - Update to latest ipld before rework ([ipld/go-ipld-prime-proto#2](https://github.com/ipld/go-ipld-prime-proto/pull/2)) - github.com/libp2p/go-libp2p (v0.9.6 -> v0.11.0): - Added parsing of IPv6 addresses for incoming mDNS requests ([libp2p/go-libp2p#990](https://github.com/libp2p/go-libp2p/pull/990)) - Switch from SECIO to Noise ([libp2p/go-libp2p#972](https://github.com/libp2p/go-libp2p/pull/972)) - fix tests ([libp2p/go-libp2p#995](https://github.com/libp2p/go-libp2p/pull/995)) - Bump Autonat version & validate fixed call loop in `.Addrs` (#988) ([libp2p/go-libp2p#988](https://github.com/libp2p/go-libp2p/pull/988)) - fix: use the correct external address when NAT port-mapping ([libp2p/go-libp2p#987](https://github.com/libp2p/go-libp2p/pull/987)) - upgrade deps + interoperable uvarint delimited writer/reader. (#985) ([libp2p/go-libp2p#985](https://github.com/libp2p/go-libp2p/pull/985)) - fix host can be dialed by autonat public addr, but lost the public addr to announce ([libp2p/go-libp2p#983](https://github.com/libp2p/go-libp2p/pull/983)) - Fix address advertisement bugs (#974) ([libp2p/go-libp2p#974](https://github.com/libp2p/go-libp2p/pull/974)) - fix: avoid a close deadlock in the natmanager ([libp2p/go-libp2p#971](https://github.com/libp2p/go-libp2p/pull/971)) - upgrade swarm; add ID() on mock conns and streams. (#970) ([libp2p/go-libp2p#970](https://github.com/libp2p/go-libp2p/pull/970)) - github.com/libp2p/go-libp2p-asn-util (null -> v0.0.0-20200825225859-85005c6cf052): - chore: go fmt - feat: use deferred initialization of the asnStore ([libp2p/go-libp2p-asn-util#3](https://github.com/libp2p/go-libp2p-asn-util/pull/3)) - chore: switch to forked cidranger - fixed code - library for ASN mappings - github.com/libp2p/go-libp2p-autonat (v0.2.3 -> v0.3.2): - static nat shouldn't call host.Addrs() - upgrade deps + interoperable uvarint delimited writer/reader. (#95) ([libp2p/go-libp2p-autonat#95](https://github.com/libp2p/go-libp2p-autonat/pull/95)) - fix: a type switch nit ([libp2p/go-libp2p-autonat#83](https://github.com/libp2p/go-libp2p-autonat/pull/83)) - github.com/libp2p/go-libp2p-blankhost (v0.1.6 -> v0.2.0): - call reset where appropriate (and update deps) ([libp2p/go-libp2p-blankhost#52](https://github.com/libp2p/go-libp2p-blankhost/pull/52)) - github.com/libp2p/go-libp2p-circuit (v0.2.3 -> v0.3.1): - upgrade deps + interoperable uvarints. (#122) ([libp2p/go-libp2p-circuit#122](https://github.com/libp2p/go-libp2p-circuit/pull/122)) - Fix/remove deprecated logging ([libp2p/go-libp2p-circuit#85](https://github.com/libp2p/go-libp2p-circuit/pull/85)) - github.com/libp2p/go-libp2p-core (v0.5.7 -> v0.6.1): - experimental introspection support (#159) ([libp2p/go-libp2p-core#159](https://github.com/libp2p/go-libp2p-core/pull/159)) - github.com/libp2p/go-libp2p-discovery (v0.4.0 -> v0.5.0): - Put period at end of sentence ([libp2p/go-libp2p-discovery#65](https://github.com/libp2p/go-libp2p-discovery/pull/65)) - github.com/libp2p/go-libp2p-kad-dht (v0.8.2 -> v0.9.0): - chore: update deps ([libp2p/go-libp2p-kad-dht#689](https://github.com/libp2p/go-libp2p-kad-dht/pull/689)) - allow overwriting builtin dual DHT options ([libp2p/go-libp2p-kad-dht#688](https://github.com/libp2p/go-libp2p-kad-dht/pull/688)) - Hardening Improvements: RT diversity and decreased RT churn ([libp2p/go-libp2p-kad-dht#687](https://github.com/libp2p/go-libp2p-kad-dht/pull/687)) - Fix key log encoding ([libp2p/go-libp2p-kad-dht#682](https://github.com/libp2p/go-libp2p-kad-dht/pull/682)) - upgrade deps + uvarint delimited writer/reader. (#684) ([libp2p/go-libp2p-kad-dht#684](https://github.com/libp2p/go-libp2p-kad-dht/pull/684)) - periodicBootstrapInterval should be ticker? (#678) ([libp2p/go-libp2p-kad-dht#678](https://github.com/libp2p/go-libp2p-kad-dht/pull/678)) - removes duplicate comment ([libp2p/go-libp2p-kad-dht#674](https://github.com/libp2p/go-libp2p-kad-dht/pull/674)) - Revert "Peer Diversity in the Routing Table (#658)" ([libp2p/go-libp2p-kad-dht#670](https://github.com/libp2p/go-libp2p-kad-dht/pull/670)) - Fixed problem with refresh logging ([libp2p/go-libp2p-kad-dht#667](https://github.com/libp2p/go-libp2p-kad-dht/pull/667)) - feat: protect all peers in low buckets, tag everyone else with 5 ([libp2p/go-libp2p-kad-dht#666](https://github.com/libp2p/go-libp2p-kad-dht/pull/666)) - Peer Diversity in the Routing Table (#658) ([libp2p/go-libp2p-kad-dht#658](https://github.com/libp2p/go-libp2p-kad-dht/pull/658)) - github.com/libp2p/go-libp2p-kbucket (v0.4.2 -> v0.4.7): - chore: switch from go-multiaddr-net to go-multiaddr/net - Use crypto/rand for generating random prefixes - feat: when using the diversity filter for ipv6 addresses if the ASN cannot be found for a particular address then fallback on using the /32 mask of the address as the group name instead of simply rejecting the peer from routing table - simplify filter (#92) ([libp2p/go-libp2p-kbucket#92](https://github.com/libp2p/go-libp2p-kbucket/pull/92)) - fix: switch to forked cid ranger dep ([libp2p/go-libp2p-kbucket#91](https://github.com/libp2p/go-libp2p-kbucket/pull/91)) - Reduce Routing Table churn (#90) ([libp2p/go-libp2p-kbucket#90](https://github.com/libp2p/go-libp2p-kbucket/pull/90)) - Peer Diversity for Routing Table and Querying (#88) ([libp2p/go-libp2p-kbucket#88](https://github.com/libp2p/go-libp2p-kbucket/pull/88)) - fix bug in peer eviction (#87) ([libp2p/go-libp2p-kbucket#87](https://github.com/libp2p/go-libp2p-kbucket/pull/87)) - feat: add an AddedAt timestamp (#84) ([libp2p/go-libp2p-kbucket#84](https://github.com/libp2p/go-libp2p-kbucket/pull/84)) - github.com/libp2p/go-libp2p-pubsub (v0.3.1 -> v0.3.5): - regenerate protobufs (#381) ([libp2p/go-libp2p-pubsub#381](https://github.com/libp2p/go-libp2p-pubsub/pull/381)) - track validation time - fulfill promise as soon as a message begins validation - don't apply penalty in self origin rejections - add behaviour penalty threshold - Add String() method to Topic. - add regression test for issue 371 - don't add direct peers to fanout - reference spec change in comment. - fix backoff slack time - use the heartbeat interval for slack time - add slack time to prune backoff clearance - fix: call the correct tracer function in FloodSubRouter.Leave (#373) ([libp2p/go-libp2p-pubsub#373](https://github.com/libp2p/go-libp2p-pubsub/pull/373)) - downgrade trace buffer overflow log to debug - track topics in Reject/Duplicate/Deliver events - add topics to Reject/Duplicate/Deliver events - fix flaky test - refactor ip colocation factor computation that is common for score and inspection - better handling of intermediate topic score snapshots - disallow duplicate score inspectors - make peer score inspect function types aliases - extended peer score inspection - upgrade deps + interoperable uvarint delimited writer/reader. - Add warning about messageIDs - Signing policy + optional Signature, From and Seqno ([libp2p/go-libp2p-pubsub#359](https://github.com/libp2p/go-libp2p-pubsub/pull/359)) - Update pubsub.go - Define a public error ErrSubscriptionCancelled. - only do PX on leave if PX was enabled in the node - drop warning about failure to open stream to a debug log - reinstate tagging (now protection) tests - disable tests for direct/mesh tags, we don't have an interface to query the connman yet - protect direct and mesh peers in the connection manager - feat: add direct connect ticks option - github.com/libp2p/go-libp2p-pubsub-router (v0.3.0 -> v0.3.2): - upgrade deps + interoperable uvarint delimited writer/reader. (#79) ([libp2p/go-libp2p-pubsub-router#79](https://github.com/libp2p/go-libp2p-pubsub-router/pull/79)) - github.com/libp2p/go-libp2p-quic-transport (v0.6.0 -> v0.8.0): - update quic-go to v0.18.0 (#171) ([libp2p/go-libp2p-quic-transport#171](https://github.com/libp2p/go-libp2p-quic-transport/pull/171)) - github.com/libp2p/go-libp2p-swarm (v0.2.6 -> v0.2.8): - slim down dependencies ([libp2p/go-libp2p-swarm#225](https://github.com/libp2p/go-libp2p-swarm/pull/225)) - `ID()` method on connections and streams + record opening time (#224) ([libp2p/go-libp2p-swarm#224](https://github.com/libp2p/go-libp2p-swarm/pull/224)) - github.com/libp2p/go-libp2p-testing (v0.1.1 -> v0.2.0): - Add net benchmark harness ([libp2p/go-libp2p-testing#21](https://github.com/libp2p/go-libp2p-testing/pull/21)) - Update suite to check that streams respect mux.ErrReset. ([libp2p/go-libp2p-testing#16](https://github.com/libp2p/go-libp2p-testing/pull/16)) - github.com/libp2p/go-maddr-filter (v0.0.5 -> v0.1.0): - deprecate this package; moved to multiformats/go-multiaddr. (#23) ([libp2p/go-maddr-filter#23](https://github.com/libp2p/go-maddr-filter/pull/23)) - chore(dep): update ([libp2p/go-maddr-filter#18](https://github.com/libp2p/go-maddr-filter/pull/18)) - github.com/libp2p/go-msgio (v0.0.4 -> v0.0.6): - interoperable uvarints. (#21) ([libp2p/go-msgio#21](https://github.com/libp2p/go-msgio/pull/21)) - upgrade deps + interoperable uvarint delimited writer/reader. (#20) ([libp2p/go-msgio#20](https://github.com/libp2p/go-msgio/pull/20)) - github.com/libp2p/go-netroute (v0.1.2 -> v0.1.3): - add Plan 9 support - github.com/libp2p/go-openssl (v0.0.5 -> v0.0.7): - make ed25519 less special ([libp2p/go-openssl#7](https://github.com/libp2p/go-openssl/pull/7)) - Add required bindings to support openssl in libp2p-tls ([libp2p/go-openssl#6](https://github.com/libp2p/go-openssl/pull/6)) - github.com/libp2p/go-reuseport (v0.0.1 -> v0.0.2): - Fix build on Plan 9 ([libp2p/go-reuseport#79](https://github.com/libp2p/go-reuseport/pull/79)) - farewell gx; thanks for serving us well. - update readme badges - remove Jenkinsfile. - github.com/libp2p/go-reuseport-transport (v0.0.3 -> v0.0.4): - Update go-netroute and go-reuseport for Plan 9 support - Fix build on Plan 9 - github.com/lucas-clemente/quic-go (v0.16.2 -> v0.18.0): - create a milestone version for v0.18.x - add Changelog entries for v0.17 ([lucas-clemente/quic-go#2726](https://github.com/lucas-clemente/quic-go/pull/2726)) - regenerate the testdata certificate with SAN instead of CommonName ([lucas-clemente/quic-go#2723](https://github.com/lucas-clemente/quic-go/pull/2723)) - make it possible to use multiple qtls versions at the same time, add support for Go 1.15 ([lucas-clemente/quic-go#2720](https://github.com/lucas-clemente/quic-go/pull/2720)) - add fuzzing for transport parameters ([lucas-clemente/quic-go#2713](https://github.com/lucas-clemente/quic-go/pull/2713)) - run golangci-lint on GitHub Actions ([lucas-clemente/quic-go#2700](https://github.com/lucas-clemente/quic-go/pull/2700)) - disallow values above 2^60 for Config.MaxIncoming{Uni}Streams ([lucas-clemente/quic-go#2711](https://github.com/lucas-clemente/quic-go/pull/2711)) - never send a value larger than 2^60 in MAX_STREAMS frames ([lucas-clemente/quic-go#2710](https://github.com/lucas-clemente/quic-go/pull/2710)) - run the check for go generated files on GitHub Actions instead of Travis ([lucas-clemente/quic-go#2703](https://github.com/lucas-clemente/quic-go/pull/2703)) - update QUIC draft version information in README ([lucas-clemente/quic-go#2715](https://github.com/lucas-clemente/quic-go/pull/2715)) - remove Fuzzit badge from README ([lucas-clemente/quic-go#2714](https://github.com/lucas-clemente/quic-go/pull/2714)) - use the correct return values in Fuzz() functions ([lucas-clemente/quic-go#2705](https://github.com/lucas-clemente/quic-go/pull/2705)) - simplify the connection, rename it to sendConn ([lucas-clemente/quic-go#2707](https://github.com/lucas-clemente/quic-go/pull/2707)) - update qpack to v0.2.0 ([lucas-clemente/quic-go#2704](https://github.com/lucas-clemente/quic-go/pull/2704)) - remove redundant error check in the stream ([lucas-clemente/quic-go#2718](https://github.com/lucas-clemente/quic-go/pull/2718)) - put back the packet buffer when parsing the connection ID fails ([lucas-clemente/quic-go#2708](https://github.com/lucas-clemente/quic-go/pull/2708)) - update fuzzing code for oss-fuzz ([lucas-clemente/quic-go#2702](https://github.com/lucas-clemente/quic-go/pull/2702)) - fix travis script ([lucas-clemente/quic-go#2701](https://github.com/lucas-clemente/quic-go/pull/2701)) - remove Fuzzit from Travis config ([lucas-clemente/quic-go#2699](https://github.com/lucas-clemente/quic-go/pull/2699)) - add a script to check if go generated files are correct ([lucas-clemente/quic-go#2692](https://github.com/lucas-clemente/quic-go/pull/2692)) - only arm the application data PTO timer after the handshake is confirmed ([lucas-clemente/quic-go#2689](https://github.com/lucas-clemente/quic-go/pull/2689)) - fix tracing of congestion state updates ([lucas-clemente/quic-go#2691](https://github.com/lucas-clemente/quic-go/pull/2691)) - fix reading of flag values in integration tests ([lucas-clemente/quic-go#2690](https://github.com/lucas-clemente/quic-go/pull/2690)) - remove ACK decimation ([lucas-clemente/quic-go#2599](https://github.com/lucas-clemente/quic-go/pull/2599)) - add a metric for PTOs ([lucas-clemente/quic-go#2686](https://github.com/lucas-clemente/quic-go/pull/2686)) - remove the H3_EARLY_RESPONSE error ([lucas-clemente/quic-go#2687](https://github.com/lucas-clemente/quic-go/pull/2687)) - implement tracing for congestion state changes ([lucas-clemente/quic-go#2684](https://github.com/lucas-clemente/quic-go/pull/2684)) - remove the N connection simulation from the Reno code ([lucas-clemente/quic-go#2682](https://github.com/lucas-clemente/quic-go/pull/2682)) - remove the SSLR (slow start large reduction) experiment ([lucas-clemente/quic-go#2680](https://github.com/lucas-clemente/quic-go/pull/2680)) - remove unused connectionStats counters from the Reno implementation ([lucas-clemente/quic-go#2683](https://github.com/lucas-clemente/quic-go/pull/2683)) - add an integration test that randomly sets tracers ([lucas-clemente/quic-go#2679](https://github.com/lucas-clemente/quic-go/pull/2679)) - privatize some methods in the congestion controller package ([lucas-clemente/quic-go#2681](https://github.com/lucas-clemente/quic-go/pull/2681)) - fix out-of-bounds read when creating a multiplexed tracer ([lucas-clemente/quic-go#2678](https://github.com/lucas-clemente/quic-go/pull/2678)) - run integration tests with qlog and metrics on CircleCI ([lucas-clemente/quic-go#2677](https://github.com/lucas-clemente/quic-go/pull/2677)) - add a metric for closed connections ([lucas-clemente/quic-go#2676](https://github.com/lucas-clemente/quic-go/pull/2676)) - trace packets that are sent outside of a connection ([lucas-clemente/quic-go#2675](https://github.com/lucas-clemente/quic-go/pull/2675)) - trace dropped packets that are dropped before they are passed to any session ([lucas-clemente/quic-go#2670](https://github.com/lucas-clemente/quic-go/pull/2670)) - add a metric for sent packets ([lucas-clemente/quic-go#2673](https://github.com/lucas-clemente/quic-go/pull/2673)) - add a metric for lost packets ([lucas-clemente/quic-go#2672](https://github.com/lucas-clemente/quic-go/pull/2672)) - simplify the Tracer interface by combining the TracerFor... methods ([lucas-clemente/quic-go#2671](https://github.com/lucas-clemente/quic-go/pull/2671)) - add a metrics package using OpenCensus, trace connections ([lucas-clemente/quic-go#2646](https://github.com/lucas-clemente/quic-go/pull/2646)) - add a multiplexer for the tracer ([lucas-clemente/quic-go#2665](https://github.com/lucas-clemente/quic-go/pull/2665)) - introduce a type for stateless reset tokens ([lucas-clemente/quic-go#2668](https://github.com/lucas-clemente/quic-go/pull/2668)) - log all reasons why a connection is closed ([lucas-clemente/quic-go#2669](https://github.com/lucas-clemente/quic-go/pull/2669)) - add integration tests using faulty packet conns ([lucas-clemente/quic-go#2663](https://github.com/lucas-clemente/quic-go/pull/2663)) - don't block sendQueue.Send() if the runloop already exited. ([lucas-clemente/quic-go#2656](https://github.com/lucas-clemente/quic-go/pull/2656)) - move the SupportedVersions slice out of the wire.Header ([lucas-clemente/quic-go#2664](https://github.com/lucas-clemente/quic-go/pull/2664)) - add a flag to disable conn ID generation and the check for retired conn IDs ([lucas-clemente/quic-go#2660](https://github.com/lucas-clemente/quic-go/pull/2660)) - put the session in the packet handler map directly (for client sessions) ([lucas-clemente/quic-go#2667](https://github.com/lucas-clemente/quic-go/pull/2667)) - don't send write error in CONNECTION_CLOSE frames ([lucas-clemente/quic-go#2666](https://github.com/lucas-clemente/quic-go/pull/2666)) - reset the PTO count before setting the timer when dropping a PN space ([lucas-clemente/quic-go#2657](https://github.com/lucas-clemente/quic-go/pull/2657)) - enforce that a connection ID is not retired in a packet that uses that connection ID ([lucas-clemente/quic-go#2651](https://github.com/lucas-clemente/quic-go/pull/2651)) - don't retire the conn ID that's in use when receiving a retransmission ([lucas-clemente/quic-go#2652](https://github.com/lucas-clemente/quic-go/pull/2652)) - fix flaky cancellation integration test ([lucas-clemente/quic-go#2649](https://github.com/lucas-clemente/quic-go/pull/2649)) - fix crash when the qlog callbacks returns a nil io.WriteCloser ([lucas-clemente/quic-go#2648](https://github.com/lucas-clemente/quic-go/pull/2648)) - fix flaky server test on Travis ([lucas-clemente/quic-go#2645](https://github.com/lucas-clemente/quic-go/pull/2645)) - fix a typo in the logging package test suite - introduce type aliases in the logging package ([lucas-clemente/quic-go#2643](https://github.com/lucas-clemente/quic-go/pull/2643)) - rename frame fields to the names used in the draft ([lucas-clemente/quic-go#2644](https://github.com/lucas-clemente/quic-go/pull/2644)) - split the qlog package into a logging and a qlog package, use a tracer interface in the quic.Config ([lucas-clemente/quic-go#2638](https://github.com/lucas-clemente/quic-go/pull/2638)) - fix HTTP request writing if the Request.Body reads data and returns EOF ([lucas-clemente/quic-go#2642](https://github.com/lucas-clemente/quic-go/pull/2642)) - handle Version Negotiation packets in the session ([lucas-clemente/quic-go#2640](https://github.com/lucas-clemente/quic-go/pull/2640)) - increase the packet size of the client's Initial packet ([lucas-clemente/quic-go#2634](https://github.com/lucas-clemente/quic-go/pull/2634)) - introduce an assertion in the server ([lucas-clemente/quic-go#2637](https://github.com/lucas-clemente/quic-go/pull/2637)) - use the new qtls interface for (re)storing app data with a session state ([lucas-clemente/quic-go#2631](https://github.com/lucas-clemente/quic-go/pull/2631)) - remove buffering of HTTP requests ([lucas-clemente/quic-go#2626](https://github.com/lucas-clemente/quic-go/pull/2626)) - remove superfluous parameters logged when not doing 0-RTT ([lucas-clemente/quic-go#2632](https://github.com/lucas-clemente/quic-go/pull/2632)) - return an infinite bandwidth if the RTT is zero ([lucas-clemente/quic-go#2636](https://github.com/lucas-clemente/quic-go/pull/2636)) - drop support for Go 1.13 ([lucas-clemente/quic-go#2628](https://github.com/lucas-clemente/quic-go/pull/2628)) - remove superfluous handleResetStreamFrame method on the stream ([lucas-clemente/quic-go#2623](https://github.com/lucas-clemente/quic-go/pull/2623)) - implement a token-bucket pacing algorithm ([lucas-clemente/quic-go#2615](https://github.com/lucas-clemente/quic-go/pull/2615)) - gracefully handle concurrent stream writes and cancellations ([lucas-clemente/quic-go#2624](https://github.com/lucas-clemente/quic-go/pull/2624)) - log sent packets right before sending them out ([lucas-clemente/quic-go#2613](https://github.com/lucas-clemente/quic-go/pull/2613)) - remove unused packet counter in the receivedPacketTracker ([lucas-clemente/quic-go#2611](https://github.com/lucas-clemente/quic-go/pull/2611)) - rewrite the proxy to avoid packet reordering ([lucas-clemente/quic-go#2617](https://github.com/lucas-clemente/quic-go/pull/2617)) - fix flaky INVALID_TOKEN integration test ([lucas-clemente/quic-go#2610](https://github.com/lucas-clemente/quic-go/pull/2610)) - make DialEarly return EarlySession ([lucas-clemente/quic-go#2621](https://github.com/lucas-clemente/quic-go/pull/2621)) - add debug logging to the packet handler map ([lucas-clemente/quic-go#2608](https://github.com/lucas-clemente/quic-go/pull/2608)) - increase the minimum pacing delay to 1ms ([lucas-clemente/quic-go#2605](https://github.com/lucas-clemente/quic-go/pull/2605)) - github.com/marten-seemann/qpack (v0.1.0 -> v0.2.0): - don't reuse the encoder in the integration tests ([marten-seemann/qpack#18](https://github.com/marten-seemann/qpack/pull/18)) - use Huffman encoding for field names and values ([marten-seemann/qpack#16](https://github.com/marten-seemann/qpack/pull/16)) - add more tests for encoding using the static table ([marten-seemann/qpack#15](https://github.com/marten-seemann/qpack/pull/15)) - Encoder uses the static table. ([marten-seemann/qpack#10](https://github.com/marten-seemann/qpack/pull/10)) - add gofmt to golangci-lint - update qifs to the current version ([marten-seemann/qpack#14](https://github.com/marten-seemann/qpack/pull/14)) - use golangci-lint for linting ([marten-seemann/qpack#12](https://github.com/marten-seemann/qpack/pull/12)) - add fuzzing ([marten-seemann/qpack#9](https://github.com/marten-seemann/qpack/pull/9)) - update qifs - use https protocol for submodule clone ([marten-seemann/qpack#7](https://github.com/marten-seemann/qpack/pull/7)) - github.com/marten-seemann/qtls (v0.9.1 -> v0.10.0): - add callbacks to store and restore app data along a session state - remove support for Go 1.13 - github.com/marten-seemann/qtls-go1-15 (null -> v0.1.0): - use a prefix for client session cache keys - add callbacks to store and restore app data along a session state - don't use TLS 1.3 compatibility mode when using alternative record layer - delete the session ticket after attempting 0-RTT - reject 0-RTT when a different ALPN is chosen - encode the ALPN into the session ticket - add a field to the ConnectionState to tell if 0-RTT was used - add a callback to tell the client about rejection of 0-RTT - don't offer 0-RTT after a HelloRetryRequest - add Accept0RTT to Config callback to decide if 0-RTT should be accepted - add the option to encode application data into the session ticket - export the 0-RTT write key - abuse the nonce field of ClientSessionState to save max_early_data_size - export the 0-RTT read key - close connection if client attempts 0-RTT, but ticket didn't allow it - encode the max early data size into the session ticket - implement parsing of the early_data extension in the EncryptedExtensions - add a tls.Config.MaxEarlyData option to enable 0-RTT - accept TLS 1.3 cipher suites in Config.CipherSuites - introduce a function on the connection to generate a session ticket - add a config option to enforce selection of an application protocol - export Conn.HandlePostHandshakeMessage - export Alert - reject Configs that set MaxVersion < 1.3 when using a record layer - enforce TLS 1.3 when using an alternative record layer - github.com/multiformats/go-multiaddr (v0.2.2 -> v0.3.1): - dep: add "codependencies" for handling version conflicts ([multiformats/go-multiaddr#132](https://github.com/multiformats/go-multiaddr/pull/132)) - Support /p2p addresses encoded as CIDs ([multiformats/go-multiaddr#130](https://github.com/multiformats/go-multiaddr/pull/130)) - Merge go-multiaddr-net - github.com/multiformats/go-multiaddr-net (v0.1.5 -> v0.2.0): - Deprecate ([multiformats/go-multiaddr-net#72](https://github.com/multiformats/go-multiaddr-net/pull/72)) - github.com/multiformats/go-multihash (v0.0.13 -> v0.0.14): - fix: only register one blake2s length ([multiformats/go-multihash#129](https://github.com/multiformats/go-multihash/pull/129)) - feat: add two filecoin hashes, without Sum() implementations ([multiformats/go-multihash#128](https://github.com/multiformats/go-multihash/pull/128)) - feat: reduce blake2b allocations by special-casing the 256/512 variants ([multiformats/go-multihash#126](https://github.com/multiformats/go-multihash/pull/126)) - github.com/multiformats/go-multistream (v0.1.1 -> v0.1.2): - upgrade deps + interoperable varints. (#51) ([multiformats/go-multistream#51](https://github.com/multiformats/go-multistream/pull/51)) - github.com/multiformats/go-varint (v0.0.5 -> v0.0.6): - fix minor interoperability issues. (#6) ([multiformats/go-varint#6](https://github.com/multiformats/go-varint/pull/6)) - github.com/warpfork/go-wish (v0.0.0-20190328234359-8b3e70f8e830 -> v0.0.0-20200122115046-b9ea61034e4a): - Add ShouldBeSameTypeAs checker. - Integration test update for go versions. - github.com/whyrusleeping/cbor-gen (v0.0.0-20200123233031-1cdf64d27158 -> v0.0.0-20200402171437-3d27c146c105): - Handle Nil values for cbg.Deferred ([whyrusleeping/cbor-gen#14](https://github.com/whyrusleeping/cbor-gen/pull/14)) - add name of struct field to error messages - Support uint64 pointers ([whyrusleeping/cbor-gen#13](https://github.com/whyrusleeping/cbor-gen/pull/13)) - int64 support in map encoders ([whyrusleeping/cbor-gen#12](https://github.com/whyrusleeping/cbor-gen/pull/12)) - Fix uint64 typed array gen ([whyrusleeping/cbor-gen#10](https://github.com/whyrusleeping/cbor-gen/pull/10)) - Fix cbg self referencing import path ([whyrusleeping/cbor-gen#8](https://github.com/whyrusleeping/cbor-gen/pull/8)) ### Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Marten Seemann | 156 | +16428/-42621 | 979 | | hannahhoward | 42 | +15132/-9819 | 467 | | Eric Myhre | 114 | +13709/-6898 | 586 | | Steven Allen | 55 | +1211/-2714 | 95 | | Adin Schmahmann | 54 | +1660/-783 | 117 | | Petar Maymounkov | 23 | +1677/-671 | 75 | | Aarsh Shah | 10 | +1926/-341 | 39 | | Raúl Kripalani | 17 | +1134/-537 | 53 | | Will | 1 | +841/-0 | 9 | | rendaw | 3 | +425/-195 | 12 | | Will Scott | 8 | +302/-229 | 15 | | vyzo | 22 | +345/-166 | 23 | | Fazlul Shahriar | 7 | +452/-44 | 19 | | Peter Rabbitson | 1 | +353/-118 | 5 | | Hector Sanjuan | 10 | +451/-3 | 14 | | Marcin Rataj | 9 | +298/-106 | 16 | | Łukasz Magiera | 4 | +329/-51 | 12 | | RubenKelevra | 9 | +331/-7 | 12 | | Michael Muré | 2 | +259/-69 | 6 | | jstordeur | 1 | +252/-2 | 5 | | Diederik Loerakker | 1 | +168/-35 | 7 | | Tiger | 3 | +138/-52 | 8 | | Kevin Neaton | 3 | +103/-21 | 9 | | Rod Vagg | 1 | +50/-40 | 4 | | Oli Evans | 4 | +60/-9 | 6 | | achingbrain | 4 | +30/-30 | 5 | | Cyril Fougeray | 2 | +34/-24 | 2 | | Luke Tucker | 1 | +31/-1 | 2 | | sandman | 2 | +23/-7 | 3 | | Alan Shaw | 1 | +18/-9 | 2 | | Jacob Heun | 4 | +13/-3 | 4 | | Jessica Schilling | 3 | +7/-7 | 3 | | Rafael Ramalho | 4 | +9/-4 | 4 | | Jeromy Johnson | 2 | +6/-6 | 4 | | Nick Cabatoff | 1 | +7/-2 | 1 | | Stephen Solka | 1 | +1/-7 | 1 | | Preston Van Loon | 2 | +6/-2 | 2 | | Jakub Sztandera | 2 | +5/-2 | 2 | | llx | 1 | +3/-3 | 1 | | Adrian Lanzafame | 1 | +3/-3 | 1 | | Yusef Napora | 1 | +3/-2 | 1 | | Louis Thibault | 1 | +5/-0 | 1 | | Martín Triay | 1 | +4/-0 | 1 | | Hlib | 1 | +2/-2 | 1 | | Shotaro Yamada | 1 | +2/-1 | 1 | | phuslu | 1 | +1/-1 | 1 | | Zero King | 1 | +1/-1 | 1 | | Rüdiger Klaehn | 1 | +2/-0 | 1 | | Nex | 1 | +1/-1 | 1 | | Mark Gaiser | 1 | +1/-1 | 1 | | Luflosi | 1 | +1/-1 | 1 | | David Florness | 1 | +1/-1 | 1 | | Dean Eigenmann | 1 | +0/-1 | 1 | ================================================ FILE: docs/changelogs/v0.8.md ================================================ # go-ipfs changelog v0.8 ## v0.8.0 2021-02-18 We're happy to announce go-ipfs 0.8.0! This is planned to be a fairly small release focused on integrating in the new pinning service/remote pinning [API](https://github.com/ipfs/pinning-services-api-spec) that makes the experience of managing pins across pinning services easier and more uniform. ### 🔦 Highlights #### 🧷 Remote pinning services There is now support for asking remote services to pin data for you. This means anyone can implement the [spec](https://ipfs.github.io/pinning-services-api-spec/) (developed in this [repo](https://github.com/ipfs/pinning-services-api-spec)) and allow for pin management. All of the CLI (and corresponding HTTP API) commands are available under `ipfs pin remote`. This remote pinning service comes with a redesign of how we're thinking about pinning and includes some commonly requested features such as: - Pins can have names (and coming soon metadata) - The same content can be pinned multiple times, but of course stored only once - This allows applications using the same pinning service to manage their own pins without worrying about removing content important to another application - Data can be pinned in either the foreground or background Examples include: ``` ipfs pin remote service add myservice https://myservice.tld:1234/api/path myaccess key ipfs pin remote add /ipfs/bafymydata --service=myservice --name=myfile ipfs pin remote ls --service=myservice --name=myfile ipfs pin remote ls --service=myservice --cid=bafymydata ipfs pin remote rm --service=myservice --name=myfile ``` A few notes: Remote pinning services work with recursive pins. This means commands like `ipfs pin remote ls` will not list indirectly pinned CIDs. While pinning service data is stored in the configuration file it cannot be edited directly via the `ipfs config` commands due to the sensitive nature of pinning service API keys. The `ipfs pin remote service` commands can be used for interacting with remote service settings. #### 📌 Faster local pinning and unpinning The pinning subsystem has been redesigned to be much faster and more flexible in how it tracks pins. For users who are working with many pins this will lead to a big speed increase in listing and modifying the set of pinned items as well as decreased memory usage. Part of the redesign was setup to account for being able to interact with local pins the same way we can now interact with remote pins (e.g. names, being allowed to pin the same CID multiple times, etc.). Keep posted for more improvements to pinning. #### DNSLink names on https:// subdomains Previously DNSLink names would have trouble loading over subdomain gateways with HTTPS support since there is no way to get multilevel wildcard certificates (e.g. `en.wikipedia-on-ipfs.org.ipns.dweb.link` cannot be covered by `*.ipns.dweb.link`). Therefore, when trying to load DNSLink names over https:// subdomains go-ipfs we now forward to an encoded DNS name. Since DNS names cannot contain `.` in them they are escaped using `-`. `/ipns/en.wikipedia-on-ipfs.org` → `ipns://en.wikipedia-on-ipfs.org` → `https://dweb.link/ipns/en.wikipedia-on-ipfs.org` `https://en-wikipedia--on--ipfs-org.ipns.dweb.link` :point_left: _a single DNS label, no TLS error_ #### QUIC update QUIC support has received a number of upgrades, including the ability to take advantage of larger UDP receive buffers for increased performance. Linux users may notice a logged error on daemon startup if your system needs extra configuration to allow IPFS increase the buffer size. A helpful link for resolving this is in the log message as well as [here](https://github.com/lucas-clemente/quic-go/wiki/UDP-Receive-Buffer-Size). #### 👋 No more Darwin 386 builds Go 1.15 (the latest version of Go) [no longer supports](https://github.com/golang/go/issues/34749) Darwin 386 and so we are dropping support as well. ### Changelog - github.com/ipfs/go-ipfs: - Release v0.8.0 - docs: RepinInterval - style: docs/config.md - style: improved MFS PinName example - docs: Pinning.RemoteServices.Policies - fix: decrease log level of opencensus initialization ([ipfs/go-ipfs#7815](https://github.com/ipfs/go-ipfs/pull/7815)) - Register oc metrics ([ipfs/go-ipfs#7593](https://github.com/ipfs/go-ipfs/pull/7593)) - add remote pinning to ipfs command (#7661) ([ipfs/go-ipfs#7661](https://github.com/ipfs/go-ipfs/pull/7661)) - More p2p proxy checks ([ipfs/go-ipfs#7797](https://github.com/ipfs/go-ipfs/pull/7797)) - Use datastore based pinning ([ipfs/go-ipfs#7750](https://github.com/ipfs/go-ipfs/pull/7750)) - fix: return an error when an unknown object type is passed ([ipfs/go-ipfs#7795](https://github.com/ipfs/go-ipfs/pull/7795)) - clarify why ipfs file ls is being deprecated ([ipfs/go-ipfs#7755](https://github.com/ipfs/go-ipfs/pull/7755)) - fix: ipfs dag export uses the CoreAPI and respects the offline flag ([ipfs/go-ipfs#7753](https://github.com/ipfs/go-ipfs/pull/7753)) - return an error when trying to download fs-repo-migrations for linux + musl ([ipfs/go-ipfs#7735](https://github.com/ipfs/go-ipfs/pull/7735)) - fix: do not create a new (unused) peerID when initializing from config ([ipfs/go-ipfs#7730](https://github.com/ipfs/go-ipfs/pull/7730)) - docs: Add a link in config.md ([ipfs/go-ipfs#7780](https://github.com/ipfs/go-ipfs/pull/7780)) - update libp2p for stream closure refactor ([ipfs/go-ipfs#7747](https://github.com/ipfs/go-ipfs/pull/7747)) - Fix typo in ipfs dag stat command ([ipfs/go-ipfs#7761](https://github.com/ipfs/go-ipfs/pull/7761)) - docs(readme): key rotation in docker (#7721) ([ipfs/go-ipfs#7721](https://github.com/ipfs/go-ipfs/pull/7721)) - fix(dnslink-gw): breadcrumbs and CID column when dir listing ([ipfs/go-ipfs#7699](https://github.com/ipfs/go-ipfs/pull/7699)) - fix(gw): preserve query on website redirect ([ipfs/go-ipfs#7727](https://github.com/ipfs/go-ipfs/pull/7727)) - feat: ipfs-webui v2.11.4 ([ipfs/go-ipfs#7716](https://github.com/ipfs/go-ipfs/pull/7716)) - docs: how the ipfs snap is built and published ([ipfs/go-ipfs#7725](https://github.com/ipfs/go-ipfs/pull/7725)) - fix: webui on ipv6 localhost ([ipfs/go-ipfs#7731](https://github.com/ipfs/go-ipfs/pull/7731)) - Add missing plugin support on FreeBSD ([ipfs/go-ipfs#7722](https://github.com/ipfs/go-ipfs/pull/7722)) - fix error when computing coverage ([ipfs/go-ipfs#7726](https://github.com/ipfs/go-ipfs/pull/7726)) - docs(config): X-Forwarded-Host ([ipfs/go-ipfs#7651](https://github.com/ipfs/go-ipfs/pull/7651)) - chore: webui v2.11.2 ([ipfs/go-ipfs#7703](https://github.com/ipfs/go-ipfs/pull/7703)) - Add task for updating CLI docs right after updating the HTTP-api docs ([ipfs/go-ipfs#7711](https://github.com/ipfs/go-ipfs/pull/7711)) - feat(gateway): Content-Disposition improvements ([ipfs/go-ipfs#7677](https://github.com/ipfs/go-ipfs/pull/7677)) - fix build on Plan 9 ([ipfs/go-ipfs#7690](https://github.com/ipfs/go-ipfs/pull/7690)) - docs: update changelog for v0.7.0 - chore: bump webui version - fix: remove the (empty) alias for --peerid-base - fix: use override GOFLAGS changes from 480defab689610550ee3d346e31441a2bb881fcb but keep trimpath usage as is - Revert "fix: override GOFLAGS" - Fix --ipns-base alias ([ipfs/go-ipfs#7659](https://github.com/ipfs/go-ipfs/pull/7659)) - docs: update config to indicate SECIO deprecation ([ipfs/go-ipfs#7630](https://github.com/ipfs/go-ipfs/pull/7630)) - fix: ipfs dht put/get commands with peerIDs encoded as CIDs ([ipfs/go-ipfs#7633](https://github.com/ipfs/go-ipfs/pull/7633)) - update version to 0.8.0-dev ([ipfs/go-ipfs#7629](https://github.com/ipfs/go-ipfs/pull/7629)) - github.com/ipfs/go-bitswap (v0.2.20 -> v0.3.3): - feat: configurable engine blockstore worker count (#449) ([ipfs/go-bitswap#449](https://github.com/ipfs/go-bitswap/pull/449)) - fix: set the score ledger on start ([ipfs/go-bitswap#447](https://github.com/ipfs/go-bitswap/pull/447)) - feat: update for go-libp2p-core 0.7.0 interface changes ([ipfs/go-bitswap#445](https://github.com/ipfs/go-bitswap/pull/445)) - fix: guard access to the mock wiretap with a lock ([ipfs/go-bitswap#446](https://github.com/ipfs/go-bitswap/pull/446)) - Add WireTap interface (#444) ([ipfs/go-bitswap#444](https://github.com/ipfs/go-bitswap/pull/444)) - Fix: Increment stats.MessagesSent in msgToStream() function (#441) ([ipfs/go-bitswap#441](https://github.com/ipfs/go-bitswap/pull/441)) - refactor: remove extraneous ledger field init (#437) ([ipfs/go-bitswap#437](https://github.com/ipfs/go-bitswap/pull/437)) - Added `WithScoreLedger` Bitswap option (#430) ([ipfs/go-bitswap#430](https://github.com/ipfs/go-bitswap/pull/430)) - github.com/ipfs/go-blockservice (v0.1.3 -> v0.1.4): - Avoid modifying passed in slice of cids ([ipfs/go-blockservice#65](https://github.com/ipfs/go-blockservice/pull/65)) - github.com/ipfs/go-ds-badger (v0.2.4 -> v0.2.6): - Log error if batch not committed or canceled ([ipfs/go-ds-badger#108](https://github.com/ipfs/go-ds-badger/pull/108)) - Add Cancel function; add finalizer to cleanup abandoned batch ([ipfs/go-ds-badger#105](https://github.com/ipfs/go-ds-badger/pull/105)) - Do not implement batches using transactions ([ipfs/go-ds-badger#104](https://github.com/ipfs/go-ds-badger/pull/104)) - readme: add information on Badger2 datastore ([ipfs/go-ds-badger#102](https://github.com/ipfs/go-ds-badger/pull/102)) - update contributing link ([ipfs/go-ds-badger#91](https://github.com/ipfs/go-ds-badger/pull/91)) - Use current go-log (#89) ([ipfs/go-ds-badger#89](https://github.com/ipfs/go-ds-badger/pull/89)) - github.com/ipfs/go-graphsync (v0.1.1 -> v0.6.0): - docs(CHANGELOG): revise for 0.6.0 - Merge branch 'master' into release/v0.6.0 - docs(CHANGELOG): update for 0.6.0 release - move block allocation into message queue (#140) ([ipfs/go-graphsync#140](https://github.com/ipfs/go-graphsync/pull/140)) - Response Assembler Refactor (#138) ([ipfs/go-graphsync#138](https://github.com/ipfs/go-graphsync/pull/138)) - Add error listener on receiver (#136) ([ipfs/go-graphsync#136](https://github.com/ipfs/go-graphsync/pull/136)) - Run testplan on in CI (#137) ([ipfs/go-graphsync#137](https://github.com/ipfs/go-graphsync/pull/137)) - fix(responsemanager): fix network error propagation (#133) ([ipfs/go-graphsync#133](https://github.com/ipfs/go-graphsync/pull/133)) - testground test for graphsync (#132) ([ipfs/go-graphsync#132](https://github.com/ipfs/go-graphsync/pull/132)) - docs(CHANGELOG): update for v0.5.2 ([ipfs/go-graphsync#130](https://github.com/ipfs/go-graphsync/pull/130)) - RegisterNetworkErrorListener should fire when there's an error connecting to the peer (#127) ([ipfs/go-graphsync#127](https://github.com/ipfs/go-graphsync/pull/127)) - Permit multiple data subscriptions per original topic (#128) ([ipfs/go-graphsync#128](https://github.com/ipfs/go-graphsync/pull/128)) - release: v0.5.1 (#123) ([ipfs/go-graphsync#123](https://github.com/ipfs/go-graphsync/pull/123)) - feat(responsemanager): allow configuration of max requests (#122) ([ipfs/go-graphsync#122](https://github.com/ipfs/go-graphsync/pull/122)) - docs(CHANGELOG): update for 0.5.0 ([ipfs/go-graphsync#120](https://github.com/ipfs/go-graphsync/pull/120)) - feat: use go-libp2p-core 0.7.0 stream interfaces (#116) ([ipfs/go-graphsync#116](https://github.com/ipfs/go-graphsync/pull/116)) - Merge branch 'release/v0.4.3' - chore(benchmarks): remove extra files - fix(peerresponsemanager): avoid race condition that could result in NPE in link tracker (#118) ([ipfs/go-graphsync#118](https://github.com/ipfs/go-graphsync/pull/118)) - docs(CHANGELOG): update for 0.4.2 ([ipfs/go-graphsync#117](https://github.com/ipfs/go-graphsync/pull/117)) - feat(memory): improve memory usage (#110) ([ipfs/go-graphsync#110](https://github.com/ipfs/go-graphsync/pull/110)) - fix(notifications): fix lock in close (#115) ([ipfs/go-graphsync#115](https://github.com/ipfs/go-graphsync/pull/115)) - docs(CHANGELOG): update for v0.4.1 ([ipfs/go-graphsync#114](https://github.com/ipfs/go-graphsync/pull/114)) - fix(allocator): remove peer from peer status list - docs(CHANGELOG): update for v0.4.0 - docs(CHANGELOG): update for 0.3.1 ([ipfs/go-graphsync#112](https://github.com/ipfs/go-graphsync/pull/112)) - Add allocator for memory backpressure (#108) ([ipfs/go-graphsync#108](https://github.com/ipfs/go-graphsync/pull/108)) - Shutdown notifications go routines (#109) ([ipfs/go-graphsync#109](https://github.com/ipfs/go-graphsync/pull/109)) - Switch to google protobuf generator (#105) ([ipfs/go-graphsync#105](https://github.com/ipfs/go-graphsync/pull/105)) - feat(CHANGELOG): update for 0.3.0 ([ipfs/go-graphsync#104](https://github.com/ipfs/go-graphsync/pull/104)) - docs(CHANGELOG): update for 0.2.1 ([ipfs/go-graphsync#103](https://github.com/ipfs/go-graphsync/pull/103)) - Track actual network operations in a response (#102) ([ipfs/go-graphsync#102](https://github.com/ipfs/go-graphsync/pull/102)) - feat(responsecache): prune blocks more intelligently (#101) ([ipfs/go-graphsync#101](https://github.com/ipfs/go-graphsync/pull/101)) - Release/0.2.0 ([ipfs/go-graphsync#99](https://github.com/ipfs/go-graphsync/pull/99)) - fix(metadata): fix cbor-gen (#98) ([ipfs/go-graphsync#98](https://github.com/ipfs/go-graphsync/pull/98)) - fix(selectorvalidator): memory optimization (#97) ([ipfs/go-graphsync#97](https://github.com/ipfs/go-graphsync/pull/97)) - Update go-ipld-prime@v0.5.0 (#92) ([ipfs/go-graphsync#92](https://github.com/ipfs/go-graphsync/pull/92)) - refactor(metadata): use cbor-gen encoding (#96) ([ipfs/go-graphsync#96](https://github.com/ipfs/go-graphsync/pull/96)) - Release/v0.1.2 ([ipfs/go-graphsync#95](https://github.com/ipfs/go-graphsync/pull/95)) - Return Request context canceled error (#93) ([ipfs/go-graphsync#93](https://github.com/ipfs/go-graphsync/pull/93)) - feat(benchmarks): add p2p stress test (#91) ([ipfs/go-graphsync#91](https://github.com/ipfs/go-graphsync/pull/91)) - Benchmark framework + First memory fixes (#89) ([ipfs/go-graphsync#89](https://github.com/ipfs/go-graphsync/pull/89)) - docs(CHANGELOG): update for v0.1.1 ([ipfs/go-graphsync#85](https://github.com/ipfs/go-graphsync/pull/85)) - github.com/ipfs/go-ipfs-cmds (v0.4.0 -> v0.6.0): - Added DelimitedStringsOption for enabling delimited strings on the CLI ([ipfs/go-ipfs-cmds#204](https://github.com/ipfs/go-ipfs-cmds/pull/204)) - feat: support strings option over HTTP API ([ipfs/go-ipfs-cmds#203](https://github.com/ipfs/go-ipfs-cmds/pull/203)) - github.com/ipfs/go-ipfs-config (v0.9.0 -> v0.12.0): - add support for pinning mfs (#116) ([ipfs/go-ipfs-config#116](https://github.com/ipfs/go-ipfs-config/pull/116)) - add remote pinning services config ([ipfs/go-ipfs-config#113](https://github.com/ipfs/go-ipfs-config/pull/113)) - Remove badger2 profile ([ipfs/go-ipfs-config#115](https://github.com/ipfs/go-ipfs-config/pull/115)) - Add badger2 profile and config spec - github.com/ipfs/go-ipfs-pinner (v0.0.4 -> v0.1.1): - Avoid loading all pins into memory during migration (#5) ([ipfs/go-ipfs-pinner#5](https://github.com/ipfs/go-ipfs-pinner/pull/5)) - Datastore based pinner (#4) ([ipfs/go-ipfs-pinner#4](https://github.com/ipfs/go-ipfs-pinner/pull/4)) - github.com/ipfs/go-ipld-cbor (v0.0.4 -> v0.0.5): - add the ability to leverage zero-copy on blockstores. (#75) ([ipfs/go-ipld-cbor#75](https://github.com/ipfs/go-ipld-cbor/pull/75)) - ipldstore: Also wrap Put serialization errors ([ipfs/go-ipld-cbor#74](https://github.com/ipfs/go-ipld-cbor/pull/74)) - add helper constructor for inmem cbor store - docs: add comments describing methods & interfaces ([ipfs/go-ipld-cbor#71](https://github.com/ipfs/go-ipld-cbor/pull/71)) - github.com/ipfs/go-path (v0.0.8 -> v0.0.9): - fix: improved error message on broken CIDv0 ([ipfs/go-path#33](https://github.com/ipfs/go-path/pull/33)) - github.com/ipfs/go-pinning-service-http-client (null -> v0.1.0): - feat: LsBatchSync to fetch single batch of results ([ipfs/go-pinning-service-http-client#6](https://github.com/ipfs/go-pinning-service-http-client/pull/6)) - Initial Implementation ([ipfs/go-pinning-service-http-client#1](https://github.com/ipfs/go-pinning-service-http-client/pull/1)) - github.com/ipld/go-car (v0.1.1-0.20200429200904-c222d793c339 -> v0.1.1-0.20201015032735-ff6ccdc46acc): - Update ipld libs ([ipld/go-car#35](https://github.com/ipld/go-car/pull/35)) - github.com/ipld/go-ipld-prime (v0.0.2-0.20200428162820-8b59dc292b8e -> v0.5.1-0.20201021195245-109253e8a018): - Merge branch 'codec-hardening' - Add fluent.MustReflect convenience method. - codegen: make error info available when tuples process data that is too long. ([ipld/go-ipld-prime#99](https://github.com/ipld/go-ipld-prime/pull/99)) - Merge branch 'codegen-typofixes' - Implement resource budgets in dagcbor parsing. ([ipld/go-ipld-prime#85](https://github.com/ipld/go-ipld-prime/pull/85)) - Codegen for links should emit the methods to conform to the schema.TypedLinkNode interface where applicable. ([ipld/go-ipld-prime#91](https://github.com/ipld/go-ipld-prime/pull/91)) - Introduce fluent.Reflect convenience functions. ([ipld/go-ipld-prime#81](https://github.com/ipld/go-ipld-prime/pull/81)) - schema/gen/go: make all top-level tests parallel - all: don't use buffers where readers suffice - fix typo in documentation - schema-schema codegen demo now includes unmarshal exercise ([ipld/go-ipld-prime#76](https://github.com/ipld/go-ipld-prime/pull/76)) - Update tests for unions; several fixes ([ipld/go-ipld-prime#75](https://github.com/ipld/go-ipld-prime/pull/75)) - New testcase system for exercising typed nodes; Revamp struct tests to use it. ([ipld/go-ipld-prime#66](https://github.com/ipld/go-ipld-prime/pull/66)) - small docs fixes on an internal component. - Fix formatting in README. - fix(cidlink): check for byte buffer ([ipld/go-ipld-prime#70](https://github.com/ipld/go-ipld-prime/pull/70)) - linking/cid: check a previously unused error ([ipld/go-ipld-prime#68](https://github.com/ipld/go-ipld-prime/pull/68)) - all: make 'go test ./...' pass on Go 1.15 ([ipld/go-ipld-prime#67](https://github.com/ipld/go-ipld-prime/pull/67)) - Merge branch 'kinded-union-gen' - Add traversal.Get function ([ipld/go-ipld-prime#65](https://github.com/ipld/go-ipld-prime/pull/65)) - Kinded union gen ([ipld/go-ipld-prime#64](https://github.com/ipld/go-ipld-prime/pull/64)) - Struct tuple representation codegen ([ipld/go-ipld-prime#63](https://github.com/ipld/go-ipld-prime/pull/63)) - Merge branch 'moar-codegen' - Self-hosting gen of the schema-schema. ([ipld/go-ipld-prime#62](https://github.com/ipld/go-ipld-prime/pull/62)) - Codegen: approaching self-host ([ipld/go-ipld-prime#61](https://github.com/ipld/go-ipld-prime/pull/61)) - Codegen of unions, and their keyed representations ([ipld/go-ipld-prime#60](https://github.com/ipld/go-ipld-prime/pull/60)) - mark v0.5 - API updates for v0.5: the renamening ([ipld/go-ipld-prime#59](https://github.com/ipld/go-ipld-prime/pull/59)) - mark v0.4 - changelog: note the codegen work. - Codegen update -- Assemblers, and many new representations ([ipld/go-ipld-prime#52](https://github.com/ipld/go-ipld-prime/pull/52)) - Merge branch 'json-tables-codec' - Merge branch 'docs-updates' - Introduce changelog! - Add examples of creating and loading links. - github.com/ipld/go-ipld-prime-proto (v0.0.0-20200428191222-c1ffdadc01e1 -> v0.1.0): - Update go-ipld-prime ([ipld/go-ipld-prime-proto#6](https://github.com/ipld/go-ipld-prime-proto/pull/6)) - feat(coding use -1 instead of 0): - Update ipld prime, use proper code-gen ([ipld/go-ipld-prime-proto#5](https://github.com/ipld/go-ipld-prime-proto/pull/5)) - Updates to dependencies ([ipld/go-ipld-prime-proto#4](https://github.com/ipld/go-ipld-prime-proto/pull/4)) - Check for byte buffer on decode ([ipld/go-ipld-prime-proto#3](https://github.com/ipld/go-ipld-prime-proto/pull/3)) - github.com/libp2p/go-libp2p (v0.11.0 -> v0.13.0): - use a context when opening streams ([libp2p/go-libp2p#1033](https://github.com/libp2p/go-libp2p/pull/1033)) - fix: obey new stream timeout ([libp2p/go-libp2p#1029](https://github.com/libp2p/go-libp2p/pull/1029)) - feat: update to go-libp2p-core 0.7.0 interface changes ([libp2p/go-libp2p#1001](https://github.com/libp2p/go-libp2p/pull/1001)) - Basic Connection Gater Implementation ([libp2p/go-libp2p#1005](https://github.com/libp2p/go-libp2p/pull/1005)) - Fixed bug for inbound connections gated by the deprecated filter option (#1004) ([libp2p/go-libp2p#1004](https://github.com/libp2p/go-libp2p/pull/1004)) - github.com/libp2p/go-libp2p-autonat (v0.3.2 -> v0.4.0): - feat: update to go-libp2p-core 0.7.0 ([libp2p/go-libp2p-autonat#97](https://github.com/libp2p/go-libp2p-autonat/pull/97)) - github.com/libp2p/go-libp2p-circuit (v0.3.1 -> v0.4.0): - feat: update to go-libp2p-core 0.7.0 ([libp2p/go-libp2p-circuit#123](https://github.com/libp2p/go-libp2p-circuit/pull/123)) - github.com/libp2p/go-libp2p-core (v0.6.1 -> v0.8.0): - add a context to OpenStream and NewStream (#172) ([libp2p/go-libp2p-core#172](https://github.com/libp2p/go-libp2p-core/pull/172)) - sec/insecure/insecure.go: Fix typo (#167) ([libp2p/go-libp2p-core#167](https://github.com/libp2p/go-libp2p-core/pull/167)) - add CloseRead/CloseWrite on streams (#166) ([libp2p/go-libp2p-core#166](https://github.com/libp2p/go-libp2p-core/pull/166)) - Fix typo in docs (#163) ([libp2p/go-libp2p-core#163](https://github.com/libp2p/go-libp2p-core/pull/163)) - github.com/libp2p/go-libp2p-gostream (v0.2.1 -> v0.3.0): - feat: use go-libp2p-core 0.7.0 stream interfaces ([libp2p/go-libp2p-gostream#60](https://github.com/libp2p/go-libp2p-gostream/pull/60)) - github.com/libp2p/go-libp2p-http (v0.1.5 -> v0.2.0): - Fix var name in README ([libp2p/go-libp2p-http#63](https://github.com/libp2p/go-libp2p-http/pull/63)) - Fix var name in doc ([libp2p/go-libp2p-http#62](https://github.com/libp2p/go-libp2p-http/pull/62)) - github.com/libp2p/go-libp2p-kad-dht (v0.9.0 -> v0.11.1): - Fix constructor ordering ([libp2p/go-libp2p-kad-dht#698](https://github.com/libp2p/go-libp2p-kad-dht/pull/698)) - feat: update to go-libp2p-core 0.7.0 ([libp2p/go-libp2p-kad-dht#693](https://github.com/libp2p/go-libp2p-kad-dht/pull/693)) - Run fixLowPeers on startup ([libp2p/go-libp2p-kad-dht#694](https://github.com/libp2p/go-libp2p-kad-dht/pull/694)) - feat: add advanced V1ProtocolOverride option to be used by legacy networks - feat: remove dht v2 as it's not actually in use and could be confusing - github.com/libp2p/go-libp2p-mplex (v0.2.4 -> v0.4.1): - update go-mplex, use the context passed to OpenStream ([libp2p/go-libp2p-mplex#23](https://github.com/libp2p/go-libp2p-mplex/pull/23)) - change OpenStream to accept a context ([libp2p/go-libp2p-mplex#21](https://github.com/libp2p/go-libp2p-mplex/pull/21)) - feat: update stream interfaces ([libp2p/go-libp2p-mplex#20](https://github.com/libp2p/go-libp2p-mplex/pull/20)) - github.com/libp2p/go-libp2p-noise (v0.1.1 -> v0.1.2): - optimize: reduce syscalls using a buffered reader. - github.com/libp2p/go-libp2p-pubsub (v0.3.5 -> v0.4.1): - defer stream removal instead of doing it inline. - add test for inbound stream deduplication - deduplicate inbound streams - populate receivedFrom field in delivery trace - add receivedFrom field in delivery trace - fix: reduce log spam (#394) ([libp2p/go-libp2p-pubsub#394](https://github.com/libp2p/go-libp2p-pubsub/pull/394)) - fix: treat peers already connected to the host before pubsub is initialized as valid potential pubsub peers - test: add test for if nodes are connected before pubsub is started - feat: update to go-libp2p-core 0.7.0 - Add go-libp2p example in README.md (#392) ([libp2p/go-libp2p-pubsub#392](https://github.com/libp2p/go-libp2p-pubsub/pull/392)) - subscription filters - remove multi-topic message support - satisfy race detector - clean up - copy string topic - add test for score adjustment from topic params reset - prettify things - add test for topic score parameter reset method - add test for topic score parameter reset - add api for dynamically setting and resetting topic score parameters - add support for priority topic delivery weights - tweak duplicate/reject weights - decay global counters after 2 min - decouple global counter decay from source counter decay - add warning for failure to parse IP out of remote multiaddr - more docs - configure the peer gater using a parameter object, docs and stuff - disable codecov annotations, makes things unreadable - further tweak gate threshold weights - fix test races - use IPs for peer gater stat tracking - mix total accounting components with different weights - count all rejections by default - fix non-determinism in test - tweak probability threshold - also account for duplicates in gating decisions - test throttle code path in gossip tracer - add test for peer gater - more efficient promise processing on throttling - trace throttle peers to avoid breaking promises unfairly - better log messages around gating - implement peer gater - peer gater scaffolding - rich router acceptance semantics - reduce log verbosity; debug mostly - github.com/libp2p/go-libp2p-pubsub-router (v0.3.2 -> v0.4.0): - feat: use new stream interfaces from go-libp2p-core 0.7.0 ([libp2p/go-libp2p-pubsub-router#81](https://github.com/libp2p/go-libp2p-pubsub-router/pull/81)) - github.com/libp2p/go-libp2p-quic-transport (v0.8.0 -> v0.10.0): - change OpenStream to accept a context ([libp2p/go-libp2p-quic-transport#189](https://github.com/libp2p/go-libp2p-quic-transport/pull/189)) - update quic-go to v0.19.1 ([libp2p/go-libp2p-quic-transport#182](https://github.com/libp2p/go-libp2p-quic-transport/pull/182)) - pass a conn that can be type asserted to a net.UDPConn to quic-go ([libp2p/go-libp2p-quic-transport#180](https://github.com/libp2p/go-libp2p-quic-transport/pull/180)) - add more integration tests ([libp2p/go-libp2p-quic-transport#181](https://github.com/libp2p/go-libp2p-quic-transport/pull/181)) - always close the connection in the cmd client ([libp2p/go-libp2p-quic-transport#175](https://github.com/libp2p/go-libp2p-quic-transport/pull/175)) - use GitHub Actions to test interoperability of releases ([libp2p/go-libp2p-quic-transport#173](https://github.com/libp2p/go-libp2p-quic-transport/pull/173)) - Implement CloseRead/CloseWrite ([libp2p/go-libp2p-quic-transport#174](https://github.com/libp2p/go-libp2p-quic-transport/pull/174)) - enable quic-go metrics collection ([libp2p/go-libp2p-quic-transport#172](https://github.com/libp2p/go-libp2p-quic-transport/pull/172)) - github.com/libp2p/go-libp2p-swarm (v0.2.8 -> v0.4.0): - use a context for OpenStream and NewStream ([libp2p/go-libp2p-swarm#232](https://github.com/libp2p/go-libp2p-swarm/pull/232)) - fix: handle case where swarm closes before stream ([libp2p/go-libp2p-swarm#229](https://github.com/libp2p/go-libp2p-swarm/pull/229)) - feat: update to latest go-libp2p-core interfaces ([libp2p/go-libp2p-swarm#228](https://github.com/libp2p/go-libp2p-swarm/pull/228)) - github.com/libp2p/go-libp2p-testing (v0.2.0 -> v0.4.0): - pass contexts to OpenStream in tests ([libp2p/go-libp2p-testing#31](https://github.com/libp2p/go-libp2p-testing/pull/31)) - chore: Adding LICENSE. ([libp2p/go-libp2p-testing#30](https://github.com/libp2p/go-libp2p-testing/pull/30)) - feat: update to go-libp2p-core 0.7.0 ([libp2p/go-libp2p-testing#29](https://github.com/libp2p/go-libp2p-testing/pull/29)) - github.com/libp2p/go-libp2p-transport-upgrader (v0.3.0 -> v0.4.0): - pass contexts to OpenStream in tests ([libp2p/go-libp2p-transport-upgrader#70](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/70)) - fix int to string conversion in tests, update Go version on CI ([libp2p/go-libp2p-transport-upgrader#69](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/69)) - github.com/libp2p/go-libp2p-yamux (v0.2.8 -> v0.5.1): - update go-yamux to v2.0.0, use context passed to OpenStream ([libp2p/go-libp2p-yamux#31](https://github.com/libp2p/go-libp2p-yamux/pull/31)) - change OpenStream to accept a context ([libp2p/go-libp2p-yamux#29](https://github.com/libp2p/go-libp2p-yamux/pull/29)) - feat: update to new stream interfaces ([libp2p/go-libp2p-yamux#27](https://github.com/libp2p/go-libp2p-yamux/pull/27)) - github.com/libp2p/go-mplex (v0.1.2 -> v0.3.0): - add a context to NewStream, remove the NewStreamTimeout ([libp2p/go-mplex#82](https://github.com/libp2p/go-mplex/pull/82)) - Implement new CloseWrite/CloseRead interface ([libp2p/go-mplex#81](https://github.com/libp2p/go-mplex/pull/81)) - Bump lodash from 4.17.15 to 4.17.19 in /interop/js ([libp2p/go-mplex#79](https://github.com/libp2p/go-mplex/pull/79)) - upgrade deps + interoperable varints. (#80) ([libp2p/go-mplex#80](https://github.com/libp2p/go-mplex/pull/80)) - write benchmarks (#77) ([libp2p/go-mplex#77](https://github.com/libp2p/go-mplex/pull/77)) - github.com/libp2p/go-ws-transport (v0.3.1 -> v0.4.0): - pass a context to OpenStream in tests ([libp2p/go-ws-transport#98](https://github.com/libp2p/go-ws-transport/pull/98)) - Dependency: Remove deprecated multiaddr-net ([libp2p/go-ws-transport#97](https://github.com/libp2p/go-ws-transport/pull/97)) - Update for go 1.14 Wasm changes ([libp2p/go-ws-transport#96](https://github.com/libp2p/go-ws-transport/pull/96)) - github.com/libp2p/go-yamux (v1.3.7 -> v1.4.1): - feat: improve ping accuracy ([libp2p/go-yamux#35](https://github.com/libp2p/go-yamux/pull/35)) - implement CloseRead/CloseWrite ([libp2p/go-yamux#5](https://github.com/libp2p/go-yamux/pull/5)) - fix space accounting in the receive buffer ([libp2p/go-yamux#33](https://github.com/libp2p/go-yamux/pull/33)) - Limit pings ([libp2p/go-yamux#32](https://github.com/libp2p/go-yamux/pull/32)) - fix: simplify inflight fix ([libp2p/go-yamux#31](https://github.com/libp2p/go-yamux/pull/31)) - Clearing inflight along with streams to avoid memory leak ([libp2p/go-yamux#30](https://github.com/libp2p/go-yamux/pull/30)) - github.com/lucas-clemente/quic-go (v0.18.0 -> v0.19.3): - create a v0.19.x release - improve the warning about the UDP receive buffer size ([lucas-clemente/quic-go#2923](https://github.com/lucas-clemente/quic-go/pull/2923)) - immediately remove reset tokens when retiring a connection ID ([lucas-clemente/quic-go#2897](https://github.com/lucas-clemente/quic-go/pull/2897)) - add common temporary file patterns to .gitignore ([lucas-clemente/quic-go#2917](https://github.com/lucas-clemente/quic-go/pull/2917)) - disable key updates when using HTTP/3 to avoid breaking Chrome 87 ([lucas-clemente/quic-go#2906](https://github.com/lucas-clemente/quic-go/pull/2906)) - fix decoding of packet numbers in different packet number spaces ([lucas-clemente/quic-go#2903](https://github.com/lucas-clemente/quic-go/pull/2903)) - log sent packet before logging its congestion / loss recovery effects ([lucas-clemente/quic-go#2912](https://github.com/lucas-clemente/quic-go/pull/2912)) - fix a crash in the http3.Server when GetConfigForClient returns nil ([lucas-clemente/quic-go#2925](https://github.com/lucas-clemente/quic-go/pull/2925)) - set the UDP receive buffer size on Windows ([lucas-clemente/quic-go#2896](https://github.com/lucas-clemente/quic-go/pull/2896)) - remove superfluous sleep in packet handler map test ([lucas-clemente/quic-go#2894](https://github.com/lucas-clemente/quic-go/pull/2894)) - fix setting of http.Handler in the example server ([lucas-clemente/quic-go#2900](https://github.com/lucas-clemente/quic-go/pull/2900)) - remove stray print statement - remove unnecessary mutex locking in the stream flow controller ([lucas-clemente/quic-go#2869](https://github.com/lucas-clemente/quic-go/pull/2869)) - only use syscalls on platforms that we're actually testing ([lucas-clemente/quic-go#2886](https://github.com/lucas-clemente/quic-go/pull/2886)) - only write headers with a length that fits into 2 bytes in fuzz test ([lucas-clemente/quic-go#2884](https://github.com/lucas-clemente/quic-go/pull/2884)) - fix packing of 1-RTT probe packets ([lucas-clemente/quic-go#2882](https://github.com/lucas-clemente/quic-go/pull/2882)) - use PADDING frames to pad packets ([lucas-clemente/quic-go#2876](https://github.com/lucas-clemente/quic-go/pull/2876)) - fix race condition when accepting streams ([lucas-clemente/quic-go#2874](https://github.com/lucas-clemente/quic-go/pull/2874)) - only trace dropped 0-RTT packets when a tracer is set ([lucas-clemente/quic-go#2871](https://github.com/lucas-clemente/quic-go/pull/2871)) - use consistent version numbers in client test ([lucas-clemente/quic-go#2870](https://github.com/lucas-clemente/quic-go/pull/2870)) - replace the RWMutex with a Mutex in the flow controller ([lucas-clemente/quic-go#2865](https://github.com/lucas-clemente/quic-go/pull/2865)) - replace the RWMutex with a Mutex in the packet handler map ([lucas-clemente/quic-go#2864](https://github.com/lucas-clemente/quic-go/pull/2864)) - wait until the handshake is complete before updating the connection ID ([lucas-clemente/quic-go#2856](https://github.com/lucas-clemente/quic-go/pull/2856)) - only check the SCID for Initial packets ([lucas-clemente/quic-go#2857](https://github.com/lucas-clemente/quic-go/pull/2857)) - add the NO_VIABLE_PATH error ([lucas-clemente/quic-go#2861](https://github.com/lucas-clemente/quic-go/pull/2861)) - implement qlogging of the preferred address in the transport parameters ([lucas-clemente/quic-go#2853](https://github.com/lucas-clemente/quic-go/pull/2853)) - explicitly set the supported versions in the HTTP/3 server test ([lucas-clemente/quic-go#2854](https://github.com/lucas-clemente/quic-go/pull/2854)) - allow an amplification factor of 3.x ([lucas-clemente/quic-go#2862](https://github.com/lucas-clemente/quic-go/pull/2862)) - only allow the HTTP/3 client to dial with a single QUIC version ([lucas-clemente/quic-go#2848](https://github.com/lucas-clemente/quic-go/pull/2848)) - send STREAMS_BLOCKED frame when MAX_STREAMS frame allows too few streams ([lucas-clemente/quic-go#2828](https://github.com/lucas-clemente/quic-go/pull/2828)) - set the ALPN based on the QUIC version in the HTTP3 server ([lucas-clemente/quic-go#2847](https://github.com/lucas-clemente/quic-go/pull/2847)) - pad datagrams containing ack-eliciting Initial packets sent by the server ([lucas-clemente/quic-go#2841](https://github.com/lucas-clemente/quic-go/pull/2841)) - fix OpenStreamSync busy looping ([lucas-clemente/quic-go#2827](https://github.com/lucas-clemente/quic-go/pull/2827)) - fix deadlock when closing the server and the connection at the same time ([lucas-clemente/quic-go#2849](https://github.com/lucas-clemente/quic-go/pull/2849)) - run gofumpt, enable the gofumpt linter ([lucas-clemente/quic-go#2839](https://github.com/lucas-clemente/quic-go/pull/2839)) - prepare for draft-32 ([lucas-clemente/quic-go#2831](https://github.com/lucas-clemente/quic-go/pull/2831)) - update the invalid packet limit for AES ([lucas-clemente/quic-go#2825](https://github.com/lucas-clemente/quic-go/pull/2825)) - increase UDP receive buffer size ([lucas-clemente/quic-go#2791](https://github.com/lucas-clemente/quic-go/pull/2791)) - listen on both IPv4 and IPv6 in the interop runner server ([lucas-clemente/quic-go#2822](https://github.com/lucas-clemente/quic-go/pull/2822)) - only send Version Negotiation packets for packets larger than 1200 bytes ([lucas-clemente/quic-go#2820](https://github.com/lucas-clemente/quic-go/pull/2820)) - don't send a version negotiation packet in response to a version negotiation packet ([lucas-clemente/quic-go#2818](https://github.com/lucas-clemente/quic-go/pull/2818)) - client: Add DialEarlyContext and DialAddrEarlyContext API ([lucas-clemente/quic-go#2814](https://github.com/lucas-clemente/quic-go/pull/2814)) - qlog the key phase bit ([lucas-clemente/quic-go#2817](https://github.com/lucas-clemente/quic-go/pull/2817)) - only include quic-trace when the quictrace build flag is set ([lucas-clemente/quic-go#2799](https://github.com/lucas-clemente/quic-go/pull/2799)) - fix error handling when receiving post handshake messages ([lucas-clemente/quic-go#2807](https://github.com/lucas-clemente/quic-go/pull/2807)) - add support for the ChaCha20 test on the server side ([lucas-clemente/quic-go#2816](https://github.com/lucas-clemente/quic-go/pull/2816)) - allow the first key update immediately after handshake confirmation ([lucas-clemente/quic-go#2811](https://github.com/lucas-clemente/quic-go/pull/2811)) - ignore temporary errors when reading from the packet conn ([lucas-clemente/quic-go#2806](https://github.com/lucas-clemente/quic-go/pull/2806)) - fix linting error on OSX ([lucas-clemente/quic-go#2813](https://github.com/lucas-clemente/quic-go/pull/2813)) - add the exhaustive linter, replace panics by return values in logging stringers ([lucas-clemente/quic-go#2729](https://github.com/lucas-clemente/quic-go/pull/2729)) - include the error code in the string for CRYPTO_ERRORs ([lucas-clemente/quic-go#2805](https://github.com/lucas-clemente/quic-go/pull/2805)) - fail the handshake if the quic_transport_parameter extension is missing ([lucas-clemente/quic-go#2804](https://github.com/lucas-clemente/quic-go/pull/2804)) - fix logging of received Retry packets ([lucas-clemente/quic-go#2803](https://github.com/lucas-clemente/quic-go/pull/2803)) - fix deadlock in crypto setup when it is closed while handling a message ([lucas-clemente/quic-go#2802](https://github.com/lucas-clemente/quic-go/pull/2802)) - make the key update integration test more rigorous ([lucas-clemente/quic-go#2760](https://github.com/lucas-clemente/quic-go/pull/2760)) - add support for the new keyupdate interop runner test case ([lucas-clemente/quic-go#2782](https://github.com/lucas-clemente/quic-go/pull/2782)) - remove unneeded mutex in the client ([lucas-clemente/quic-go#2798](https://github.com/lucas-clemente/quic-go/pull/2798)) - correctly handle key updates within the 3 PTO period ([lucas-clemente/quic-go#2787](https://github.com/lucas-clemente/quic-go/pull/2787)) - introduce an ECNCapablePacketConn interface to determine ECN support ([lucas-clemente/quic-go#2788](https://github.com/lucas-clemente/quic-go/pull/2788)) - use certificates from /certs directory for the server ([lucas-clemente/quic-go#2794](https://github.com/lucas-clemente/quic-go/pull/2794)) - remove support for the ECN test case ([lucas-clemente/quic-go#2793](https://github.com/lucas-clemente/quic-go/pull/2793)) - check that the peer updated its keys when acknowledging a key update ([lucas-clemente/quic-go#2781](https://github.com/lucas-clemente/quic-go/pull/2781)) - fix flaky packet number skipping test ([lucas-clemente/quic-go#2786](https://github.com/lucas-clemente/quic-go/pull/2786)) - read ECN bits and send ECN counters in ACK frames ([lucas-clemente/quic-go#2741](https://github.com/lucas-clemente/quic-go/pull/2741)) - implement the limit of unsuccessful decryptions for the AEADs ([lucas-clemente/quic-go#2771](https://github.com/lucas-clemente/quic-go/pull/2771)) - use the KEY_UPDATE_ERROR ([lucas-clemente/quic-go#2770](https://github.com/lucas-clemente/quic-go/pull/2770)) - fix dropping of key phase 0 ([lucas-clemente/quic-go#2769](https://github.com/lucas-clemente/quic-go/pull/2769)) - reduce the handshake timeout to two minutes in the handshake drop tests ([lucas-clemente/quic-go#2768](https://github.com/lucas-clemente/quic-go/pull/2768)) - fix handling of multiple handshake messages in the case of errors ([lucas-clemente/quic-go#2777](https://github.com/lucas-clemente/quic-go/pull/2777)) - enable more linters, update golangci-lint to v1.31 ([lucas-clemente/quic-go#2775](https://github.com/lucas-clemente/quic-go/pull/2775)) - increase the threshold for the receive stream deadline test ([lucas-clemente/quic-go#2774](https://github.com/lucas-clemente/quic-go/pull/2774)) - add an assertion that bytes_in_flight never becomes negative ([lucas-clemente/quic-go#2779](https://github.com/lucas-clemente/quic-go/pull/2779)) - fix race condition in handshake fuzz code ([lucas-clemente/quic-go#2778](https://github.com/lucas-clemente/quic-go/pull/2778)) - use more tls.Config options in the handshake fuzzer ([lucas-clemente/quic-go#2746](https://github.com/lucas-clemente/quic-go/pull/2746)) - run two handshakes in the handshake fuzzer ([lucas-clemente/quic-go#2743](https://github.com/lucas-clemente/quic-go/pull/2743)) - send post-handshake message in the handshake fuzzer ([lucas-clemente/quic-go#2742](https://github.com/lucas-clemente/quic-go/pull/2742)) - skip a packet number when sending a 1-RTT PTO packet ([lucas-clemente/quic-go#2754](https://github.com/lucas-clemente/quic-go/pull/2754)) - save dummy packets in the packet history when skipping packet numbers ([lucas-clemente/quic-go#2753](https://github.com/lucas-clemente/quic-go/pull/2753)) - delete unacknowledged packets from the packet history after 3 PTOs ([lucas-clemente/quic-go#2750](https://github.com/lucas-clemente/quic-go/pull/2750)) - add support for the HTTP CONNECT method (#2761) ([lucas-clemente/quic-go#2761](https://github.com/lucas-clemente/quic-go/pull/2761)) - don't drop keys for key phase N before receiving a N+1-protected packet ([lucas-clemente/quic-go#2762](https://github.com/lucas-clemente/quic-go/pull/2762)) - close session on errors unpacking errors other than decryption errors ([lucas-clemente/quic-go#2756](https://github.com/lucas-clemente/quic-go/pull/2756)) - log when an old 1-RTT key is retired ([lucas-clemente/quic-go#2765](https://github.com/lucas-clemente/quic-go/pull/2765)) - only return an invalid first key phase error for decryptable packets ([lucas-clemente/quic-go#2757](https://github.com/lucas-clemente/quic-go/pull/2757)) - fix logging of locally initiated key updates ([lucas-clemente/quic-go#2764](https://github.com/lucas-clemente/quic-go/pull/2764)) - test that both endpoints time out in the timeout integration test ([lucas-clemente/quic-go#2744](https://github.com/lucas-clemente/quic-go/pull/2744)) - refactor RTT measurements to simplify the sentPacketHistory ([lucas-clemente/quic-go#2747](https://github.com/lucas-clemente/quic-go/pull/2747)) - fix dropping of 0-RTT packets ([lucas-clemente/quic-go#2752](https://github.com/lucas-clemente/quic-go/pull/2752)) - always qlog the generation of 1-RTT key updates ([lucas-clemente/quic-go#2763](https://github.com/lucas-clemente/quic-go/pull/2763)) - move the PacketHeader struct from logging to qlog package ([lucas-clemente/quic-go#2766](https://github.com/lucas-clemente/quic-go/pull/2766)) - use a uint8 for the EncryptionLevel ([lucas-clemente/quic-go#2751](https://github.com/lucas-clemente/quic-go/pull/2751)) - make sure to only pass handshake messages that keys are available for ([lucas-clemente/quic-go#2739](https://github.com/lucas-clemente/quic-go/pull/2739)) - only close the handshake fuzz runner once ([lucas-clemente/quic-go#2740](https://github.com/lucas-clemente/quic-go/pull/2740)) - generate a self-signed certificate for the handshake fuzzer ([lucas-clemente/quic-go#2738](https://github.com/lucas-clemente/quic-go/pull/2738)) - use the os.ErrDeadlineExceeded for stream deadline errors on Go 1.15 ([lucas-clemente/quic-go#2734](https://github.com/lucas-clemente/quic-go/pull/2734)) - use GitHub Actions to run unit tests ([lucas-clemente/quic-go#2732](https://github.com/lucas-clemente/quic-go/pull/2732)) - add a basic fuzzer for the handshake ([lucas-clemente/quic-go#2733](https://github.com/lucas-clemente/quic-go/pull/2733)) - export seed corpus files using the SHA1 of the content as the filename ([lucas-clemente/quic-go#2731](https://github.com/lucas-clemente/quic-go/pull/2731)) - add a fuzz target for the token generator ([lucas-clemente/quic-go#2730](https://github.com/lucas-clemente/quic-go/pull/2730)) - fix typo in error message in sent packet handler - fix missing OnLost callback for frames sent in 0-RTT packets ([lucas-clemente/quic-go#2728](https://github.com/lucas-clemente/quic-go/pull/2728)) - fix overflow of the max_ack_delay when parsing transport parameters ([lucas-clemente/quic-go#2725](https://github.com/lucas-clemente/quic-go/pull/2725)) - github.com/marten-seemann/qpack (v0.2.0 -> v0.2.1): - run gofumpt, add a few more linters ([marten-seemann/qpack#21](https://github.com/marten-seemann/qpack/pull/21)) - fix static table entry 80 ([marten-seemann/qpack#20](https://github.com/marten-seemann/qpack/pull/20)) - github.com/marten-seemann/qtls-go1-15 (v0.1.0 -> v0.1.1): - use a prefix for client session cache keys - add callbacks to store and restore app data along a session state - don't use TLS 1.3 compatibility mode when using alternative record layer - delete the session ticket after attempting 0-RTT - reject 0-RTT when a different ALPN is chosen - encode the ALPN into the session ticket - add a field to the ConnectionState to tell if 0-RTT was used - add a callback to tell the client about rejection of 0-RTT - don't offer 0-RTT after a HelloRetryRequest - add Accept0RTT to Config callback to decide if 0-RTT should be accepted - add the option to encode application data into the session ticket - export the 0-RTT write key - abuse the nonce field of ClientSessionState to save max_early_data_size - export the 0-RTT read key - close connection if client attempts 0-RTT, but ticket didn't allow it - encode the max early data size into the session ticket - implement parsing of the early_data extension in the EncryptedExtensions - add a tls.Config.MaxEarlyData option to enable 0-RTT - accept TLS 1.3 cipher suites in Config.CipherSuites - introduce a function on the connection to generate a session ticket - add a config option to enforce selection of an application protocol - export Conn.HandlePostHandshakeMessage - export Alert - reject Configs that set MaxVersion < 1.3 when using a record layer - enforce TLS 1.3 when using an alternative record layer - github.com/multiformats/go-multistream (v0.1.2 -> v0.2.0): - improve negotiation flushing ([multiformats/go-multistream#52](https://github.com/multiformats/go-multistream/pull/52)) - github.com/whyrusleeping/cbor-gen (v0.0.0-20200402171437-3d27c146c105 -> v0.0.0-20200710004633-5379fc63235d): - correctly map typegen to cbg in all cases ([whyrusleeping/cbor-gen#26](https://github.com/whyrusleeping/cbor-gen/pull/26)) - fix: clear struct state on unmarshal ([whyrusleeping/cbor-gen#22](https://github.com/whyrusleeping/cbor-gen/pull/22)) - deferred: restrict max length ([whyrusleeping/cbor-gen#25](https://github.com/whyrusleeping/cbor-gen/pull/25)) - reduce number of allocations in ScanForLinks ([whyrusleeping/cbor-gen#24](https://github.com/whyrusleeping/cbor-gen/pull/24)) - attempt to allocate less by using shared buffers ([whyrusleeping/cbor-gen#18](https://github.com/whyrusleeping/cbor-gen/pull/18)) - add benchmark - use new cid methods for less allocs ([whyrusleeping/cbor-gen#17](https://github.com/whyrusleeping/cbor-gen/pull/17)) - properly handle roundtripping Deferred with 'null' value ([whyrusleeping/cbor-gen#16](https://github.com/whyrusleeping/cbor-gen/pull/16)) - Support array types ([whyrusleeping/cbor-gen#15](https://github.com/whyrusleeping/cbor-gen/pull/15)) - github.com/whyrusleeping/tar-utils (v0.0.0-20180509141711-8c6c8ba81d5c -> v0.0.0-20201201191210-20a61371de5b): - more closely match default tar errors (GNU + BSD binaries) Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Eric Myhre | 180 | +26453/-11032 | 883 | | Marten Seemann | 212 | +14876/-9352 | 794 | | hannahhoward | 41 | +9195/-3113 | 186 | | Alex Cruikshank | 5 | +3323/-1895 | 58 | | Andrew Gillis | 3 | +3792/-581 | 21 | | vyzo | 49 | +2675/-949 | 95 | | Adin Schmahmann | 57 | +1473/-837 | 90 | | Steven Allen | 43 | +1252/-780 | 99 | | Petar Maymounkov | 3 | +1755/-113 | 17 | | Marcin Rataj | 35 | +979/-210 | 61 | | Paul Wolneykien | 2 | +670/-338 | 9 | | Jeromy Johnson | 9 | +525/-221 | 21 | | gammazero | 11 | +366/-101 | 26 | | Hector Sanjuan | 7 | +312/-0 | 11 | | Dirk McCormick | 4 | +190/-90 | 15 | | Will Scott | 1 | +252/-0 | 1 | | Oli Evans | 1 | +201/-0 | 1 | | Tomasz Zdybał | 2 | +182/-3 | 6 | | Daniel Martí | 6 | +104/-66 | 35 | | Sam | 3 | +76/-59 | 5 | | Łukasz Magiera | 2 | +92/-3 | 5 | | whyrusleeping | 3 | +77/-15 | 3 | | nisdas | 3 | +76/-15 | 4 | | Raúl Kripalani | 3 | +59/-31 | 5 | | Lucas Molas | 1 | +66/-3 | 2 | | Alex Towle | 1 | +52/-8 | 2 | | Dennis Trautwein | 1 | +58/-0 | 2 | | Adrian Lanzafame | 2 | +49/-7 | 4 | | klzgrad | 1 | +49/-5 | 2 | | Fazlul Shahriar | 1 | +35/-14 | 17 | | Yingrong Zhao | 1 | +45/-2 | 2 | | Jakub Sztandera | 2 | +22/-13 | 2 | | Chaitanya | 8 | +16/-16 | 8 | | Aarsh Shah | 1 | +27/-1 | 3 | | Rod Vagg | 1 | +23/-4 | 2 | | M. Hawn | 4 | +11/-11 | 8 | | Will | 1 | +12/-2 | 1 | | frrist | 1 | +7/-0 | 1 | | Rafael Ramalho | 2 | +5/-2 | 2 | | dependabot[bot] | 1 | +3/-3 | 1 | | Zaurbek Zhakupov | 1 | +3/-3 | 1 | | Tom Worrall | 1 | +4/-2 | 1 | | Jorropo | 2 | +5/-1 | 2 | | Chaitanya Raju | 1 | +3/-3 | 2 | | Egon Elbre | 1 | +0/-5 | 1 | | incognitomode | 1 | +2/-2 | 1 | | achingbrain | 1 | +2/-2 | 1 | | Michael Burns | 1 | +2/-2 | 1 | | David Florness | 2 | +2/-2 | 2 | | RubenKelevra | 1 | +2/-1 | 1 | | Andrew Nesbitt | 2 | +2/-1 | 2 | | Tarun Bansal | 1 | +1/-1 | 1 | | Max Inden | 1 | +1/-1 | 1 | | K | 1 | +2/-0 | 1 | | Jacob Heun | 1 | +1/-1 | 1 | | Henrique Dias | 1 | +1/-1 | 1 | | Bryan White | 1 | +1/-1 | 1 | | Bryan Stenson | 1 | +1/-1 | 1 | ================================================ FILE: docs/changelogs/v0.9.md ================================================ # go-ipfs changelog v0.9 ## v0.9.1 2021-07-20 This is a small bug fix release resolving the following issues: 1. A regression where the empty CID bafkqaaa could not resolve on gateways [#8230](https://github.com/ipfs/go-ipfs/issues/8230) 2. A panic on OpenBSD [#8211](https://github.com/ipfs/go-ipfs/issues/8211) 3. High CPU usage with QUIC [#8256](https://github.com/ipfs/go-ipfs/issues/8256) 4. High memory usage with TCP [#8219](https://github.com/ipfs/go-ipfs/issues/8219) 5. Some pubsub issues ([libp2p/go-libp2p-pubsub#427](https://github.com/libp2p/go-libp2p-pubsub/pull/427), [libp2p/go-libp2p-pubsub#430](https://github.com/libp2p/go-libp2p-pubsub/pull/430)) 6. Updated WebUI to [v2.12.4](https://github.com/ipfs/ipfs-webui/releases/tag/v2.12.4) 7. Fixed the snap deployment [#8212](https://github.com/ipfs/go-ipfs/pull/8212) ### Changelog - github.com/ipfs/go-ipfs: - chore: update deps - feat: webui v2.12.4 - test: gateway response for bafkqaaa - fix: downgrade mimetype dependency - update go-libp2p to v0.14.3 - bump snap to build with Go 1.16 - github.com/libp2p/go-libp2p (v0.14.2 -> v0.14.3): - update go-tcp-transport to v0.2.3 and go-multiaddr to v0.3.3 ([libp2p/go-libp2p#1121](https://github.com/libp2p/go-libp2p/pull/1121)) - github.com/libp2p/go-libp2p-pubsub (v0.4.1 -> v0.4.2): - release priority locks early when handling batches - don't respawn writer if we fail to open a stream; declare it a peer error - batch process dead peer notifications - use a priority lock instead of a semaphore - do the notification in a goroutine - emit new peer notification without holding the semaphore - use a semaphore for new peer notifications so that we don't block the event loop - don't accumulate pending goroutines from new connections - Make close concurrent safe - Fix close of closed channel - github.com/libp2p/go-libp2p-quic-transport (v0.11.1 -> v0.11.2): - update quic-go to v0.21.2 - github.com/libp2p/go-tcp-transport (v0.2.2 -> v0.2.4): - collect metrics in a separate go routine ([libp2p/go-tcp-transport#82](https://github.com/libp2p/go-tcp-transport/pull/82)) - fix: avoid logging "invalid argument" errors when setting keepalive ([libp2p/go-tcp-transport#83](https://github.com/libp2p/go-tcp-transport/pull/83)) - Skip SetKeepAlivePeriod call on OpenBSD ([libp2p/go-tcp-transport#80](https://github.com/libp2p/go-tcp-transport/pull/80)) - sync: update CI config files (#79) ([libp2p/go-tcp-transport#79](https://github.com/libp2p/go-tcp-transport/pull/79)) - github.com/lucas-clemente/quic-go (v0.21.1 -> v0.21.2): - update qtls to include the crypto/tls fix of Go 1.16.6 / 1.15.14 - cancel the PTO timer when all Handshake packets are acknowledged - update to Go 1.17rc1 - update Ginkgo to v1.16.4 and Gomega to v1.13.0 ([lucas-clemente/quic-go#3139](https://github.com/lucas-clemente/quic-go/pull/3139)) - github.com/multiformats/go-multiaddr (v0.3.2 -> v0.3.3): - guard against nil {Local,Remote}Addr() return values ([multiformats/go-multiaddr#155](https://github.com/multiformats/go-multiaddr/pull/155)) - sync: update CI config files (#154) ([multiformats/go-multiaddr#154](https://github.com/multiformats/go-multiaddr/pull/154)) ### Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | vyzo | 8 | +205/-141 | 12 | | Marten Seemann | 7 | +127/-74 | 11 | | gammazero | 2 | +43/-5 | 3 | | Steven Allen | 1 | +13/-2 | 1 | | Adin Schmahmann | 3 | +13/-2 | 3 | | Marcin Rataj | 2 | +9/-1 | 2 | | Aaron Bieber | 1 | +6/-2 | 1 | ## v0.9.0 2021-06-22 We're happy to announce go-ipfs 0.9.0. This release makes go-ipfs even more configurable with some fun experiments to boot. We're also deprecating or removing some uncommonly used features to make it easier for users to discover the easy ways to use go-ipfs safely and efficiently. As usual, this release includes important fixes, some of which may be critical for security. Unless the fix addresses a bug being exploited in the wild, the fix will _not_ be called out in the release notes. Please make sure to update ASAP. See our [release process](https://github.com/ipfs/go-ipfs/tree/master/docs/releases.md#security-fix-policy) for details. ### 🔦 Highlights #### 📦 Exporting of DAGs via Gateways Gateways now support downloading arbitrary IPLD graphs via the `/api/v0/dag/export` endpoint. This endpoint works in the same way as the `ipfs dag export` command. One major thing this enables is ability to verify data downloaded from public gateways. If you go to `https://somegateway.example.net/ipfs/bafyexample` you are using the old school HTTP transport, and trusting that the gateway is being well behaved. However, if you download the graph as a [DAG archive](https://github.com/ipld/specs/blob/master/block-layer/content-addressable-archives.md) then it is possible to verify that the data you downloaded does in fact match `bafyexample`. Additionally, it was previously quite painful to download things other than UnixFS (files + directories) using gateways. It is now possible to download arbitrary IPLD graphs from gateways, making them useful as a general-purpose alternative to p2p transports. This opens exciting opportunities in areas like thin clients, mobile browsers and IoT devices, which now can delegate IPFS resolution to any public gateway, and have ability to verify that the data received matches the requested hash. #### ☁ Custom DNS Resolvers Resolution of DNS records for DNSLink and DNSAddrs means that names are sent in cleartext between the operating system and the DNS server provided by an ISP. In the past, the only way to customize DNS resolution in IPFS stack was to set up own DNS proxy server. There is now the ability to [customize DNS resolution](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#dns) and override the default resolver from the OS with [DNS over HTTPS](https://en.wikipedia.org/wiki/DNS_over_HTTPS) (DoH) one. We made it really flexible: override can be applied globally, or per specific [TLD](https://en.wikipedia.org/wiki/Top-level_domain)/[FQDN](https://en.wikipedia.org/wiki/Fully_qualified_domain_name). Examples can be found in the [documentation](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#dns). #### 👪 Support for non-ICANN DNSLink names Building off of the support for custom DNS resolvers it is now possible to create DNSLink names not handled by ICANN and choose how that domain name will be resolved. An example of this is how ENS is supported, despite `.eth` not being an ICANN TLD you can point `.eth` to any ENS resolver you want (including a local one). While go-ipfs may have some DoH defaults for a few popular non-ICANN DNSLink names (e.g. ENS), you are free to use any protocol for a naming system and as long as it exposes a DNSLink record via a DNS endpoint you can make it work. #### 🖥️ Updated to the latest WebUI Our web interface now includes experimental support for pinning services, and various updates to _Files_ and _Peers_ screens. Remote pinning services added via the `ipfs pin remote service add` command are already detected, one can also add one from _Settings_ screen, and it will appear in _Set pinning_ interface on the _Files_ screen. Data presented on the _Peers_ screen can now be copied by simply clicking on a specific cell, and a list of open streams gives better insight into how a local node interacts with a specific peer. See release notes for [ipfs-webui v2.12](https://github.com/ipfs/ipfs-webui/releases/tag/v2.12.0) for screenshots and more details. #### 🔑 IPNS keys can now be exported via the CLI without stopping the daemon `ipfs key export` no longer requires interrupting `ipfs daemon` ✨ #### 🕸 Experimental DHT Client and Provider System An area of go-ipfs that has been historically tricky is how go-ipfs finds who has the data they are looking for. While the IPFS Public DHT is only one of the ways go-ipfs can find data it tends to be an important one. While since go-ipfs v0.5.0 the time to find content in the network has dropped significantly the time to put/get IPNS records or for a node to advertise the content it has still has much room for improvement. We have been doing some experimenting and have an alternative DHT client that essentially trades off some resources and in return is much more performant. We have also included with the experimental DHT client a bulk provider system that takes advantage of the new client to more efficiently do many advertisements at a time This work is quite new and still under development, however, the results so far have been promising especially for users with lots of data who have otherwise been having difficulty advertising their data into the IPFS Public DHT As described in the experimental features [documentation](https://github.com/ipfs/go-ipfs/blob/master/docs/experimental-features.md#accelerated-dht-client) the experimental client can be enabled using the command below (or modifying the config file). ``` ipfs config --json Experimental.AcceleratedDHTClient true ``` A few things to take note of when `AcceleratedDHTClient` is enabled: - go-ipfs will likely use more resources then previously - DHT queries will not be usable (i.e. finding which peers have some data, finding where a particular peer is, etc.) for the first 5-10 minutes of operation depending on your network conditions - There is an `ipfs stats provide` command that will help you track your provide/reprovide usage, if you are providing lots of data you may want to consider how to reduce the amount you are providing (e.g. [Reprovider Strategies](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#reproviderstrategy) and/or [Strategic Providing](https://github.com/ipfs/go-ipfs/blob/master/docs/experimental-features.md#strategic-providing)) See the [documentation](https://github.com/ipfs/go-ipfs/blob/master/docs/experimental-features.md#accelerated-dht-client) for more details. #### 🚶‍♀️ Migrations ##### Migrations are now individually packaged While previously the go-ipfs [repo migration](https://github.com/ipfs/fs-repo-migrations) binary was monolithic and contained all migrations from previous go-ipfs versions the binaries are now packaged individually. However, the fs-repo-migrations binary is still there to help those who manually upgrade their repos to download all the individual migrations. This means faster download times for upgrades, a much easier time building migrations for those who make use of custom plugins, and an easier time developing new migrations going forward. ##### Configurable migration downloads enable downloading over IPFS Previously the migration downloader built into go-ipfs downloaded the migrations from [dist.ipfs.tech](https://dist.ipfs.tech). While users could use tools like [ipfs-update](https://github.com/ipfs/ipfs-update) to download the migrations over IPFS or manually download the migrations (over IPFS or otherwise) themselves, this is now automated and configurable. Users can choose to download the migrations over IPFS or from any specified IPFS Gateway. The configurable migration options are described in the config file [documentation](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#migration), although most users should not need to change the default settings. The main benefit here is that users behind restrictive firewalls, or in offline/private deployments, won't have to run migrations manually, which is especially important for desktop use cases where go-ipfs is running inside of [IPFS Desktop](https://github.com/ipfs-shipyard/ipfs-desktop#readme) and [Brave](https://brave.com/ipfs-support/). #### 🍎 Published builds for Apple M1 hardware Go now supports building for Darwin ARM64, and we are now publishing those builds #### 👋 Deprecations and Feature Removals ##### The `ipfs object` commands are now deprecated In the last couple years most of the Object API's commands have become fulfillable using alternative APIs. The utility of Object API's is limited to data in UnixFS-v1 (`dag-pb`) format. If you are still using it, it is highly recommended that you switch to the DAG `ipfs dag` (supports modern data types like `dag-cbor`) or Files `ipfs files` (more intuitive for working with `dag-pb`) APIs. While the Object API and commands are still usable they are now marked as deprecated and hidden from users on the command line to discourage further use. We also updated their `--help` text to point at the modern replacements. ##### `X-Ipfs-Gateway-Prefix` is now deprecated IPFS community moved towards dedicated Origins (DNSLink and [subdomain gateways](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway)) which are much easier to isolate and reason about. Setting up `Gateway.PathPrefixes` and `X-Ipfs-Gateway-Prefix` is no longer necessary and support [will be removed in near future](https://github.com/ipfs/go-ipfs/issues/7702). ##### Proquints support removed A little known feature that was not well used or documented and was more well known for the error message `Error: not a valid proquint string` users received when trying to download invalid IPNS or DNSLink names (e.g. `https://dweb.link/ipns/badname`). We have removed support for proquints as they were out of place and largely unused, however proquints are [valid multibases](https://github.com/multiformats/multibase/pull/78) so if there is renewed interest in them there is a way forward. ##### SECIO support removed SECIO was deprecated and turned off by default given the prevalence of TLS and Noise support, SECIO support is now removed entirely. ### Changelog - github.com/ipfs/go-ipfs: - chore: switch tar-utils dep to ipfs org - chore: update CHANGELOG - refactor: warning when bootstrap disabled by user - feat: print error on bootstrap failure - fix: typo in migration error - Release v0.9.0-rc2 - refactor: improved humanNumber and humanSI - feat: humanized durations in stat provide - feat: humanized numbers in stat provide - feat: add a text output encoding for the stats provide command - fix: webui-2.12.3 - refactor(pinmfs): log error if pre-existing pin failed (#8056) ([ipfs/go-ipfs#8056](https://github.com/ipfs/go-ipfs/pull/8056)) - Release v0.9.0-rc1 - Added support for an experimental DHT client and provider system via the Experiments.AcceleratedDHTClient config option ([ipfs/go-ipfs#8061](https://github.com/ipfs/go-ipfs/pull/8061)) - feat: support DNSLink on non-ICANN DNS names ([ipfs/go-ipfs#8071](https://github.com/ipfs/go-ipfs/pull/8071)) - update go-tcp-transport to v0.2.2 ([ipfs/go-ipfs#8129](https://github.com/ipfs/go-ipfs/pull/8129)) - update quic-go to v0.21.0-rc.1 ([ipfs/go-ipfs#8125](https://github.com/ipfs/go-ipfs/pull/8125)) - chore: bump minimum go version to 1.15 ([ipfs/go-ipfs#7944](https://github.com/ipfs/go-ipfs/pull/7944)) - chore: update deps ([ipfs/go-ipfs#8128](https://github.com/ipfs/go-ipfs/pull/8128)) - feat: allow key export in online mode ([ipfs/go-ipfs#8113](https://github.com/ipfs/go-ipfs/pull/8113)) - Feat/migration ipfs download (#8064) ([ipfs/go-ipfs#8064](https://github.com/ipfs/go-ipfs/pull/8064)) - feat: support custom DoH resolvers ([ipfs/go-ipfs#8068](https://github.com/ipfs/go-ipfs/pull/8068)) - update go-libp2p to v0.14.0 ([ipfs/go-ipfs#8122](https://github.com/ipfs/go-ipfs/pull/8122)) - feat(gw): expose /api/v0/dag/export on gateway port ([ipfs/go-ipfs#8111](https://github.com/ipfs/go-ipfs/pull/8111)) - chore: update webui to 2.12.2 ([ipfs/go-ipfs#8097](https://github.com/ipfs/go-ipfs/pull/8097)) - docs: deprecate object commands ([ipfs/go-ipfs#8098](https://github.com/ipfs/go-ipfs/pull/8098)) - fix: omit empty pins slice when reporting pin progress ([ipfs/go-ipfs#8023](https://github.com/ipfs/go-ipfs/pull/8023)) - Fix typo in comment ([ipfs/go-ipfs#8087](https://github.com/ipfs/go-ipfs/pull/8087)) - fix: set systemd startup timeout to infinity ([ipfs/go-ipfs#8040](https://github.com/ipfs/go-ipfs/pull/8040)) - fix(mkreleaselog): partially handle v2 modules ([ipfs/go-ipfs#8073](https://github.com/ipfs/go-ipfs/pull/8073)) - Update migration sharness tests for new migrations (#8053) ([ipfs/go-ipfs#8053](https://github.com/ipfs/go-ipfs/pull/8053)) - fix: make migrations log output to stdout ([ipfs/go-ipfs#8054](https://github.com/ipfs/go-ipfs/pull/8054)) - fix(gw): remove hardcoded hostnames ([ipfs/go-ipfs#8069](https://github.com/ipfs/go-ipfs/pull/8069)) - Fix transposed words in docs/config.md ([ipfs/go-ipfs#8051](https://github.com/ipfs/go-ipfs/pull/8051)) - fix: update root help ([ipfs/go-ipfs#8052](https://github.com/ipfs/go-ipfs/pull/8052)) - chore: don't docker tag rc as latest ([ipfs/go-ipfs#8055](https://github.com/ipfs/go-ipfs/pull/8055)) - Add info to "pin rm" help about how to tell if pin is indirect ([ipfs/go-ipfs#8044](https://github.com/ipfs/go-ipfs/pull/8044)) - build(deps): bump contrib.go.opencensus.io/exporter/prometheus from 0.2.0 to 0.3.0 ([ipfs/go-ipfs#8020](https://github.com/ipfs/go-ipfs/pull/8020)) - fix(gw): remove use of Clear-Site-Data in subdomain router ([ipfs/go-ipfs#7890](https://github.com/ipfs/go-ipfs/pull/7890)) - ([ipfs/go-ipfs#7857](https://github.com/ipfs/go-ipfs/pull/7857)) - docs: clarification of the Strategic Providing functionality ([ipfs/go-ipfs#8035](https://github.com/ipfs/go-ipfs/pull/8035)) - docs: cosmetic fixes of help text ([ipfs/go-ipfs#8010](https://github.com/ipfs/go-ipfs/pull/8010)) - chore: deprecate Gateway.PathPrefixes ([ipfs/go-ipfs#7994](https://github.com/ipfs/go-ipfs/pull/7994)) - Fix text contrast for dark mode ([ipfs/go-ipfs#8027](https://github.com/ipfs/go-ipfs/pull/8027)) - Do not fetch recursive pins from pinner unnecessarily ([ipfs/go-ipfs#7883](https://github.com/ipfs/go-ipfs/pull/7883)) - test(sharness): verify the list of exported metrics ([ipfs/go-ipfs#7987](https://github.com/ipfs/go-ipfs/pull/7987)) - fix: return an error if repo verify is canceled ([ipfs/go-ipfs#7973](https://github.com/ipfs/go-ipfs/pull/7973)) - doc: document security fix policy ([ipfs/go-ipfs#7991](https://github.com/ipfs/go-ipfs/pull/7991)) - feat(gw): /ipfs/ipfs/{cid} → /ipfs/{cid} ([ipfs/go-ipfs#7930](https://github.com/ipfs/go-ipfs/pull/7930)) - Fix: inaccuracies in MFS command documentation. ([ipfs/go-ipfs#8001](https://github.com/ipfs/go-ipfs/pull/8001)) - Feat: Re-import InitializeKeyspace code from go-namesys ([ipfs/go-ipfs#7984](https://github.com/ipfs/go-ipfs/pull/7984)) - revert registration of metrics against unexposed prom registry ([ipfs/go-ipfs#7986](https://github.com/ipfs/go-ipfs/pull/7986)) - Extract the namesys and the keystore submodules ([ipfs/go-ipfs#7925](https://github.com/ipfs/go-ipfs/pull/7925)) - split core/commands/dag into individual files for different subcommands ([ipfs/go-ipfs#7970](https://github.com/ipfs/go-ipfs/pull/7970)) - test(sharness): pass correct timeout format to go-timeout ([ipfs/go-ipfs#7971](https://github.com/ipfs/go-ipfs/pull/7971)) - fix race condition when logging requests ([ipfs/go-ipfs#7953](https://github.com/ipfs/go-ipfs/pull/7953)) - fix some sharness-in-CI issues ([ipfs/go-ipfs#7946](https://github.com/ipfs/go-ipfs/pull/7946)) - chore: update deps ([ipfs/go-ipfs#7941](https://github.com/ipfs/go-ipfs/pull/7941)) - fix: correctly return pin ls errors ([ipfs/go-ipfs#7942](https://github.com/ipfs/go-ipfs/pull/7942)) - feat: remove secio support ([ipfs/go-ipfs#7943](https://github.com/ipfs/go-ipfs/pull/7943)) - Set supported platforms by go-version ([ipfs/go-ipfs#7927](https://github.com/ipfs/go-ipfs/pull/7927)) - docs: tips on debugging Policies.MFS (#7929) ([ipfs/go-ipfs#7929](https://github.com/ipfs/go-ipfs/pull/7929)) - docs: fix DNSLink gw recipe ([ipfs/go-ipfs#7932](https://github.com/ipfs/go-ipfs/pull/7932)) - Merge branch 'release' - docs: RepinInterval - style: docs/config.md - style: improved MFS PinName example - docs: Pinning.RemoteServices.Policies - peering: add logs before many-second waits ([ipfs/go-ipfs#7904](https://github.com/ipfs/go-ipfs/pull/7904)) - all: gofmt -s ([ipfs/go-ipfs#7900](https://github.com/ipfs/go-ipfs/pull/7900)) - github.com/ipfs/go-bitswap (v0.3.3 -> v0.3.4): - remove Makefile ([ipfs/go-bitswap#483](https://github.com/ipfs/go-bitswap/pull/483)) - test: deflake engine test ([ipfs/go-bitswap#480](https://github.com/ipfs/go-bitswap/pull/480)) - test: deflake large-message test ([ipfs/go-bitswap#479](https://github.com/ipfs/go-bitswap/pull/479)) - fix: fix alignment of stats struct in virtual network ([ipfs/go-bitswap#478](https://github.com/ipfs/go-bitswap/pull/478)) - fix(network): impl: add timeout in newStreamToPeer call ([ipfs/go-bitswap#477](https://github.com/ipfs/go-bitswap/pull/477)) - fix staticcheck ([ipfs/go-bitswap#474](https://github.com/ipfs/go-bitswap/pull/474)) - ignore transient connections ([ipfs/go-bitswap#470](https://github.com/ipfs/go-bitswap/pull/470)) - fix a startup race by creating the blockstoremanager process on init ([ipfs/go-bitswap#465](https://github.com/ipfs/go-bitswap/pull/465)) - github.com/ipfs/go-block-format (v0.0.2 -> v0.0.3): - doc: add a lead maintainer ([ipfs/go-block-format#16](https://github.com/ipfs/go-block-format/pull/16)) - github.com/ipfs/go-graphsync (v0.6.0 -> v0.8.0): - docs(CHANGELOG): update for v0.8.0 - Update for LinkSystem (#161) ([ipfs/go-graphsync#161](https://github.com/ipfs/go-graphsync/pull/161)) - Round out diagnostic parameters (#157) ([ipfs/go-graphsync#157](https://github.com/ipfs/go-graphsync/pull/157)) - map response codes to names (#148) ([ipfs/go-graphsync#148](https://github.com/ipfs/go-graphsync/pull/148)) - Discard http output (#156) ([ipfs/go-graphsync#156](https://github.com/ipfs/go-graphsync/pull/156)) - Add debug logging (#121) ([ipfs/go-graphsync#121](https://github.com/ipfs/go-graphsync/pull/121)) - Add optional HTTP comparison (#153) ([ipfs/go-graphsync#153](https://github.com/ipfs/go-graphsync/pull/153)) - docs(architecture): update architecture docs (#154) ([ipfs/go-graphsync#154](https://github.com/ipfs/go-graphsync/pull/154)) - release v0.7.0 ([ipfs/go-graphsync#152](https://github.com/ipfs/go-graphsync/pull/152)) - chore: update deps (#151) ([ipfs/go-graphsync#151](https://github.com/ipfs/go-graphsync/pull/151)) - Automatically record heap profiles in test plans (#147) ([ipfs/go-graphsync#147](https://github.com/ipfs/go-graphsync/pull/147)) - feat(deps): update go-ipld-prime v0.7.0 (#145) ([ipfs/go-graphsync#145](https://github.com/ipfs/go-graphsync/pull/145)) - Release/v0.6.0 ([ipfs/go-graphsync#144](https://github.com/ipfs/go-graphsync/pull/144)) - github.com/ipfs/go-ipfs-blockstore (v0.1.4 -> v0.1.6): - use bloom filter in GetSize - github.com/ipfs/go-ipfs-config (v0.12.0 -> v0.14.0): - Added Experiments.AcceleratedDHTClient option ([ipfs/go-ipfs-config#125](https://github.com/ipfs/go-ipfs-config/pull/125)) - Add config for downloading repo migrations ([ipfs/go-ipfs-config#128](https://github.com/ipfs/go-ipfs-config/pull/128)) - remove duplicate entries in defaultServerFilters ([ipfs/go-ipfs-config#121](https://github.com/ipfs/go-ipfs-config/pull/121)) - add custom DNS Resolver configuration ([ipfs/go-ipfs-config#126](https://github.com/ipfs/go-ipfs-config/pull/126)) - github.com/ipfs/go-ipfs-provider (v0.4.3 -> v0.5.1): - Fix batched providing of empty keys ([ipfs/go-ipfs-provider#37](https://github.com/ipfs/go-ipfs-provider/pull/37)) - Bulk Provide/Reproviding System (#34) ([ipfs/go-ipfs-provider#34](https://github.com/ipfs/go-ipfs-provider/pull/34)) - chore: update the Usage part of readme ([ipfs/go-ipfs-provider#33](https://github.com/ipfs/go-ipfs-provider/pull/33)) - Retract and revert 1.0.0 ([ipfs/go-ipfs-provider#31](https://github.com/ipfs/go-ipfs-provider/pull/31)) - replace go-merkledag with go-fetcher ([ipfs/go-ipfs-provider#30](https://github.com/ipfs/go-ipfs-provider/pull/30)) - github.com/ipfs/go-ipld-git (v0.0.3 -> v0.0.4): - add license file so it can be found by go-licenses ([ipfs/go-ipld-git#42](https://github.com/ipfs/go-ipld-git/pull/42)) - github.com/ipfs/go-ipns (v0.0.2 -> v0.1.0): - Add support for extensible records (and v2 signature) - github.com/ipfs/go-log (v1.0.4 -> v1.0.5): - chore: update v1 deps ([ipfs/go-log#108](https://github.com/ipfs/go-log/pull/108)) - github.com/ipfs/go-log/v2 (v2.1.1 -> v2.1.3): - doc(README): use circle-ci badge ([ipfs/go-log#106](https://github.com/ipfs/go-log/pull/106)) - feat: add ability to specify labels for all loggers ([ipfs/go-log#105](https://github.com/ipfs/go-log/pull/105)) - Add an option to pass URL to zap ([ipfs/go-log#101](https://github.com/ipfs/go-log/pull/101)) - enable configuring several log outputs ([ipfs/go-log#98](https://github.com/ipfs/go-log/pull/98)) - Fix caller not being added ([ipfs/go-log#96](https://github.com/ipfs/go-log/pull/96)) - github.com/ipfs/go-unixfs (v0.2.4 -> v0.2.5): - correct file size for raw node ([ipfs/go-unixfs#88](https://github.com/ipfs/go-unixfs/pull/88)) - github.com/ipld/go-car (v0.1.1-0.20201015032735-ff6ccdc46acc -> v0.3.1): - chore: make sure we get an error where we expect one - chore: refactor header tests to iterate over a struct - chore: add header error tests - fix: lint errors - fix: go mod tidy - chore: update go.mod to 1.15 - fix: ReadHeader return value mismatch - Updates for ipld linksystem branch ([ipld/go-car#56](https://github.com/ipld/go-car/pull/56)) - replace go-ipld-prime-proto with go-codec-dagpb - fix staticcheck errors ([ipld/go-car#67](https://github.com/ipld/go-car/pull/67)) - chore: switch to a single license file ([ipld/go-car#59](https://github.com/ipld/go-car/pull/59)) - chore: remove LICENSE ([ipld/go-car#58](https://github.com/ipld/go-car/pull/58)) - chore: relicense ([ipld/go-car#57](https://github.com/ipld/go-car/pull/57)) - ci: remove travis support ([ipld/go-car#55](https://github.com/ipld/go-car/pull/55)) - run gofmt -s - Allow user defined block hooks when using two step write for selective cars ([ipld/go-car#37](https://github.com/ipld/go-car/pull/37)) - feat: handle mid-varint EOF case as UnexpectedEOF - fix: main NewReader call - github.com/ipld/go-ipld-prime (v0.5.1-0.20201021195245-109253e8a018 -> v0.9.1-0.20210324083106-dc342a9917db): - Add option to tell link system storage is trusted and we can skip hash on read ([ipld/go-ipld-prime#149](https://github.com/ipld/go-ipld-prime/pull/149)) - implement non-dag cbor codec ([ipld/go-ipld-prime#153](https://github.com/ipld/go-ipld-prime/pull/153)) - add non-dag json codec ([ipld/go-ipld-prime#152](https://github.com/ipld/go-ipld-prime/pull/152)) - typo fixes - mark v0.9.0 - Changelog: more backfill :) - hackme: about merge strategies. - Dropping .gopath and other unmaintained scripts. - introduce LinkSystem ([ipld/go-ipld-prime#143](https://github.com/ipld/go-ipld-prime/pull/143)) - Readme updates. - codec/raw: implement the raw codec - add an ADL interface type - schema/gen/go: cache genned code in os.TempDir - fluent/qp: finish writing all data model helpers - fluent: add qp, a different spin on quip - schema/gen/go: prevent some unkeyed literal vet errors - schema/gen/go: remove two common subtest levels - use %q in error strings - schema/gen/go: please vet a bit more - Introduce 'quip' data building helpers. ([ipld/go-ipld-prime#134](https://github.com/ipld/go-ipld-prime/pull/134)) - gengo: support for unions with stringprefix representation. ([ipld/go-ipld-prime#133](https://github.com/ipld/go-ipld-prime/pull/133)) - target of opportunity DRY improvement: use more shared templates for structs with stringjoin representations. - fix small consistency typo in gen function names. - drop old generation mechanisms that were already deprecated. - error type cleanup, and helpers. - v0.7.0 and changelog update - Revert "rename AssignNode to ConvertFrom" - Implement traversal.FocusedTransform. ([ipld/go-ipld-prime#130](https://github.com/ipld/go-ipld-prime/pull/130)) - Update a few more lingering ReprKind references. - all: rename schema.Kind to TypeKind, ipld.ReprKind to Kind ([ipld/go-ipld-prime#127](https://github.com/ipld/go-ipld-prime/pull/127)) - all: rename AssignNode to ConvertFrom - all: rewrite interfaces and APIs to support int64 - mark v0.6.0 - clean up node/gendemo regeneration ([ipld/go-ipld-prime#123](https://github.com/ipld/go-ipld-prime/pull/123)) - cleanup: drop orphaned gitignore file. - Schema types rebased to use codegen types for the data ([ipld/go-ipld-prime#107](https://github.com/ipld/go-ipld-prime/pull/107)) - codegen: assembler for struct with map representation validates all non-optional fields are present ([ipld/go-ipld-prime#121](https://github.com/ipld/go-ipld-prime/pull/121)) - changelog: backfill. - fluent: finish out matrix of helper methods, and fix error handling of the non-Must methods. - all: fix a lot of "unkeyed literal" vet warnings - node/mixins: use simpler filenames - node/gendemo: use the new code generator - Merge pull request #96 , originally known as ipld/cidlink-only-usable-as-ptr - Codec revamp ([ipld/go-ipld-prime#112](https://github.com/ipld/go-ipld-prime/pull/112)) - Allow overridden types (#116) ([ipld/go-ipld-prime#116](https://github.com/ipld/go-ipld-prime/pull/116)) - add import to ipld in ipldsch_types.go ([ipld/go-ipld-prime#115](https://github.com/ipld/go-ipld-prime/pull/115)) - Codegen output rearrange ([ipld/go-ipld-prime#105](https://github.com/ipld/go-ipld-prime/pull/105)) - Validate struct builder sufficiency ([ipld/go-ipld-prime#111](https://github.com/ipld/go-ipld-prime/pull/111)) - Fresh take on codec APIs, and some tokenization utilities. ([ipld/go-ipld-prime#101](https://github.com/ipld/go-ipld-prime/pull/101)) - Add a demo ADL (rot13adl) ([ipld/go-ipld-prime#98](https://github.com/ipld/go-ipld-prime/pull/98)) - Introduce traversal function that selects links out of a tree. ([ipld/go-ipld-prime#110](https://github.com/ipld/go-ipld-prime/pull/110)) - Codegen various improvements ([ipld/go-ipld-prime#106](https://github.com/ipld/go-ipld-prime/pull/106)) - github.com/libp2p/go-conn-security-multistream (v0.2.0 -> v0.2.1): - Implement support for simultaneous open (#14) ([libp2p/go-conn-security-multistream#14](https://github.com/libp2p/go-conn-security-multistream/pull/14)) - github.com/libp2p/go-libp2p (v0.13.0 -> v0.14.2): - Fix race in adding connections to connsByPeer ([libp2p/go-libp2p#1116](https://github.com/libp2p/go-libp2p/pull/1116)) - speed up the mock tests ([libp2p/go-libp2p#1103](https://github.com/libp2p/go-libp2p/pull/1103)) - remove slow ObservedAddrManager test that doesn't test anything ([libp2p/go-libp2p#1104](https://github.com/libp2p/go-libp2p/pull/1104)) - remove Codecov config ([libp2p/go-libp2p#1100](https://github.com/libp2p/go-libp2p/pull/1100)) - doc: document standard connection manager ([libp2p/go-libp2p#1099](https://github.com/libp2p/go-libp2p/pull/1099)) - run go mod tidy in the examples ([libp2p/go-libp2p#1098](https://github.com/libp2p/go-libp2p/pull/1098)) - Cleanup some remaining examples nits ([libp2p/go-libp2p#1097](https://github.com/libp2p/go-libp2p/pull/1097)) - chore: bring examples back into repository and add tests ([libp2p/go-libp2p#1092](https://github.com/libp2p/go-libp2p/pull/1092)) - fix(mkreleasenotes): handle first commit ([libp2p/go-libp2p#1095](https://github.com/libp2p/go-libp2p/pull/1095)) - doc: add a basic release process ([libp2p/go-libp2p#1080](https://github.com/libp2p/go-libp2p/pull/1080)) - chore: update yamux ([libp2p/go-libp2p#1089](https://github.com/libp2p/go-libp2p/pull/1089)) - fix: re-expose AutoNAT service on BasicHost ([libp2p/go-libp2p#1088](https://github.com/libp2p/go-libp2p/pull/1088)) - remove NEWS.md ([libp2p/go-libp2p#1086](https://github.com/libp2p/go-libp2p/pull/1086)) - test: deflake TestProtoDowngrade ([libp2p/go-libp2p#1084](https://github.com/libp2p/go-libp2p/pull/1084)) - sync: update CI config files (and fix tests) ([libp2p/go-libp2p#1083](https://github.com/libp2p/go-libp2p/pull/1083)) - static check fixes ([libp2p/go-libp2p#1076](https://github.com/libp2p/go-libp2p/pull/1076)) - fix go vet ([libp2p/go-libp2p#1075](https://github.com/libp2p/go-libp2p/pull/1075)) - option for custom dns resolver ([libp2p/go-libp2p#1073](https://github.com/libp2p/go-libp2p/pull/1073)) - chore: update deps ([libp2p/go-libp2p#1066](https://github.com/libp2p/go-libp2p/pull/1066)) - fix autonat race ([libp2p/go-libp2p#1062](https://github.com/libp2p/go-libp2p/pull/1062)) - use transient connections in identify streams ([libp2p/go-libp2p#1061](https://github.com/libp2p/go-libp2p/pull/1061)) - Emit event for User's NAT Type i.e. Hard NAT or Easy NAT (#1042) ([libp2p/go-libp2p#1042](https://github.com/libp2p/go-libp2p/pull/1042)) - Finish and Test the simultaneous connect problem in libp2p peers (#1041) ([libp2p/go-libp2p#1041](https://github.com/libp2p/go-libp2p/pull/1041)) - Close peerstore and document Host Close (#1037) ([libp2p/go-libp2p#1037](https://github.com/libp2p/go-libp2p/pull/1037)) - Timeout all Identify stream reads (#1032) ([libp2p/go-libp2p#1032](https://github.com/libp2p/go-libp2p/pull/1032)) - github.com/libp2p/go-libp2p-autonat (v0.4.0 -> v0.4.2): - Fix: Stream read timeout ([libp2p/go-libp2p-autonat#99](https://github.com/libp2p/go-libp2p-autonat/pull/99)) - fix: simplify address replacement ([libp2p/go-libp2p-autonat#102](https://github.com/libp2p/go-libp2p-autonat/pull/102)) - replace the port number for double NAT mapping ([libp2p/go-libp2p-autonat#101](https://github.com/libp2p/go-libp2p-autonat/pull/101)) - github.com/libp2p/go-libp2p-core (v0.8.0 -> v0.8.5): - mind the dot. - context option for simultaneous connect - Event for user's NAT Device Type: Tell user if the node is behind an Easy or Hard NAT (#173) ([libp2p/go-libp2p-core#173](https://github.com/libp2p/go-libp2p-core/pull/173)) - address aarshian nitpicks - make UseTransient context option take a reason argument, for consistency with other options - abstract Conn Stat interface for threading - Update network/context.go - add ErrTransientConn error - add support for transient connections - more docs for stream fncs (#183) ([libp2p/go-libp2p-core#183](https://github.com/libp2p/go-libp2p-core/pull/183)) - refactor: use a helper type to decode AddrInfo from JSON (#178) ([libp2p/go-libp2p-core#178](https://github.com/libp2p/go-libp2p-core/pull/178)) - fix stream docs (#182) ([libp2p/go-libp2p-core#182](https://github.com/libp2p/go-libp2p-core/pull/182)) - context to force direct dial (#181) ([libp2p/go-libp2p-core#181](https://github.com/libp2p/go-libp2p-core/pull/181)) - Secure Muxer Interface (#180) ([libp2p/go-libp2p-core#180](https://github.com/libp2p/go-libp2p-core/pull/180)) - github.com/libp2p/go-libp2p-discovery (v0.5.0 -> v0.5.1): - Fix hang in BackoffDiscovery.FindPeers when requesting limit lower than number of peers available ([libp2p/go-libp2p-discovery#69](https://github.com/libp2p/go-libp2p-discovery/pull/69)) - fix staticcheck ([libp2p/go-libp2p-discovery#70](https://github.com/libp2p/go-libp2p-discovery/pull/70)) - github.com/libp2p/go-libp2p-kad-dht (v0.11.1 -> v0.12.2): - fullrt rework batching (#720) ([libp2p/go-libp2p-kad-dht#720](https://github.com/libp2p/go-libp2p-kad-dht/pull/720)) - sync: update CI config files ([libp2p/go-libp2p-kad-dht#712](https://github.com/libp2p/go-libp2p-kad-dht/pull/712)) - fix staticcheck ([libp2p/go-libp2p-kad-dht#721](https://github.com/libp2p/go-libp2p-kad-dht/pull/721)) - fix: fullrt dht bug fixes ([libp2p/go-libp2p-kad-dht#719](https://github.com/libp2p/go-libp2p-kad-dht/pull/719)) - Crawler based DHT client (#709) ([libp2p/go-libp2p-kad-dht#709](https://github.com/libp2p/go-libp2p-kad-dht/pull/709)) - test: fix unique addr check ([libp2p/go-libp2p-kad-dht#714](https://github.com/libp2p/go-libp2p-kad-dht/pull/714)) - chore: update deps ([libp2p/go-libp2p-kad-dht#713](https://github.com/libp2p/go-libp2p-kad-dht/pull/713)) - Add basic crawler (#663) ([libp2p/go-libp2p-kad-dht#663](https://github.com/libp2p/go-libp2p-kad-dht/pull/663)) - various staticcheck fixes ([libp2p/go-libp2p-kad-dht#710](https://github.com/libp2p/go-libp2p-kad-dht/pull/710)) - findpeer should work even on peers that are not part of DHT queries ([libp2p/go-libp2p-kad-dht#711](https://github.com/libp2p/go-libp2p-kad-dht/pull/711)) - Extract DHT message sender from the DHT ([libp2p/go-libp2p-kad-dht#659](https://github.com/libp2p/go-libp2p-kad-dht/pull/659)) - github.com/libp2p/go-libp2p-noise (v0.1.2 -> v0.2.0): - Update github.com/flynn/noise to address nonce handling security issues ([libp2p/go-libp2p-noise#95](https://github.com/libp2p/go-libp2p-noise/pull/95)) - fix staticcheck ([libp2p/go-libp2p-noise#96](https://github.com/libp2p/go-libp2p-noise/pull/96)) - chore: update deps ([libp2p/go-libp2p-noise#94](https://github.com/libp2p/go-libp2p-noise/pull/94)) - chore: relicense MIT/Apache-2.0 ([libp2p/go-libp2p-noise#93](https://github.com/libp2p/go-libp2p-noise/pull/93)) - github.com/libp2p/go-libp2p-peerstore (v0.2.6 -> v0.2.7): - fix: delete addrs when "updating" them to zero ([libp2p/go-libp2p-peerstore#157](https://github.com/libp2p/go-libp2p-peerstore/pull/157)) - github.com/libp2p/go-libp2p-quic-transport (v0.10.0 -> v0.11.1): - update quic-go, enable QUIC v1 (RFC 9000) ([libp2p/go-libp2p-quic-transport#207](https://github.com/libp2p/go-libp2p-quic-transport/pull/207)) - update quic-go to v0.21.0-rc2 ([libp2p/go-libp2p-quic-transport#206](https://github.com/libp2p/go-libp2p-quic-transport/pull/206)) - increase test timeout to reduce flakiness of test on Windows ([libp2p/go-libp2p-quic-transport#204](https://github.com/libp2p/go-libp2p-quic-transport/pull/204)) - correctly export version negotiation failures to Prometheus ([libp2p/go-libp2p-quic-transport#205](https://github.com/libp2p/go-libp2p-quic-transport/pull/205)) - update quic-go to v0.20.1 ([libp2p/go-libp2p-quic-transport#201](https://github.com/libp2p/go-libp2p-quic-transport/pull/201)) - expose some Prometheus metrics ([libp2p/go-libp2p-quic-transport#200](https://github.com/libp2p/go-libp2p-quic-transport/pull/200)) - update quic-go to v0.20.0 ([libp2p/go-libp2p-quic-transport#198](https://github.com/libp2p/go-libp2p-quic-transport/pull/198)) - reduce the zstd window size from 8 MB to 32 KB ([libp2p/go-libp2p-quic-transport#195](https://github.com/libp2p/go-libp2p-quic-transport/pull/195)) - compress qlogs when the QUIC connection is closed ([libp2p/go-libp2p-quic-transport#193](https://github.com/libp2p/go-libp2p-quic-transport/pull/193)) - switch from gzip to zstd for qlog compression ([libp2p/go-libp2p-quic-transport#190](https://github.com/libp2p/go-libp2p-quic-transport/pull/190)) - github.com/libp2p/go-libp2p-swarm (v0.4.0 -> v0.5.0): - run connection gating tests on both TCP and QUIC ([libp2p/go-libp2p-swarm#258](https://github.com/libp2p/go-libp2p-swarm/pull/258)) - fix: avoid returning typed nils ([libp2p/go-libp2p-swarm#257](https://github.com/libp2p/go-libp2p-swarm/pull/257)) - fix staticcheck ([libp2p/go-libp2p-swarm#255](https://github.com/libp2p/go-libp2p-swarm/pull/255)) - fix go vet ([libp2p/go-libp2p-swarm#253](https://github.com/libp2p/go-libp2p-swarm/pull/253)) - New Dialer ([libp2p/go-libp2p-swarm#243](https://github.com/libp2p/go-libp2p-swarm/pull/243)) - fix: use 64bit stream/conn IDs ([libp2p/go-libp2p-swarm#247](https://github.com/libp2p/go-libp2p-swarm/pull/247)) - feat: close transports that implement io.Closer ([libp2p/go-libp2p-swarm#227](https://github.com/libp2p/go-libp2p-swarm/pull/227)) - fix swarm transient conn (#241) ([libp2p/go-libp2p-swarm#241](https://github.com/libp2p/go-libp2p-swarm/pull/241)) - Support for Hole punching (#233) ([libp2p/go-libp2p-swarm#233](https://github.com/libp2p/go-libp2p-swarm/pull/233)) - Treat transient connections as opt-in when opening new streams ([libp2p/go-libp2p-swarm#236](https://github.com/libp2p/go-libp2p-swarm/pull/236)) - avoid assigning a function to a variable ([libp2p/go-libp2p-swarm#239](https://github.com/libp2p/go-libp2p-swarm/pull/239)) - only listen on localhost in tests ([libp2p/go-libp2p-swarm#238](https://github.com/libp2p/go-libp2p-swarm/pull/238)) - prevent dialing addresses that we're listening on ([libp2p/go-libp2p-swarm#237](https://github.com/libp2p/go-libp2p-swarm/pull/237)) - Enable QUIC in Test Swarm (#235) ([libp2p/go-libp2p-swarm#235](https://github.com/libp2p/go-libp2p-swarm/pull/235)) - github.com/libp2p/go-libp2p-transport-upgrader (v0.4.0 -> v0.4.2): - Expose underlying transport connection stat where available ([libp2p/go-libp2p-transport-upgrader#71](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/71)) - Implement support for simultaneous open (#25) ([libp2p/go-libp2p-transport-upgrader#25](https://github.com/libp2p/go-libp2p-transport-upgrader/pull/25)) - github.com/libp2p/go-libp2p-yamux (v0.5.1 -> v0.5.4): - remove Makefile ([libp2p/go-libp2p-yamux#35](https://github.com/libp2p/go-libp2p-yamux/pull/35)) - github.com/libp2p/go-netroute (v0.1.3 -> v0.1.6): - add js stub impl - github.com/libp2p/go-sockaddr (v0.0.2 -> v0.1.1): - fix: allocate "any" socket type then cast ([libp2p/go-sockaddr#20](https://github.com/libp2p/go-sockaddr/pull/20)) - fix: remove CGO functions ([libp2p/go-sockaddr#18](https://github.com/libp2p/go-sockaddr/pull/18)) - github.com/libp2p/go-tcp-transport (v0.2.1 -> v0.2.2): - use log.Warn instead of log.Warning ([libp2p/go-tcp-transport#77](https://github.com/libp2p/go-tcp-transport/pull/77)) - add bandwidth-related metrics (for Linux and OSX) ([libp2p/go-tcp-transport#76](https://github.com/libp2p/go-tcp-transport/pull/76)) - expose some Prometheus metrics ([libp2p/go-tcp-transport#75](https://github.com/libp2p/go-tcp-transport/pull/75)) - enable TCP keepalives ([libp2p/go-tcp-transport#73](https://github.com/libp2p/go-tcp-transport/pull/73)) - stop using the deprecated go-multiaddr-net package ([libp2p/go-tcp-transport#72](https://github.com/libp2p/go-tcp-transport/pull/72)) - github.com/libp2p/go-yamux/v2 (v2.0.0 -> v2.2.0): - make the initial stream receive window configurable ([libp2p/go-yamux#59](https://github.com/libp2p/go-yamux/pull/59)) - set initial window size to spec value (256 kB), remove config option ([libp2p/go-yamux#57](https://github.com/libp2p/go-yamux/pull/57)) - fix: don't change the receive window if we're forcing an update ([libp2p/go-yamux#56](https://github.com/libp2p/go-yamux/pull/56)) - sync: update CI config files ([libp2p/go-yamux#55](https://github.com/libp2p/go-yamux/pull/55)) - increase the receive window size if we're sending updates to frequently ([libp2p/go-yamux#54](https://github.com/libp2p/go-yamux/pull/54)) - remove unused Stream.Shrink() method ([libp2p/go-yamux#52](https://github.com/libp2p/go-yamux/pull/52)) - remove misleading comment about the MaxMessageSize ([libp2p/go-yamux#50](https://github.com/libp2p/go-yamux/pull/50)) - clean up the receive window check ([libp2p/go-yamux#49](https://github.com/libp2p/go-yamux/pull/49)) - don't reslice byte slices taking from the buffer ([libp2p/go-yamux#48](https://github.com/libp2p/go-yamux/pull/48)) - don't reimplement io.ReadFull ([libp2p/go-yamux#38](https://github.com/libp2p/go-yamux/pull/38)) - remove the recvLock in the stream ([libp2p/go-yamux#42](https://github.com/libp2p/go-yamux/pull/42)) - remove the sendLock in the stream ([libp2p/go-yamux#41](https://github.com/libp2p/go-yamux/pull/41)) - remove misleading statement about NAT traversal ([libp2p/go-yamux#45](https://github.com/libp2p/go-yamux/pull/45)) - remove .gx directory, add last gx version to README ([libp2p/go-yamux#43](https://github.com/libp2p/go-yamux/pull/43)) - reduce usage of goto ([libp2p/go-yamux#40](https://github.com/libp2p/go-yamux/pull/40)) - remove unused error return value in Stream.processFlags ([libp2p/go-yamux#39](https://github.com/libp2p/go-yamux/pull/39)) - github.com/lucas-clemente/quic-go (v0.19.3 -> v0.21.1): - add support for Go 1.17 Beta 1 ([lucas-clemente/quic-go#3203](https://github.com/lucas-clemente/quic-go/pull/3203)) - add a CI test that go mod vendor works ([lucas-clemente/quic-go#3202](https://github.com/lucas-clemente/quic-go/pull/3202)) - prevent go mod vendor from stumbling over the Go 1.18 file ([lucas-clemente/quic-go#3195](https://github.com/lucas-clemente/quic-go/pull/3195)) - remove CipherSuiteName and HkdfExtract for Go 1.17 ([lucas-clemente/quic-go#3192](https://github.com/lucas-clemente/quic-go/pull/3192)) - fix relocation target for cipherSuiteTLS13ByID in Go 1.17 - use HkdfExtract from x/crypto ([lucas-clemente/quic-go#3173](https://github.com/lucas-clemente/quic-go/pull/3173)) - add support for QUIC v1, RFC 9000 ([lucas-clemente/quic-go#3190](https://github.com/lucas-clemente/quic-go/pull/3190)) - use tls.CipherSuiteName, instead of wrapping it in the qtls package ([lucas-clemente/quic-go#3174](https://github.com/lucas-clemente/quic-go/pull/3174)) - use a pre-generated test vectors to test hkdfExpandLabel ([lucas-clemente/quic-go#3175](https://github.com/lucas-clemente/quic-go/pull/3175)) - reduce flakiness of packet number generation test ([lucas-clemente/quic-go#3181](https://github.com/lucas-clemente/quic-go/pull/3181)) - simplify the qtls tests ([lucas-clemente/quic-go#3185](https://github.com/lucas-clemente/quic-go/pull/3185)) - add support for Go 1.17 (tip) ([lucas-clemente/quic-go#3182](https://github.com/lucas-clemente/quic-go/pull/3182)) - prevent quic-go from building on Go 1.17 ([lucas-clemente/quic-go#3180](https://github.com/lucas-clemente/quic-go/pull/3180)) - fix DONT_FRAGMENT error when using a IPv6 connection on Windows ([lucas-clemente/quic-go#3178](https://github.com/lucas-clemente/quic-go/pull/3178)) - use net.ErrClosed (for Go 1.16) ([lucas-clemente/quic-go#3163](https://github.com/lucas-clemente/quic-go/pull/3163)) - use the new error types to log the reason why a connection is closed ([lucas-clemente/quic-go#3166](https://github.com/lucas-clemente/quic-go/pull/3166)) - fix race condition in deadline integration test ([lucas-clemente/quic-go#3165](https://github.com/lucas-clemente/quic-go/pull/3165)) - add support for QUIC v1 ([lucas-clemente/quic-go#3160](https://github.com/lucas-clemente/quic-go/pull/3160)) - rework error return values ([lucas-clemente/quic-go#3159](https://github.com/lucas-clemente/quic-go/pull/3159)) - declare Path MTU probe packets lost with the early retransmit timer ([lucas-clemente/quic-go#3152](https://github.com/lucas-clemente/quic-go/pull/3152)) - declare the handshake confirmed when receiving an ACK for a 1-RTT packet ([lucas-clemente/quic-go#3148](https://github.com/lucas-clemente/quic-go/pull/3148)) - trace and qlog version selection / negotiation ([lucas-clemente/quic-go#3153](https://github.com/lucas-clemente/quic-go/pull/3153)) - set the don't fragment (DF) bit on Windows (#3155) ([lucas-clemente/quic-go#3155](https://github.com/lucas-clemente/quic-go/pull/3155)) - fix doc comment for Tracer.TracerForConnection ([lucas-clemente/quic-go#3154](https://github.com/lucas-clemente/quic-go/pull/3154)) - make it possible to associate a ConnectionTracer with a Session ([lucas-clemente/quic-go#3146](https://github.com/lucas-clemente/quic-go/pull/3146)) - remove the .editorconfig ([lucas-clemente/quic-go#3147](https://github.com/lucas-clemente/quic-go/pull/3147)) - don't use a lower RTT than 5ms after receiving a Retry packet ([lucas-clemente/quic-go#3129](https://github.com/lucas-clemente/quic-go/pull/3129)) - don't pass the QUIC version to the StartedConnection event ([lucas-clemente/quic-go#3109](https://github.com/lucas-clemente/quic-go/pull/3109)) - update the packet numbers in decoding test to the ones from the draft ([lucas-clemente/quic-go#3137](https://github.com/lucas-clemente/quic-go/pull/3137)) - various amplification limit fixes ([lucas-clemente/quic-go#3132](https://github.com/lucas-clemente/quic-go/pull/3132)) - fix calculation of the handshake idle timeout ([lucas-clemente/quic-go#3120](https://github.com/lucas-clemente/quic-go/pull/3120)) - only start PMTUD after handshake confirmation ([lucas-clemente/quic-go#3138](https://github.com/lucas-clemente/quic-go/pull/3138)) - don't regard PMTU probe packets as outstanding ([lucas-clemente/quic-go#3126](https://github.com/lucas-clemente/quic-go/pull/3126)) - expose the draft-34 version ([lucas-clemente/quic-go#3100](https://github.com/lucas-clemente/quic-go/pull/3100)) - clean up the testutils ([lucas-clemente/quic-go#3104](https://github.com/lucas-clemente/quic-go/pull/3104)) - initialize the congestion controller with the actual max datagram size ([lucas-clemente/quic-go#3107](https://github.com/lucas-clemente/quic-go/pull/3107)) - make it possible to trace acknowledged packets ([lucas-clemente/quic-go#3134](https://github.com/lucas-clemente/quic-go/pull/3134)) - avoid type confusion between protocol.PacketType and logging.PacketType ([lucas-clemente/quic-go#3108](https://github.com/lucas-clemente/quic-go/pull/3108)) - fix duplicate logging of errors when the first error was a timeout error ([lucas-clemente/quic-go#3112](https://github.com/lucas-clemente/quic-go/pull/3112)) - use a tracer to make the packetization test more useful ([lucas-clemente/quic-go#3136](https://github.com/lucas-clemente/quic-go/pull/3136)) - improve string representation of timeout errors ([lucas-clemente/quic-go#3118](https://github.com/lucas-clemente/quic-go/pull/3118)) - fix flaky timeout test ([lucas-clemente/quic-go#3105](https://github.com/lucas-clemente/quic-go/pull/3105)) - fix calculation of the time for the next keep alive - add a 0-RTT test with different connection ID lengths ([lucas-clemente/quic-go#3098](https://github.com/lucas-clemente/quic-go/pull/3098)) - only run Ginkgo focus detection in staged files in pre-commit hook ([lucas-clemente/quic-go#3099](https://github.com/lucas-clemente/quic-go/pull/3099)) - allow 0-RTT when flow control windows are increased ([lucas-clemente/quic-go#3096](https://github.com/lucas-clemente/quic-go/pull/3096)) - improve the 0-RTT rejection integration test ([lucas-clemente/quic-go#3097](https://github.com/lucas-clemente/quic-go/pull/3097)) - rename config values for flow control limits ([lucas-clemente/quic-go#3089](https://github.com/lucas-clemente/quic-go/pull/3089)) - allow 0-RTT resumption if the server's stream limit was increased ([lucas-clemente/quic-go#3086](https://github.com/lucas-clemente/quic-go/pull/3086)) - cache the serialized OOB in the conn, not in the packet info ([lucas-clemente/quic-go#3093](https://github.com/lucas-clemente/quic-go/pull/3093)) - use code points from x/sys/unix for PKTINFO syscalls ([lucas-clemente/quic-go#3094](https://github.com/lucas-clemente/quic-go/pull/3094)) - make it possible to detect version negotiation failures in logging, fix qlogging of those ([lucas-clemente/quic-go#3092](https://github.com/lucas-clemente/quic-go/pull/3092)) - make the initial stream / connection flow control windows configurable ([lucas-clemente/quic-go#3083](https://github.com/lucas-clemente/quic-go/pull/3083)) - only apply server's transport parameters after handshake completion ([lucas-clemente/quic-go#3085](https://github.com/lucas-clemente/quic-go/pull/3085)) - fix documentation for baseFlowController.UpdateSendWindow ([lucas-clemente/quic-go#3087](https://github.com/lucas-clemente/quic-go/pull/3087)) - set the Content-Length for HTTP/3 responses ([lucas-clemente/quic-go#3091](https://github.com/lucas-clemente/quic-go/pull/3091)) - update the flow control windows of streams opened in 0-RTT ([lucas-clemente/quic-go#3088](https://github.com/lucas-clemente/quic-go/pull/3088)) - Use the correct source IP when binding multiple IPs ([lucas-clemente/quic-go#3067](https://github.com/lucas-clemente/quic-go/pull/3067)) - fix race condition when receiving 0-RTT packets ([lucas-clemente/quic-go#3074](https://github.com/lucas-clemente/quic-go/pull/3074)) - require the application to handle 0-RTT rejection ([lucas-clemente/quic-go#3066](https://github.com/lucas-clemente/quic-go/pull/3066)) - add an internal queue to signal that a datagram frame has been dequeued ([lucas-clemente/quic-go#3081](https://github.com/lucas-clemente/quic-go/pull/3081)) - increase the maximum size of DATAGRAM frames ([lucas-clemente/quic-go#2966](https://github.com/lucas-clemente/quic-go/pull/2966)) - remove non-functioning 0-RTT test with different conn ID lengths ([lucas-clemente/quic-go#3079](https://github.com/lucas-clemente/quic-go/pull/3079)) - remove stray struct equality check ([lucas-clemente/quic-go#3078](https://github.com/lucas-clemente/quic-go/pull/3078)) - fix issuing of connection IDs when dialing a 0-RTT connections ([lucas-clemente/quic-go#3058](https://github.com/lucas-clemente/quic-go/pull/3058)) - only accept 0-RTT it the active_connection_id_limit didn't change ([lucas-clemente/quic-go#3060](https://github.com/lucas-clemente/quic-go/pull/3060)) - remove unused error return value from HandleMaxStreamsFrame ([lucas-clemente/quic-go#3072](https://github.com/lucas-clemente/quic-go/pull/3072)) - fix flaky accept queue integration test ([lucas-clemente/quic-go#3068](https://github.com/lucas-clemente/quic-go/pull/3068)) - don't reset the QPACK encoder / decoder streams ([lucas-clemente/quic-go#3063](https://github.com/lucas-clemente/quic-go/pull/3063)) - remove incorrect logging for client side retry packet ([lucas-clemente/quic-go#3071](https://github.com/lucas-clemente/quic-go/pull/3071)) - allow sending 1xx responses (#3047) ([lucas-clemente/quic-go#3047](https://github.com/lucas-clemente/quic-go/pull/3047)) - fix retry key and nonce for draft-34 ([lucas-clemente/quic-go#3062](https://github.com/lucas-clemente/quic-go/pull/3062)) - implement DPLPMTUD ([lucas-clemente/quic-go#3028](https://github.com/lucas-clemente/quic-go/pull/3028)) - only read multiple packets at a time after handshake completion ([lucas-clemente/quic-go#3041](https://github.com/lucas-clemente/quic-go/pull/3041)) - make the certificate verification integration tests more explicit ([lucas-clemente/quic-go#3040](https://github.com/lucas-clemente/quic-go/pull/3040)) - update gomock to v1.5.0, use mockgen source mode ([lucas-clemente/quic-go#3049](https://github.com/lucas-clemente/quic-go/pull/3049)) - trace dropping of 0-RTT keys ([lucas-clemente/quic-go#3054](https://github.com/lucas-clemente/quic-go/pull/3054)) - improve timeout measurement in the timeout test ([lucas-clemente/quic-go#3042](https://github.com/lucas-clemente/quic-go/pull/3042)) - add a randomized test for the received_packet_history ([lucas-clemente/quic-go#3052](https://github.com/lucas-clemente/quic-go/pull/3052)) - fix documentation of default values for MaxReceive{Stream, Connection}FlowControlWindow ([lucas-clemente/quic-go#3055](https://github.com/lucas-clemente/quic-go/pull/3055)) - refactor merge packet number ranges ([lucas-clemente/quic-go#3051](https://github.com/lucas-clemente/quic-go/pull/3051)) - add draft-34 to support versions in README - update README to reflect dropped Go 1.14 support - remove redundant nil-check in the packet packer ([lucas-clemente/quic-go#3048](https://github.com/lucas-clemente/quic-go/pull/3048)) - avoid using rand.Source ([lucas-clemente/quic-go#3046](https://github.com/lucas-clemente/quic-go/pull/3046)) - update Go to 1.16, drop support for 1.14 ([lucas-clemente/quic-go#3045](https://github.com/lucas-clemente/quic-go/pull/3045)) - fix error message when the UDP receive buffer size can't be increased ([lucas-clemente/quic-go#3039](https://github.com/lucas-clemente/quic-go/pull/3039)) - add the time_format field to qlog common_fields ([lucas-clemente/quic-go#3038](https://github.com/lucas-clemente/quic-go/pull/3038)) - log connection IDs without the 0x prefix ([lucas-clemente/quic-go#3036](https://github.com/lucas-clemente/quic-go/pull/3036)) - add support for QUIC draft-34 ([lucas-clemente/quic-go#3031](https://github.com/lucas-clemente/quic-go/pull/3031)) - fix qtls imports in mockgen generated mocks ([lucas-clemente/quic-go#3037](https://github.com/lucas-clemente/quic-go/pull/3037)) - improve error message when the read buffer size can't be set ([lucas-clemente/quic-go#3030](https://github.com/lucas-clemente/quic-go/pull/3030)) - qlog the quic-go version ([lucas-clemente/quic-go#3033](https://github.com/lucas-clemente/quic-go/pull/3033)) - remove the metrics package ([lucas-clemente/quic-go#3032](https://github.com/lucas-clemente/quic-go/pull/3032)) - expose the constructor for the qlog connection tracer ([lucas-clemente/quic-go#3034](https://github.com/lucas-clemente/quic-go/pull/3034)) - expose the constructor for the multipexed connection tracer ([lucas-clemente/quic-go#3035](https://github.com/lucas-clemente/quic-go/pull/3035)) - make sure the server is stopped before closing all server sessions ([lucas-clemente/quic-go#3020](https://github.com/lucas-clemente/quic-go/pull/3020)) - increase the size of the send queue ([lucas-clemente/quic-go#3016](https://github.com/lucas-clemente/quic-go/pull/3016)) - prioritize receiving packets over sending out more packets ([lucas-clemente/quic-go#3015](https://github.com/lucas-clemente/quic-go/pull/3015)) - re-enable key updates for HTTP/3 ([lucas-clemente/quic-go#3017](https://github.com/lucas-clemente/quic-go/pull/3017)) - check for errors after handling each previously undecryptable packet ([lucas-clemente/quic-go#3011](https://github.com/lucas-clemente/quic-go/pull/3011)) - fix flaky streams map test on Windows ([lucas-clemente/quic-go#3013](https://github.com/lucas-clemente/quic-go/pull/3013)) - fix flaky stream cancellation integration test ([lucas-clemente/quic-go#3014](https://github.com/lucas-clemente/quic-go/pull/3014)) - preallocate a slice of one frame when packing a packet ([lucas-clemente/quic-go#3018](https://github.com/lucas-clemente/quic-go/pull/3018)) - allow sending of ACKs when pacing limited ([lucas-clemente/quic-go#3010](https://github.com/lucas-clemente/quic-go/pull/3010)) - fix qlogging of the packet payload length ([lucas-clemente/quic-go#3004](https://github.com/lucas-clemente/quic-go/pull/3004)) - corrupt more ACKs in the MITM test ([lucas-clemente/quic-go#3007](https://github.com/lucas-clemente/quic-go/pull/3007)) - fix flaky key update integration test ([lucas-clemente/quic-go#3005](https://github.com/lucas-clemente/quic-go/pull/3005)) - immediately complete streams that were canceled, drop retransmissions ([lucas-clemente/quic-go#3003](https://github.com/lucas-clemente/quic-go/pull/3003)) - stop generating new packets when the send queue is full ([lucas-clemente/quic-go#2971](https://github.com/lucas-clemente/quic-go/pull/2971)) - allow access to the underlying quic.Stream from a http.ResponseWriter ([lucas-clemente/quic-go#2993](https://github.com/lucas-clemente/quic-go/pull/2993)) - remove stay print statement from session test - allow receiving of multiple packets before sending a packet ([lucas-clemente/quic-go#2984](https://github.com/lucas-clemente/quic-go/pull/2984)) - use cryptographic random for determining skipped packet numbers ([lucas-clemente/quic-go#2940](https://github.com/lucas-clemente/quic-go/pull/2940)) - fix interpretation of time.Time{} as a pacing deadline ([lucas-clemente/quic-go#2980](https://github.com/lucas-clemente/quic-go/pull/2980)) - qlog restored transport parameters ([lucas-clemente/quic-go#2991](https://github.com/lucas-clemente/quic-go/pull/2991)) - use a pkg.go.dev instead of a GoDoc badge ([lucas-clemente/quic-go#2982](https://github.com/lucas-clemente/quic-go/pull/2982)) - introduce a separate queue for undecryptable packets ([lucas-clemente/quic-go#2988](https://github.com/lucas-clemente/quic-go/pull/2988)) - improve 0-RTT queue ([lucas-clemente/quic-go#2990](https://github.com/lucas-clemente/quic-go/pull/2990)) - simplify switch statement in the transport parameter parser ([lucas-clemente/quic-go#2995](https://github.com/lucas-clemente/quic-go/pull/2995)) - remove unneeded overflow check when parsing the max_ack_delay ([lucas-clemente/quic-go#2996](https://github.com/lucas-clemente/quic-go/pull/2996)) - remove unneeded check in receivedPacketHandler.IsPotentiallyDuplicate ([lucas-clemente/quic-go#2998](https://github.com/lucas-clemente/quic-go/pull/2998)) - qlog the max_datagram_frame_size transport parameter ([lucas-clemente/quic-go#2997](https://github.com/lucas-clemente/quic-go/pull/2997)) - qlog draft-02 fixes ([lucas-clemente/quic-go#2987](https://github.com/lucas-clemente/quic-go/pull/2987)) - fix flaky qlog test ([lucas-clemente/quic-go#2981](https://github.com/lucas-clemente/quic-go/pull/2981)) - only run gofumpt on .go files in pre-commit hook ([lucas-clemente/quic-go#2983](https://github.com/lucas-clemente/quic-go/pull/2983)) - fix outdated comment for the http3.Server - make the OpenStreamSync cancellation test less flaky ([lucas-clemente/quic-go#2978](https://github.com/lucas-clemente/quic-go/pull/2978)) - add some useful pre-commit hooks ([lucas-clemente/quic-go#2979](https://github.com/lucas-clemente/quic-go/pull/2979)) - publicize QUIC varint reading and writing ([lucas-clemente/quic-go#2973](https://github.com/lucas-clemente/quic-go/pull/2973)) - add a http3.RoundTripOpt to skip the request scheme check ([lucas-clemente/quic-go#2962](https://github.com/lucas-clemente/quic-go/pull/2962)) - use the standard quic.Config in the deadline tests ([lucas-clemente/quic-go#2970](https://github.com/lucas-clemente/quic-go/pull/2970)) - update golangci-lint to v1.34.1 ([lucas-clemente/quic-go#2964](https://github.com/lucas-clemente/quic-go/pull/2964)) - update text about QUIC versions in the README ([lucas-clemente/quic-go#2975](https://github.com/lucas-clemente/quic-go/pull/2975)) - remove stray TODO in the http3.Server - add support for Go 1.16 ([lucas-clemente/quic-go#2953](https://github.com/lucas-clemente/quic-go/pull/2953)) - cancel reading on unidirectional streams when the stream type is unknown ([lucas-clemente/quic-go#2952](https://github.com/lucas-clemente/quic-go/pull/2952)) - remove duplicate check of the URL scheme in the HTTP/3 client ([lucas-clemente/quic-go#2956](https://github.com/lucas-clemente/quic-go/pull/2956)) - increase queueing duration in 0-RTT queue test to reduce flakiness ([lucas-clemente/quic-go#2954](https://github.com/lucas-clemente/quic-go/pull/2954)) - implement the HTTP/3 Datagram negotiation ([lucas-clemente/quic-go#2951](https://github.com/lucas-clemente/quic-go/pull/2951)) - implement HTTP/3 control stream handling ([lucas-clemente/quic-go#2949](https://github.com/lucas-clemente/quic-go/pull/2949)) - fix flaky sentPacketHandler test ([lucas-clemente/quic-go#2950](https://github.com/lucas-clemente/quic-go/pull/2950)) - don't retransmit PING frames added to ACK-only packets ([lucas-clemente/quic-go#2942](https://github.com/lucas-clemente/quic-go/pull/2942)) - move the transport parameter stream limit check to the parser ([lucas-clemente/quic-go#2944](https://github.com/lucas-clemente/quic-go/pull/2944)) - remove unused initialVersion variable in session ([lucas-clemente/quic-go#2946](https://github.com/lucas-clemente/quic-go/pull/2946)) - remove unneeded check for the peer's transport parameters ([lucas-clemente/quic-go#2945](https://github.com/lucas-clemente/quic-go/pull/2945)) - add the H3_MESSAGE_ERROR ([lucas-clemente/quic-go#2947](https://github.com/lucas-clemente/quic-go/pull/2947)) - simplify Read and Write mock calls in http3 tests ([lucas-clemente/quic-go#2948](https://github.com/lucas-clemente/quic-go/pull/2948)) - implement the datagram draft ([lucas-clemente/quic-go#2162](https://github.com/lucas-clemente/quic-go/pull/2162)) - fix logging of bytes_in_flight when receiving an ACK ([lucas-clemente/quic-go#2937](https://github.com/lucas-clemente/quic-go/pull/2937)) - trace when a packet is dropped because the receivedPackets chan is full ([lucas-clemente/quic-go#2939](https://github.com/lucas-clemente/quic-go/pull/2939)) - various improvements to the packet number generator ([lucas-clemente/quic-go#2905](https://github.com/lucas-clemente/quic-go/pull/2905)) - introduce a quic.Config.HandshakeIdleTimeout, remove HandshakeTimeout ([lucas-clemente/quic-go#2930](https://github.com/lucas-clemente/quic-go/pull/2930)) - allow up to 20 byte for the initial connection IDs ([lucas-clemente/quic-go#2936](https://github.com/lucas-clemente/quic-go/pull/2936)) - reduce memory footprint of undecryptable packet handling ([lucas-clemente/quic-go#2932](https://github.com/lucas-clemente/quic-go/pull/2932)) - use a buffer from the pool for composing Retry packets ([lucas-clemente/quic-go#2934](https://github.com/lucas-clemente/quic-go/pull/2934)) - release the packet buffer after sending a CONNECTION_CLOSE in the server ([lucas-clemente/quic-go#2935](https://github.com/lucas-clemente/quic-go/pull/2935)) - move integration tests to GitHub Actions, disable Travis ([lucas-clemente/quic-go#2891](https://github.com/lucas-clemente/quic-go/pull/2891)) - use golang.org/x/sys/unix instead of syscall ([lucas-clemente/quic-go#2927](https://github.com/lucas-clemente/quic-go/pull/2927)) - add support for the connection_closed qlog event ([lucas-clemente/quic-go#2921](https://github.com/lucas-clemente/quic-go/pull/2921)) - qlog tokens in NEW_TOKEN frames, Retry packets and Initial packets ([lucas-clemente/quic-go#2863](https://github.com/lucas-clemente/quic-go/pull/2863)) - qlog the packet_type as part of the packet header, not the event itself ([lucas-clemente/quic-go#2758](https://github.com/lucas-clemente/quic-go/pull/2758)) - use the new, streaming-friendly NDJSON-based qlog encoding ([lucas-clemente/quic-go#2736](https://github.com/lucas-clemente/quic-go/pull/2736)) - add a generic Debug() function to the connection tracer ([lucas-clemente/quic-go#2909](https://github.com/lucas-clemente/quic-go/pull/2909)) - remove unnecessary call to time.Now() when sending a packet ([lucas-clemente/quic-go#2911](https://github.com/lucas-clemente/quic-go/pull/2911)) - remove support for quic-trace ([lucas-clemente/quic-go#2913](https://github.com/lucas-clemente/quic-go/pull/2913)) - reduce the maximum number of ACK ranges ([lucas-clemente/quic-go#2887](https://github.com/lucas-clemente/quic-go/pull/2887)) - don't allocate for acked packets ([lucas-clemente/quic-go#2899](https://github.com/lucas-clemente/quic-go/pull/2899)) - avoid allocating when detecting lost packets ([lucas-clemente/quic-go#2898](https://github.com/lucas-clemente/quic-go/pull/2898)) - use the string optimization for map keys in the packet handler map ([lucas-clemente/quic-go#2892](https://github.com/lucas-clemente/quic-go/pull/2892)) - use a single map in the incoming streams map ([lucas-clemente/quic-go#2890](https://github.com/lucas-clemente/quic-go/pull/2890)) - github.com/marten-seemann/qtls-go1-15 (v0.1.1 -> v0.1.4): - use a prefix for client session cache keys - add callbacks to store and restore app data along a session state - don't use TLS 1.3 compatibility mode when using alternative record layer - delete the session ticket after attempting 0-RTT - reject 0-RTT when a different ALPN is chosen - encode the ALPN into the session ticket - add a field to the ConnectionState to tell if 0-RTT was used - add a callback to tell the client about rejection of 0-RTT - don't offer 0-RTT after a HelloRetryRequest - add Accept0RTT to Config callback to decide if 0-RTT should be accepted - github.com/marten-seemann/qtls-go1-16 (null -> v0.1.3): - use a prefix for client session cache keys - add callbacks to store and restore app data along a session state - don't use TLS 1.3 compatibility mode when using alternative record layer - delete the session ticket after attempting 0-RTT - reject 0-RTT when a different ALPN is chosen - github.com/multiformats/go-multiaddr (v0.3.1 -> v0.3.2): - fix(net): export new net.Addr conversion registration functions ([multiformats/go-multiaddr#152](https://github.com/multiformats/go-multiaddr/pull/152)) - sync: run go mod tidy (and set Go 1.15) and gofmt -s in copy workflow (#146) ([multiformats/go-multiaddr#146](https://github.com/multiformats/go-multiaddr/pull/146)) - more linter fixes ([multiformats/go-multiaddr#145](https://github.com/multiformats/go-multiaddr/pull/145)) - fix go vet and staticcheck failures ([multiformats/go-multiaddr#143](https://github.com/multiformats/go-multiaddr/pull/143)) - don't listen on all interfaces in tests, unless on CI ([multiformats/go-multiaddr#136](https://github.com/multiformats/go-multiaddr/pull/136)) - Fix Local Address on TCP connections ([multiformats/go-multiaddr#135](https://github.com/multiformats/go-multiaddr/pull/135)) - github.com/multiformats/go-multiaddr-dns (v0.2.0 -> v0.3.1): - Normalize domains to fqdn for resolver selection ([multiformats/go-multiaddr-dns#27](https://github.com/multiformats/go-multiaddr-dns/pull/27)) - refactor Resolver to support custom per-TLD resolvers ([multiformats/go-multiaddr-dns#26](https://github.com/multiformats/go-multiaddr-dns/pull/26)) - feat: exposes backend ([multiformats/go-multiaddr-dns#25](https://github.com/multiformats/go-multiaddr-dns/pull/25)) - github.com/multiformats/go-multihash (v0.0.14 -> v0.0.15): - Refactor registry system: no direct dependencies; expose standard hash.Hash; be a data carrier. ([multiformats/go-multihash#136](https://github.com/multiformats/go-multihash/pull/136)) - github.com/multiformats/go-multistream (v0.2.0 -> v0.2.2): - change the simultaneous open protocol to /libp2p/simultaneous-connect ([multiformats/go-multistream#66](https://github.com/multiformats/go-multistream/pull/66)) - fix the lazy stress read test on Windows ([multiformats/go-multistream#61](https://github.com/multiformats/go-multistream/pull/61)) - fix go vet and staticcheck errors ([multiformats/go-multistream#60](https://github.com/multiformats/go-multistream/pull/60)) - Implement simultaneous open extension ([multiformats/go-multistream#42](https://github.com/multiformats/go-multistream/pull/42)) - reduce the number of streams in the stress tests, fix error handling ([multiformats/go-multistream#54](https://github.com/multiformats/go-multistream/pull/54)) - github.com/whyrusleeping/cbor-gen (v0.0.0-20200710004633-5379fc63235d -> v0.0.0-20210219115102-f37d292932f2): - feat: allow unmarshalling of struct with more fields than marshaled struct ([whyrusleeping/cbor-gen#50](https://github.com/whyrusleeping/cbor-gen/pull/50)) - chore: add a license file ([whyrusleeping/cbor-gen#49](https://github.com/whyrusleeping/cbor-gen/pull/49)) - fix: enforce maxlen in ReadByteArray() ([whyrusleeping/cbor-gen#43](https://github.com/whyrusleeping/cbor-gen/pull/43)) - use unix nanoseconds for encoding Cbortime ([whyrusleeping/cbor-gen#41](https://github.com/whyrusleeping/cbor-gen/pull/41)) - add json marshalers to CborTime - add a helper for roundtripping time.time objects ([whyrusleeping/cbor-gen#40](https://github.com/whyrusleeping/cbor-gen/pull/40)) - Add a validate function. ([whyrusleeping/cbor-gen#39](https://github.com/whyrusleeping/cbor-gen/pull/39)) - Fix import handling ([whyrusleeping/cbor-gen#38](https://github.com/whyrusleeping/cbor-gen/pull/38)) - Optimize discarding in ScanForLinks ([whyrusleeping/cbor-gen#36](https://github.com/whyrusleeping/cbor-gen/pull/36)) - Always allocate scratch space when marshalling into a map. ([whyrusleeping/cbor-gen#37](https://github.com/whyrusleeping/cbor-gen/pull/37)) - optimize byte reading ([whyrusleeping/cbor-gen#35](https://github.com/whyrusleeping/cbor-gen/pull/35)) - Optimize decoding ([whyrusleeping/cbor-gen#34](https://github.com/whyrusleeping/cbor-gen/pull/34)) - Fix named string issue ([whyrusleeping/cbor-gen#30](https://github.com/whyrusleeping/cbor-gen/pull/30)) - Fix encoding/decoding fixed byte arrays ([whyrusleeping/cbor-gen#29](https://github.com/whyrusleeping/cbor-gen/pull/29)) - fix overread on scanforlinks ([whyrusleeping/cbor-gen#28](https://github.com/whyrusleeping/cbor-gen/pull/28)) ### ❤️ Contributors | Contributor | Commits | Lines ± | Files Changed | |-------------|---------|---------|---------------| | Marten Seemann | 358 | +17444/-12000 | 1268 | | Eric Myhre | 82 | +9672/-2459 | 328 | | Ian Davis | 7 | +8421/-737 | 116 | | Daniel Martí | 18 | +2733/-4377 | 313 | | Adin Schmahmann | 46 | +5387/-1289 | 125 | | Steven Allen | 95 | +3278/-1861 | 200 | | hannahhoward | 14 | +1380/-3667 | 84 | | gammazero | 29 | +2520/-1161 | 88 | | Hector Sanjuan | 12 | +511/-3129 | 52 | | vyzo | 77 | +2198/-940 | 117 | | Will Scott | 12 | +912/-593 | 37 | | Dirk McCormick | 3 | +1384/-63 | 14 | | Andrew Gillis | 3 | +1231/-39 | 19 | | Marcin Rataj | 37 | +549/-308 | 72 | | Aarsh Shah | 13 | +668/-86 | 30 | | Olivier Poitrey | 1 | +469/-182 | 15 | | Rod Vagg | 9 | +364/-184 | 14 | | whyrusleeping | 5 | +253/-32 | 11 | | Cory Schwartz | 10 | +162/-115 | 37 | | Adrian Lanzafame | 8 | +212/-60 | 11 | | aarshkshah1992 | 7 | +102/-110 | 9 | | Jakub Sztandera | 7 | +126/-75 | 16 | | huoju | 4 | +127/-41 | 6 | | acruikshank | 6 | +32/-24 | 7 | | Toby | 1 | +41/-1 | 2 | | Naveen | 1 | +40/-0 | 1 | | Bogdan Stirbat | 1 | +22/-16 | 2 | | Kévin Dunglas | 1 | +32/-2 | 2 | | Nicholas Bollweg | 1 | +22/-0 | 1 | | q191201771 | 2 | +4/-11 | 2 | | Mathis Engelbart | 1 | +12/-2 | 1 | | requilence | 1 | +13/-0 | 1 | | divingpetrel | 1 | +7/-4 | 2 | | Oli Evans | 2 | +9/-2 | 3 | | Lucas Molas | 3 | +7/-3 | 3 | | RubenKelevra | 3 | +2/-6 | 3 | | Will | 1 | +1/-5 | 1 | | Jorropo | 1 | +4/-2 | 1 | | Ju Huo | 1 | +2/-2 | 1 | | zhoujiajie | 1 | +1/-1 | 1 | | Luflosi | 1 | +1/-1 | 1 | | Jonathan Rudenberg | 1 | +1/-1 | 1 | | David Pflug | 1 | +1/-1 | 1 | | Ari Mattila | 1 | +1/-1 | 1 | | Yingrong Zhao | 1 | +0/-1 | 1 | ================================================ FILE: docs/command-completion.md ================================================ # Command Completion Shell command completions can be generated by running one of the `ipfs commands completions` sub-commands. The simplest way to "eval" the completions logic: ```bash > eval "$(ipfs commands completion bash)" ``` To install the completions permanently, they can be moved to `/etc/bash_completion.d` or sourced from your `~/.bashrc` file. ## Fish The fish shell is also supported: The simplest way to use the completions logic: ```bash > ipfs commands completion fish | source ``` To install the completions permanently, they can be moved to `/etc/fish/completions` or `~/.config/fish/completions` or sourced from your `~/.config/fish/config.fish` file. ## ZSH The zsh shell is also supported: The simplest way to "eval" the completions logic: ```bash > eval "$(ipfs commands completion zsh)" ``` To install the completions permanently, they can be moved to `/etc/bash_completion.d` or sourced from your `~/.zshrc` file. ================================================ FILE: docs/config.md ================================================ # The Kubo config file The Kubo config file is a JSON document located at `$IPFS_PATH/config`. It is read once at node instantiation, either for an offline command, or when starting the daemon. Commands that execute on a running daemon do not read the config file at runtime. # Table of Contents - [The Kubo config file](#the-kubo-config-file) - [Table of Contents](#table-of-contents) - [`Addresses`](#addresses) - [`Addresses.API`](#addressesapi) - [`Addresses.Gateway`](#addressesgateway) - [`Addresses.Swarm`](#addressesswarm) - [`Addresses.Announce`](#addressesannounce) - [`Addresses.AppendAnnounce`](#addressesappendannounce) - [`Addresses.NoAnnounce`](#addressesnoannounce) - [`API`](#api) - [`API.HTTPHeaders`](#apihttpheaders) - [`API.Authorizations`](#apiauthorizations) - [`API.Authorizations: AuthSecret`](#apiauthorizations-authsecret) - [`API.Authorizations: AllowedPaths`](#apiauthorizations-allowedpaths) - [`AutoNAT`](#autonat) - [`AutoNAT.ServiceMode`](#autonatservicemode) - [`AutoNAT.Throttle`](#autonatthrottle) - [`AutoNAT.Throttle.GlobalLimit`](#autonatthrottlegloballimit) - [`AutoNAT.Throttle.PeerLimit`](#autonatthrottlepeerlimit) - [`AutoNAT.Throttle.Interval`](#autonatthrottleinterval) - [`AutoTLS`](#autotls) - [`AutoTLS.Enabled`](#autotlsenabled) - [`AutoTLS.AutoWSS`](#autotlsautowss) - [`AutoTLS.ShortAddrs`](#autotlsshortaddrs) - [`AutoTLS.DomainSuffix`](#autotlsdomainsuffix) - [`AutoTLS.RegistrationEndpoint`](#autotlsregistrationendpoint) - [`AutoTLS.RegistrationToken`](#autotlsregistrationtoken) - [`AutoTLS.RegistrationDelay`](#autotlsregistrationdelay) - [`AutoTLS.CAEndpoint`](#autotlscaendpoint) - [`AutoConf`](#autoconf) - [`AutoConf.URL`](#autoconfurl) - [`AutoConf.Enabled`](#autoconfenabled) - [`AutoConf.RefreshInterval`](#autoconfrefreshinterval) - [`AutoConf.TLSInsecureSkipVerify`](#autoconftlsinsecureskipverify) - [`Bitswap`](#bitswap) - [`Bitswap.Libp2pEnabled`](#bitswaplibp2penabled) - [`Bitswap.ServerEnabled`](#bitswapserverenabled) - [`Bootstrap`](#bootstrap) - [`Datastore`](#datastore) - [`Datastore.StorageMax`](#datastorestoragemax) - [`Datastore.StorageGCWatermark`](#datastorestoragegcwatermark) - [`Datastore.GCPeriod`](#datastoregcperiod) - [`Datastore.HashOnRead`](#datastorehashonread) - [`Datastore.BloomFilterSize`](#datastorebloomfiltersize) - [`Datastore.WriteThrough`](#datastorewritethrough) - [`Datastore.BlockKeyCacheSize`](#datastoreblockkeycachesize) - [`Datastore.Spec`](#datastorespec) - [`Discovery`](#discovery) - [`Discovery.MDNS`](#discoverymdns) - [`Discovery.MDNS.Enabled`](#discoverymdnsenabled) - [`Discovery.MDNS.Interval`](#discoverymdnsinterval) - [`Experimental`](#experimental) - [`Experimental.Libp2pStreamMounting`](#experimentallibp2pstreammounting) - [`Gateway`](#gateway) - [`Gateway.NoFetch`](#gatewaynofetch) - [`Gateway.NoDNSLink`](#gatewaynodnslink) - [`Gateway.DeserializedResponses`](#gatewaydeserializedresponses) - [`Gateway.AllowCodecConversion`](#gatewayallowcodecconversion) - [`Gateway.DisableHTMLErrors`](#gatewaydisablehtmlerrors) - [`Gateway.ExposeRoutingAPI`](#gatewayexposeroutingapi) - [`Gateway.RetrievalTimeout`](#gatewayretrievaltimeout) - [`Gateway.MaxRequestDuration`](#gatewaymaxrequestduration) - [`Gateway.MaxRangeRequestFileSize`](#gatewaymaxrangerequestfilesize) - [`Gateway.MaxConcurrentRequests`](#gatewaymaxconcurrentrequests) - [`Gateway.HTTPHeaders`](#gatewayhttpheaders) - [`Gateway.RootRedirect`](#gatewayrootredirect) - [`Gateway.DiagnosticServiceURL`](#gatewaydiagnosticserviceurl) - [`Gateway.FastDirIndexThreshold`](#gatewayfastdirindexthreshold) - [`Gateway.Writable`](#gatewaywritable) - [`Gateway.PathPrefixes`](#gatewaypathprefixes) - [`Gateway.PublicGateways`](#gatewaypublicgateways) - [`Gateway.PublicGateways: Paths`](#gatewaypublicgateways-paths) - [`Gateway.PublicGateways: UseSubdomains`](#gatewaypublicgateways-usesubdomains) - [`Gateway.PublicGateways: NoDNSLink`](#gatewaypublicgateways-nodnslink) - [`Gateway.PublicGateways: InlineDNSLink`](#gatewaypublicgateways-inlinednslink) - [`Gateway.PublicGateways: DeserializedResponses`](#gatewaypublicgateways-deserializedresponses) - [Implicit defaults of `Gateway.PublicGateways`](#implicit-defaults-of-gatewaypublicgateways) - [`Gateway` recipes](#gateway-recipes) - [`Identity`](#identity) - [`Identity.PeerID`](#identitypeerid) - [`Identity.PrivKey`](#identityprivkey) - [`Internal`](#internal) - [`Internal.Bitswap`](#internalbitswap) - [`Internal.Bitswap.TaskWorkerCount`](#internalbitswaptaskworkercount) - [`Internal.Bitswap.EngineBlockstoreWorkerCount`](#internalbitswapengineblockstoreworkercount) - [`Internal.Bitswap.EngineTaskWorkerCount`](#internalbitswapenginetaskworkercount) - [`Internal.Bitswap.MaxOutstandingBytesPerPeer`](#internalbitswapmaxoutstandingbytesperpeer) - [`Internal.Bitswap.ProviderSearchDelay`](#internalbitswapprovidersearchdelay) - [`Internal.Bitswap.ProviderSearchMaxResults`](#internalbitswapprovidersearchmaxresults) - [`Internal.Bitswap.BroadcastControl`](#internalbitswapbroadcastcontrol) - [`Internal.Bitswap.BroadcastControl.Enable`](#internalbitswapbroadcastcontrolenable) - [`Internal.Bitswap.BroadcastControl.MaxPeers`](#internalbitswapbroadcastcontrolmaxpeers) - [`Internal.Bitswap.BroadcastControl.LocalPeers`](#internalbitswapbroadcastcontrollocalpeers) - [`Internal.Bitswap.BroadcastControl.PeeredPeers`](#internalbitswapbroadcastcontrolpeeredpeers) - [`Internal.Bitswap.BroadcastControl.MaxRandomPeers`](#internalbitswapbroadcastcontrolmaxrandompeers) - [`Internal.Bitswap.BroadcastControl.SendToPendingPeers`](#internalbitswapbroadcastcontrolsendtopendingpeers) - [`Internal.UnixFSShardingSizeThreshold`](#internalunixfsshardingsizethreshold) - [`Ipns`](#ipns) - [`Ipns.RepublishPeriod`](#ipnsrepublishperiod) - [`Ipns.RecordLifetime`](#ipnsrecordlifetime) - [`Ipns.ResolveCacheSize`](#ipnsresolvecachesize) - [`Ipns.MaxCacheTTL`](#ipnsmaxcachettl) - [`Ipns.UsePubsub`](#ipnsusepubsub) - [`Ipns.DelegatedPublishers`](#ipnsdelegatedpublishers) - [`Migration`](#migration) - [`Migration.DownloadSources`](#migrationdownloadsources) - [`Migration.Keep`](#migrationkeep) - [`Mounts`](#mounts) - [`Mounts.IPFS`](#mountsipfs) - [`Mounts.IPNS`](#mountsipns) - [`Mounts.MFS`](#mountsmfs) - [`Mounts.FuseAllowOther`](#mountsfuseallowother) - [`Pinning`](#pinning) - [`Pinning.RemoteServices`](#pinningremoteservices) - [`Pinning.RemoteServices: API`](#pinningremoteservices-api) - [`Pinning.RemoteServices: API.Endpoint`](#pinningremoteservices-apiendpoint) - [`Pinning.RemoteServices: API.Key`](#pinningremoteservices-apikey) - [`Pinning.RemoteServices: Policies`](#pinningremoteservices-policies) - [`Pinning.RemoteServices: Policies.MFS`](#pinningremoteservices-policiesmfs) - [`Pinning.RemoteServices: Policies.MFS.Enabled`](#pinningremoteservices-policiesmfsenabled) - [`Pinning.RemoteServices: Policies.MFS.PinName`](#pinningremoteservices-policiesmfspinname) - [`Pinning.RemoteServices: Policies.MFS.RepinInterval`](#pinningremoteservices-policiesmfsrepininterval) - [`Provide`](#provide) - [`Provide.Enabled`](#provideenabled) - [`Provide.Strategy`](#providestrategy) - [`Provide.DHT`](#providedht) - [`Provide.DHT.MaxWorkers`](#providedhtmaxworkers) - [`Provide.DHT.Interval`](#providedhtinterval) - [`Provide.DHT.SweepEnabled`](#providedhtsweepenabled) - [`Provide.DHT.ResumeEnabled`](#providedhtresumeenabled) - [`Provide.DHT.DedicatedPeriodicWorkers`](#providedhtdedicatedperiodicworkers) - [`Provide.DHT.DedicatedBurstWorkers`](#providedhtdedicatedburstworkers) - [`Provide.DHT.MaxProvideConnsPerWorker`](#providedhtmaxprovideconnsperworker) - [`Provide.DHT.KeystoreBatchSize`](#providedhtkeystorebatchsize) - [`Provide.DHT.OfflineDelay`](#providedhtofflinedelay) - [`Provider`](#provider) - [`Provider.Enabled`](#providerenabled) - [`Provider.Strategy`](#providerstrategy) - [`Provider.WorkerCount`](#providerworkercount) - [`Pubsub`](#pubsub) - [When to use a dedicated pubsub node](#when-to-use-a-dedicated-pubsub-node) - [Message deduplication](#message-deduplication) - [`Pubsub.Enabled`](#pubsubenabled) - [`Pubsub.Router`](#pubsubrouter) - [`Pubsub.DisableSigning`](#pubsubdisablesigning) - [`Pubsub.SeenMessagesTTL`](#pubsubseenmessagesttl) - [`Pubsub.SeenMessagesStrategy`](#pubsubseenmessagesstrategy) - [`Peering`](#peering) - [`Peering.Peers`](#peeringpeers) - [`Reprovider`](#reprovider) - [`Reprovider.Interval`](#reproviderinterval) - [`Reprovider.Strategy`](#providestrategy) - [`Routing`](#routing) - [`Routing.Type`](#routingtype) - [`Routing.DelegatedRouters`](#routingdelegatedrouters) - [`Routing.AcceleratedDHTClient`](#routingaccelerateddhtclient) - [`Routing.LoopbackAddressesOnLanDHT`](#routingloopbackaddressesonlandht) - [`Routing.IgnoreProviders`](#routingignoreproviders) - [`Routing.Routers`](#routingrouters) - [`Routing.Routers.[name].Type`](#routingroutersnametype) - [`Routing.Routers.[name].Parameters`](#routingroutersnameparameters) - [`Routing.Methods`](#routingmethods) - [`Swarm`](#swarm) - [`Swarm.AddrFilters`](#swarmaddrfilters) - [`Swarm.DisableBandwidthMetrics`](#swarmdisablebandwidthmetrics) - [`Swarm.DisableNatPortMap`](#swarmdisablenatportmap) - [`Swarm.EnableHolePunching`](#swarmenableholepunching) - [`Swarm.EnableAutoRelay`](#swarmenableautorelay) - [`Swarm.RelayClient`](#swarmrelayclient) - [`Swarm.RelayClient.Enabled`](#swarmrelayclientenabled) - [`Swarm.RelayClient.StaticRelays`](#swarmrelayclientstaticrelays) - [`Swarm.RelayService`](#swarmrelayservice) - [`Swarm.RelayService.Enabled`](#swarmrelayserviceenabled) - [`Swarm.RelayService.Limit`](#swarmrelayservicelimit) - [`Swarm.RelayService.ConnectionDurationLimit`](#swarmrelayserviceconnectiondurationlimit) - [`Swarm.RelayService.ConnectionDataLimit`](#swarmrelayserviceconnectiondatalimit) - [`Swarm.RelayService.ReservationTTL`](#swarmrelayservicereservationttl) - [`Swarm.RelayService.MaxReservations`](#swarmrelayservicemaxreservations) - [`Swarm.RelayService.MaxCircuits`](#swarmrelayservicemaxcircuits) - [`Swarm.RelayService.BufferSize`](#swarmrelayservicebuffersize) - [`Swarm.RelayService.MaxReservationsPerPeer`](#swarmrelayservicemaxreservationsperpeer) - [`Swarm.RelayService.MaxReservationsPerIP`](#swarmrelayservicemaxreservationsperip) - [`Swarm.RelayService.MaxReservationsPerASN`](#swarmrelayservicemaxreservationsperasn) - [`Swarm.EnableRelayHop`](#swarmenablerelayhop) - [`Swarm.DisableRelay`](#swarmdisablerelay) - [`Swarm.EnableAutoNATService`](#swarmenableautonatservice) - [`Swarm.ConnMgr`](#swarmconnmgr) - [`Swarm.ConnMgr.Type`](#swarmconnmgrtype) - [Basic Connection Manager](#basic-connection-manager) - [`Swarm.ConnMgr.LowWater`](#swarmconnmgrlowwater) - [`Swarm.ConnMgr.HighWater`](#swarmconnmgrhighwater) - [`Swarm.ConnMgr.GracePeriod`](#swarmconnmgrgraceperiod) - [`Swarm.ConnMgr.SilencePeriod`](#swarmconnmgrsilenceperiod) - [`Swarm.ResourceMgr`](#swarmresourcemgr) - [`Swarm.ResourceMgr.Enabled`](#swarmresourcemgrenabled) - [`Swarm.ResourceMgr.MaxMemory`](#swarmresourcemgrmaxmemory) - [`Swarm.ResourceMgr.MaxFileDescriptors`](#swarmresourcemgrmaxfiledescriptors) - [`Swarm.ResourceMgr.Allowlist`](#swarmresourcemgrallowlist) - [`Swarm.Transports`](#swarmtransports) - [`Swarm.Transports.Network`](#swarmtransportsnetwork) - [`Swarm.Transports.Network.TCP`](#swarmtransportsnetworktcp) - [`Swarm.Transports.Network.Websocket`](#swarmtransportsnetworkwebsocket) - [`Swarm.Transports.Network.QUIC`](#swarmtransportsnetworkquic) - [`Swarm.Transports.Network.Relay`](#swarmtransportsnetworkrelay) - [`Swarm.Transports.Network.WebTransport`](#swarmtransportsnetworkwebtransport) - [`Swarm.Transports.Network.WebRTCDirect`](#swarmtransportsnetworkwebrtcdirect) - [`Swarm.Transports.Security`](#swarmtransportssecurity) - [`Swarm.Transports.Security.TLS`](#swarmtransportssecuritytls) - [`Swarm.Transports.Security.SECIO`](#swarmtransportssecuritysecio) - [`Swarm.Transports.Security.Noise`](#swarmtransportssecuritynoise) - [`Swarm.Transports.Multiplexers`](#swarmtransportsmultiplexers) - [`Swarm.Transports.Multiplexers.Yamux`](#swarmtransportsmultiplexersyamux) - [`Swarm.Transports.Multiplexers.Mplex`](#swarmtransportsmultiplexersmplex) - [`DNS`](#dns) - [`DNS.Resolvers`](#dnsresolvers) - [`DNS.MaxCacheTTL`](#dnsmaxcachettl) - [`HTTPRetrieval`](#httpretrieval) - [`HTTPRetrieval.Enabled`](#httpretrievalenabled) - [`HTTPRetrieval.Allowlist`](#httpretrievalallowlist) - [`HTTPRetrieval.Denylist`](#httpretrievaldenylist) - [`HTTPRetrieval.NumWorkers`](#httpretrievalnumworkers) - [`HTTPRetrieval.MaxBlockSize`](#httpretrievalmaxblocksize) - [`HTTPRetrieval.TLSInsecureSkipVerify`](#httpretrievaltlsinsecureskipverify) - [`Import`](#import) - [`Import.CidVersion`](#importcidversion) - [`Import.UnixFSRawLeaves`](#importunixfsrawleaves) - [`Import.UnixFSChunker`](#importunixfschunker) - [`Import.HashFunction`](#importhashfunction) - [`Import.FastProvideRoot`](#importfastprovideroot) - [`Import.FastProvideWait`](#importfastprovidewait) - [`Import.BatchMaxNodes`](#importbatchmaxnodes) - [`Import.BatchMaxSize`](#importbatchmaxsize) - [`Import.UnixFSFileMaxLinks`](#importunixfsfilemaxlinks) - [`Import.UnixFSDirectoryMaxLinks`](#importunixfsdirectorymaxlinks) - [`Import.UnixFSHAMTDirectoryMaxFanout`](#importunixfshamtdirectorymaxfanout) - [`Import.UnixFSHAMTDirectorySizeThreshold`](#importunixfshamtdirectorysizethreshold) - [`Import.UnixFSHAMTDirectorySizeEstimation`](#importunixfshamtdirectorysizeestimation) - [`Import.UnixFSDAGLayout`](#importunixfsdaglayout) - [`Version`](#version) - [`Version.AgentSuffix`](#versionagentsuffix) - [`Version.SwarmCheckEnabled`](#versionswarmcheckenabled) - [`Version.SwarmCheckPercentThreshold`](#versionswarmcheckpercentthreshold) - [Profiles](#profiles) - [`server` profile](#server-profile) - [`randomports` profile](#randomports-profile) - [`default-datastore` profile](#default-datastore-profile) - [`local-discovery` profile](#local-discovery-profile) - [`default-networking` profile](#default-networking-profile) - [`autoconf-on` profile](#autoconf-on-profile) - [`autoconf-off` profile](#autoconf-off-profile) - [`flatfs` profile](#flatfs-profile) - [`flatfs-measure` profile](#flatfs-measure-profile) - [`pebbleds` profile](#pebbleds-profile) - [`pebbleds-measure` profile](#pebbleds-measure-profile) - [`badgerds` profile](#badgerds-profile) - [`badgerds-measure` profile](#badgerds-measure-profile) - [`lowpower` profile](#lowpower-profile) - [`announce-off` profile](#announce-off-profile) - [`announce-on` profile](#announce-on-profile) - [`unixfs-v0-2015` profile](#unixfs-v0-2015-profile) - [`legacy-cid-v0` profile](#legacy-cid-v0-profile) - [`unixfs-v1-2025` profile](#unixfs-v1-2025-profile) - [Security](#security) - [Port and Network Exposure](#port-and-network-exposure) - [Security Best Practices](#security-best-practices) - [Types](#types) - [`flag`](#flag) - [`priority`](#priority) - [`strings`](#strings) - [`duration`](#duration) - [`optionalInteger`](#optionalinteger) - [`optionalBytes`](#optionalbytes) - [`optionalString`](#optionalstring) - [`optionalDuration`](#optionalduration) ## `Addresses` Contains information about various listener addresses to be used by this node. ### `Addresses.API` [Multiaddr][multiaddr] or array of multiaddrs describing the addresses to serve the local [Kubo RPC API](https://docs.ipfs.tech/reference/kubo/rpc/) (`/api/v0`). Supported Transports: - tcp/ip{4,6} - `/ipN/.../tcp/...` - unix - `/unix/path/to/socket` > [!CAUTION] > **NEVER EXPOSE UNPROTECTED ADMIN RPC TO LAN OR THE PUBLIC INTERNET** > > The RPC API grants admin-level access to your Kubo IPFS node, including > configuration and secret key management. > > By default, it is bound to localhost for security reasons. Exposing it to LAN > or the public internet is highly risky—similar to exposing a SQL database or > backend service without authentication middleware > > - If you need secure access to a subset of RPC, secure it with [`API.Authorizations`](#apiauthorizations) or custom auth middleware running in front of the localhost-only RPC port defined here. > - If you are looking for an interface designed for browsers and public internet, use [`Addresses.Gateway`](#addressesgateway) port instead. > - See [Security section](#security) for network exposure considerations. Default: `/ip4/127.0.0.1/tcp/5001` Type: `strings` ([multiaddrs][multiaddr]) ### `Addresses.Gateway` [Multiaddr][multiaddr] or array of multiaddrs describing the address to serve the local [HTTP gateway](https://specs.ipfs.tech/http-gateways/) (`/ipfs`, `/ipns`) on. Supported Transports: - tcp/ip{4,6} - `/ipN/.../tcp/...` - unix - `/unix/path/to/socket` > [!CAUTION] > **SECURITY CONSIDERATIONS FOR GATEWAY EXPOSURE** > > By default, the gateway is bound to localhost for security. If you bind to `0.0.0.0` > or a public IP, anyone with access can trigger retrieval of arbitrary CIDs, causing > bandwidth usage and potential exposure to malicious content. Limit with > [`Gateway.NoFetch`](#gatewaynofetch). Consider firewall rules, authentication, > and [`Gateway.PublicGateways`](#gatewaypublicgateways) for public exposure. > See [Security section](#security) for network exposure considerations. Default: `/ip4/127.0.0.1/tcp/8080` Type: `strings` ([multiaddrs][multiaddr]) ### `Addresses.Swarm` An array of [multiaddrs][multiaddr] describing which addresses to listen on for p2p swarm connections. Supported Transports: - tcp/ip{4,6} - `/ipN/.../tcp/...` - websocket - `/ipN/.../tcp/.../ws` - quicv1 (RFC9000) - `/ipN/.../udp/.../quic-v1` - can share the same two tuple with `/quic-v1/webtransport` - webtransport `/ipN/.../udp/.../quic-v1/webtransport` - can share the same two tuple with `/quic-v1` > [!IMPORTANT] > Make sure your firewall rules allow incoming connections on both TCP and UDP ports defined here. > See [Security section](#security) for network exposure considerations. Note that quic (Draft-29) used to be supported with the format `/ipN/.../udp/.../quic`, but has since been [removed](https://github.com/libp2p/go-libp2p/releases/tag/v0.30.0). Default: ```json [ "/ip4/0.0.0.0/tcp/4001", "/ip6/::/tcp/4001", "/ip4/0.0.0.0/udp/4001/quic-v1", "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", "/ip6/::/udp/4001/quic-v1", "/ip6/::/udp/4001/quic-v1/webtransport" ] ``` Type: `array[string]` ([multiaddrs][multiaddr]) ### `Addresses.Announce` If non-empty, this array specifies the swarm addresses to announce to the network. If empty, the daemon will announce inferred swarm addresses. Default: `[]` Type: `array[string]` ([multiaddrs][multiaddr]) ### `Addresses.AppendAnnounce` Similar to [`Addresses.Announce`](#addressesannounce) except this doesn't override inferred swarm addresses if non-empty. Default: `[]` Type: `array[string]` ([multiaddrs][multiaddr]) ### `Addresses.NoAnnounce` An array of swarm addresses not to announce to the network. Takes precedence over `Addresses.Announce` and `Addresses.AppendAnnounce`. > [!TIP] > The [`server` configuration profile](#server-profile) fills up this list with sensible defaults, > preventing announcement of non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`, > which is the [multiaddress][multiaddr] representation of `192.168.0.0/16`) but you should always > check settings against your own network and/or hosting provider. Default: `[]` Type: `array[string]` ([multiaddrs][multiaddr]) ## `API` Contains information used by the [Kubo RPC API](https://docs.ipfs.tech/reference/kubo/rpc/). ### `API.HTTPHeaders` Map of HTTP headers to set on responses from the RPC (`/api/v0`) HTTP server. Example: ```json { "Foo": ["bar"] } ``` Default: `null` Type: `object[string -> array[string]]` (header names -> array of header values) ### `API.Authorizations` The `API.Authorizations` field defines user-based access restrictions for the [Kubo RPC API](https://docs.ipfs.tech/reference/kubo/rpc/), which is located at `Addresses.API` under `/api/v0` paths. By default, the admin-level RPC API is accessible without restrictions as it is only exposed on `127.0.0.1` and safeguarded with Origin check and implicit [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) headers that block random websites from accessing the RPC. When entries are defined in `API.Authorizations`, RPC requests will be declined unless a corresponding secret is present in the HTTP [`Authorization` header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization), and the requested path is included in the `AllowedPaths` list for that specific secret. > [!CAUTION] > **NEVER EXPOSE UNPROTECTED ADMIN RPC TO LAN OR THE PUBLIC INTERNET** > > The RPC API is vast. It grants admin-level access to your Kubo IPFS node, including > configuration and secret key management. > > - If you need secure access to a subset of RPC, make sure you understand the risk, block everything by default and allow basic auth access with [`API.Authorizations`](#apiauthorizations) or custom auth middleware running in front of the localhost-only port defined in [`Addresses.API`](#addressesapi). > - If you are looking for an interface designed for browsers and public internet, use [`Addresses.Gateway`](#addressesgateway) port instead. Default: `null` Type: `object[string -> object]` (user name -> authorization object, see below) For example, to limit RPC access to Alice (access `id` and MFS `files` commands with HTTP Basic Auth) and Bob (full access with Bearer token): ```json { "API": { "Authorizations": { "Alice": { "AuthSecret": "basic:alice:password123", "AllowedPaths": ["/api/v0/id", "/api/v0/files"] }, "Bob": { "AuthSecret": "bearer:secret-token123", "AllowedPaths": ["/api/v0"] } } } } ``` #### `API.Authorizations: AuthSecret` The `AuthSecret` field denotes the secret used by a user to authenticate, usually via HTTP [`Authorization` header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization). Field format is `type:value`, and the following types are supported: - `bearer:` For secret Bearer tokens, set as `bearer:token`. - If no known `type:` prefix is present, `bearer:` is assumed. - `basic`: For HTTP Basic Auth introduced in [RFC7617](https://datatracker.ietf.org/doc/html/rfc7617). Value can be: - `basic:user:pass` - `basic:base64EncodedBasicAuth` One can use the config value for authentication via the command line: ``` ipfs id --api-auth basic:user:pass ``` Type: `string` #### `API.Authorizations: AllowedPaths` The `AllowedPaths` field is an array of strings containing allowed RPC path prefixes. Users authorized with the related `AuthSecret` will only be able to access paths prefixed by the specified prefixes. For instance: - If set to `["/api/v0"]`, the user will have access to the complete RPC API. - If set to `["/api/v0/id", "/api/v0/files"]`, the user will only have access to the `id` command and all MFS commands under `files`. Note that `/api/v0/version` is always permitted access to allow version check to ensure compatibility. Default: `[]` Type: `array[string]` ## `AutoNAT` Contains the configuration options for the libp2p's [AutoNAT](https://github.com/libp2p/specs/tree/master/autonat) service. The AutoNAT service helps other nodes on the network determine if they're publicly reachable from the rest of the internet. ### `AutoNAT.ServiceMode` When unset (default), the AutoNAT service defaults to _enabled_. Otherwise, this field can take one of two values: - `enabled` - Enable the V1+V2 service (unless the node determines that it, itself, isn't reachable by the public internet). - `legacy-v1` - **DEPRECATED** Same as `enabled` but only V1 service is enabled. Used for testing during as few releases as we [transition to V2](https://github.com/ipfs/kubo/issues/10091), will be removed in the future. - `disabled` - Disable the service. Additional modes may be added in the future. > [!IMPORTANT] > We are in the progress of [rolling out AutoNAT V2](https://github.com/ipfs/kubo/issues/10091). > Right now, by default, a publicly dialable Kubo provides both V1 and V2 service to other peers, > and V1 is still used by Kubo for Autorelay feature. In a future release we will remove V1 and switch all features to use V2. Default: `enabled` Type: `optionalString` ### `AutoNAT.Throttle` When set, this option configures the AutoNAT services throttling behavior. By default, Kubo will rate-limit the number of NAT checks performed for other nodes to 30 per minute, and 3 per peer. ### `AutoNAT.Throttle.GlobalLimit` Configures how many AutoNAT requests to service per `AutoNAT.Throttle.Interval`. Default: 30 Type: `integer` (non-negative, `0` means unlimited) ### `AutoNAT.Throttle.PeerLimit` Configures how many AutoNAT requests per-peer to service per `AutoNAT.Throttle.Interval`. Default: 3 Type: `integer` (non-negative, `0` means unlimited) ### `AutoNAT.Throttle.Interval` Configures the interval for the above limits. Default: 1 Minute Type: `duration` (when `0`/unset, the default value is used) ## `AutoConf` The AutoConf feature enables Kubo nodes to automatically fetch and apply network configuration from a remote JSON endpoint. This system allows dynamic configuration updates for bootstrap peers, DNS resolvers, delegated routing, and IPNS publishing endpoints without requiring manual updates to each node's local config. AutoConf works by using special `"auto"` placeholder values in configuration fields. When Kubo encounters these placeholders, it fetches the latest configuration from the specified URL and resolves the placeholders with the appropriate values at runtime. The original configuration file remains unchanged - `"auto"` values are preserved in the JSON and only resolved in memory during node operation. ### Key Features - **Remote Configuration**: Fetch network defaults from a trusted URL - **Automatic Updates**: Periodic background checks for configuration updates - **Graceful Fallback**: Uses hardcoded IPFS Mainnet bootstrappers when remote config is unavailable - **Validation**: Ensures all fetched configuration values are valid multiaddrs and URLs - **Caching**: Stores multiple versions locally with ETags for efficient updates - **User Notification**: Logs ERROR when new configuration is available requiring node restart - **Debug Logging**: AutoConf operations can be inspected by setting `GOLOG_LOG_LEVEL="error,autoconf=debug"` ### Supported Fields AutoConf can resolve `"auto"` placeholders in the following configuration fields: - `Bootstrap` - Bootstrap peer addresses - `DNS.Resolvers` - DNS-over-HTTPS resolver endpoints - `Routing.DelegatedRouters` - Delegated routing HTTP API endpoints - `Ipns.DelegatedPublishers` - IPNS delegated publishing HTTP API endpoints ### Usage Example ```json { "AutoConf": { "URL": "https://example.com/autoconf.json", "Enabled": true, "RefreshInterval": "24h" }, "Bootstrap": ["auto"], "DNS": { "Resolvers": { ".": ["auto"], "eth.": ["auto"], "custom.": ["https://dns.example.com/dns-query"] } }, "Routing": { "DelegatedRouters": ["auto", "https://router.example.org/routing/v1"] } } ``` **Notes:** - Configuration fetching happens at daemon startup and periodically in the background - When new configuration is detected, users must restart their node to apply changes - Mixed configurations are supported: you can use both `"auto"` and static values - If AutoConf is disabled but `"auto"` values exist, daemon startup will fail with validation errors - Cache is stored in `$IPFS_PATH/autoconf/` with up to 3 versions retained ### Path-Based Routing Configuration AutoConf supports path-based routing URLs that automatically enable specific routing operations based on the URL path. This allows precise control over which HTTP Routing V1 endpoints are used for different operations: **Supported paths:** - `/routing/v1/providers` - Enables provider record lookups only - `/routing/v1/peers` - Enables peer routing lookups only - `/routing/v1/ipns` - Enables IPNS record operations only - No path - Enables all routing operations (backward compatibility) **AutoConf JSON structure with path-based routing:** ```json { "DelegatedRouters": { "mainnet-for-nodes-with-dht": [ "https://cid.contact/routing/v1/providers" ], "mainnet-for-nodes-without-dht": [ "https://delegated-ipfs.dev/routing/v1/providers", "https://delegated-ipfs.dev/routing/v1/peers", "https://delegated-ipfs.dev/routing/v1/ipns" ] }, "DelegatedPublishers": { "mainnet-for-ipns-publishers-with-http": [ "https://delegated-ipfs.dev/routing/v1/ipns" ] } } ``` **Node type categories:** - `mainnet-for-nodes-with-dht`: Mainnet nodes with DHT enabled (typically only need additional provider lookups) - `mainnet-for-nodes-without-dht`: Mainnet nodes without DHT (need comprehensive routing services) - `mainnet-for-ipns-publishers-with-http`: Mainnet nodes that publish IPNS records via HTTP This design enables efficient, selective routing where each endpoint URL automatically determines its capabilities based on the path, while maintaining semantic grouping by node configuration type. Default: `{}` Type: `object` ### `AutoConf.Enabled` Controls whether the AutoConf system is active. When enabled, Kubo will fetch configuration from the specified URL and resolve `"auto"` placeholders at runtime. When disabled, any `"auto"` values in the configuration will cause daemon startup to fail with validation errors. This provides a safety mechanism to ensure nodes don't start with unresolved placeholders when AutoConf is intentionally disabled. Default: `true` Type: `flag` ### `AutoConf.URL` Specifies the HTTP(S) URL from which to fetch the autoconf JSON. The endpoint should return a JSON document containing Bootstrap peers, DNS resolvers, delegated routing endpoints, and IPNS publishing endpoints that will replace `"auto"` placeholders in the local configuration. The URL must serve a JSON document matching the AutoConf schema. Kubo validates all multiaddr and URL values before caching to ensure they are properly formatted. When not specified in the configuration, the default mainnet URL is used automatically. > [!NOTE] > Public good autoconf manifest at `conf.ipfs-mainnet.org` is provided by the team at [Shipyard](https://ipshipyard.com). Default: `"https://conf.ipfs-mainnet.org/autoconf.json"` (when not specified) Type: `optionalString` ### `AutoConf.RefreshInterval` Specifies how frequently Kubo should refresh autoconf data. This controls both how often cached autoconf data is considered fresh and how frequently the background service checks for new configuration updates. When a new configuration version is detected during background updates, Kubo logs an ERROR message informing the user that a node restart is required to apply the changes to any `"auto"` entries in their configuration. Default: `24h` Type: `optionalDuration` ### `AutoConf.TLSInsecureSkipVerify` **FOR TESTING ONLY** - Allows skipping TLS certificate verification when fetching autoconf from HTTPS URLs. This should never be enabled in production as it makes the configuration fetching vulnerable to man-in-the-middle attacks. Default: `false` Type: `flag` ## `AutoTLS` The [AutoTLS](https://web.archive.org/web/20260112031855/https://blog.libp2p.io/autotls/) feature enables publicly reachable Kubo nodes (those dialable from the public internet) to automatically obtain a wildcard TLS certificate for a DNS name unique to their PeerID at `*.[PeerID].libp2p.direct`. This enables direct libp2p connections and retrieval of IPFS content from browsers [Secure Context](https://developer.mozilla.org/en-US/docs/Web/Security/Secure_Contexts) using transports such as [Secure WebSockets](https://github.com/libp2p/specs/blob/master/websockets/README.md), without requiring user to do any manual domain registration and certificate configuration. Under the hood, [p2p-forge] client uses public utility service at `libp2p.direct` as an [ACME DNS-01 Challenge](https://letsencrypt.org/docs/challenge-types/#dns-01-challenge) broker enabling peer to obtain a wildcard TLS certificate tied to public key of their [PeerID](https://web.archive.org/web/20251112181025/https://docs.libp2p.io/concepts/fundamentals/peers/#peer-id). By default, the certificates are requested from Let's Encrypt. Origin and rationale for this project can be found in [community.letsencrypt.org discussion](https://community.letsencrypt.org/t/feedback-on-raising-certificates-per-registered-domain-to-enable-peer-to-peer-networking/223003). > [!NOTE] > Public good DNS and [p2p-forge] infrastructure at `libp2p.direct` is run by the team at [Interplanetary Shipyard](https://ipshipyard.com). > [p2p-forge]: https://github.com/ipshipyard/p2p-forge Default: `{}` Type: `object` ### `AutoTLS.Enabled` Enables the AutoTLS feature to provide DNS and TLS support for [libp2p Secure WebSocket](https://github.com/libp2p/specs/blob/master/websockets/README.md) over a `/tcp` port, to allow JS clients running in web browser [Secure Context](https://w3c.github.io/webappsec-secure-contexts/) to connect to Kubo directly. When activated, together with [`AutoTLS.AutoWSS`](#autotlsautowss) (default) or manually including a `/tcp/{port}/tls/sni/*.libp2p.direct/ws` multiaddr in [`Addresses.Swarm`](#addressesswarm) (with SNI suffix matching [`AutoTLS.DomainSuffix`](#autotlsdomainsuffix)), Kubo retrieves a trusted PKI TLS certificate for `*.{peerid}.libp2p.direct` and configures the `/ws` listener to use it. **Note:** - This feature requires a publicly reachable node. If behind NAT, manual port forwarding or UPnP (`Swarm.DisableNatPortMap=false`) is required. - The first time AutoTLS is used, it may take 5-15 minutes + [`AutoTLS.RegistrationDelay`](#autotlsregistrationdelay) before `/ws` listener is added. Be patient. - Avoid manual configuration. [`AutoTLS.AutoWSS=true`](#autotlsautowss) should automatically add `/ws` listener to existing, firewall-forwarded `/tcp` ports. - To troubleshoot, use `GOLOG_LOG_LEVEL="error,autotls=debug` for detailed logs, or `GOLOG_LOG_LEVEL="error,autotls=info` for quieter output. - Certificates are stored in `$IPFS_PATH/p2p-forge-certs`; deleting this directory and restarting the daemon forces a certificate rotation. - For now, the TLS cert applies solely to `/ws` libp2p WebSocket connections, not HTTP [`Gateway`](#gateway), which still need separate reverse proxy TLS setup with a custom domain. Default: `true` Type: `flag` ### `AutoTLS.AutoWSS` Optional. Controls if Kubo should add `/tls/sni/*.libp2p.direct/ws` listener to every pre-existing `/tcp` port IFF no explicit `/ws` is defined in [`Addresses.Swarm`](#addressesswarm) already. Default: `true` (if `AutoTLS.Enabled`) Type: `flag` ### `AutoTLS.ShortAddrs` Optional. Controls if final AutoTLS listeners are announced under shorter `/dnsX/A.B.C.D.peerid.libp2p.direct/tcp/4001/tls/ws` addresses instead of fully resolved `/ip4/A.B.C.D/tcp/4001/tls/sni/A-B-C-D.peerid.libp2p.direct/tls/ws`. The main use for AutoTLS is allowing connectivity from Secure Context in a web browser, and DNS lookup needs to happen there anyway, making `/dnsX` a more compact, more interoperable option without obvious downside. Default: `true` Type: `flag` ### `AutoTLS.SkipDNSLookup` Optional. Controls whether to skip network DNS lookups for [p2p-forge] domains like `*.libp2p.direct`. This applies to DNS resolution performed via [`DNS.Resolvers`](#dnsresolvers), including `/dns*` multiaddrs resolved by go-libp2p (e.g., peer addresses from DHT or delegated routing). When enabled (default), A/AAAA queries for hostnames matching [`AutoTLS.DomainSuffix`](#autotlsdomainsuffix) are resolved locally by parsing the IP address directly from the hostname (e.g., `1-2-3-4.peerID.libp2p.direct` resolves to `1.2.3.4` without network I/O). This avoids unnecessary DNS queries since the IP is already encoded in the hostname. If the hostname format is invalid (wrong peerID, malformed IP encoding), the resolver falls back to network DNS, ensuring forward compatibility with potential future DNS record types. Set to `false` to always use network DNS for these domains. This is primarily useful for debugging or if you need to override resolution behavior via [`DNS.Resolvers`](#dnsresolvers). Default: `true` Type: `flag` ### `AutoTLS.DomainSuffix` Optional override of the parent domain suffix that will be used in DNS+TLS+WebSockets multiaddrs generated by [p2p-forge] client. Do not change this unless you self-host [p2p-forge]. Default: `libp2p.direct` (public good run by [Interplanetary Shipyard](https://ipshipyard.com)) Type: `optionalString` ### `AutoTLS.RegistrationEndpoint` Optional override of [p2p-forge] HTTP registration API. Do not change this unless you self-host [p2p-forge] under own domain. > [!IMPORTANT] > The default endpoint performs [libp2p Peer ID Authentication over HTTP](https://github.com/libp2p/specs/blob/master/http/peer-id-auth.md) > (proving ownership of PeerID), probes if your Kubo node can correctly answer to a [libp2p Identify](https://github.com/libp2p/specs/tree/master/identify) query. > This ensures only a correctly configured, publicly dialable Kubo can initiate [ACME DNS-01 challenge](https://letsencrypt.org/docs/challenge-types/#dns-01-challenge) for `peerid.libp2p.direct`. Default: `https://registration.libp2p.direct` (public good run by [Interplanetary Shipyard](https://ipshipyard.com)) Type: `optionalString` ### `AutoTLS.RegistrationToken` Optional value for `Forge-Authorization` token sent with request to `RegistrationEndpoint` (useful for private/self-hosted/test instances of [p2p-forge], unset by default). Default: `""` Type: `optionalString` ### `AutoTLS.RegistrationDelay` An additional delay applied before sending a request to the `RegistrationEndpoint`. The default delay is bypassed if the user explicitly set `AutoTLS.Enabled=true` in the JSON configuration file. This ensures that ephemeral nodes using the default configuration do not spam the`AutoTLS.CAEndpoint` with unnecessary ACME requests. Default: `1h` (or `0` if explicit `AutoTLS.Enabled=true`) Type: `optionalDuration` ### `AutoTLS.CAEndpoint` Optional override of CA ACME API used by [p2p-forge] system. Do not change this unless you self-host [p2p-forge] under own domain. > [!IMPORTANT] > CAA DNS record at `libp2p.direct` limits CA choice to Let's Encrypt. If you want to use a different CA, use your own domain. Default: [certmagic.LetsEncryptProductionCA](https://pkg.go.dev/github.com/caddyserver/certmagic#pkg-constants) (see [community.letsencrypt.org discussion](https://community.letsencrypt.org/t/feedback-on-raising-certificates-per-registered-domain-to-enable-peer-to-peer-networking/223003)) Type: `optionalString` ## `Bitswap` High level client and server configuration of the [Bitswap Protocol](https://specs.ipfs.tech/bitswap-protocol/) over libp2p. For internal configuration see [`Internal.Bitswap`](#internalbitswap). For HTTP version see [`HTTPRetrieval`](#httpretrieval). ### `Bitswap.Libp2pEnabled` Determines whether Kubo will use Bitswap over libp2p. Disabling this, will remove `/ipfs/bitswap/*` protocol support from [libp2p identify](https://github.com/libp2p/specs/blob/master/identify/README.md) responses, effectively shutting down both Bitswap libp2p client and server. > [!WARNING] > Bitswap over libp2p is a core component of Kubo and the oldest way of exchanging blocks. Disabling it completely may cause unpredictable outcomes, such as retrieval failures, if the only providers were libp2p ones. Treat this as experimental and use it solely for testing purposes with `HTTPRetrieval.Enabled`. Default: `true` Type: `flag` ### `Bitswap.ServerEnabled` Determines whether Kubo functions as a Bitswap server to host and respond to block requests. Disabling the server retains client and protocol support in [libp2p identify](https://github.com/libp2p/specs/blob/master/identify/README.md) responses but causes Kubo to reply with "don't have" to all block requests. Default: `true` (requires `Bitswap.Libp2pEnabled`) Type: `flag` ## `Bootstrap` Bootstrap peers help your node discover and connect to the IPFS network when starting up. This array contains [multiaddrs][multiaddr] of trusted nodes that your node contacts first to find other peers and content. The special value `"auto"` automatically uses curated, up-to-date bootstrap peers from [AutoConf](#autoconf), ensuring your node can always connect to the healthy network without manual maintenance. **What this gives you:** - **Reliable startup**: Your node can always find the network, even if some bootstrap peers go offline - **Automatic updates**: New bootstrap peers are added as the network evolves - **Custom control**: Add your own trusted peers alongside or instead of the defaults Default: `["auto"]` Type: `array[string]` ([multiaddrs][multiaddr] or `"auto"`) ## `Datastore` Contains information related to the construction and operation of the on-disk storage system. ### `Datastore.StorageMax` A soft upper limit for the size of the ipfs repository's datastore. With `StorageGCWatermark`, is used to calculate whether to trigger a gc run (only if `--enable-gc` flag is set). > [!NOTE] > This only controls when automatic GC of raw blocks is triggered. It is not a > hard limit on total disk usage. The metadata stored alongside blocks (pins, > MFS, provider system state, pubsub message ID tracking, and other internal > data) is not counted against this limit. Always include extra headroom to > account for metadata overhead. See [datastores.md](datastores.md) for details > on how different datastore backends handle disk space reclamation. Default: `"10GB"` Type: `string` (size) ### `Datastore.StorageGCWatermark` The percentage of the `StorageMax` value at which a garbage collection will be triggered automatically if the daemon was run with automatic gc enabled (that option defaults to false currently). Default: `90` Type: `integer` (0-100%) ### `Datastore.GCPeriod` A time duration specifying how frequently to run a garbage collection. Only used if automatic gc is enabled. Default: `1h` Type: `duration` (an empty string means the default value) ### `Datastore.HashOnRead` A boolean value. If set to true, all block reads from the disk will be hashed and verified. This will cause increased CPU utilization. Default: `false` Type: `bool` ### `Datastore.BloomFilterSize` A number representing the size in bytes of the blockstore's [bloom filter](https://en.wikipedia.org/wiki/Bloom_filter). A value of zero represents the feature is disabled. This site generates useful graphs for various bloom filter values: You may use it to find a preferred optimal value, where `m` is `BloomFilterSize` in bits. Remember to convert the value `m` from bits, into bytes for use as `BloomFilterSize` in the config file. For example, for 1,000,000 blocks, expecting a 1% false-positive rate, you'd end up with a filter size of 9592955 bits, so for `BloomFilterSize` we'd want to use 1199120 bytes. As of writing, [7 hash functions](https://github.com/ipfs/go-ipfs-blockstore/blob/547442836ade055cc114b562a3cc193d4e57c884/caching.go#L22) are used, so the constant `k` is 7 in the formula. Enabling the BloomFilter can provide performance improvements specially when responding to many requests for inexistent blocks. It however requires a full sweep of all the datastore keys on daemon start. On very large datastores this can be a very taxing operation, particularly if the datastore does not support querying existing keys without reading their values at the same time (blocks). Default: `0` (disabled) Type: `integer` (non-negative, bytes) ### `Datastore.WriteThrough` This option controls whether a block that already exist in the datastore should be written to it. When set to `false`, a `Has()` call is performed against the datastore prior to writing every block. If the block is already stored, the write is skipped. This check happens both on the Blockservice and the Blockstore layers and this setting affects both. When set to `true`, no checks are performed and blocks are written to the datastore, which depending on the implementation may perform its own checks. This option can affect performance and the strategy should be taken in conjunction with [`BlockKeyCacheSize`](#datastoreblockkeycachesize) and [`BloomFilterSize`](#datastoreboomfiltersize`). Default: `true` Type: `bool` ### `Datastore.BlockKeyCacheSize` A number representing the maximum size in bytes of the blockstore's Two-Queue cache, which caches block-cids and their block-sizes. Use `0` to disable. This cache, once primed, can greatly speed up operations like `ipfs repo stat` as there is no need to read full blocks to know their sizes. Size should be adjusted depending on the number of CIDs on disk (`NumObjects in`ipfs repo stat`). Default: `65536` (64KiB) Type: `optionalInteger` (non-negative, bytes) ### `Datastore.Spec` Spec defines the structure of the ipfs datastore. It is a composable structure, where each datastore is represented by a json object. Datastores can wrap other datastores to provide extra functionality (eg metrics, logging, or caching). > [!NOTE] > For more information on possible values for this configuration option, see [`kubo/docs/datastores.md`](datastores.md) Default: ``` { "mounts": [ { "mountpoint": "/blocks", "path": "blocks", "prefix": "flatfs.datastore", "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2", "sync": false, "type": "flatfs" }, { "compression": "none", "mountpoint": "/", "path": "datastore", "prefix": "leveldb.datastore", "type": "levelds" } ], "type": "mount" } ``` With `flatfs-measure` profile: ``` { "mounts": [ { "child": { "path": "blocks", "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2", "sync": true, "type": "flatfs" }, "mountpoint": "/blocks", "prefix": "flatfs.datastore", "type": "measure" }, { "child": { "compression": "none", "path": "datastore", "type": "levelds" }, "mountpoint": "/", "prefix": "leveldb.datastore", "type": "measure" } ], "type": "mount" } ``` Type: `object` ## `Discovery` Contains options for configuring IPFS node discovery mechanisms. ### `Discovery.MDNS` Options for [ZeroConf](https://github.com/libp2p/zeroconf#readme) Multicast DNS-SD peer discovery. #### `Discovery.MDNS.Enabled` A boolean value to activate or deactivate Multicast DNS-SD. Default: `true` Type: `bool` #### `Discovery.MDNS.Interval` **REMOVED:** this is not configurable anymore in the [new mDNS implementation](https://github.com/libp2p/zeroconf#readme). ## `Experimental` Toggle and configure experimental features of Kubo. Experimental features are listed [here](./experimental-features.md). ### `Experimental.Libp2pStreamMounting` Enables the `ipfs p2p` commands for tunneling TCP connections through libp2p streams, similar to SSH port forwarding. See [docs/p2p-tunnels.md](p2p-tunnels.md) for usage examples. Default: `false` Type: `bool` ## `Gateway` Options for the HTTP gateway. > [!IMPORTANT] > By default, Kubo's gateway is configured for local use at `127.0.0.1` and `localhost`. > To run a public gateway, configure your domain names in [`Gateway.PublicGateways`](#gatewaypublicgateways). > For production deployment considerations (reverse proxy, timeouts, rate limiting, CDN), > see [Running in Production](gateway.md#running-in-production). ### `Gateway.NoFetch` When set to true, the gateway will only serve content already in the local repo and will not fetch files from the network. Default: `false` Type: `bool` ### `Gateway.NoDNSLink` A boolean to configure whether DNSLink lookup for value in `Host` HTTP header should be performed. If DNSLink is present, the content path stored in the DNS TXT record becomes the `/` and the respective payload is returned to the client. Default: `false` Type: `bool` ### `Gateway.DeserializedResponses` An optional flag to explicitly configure whether this gateway responds to deserialized requests, or not. By default, it is enabled. When disabling this option, the gateway operates as a Trustless Gateway only: . Default: `true` Type: `flag` ### `Gateway.AllowCodecConversion` An optional flag to enable automatic conversion between codecs when the requested format differs from the block's native codec (e.g., converting dag-pb or dag-cbor to dag-json). When disabled (the default), the gateway returns `406 Not Acceptable` for codec mismatches, following behavior specified in [IPIP-524](https://specs.ipfs.tech/ipips/ipip-0524/). Most users should keep this disabled unless legacy [IPLD Logical Format](https://web.archive.org/web/20260204204727/https://ipld.io/specs/codecs/dag-pb/spec/#logical-format) support is needed as a stop-gap while switching clients to `?format=raw` and converting client-side. Instead of relying on gateway-side conversion, fetch the raw block using `?format=raw` (`application/vnd.ipld.raw`) and convert client-side. This: - Allows clients to use any codec without waiting for gateway support - Enables ecosystem innovation without gateway operator coordination - Works with libraries like [@helia/verified-fetch](https://www.npmjs.com/package/@helia/verified-fetch) in JavaScript Default: `false` Type: `flag` ### `Gateway.DisableHTMLErrors` An optional flag to disable the pretty HTML error pages of the gateway. Instead, a `text/plain` page will be returned with the raw error message from Kubo. It is useful for whitelabel or middleware deployments that wish to avoid `text/html` responses with IPFS branding and links on error pages in browsers. Default: `false` Type: `flag` ### `Gateway.ExposeRoutingAPI` An optional flag to expose Kubo `Routing` system on the gateway port as an [HTTP `/routing/v1`](https://specs.ipfs.tech/routing/http-routing-v1/) endpoint on `127.0.0.1`. Use reverse proxy to expose it on a different hostname. This endpoint can be used by other Kubo instances, as illustrated in [`delegated_routing_v1_http_proxy_test.go`](https://github.com/ipfs/kubo/blob/master/test/cli/delegated_routing_v1_http_proxy_test.go). Kubo will filter out routing results which are not actionable, for example, all graphsync providers will be skipped. If you need a generic pass-through, see standalone router implementation named [someguy](https://github.com/ipfs/someguy). Default: `true` Type: `flag` ### `Gateway.RetrievalTimeout` Maximum duration Kubo will wait for content retrieval (new bytes to arrive). **Timeout behavior:** - **Time to first byte**: Returns 504 Gateway Timeout if the gateway cannot start writing within this duration (e.g., stuck searching for providers) - **Time between writes**: After first byte, timeout resets with each write. Response terminates if no new data can be written within this duration **Truncation handling:** When timeout occurs after HTTP 200 headers are sent (e.g., during CAR streams), the gateway: - Appends error message to indicate truncation - Forces TCP reset (RST) to prevent caching incomplete responses - Records in metrics with original status code and `truncated=true` flag **Monitoring:** Track `ipfs_http_gw_retrieval_timeouts_total` by status code and truncation status. **Tuning guidance:** - Compare timeout rates (`ipfs_http_gw_retrieval_timeouts_total`) with success rates (`ipfs_http_gw_responses_total{status="200"}`) - High timeout rate: consider increasing timeout or scaling horizontally if hardware is constrained - Many 504s may indicate routing problems - check requested CIDs and provider availability using - `truncated=true` timeouts indicate retrieval stalled mid-file with no new bytes for the timeout duration A value of 0 disables this timeout. Default: `30s` Type: `optionalDuration` ### `Gateway.MaxRequestDuration` An absolute deadline for the entire gateway request. Unlike [`RetrievalTimeout`](#gatewayretrievaltimeout) (which resets on each data write and catches stalled transfers), this is a hard limit on the total time a request can take. Returns 504 Gateway Timeout when exceeded. This protects the gateway from edge cases and slow client attacks. Default: `1h` Type: `optionalDuration` ### `Gateway.MaxRangeRequestFileSize` Maximum file size for HTTP range requests on deserialized responses. Range requests for files larger than this limit return 501 Not Implemented. **Why this exists:** Some CDNs like Cloudflare intercept HTTP range requests and convert them to full file downloads when files exceed their cache bucket limits. Cloudflare's default plan only caches range requests for files up to 5GiB. Files larger than this receive HTTP 200 with the entire file instead of HTTP 206 with the requested byte range. A client requesting 1MB from a 40GiB file would unknowingly download all 40GiB, causing bandwidth overcharges for the gateway operator, unexpected data costs for the client, and potential browser crashes. This only affects deserialized responses. Clients fetching verifiable blocks as `application/vnd.ipld.raw` are not impacted because they work with small chunks that stay well below CDN cache limits. **How to use:** Set this to your CDN's range request cache limit (e.g., `"5GiB"` for Cloudflare's default plan). The gateway returns 501 Not Implemented for range requests over files larger than this limit, with an error message suggesting verifiable block requests as an alternative. > [!NOTE] > Cloudflare users running open gateway hosting deserialized responses should deploy additional protection via Cloudflare Snippets (requires Enterprise plan). The Kubo configuration alone is not sufficient because Cloudflare has already intercepted and cached the response by the time it reaches your origin. See [boxo#856](https://github.com/ipfs/boxo/issues/856#issuecomment-3523944976) for a snippet that aborts HTTP 200 responses when Content-Length exceeds the limit. Default: `0` (no limit) Type: [`optionalBytes`](#optionalbytes) ### `Gateway.MaxConcurrentRequests` Limits concurrent HTTP requests. Requests beyond limit receive 429 Too Many Requests. Protects nodes from traffic spikes and resource exhaustion, especially behind reverse proxies without rate-limiting. Default (4096) aligns with common reverse proxy configurations (e.g., nginx: 8 workers × 1024 connections). **Monitoring:** `ipfs_http_gw_concurrent_requests` tracks current requests in flight. **Tuning guidance:** - Monitor `ipfs_http_gw_concurrent_requests` gauge for usage patterns - Track 429s (`ipfs_http_gw_responses_total{status="429"}`) and success rate (`{status="200"}`) - Near limit with low resource usage → increase value - Memory pressure or OOMs → decrease value and consider scaling - Set slightly below reverse proxy limit for graceful degradation - Start with default, adjust based on observed performance for your hardware A value of 0 disables the limit. Default: `4096` Type: `optionalInteger` ### `Gateway.HTTPHeaders` Headers to set on gateway responses. Default: `{}` + implicit CORS headers from `boxo/gateway#AddAccessControlHeaders` and [ipfs/specs#423](https://github.com/ipfs/specs/issues/423) Type: `object[string -> array[string]]` ### `Gateway.RootRedirect` A URL to redirect requests for `/` to. Default: `""` Type: `string` (url) ### `Gateway.DiagnosticServiceURL` URL for a service to diagnose CID retrievability issues. When the gateway returns a 504 Gateway Timeout error, an "Inspect retrievability of CID" button will be shown that links to this service with the CID appended as `?cid=`. Set to empty string to disable the button. Default: `"https://check.ipfs.network"` Type: `optionalstring` (url) ### `Gateway.FastDirIndexThreshold` **REMOVED**: this option is [no longer necessary](https://github.com/ipfs/kubo/pull/9481). Ignored since [Kubo 0.18](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.18.md). ### `Gateway.Writable` **REMOVED**: this option no longer available as of [Kubo 0.20](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.20.md). We are working on developing a modern replacement. To support our efforts, please leave a comment describing your use case in [ipfs/specs#375](https://github.com/ipfs/specs/issues/375). ### `Gateway.PathPrefixes` **REMOVED:** see [go-ipfs#7702](https://github.com/ipfs/go-ipfs/issues/7702) ### `Gateway.PublicGateways` > [!IMPORTANT] > This configuration is **NOT** for HTTP Client, it is for HTTP Server – use this ONLY if you want to run your own IPFS gateway. `PublicGateways` is a configuration map used for dictionary for customizing gateway behavior on specified hostnames that point at your Kubo instance. It is useful when you want to run [Path gateway](https://specs.ipfs.tech/http-gateways/path-gateway/) on `example.com/ipfs/cid`, and [Subdomain gateway](https://specs.ipfs.tech/http-gateways/subdomain-gateway/) on `cid.ipfs.example.org`, or limit `verifiable.example.net` to response types defined in [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/) specification. > [!CAUTION] > Keys (Hostnames) MUST be unique. Do not use the same parent domain for multiple gateway types, it will break origin isolation. Hostnames can optionally be defined with one or more wildcards. Examples: - `*.example.com` will match requests to `http://foo.example.com/ipfs/*` or `http://{cid}.ipfs.bar.example.com/*`. - `foo-*.example.com` will match requests to `http://foo-bar.example.com/ipfs/*` or `http://{cid}.ipfs.foo-xyz.example.com/*`. > [!IMPORTANT] > **Reverse Proxy:** If running behind nginx or another reverse proxy, ensure > `Host` and `X-Forwarded-*` headers are forwarded correctly. > See [Reverse Proxy Caveats](gateway.md#reverse-proxy) in gateway documentation. #### `Gateway.PublicGateways: Paths` An array of paths that should be exposed on the hostname. Example: ```json { "Gateway": { "PublicGateways": { "example.com": { "Paths": ["/ipfs"], } } } } ``` Above enables `http://example.com/ipfs/*` but not `http://example.com/ipns/*` Default: `[]` Type: `array[string]` #### `Gateway.PublicGateways: UseSubdomains` A boolean to configure whether the gateway at the hostname should be a [Subdomain Gateway](https://specs.ipfs.tech/http-gateways/subdomain-gateway/) and provide [Origin isolation](https://developer.mozilla.org/en-US/docs/Web/Security/Same-origin_policy) between content roots. - `true` - enables [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://*.{hostname}/` - **Requires whitelist:** make sure respective `Paths` are set. For example, `Paths: ["/ipfs", "/ipns"]` are required for `http://{cid}.ipfs.{hostname}` and `http://{foo}.ipns.{hostname}` to work: ```json "Gateway": { "PublicGateways": { "dweb.link": { "UseSubdomains": true, "Paths": ["/ipfs", "/ipns"] } } } ``` - **Backward-compatible:** requests for content paths such as `http://{hostname}/ipfs/{cid}` produce redirect to `http://{cid}.ipfs.{hostname}` - `false` - enables [path gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#path-gateway) at `http://{hostname}/*` - Example: ```json "Gateway": { "PublicGateways": { "ipfs.io": { "UseSubdomains": false, "Paths": ["/ipfs", "/ipns"] } } } ``` Default: `false` Type: `bool` > [!IMPORTANT] > See [Reverse Proxy Caveats](gateway.md#reverse-proxy) if running behind nginx or another reverse proxy. #### `Gateway.PublicGateways: NoDNSLink` A boolean to configure whether DNSLink for hostname present in `Host` HTTP header should be resolved. Overrides global setting. If `Paths` are defined, they take priority over DNSLink. Default: `false` (DNSLink lookup enabled by default for every defined hostname) Type: `bool` > [!IMPORTANT] > See [Reverse Proxy Caveats](gateway.md#reverse-proxy) if running behind nginx or another reverse proxy. #### `Gateway.PublicGateways: InlineDNSLink` An optional flag to explicitly configure whether subdomain gateway's redirects (enabled by `UseSubdomains: true`) should always inline a DNSLink name (FQDN) into a single DNS label ([specification](https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header)): ``` //example.com/ipns/example.net → HTTP 301 → //example-net.ipns.example.com ``` DNSLink name inlining allows for HTTPS on public subdomain gateways with single label wildcard TLS certs (also enabled when passing `X-Forwarded-Proto: https`), and provides disjoint Origin per root CID when special rules like , or a custom localhost logic in browsers like Brave has to be applied. Default: `false` Type: `flag` #### `Gateway.PublicGateways: DeserializedResponses` An optional flag to explicitly configure whether this gateway responds to deserialized requests, or not. By default, it is enabled. When disabled, the gateway operates strictly as a [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/). > [!TIP] > Disabling deserialized responses will protect you from acting as a free web hosting, > while still allowing trustless clients like [@helia/verified-fetch](https://www.npmjs.com/package/@helia/verified-fetch) > to utilize it for [trustless, verifiable data retrieval](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval). Default: same as global `Gateway.DeserializedResponses` Type: `flag` #### Implicit defaults of `Gateway.PublicGateways` Default entries for `localhost` hostname and loopback IPs are always present. If additional config is provided for those hostnames, it will be merged on top of implicit values: ```json { "Gateway": { "PublicGateways": { "localhost": { "Paths": ["/ipfs", "/ipns"], "UseSubdomains": true } } } } ``` It is also possible to remove a default by setting it to `null`. For example, to disable subdomain gateway on `localhost` and make that hostname act the same as `127.0.0.1`: ```console ipfs config --json Gateway.PublicGateways '{"localhost": null }' ``` ### `Gateway` recipes Below is a list of the most common gateway setups. > [!IMPORTANT] > See [Reverse Proxy Caveats](gateway.md#reverse-proxy) if running behind nginx or another reverse proxy. - Public [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://{cid}.ipfs.dweb.link` (each content root gets its own Origin) ```console $ ipfs config --json Gateway.PublicGateways '{ "dweb.link": { "UseSubdomains": true, "Paths": ["/ipfs", "/ipns"] } }' ``` - **Performance:** Consider enabling `Routing.AcceleratedDHTClient=true` to improve content routing lookups. Separately, gateway operators should decide if the gateway node should also co-host and provide (announce) fetched content to the DHT. If providing content, enable `Provide.DHT.SweepEnabled=true` for efficient announcements. If announcements are still not fast enough, adjust `Provide.DHT.MaxWorkers`. For a read-only gateway that doesn't announce content, use `Provide.Enabled=false`. - **Backward-compatible:** this feature enables automatic redirects from content paths to subdomains: `http://dweb.link/ipfs/{cid}` → `http://{cid}.ipfs.dweb.link` - **X-Forwarded-Proto:** if you run Kubo behind a reverse proxy that provides TLS, make it add a `X-Forwarded-Proto: https` HTTP header to ensure users are redirected to `https://`, not `http://`. It will also ensure DNSLink names are inlined to fit in a single DNS label, so they work fine with a wildcard TLS cert ([details](https://github.com/ipfs/in-web-browsers/issues/169)). The NGINX directive is `proxy_set_header X-Forwarded-Proto "https";`.: `http://dweb.link/ipfs/{cid}` → `https://{cid}.ipfs.dweb.link` `http://dweb.link/ipns/your-dnslink.site.example.com` → `https://your--dnslink-site-example-com.ipfs.dweb.link` - **X-Forwarded-Host:** we also support `X-Forwarded-Host: example.com` if you want to override subdomain gateway host from the original request: `http://dweb.link/ipfs/{cid}` → `http://{cid}.ipfs.example.com` - Public [path gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#path-gateway) at `http://ipfs.io/ipfs/{cid}` (no Origin separation) ```console $ ipfs config --json Gateway.PublicGateways '{ "ipfs.io": { "UseSubdomains": false, "Paths": ["/ipfs", "/ipns"] } }' ``` - **Performance:** Consider enabling `Routing.AcceleratedDHTClient=true` to improve content routing lookups. When running an open, recursive gateway, decide if the gateway should also co-host and provide (announce) fetched content to the DHT. If providing content, enable `Provide.DHT.SweepEnabled=true` for efficient announcements. If announcements are still not fast enough, adjust `Provide.DHT.MaxWorkers`. For a read-only gateway that doesn't announce content, use `Provide.Enabled=false`. - Public [DNSLink](https://dnslink.io/) gateway resolving every hostname passed in `Host` header. ```console ipfs config --json Gateway.NoDNSLink false ``` - Note that `NoDNSLink: false` is the default (it works out of the box unless set to `true` manually) - Hardened, site-specific [DNSLink gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#dnslink-gateway). Disable fetching of remote data (`NoFetch: true`) and resolving DNSLink at unknown hostnames (`NoDNSLink: true`). Then, enable DNSLink gateway only for the specific hostname (for which data is already present on the node), without exposing any content-addressing `Paths`: ```console $ ipfs config --json Gateway.NoFetch true $ ipfs config --json Gateway.NoDNSLink true $ ipfs config --json Gateway.PublicGateways '{ "en.wikipedia-on-ipfs.org": { "NoDNSLink": false, "Paths": [] } }' ``` ## `Identity` ### `Identity.PeerID` The unique PKI identity label for this configs peer. Set on init and never read, it's merely here for convenience. Ipfs will always generate the peerID from its keypair at runtime. Type: `string` (peer ID) ### `Identity.PrivKey` The base64 encoded protobuf describing (and containing) the node's private key. Type: `string` (base64 encoded) ## `Internal` This section includes internal knobs for various subsystems to allow advanced users with big or private infrastructures to fine-tune some behaviors without the need to recompile Kubo. **Be aware that making informed change here requires in-depth knowledge and most users should leave these untouched. All knobs listed here are subject to breaking changes between versions.** ### `Internal.Bitswap` `Internal.Bitswap` contains knobs for tuning bitswap resource utilization. > [!TIP] > For high level configuration see [`Bitswap`](#bitswap). The knobs (below) document how their value should related to each other. Whether their values should be raised or lowered should be determined based on the metrics `ipfs_bitswap_active_tasks`, `ipfs_bitswap_pending_tasks`, `ipfs_bitswap_pending_block_tasks` and `ipfs_bitswap_active_block_tasks` reported by bitswap. These metrics can be accessed as the Prometheus endpoint at `{Addresses.API}/debug/metrics/prometheus` (default: `http://127.0.0.1:5001/debug/metrics/prometheus`) The value of `ipfs_bitswap_active_tasks` is capped by `EngineTaskWorkerCount`. The value of `ipfs_bitswap_pending_tasks` is generally capped by the knobs below, however its exact maximum value is hard to predict as it depends on task sizes as well as number of requesting peers. However, as a rule of thumb, during healthy operation this value should oscillate around a "typical" low value (without hitting a plateau continuously). If `ipfs_bitswap_pending_tasks` is growing while `ipfs_bitswap_active_tasks` is at its maximum then the node has reached its resource limits and new requests are unable to be processed as quickly as they are coming in. Raising resource limits (using the knobs below) could help, assuming the hardware can support the new limits. The value of `ipfs_bitswap_active_block_tasks` is capped by `EngineBlockstoreWorkerCount`. The value of `ipfs_bitswap_pending_block_tasks` is indirectly capped by `ipfs_bitswap_active_tasks`, but can be hard to predict as it depends on the number of blocks involved in a peer task which can vary. If the value of `ipfs_bitswap_pending_block_tasks` is observed to grow, while `ipfs_bitswap_active_block_tasks` is at its maximum, there is indication that the number of available block tasks is creating a bottleneck (either due to high-latency block operations, or due to high number of block operations per bitswap peer task). In such cases, try increasing the `EngineBlockstoreWorkerCount`. If this adjustment still does not increase the throughput of the node, there might be hardware limitations like I/O or CPU. #### `Internal.Bitswap.TaskWorkerCount` Number of threads (goroutines) sending outgoing messages. Throttles the number of concurrent send operations. Type: `optionalInteger` (thread count, `null` means default which is 8) #### `Internal.Bitswap.EngineBlockstoreWorkerCount` Number of threads for blockstore operations. Used to throttle the number of concurrent requests to the block store. The optimal value can be informed by the metrics `ipfs_bitswap_pending_block_tasks` and `ipfs_bitswap_active_block_tasks`. This would be a number that depends on your hardware (I/O and CPU). Type: `optionalInteger` (thread count, `null` means default which is 128) #### `Internal.Bitswap.EngineTaskWorkerCount` Number of worker threads used for preparing and packaging responses before they are sent out. This number should generally be equal to `TaskWorkerCount`. Type: `optionalInteger` (thread count, `null` means default which is 8) #### `Internal.Bitswap.MaxOutstandingBytesPerPeer` Maximum number of bytes (across all tasks) pending to be processed and sent to any individual peer. This number controls fairness and can vary from 250Kb (very fair) to 10Mb (less fair, with more work dedicated to peers who ask for more). Values below 250Kb could cause thrashing. Values above 10Mb open the potential for aggressively-wanting peers to consume all resources and deteriorate the quality provided to less aggressively-wanting peers. Type: `optionalInteger` (byte count, `null` means default which is 1MB) #### `Internal.Bitswap.ProviderSearchDelay` This parameter determines how long to wait before looking for providers outside of bitswap. Other routing systems like the Amino DHT are able to provide results in less than a second, so lowering this number will allow faster peers lookups in some cases. Type: `optionalDuration` (`null` means default which is 1s) #### `Internal.Bitswap.ProviderSearchMaxResults` Maximum number of providers bitswap client should aim at before it stops searching for new ones. Setting to 0 means unlimited. Type: `optionalInteger` (`null` means default which is 10) #### `Internal.Bitswap.BroadcastControl` `Internal.Bitswap.BroadcastControl` contains settings for the bitswap client's broadcast control functionality. Broadcast control tries to reduce the number of bitswap broadcast messages sent to peers by choosing a subset of of the peers to send to. Peers are chosen based on whether they have previously responded indicating they have wanted blocks, as well as other configurable criteria. The settings here change how peers are selected as broadcast targets. Broadcast control can also be completely disabled to return bitswap to its previous behavior before broadcast control was introduced. Enabling broadcast control should generally reduce the number of broadcasts significantly without significantly degrading the ability to discover which peers have wanted blocks. However, if block discovery on your network relies sufficiently on broadcasts to discover peers that have wanted blocks, then adjusting the broadcast control configuration or disabling it altogether, may be helpful. ##### `Internal.Bitswap.BroadcastControl.Enable` Enables or disables broadcast control functionality. Setting this to `false` disables broadcast reduction logic and restores the previous (Kubo < 0.36) broadcast behavior of sending broadcasts to all peers. When disabled, all other `Bitswap.BroadcastControl` configuration items are ignored. Default: `true` (Enabled) Type: `flag` ##### `Internal.Bitswap.BroadcastControl.MaxPeers` Sets a hard limit on the number of peers to send broadcasts to. A value of `0` means no broadcasts are sent. A value of `-1` means there is no limit. Default: `0` (no limit) Type: `optionalInteger` (non-negative, 0 means no limit) ##### `Internal.Bitswap.BroadcastControl.LocalPeers` Enables or disables broadcast control for peers on the local network. Peers that have private or loopback addresses are considered to be on the local network. If this setting is `false`, than always broadcast to peers on the local network. If `true`, apply broadcast control to local peers. Default: `false` (Always broadcast to peers on local network) Type: `flag` ##### `Internal.Bitswap.BroadcastControl.PeeredPeers` Enables or disables broadcast reduction for peers configured for peering. If `false`, than always broadcast to peers configured for peering. If `true`, apply broadcast reduction to peered peers. Default: `false` (Always broadcast to peers configured for peering) Type: `flag` ##### `Internal.Bitswap.BroadcastControl.MaxRandomPeers` Sets the number of peers to broadcast to anyway, even though broadcast control logic has determined that they are not broadcast targets. Setting this to a non-zero value ensures at least this number of random peers receives a broadcast. This may be helpful in cases where peers that are not receiving broadcasts my have wanted blocks. Default: `0` (do not send broadcasts to peers not already targeted broadcast control) Type: `optionalInteger` (non-negative, 0 means do not broadcast to any random peers) ##### `Internal.Bitswap.BroadcastControl.SendToPendingPeers` Enables or disables sending broadcasts to any peers to which there is a pending message to send. When enabled, this sends broadcasts to many more peers, but does so in a way that does not increase the number of separate broadcast messages. There is still the increased cost of the recipients having to process and respond to the broadcasts. Default: `false` (Do not send broadcasts to all peers for which there are pending messages) Type: `flag` ### `Internal.UnixFSShardingSizeThreshold` **MOVED:** see [`Import.UnixFSHAMTDirectorySizeThreshold`](#importunixfshamtdirectorysizethreshold) ### `Internal.MFSNoFlushLimit` Controls the maximum number of consecutive MFS operations allowed with `--flush=false` before requiring a manual flush. This prevents unbounded memory growth and ensures data consistency when using deferred flushing with `ipfs files` commands. When the limit is reached, further operations will fail with an error message instructing the user to run `ipfs files flush`, use `--flush=true`, or increase this limit in the configuration. **Why operations fail instead of auto-flushing:** Automatic flushing once the limit is reached was considered but rejected because it can lead to data corruption issues that are difficult to debug. When the system decides to flush without user knowledge, it can: - Create partial states that violate user expectations about atomicity - Interfere with concurrent operations in unexpected ways - Make debugging and recovery much harder when issues occur By failing explicitly, users maintain control over when their data is persisted, allowing them to: - Batch related operations together before flushing - Handle errors predictably at natural transaction boundaries - Understand exactly when and why their data is written to disk If you expect automatic flushing behavior, simply use the default `--flush=true` (or omit the flag entirely) instead of `--flush=false`. **⚠️ WARNING:** Increasing this limit or disabling it (setting to 0) can lead to: - **Out-of-memory errors (OOM)** - Each unflushed operation consumes memory - **Data loss** - If the daemon crashes before flushing, all unflushed changes are lost - **Degraded performance** - Large unflushed caches slow down MFS operations Default: `256` Type: `optionalInteger` (0 disables the limit, strongly discouraged) **Note:** This is an EXPERIMENTAL feature and may change or be removed in future releases. See [#10842](https://github.com/ipfs/kubo/issues/10842) for more information. ## `Ipns` ### `Ipns.RepublishPeriod` A time duration specifying how frequently to republish ipns records to ensure they stay fresh on the network. Default: 4 hours. Type: `interval` or an empty string for the default. ### `Ipns.RecordLifetime` A time duration specifying the value to set on ipns records for their validity lifetime. Default: 48 hours. Type: `interval` or an empty string for the default. ### `Ipns.ResolveCacheSize` The number of entries to store in an LRU cache of resolved ipns entries. Entries will be kept cached until their lifetime is expired. Default: `128` Type: `integer` (non-negative, 0 means the default) ### `Ipns.MaxCacheTTL` Maximum duration for which entries are valid in the name system cache. Applied to everything under `/ipns/` namespace, allows you to cap the [Time-To-Live (TTL)](https://specs.ipfs.tech/ipns/ipns-record/#ttl-uint64) of [IPNS Records](https://specs.ipfs.tech/ipns/ipns-record/) AND also DNSLink TXT records (when DoH-specific [`DNS.MaxCacheTTL`](https://github.com/ipfs/kubo/blob/master/docs/config.md#dnsmaxcachettl) is not set to a lower value). When `Ipns.MaxCacheTTL` is set, it defines the upper bound limit of how long a [IPNS Name](https://specs.ipfs.tech/ipns/ipns-record/#ipns-name) lookup result will be cached and read from cache before checking for updates. **Examples:** - `"1m"` IPNS results are cached 1m or less (good compromise for system where faster updates are desired). - `"0s"` IPNS caching is effectively turned off (useful for testing, bad for production use) - **Note:** setting this to `0` will turn off TTL-based caching entirely. This is discouraged in production environments. It will make IPNS websites artificially slow because IPNS resolution results will expire as soon as they are retrieved, forcing expensive IPNS lookup to happen on every request. If you want near-real-time IPNS, set it to a low, but still sensible value, such as `1m`. Default: No upper bound, [TTL from IPNS Record](https://specs.ipfs.tech/ipns/ipns-record/#ttl-uint64) (see `ipns name publish --help`) is always respected. Type: `optionalDuration` ### `Ipns.UsePubsub` Enables [IPNS over PubSub](https://specs.ipfs.tech/ipns/ipns-pubsub-router/) for publishing and resolving IPNS records in real time. **EXPERIMENTAL:** read about current limitations at [experimental-features.md#ipns-pubsub](./experimental-features.md#ipns-pubsub). Default: `disabled` Type: `flag` ### `Ipns.DelegatedPublishers` HTTP endpoints for delegated IPNS publishing operations. These endpoints must support the [IPNS API](https://specs.ipfs.tech/routing/http-routing-v1/#ipns-api) from the Delegated Routing V1 HTTP specification. The special value `"auto"` loads delegated publishers from [AutoConf](#autoconf) when enabled. **Publishing behavior depends on routing configuration:** - `Routing.Type=auto` (default): Uses DHT for publishing, `"auto"` resolves to empty list - `Routing.Type=delegated`: Uses HTTP delegated publishers only, `"auto"` resolves to configured endpoints When using `"auto"`, inspect the effective publishers with: `ipfs config Ipns.DelegatedPublishers --expand-auto` **Command flags override publishing behavior:** - `--allow-offline` - Publishes to local datastore without requiring network connectivity - `--allow-delegated` - Uses local datastore and HTTP delegated publishers only (no DHT connectivity required) For self-hosting, you can run your own `/routing/v1/ipns` endpoint using [someguy](https://github.com/ipfs/someguy/). Default: `["auto"]` Type: `array[string]` (URLs or `"auto"`) ## `Migration` > [!WARNING] > **DEPRECATED:** Only applies to legacy migrations (repo versions <16). Modern repos (v16+) use embedded migrations. > This section is optional and will not appear in new configurations. ### `Migration.DownloadSources` **DEPRECATED:** Download sources for legacy migrations. Only `"HTTPS"` is supported. Type: `array[string]` (optional) Default: `["HTTPS"]` ### `Migration.Keep` **DEPRECATED:** Controls retention of legacy migration binaries. Options: `"cache"` (default), `"discard"`, `"keep"`. Type: `string` (optional) Default: `"cache"` ## `Mounts` > [!CAUTION] > **EXPERIMENTAL:** > This feature is disabled by default, requires an explicit opt-in with `ipfs mount` or `ipfs daemon --mount`. > > Read about current limitations at [fuse.md](./fuse.md). FUSE mount point configuration options. ### `Mounts.IPFS` Mountpoint for `/ipfs/`. Default: `/ipfs` Type: `string` (filesystem path) ### `Mounts.IPNS` Mountpoint for `/ipns/`. Default: `/ipns` Type: `string` (filesystem path) ### `Mounts.MFS` Mountpoint for Mutable File System (MFS) behind the `ipfs files` API. > [!CAUTION] > > - Write support is highly experimental and not recommended for mission-critical deployments. > - Avoid storing lazy-loaded datasets in MFS. Exposing a partially local, lazy-loaded DAG risks operating system search indexers crawling it, which may trigger unintended network prefetching of non-local DAG components. Default: `/mfs` Type: `string` (filesystem path) ### `Mounts.FuseAllowOther` Sets the 'FUSE allow-other' option on the mount point. ## `Pinning` Pinning configures the options available for pinning content (i.e. keeping content longer-term instead of as temporarily cached storage). ### `Pinning.RemoteServices` `RemoteServices` maps a name for a remote pinning service to its configuration. A remote pinning service is a remote service that exposes an API for managing that service's interest in long-term data storage. The exposed API conforms to the specification defined at #### `Pinning.RemoteServices: API` Contains information relevant to utilizing the remote pinning service Example: ```json { "Pinning": { "RemoteServices": { "myPinningService": { "API" : { "Endpoint" : "https://pinningservice.tld:1234/my/api/path", "Key" : "someOpaqueKey" } } } } } ``` ##### `Pinning.RemoteServices: API.Endpoint` The HTTP(S) endpoint through which to access the pinning service Example: "" Type: `string` ##### `Pinning.RemoteServices: API.Key` The key through which access to the pinning service is granted Type: `string` #### `Pinning.RemoteServices: Policies` Contains additional opt-in policies for the remote pinning service. ##### `Pinning.RemoteServices: Policies.MFS` When this policy is enabled, it follows changes to MFS and updates the pin for MFS root on the configured remote service. A pin request to the remote service is sent only when MFS root CID has changed and enough time has passed since the previous request (determined by `RepinInterval`). One can observe MFS pinning details by enabling debug via `ipfs log level remotepinning/mfs debug` and switching back to `error` when done. ###### `Pinning.RemoteServices: Policies.MFS.Enabled` Controls if this policy is active. Default: `false` Type: `bool` ###### `Pinning.RemoteServices: Policies.MFS.PinName` Optional name to use for a remote pin that represents the MFS root CID. When left empty, a default name will be generated. Default: `"policy/{PeerID}/mfs"`, e.g. `"policy/12.../mfs"` Type: `string` ###### `Pinning.RemoteServices: Policies.MFS.RepinInterval` Defines how often (at most) the pin request should be sent to the remote service. If left empty, the default interval will be used. Values lower than `1m` will be ignored. Default: `"5m"` Type: `duration` ## `Provide` Configures how your node advertises content to make it discoverable by other peers. **What is providing?** When your node stores content, it publishes provider records to the routing system announcing "I have this content". These records map CIDs to your peer ID, enabling content discovery across the network. While designed to support multiple routing systems in the future, the current default configuration only supports [providing to the Amino DHT](#providedht). ### `Provide.Enabled` Controls whether Kubo provide and reprovide systems are enabled. > [!CAUTION] > Disabling this will prevent other nodes from discovering your content. > Your node will stop announcing data to the routing system, making it > inaccessible unless peers connect to you directly. Default: `true` Type: `flag` ### `Provide.Strategy` Tells the provide system what should be announced. Valid strategies are: - `"all"` - announce all CIDs of stored blocks - `"pinned"` - only announce recursively pinned CIDs (`ipfs pin add -r`, both roots and child blocks) - Order: root blocks of direct and recursive pins are announced first, then the child blocks of recursive pins - `"roots"` - only announce the root block of explicitly pinned CIDs (`ipfs pin add`) - **⚠️ BE CAREFUL:** node with `roots` strategy will not announce child blocks. It makes sense only for use cases where the entire DAG is fetched in full, and a graceful resume does not have to be guaranteed: the lack of child announcements means an interrupted retrieval won't be able to find providers for the missing block in the middle of a file, unless the peer happens to already be connected to a provider and asks for child CID over bitswap. - `"mfs"` - announce only the local CIDs that are part of the MFS (`ipfs files`) - Note: MFS is lazy-loaded. Only the MFS blocks present in local datastore are announced. - `"pinned+mfs"` - a combination of the `pinned` and `mfs` strategies. - **ℹ️ NOTE:** This is the suggested strategy for users who run without GC and don't want to provide everything in cache. - Order: first `pinned` and then the locally available part of `mfs`. **Strategy changes automatically clear the provide queue.** When you change `Provide.Strategy` and restart Kubo, the provide queue is automatically cleared to ensure only content matching your new strategy is announced. You can also manually clear the queue using `ipfs provide clear`. **Memory requirements:** - Reproviding larger pinsets using the `mfs`, `pinned`, `pinned+mfs` or `roots` strategies requires additional memory, with an estimated ~1 GiB of RAM per 20 million CIDs for reproviding to the Amino DHT. - This is due to the use of a buffered provider, which loads all CIDs into memory to avoid holding a lock on the entire pinset during the reprovide cycle. Default: `"all"` Type: `optionalString` (unset for the default) ### `Provide.DHT` Configuration for providing data to Amino DHT peers. **Provider record lifecycle:** On the Amino DHT, provider records expire after [`amino.DefaultProvideValidity`](https://github.com/libp2p/go-libp2p-kad-dht/blob/v0.34.0/amino/defaults.go#L40-L43). Your node must re-announce (reprovide) content periodically to keep it discoverable. The [`Provide.DHT.Interval`](#providedhtinterval) setting controls this timing, with the default ensuring records refresh well before expiration or negative churn effects kick in. **Two provider systems:** - **Sweep provider**: Divides the DHT keyspace into regions and systematically sweeps through them over the reprovide interval. This batches CIDs allocated to the same DHT servers, dramatically reducing the number of DHT lookups and PUTs needed. Spreads work evenly over time with predictable resource usage. - **Legacy provider**: Processes each CID individually with separate DHT lookups. Works well for small content collections but struggles to complete reprovide cycles when managing thousands of CIDs. #### Monitoring Provide Operations **Quick command-line monitoring:** Use `ipfs provide stat` to view the current state of the provider system. For real-time monitoring, run `watch ipfs provide stat --all --compact` to see detailed statistics refreshed continuously in a 2-column layout. **Long-term monitoring:** For in-depth or long-term monitoring, metrics are exposed at the Prometheus endpoint: `{Addresses.API}/debug/metrics/prometheus` (default: `http://127.0.0.1:5001/debug/metrics/prometheus`). Different metrics are available depending on whether you use legacy mode (`SweepEnabled=false`) or sweep mode (`SweepEnabled=true`). See [Provide metrics documentation](https://github.com/ipfs/kubo/blob/master/docs/metrics.md#provide) for details. **Debug logging:** For troubleshooting, enable detailed logging by setting: ```sh GOLOG_LOG_LEVEL=error,provider=debug,dht/provider=debug ``` - `provider=debug` enables generic logging (legacy provider and any non-dht operations) - `dht/provider=debug` enables logging for the sweep provider #### `Provide.DHT.Interval` Sets how often to re-announce content to the DHT. Provider records on Amino DHT expire after [`amino.DefaultProvideValidity`](https://github.com/libp2p/go-libp2p-kad-dht/blob/v0.34.0/amino/defaults.go#L40-L43). **Why this matters:** The interval must be shorter than the expiration window to ensure provider records refresh before they expire. The default value is approximately half of [`amino.DefaultProvideValidity`](https://github.com/libp2p/go-libp2p-kad-dht/blob/v0.34.0/amino/defaults.go#L40-L43), which accounts for network churn and ensures records stay alive without overwhelming the network with unnecessary announcements. **With sweep mode enabled ([`Provide.DHT.SweepEnabled`](#providedhtsweepenabled)):** The system spreads reprovide operations smoothly across this entire interval. Each keyspace region is reprovided at scheduled times throughout the period, ensuring each region's announcements complete before records expire. **With legacy mode:** The system attempts to reprovide all CIDs as quickly as possible at the start of each interval. If reproviding takes longer than this interval (common with large datasets), the next cycle is skipped and provider records may expire. - If unset, it uses the implicit safe default. - If set to the value `"0"` it will disable content reproviding to DHT. > [!CAUTION] > Disabling this will prevent other nodes from discovering your content via the DHT. > Your node will stop announcing data to the DHT, making it > inaccessible unless peers connect to you directly. Since provider > records expire after `amino.DefaultProvideValidity`, your content will become undiscoverable > after this period. Default: `22h` Type: `optionalDuration` (unset for the default) #### `Provide.DHT.MaxWorkers` Sets the maximum number of _concurrent_ DHT provide operations. **When `Provide.DHT.SweepEnabled` is false (legacy mode):** - Controls NEW CID announcements only - Reprovide operations do **not** count against this limit - A value of `0` allows unlimited provide workers **When `Provide.DHT.SweepEnabled` is true:** - Controls the total worker pool for both provide and reprovide operations - Workers are split between periodic reprovides and burst provides - Use a positive value to control resource usage - See [`DedicatedPeriodicWorkers`](#providedhtdedicatedperiodicworkers) and [`DedicatedBurstWorkers`](#providedhtdedicatedburstworkers) for task allocation If the [accelerated DHT client](#routingaccelerateddhtclient) is enabled, each provide operation opens ~20 connections in parallel. With the standard DHT client (accelerated disabled), each provide opens between 20 and 60 connections, with at most 10 active at once. Provides complete more quickly when using the accelerated client. Be mindful of how many simultaneous connections this setting can generate. > [!CAUTION] > For nodes without strict connection limits that need to provide large volumes > of content, we recommend first trying `Provide.DHT.SweepEnabled=true` for efficient > announcements. If announcements are still not fast enough, adjust `Provide.DHT.MaxWorkers`. > As a last resort, consider enabling `Routing.AcceleratedDHTClient=true` but be aware that it is very resource hungry. > > At the same time, mind that raising this value too high may lead to increased load. > Proceed with caution, ensure proper hardware and networking are in place. > [!TIP] > **When `SweepEnabled` is true:** Users providing millions of CIDs or more > should increase the worker count accordingly. Underprovisioning can lead to > slow provides (burst workers) and inability to keep up with content > reproviding (periodic workers). For nodes with sufficient resources (CPU, > bandwidth, number of connections), dedicating `1024` for [periodic > workers](#providedhtdedicatedperiodicworkers) and `512` for [burst > workers](#providedhtdedicatedburstworkers), and `2048` [max > workers](#providedhtmaxworkers) should be adequate even for the largest > users. The system will only use workers as needed - unused resources won't be > consumed. Ensure you adjust the swarm [connection manager](#swarmconnmgr) and > [resource manager](#swarmresourcemgr) configuration accordingly. > See [Capacity Planning](https://github.com/ipfs/kubo/blob/master/docs/provide-stats.md#capacity-planning) for more details. Default: `16` Type: `optionalInteger` (non-negative; `0` means unlimited number of workers) #### `Provide.DHT.SweepEnabled` Enables the sweep provider for efficient content announcements. When disabled, the legacy [`boxo/provider`](https://github.com/ipfs/boxo/tree/main/provider) is used instead. **The legacy provider problem:** The legacy system processes CIDs one at a time, requiring a separate DHT lookup (10-20 seconds each) to find the 20 closest peers for each CID. This sequential approach typically handles less than 10,000 CID over 22h ([`Provide.DHT.Interval`](#providedhtinterval)). If your node has more CIDs than can be reprovided within [`Provide.DHT.Interval`](#providedhtinterval), provider records start expiring after [`amino.DefaultProvideValidity`](https://github.com/libp2p/go-libp2p-kad-dht/blob/v0.34.0/amino/defaults.go#L40-L43), making content undiscoverable. **How sweep mode works:** The sweep provider divides the DHT keyspace into regions based on keyspace prefixes. It estimates the Amino DHT size, calculates how many regions are needed (sized to contain at least 20 peers each), then schedules region processing evenly across [`Provide.DHT.Interval`](#providedhtinterval). When processing a region, it discovers the peers in that region once, then sends all provider records for CIDs allocated to those peers in a batch. This batching is the key efficiency: instead of N lookups for N CIDs, the number of lookups is bounded by a constant fraction of the Amino DHT size (e.g., ~3,000 lookups when there are ~10,000 DHT servers), regardless of how many CIDs you're providing. **Efficiency gains:** For a node providing 100,000 CIDs, sweep mode reduces lookups by 97% compared to legacy. The work spreads smoothly over time rather than completing in bursts, preventing resource spikes and duplicate announcements. Long-running nodes reprovide systematically just before records would expire, keeping content continuously discoverable without wasting bandwidth. **Implementation details:** The sweep provider tracks CIDs in a persistent keystore. New content added via `StartProviding()` enters the provide queue and gets batched by keyspace region. The keystore is periodically refreshed at each [`Provide.DHT.Interval`](#providedhtinterval) with CIDs matching [`Provide.Strategy`](#providestrategy) to ensure only current content remains scheduled. This handles cases where content is unpinned or removed. **Persistent reprovide cycle state:** When Provide Sweep is enabled, the reprovide cycle state is persisted to the datastore by default. On restart, Kubo automatically resumes from where it left off. If the node was offline for an extended period, all CIDs that haven't been reprovided within the configured [`Provide.DHT.Interval`](#providedhtinterval) are immediately queued for reproviding. Additionally, the provide queue is persisted on shutdown and restored on startup, ensuring no pending provide operations are lost. If you don't want to keep the persisted provider state from a previous run, you can disable this behavior by setting [`Provide.DHT.ResumeEnabled`](#providedhtresumeenabled) to `false`. > > > > Reprovide Cycle Comparison > > > The diagram compares performance patterns: > > - **Legacy mode**: Sequential processing, one lookup per CID, struggles with large datasets > - **Sweep mode**: Smooth distribution over time, batched lookups by keyspace region, predictable resource usage > - **Accelerated DHT**: Hourly network crawls creating traffic spikes, high resource usage > > Sweep mode achieves similar effectiveness to the Accelerated DHT client but with steady resource consumption. For background on the sweep provider design and motivations, see Shipyard's blogpost [Provide Sweep: Solving the DHT Provide Bottleneck](https://ipshipyard.com/blog/2025-dht-provide-sweep/). You can compare the effectiveness of sweep mode vs legacy mode by monitoring the appropriate metrics (see [Monitoring Provide Operations](#monitoring-provide-operations) above). > [!NOTE] > This is the default provider system as of Kubo v0.39. To use the legacy provider instead, set `Provide.DHT.SweepEnabled=false`. > [!NOTE] > When DHT routing is unavailable (e.g., `Routing.Type=custom` with only HTTP routers), the provider automatically falls back to the legacy provider regardless of this setting. Default: `true` Type: `flag` #### `Provide.DHT.ResumeEnabled` Controls whether the provider resumes from its previous state on restart. Only applies when `Provide.DHT.SweepEnabled` is true. When enabled (the default), the provider persists its reprovide cycle state and provide queue to the datastore, and restores them on restart. This ensures: - The reprovide cycle continues from where it left off instead of starting over - Any CIDs in the provide queue during shutdown are restored and provided after restart - CIDs that missed their reprovide window while the node was offline are queued for immediate reproviding When disabled, the provider starts fresh on each restart, discarding any previous reprovide cycle state and provide queue. On a fresh start, all CIDs matching the [`Provide.Strategy`](#providestrategy) will be provided ASAP (as burst provides), and then keyspace regions are reprovided according to the regular schedule starting from the beginning of the reprovide cycle. > [!NOTE] > Disabling this option means the provider will provide all content matching > your strategy on every restart (which can be resource-intensive for large > datasets), then start from the beginning of the reprovide cycle. For nodes > with large datasets or frequent restarts, keeping this enabled (the default) > is recommended for better resource efficiency and more consistent reproviding > behavior. Default: `true` Type: `flag` #### `Provide.DHT.DedicatedPeriodicWorkers` Number of workers dedicated to periodic keyspace region reprovides. Only applies when `Provide.DHT.SweepEnabled` is true. Among the [`Provide.DHT.MaxWorkers`](#providedhtmaxworkers), this number of workers will be dedicated to the periodic region reprovide only. The sum of `DedicatedPeriodicWorkers` and `DedicatedBurstWorkers` should not exceed `MaxWorkers`. Any remaining workers (MaxWorkers - DedicatedPeriodicWorkers - DedicatedBurstWorkers) form a shared pool that can be used for either type of work as needed. > [!NOTE] > If the provider system isn't able to keep up with reproviding all your > content within the [Provide.DHT.Interval](#providedhtinterval), consider > increasing this value. Default: `2` Type: `optionalInteger` (`0` means there are no dedicated workers, but the operation can be performed by free non-dedicated workers) #### `Provide.DHT.DedicatedBurstWorkers` Number of workers dedicated to burst provides. Only applies when `Provide.DHT.SweepEnabled` is true. Burst provides are triggered by: - Manual provide commands (`ipfs routing provide`) - New content matching your `Provide.Strategy` (blocks from `ipfs add`, bitswap, or trustless gateway requests) - Catch-up reprovides after being disconnected/offline for a while Having dedicated burst workers ensures that bulk operations (like adding many CIDs or reconnecting to the network) don't delay regular periodic reprovides, and vice versa. Among the [`Provide.DHT.MaxWorkers`](#providedhtmaxworkers), this number of workers will be dedicated to burst provides only. In addition to these, if there are available workers in the pool, they can also be used for burst provides. > [!NOTE] > If CIDs aren't provided quickly enough to your taste, and you can afford more > CPU and bandwidth, consider increasing this value. Default: `1` Type: `optionalInteger` (`0` means there are no dedicated workers, but the operation can be performed by free non-dedicated workers) #### `Provide.DHT.MaxProvideConnsPerWorker` Maximum number of connections that a single worker can use to send provider records over the network. When reproviding CIDs corresponding to a keyspace region, the reprovider must send a provider record to the 20 closest peers to the CID (in XOR distance) for each CID belonging to this keyspace region. The reprovider opens a connection to a peer from that region, sends it all its allocated provider records. Once done, it opens a connection to the next peer from that keyspace region until all provider records are assigned. This option defines how many such connections can be open concurrently by a single worker. > [!NOTE] > Increasing this value can speed up the provide operation, at the cost of > opening more simultaneous connections to DHT servers. A keyspace typically > has less than 60 peers, so you may hit a performance ceiling beyond which > increasing this value has no effect. Default: `20` Type: `optionalInteger` (non-negative) #### `Provide.DHT.KeystoreBatchSize` During the garbage collection, all keys stored in the Keystore are removed, and the keys are streamed from a channel to fill the Keystore again with up-to-date keys. Since a high number of CIDs to reprovide can easily fill up the memory, keys are read and written in batches to optimize for memory usage. This option defines how many multihashes should be contained within a batch. A multihash is usually represented by 34 bytes. Default: `16384` (~544 KiB per batch) Type: `optionalInteger` (non-negative) #### `Provide.DHT.OfflineDelay` The `SweepingProvider` has 3 states: `ONLINE`, `DISCONNECTED` and `OFFLINE`. It starts `OFFLINE`, and as the node bootstraps, it changes its state to `ONLINE`. When the provider loses connection to all DHT peers, it switches to the `DISCONNECTED` state. In this state, new provides will be added to the provide queue, and provided as soon as the node comes back online. After a node has been `DISCONNECTED` for `OfflineDelay`, it goes to `OFFLINE` state. When `OFFLINE`, the provider drops the provide queue, and returns errors to new provide requests. However, when `OFFLINE` the provider still adds the keys to its state, so keys will eventually be provided in the [`Provide.DHT.Interval`](#providedhtinterval) after the provider comes back `ONLINE`. Default: `2h` Type: `optionalDuration` ## `Provider` ### `Provider.Enabled` **REMOVED** Replaced with [`Provide.Enabled`](#provideenabled). ### `Provider.Strategy` **REMOVED** This field was unused. Use [`Provide.Strategy`](#providestrategy) instead. ### `Provider.WorkerCount` **REMOVED** Replaced with [`Provide.DHT.MaxWorkers`](#providedhtmaxworkers). ## `Pubsub` Pubsub configures Kubo's opt-in, opinionated [libp2p pubsub](https://web.archive.org/web/20260116065034/https://docs.libp2p.io/concepts/pubsub/overview/) instance. To enable, set `Pubsub.Enabled` to `true`. **EXPERIMENTAL:** This is an opt-in feature. Its primary use case is [IPNS over PubSub](https://specs.ipfs.tech/ipns/ipns-pubsub-router/), which enables real-time IPNS record propagation. See [`Ipns.UsePubsub`](#ipnsusepubsub) for details. The `ipfs pubsub` commands can also be used for basic publish/subscribe operations, but only if Kubo's built-in message validation (described below) is acceptable for your use case. ### When to use a dedicated pubsub node Kubo's pubsub is optimized for IPNS. It uses opinionated message validation that may not fit all applications. If you need custom Message ID computation, different deduplication logic, or validation rules beyond what Kubo provides, consider building a dedicated pubsub node using [go-libp2p-pubsub](https://github.com/libp2p/go-libp2p-pubsub) directly. ### Message deduplication Kubo uses two layers of message deduplication to handle duplicate messages that may arrive via different network paths: **Layer 1: In-memory TimeCache (Message ID)** When a message arrives, Kubo computes its Message ID (hash of the message content) and checks an in-memory cache. If the ID was seen recently, the message is dropped. This cache is controlled by: - [`Pubsub.SeenMessagesTTL`](#pubsubseenmessagesttl) - how long Message IDs are remembered (default: 120s) - [`Pubsub.SeenMessagesStrategy`](#pubsubseenmessagesstrategy) - whether TTL resets on each sighting This cache is fast but limited: it only works within the TTL window and is cleared on node restart. **Layer 2: Persistent Seqno Validator (per-peer)** For stronger deduplication, Kubo tracks the maximum sequence number seen from each peer and persists it to the datastore. Messages with sequence numbers lower than the recorded maximum are rejected. This prevents replay attacks and handles message cycles in large networks where messages may take longer than the TimeCache TTL to propagate. This layer survives node restarts. The state can be inspected or cleared using `ipfs pubsub reset` (for testing/recovery only). ### `Pubsub.Enabled` Enables the pubsub system. Default: `false` Type: `flag` ### `Pubsub.Router` Sets the default router used by pubsub to route messages to peers. This can be one of: - `"floodsub"` - floodsub is a basic router that simply _floods_ messages to all connected peers. This router is extremely inefficient but _very_ reliable. - `"gossipsub"` - [gossipsub][] is a more advanced routing algorithm that will build an overlay mesh from a subset of the links in the network. Default: `"gossipsub"` Type: `string` (one of `"floodsub"`, `"gossipsub"`, or `""` (apply default)) [gossipsub]: https://github.com/libp2p/specs/tree/master/pubsub/gossipsub ### `Pubsub.DisableSigning` Disables message signing and signature verification. **FOR TESTING ONLY - DO NOT USE IN PRODUCTION** It is _not_ safe to disable signing even if you don't care _who_ sent the message because spoofed messages can be used to silence real messages by intentionally re-using the real message's message ID. Default: `false` Type: `bool` ### `Pubsub.SeenMessagesTTL` Controls the time window for the in-memory Message ID cache (Layer 1 deduplication). Messages with the same ID seen within this window are dropped. A smaller value reduces memory usage but may cause more duplicates in networks with slow nodes. A larger value uses more memory but provides better duplicate detection within the time window. Default: see `TimeCacheDuration` from [go-libp2p-pubsub](https://github.com/libp2p/go-libp2p-pubsub) Type: `optionalDuration` ### `Pubsub.SeenMessagesStrategy` Determines how the TTL countdown for the Message ID cache works. - `last-seen` - Sliding window: TTL resets each time the message is seen again. Keeps frequently-seen messages in cache longer, preventing continued propagation. - `first-seen` - Fixed window: TTL counts from first sighting only. Messages are purged after the TTL regardless of how many times they're seen. Default: `last-seen` (see [go-libp2p-pubsub](https://github.com/libp2p/go-libp2p-pubsub)) Type: `optionalString` ## `Peering` Configures the peering subsystem. The peering subsystem configures Kubo to connect to, remain connected to, and reconnect to a set of nodes. Nodes should use this subsystem to create "sticky" links between frequently useful peers to improve reliability. Use-cases: - An IPFS gateway connected to an IPFS cluster should peer to ensure that the gateway can always fetch content from the cluster. - A dapp may peer embedded Kubo nodes with a set of pinning services or textile cafes/hubs. - A set of friends may peer to ensure that they can always fetch each other's content. When a node is added to the set of peered nodes, Kubo will: 1. Protect connections to this node from the connection manager. That is, Kubo will never automatically close the connection to this node and connections to this node will not count towards the connection limit. 2. Connect to this node on startup. 3. Repeatedly try to reconnect to this node if the last connection dies or the node goes offline. This repeated re-connect logic is governed by a randomized exponential backoff delay ranging from ~5 seconds to ~10 minutes to avoid repeatedly reconnect to a node that's offline. Peering can be asymmetric or symmetric: - When symmetric, the connection will be protected by both nodes and will likely be very stable. - When asymmetric, only one node (the node that configured peering) will protect the connection and attempt to re-connect to the peered node on disconnect. If the peered node is under heavy load and/or has a low connection limit, the connection may flap repeatedly. Be careful when asymmetrically peering to not overload peers. ### `Peering.Peers` The set of peers with which to peer. ```json { "Peering": { "Peers": [ { "ID": "QmPeerID1", "Addrs": ["/ip4/18.1.1.1/tcp/4001"] }, { "ID": "QmPeerID2", "Addrs": ["/ip4/18.1.1.2/tcp/4001", "/ip4/18.1.1.2/udp/4001/quic-v1"] } ] } ... } ``` Where `ID` is the peer ID and `Addrs` is a set of known addresses for the peer. If no addresses are specified, the Amino DHT will be queried. Additional fields may be added in the future. Default: empty. Type: `array[peering]` ## `Reprovider` ### `Reprovider.Interval` **REMOVED** Replaced with [`Provide.DHT.Interval`](#providedhtinterval). ### `Reprovider.Strategy` **REMOVED** Replaced with [`Provide.Strategy`](#providestrategy). ## `Routing` Contains options for content, peer, and IPNS routing mechanisms. ### `Routing.Type` Controls how your node discovers content and peers on the network. **Production options:** - **`auto`** (default): Uses both the public IPFS DHT (Amino) and HTTP routers from [`Routing.DelegatedRouters`](#routingdelegatedrouters) for faster lookups. Your node starts as a DHT client and automatically switches to server mode when reachable from the public internet. - **`autoclient`**: Same as `auto`, but never runs a DHT server. Use this if your node is behind a firewall or NAT. - **`dht`**: Uses only the Amino DHT (no HTTP routers). Automatically switches between client and server mode based on reachability. - **`dhtclient`**: DHT-only, always running as a client. Lower resource usage. - **`dhtserver`**: DHT-only, always running as a server. Only use this if your node is reachable from the public internet. - **`none`**: Disables all routing. You must manually connect to peers. **About DHT client vs server mode:** When the DHT is enabled, your node can operate as either a client or server. In server mode, it queries other peers and responds to their queries - this helps the network but uses more resources. In client mode, it only queries others without responding, which is less resource-intensive. With `auto` or `dht`, your node starts as a client and switches to server when it detects public reachability. > [!CAUTION] > **`Routing.Type` Experimental options:** > > These modes are for research and testing only, not production use. > They may change without notice between releases. > > - **`delegated`**: Uses only HTTP routers from [`Routing.DelegatedRouters`](#routingdelegatedrouters) > and IPNS publishers from [`Ipns.DelegatedPublishers`](#ipnsdelegatedpublishers), > without initializing the DHT. Useful when peer-to-peer connectivity is unavailable. > Note: cannot provide content to the network (no DHT means no provider records). > > - **`custom`**: Disables all default routers. You define your own routing in > [`Routing.Routers`](#routingrouters). See [delegated-routing.md](delegated-routing.md). Default: `auto` Type: `optionalString` (`null`/missing means the default) ### `Routing.DelegatedRouters` An array of URL hostnames for delegated routers to be queried in addition to the Amino DHT when `Routing.Type` is set to `auto` (default) or `autoclient`. These endpoints must support the [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/). The special value `"auto"` uses delegated routers from [AutoConf](#autoconf) when enabled. You can combine `"auto"` with custom URLs (e.g., `["auto", "https://custom.example.com"]`) to query both the default delegated routers and your own endpoints. The first `"auto"` entry gets substituted with autoconf values, and other URLs are preserved. > [!TIP] > Delegated routing allows IPFS implementations to offload tasks like content routing, peer routing, and naming to a separate process or server while also benefiting from HTTP caching. > > One can run their own delegated router either by implementing the [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) themselves, or by using [Someguy](https://github.com/ipfs/someguy), a turn-key implementation that proxies requests to other routing systems. A public utility instance of Someguy is hosted at [`https://delegated-ipfs.dev`](https://docs.ipfs.tech/concepts/public-utilities/#delegated-routing). Default: `["auto"]` Type: `array[string]` (URLs or `"auto"`) ### `Routing.AcceleratedDHTClient` This alternative Amino DHT client with a Full-Routing-Table strategy will do a complete scan of the DHT every hour and record all nodes found. Then when a lookup is tried instead of having to go through multiple Kad hops it is able to find the 20 final nodes by looking up the in-memory recorded network table. This means sustained higher memory to store the routing table and extra CPU and network bandwidth for each network scan. However the latency of individual read/write operations should be ~10x faster and provide throughput up to 6 million times faster on larger datasets! This is not compatible with `Routing.Type` `custom`. If you are using composable routers you can configure this individually on each router. When it is enabled: - Client DHT operations (reads and writes) should complete much faster - The provider will now use a keyspace sweeping mode allowing to keep alive CID sets that are multiple orders of magnitude larger. - **Note:** For improved provide/reprovide operations specifically, consider using [`Provide.DHT.SweepEnabled`](#providedhtsweepenabled) instead, which offers similar benefits without the hourly traffic spikes. - The standard Bucket-Routing-Table DHT will still run for the DHT server (if the DHT server is enabled). This means the classical routing table will still be used to answer other nodes. This is critical to maintain to not harm the network. - The operations `ipfs stats dht` will default to showing information about the accelerated DHT client > [!CAUTION] > **`Routing.AcceleratedDHTClient` Caveats:** > > 1. Running the accelerated client likely will result in more resource consumption (connections, RAM, CPU, bandwidth) > - Users that are limited in the number of parallel connections their machines/networks can perform will be most affected > - The resource usage is not smooth as the client crawls the network in rounds and reproviding is similarly done in rounds > - Users who previously had a lot of content but were unable to advertise it on the network will see an increase in > egress bandwidth as their nodes start to advertise all of their CIDs into the network. If you have lots of data > entering your node that you don't want to advertise, consider using [`Provide.*`](#provide) configuration > to control which CIDs are reprovided. > 2. Currently, the DHT is not usable for queries for the first 5-10 minutes of operation as the routing table is being > prepared. This means operations like searching the DHT for particular peers or content will not work initially. > - You can see if the DHT has been initially populated by running `ipfs stats dht` > 3. Currently, the accelerated DHT client is not compatible with LAN-based DHTs and will not perform operations against > them. Default: `false` Type: `flag` ### `Routing.LoopbackAddressesOnLanDHT` **EXPERIMENTAL: `Routing.LoopbackAddressesOnLanDHT` configuration may change in future release** Whether loopback addresses (e.g. 127.0.0.1) should not be ignored on the local LAN DHT. Most users do not need this setting. It can be useful during testing, when multiple Kubo nodes run on the same machine but some of them do not have `Discovery.MDNS.Enabled`. Default: `false` Type: `bool` (missing means `false`) ### `Routing.IgnoreProviders` An array of [string-encoded PeerIDs](https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#string-representation). Any provider record associated to one of these peer IDs is ignored. Apart from ignoring specific providers for reasons like misbehaviour etc. this setting is useful to ignore providers as a way to indicate preference, when the same provider is found under different peerIDs (i.e. one for HTTP and one for Bitswap retrieval). > [!TIP] > This denylist operates on PeerIDs. > To deny specific HTTP Provider URL, use [`HTTPRetrieval.Denylist`](#httpretrievaldenylist) instead. Default: `[]` Type: `array[string]` ### `Routing.Routers` Alternative configuration used when `Routing.Type=custom`. > [!CAUTION] > **EXPERIMENTAL: `Routing.Routers` is for research and testing only, not production use.** > > - The configuration format and behavior may change without notice between releases. > - Bugs and regressions may not be prioritized. > - HTTP-only configurations cannot reliably provide content. See [delegated-routing.md](delegated-routing.md#limitations). > > Most users should use `Routing.Type=auto` or `autoclient` with [`Routing.DelegatedRouters`](#routingdelegatedrouters). Allows for replacing the default routing (Amino DHT) with alternative Router implementations. The map key is a name of a Router, and the value is its configuration. Default: `{}` Type: `object[string->object]` #### `Routing.Routers.[name].Type` **⚠️ EXPERIMENTAL: For research and testing only. May change without notice.** It specifies the routing type that will be created. Currently supported types: - `http` simple delegated routing based on HTTP protocol from [IPIP-337](https://specs.ipfs.tech/ipips/ipip-0337/) - `dht` provides decentralized routing based on [libp2p's kad-dht](https://github.com/libp2p/specs/tree/master/kad-dht) - `parallel` and `sequential`: Helpers that can be used to run several routers sequentially or in parallel. Type: `string` #### `Routing.Routers.[name].Parameters` **⚠️ EXPERIMENTAL: For research and testing only. May change without notice.** Parameters needed to create the specified router. Supported params per router type: HTTP: - `Endpoint` (mandatory): URL that will be used to connect to a specified router. - `MaxProvideBatchSize`: This number determines the maximum amount of CIDs sent per batch. Servers might not accept more than 100 elements per batch. 100 elements by default. - `MaxProvideConcurrency`: It determines the number of threads used when providing content. GOMAXPROCS by default. DHT: - `"Mode"`: Mode used by the Amino DHT. Possible values: "server", "client", "auto" - `"AcceleratedDHTClient"`: Set to `true` if you want to use the acceleratedDHT. - `"PublicIPNetwork"`: Set to `true` to create a `WAN` DHT. Set to `false` to create a `LAN` DHT. Parallel: - `Routers`: A list of routers that will be executed in parallel: - `Name:string`: Name of the router. It should be one of the previously added to `Routers` list. - `Timeout:duration`: Local timeout. It accepts strings compatible with Go `time.ParseDuration(string)` (`10s`, `1m`, `2h`). Time will start counting when this specific router is called, and it will stop when the router returns, or we reach the specified timeout. - `ExecuteAfter:duration`: Providing this param will delay the execution of that router at the specified time. It accepts strings compatible with Go `time.ParseDuration(string)` (`10s`, `1m`, `2h`). - `IgnoreErrors:bool`: It will specify if that router should be ignored if an error occurred. - `Timeout:duration`: Global timeout. It accepts strings compatible with Go `time.ParseDuration(string)` (`10s`, `1m`, `2h`). Sequential: - `Routers`: A list of routers that will be executed in order: - `Name:string`: Name of the router. It should be one of the previously added to `Routers` list. - `Timeout:duration`: Local timeout. It accepts strings compatible with Go `time.ParseDuration(string)`. Time will start counting when this specific router is called, and it will stop when the router returns, or we reach the specified timeout. - `IgnoreErrors:bool`: It will specify if that router should be ignored if an error occurred. - `Timeout:duration`: Global timeout. It accepts strings compatible with Go `time.ParseDuration(string)`. Default: `{}` (use the safe implicit defaults) Type: `object[string->string]` ### `Routing.Methods` `Methods:map` will define which routers will be executed per method used when `Routing.Type=custom`. > [!CAUTION] > **EXPERIMENTAL: `Routing.Methods` is for research and testing only, not production use.** > > - The configuration format and behavior may change without notice between releases. > - Bugs and regressions may not be prioritized. > - HTTP-only configurations cannot reliably provide content. See [delegated-routing.md](delegated-routing.md#limitations). > > Most users should use `Routing.Type=auto` or `autoclient` with [`Routing.DelegatedRouters`](#routingdelegatedrouters). The key will be the name of the method: `"provide"`, `"find-providers"`, `"find-peers"`, `"put-ipns"`, `"get-ipns"`. All methods must be added to the list. The value will contain: - `RouterName:string`: Name of the router. It should be one of the previously added to `Routing.Routers` list. Type: `object[string->object]` **Examples:** Complete example using 2 Routers, Amino DHT (LAN/WAN) and parallel. ``` $ ipfs config Routing.Type --json '"custom"' $ ipfs config Routing.Routers.WanDHT --json '{ "Type": "dht", "Parameters": { "Mode": "auto", "PublicIPNetwork": true, "AcceleratedDHTClient": false } }' $ ipfs config Routing.Routers.LanDHT --json '{ "Type": "dht", "Parameters": { "Mode": "auto", "PublicIPNetwork": false, "AcceleratedDHTClient": false } }' $ ipfs config Routing.Routers.ParallelHelper --json '{ "Type": "parallel", "Parameters": { "Routers": [ { "RouterName" : "LanDHT", "IgnoreErrors" : true, "Timeout": "3s" }, { "RouterName" : "WanDHT", "IgnoreErrors" : false, "Timeout": "5m", "ExecuteAfter": "2s" } ] } }' ipfs config Routing.Methods --json '{ "find-peers": { "RouterName": "ParallelHelper" }, "find-providers": { "RouterName": "ParallelHelper" }, "get-ipns": { "RouterName": "ParallelHelper" }, "provide": { "RouterName": "ParallelHelper" }, "put-ipns": { "RouterName": "ParallelHelper" } }' ``` ## `Swarm` Options for configuring the swarm. ### `Swarm.AddrFilters` An array of addresses (multiaddr netmasks) to not dial. By default, IPFS nodes advertise _all_ addresses, even internal ones. This makes it easier for nodes on the same network to reach each other. Unfortunately, this means that an IPFS node will try to connect to one or more private IP addresses whenever dialing another node, even if this other node is on a different network. This may trigger netscan alerts on some hosting providers or cause strain in some setups. > [!TIP] > The [`server` configuration profile](#server-profile) fills up this list with sensible defaults, > preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`, > which is the [multiaddress][multiaddr] representation of `192.168.0.0/16`) but you should always > check settings against your own network and/or hosting provider. Default: `[]` Type: `array[string]` ### `Swarm.DisableBandwidthMetrics` A boolean value that when set to true, will cause ipfs to not keep track of bandwidth metrics. Disabling bandwidth metrics can lead to a slight performance improvement, as well as a reduction in memory usage. Default: `false` Type: `bool` ### `Swarm.DisableNatPortMap` Disable automatic NAT port forwarding (turn off [UPnP](https://en.wikipedia.org/wiki/Universal_Plug_and_Play)). When not disabled (default), Kubo asks NAT devices (e.g., routers), to open up an external port and forward it to the port Kubo is running on. When this works (i.e., when your router supports NAT port forwarding), it makes the local Kubo node accessible from the public internet. Default: `false` Type: `bool` ### `Swarm.EnableHolePunching` Enable hole punching for NAT traversal when port forwarding is not possible. When enabled, Kubo will coordinate with the counterparty using a [relayed connection](https://github.com/libp2p/specs/blob/master/relay/circuit-v2.md), to [upgrade to a direct connection](https://github.com/libp2p/specs/blob/master/relay/DCUtR.md) through a NAT/firewall whenever possible. This feature requires `Swarm.RelayClient.Enabled` to be set to `true`. Default: `true` Type: `flag` ### `Swarm.EnableAutoRelay` **REMOVED** See `Swarm.RelayClient` instead. ### `Swarm.RelayClient` Configuration options for the relay client to use relay services. Default: `{}` Type: `object` #### `Swarm.RelayClient.Enabled` Enables "automatic relay user" mode for this node. Your node will automatically _use_ public relays from the network if it detects that it cannot be reached from the public internet (e.g., it's behind a firewall) and get a `/p2p-circuit` address from a public relay. Default: `true` Type: `flag` #### `Swarm.RelayClient.StaticRelays` Your node will use these statically configured relay servers instead of discovering public relays ([Circuit Relay v2](https://github.com/libp2p/specs/blob/master/relay/circuit-v2.md)) from the network. Default: `[]` Type: `array[string]` ### `Swarm.RelayService` Configuration options for the relay service that can be provided to _other_ peers on the network ([Circuit Relay v2](https://github.com/libp2p/specs/blob/master/relay/circuit-v2.md)). Default: `{}` Type: `object` #### `Swarm.RelayService.Enabled` Enables providing `/p2p-circuit` v2 relay service to other peers on the network. NOTE: This is the service/server part of the relay system. Disabling this will prevent this node from running as a relay server. Use [`Swarm.RelayClient.Enabled`](#swarmrelayclientenabled) for turning your node into a relay user. Default: `true` Type: `flag` #### `Swarm.RelayService.Limit` Limits are applied to every relayed connection. Default: `{}` Type: `object[string -> string]` ##### `Swarm.RelayService.ConnectionDurationLimit` Time limit before a relayed connection is reset. Default: `"2m"` Type: `duration` ##### `Swarm.RelayService.ConnectionDataLimit` Limit of data relayed (in each direction) before a relayed connection is reset. Default: `131072` (128 kb) Type: `optionalInteger` #### `Swarm.RelayService.ReservationTTL` Duration of a new or refreshed reservation. Default: `"1h"` Type: `duration` #### `Swarm.RelayService.MaxReservations` Maximum number of active relay slots. Default: `128` Type: `optionalInteger` #### `Swarm.RelayService.MaxCircuits` Maximum number of open relay connections for each peer. Default: `16` Type: `optionalInteger` #### `Swarm.RelayService.BufferSize` Size of the relayed connection buffers. Default: `2048` Type: `optionalInteger` #### `Swarm.RelayService.MaxReservationsPerPeer` **REMOVED in kubo 0.32 due to [go-libp2p#2974](https://github.com/libp2p/go-libp2p/pull/2974)** #### `Swarm.RelayService.MaxReservationsPerIP` Maximum number of reservations originating from the same IP. Default: `8` Type: `optionalInteger` #### `Swarm.RelayService.MaxReservationsPerASN` Maximum number of reservations originating from the same ASN. Default: `32` Type: `optionalInteger` ### `Swarm.EnableRelayHop` **REMOVED** Replaced with [`Swarm.RelayService.Enabled`](#swarmrelayserviceenabled). ### `Swarm.DisableRelay` **REMOVED** Set `Swarm.Transports.Network.Relay` to `false` instead. ### `Swarm.EnableAutoNATService` **REMOVED** Please use [`AutoNAT.ServiceMode`](#autonatservicemode). ### `Swarm.ConnMgr` The connection manager determines which and how many connections to keep and can be configured to keep. Kubo currently supports two connection managers: - none: never close idle connections. - basic: the default connection manager. By default, this section is empty and the implicit defaults defined below are used. #### `Swarm.ConnMgr.Type` Sets the type of connection manager to use, options are: `"none"` (no connection management) and `"basic"`. Default: "basic". Type: `optionalString` (default when unset or empty) #### Basic Connection Manager The basic connection manager uses a "high water", a "low water", and internal scoring to periodically close connections to free up resources. When a node using the basic connection manager reaches `HighWater` idle connections, it will close the least useful ones until it reaches `LowWater` idle connections. The process of closing connections happens every `SilencePeriod`. The connection manager considers a connection idle if: - It has not been explicitly _protected_ by some subsystem. For example, Bitswap will protect connections to peers from which it is actively downloading data, the DHT will protect some peers for routing, and the peering subsystem will protect all "peered" nodes. - It has existed for longer than the `GracePeriod`. **Example:** ```json { "Swarm": { "ConnMgr": { "Type": "basic", "LowWater": 100, "HighWater": 200, "GracePeriod": "30s", "SilencePeriod": "10s" } } } ``` ##### `Swarm.ConnMgr.LowWater` LowWater is the number of connections that the basic connection manager will trim down to. Default: `32` Type: `optionalInteger` ##### `Swarm.ConnMgr.HighWater` HighWater is the number of connections that, when exceeded, will trigger a connection GC operation. Note: protected/recently formed connections don't count towards this limit. Default: `96` Type: `optionalInteger` ##### `Swarm.ConnMgr.GracePeriod` GracePeriod is a time duration that new connections are immune from being closed by the connection manager. Default: `"20s"` Type: `optionalDuration` ##### `Swarm.ConnMgr.SilencePeriod` SilencePeriod is the time duration between connection manager runs, when connections that are idle are closed. Default: `"10s"` Type: `optionalDuration` ### `Swarm.ResourceMgr` Learn more about Kubo's usage of libp2p Network Resource Manager in the [dedicated resource management docs](./libp2p-resource-management.md). #### `Swarm.ResourceMgr.Enabled` Enables the libp2p Resource Manager using limits based on the defaults and/or other configuration as discussed in [libp2p resource management](./libp2p-resource-management.md). Default: `true` Type: `flag` #### `Swarm.ResourceMgr.MaxMemory` This is the max amount of memory to allow go-libp2p to use. libp2p's resource manager will prevent additional resource creation while this limit is reached. This value is also used to scale the limit on various resources at various scopes when the default limits (discussed in [libp2p resource management](./libp2p-resource-management.md)) are used. For example, increasing this value will increase the default limit for incoming connections. It is possible to inspect the runtime limits via `ipfs swarm resources --help`. > [!IMPORTANT] > `Swarm.ResourceMgr.MaxMemory` is the memory limit for go-libp2p networking stack alone, and not for entire Kubo or Bitswap. > > To set memory limit for the entire Kubo process, use [`GOMEMLIMIT` environment variable](http://web.archive.org/web/20240222201412/https://kupczynski.info/posts/go-container-aware/) which all Go programs recognize, and then set `Swarm.ResourceMgr.MaxMemory` to less than your custom `GOMEMLIMIT`. Default: `[TOTAL_SYSTEM_MEMORY]/2` Type: [`optionalBytes`](#optionalbytes) #### `Swarm.ResourceMgr.MaxFileDescriptors` This is the maximum number of file descriptors to allow libp2p to use. libp2p's resource manager will prevent additional file descriptor consumption while this limit is reached. This param is ignored on Windows. Default `[TOTAL_SYSTEM_FILE_DESCRIPTORS]/2` Type: `optionalInteger` #### `Swarm.ResourceMgr.Allowlist` A list of [multiaddrs][libp2p-multiaddrs] that can bypass normal system limits (but are still limited by the allowlist scope). Convenience config around [go-libp2p-resource-manager#Allowlist.Add](https://pkg.go.dev/github.com/libp2p/go-libp2p/p2p/host/resource-manager#Allowlist.Add). Default: `[]` Type: `array[string]` ([multiaddrs][multiaddr]) ### `Swarm.Transports` Configuration section for libp2p transports. An empty configuration will apply the defaults. ### `Swarm.Transports.Network` Configuration section for libp2p _network_ transports. Transports enabled in this section will be used for dialing. However, to receive connections on these transports, multiaddrs for these transports must be added to `Addresses.Swarm`. Supported transports are: QUIC, TCP, WS, Relay, WebTransport and WebRTCDirect. > [!CAUTION] > **SECURITY CONSIDERATIONS FOR NETWORK TRANSPORTS** > > Enabling network transports allows your node to accept connections from the internet. > Ensure your firewall rules and [`Addresses.Swarm`](#addressesswarm) configuration > align with your security requirements. > See [Security section](#security) for network exposure considerations. Each field in this section is a `flag`. #### `Swarm.Transports.Network.TCP` [TCP](https://en.wikipedia.org/wiki/Transmission_Control_Protocol) is a simple and widely deployed transport, it should be compatible with most implementations and network configurations. TCP doesn't directly support encryption and/or multiplexing, so libp2p will layer a security & multiplexing transport over it. Default: Enabled Type: `flag` Listen Addresses: - /ip4/0.0.0.0/tcp/4001 (default) - /ip6/::/tcp/4001 (default) #### `Swarm.Transports.Network.Websocket` [Websocket](https://en.wikipedia.org/wiki/WebSocket) is a transport usually used to connect to non-browser-based IPFS nodes from browser-based js-ipfs nodes. While it's enabled by default for dialing, Kubo doesn't listen on this transport by default. Default: Enabled Type: `flag` Listen Addresses: - /ip4/0.0.0.0/tcp/4001/ws - /ip6/::/tcp/4001/ws #### `Swarm.Transports.Network.QUIC` [QUIC](https://en.wikipedia.org/wiki/QUIC) is the most widely used transport by Kubo nodes. It is a UDP-based transport with built-in encryption and multiplexing. The primary benefits over TCP are: 1. It takes 1 round trip to establish a connection (our TCP transport currently takes 4). 2. No [Head-of-Line blocking](https://en.wikipedia.org/wiki/Head-of-line_blocking). 3. It doesn't require a file descriptor per connection, easing the load on the OS. Default: Enabled Type: `flag` Listen Addresses: - `/ip4/0.0.0.0/udp/4001/quic-v1` (default) - `/ip6/::/udp/4001/quic-v1` (default) #### `Swarm.Transports.Network.Relay` [Libp2p Relay](https://github.com/libp2p/specs/tree/master/relay) proxy transport that forms connections by hopping between multiple libp2p nodes. Allows IPFS node to connect to other peers using their `/p2p-circuit` [multiaddrs][libp2p-multiaddrs]. This transport is primarily useful for bypassing firewalls and NATs. See also: - Docs: [Libp2p Circuit Relay](https://web.archive.org/web/20260128152445/https://docs.libp2p.io/concepts/nat/circuit-relay/) - [`Swarm.RelayClient.Enabled`](#swarmrelayclientenabled) for getting a public - `/p2p-circuit` address when behind a firewall. - [`Swarm.EnableHolePunching`](#swarmenableholepunching) for direct connection upgrade through relay - [`Swarm.RelayService.Enabled`](#swarmrelayserviceenabled) for becoming a limited relay for other peers Default: Enabled Type: `flag` Listen Addresses: - This transport is special. Any node that enables this transport can receive inbound connections on this transport, without specifying a listen address. #### `Swarm.Transports.Network.WebTransport` A new feature of [`go-libp2p`](https://github.com/libp2p/go-libp2p/releases/tag/v0.23.0) is the [WebTransport](https://github.com/libp2p/go-libp2p/issues/1717) transport. This is a spiritual descendant of WebSocket but over `HTTP/3`. Since this runs on top of `HTTP/3` it uses `QUIC` under the hood. We expect it to perform worst than `QUIC` because of the extra overhead, this transport is really meant at agents that cannot do `TCP` or `QUIC` (like browsers). WebTransport is a new transport protocol currently under development by the IETF and the W3C, and already implemented by Chrome. Conceptually, it’s like WebSocket run over QUIC instead of TCP. Most importantly, it allows browsers to establish (secure!) connections to WebTransport servers without the need for CA-signed certificates, thereby enabling any js-libp2p node running in a browser to connect to any kubo node, with zero manual configuration involved. The previous alternative is websocket secure, which require installing a reverse proxy and TLS certificates manually. Default: Enabled Type: `flag` Listen Addresses: - `/ip4/0.0.0.0/udp/4001/quic-v1/webtransport` (default) - `/ip6/::/udp/4001/quic-v1/webtransport` (default) #### `Swarm.Transports.Network.WebRTCDirect` [WebRTC Direct](https://github.com/libp2p/specs/blob/master/webrtc/webrtc-direct.md) is a transport protocol that provides another way for browsers to connect to the rest of the libp2p network. WebRTC Direct allows for browser nodes to connect to other nodes without special configuration, such as TLS certificates. This can be useful for browser nodes that do not yet support [WebTransport](https://web.archive.org/web/20260107053250/https://blog.libp2p.io/2022-12-19-libp2p-webtransport/), which is still relatively new and has [known issues](https://github.com/libp2p/js-libp2p/issues/2572). Enabling this transport allows Kubo node to act on `/udp/4001/webrtc-direct` listeners defined in `Addresses.Swarm`, `Addresses.Announce` or `Addresses.AppendAnnounce`. > [!NOTE] > WebRTC Direct is browser-to-node. It cannot be used to connect a browser > node to a node that is behind a NAT or firewall (without UPnP port mapping). > The browser-to-private requires using normal > [WebRTC](https://github.com/libp2p/specs/blob/master/webrtc/webrtc.md), > which is currently being worked on in > [go-libp2p#2009](https://github.com/libp2p/go-libp2p/issues/2009). Default: Enabled Type: `flag` Listen Addresses: - `/ip4/0.0.0.0/udp/4001/webrtc-direct` (default) - `/ip6/::/udp/4001/webrtc-direct` (default) ### `Swarm.Transports.Security` Configuration section for libp2p _security_ transports. Transports enabled in this section will be used to secure unencrypted connections. This does not concern all the QUIC transports which use QUIC's builtin encryption. Security transports are configured with the `priority` type. When establishing an _outbound_ connection, Kubo will try each security transport in priority order (lower first), until it finds a protocol that the receiver supports. When establishing an _inbound_ connection, Kubo will let the initiator choose the protocol, but will refuse to use any of the disabled transports. Supported transports are: TLS (priority 100) and Noise (priority 200). No default priority will ever be less than 100. Lower values have precedence. #### `Swarm.Transports.Security.TLS` [TLS](https://github.com/libp2p/specs/tree/master/tls) (1.3) is the default security transport as of Kubo 0.5.0. It's also the most scrutinized and trusted security transport. Default: `100` Type: `priority` #### `Swarm.Transports.Security.SECIO` **REMOVED**: support for SECIO has been removed. Please remove this option from your config. #### `Swarm.Transports.Security.Noise` [Noise](https://github.com/libp2p/specs/tree/master/noise) is slated to replace TLS as the cross-platform, default libp2p protocol due to ease of implementation. It is currently enabled by default but with low priority as it's not yet widely supported. Default: `200` Type: `priority` ### `Swarm.Transports.Multiplexers` Configuration section for libp2p _multiplexer_ transports. Transports enabled in this section will be used to multiplex duplex connections. This does not concern all the QUIC transports which use QUIC's builtin muxing. Multiplexer transports are configured the same way security transports are, with the `priority` type. Like with security transports, the initiator gets their first choice. Supported transport is only: Yamux (priority 100) No default priority will ever be less than 100. ### `Swarm.Transports.Multiplexers.Yamux` Yamux is the default multiplexer used when communicating between Kubo nodes. Default: `100` Type: `priority` ### `Swarm.Transports.Multiplexers.Mplex` **REMOVED**: See Support for Mplex has been [removed from Kubo and go-libp2p](https://github.com/libp2p/specs/issues/553). Please remove this option from your config. ## `DNS` Options for configuring DNS resolution for [DNSLink](https://docs.ipfs.tech/concepts/dnslink/) and `/dns*` [Multiaddrs][libp2p-multiaddrs] (including peer addresses discovered via DHT or delegated routing). ### `DNS.Resolvers` Map of [FQDNs](https://en.wikipedia.org/wiki/Fully_qualified_domain_name) to custom resolver URLs. This allows for overriding the default DNS resolver provided by the operating system, and using different resolvers per domain or TLD (including ones from alternative, non-ICANN naming systems). Example: ```json { "DNS": { "Resolvers": { "eth.": "https://dns.eth.limo/dns-query", "crypto.": "https://resolver.unstoppable.io/dns-query", "libre.": "https://ns1.iriseden.fr/dns-query", ".": "https://cloudflare-dns.com/dns-query" } } } ``` Be mindful that: - Currently only `https://` URLs for [DNS over HTTPS (DoH)](https://en.wikipedia.org/wiki/DNS_over_HTTPS) endpoints are supported as values. - The default catch-all resolver is the cleartext one provided by your operating system. It can be overridden by adding a DoH entry for the DNS root indicated by `.` as illustrated above. - Out-of-the-box support for selected non-ICANN TLDs relies on third-party centralized services provided by respective communities on best-effort basis. - The special value `"auto"` uses DNS resolvers from [AutoConf](#autoconf) when enabled. For example: `{".": "auto"}` uses any custom DoH resolver (global or per TLD) provided by AutoConf system. - When [`AutoTLS.SkipDNSLookup`](#autotlsskipdnslookup) is enabled (default), domains matching [`AutoTLS.DomainSuffix`](#autotlsdomainsuffix) (default: `libp2p.direct`) are resolved locally by parsing the IP directly from the hostname. Set `AutoTLS.SkipDNSLookup=false` to force network DNS lookups for these domains. Default: `{".": "auto"}` Type: `object[string -> string]` ### `DNS.MaxCacheTTL` Maximum duration for which entries are valid in the DoH cache. This allows you to cap the Time-To-Live suggested by the DNS response ([RFC2181](https://datatracker.ietf.org/doc/html/rfc2181#section-8)). If present, the upper bound is applied to DoH resolvers in [`DNS.Resolvers`](#dnsresolvers). Note: this does NOT work with Go's default DNS resolver. To make this a global setting, add a `.` entry to `DNS.Resolvers` first. **Examples:** - `"1m"` DNS entries are kept for 1 minute or less. - `"0s"` DNS entries expire as soon as they are retrieved. Default: Respect DNS Response TTL Type: `optionalDuration` ## `HTTPRetrieval` `HTTPRetrieval` is configuration for pure HTTP retrieval based on Trustless HTTP Gateways' [Block Responses (`application/vnd.ipld.raw`)](https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw) which can be used in addition to or instead of retrieving blocks with [Bitswap over Libp2p](#bitswap). Default: `{}` Type: `object` ### `HTTPRetrieval.Enabled` Controls whether HTTP-based block retrieval is enabled. When enabled, Kubo will act on `/tls/http` (HTTP/2) providers ([Trustless HTTP Gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/)) returned by the [`Routing.DelegatedRouters`](#routingdelegatedrouters) to perform pure HTTP [block retrievals](https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw) (`/ipfs/cid?format=raw`, `Accept: application/vnd.ipld.raw`) alongside [Bitswap over Libp2p](#bitswap). HTTP requests for `application/vnd.ipld.raw` will be made instead of Bitswap when a peer has a `/tls/http` multiaddr and the HTTPS server returns HTTP 200 for the [probe path](https://specs.ipfs.tech/http-gateways/trustless-gateway/#dedicated-probe-paths). > [!IMPORTANT] > This feature is relatively new. Please report any issues via [Github](https://github.com/ipfs/kubo/issues/new). > > Important notes: > > - TLS and HTTP/2 are required. For privacy reasons, and to maintain feature-parity with browsers, unencrypted `http://` providers are ignored and not used. > - This feature works in the same way as Bitswap: connected HTTP-peers receive optimistic block requests even for content that they are not announcing. > - For performance reasons, and to avoid loops, the HTTP client does not follow redirects. Providers should keep announcements up to date. > - IPFS ecosystem is working towards [supporting HTTP providers on Amino DHT](https://github.com/ipfs/specs/issues/496). Currently, HTTP providers are mostly limited to results from [`Routing.DelegatedRouters`](#routingdelegatedrouters) endpoints and requires `Routing.Type=auto|autoclient`. Default: `true` Type: `flag` ### `HTTPRetrieval.Allowlist` Optional list of hostnames for which HTTP retrieval is allowed for. If this list is not empty, only hosts matching these entries will be allowed for HTTP retrieval. > [!TIP] > To limit HTTP retrieval to a provider at `/dns4/example.com/tcp/443/tls/http` (which would serve `HEAD|GET https://example.com/ipfs/cid?format=raw`), set this to `["example.com"]` Default: `[]` Type: `array[string]` ### `HTTPRetrieval.Denylist` Optional list of hostnames for which HTTP retrieval is not allowed. Denylist entries take precedence over Allowlist entries. > [!TIP] > This denylist operates on HTTP endpoint hostnames. > To deny specific PeerID, use [`Routing.IgnoreProviders`](#routingignoreproviders) instead. Default: `[]` Type: `array[string]` ### `HTTPRetrieval.NumWorkers` The number of worker goroutines to use for concurrent HTTP retrieval operations. This setting controls the level of parallelism for HTTP-based block retrieval operations. Higher values can improve performance when retrieving many blocks but may increase resource usage. Default: `16` Type: `optionalInteger` ### `HTTPRetrieval.MaxBlockSize` Sets the maximum size of a block that the HTTP retrieval client will accept. > [!NOTE] > This setting is a security feature designed to protect Kubo from malicious providers who might send excessively large or invalid data. > Increasing this value allows Kubo to retrieve larger blocks from compatible HTTP providers, but doing so reduces interoperability with Bitswap, and increases potential security risks. > > Learn more: [Supporting Large IPLD Blocks: Why block limits?](https://discuss.ipfs.tech/t/supporting-large-ipld-blocks/15093#why-block-limits-5) Default: `2MiB` (matching [Bitswap size limit](https://specs.ipfs.tech/bitswap-protocol/#block-sizes)) Type: `optionalString` ### `HTTPRetrieval.TLSInsecureSkipVerify` Disables TLS certificate validation. Allows making HTTPS connections to HTTP/2 test servers with self-signed TLS certificates. Only for testing, do not use in production. Default: `false` Type: `flag` ## `Import` Options to configure the default parameters used for ingesting data, in commands such as `ipfs add` or `ipfs block put`. All affected commands are detailed per option. These options implement [IPIP-499: UnixFS CID Profiles](https://specs.ipfs.tech/ipips/ipip-0499/) for reproducible CID generation across IPFS implementations. Instead of configuring individual options, you can apply a predefined profile with `ipfs config profile apply `. See [Profiles](#profiles) for available options like `unixfs-v1-2025`. Note that using CLI flags will override the options defined here. ### `Import.CidVersion` The default CID version. Commands affected: `ipfs add`. Must be either 0 or 1. CIDv0 uses SHA2-256 only, while CIDv1 supports multiple hash functions. Default: `0` Type: `optionalInteger` ### `Import.UnixFSRawLeaves` The default UnixFS raw leaves option. Commands affected: `ipfs add`, `ipfs files write`. Default: `false` if `CidVersion=0`; `true` if `CidVersion=1` Type: `flag` ### `Import.UnixFSChunker` The default UnixFS chunker. Commands affected: `ipfs add`. Valid formats: - `size-` - fixed size chunker - `rabin---` - rabin fingerprint chunker - `buzhash` - buzhash chunker The maximum accepted value for `size-` and rabin `max` parameter is `2MiB - 256 bytes` (2096896 bytes). The 256-byte overhead budget is reserved for protobuf/UnixFS framing so that serialized blocks stay within the 2MiB block size limit defined by the [bitswap spec](https://specs.ipfs.tech/bitswap-protocol/#block-sizes). The `buzhash` chunker uses a fixed internal maximum of 512KiB and is not affected by this limit. Only the fixed-size chunker (`size-`) guarantees that the same data will always produce the same CID. The `rabin` and `buzhash` chunkers may change their internal parameters in a future release. Default: `size-262144` Type: `optionalString` ### `Import.HashFunction` The default hash function. Commands affected: `ipfs add`, `ipfs block put`, `ipfs dag put`. Must be a valid multihash name (e.g., `sha2-256`, `blake3`) and must be allowed for use in IPFS according to security constraints. Run `ipfs cid hashes --supported` to see the full list of allowed hash functions. Default: `sha2-256` Type: `optionalString` ### `Import.FastProvideRoot` Immediately provide root CIDs to the DHT in addition to the regular provide queue. This complements the sweep provider system: fast-provide handles the urgent case (root CIDs that users share and reference), while the sweep provider efficiently provides all blocks according to the `Provide.Strategy` over time. Together, they optimize for both immediate discoverability of newly imported content and efficient resource usage for complete DAG provides. When disabled, only the sweep provider's queue is used. This setting applies to both `ipfs add` and `ipfs dag import` commands and can be overridden per-command with the `--fast-provide-root` flag. Ignored when DHT is not available for routing (e.g., `Routing.Type=none` or delegated-only configurations). Default: `true` Type: `flag` ### `Import.FastProvideWait` Wait for the immediate root CID provide to complete before returning. When enabled, the command blocks until the provide completes, ensuring guaranteed discoverability before returning. When disabled (default), the provide happens asynchronously in the background without blocking the command. Use this when you need certainty that content is discoverable before the command returns (e.g., sharing a link immediately after adding). This setting applies to both `ipfs add` and `ipfs dag import` commands and can be overridden per-command with the `--fast-provide-wait` flag. Ignored when DHT is not available for routing (e.g., `Routing.Type=none` or delegated-only configurations). Default: `false` Type: `flag` ### `Import.BatchMaxNodes` The maximum number of nodes in a write-batch. The total size of the batch is limited by `BatchMaxnodes` and `BatchMaxSize`. Increasing this will batch more items together when importing data with `ipfs dag import`, which can speed things up. Must be positive (> 0). Setting to 0 would cause immediate batching after each node, which is inefficient. Default: `128` Type: `optionalInteger` ### `Import.BatchMaxSize` The maximum size of a single write-batch (computed as the sum of the sizes of the blocks). The total size of the batch is limited by `BatchMaxnodes` and `BatchMaxSize`. Increasing this will batch more items together when importing data with `ipfs dag import`, which can speed things up. Must be positive (> 0). Setting to 0 would cause immediate batching after any data, which is inefficient. Default: `20971520` (20MiB) Type: `optionalInteger` ### `Import.UnixFSFileMaxLinks` The maximum number of links that a node part of a UnixFS File can have when building the DAG while importing. This setting controls both the fanout in files that are chunked into several blocks and grouped as a Unixfs (dag-pb) DAG. Must be positive (> 0). Zero or negative values would break file DAG construction. Default: `174` Type: `optionalInteger` ### `Import.UnixFSDirectoryMaxLinks` The maximum number of links that a node part of a UnixFS basic directory can have when building the DAG while importing. This setting controls both the fanout for basic, non-HAMT folder nodes. It sets a limit after which directories are converted to a HAMT-based structure. When unset (0), no limit exists for children. Directories will be converted to HAMTs based on their estimated size only. This setting will cause basic directories to be converted to HAMTs when they exceed the maximum number of children. This happens transparently during the add process. The fanout of HAMT nodes is controlled by `MaxHAMTFanout`. Must be non-negative (>= 0). Zero means no limit, negative values are invalid. Commands affected: `ipfs add` Default: `0` (no limit, because [`Import.UnixFSHAMTDirectorySizeThreshold`](#importunixfshamtdirectorysizethreshold) triggers controls when to switch to HAMT sharding when a directory grows too big) Type: `optionalInteger` ### `Import.UnixFSHAMTDirectoryMaxFanout` The maximum number of children that a node part of a UnixFS HAMT directory (aka sharded directory) can have. HAMT directories have unlimited children and are used when basic directories become too big or reach `MaxLinks`. A HAMT is a structure made of UnixFS nodes that store the list of elements in the folder. This option controls the maximum number of children that the HAMT nodes can have. According to the [UnixFS specification](https://specs.ipfs.tech/unixfs/#hamt-structure-and-parameters), this value must be a power of 2, between 8 (for byte-aligned bitfields) and 1024 (to prevent denial-of-service attacks). Commands affected: `ipfs add`, `ipfs daemon` (globally overrides [`boxo/ipld/unixfs/io.DefaultShardWidth`](https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L30C5-L30C22)) Default: `256` Type: `optionalInteger` ### `Import.UnixFSHAMTDirectorySizeThreshold` The sharding threshold to decide whether a basic UnixFS directory should be sharded (converted into HAMT Directory) or not. This value is not strictly related to the size of the UnixFS directory block and any increases in the threshold should come with being careful that block sizes stay under 2MiB in order for them to be reliably transferable through the networking stack. At the time of writing this, IPFS peers on the public swarm tend to ignore requests for blocks bigger than 2MiB. Uses implementation from `boxo/ipld/unixfs/io/directory`, where the size is not the _exact_ block size of the encoded directory but just the estimated size based byte length of DAG-PB Links names and CIDs. Setting to `1B` is functionally equivalent to always using HAMT (useful in testing). Commands affected: `ipfs add`, `ipfs daemon` (globally overrides [`boxo/ipld/unixfs/io.HAMTShardingSize`](https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L26)) Default: `256KiB` (may change, inspect `DefaultUnixFSHAMTDirectorySizeThreshold` to confirm) Type: [`optionalBytes`](#optionalbytes) ### `Import.UnixFSHAMTDirectorySizeEstimation` Controls how directory size is estimated when deciding whether to switch from a basic UnixFS directory to HAMT sharding. Accepted values: - `links` (default): Legacy estimation using sum of link names and CID byte lengths. - `block`: Full serialized dag-pb block size for accurate threshold decisions. - `disabled`: Disable HAMT sharding entirely (directories always remain basic). The `block` estimation is recommended for new profiles as it provides more accurate threshold decisions and better cross-implementation consistency. See [IPIP-499](https://specs.ipfs.tech/ipips/ipip-0499/) for more details. Commands affected: `ipfs add` Default: `links` Type: `optionalString` ### `Import.UnixFSDAGLayout` Controls the DAG layout used when chunking files. Accepted values: - `balanced` (default): Balanced DAG layout with uniform leaf depth. - `trickle`: Trickle DAG layout optimized for streaming. Commands affected: `ipfs add` Default: `balanced` Type: `optionalString` ## `Version` Options to configure agent version announced to the swarm, and leveraging other peers version for detecting when there is time to update. ### `Version.AgentSuffix` Optional suffix to the AgentVersion presented by `ipfs id` and exposed via [libp2p identify protocol](https://github.com/libp2p/specs/blob/master/identify/README.md#agentversion). The value from config takes precedence over value passed via `ipfs daemon --agent-version-suffix`. > [!NOTE] > Setting a custom version suffix helps with ecosystem analysis, such as Amino DHT reports published at Default: `""` (no suffix, or value from `ipfs daemon --agent-version-suffix=`) Type: `optionalString` ### `Version.SwarmCheckEnabled` Observe the AgentVersion of swarm peers and log warning when `SwarmCheckPercentThreshold` of peers runs version higher than this node. Default: `true` Type: `flag` ### `Version.SwarmCheckPercentThreshold` Control the percentage of `kubo/` peers running new version required to trigger update warning. Default: `5` Type: `optionalInteger` (1-100) ## Profiles Configuration profiles allow to tweak configuration quickly. Profiles can be applied with the `--profile` flag to `ipfs init` or with the `ipfs config profile apply` command. When a profile is applied a backup of the configuration file will be created in `$IPFS_PATH`. Configuration profiles can be applied additively. For example, both the `unixfs-v1-2025` and `lowpower` profiles can be applied one after the other. The available configuration profiles are listed below. You can also find them documented in `ipfs config profile --help`. ### `server` profile Disables local [`Discovery.MDNS`](#discoverymdns), [turns off uPnP NAT port mapping](#swarmdisablenatportmap), and blocks connections to IPv4 and IPv6 prefixes that are [private, local only, or unrouteable](https://github.com/ipfs/kubo/blob/b71cf0d15904bdef21fe2eee5f1118a274309a4d/config/profile.go#L24-L43). Recommended when running IPFS on machines with public IPv4 addresses (no NAT, no uPnP) at providers that interpret local IPFS discovery and traffic as netscan abuse ([example](https://github.com/ipfs/kubo/issues/10327)). ### `randomports` profile Use a random port number for the incoming swarm connections. Used for testing. ### `default-datastore` profile Configures the node to use the default datastore (flatfs). Read the "flatfs" profile description for more information on this datastore. This profile may only be applied when first initializing the node. ### `local-discovery` profile Enables local [`Discovery.MDNS`](#discoverymdns) (enabled by default). Useful to re-enable local discovery after it's disabled by another profile (e.g., the server profile). `test` profile Reduces external interference of IPFS daemon, this is useful when using the daemon in test environments. ### `default-networking` profile Restores default network settings. Inverse profile of the test profile. ### `autoconf-on` profile Safe default for joining the public IPFS Mainnet swarm with automatic configuration. Can also be used with custom AutoConf.URL for other networks. ### `autoconf-off` profile Disables AutoConf and clears all networking fields for manual configuration. Use this for private networks or when you want explicit control over all endpoints. ### `flatfs` profile Configures the node to use the flatfs datastore. Flatfs is the default, most battle-tested and reliable datastore. You should use this datastore if: - You need a very simple and very reliable datastore, and you trust your filesystem. This datastore stores each block as a separate file in the underlying filesystem so it's unlikely to lose data unless there's an issue with the underlying file system. - You need to run garbage collection in a way that reclaims free space as soon as possible. - You want to minimize memory usage. - You are ok with the default speed of data import, or prefer to use `--nocopy`. > [!WARNING] > This profile may only be applied when first initializing the node via `ipfs init --profile flatfs` > [!NOTE] > See caveats and configuration options at [`datastores.md#flatfs`](datastores.md#flatfs) ### `flatfs-measure` profile Configures the node to use the flatfs datastore with metrics. This is the same as [`flatfs` profile](#flatfs-profile) with the addition of the `measure` datastore wrapper. ### `pebbleds` profile Configures the node to use the pebble high-performance datastore. Pebble is a LevelDB/RocksDB inspired key-value store focused on performance and internal usage by CockroachDB. You should use this datastore if: - You need a datastore that is focused on performance. - You need a datastore that is good for multi-terabyte data sets. - You need reliability by default, but may choose to disable WAL for maximum performance when reliability is not critical. - You want a datastore that does not need GC cycles and does not use more space than necessary - You want a datastore that does not take several minutes to start with large repositories - You want a datastore that performs well even with default settings, but can optimized by setting configuration to tune it for your specific needs. > [!WARNING] > This profile may only be applied when first initializing the node via `ipfs init --profile pebbleds` > [!NOTE] > See other caveats and configuration options at [`datastores.md#pebbleds`](datastores.md#pebbleds) ### `pebbleds-measure` profile Configures the node to use the pebble datastore with metrics. This is the same as [`pebbleds` profile](#pebble-profile) with the addition of the `measure` datastore wrapper. ### `badgerds` profile Configures the node to use the **legacy** badgerv1 datastore. > [!CAUTION] > **Badger v1 datastore is deprecated and will be removed in a future Kubo release.** > > This is based on very old badger 1.x, which has not been maintained by its > upstream maintainers for years and has known bugs (startup timeouts, shutdown > hangs, file descriptor > exhaustion, and more). Do not use it for new deployments. > > **To migrate:** create a new `IPFS_PATH` with `flatfs` > (`ipfs init --profile=flatfs`), move pinned data via > `ipfs dag export/import` or `ipfs pin ls -t recursive|add`, and decommission the > old badger-based node. When it comes to block storage, use experimental > `pebbleds` only if you are sure modern `flatfs` does not serve your use case > (most users will be perfectly fine with `flatfs`, it is also possible to keep > `flatfs` for blocks and replace `leveldb` with `pebble` if preferred over > `leveldb`). Also, be aware that: - This datastore will not properly reclaim space when your datastore is smaller than several gigabytes. If you run IPFS with `--enable-gc`, you plan on storing very little data in your IPFS node, and disk usage is more critical than performance, consider using `flatfs`. - This datastore uses up to several gigabytes of memory. - Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte. > [!WARNING] > This profile may only be applied when first initializing the node via `ipfs init --profile badgerds` > [!NOTE] > See other caveats and configuration options at [`datastores.md#badgerds`](datastores.md#badgerds) ### `badgerds-measure` profile Configures the node to use the **legacy** badgerv1 datastore with metrics. This is the same as [`badgerds` profile](#badger-profile) with the addition of the `measure` datastore wrapper. This profile will be removed in a future Kubo release. ### `lowpower` profile Reduces daemon overhead on the system by disabling optional swarm services. - [`Routing.Type`](#routingtype) set to `autoclient` (no DHT server, only client). - `Swarm.ConnMgr` set to maintain minimum number of p2p connections at a time. - Disables [`AutoNAT`](#autonat). - Disables [`Swam.RelayService`](#swarmrelayservice). > [!NOTE] > This profile is provided for legacy reasons. > With modern Kubo setting the above should not be necessary. ### `announce-off` profile Disables [Provide](#provide) system (and announcing to Amino DHT). > [!CAUTION] > The main use case for this is setups with manual Peering.Peers config. > Data from this node will not be announced on the DHT. This will make > DHT-based routing an data retrieval impossible if this node is the only > one hosting it, and other peers are not already connected to it. ### `announce-on` profile (Re-)enables [Provide](#provide) system (reverts [`announce-off` profile](#announce-off-profile)). ### `unixfs-v0-2015` profile Legacy UnixFS import profile for backward-compatible CID generation. Produces CIDv0 with no raw leaves, sha2-256, 256 KiB chunks, and link-based HAMT size estimation. See for exact [`Import.*`](#import) settings. > [!NOTE] > Use only when legacy CIDs are required. For new projects, use [`unixfs-v1-2025`](#unixfs-v1-2025-profile). > > See [IPIP-499](https://specs.ipfs.tech/ipips/ipip-0499/) for more details. ### `legacy-cid-v0` profile Alias for [`unixfs-v0-2015`](#unixfs-v0-2015-profile) profile. ### `unixfs-v1-2025` profile Recommended UnixFS import profile for cross-implementation CID determinism. Uses CIDv1, raw leaves, sha2-256, 1 MiB chunks, 1024 links per file node, 256 HAMT fanout, and block-based size estimation for HAMT threshold. See for exact [`Import.*`](#import) settings. > [!NOTE] > This profile ensures CID consistency across different IPFS implementations. > > See [IPIP-499](https://specs.ipfs.tech/ipips/ipip-0499/) for more details. ## Security This section provides an overview of security considerations for configurations that expose network services. ### Port and Network Exposure Several configuration options expose TCP or UDP ports that can make your Kubo node accessible from the network: - **[`Addresses.API`](#addressesapi)** - Exposes the admin RPC API (default: localhost:5001) - **[`Addresses.Gateway`](#addressesgateway)** - Exposes the HTTP gateway (default: localhost:8080) - **[`Addresses.Swarm`](#addressesswarm)** - Exposes P2P connectivity (default: 0.0.0.0:4001, both UDP and TCP) - **[`Swarm.Transports.Network`](#swarmtransportsnetwork)** - Controls which P2P transport protocols are enabled over TCP and UDP ### Security Best Practices - Keep admin services ([`Addresses.API`](#addressesapi)) bound to localhost unless authentication ([`API.Authorizations`](#apiauthorizations)) is configured - Use [`Gateway.NoFetch`](#gatewaynofetch) to prevent arbitrary CID retrieval if Kubo is acting as a public gateway available to anyone - Configure firewall rules to restrict access to exposed ports. Note that [`Addresses.Swarm`](#addressesswarm) is special - all incoming traffic to swarm ports should be allowed to ensure proper P2P connectivity - Control which public-facing addresses are announced to other peers using [`Addresses.NoAnnounce`](#addressesnoannounce), [`Addresses.Announce`](#addressesannounce), and [`Addresses.AppendAnnounce`](#addressesappendannounce) - Consider using the [`server` profile](#server-profile) for production deployments ## Types This document refers to the standard JSON types (e.g., `null`, `string`, `number`, etc.), as well as a few custom types, described below. ### `flag` Flags allow enabling and disabling features. However, unlike simple booleans, they can also be `null` (or omitted) to indicate that the default value should be chosen. This makes it easier for Kubo to change the defaults in the future unless the user _explicitly_ sets the flag to either `true` (enabled) or `false` (disabled). Flags have three possible states: - `null` or missing (apply the default value). - `true` (enabled) - `false` (disabled) ### `priority` Priorities allow specifying the priority of a feature/protocol and disabling the feature/protocol. Priorities can take one of the following values: - `null`/missing (apply the default priority, same as with flags) - `false` (disabled) - `1 - 2^63` (priority, lower is preferred) ### `strings` Strings is a special type for conveniently specifying a single string, an array of strings, or null: - `null` - `"a single string"` - `["an", "array", "of", "strings"]` ### `duration` Duration is a type for describing lengths of time, using the same format go does (e.g, `"1d2h4m40.01s"`). ### `optionalInteger` Optional integers allow specifying some numerical value which has an implicit default when missing from the config file: - `null`/missing will apply the default value defined in Kubo sources (`.WithDefault(value)`) - an integer between `-2^63` and `2^63-1` (i.e. `-9223372036854775808` to `9223372036854775807`) ### `optionalBytes` Optional Bytes allow specifying some number of bytes which has an implicit default when missing from the config file: - `null`/missing (apply the default value defined in Kubo sources) - a string value indicating the number of bytes, including human readable representations: - [SI sizes](https://en.wikipedia.org/wiki/Metric_prefix#List_of_SI_prefixes) (metric units, powers of 1000), e.g. `1B`, `2kB`, `3MB`, `4GB`, `5TB`, …) - [IEC sizes](https://en.wikipedia.org/wiki/Binary_prefix#IEC_prefixes) (binary units, powers of 1024), e.g. `1B`, `2KiB`, `3MiB`, `4GiB`, `5TiB`, …) - a raw number (will be interpreted as bytes, e.g. `1048576` for 1MiB) ### `optionalString` Optional strings allow specifying some string value which has an implicit default when missing from the config file: - `null`/missing will apply the default value defined in Kubo sources (`.WithDefault("value")`) - a string ### `optionalDuration` Optional durations allow specifying some duration value which has an implicit default when missing from the config file: - `null`/missing will apply the default value defined in Kubo sources (`.WithDefault("1h2m3s")`) - a string with a valid [go duration](#duration) (e.g, `"1d2h4m40.01s"`). ---- [multiaddr]: https://docs.ipfs.tech/concepts/glossary/#multiaddr ================================================ FILE: docs/content-blocking.md ================================================


content blocking logo
Content Blocking in Kubo

Kubo ships with built-in support for denylist format from [IPIP-383](https://specs.ipfs.tech/ipips/ipip-0383/). ## Default behavior Official Kubo build does not ship with any denylists enabled by default. Content blocking is an opt-in decision made by the operator of `ipfs daemon`. ## How to enable blocking Place a `*.deny` file in one of directories: - `$IPFS_PATH/denylists/` (`$HOME/.ipfs/denylists/` if `IPFS_PATH` is not set) - `$XDG_CONFIG_HOME/ipfs/denylists/` (`$HOME/.config/ipfs/denylists/` if `XDG_CONFIG_HOME` is not set) - `/etc/ipfs/denylists/` (global) Files need to be present before starting the `ipfs daemon` in order to be watched for any new updates appended once started. Any other changes (such as removal of entries, prepending of entries, or insertion of new entries before the EOF at time of daemon starting) will not be detected or processed after boot; a restart of the daemon will be required for them to be factored in. If an entire new denylist file is added, `ipfs daemon` also needs to be restarted to track it. CLI and Gateway users will receive errors in response to request impacted by a blocklist: ``` Error: /ipfs/QmQvjk82hPkSaZsyJ8vNER5cmzKW7HyGX5XVusK7EAenCN is blocked and cannot be provided ``` End user is not informed about the exact reason, see [How to debug](#how-to-debug) if you need to find out which line of which denylist caused the request to be blocked. ## Denylist file format [NOpfs](https://github.com/ipfs-shipyard/nopfs) supports the format from [IPIP-383](https://specs.ipfs.tech/ipips/ipip-0383/). Clear-text rules are simple: just put content paths to block, one per line. Paths with unicode and whitespace need to be percent-encoded: ``` /ipfs/QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR /ipfs/bafybeihfg3d7rdltd43u3tfvncx7n5loqofbsobojcadtmokrljfthuc7y/927%20-%20Standards/927%20-%20Standards.png ``` Sensitive content paths can be double-hashed to block without revealing them. Double-hashed list example: https://badbits.dwebops.pub/badbits.deny See [IPIP-383](https://specs.ipfs.tech/ipips/ipip-0383/) for detailed format specification and more examples. ## How to suspend blocking without removing denylists Set `IPFS_CONTENT_BLOCKING_DISABLE` environment variable to `true` and restart the daemon. ## How to debug Debug logging of `nopfs` subsystem can be enabled with `GOLOG_LOG_LEVEL="nopfs=debug"` All block events are logged as warnings on a separate level named `nopfs-blocks`. To only log requests for blocked content set `GOLOG_LOG_LEVEL="nopfs-blocks=warn"`: ``` WARN (...) QmRFniDxwxoG2n4AcnGhRdjqDjCM5YeUcBE75K8WXmioH3: blocked (test.deny:9) ``` ================================================ FILE: docs/customizing.md ================================================ # Customizing Kubo You may want to customize Kubo if you want to reuse most of Kubo's machinery. This document discusses some approaches you may consider for customizing Kubo, and their tradeoffs. Some common use cases for customizing Kubo include: - Using a custom datastore for storing blocks, pins, or other Kubo metadata - Adding a custom data transfer protocol into Kubo - Customizing Kubo internals, such as adding allowlist/blocklist functionality to Bitswap - Adding new commands, interfaces, functionality, etc. to Kubo while reusing the libp2p swarm - Building on top of Kubo's configuration and config migration functionality ## Summary This table summarizes the tradeoffs between the approaches below: | | [Boxo](#boxo-build-your-own-binary) | [Kubo Plugin](#kubo-plugins) | [Bespoke Extension Point](#bespoke-extension-points) | [Go Plugin](#go-plugins) | [Fork](#fork-kubo) | |:-------------------:|:-----------------------------------:|:----------------------------:|:----------------------------------------------------:|:------------------------:|:------------------:| | Supported? | ✅ | ✅ | ✅ | ❌ | ❌ | | Future-proof? | ✅ | ❌ | ✅ | ❌ | ❌ | | Fully customizable? | ✅ | ✅ | ❌ | ✅ | ✅ | | Fast to implement? | ❌ | ✅ | ✅ | ✅ | ✅ | | Dynamic at runtime? | ❌ | ❌ | ✅ | ✅ | ❌ | | Add new commands? | ❌ | ✅ | ❌ | ✅ | ✅ | ## Boxo: build your own binary The best way to reuse Kubo functionality is to pick the functionality you need directly from [Boxo](https://github.com/ipfs/boxo) and compile your own binary. Boxo's raison d'etre is to be an IPFS component toolbox to support building custom-made implementations and applications. If your use case is not easy to implement with Boxo, you may want to consider adding whatever functionality is needed to Boxo instead of customizing Kubo, so that the community can benefit. If you are interested in this option, please reach out to Boxo maintainers, who will be happy to help you scope & plan the work. See [Boxo's FAQ](https://github.com/ipfs/boxo#help) for more info. ## Kubo Plugins Kubo plugins are a set of interfaces that may be implemented and injected into Kubo. Generally you should recompile the Kubo binary with your plugins added. A popular example of a Kubo plugin is [go-ds-s3](https://github.com/ipfs/go-ds-s3), which can be used to store blocks in Amazon S3. Some plugins, such as the `fx` plugin, allow deep customization of Kubo internals. As a result, Kubo maintainers can't guarantee backwards compatibility with these, so you may need to adapt to breaking changes when upgrading to new Kubo versions. For more information about the different types of Kubo plugins, see [plugins.md](./plugins.md). Kubo plugins can also be injected at runtime using Go plugins (see below), but these are hard to use and not well supported by Go, so we don't recommend them. ### Kubo binary imports It is possible to depend on the package `cmd/ipfs/kubo` as a way of using Kubo plugins that is an alternative to recompiling Kubo with additional preloaded plugins. This gives a more Go-centric dependency updating flow to building a new binary with preloaded plugins by simply requiring updating a Kubo dependency rather than needing to update Kubo source code and recompile. ## Bespoke Extension Points Certain Kubo functionality may have their own extension points. For example: * Kubo supports the [Routing v1](https://specs.ipfs.tech/routing/http-routing-v1/) API for delegating content routing to external processes * Kubo supports the [Pinning Service API](https://github.com/ipfs/pinning-services-api-spec) for delegating pinning to external processes * Kubo supports [DNSLink](https://dnslink.dev/) for delegating name->CID mappings to DNS (This list is not exhaustive.) These can generally be developed and deployed as sidecars (or full external services) without modifying the Kubo binary. ## Go Plugins Go provides [dynamic plugins](https://pkg.go.dev/plugin) which can be loaded at runtime into a Go binary. Kubo currently works with Go plugins. But using Go plugins requires that you compile the plugin using the exact same version of the Go toolchain with the same configuration (build flags, environment variables, etc.). As a result, you likely need to build Kubo and the plugins together at the same time, and at that point you may as well just compile the functionality directly into Kubo and avoid Go plugins. As a result, we don't recommend using Go plugins, and are likely to remove them in a future release of Kubo. ## Fork Kubo The "nuclear option" is to fork Kubo into your own repo, make your changes, and periodically sync your repo with the Kubo repo. This can be a good option if your changes are significant and you can commit to keeping your repo in sync with Kubo. Kubo maintainers can't make any backwards compatibility guarantees about Kubo internals, so by choosing this option you're accepting the risk that you may need to spend more time adapting to breaking changes. ================================================ FILE: docs/datastores.md ================================================ # Datastore Configuration Options This document describes the different possible values for the `Datastore.Spec` field in the ipfs configuration file. - [flatfs](#flatfs) - [levelds](#levelds) - [pebbleds](#pebbleds) - [badgerds](#badgerds) - [mount](#mount) - [measure](#measure) ## flatfs Stores each key-value pair as a file on the filesystem. The shardFunc is prefixed with `/repo/flatfs/shard/v1` then followed by a descriptor of the sharding strategy. Some example values are: - `/repo/flatfs/shard/v1/next-to-last/2` - Shards on the two next to last characters of the key - `/repo/flatfs/shard/v1/prefix/2` - Shards based on the two-character prefix of the key ```json { "type": "flatfs", "path": "", "shardFunc": "", "sync": true|false } ``` - `sync`: Flush every write to disk before continuing. Setting this to false is safe as kubo will automatically flush writes to disk before and after performing critical operations like pinning. However, you can set this to true to be extra-safe (at the cost of a slowdown when adding files). NOTE: flatfs must only be used as a block store (mounted at `/blocks`) as it only partially implements the datastore interface. You can mount flatfs for /blocks only using the mount datastore (described below). ## levelds Uses a [leveldb](https://github.com/syndtr/goleveldb) database to store key-value pairs via [go-ds-leveldb](https://github.com/ipfs/go-ds-leveldb). ```json { "type": "levelds", "path": "", "compression": "none" | "snappy", } ``` > [!NOTE] > LevelDB uses a log-structured merge-tree (LSM) storage engine. When keys are > deleted, the data is not removed immediately. Instead, a tombstone marker is > written, and the actual data is removed later by background compaction. > > LevelDB's compaction decides what to compact based on file counts (L0) and > total level size (L1+), without considering how many tombstones a file > contains. This means that after bulk deletions (such as pin removals or the > periodic provider keystore sync), disk space may not be reclaimed promptly. > The `datastore/` directory can grow significantly larger than the live data it > holds, especially on long-running nodes with many CIDs. > > Unlike flatfs (which deletes files immediately) or pebble (which has > tombstone-aware compaction), LevelDB has no way to prioritize reclaiming > space from deleted keys. Restarting the daemon may trigger some compaction, > but this is not guaranteed. > > If slow compaction is a problem, consider using the `pebbleds` datastore > instead (see below), which handles this workload more efficiently. ## pebbleds Uses [pebble](https://github.com/cockroachdb/pebble) as a key-value store. ```json { "type": "pebbleds", "path": "", } ``` The following options are available for tuning pebble. If they are not configured (or assigned their zero-valued), then default values are used. * `bytesPerSync`: int, Sync sstables periodically in order to smooth out writes to disk. (default: 512KB) * `disableWAL`: true|false, Disable the write-ahead log (WAL) at expense of prohibiting crash recovery. (default: false) * `cacheSize`: Size of pebble's shared block cache. (default: 8MB) * `formatVersionMajor`: int, Sets the format of pebble on-disk files. If 0 or unset, automatically convert to latest format. * `l0CompactionThreshold`: int, Count of L0 files necessary to trigger an L0 compaction. * `l0StopWritesThreshold`: int, Limit on L0 read-amplification, computed as the number of L0 sublevels. * `lBaseMaxBytes`: int, Maximum number of bytes for LBase. The base level is the level which L0 is compacted into. * `maxConcurrentCompactions`: int, Maximum number of concurrent compactions. (default: 1) * `memTableSize`: int, Size of a MemTable in steady state. The actual MemTable size starts at min(256KB, MemTableSize) and doubles for each subsequent MemTable up to MemTableSize (default: 4MB) * `memTableStopWritesThreshold`: int, Limit on the number of queued of MemTables. (default: 2) * `walBytesPerSync`: int: Sets the number of bytes to write to a WAL before calling Sync on it in the background. (default: 0, no background syncing) * `walMinSyncSeconds`: int: Sets the minimum duration between syncs of the WAL. (default: 0) > [!TIP] > Start using pebble with only default values and configure tuning items are needed for your needs. For a more complete description of these values, see: `https://pkg.go.dev/github.com/cockroachdb/pebble@vA.B.C#Options` (where `A.B.C` is pebble version from Kubo's `go.mod`). Using a pebble datastore can be set when initializing kubo `ipfs init --profile pebbleds`. #### Use of `formatMajorVersion` [Pebble's `FormatMajorVersion`](https://github.com/cockroachdb/pebble/tree/master?tab=readme-ov-file#format-major-versions) is a constant controlling the format of persisted data. Backwards incompatible changes to durable formats are gated behind new format major versions. At any point, a database's format major version may be bumped. However, once a database's format major version is increased, previous versions of Pebble will refuse to open the database. When IPFS is initialized to use the pebbleds datastore (`ipfs init --profile=pebbleds`), the latest pebble database format is configured in the pebble datastore config as `"formatMajorVersion"`. Setting this in the datastore config prevents automatically upgrading to the latest available version when kubo is upgraded. If a later version becomes available, the kubo daemon prints a startup message to indicate this. The user can them update the config to use the latest format when they are certain a downgrade will not be necessary. Without the `"formatMajorVersion"` in the pebble datastore config, the database format is automatically upgraded to the latest version. If this happens, then it is possible a downgrade back to the previous version of kubo will not work if new format is not compatible with the pebble datastore in the previous version of kubo. When installing a new version of kubo when `"formatMajorVersion"` is configured, migration does not upgrade this to the latest available version. This is done because a user may have reasons not to upgrade the pebble database format, and may want to be able to downgrade kubo if something else is not working in the new version. If the configured pebble database format in the old kubo is not supported in the new kubo, then the configured version must be updated and the old kubo run, before installing the new kubo. ## badgerds Uses [badger](https://github.com/dgraph-io/badger) as a key-value store. > [!CAUTION] > **Badger v1 datastore is deprecated and will be removed in a future Kubo release.** > > This is based on very old badger 1.x, which has not been maintained by its > upstream maintainers for years and has known bugs (startup timeouts, shutdown > hangs, file descriptor > exhaustion, and more). Do not use it for new deployments. > > **To migrate:** create a new `IPFS_PATH` with `flatfs` > (`ipfs init --profile=flatfs`), move pinned data via > `ipfs dag export/import` or `ipfs pin ls -t recursive|add`, and decommission the > old badger-based node. When it comes to block storage, use experimental > `pebbleds` only if you are sure modern `flatfs` does not serve your use case > (most users will be perfectly fine with `flatfs`, it is also possible to keep > `flatfs` for blocks and replace `leveldb` with `pebble` if preferred over > `leveldb`). - `syncWrites`: Flush every write to disk before continuing. Setting this to false is safe as kubo will automatically flush writes to disk before and after performing critical operations like pinning. However, you can set this to true to be extra-safe (at the cost of a 2-3x slowdown when adding files). - `truncate`: Truncate the DB if a partially written sector is found (defaults to true). There is no good reason to set this to false unless you want to manually recover partially written (and unpinned) blocks if kubo crashes half-way through a write operation. ```json { "type": "badgerds", "path": "", "syncWrites": true|false, "truncate": true|false, } ``` ## mount Allows specified datastores to handle keys prefixed with a given path. The mountpoints are added as keys within the child datastore definitions. ```json { "type": "mount", "mounts": [ { // Insert other datastore definition here, but add the following key: "mountpoint": "/path/to/handle" }, { // Insert other datastore definition here, but add the following key: "mountpoint": "/path/to/handle" }, ] } ``` ## measure This datastore is a wrapper that adds metrics tracking to any datastore. ```json { "type": "measure", "prefix": "sometag.datastore", "child": { datastore being wrapped } } ``` ================================================ FILE: docs/debug-guide.md ================================================ # General performance debugging guidelines This is a document for helping debug Kubo. Please add to it if you can! # Table of Contents - [General performance debugging guidelines](#general-performance-debugging-guidelines) - [Table of Contents](#table-of-contents) - [Beginning](#beginning) - [Analyzing the stack dump](#analyzing-the-stack-dump) - [Analyzing the CPU Profile](#analyzing-the-cpu-profile) - [Analyzing vars and memory statistics](#analyzing-vars-and-memory-statistics) - [Tracing](#tracing) - [Other](#other) ### Beginning > **Note:** Enable more logs by setting `GOLOG_LOG_LEVEL` env variable when troubleshooting. See [go-log documentation](https://github.com/ipfs/go-log#golog_log_level) for configuration options and available log levels. When you see ipfs doing something (using lots of CPU, memory, or otherwise being weird), the first thing you want to do is gather all the relevant profiling information. There's a command (`ipfs diag profile`) that will do this for you and bundle the results up into a zip file, ready to be attached to a bug report. If you feel intrepid, you can dump this information and investigate it yourself: - goroutine dump - `curl localhost:5001/debug/pprof/goroutine\?debug=2 > ipfs.stacks` - 30 second cpu profile - `curl localhost:5001/debug/pprof/profile > ipfs.cpuprof` - heap trace dump - `curl localhost:5001/debug/pprof/heap > ipfs.heap` - memory statistics (in json, see "memstats" object) - `curl localhost:5001/debug/vars > ipfs.vars` - system information - `ipfs diag sys > ipfs.sysinfo` ### Analyzing the stack dump The first thing to look for is hung goroutines -- any goroutine that's been stuck for over a minute will note that in the trace. It looks something like: ``` goroutine 2306090 [semacquire, 458 minutes]: sync.runtime_Semacquire(0xc8222fd3e4) /home/whyrusleeping/go/src/runtime/sema.go:47 +0x26 sync.(*Mutex).Lock(0xc8222fd3e0) /home/whyrusleeping/go/src/sync/mutex.go:83 +0x1c4 gx/ipfs/QmedFDs1WHcv3bcknfo64dw4mT1112yptW1H65Y2Wc7KTV/yamux.(*Session).Close(0xc8222fd340, 0x0, 0x0) /home/whyrusleeping/gopkg/src/gx/ipfs/QmedFDs1WHcv3bcknfo64dw4mT1112yptW1H65Y2Wc7KTV/yamux/session.go:205 +0x55 gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer/yamux.(*conn).Close(0xc8222fd340, 0x0, 0x0) /home/whyrusleeping/gopkg/src/gx/ipfs/QmWSJzRkCMJFHYUQZxKwPX8WA7XipaPtfiwMPARP51ymfn/go-stream-muxer/yamux/yamux.go:39 +0x2d gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream.(*Conn).Close(0xc8257a2000, 0x0, 0x0) /home/whyrusleeping/gopkg/src/gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream/conn.go:156 +0x1f2 created by gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream.(*Conn).GoClose /home/whyrusleeping/gopkg/src/gx/ipfs/QmZK81vcgMhpb2t7GNbozk7qzt6Rj4zFqitpvsWT9mduW8/go-peerstream/conn.go:131 +0xab ``` At the top, you can see that this goroutine (number 2306090) has been waiting to acquire a semaphore for 458 minutes. That seems bad. Looking at the rest of the trace, we see the exact line it's waiting on is line 47 of runtime/sema.go. That's not particularly helpful, so we move on. Next, we see that call was made by line 205 of yamux/session.go in the `Close` method of `yamux.Session`. This one appears to be the issue. Given that information, look for another goroutine that might be holding the semaphore in question in the rest of the stack dump. (If you need help doing this, ping and we'll stub this out.) There are a few different reasons that goroutines can be hung: - `semacquire` means we're waiting to take a lock or semaphore. - `select` means that the goroutine is hanging in a select statement and none of the cases are yielding anything. - `chan receive` and `chan send` are waiting for a channel to be received from or sent on, respectively. - `IO wait` generally means that we are waiting on a socket to read or write data, although it *can* mean we are waiting on a very slow filesystem. If you see any of those tags _without_ a `, X minutes` suffix, that generally means there isn't a problem -- you just caught that goroutine in the middle of a short wait for something. If the wait time is over a few minutes, that either means that goroutine doesn't do much, or something is pretty wrong. If you're seeing a lot of goroutines, consider using [stackparse](https://github.com/whyrusleeping/stackparse) to filter, sort, and summarize them. ### Analyzing the CPU Profile The go team wrote an [excellent article on profiling go programs](http://blog.golang.org/profiling-go-programs). If you've already gathered the above information, you can skip down to where they start talking about `go tool pprof`. My go-to method of analyzing these is to run the `web` command, which generates an SVG dotgraph and opens it in your browser. This is the quickest way to easily point out where the hot spots in the code are. ### Analyzing vars and memory statistics The output is JSON formatted and includes badger store statistics, the command line run, and the output from Go's [runtime.ReadMemStats](https://golang.org/pkg/runtime/#ReadMemStats). The [MemStats](https://golang.org/pkg/runtime/#MemStats) has useful information about memory allocation and garbage collection. ### Tracing Experimental tracing via OpenTelemetry suite of tools is available. See `tracing/doc.go` for more details. ### Other If you have any questions, or want us to analyze some weird kubo behavior, just let us know, and be sure to include all the profiling information mentioned at the top. ================================================ FILE: docs/delegated-routing.md ================================================ # Delegated Routing Notes - Status Date: 2025-12 > [!IMPORTANT] > Most users are best served by setting delegated HTTP router URLs in [`Routing.DelegatedRouters`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingdelegatedrouters) and `Routing.Type` to `auto` or `autoclient`, rather than using custom routing with `Routing.Routers` and `Routing.Methods` directly. > > The rest of this documentation describes experimental features intended only for researchers and advanced users. ---- # Custom Multi-Router Configuration (Experimental) - Start Date: 2022-08-15 - Related Issues: - https://github.com/ipfs/kubo/issues/9188 - https://github.com/ipfs/kubo/issues/9079 - https://github.com/ipfs/kubo/pull/9877 > [!CAUTION] > **`Routing.Type=custom` with `Routing.Routers` and `Routing.Methods` is EXPERIMENTAL.** > > This feature is provided for **research and testing purposes only**. It is **not suitable for production use**. > > - The configuration format and behavior may change without notice between Kubo releases. > - Bugs and regressions affecting custom routing may not be prioritized or fixed promptly. > - HTTP-only routing configurations (without DHT) cannot reliably provide content to the network (👉️ see [Limitations](#limitations) below). > > **For production deployments**, use `Routing.Type=auto` (default) or `Routing.Type=autoclient` with [`Routing.DelegatedRouters`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingdelegatedrouters). ## Motivation The actual routing implementation is not enough. Some users need to have more options when configuring the routing system. The new implementations should be able to: - [x] Be user-friendly and easy enough to configure, but also versatile - [x] Configurable Router execution order - [x] Delay some of the Router methods execution when they will be executed on parallel - [x] Configure which method of a giving router will be used - [x] Mark some router methods as mandatory to make the execution fails if that method fails ## Detailed design ### Configuration file description The `Routing` configuration section will contain the following keys: #### Type `Type` will be still in use to avoid complexity for the user that only wants to use Kubo with the default behavior. We are going to add a new type, `custom`, that will use the new router systems. `none` type will deactivate **all** routers, default dht and delegated ones. #### Routers `Routers` will be a key-value list of routers that will be available to use. The key is the router name and the value is all the needed configurations for that router. the `Type` will define the routing kind. The main router types will be `http` and `dht`, but we will implement two special routers used to execute a set of routers in parallel or sequentially: `parallel` router and `sequential` router. Depending on the routing type, it will use different parameters: ##### HTTP Params: - `"Endpoint"`: URL of HTTP server with endpoints that implement [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) protocol. ##### Amino DHT Params: - `"Mode"`: Mode used by the Amino DHT. Possible values: "server", "client", "auto" - `"AcceleratedDHTClient"`: Set to `true` if you want to use the experimentalDHT. - `"PublicIPNetwork"`: Set to `true` to create a `WAN` Amino DHT. Set to `false` to create a `LAN` DHT. ##### Parallel Params: - `Routers`: A list of routers that will be executed in parallel: - `Name:string`: Name of the router. It should be one of the previously added to `Routers` list. - `Timeout:duration`: Local timeout. It accepts strings compatible with Go `time.ParseDuration(string)`. Time will start counting when this specific router is called, and it will stop when the router returns, or we reach the specified timeout. - `ExecuteAfter:duration`: Providing this param will delay the execution of that router at the specified time. It accepts strings compatible with Go `time.ParseDuration(string)`. - `IgnoreErrors:bool`: It will specify if that router should be ignored if an error occurred. - `Timeout:duration`: Global timeout. It accepts strings compatible with Go `time.ParseDuration(string)`. ##### Sequential Params: - `Routers`: A list of routers that will be executed in order: - `Name:string`: Name of the router. It should be one of the previously added to `Routers` list. - `Timeout:duration`: Local timeout. It accepts strings compatible with Go `time.ParseDuration(string)`. Time will start counting when this specific router is called, and it will stop when the router returns, or we reach the specified timeout. - `IgnoreErrors:bool`: It will specify if that router should be ignored if an error occurred. - `Timeout:duration`: Global timeout. It accepts strings compatible with Go `time.ParseDuration(string)`. #### Methods `Methods:map` will define which routers will be executed per method. The key will be the name of the method: `"provide"`, `"find-providers"`, `"find-peers"`, `"put-ipns"`, `"get-ipns"`. All methods must be added to the list. This will make configuration discoverable giving good errors to the user if a method is missing. The value will contain: - `RouterName:string`: Name of the router. It should be one of the previously added to `Routers` list. #### Configuration file example: ```json "Routing": { "Type": "custom", "Routers": { "http-delegated": { "Type": "http", "Parameters": { "Endpoint": "https://delegated-ipfs.dev" // /routing/v1 (https://specs.ipfs.tech/routing/http-routing-v1/) } }, "dht-lan": { "Type": "dht", "Parameters": { "Mode": "server", "PublicIPNetwork": false, "AcceleratedDHTClient": false } }, "dht-wan": { "Type": "dht", "Parameters": { "Mode": "auto", "PublicIPNetwork": true, "AcceleratedDHTClient": false } }, "find-providers-router": { "Type": "parallel", "Parameters": { "Routers": [ { "RouterName": "dht-lan", "IgnoreErrors": true }, { "RouterName": "dht-wan" }, { "RouterName": "http-delegated" } ] } }, "provide-router": { "Type": "parallel", "Parameters": { "Routers": [ { "RouterName": "dht-lan", "IgnoreErrors": true }, { "RouterName": "dht-wan", "ExecuteAfter": "100ms", "Timeout": "100ms" }, { "RouterName": "http-delegated", "ExecuteAfter": "100ms" } ] } }, "get-ipns-router": { "Type": "sequential", "Parameters": { "Routers": [ { "RouterName": "dht-lan", "IgnoreErrors": true }, { "RouterName": "dht-wan", "Timeout": "300ms" }, { "RouterName": "http-delegated", "Timeout": "300ms" } ] } }, "put-ipns-router": { "Type": "parallel", "Parameters": { "Routers": [ { "RouterName": "dht-lan" }, { "RouterName": "dht-wan" }, { "RouterName": "http-delegated" } ] } } }, "Methods": { "find-providers": { "RouterName": "find-providers-router" }, "provide": { "RouterName": "provide-router" }, "get-ipns": { "RouterName": "get-ipns-router" }, "put-ipns": { "RouterName": "put-ipns-router" } } } ``` ### Error cases - If any of the routers fails, the output will be an error by default. - You can use `IgnoreErrors:true` to ignore errors for a specific router output - To avoid any error at the output, you must ignore all router errors. ### Implementation Details #### Methods All routers must implement the `routing.Routing` interface: ```go= type Routing interface { ContentRouting PeerRouting ValueStore Bootstrap(context.Context) error } ``` All methods involved: ```go= type Routing interface { Provide(context.Context, cid.Cid, bool) error FindProvidersAsync(context.Context, cid.Cid, int) <-chan peer.AddrInfo FindPeer(context.Context, peer.ID) (peer.AddrInfo, error) PutValue(context.Context, string, []byte, ...Option) error GetValue(context.Context, string, ...Option) ([]byte, error) SearchValue(context.Context, string, ...Option) (<-chan []byte, error) Bootstrap(context.Context) error } ``` We can configure which methods will be used per routing implementation. Methods names used in the configuration file will be: - `Provide`: `"provide"` - `FindProvidersAsync`: `"find-providers"` - `FindPeer`: `"find-peers"` - `PutValue`: `"put-ipns"` - `GetValue`, `SearchValue`: `"get-ipns"` - `Bootstrap`: It will be always executed when needed. #### Routers We need to implement the `parallel` and `sequential` routers and stop using `routinghelpers.Tiered` router implementation. Add cycle detection to avoid to user some headaches. Also we need to implement an internal router, that will define the router used per method. #### Other considerations - We need to refactor how DHT routers are created to be able to use and add any amount of custom DHT routers. - We need to add a new `custom` router type to be able to use the new routing system. - Bitswap WANT broadcasting is not included on this document, but it can be added in next iterations. - This document will live in docs/design-notes for historical reasons and future reference. ## Test fixtures As test fixtures we can add different use cases here and see how the configuration will look like. ### Mimic previous dual DHT config ```json "Routing": { "Type": "custom", "Routers": { "dht-lan": { "Type": "dht", "Parameters": { "Mode": "server", "PublicIPNetwork": false } }, "dht-wan": { "Type": "dht", "Parameters": { "Mode": "auto", "PublicIPNetwork": true } }, "parallel-dht-strict": { "Type": "parallel", "Parameters": { "Routers": [ { "RouterName": "dht-lan" }, { "RouterName": "dht-wan" } ] } }, "parallel-dht": { "Type": "parallel", "Parameters": { "Routers": [ { "RouterName": "dht-lan", "IgnoreError": true }, { "RouterName": "dht-wan" } ] } } }, "Methods": { "provide": { "RouterName": "dht-wan" }, "find-providers": { "RouterName": "parallel-dht-strict" }, "find-peers": { "RouterName": "parallel-dht-strict" }, "get-ipns": { "RouterName": "parallel-dht" }, "put-ipns": { "RouterName": "parallel-dht" } } } ``` ### Compatibility ~~We need to create a config migration using [fs-repo-migrations](https://github.com/ipfs/fs-repo-migrations). We should remove the `Routing.Type` param and add the configuration specified [previously](#Mimic-previous-dual-DHT-config).~~ We don't need to create any config migration! To avoid to the users the hassle of understanding how the new routing system works, we are going to keep the old behavior. We will add the Type `custom` to make available the new Routing system. ### Security No new security implications or considerations were found. ### Alternatives I got ideas from all of the following links to create this design document: - https://github.com/ipfs/kubo/issues/9079#issuecomment-1211288268 - https://github.com/ipfs/kubo/issues/9157 - https://github.com/ipfs/kubo/issues/9079#issuecomment-1205000253 - https://www.notion.so/pl-strflt/Delegated-Routing-Thoughts-very-very-WIP-0543bc51b1bd4d63a061b0f28e195d38 - https://gist.github.com/guseggert/effa027ff4cbadd7f67598efb6704d12 ### Limitations #### HTTP-only routing cannot reliably provide content Configurations that use only HTTP routers (without any DHT router) are unable to reliably announce content (provider records) to the network. This limitation exists because: 1. **No standardized HTTP API for providing**: The [Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) spec only defines read operations (`GET /routing/v1/providers/{cid}`). The write operation (`PUT /routing/v1/providers`) was never standardized. 2. **Legacy experimental API**: The only available HTTP providing mechanism is an undocumented `PUT /routing/v1/providers` request format called `ProvideBitswap`, which is a historical experiment. See [IPIP-526](https://github.com/ipfs/specs/pull/526) for ongoing discussion about formalizing HTTP-based provider announcements. 3. **Provider system integration**: Kubo's default provider system (`Provide.DHT.SweepEnabled=true` since v0.38) is designed for DHT-based providing. When no DHT is configured, the provider system may silently skip HTTP routers or behave unexpectedly. **Workarounds for testing:** If you need to test HTTP providing, you can try: - Setting `Provide.DHT.SweepEnabled=false` to use the legacy provider system - Including at least one DHT router in your custom configuration alongside HTTP routers These workarounds are not guaranteed to work across Kubo versions and should not be relied upon for production use. ### Copyright Copyright and related rights waived via [CC0](https://creativecommons.org/publicdomain/zero/1.0/). ================================================ FILE: docs/developer-certificate-of-origin ================================================ Developer Certificate of Origin Version 1.1 Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 1 Letterman Drive Suite D4700 San Francisco, CA, 94129 Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. ================================================ FILE: docs/developer-guide.md ================================================ # Developer Guide By the end of this guide, you will be able to: - Build Kubo from source - Run the test suites - Make and verify code changes This guide covers the local development workflow. For user documentation, see [docs.ipfs.tech](https://docs.ipfs.tech/). ## Table of Contents - [Prerequisites](#prerequisites) - [Quick Start](#quick-start) - [Building](#building) - [Running Tests](#running-tests) - [Running the Linter](#running-the-linter) - [Common Development Tasks](#common-development-tasks) - [Code Organization](#code-organization) - [Architecture](#architecture) - [Troubleshooting](#troubleshooting) - [Development Dependencies](#development-dependencies) - [Further Reading](#further-reading) ## Prerequisites Before you begin, ensure you have: - **Go** - see `go.mod` for the minimum required version - **Git** - **GNU Make** - **GCC** (optional) - required for CGO (Go's C interop); without it, build with `CGO_ENABLED=0` ## Quick Start ```bash git clone https://github.com/ipfs/kubo.git cd kubo make build ./cmd/ipfs/ipfs version ``` You should see output like: ``` ipfs version 0.34.0-dev ``` The binary is built to `cmd/ipfs/ipfs`. To install it system-wide: ```bash make install ``` This installs the binary to `$GOPATH/bin`. ## Building | Command | Description | |---------|-------------| | `make build` | build the `ipfs` binary to `cmd/ipfs/ipfs` | | `make install` | install to `$GOPATH/bin` | | `make nofuse` | build without FUSE support | | `make build CGO_ENABLED=0` | build without CGO (no C compiler needed) | For Windows-specific instructions, see [windows.md](windows.md). ## Running Tests Kubo has two types of tests: - **Unit tests** - test individual packages in isolation. Fast and don't require a running daemon. - **End-to-end tests** - spawn real `ipfs` nodes, run actual CLI commands, and test the full system. Slower but catch integration issues. Note that `go test ./...` runs both unit and end-to-end tests. Use `make test` to run all tests. CI runs unit and end-to-end tests in separate jobs for faster feedback. For end-to-end tests, Kubo has two suites: - **`test/cli`** - modern Go-based test harness that spawns real `ipfs` nodes and runs actual CLI commands. All new tests should be added here. - **`test/sharness`** - legacy bash-based tests. We are slowly migrating these to `test/cli`. When modifying tests: cosmetic changes to `test/sharness` are fine, but if significant rewrites are needed, remove the outdated sharness test and add a modern one to `test/cli` instead. ### Before Running Tests **Environment requirements**: some legacy tests expect default ports (8080, 5001, 4001) to be free and no mDNS (local network discovery) Kubo service on the LAN. Tests may fail if you have a local Kubo instance running. Before running the full test suite, stop any running `ipfs daemon`. Two critical setup steps: 1. **Rebuild after code changes**: if you modify any `.go` files outside of `test/`, you must run `make build` before running integration tests. 2. **Set environment variables**: integration tests use the `ipfs` binary from `PATH` and need an isolated `IPFS_PATH`. Run these commands from the repository root: ```bash export PATH="$PWD/cmd/ipfs:$PATH" export IPFS_PATH="$(mktemp -d)" ``` ### Unit Tests ```bash go test ./... ``` ### CLI Integration Tests (`test/cli`) These are Go-based integration tests that invoke the `ipfs` CLI. Instead of running the entire test suite, you can run a specific test to get faster feedback during development. Run a specific test (recommended during development): ```bash go test ./test/cli/... -run TestAdd -v ``` Run all CLI tests: ```bash go test ./test/cli/... ``` Run a specific test: ```bash go test ./test/cli/... -run TestAdd ``` Run with verbose output: ```bash go test ./test/cli/... -v ``` **Common error**: "version (16) is lower than repos (17)" means your `PATH` points to an old binary. Check `which ipfs` and rebuild with `make build`. ### Sharness Tests (`test/sharness`) Shell-based integration tests using [sharness](https://github.com/chriscool/sharness) (a portable shell testing framework). ```bash cd test/sharness ``` Run a specific test: ```bash timeout 60s ./t0080-repo.sh ``` Run with verbose output (this disables automatic cleanup): ```bash ./t0080-repo.sh -v ``` **Cleanup**: the `-v` flag disables automatic cleanup. Before re-running tests, kill any dangling `ipfs daemon` processes: ```bash pkill -f "ipfs daemon" ``` ### Full Test Suite ```bash make test # run all tests make test_short # run shorter test suite ``` ## Running the Linter Run the linter using the Makefile target (not `golangci-lint` directly): ```bash make -O test_go_lint ``` ## Common Development Tasks ### Modifying CLI Commands After editing help text in `core/commands/`, verify the output width: ```bash go test ./test/cli/... -run TestCommandDocsWidth ``` ### Updating Dependencies Use the Makefile target (not `go mod tidy` directly): ```bash make mod_tidy ``` ### Editing the Changelog When modifying `docs/changelogs/`: - update the Table of Contents when adding sections - add user-facing changes to the Highlights section (the Changelog section is auto-generated) ### Running the Daemon Always run the daemon with a timeout or shut it down promptly. With timeout: ```bash timeout 60s ipfs daemon ``` Or shut down via API: ```bash ipfs shutdown ``` For multi-step experiments, store `IPFS_PATH` in a file to ensure consistency. ## Code Organization | Directory | Description | |-----------|-------------| | `cmd/ipfs/` | CLI entry point and binary | | `core/` | core IPFS node implementation | | `core/commands/` | CLI command definitions | | `core/coreapi/` | Go API implementation | | `client/rpc/` | HTTP RPC client | | `plugin/` | plugin system | | `repo/` | repository management | | `test/cli/` | Go-based CLI integration tests | | `test/sharness/` | legacy shell-based integration tests | | `docs/` | documentation | Key external dependencies: - [go-libp2p](https://github.com/libp2p/go-libp2p) - networking stack - [go-libp2p-kad-dht](https://github.com/libp2p/go-libp2p-kad-dht) - distributed hash table - [boxo](https://github.com/ipfs/boxo) - IPFS SDK (including Bitswap, the data exchange engine) For a deep dive into how code flows through Kubo, see [The `Add` command demystified](add-code-flow.md). ## Architecture **Map of Implemented Subsystems** ([editable source](https://docs.google.com/drawings/d/1OVpBT2q-NtSJqlPX3buvjYhOnWfdzb85YEsM_njesME/edit)): **CLI, HTTP-API, Core Diagram**: ![](./cli-http-api-core-diagram.png) ## Troubleshooting ### "version (N) is lower than repos (M)" Error This means the `ipfs` binary in your `PATH` is older than expected. Check which binary is being used: ```bash which ipfs ``` Rebuild and verify PATH: ```bash make build export PATH="$PWD/cmd/ipfs:$PATH" ./cmd/ipfs/ipfs version ``` ### FUSE Issues If you don't need FUSE support, build without it: ```bash make nofuse ``` Or set the `TEST_FUSE=0` environment variable when running tests. ### Build Fails with "No such file: stdlib.h" You're missing a C compiler. Either install GCC or build without CGO: ```bash make build CGO_ENABLED=0 ``` ## Development Dependencies If you make changes to the protocol buffers, you will need to install the [protoc compiler](https://github.com/google/protobuf). ## Further Reading - [The `Add` command demystified](add-code-flow.md) - deep dive into code flow - [Configuration reference](config.md) - [Performance debugging](debug-guide.md) - [Experimental features](experimental-features.md) - [Release process](releases.md) - [Contributing guidelines](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) ## Source Code The complete source code is at [github.com/ipfs/kubo](https://github.com/ipfs/kubo). ================================================ FILE: docs/environment-variables.md ================================================ # Kubo environment variables - [Variables](#variables) - [`IPFS_PATH`](#ipfs_path) - [`IPFS_LOGGING`](#ipfs_logging) - [`IPFS_LOGGING_FMT`](#ipfs_logging_fmt) - [`GOLOG_LOG_LEVEL`](#golog_log_level) - [`GOLOG_LOG_FMT`](#golog_log_fmt) - [`GOLOG_FILE`](#golog_file) - [`GOLOG_OUTPUT`](#golog_output) - [`GOLOG_TRACING_FILE`](#golog_tracing_file) - [`IPFS_FUSE_DEBUG`](#ipfs_fuse_debug) - [`YAMUX_DEBUG`](#yamux_debug) - [`IPFS_FD_MAX`](#ipfs_fd_max) - [`IPFS_DIST_PATH`](#ipfs_dist_path) - [`IPFS_NS_MAP`](#ipfs_ns_map) - [`IPFS_HTTP_ROUTERS`](#ipfs_http_routers) - [`IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS`](#ipfs_http_routers_filter_protocols) - [`IPFS_CONTENT_BLOCKING_DISABLE`](#ipfs_content_blocking_disable) - [`IPFS_WAIT_REPO_LOCK`](#ipfs_wait_repo_lock) - [`IPFS_TELEMETRY`](#ipfs_telemetry) - [`LIBP2P_TCP_REUSEPORT`](#libp2p_tcp_reuseport) - [`LIBP2P_TCP_MUX`](#libp2p_tcp_mux) - [`LIBP2P_MUX_PREFS`](#libp2p_mux_prefs) - [`LIBP2P_RCMGR`](#libp2p_rcmgr) - [`LIBP2P_DEBUG_RCMGR`](#libp2p_debug_rcmgr) - [`LIBP2P_SWARM_FD_LIMIT`](#libp2p_swarm_fd_limit) - [Tracing](#tracing) # Variables ## `IPFS_PATH` Sets the location of the IPFS repo (where the config, blocks, etc. are stored). Default: ~/.ipfs ## `IPFS_LOGGING` Specifies the log level for Kubo. `IPFS_LOGGING` is a deprecated alias for the `GOLOG_LOG_LEVEL` environment variable. See below. ## `IPFS_LOGGING_FMT` Specifies the log message format. `IPFS_LOGGING_FMT` is a deprecated alias for the `GOLOG_LOG_FMT` environment variable. See below. ## `GOLOG_LOG_LEVEL` Specifies the log-level, both globally and on a per-subsystem basis. Level can be one of: * `debug` * `info` * `warn` * `error` * `dpanic` * `panic` * `fatal` Per-subsystem levels can be specified with `subsystem=level`. One global level and one or more per-subsystem levels can be specified by separating them with commas. Default: `error` Example: ```console GOLOG_LOG_LEVEL="error,core/server=debug" ipfs daemon ``` Logging can also be configured at runtime, both globally and on a per-subsystem basis, with the `ipfs log` command. ## `GOLOG_LOG_FMT` Specifies the log message format. It supports the following values: - `color` -- human readable, colorized (ANSI) output - `nocolor` -- human readable, plain-text output. - `json` -- structured JSON. For example, to log structured JSON (for easier parsing): ```bash export GOLOG_LOG_FMT="json" ``` The logging format defaults to `color` when the output is a terminal, and `nocolor` otherwise. ## `GOLOG_FILE` Sets the file to which Kubo logs. By default, Kubo logs to standard error. ## `GOLOG_OUTPUT` When stderr and/or stdout options are configured or specified by the `GOLOG_OUTPUT` environ variable, log only to the output(s) specified. For example: - `GOLOG_OUTPUT="stderr"` logs only to stderr - `GOLOG_OUTPUT="stdout"` logs only to stdout - `GOLOG_OUTPUT="stderr+stdout"` logs to both stderr and stdout ## `GOLOG_TRACING_FILE` Sets the file to which Kubo sends tracing events. By default, tracing is disabled. This log can be read at runtime (without writing it to a file) using the `ipfs log tail` command. Warning: Enabling tracing will likely affect performance. ## `IPFS_FUSE_DEBUG` If SET, enables fuse debug logging. Default: false ## `YAMUX_DEBUG` If SET, enables debug logging for the yamux stream muxer. Default: false ## `IPFS_FD_MAX` Sets the file descriptor limit for Kubo. If Kubo fails to set the file descriptor limit, it will log an error. Defaults: 2048 ## `IPFS_DIST_PATH` IPFS Content Path from which Kubo fetches repo migrations (when the daemon is launched with the `--migrate` flag). Default: `/ipfs/` (the exact path is hardcoded in `migrations.CurrentIpfsDist`, depends on the IPFS version) ## `IPFS_NS_MAP` Adds static namesys records for deterministic tests and debugging. Useful for testing things like DNSLink without real DNS lookup. Example: ```console $ IPFS_NS_MAP="dnslink-test1.example.com:/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am,dnslink-test2.example.com:/ipns/dnslink-test1.example.com" ipfs daemon ... $ ipfs resolve -r /ipns/dnslink-test2.example.com /ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am ``` ## `IPFS_HTTP_ROUTERS` Overrides AutoConf and all other HTTP routers when set. When `Routing.Type=auto`, this environment variable takes precedence over both AutoConf-provided endpoints and any manually configured delegated routers. The value should be a space or comma-separated list of HTTP routing endpoint URLs. This is useful for: - Testing and debugging in offline contexts - Overriding AutoConf endpoints temporarily - Using custom or private HTTP routing services Example: ```console $ ipfs config Routing.Type auto $ IPFS_HTTP_ROUTERS="http://127.0.0.1:7423" ipfs daemon ``` The above will replace all AutoConf endpoints with a single local one, allowing for inspection/debug of HTTP requests sent by Kubo via `while true ; do nc -l 7423; done` or more advanced tools like [mitmproxy](https://docs.mitmproxy.org/stable/#mitmproxy). When not set, Kubo uses endpoints from AutoConf (when enabled) or manually configured `Routing.DelegatedRouters`. ## `IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS` Overrides values passed with `filter-protocols` parameter defined in IPIP-484. Value is space-separated. ```console $ IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS="unknown transport-bitswap transport-foo" ipfs daemon ``` Default: `config.DefaultHTTPRoutersFilterProtocols` ## `IPFS_CONTENT_BLOCKING_DISABLE` Disables the content-blocking subsystem. No denylists will be watched and no content will be blocked. ## `IPFS_WAIT_REPO_LOCK` Specifies the amount of time to wait for the repo lock. Set the value of this variable to a string that can be [parsed](https://pkg.go.dev/time@go1.24.3#ParseDuration) as a golang `time.Duration`. For example: ``` IPFS_WAIT_REPO_LOCK="15s" ``` If the lock cannot be acquired because someone else has the lock, and `IPFS_WAIT_REPO_LOCK` is set to a valid value, then acquiring the lock is retried every second until the lock is acquired or the specified wait time has elapsed. ## `IPFS_TELEMETRY` Controls the behavior of the [telemetry plugin](telemetry.md). Valid values are: - `on`: Enables telemetry. - `off`: Disables telemetry. - `auto`: Like `on`, but logs an informative message about telemetry and gives user 15 minutes to opt-out before first collection. Used automatically on first run and when `IPFS_TELEMETRY` is not set. The mode can also be set in the config file under `Plugins.Plugins.telemetry.Config.Mode`. Example: ```bash export IPFS_TELEMETRY="off" ``` ## `LIBP2P_TCP_REUSEPORT` Kubo tries to reuse the same source port for all connections to improve NAT traversal. If this is an issue, you can disable it by setting `LIBP2P_TCP_REUSEPORT` to false. Default: `true` ## `LIBP2P_TCP_MUX` By default Kubo tries to reuse the same listener port for raw TCP and WebSockets transports via experimental `libp2p.ShareTCPListener()` feature introduced in [go-libp2p#2984](https://github.com/libp2p/go-libp2p/pull/2984). If this is an issue, you can disable it by setting `LIBP2P_TCP_MUX` to `false` and use separate ports for each TCP transport. > [!CAUTION] > This configuration option may be removed once `libp2p.ShareTCPListener()` becomes default in go-libp2p. Default: `true` ## `LIBP2P_MUX_PREFS` Deprecated: Use the `Swarm.Transports.Multiplexers` config field. Tells Kubo which multiplexers to use in which order. Default: "/yamux/1.0.0 /mplex/6.7.0" ## `LIBP2P_RCMGR` Forces [libp2p Network Resource Manager](https://github.com/libp2p/go-libp2p-resource-manager#readme) to be enabled (`1`) or disabled (`0`). When set, overrides [`Swarm.ResourceMgr.Enabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#swarmresourcemgrenabled) from the config. Default: use config (not set) ## `LIBP2P_DEBUG_RCMGR` Enables tracing of [libp2p Network Resource Manager](https://github.com/libp2p/go-libp2p-resource-manager#readme) and outputs it to `rcmgr.json.gz` Default: disabled (not set) ## `LIBP2P_SWARM_FD_LIMIT` This variable controls the number of concurrent outbound dials (except dials to relay addresses which have their own limiting logic). Reducing it slows down connection ballooning but might affect performance negatively. Default: [160](https://github.com/libp2p/go-libp2p/blob/master/p2p/net/swarm/swarm_dial.go#L91) (not set) # Tracing For tracing configuration, please check: https://github.com/ipfs/boxo/blob/main/docs/tracing.md ================================================ FILE: docs/examples/kubo-as-a-library/README.md ================================================ # Use Kubo (go-ipfs) as a library to spawn a node and add a file > Note: if you are trying to customize or extend Kubo, you should read the [Customizing Kubo](../../customizing.md) doc By the end of this tutorial, you will learn how to: - Spawn an IPFS node that runs in process (no separate daemon process) - Create an IPFS repo - Add files and directories to IPFS - Retrieve those files and directories using ``cat`` and ``get`` - Connect to other nodes in the network - Retrieve a file that only exists on the network - The difference between a node in DHT client mode and full DHT mode All of this using only golang! In order to complete this tutorial, you will need: - golang installed on your machine. See how at https://golang.org/doc/install - git installed on your machine (so that go can download the repo and the necessary dependencies). See how at https://git-scm.com/downloads - IPFS Desktop (for convenience) installed and running on your machine. See how at https://github.com/ipfs-shipyard/ipfs-desktop#ipfs-desktop **Disclaimer**: The example code is quite large (more than 300 lines of code) and it has been a great way to understand the scope of the [go-ipfs Core API](https://godoc.org/github.com/ipfs/interface-go-ipfs-core), and how it can be improved to further the user experience. You can expect to be able to come back to this example in the future and see how the number of lines of code have decreased and how the example have become simpler, making other go-ipfs programs simpler as well. ## Getting started **Note:** Make sure you have [![](https://img.shields.io/badge/go-%3E%3D1.13.0-blue.svg?style=flat-square)](https://golang.org/dl/) installed. Download Kubo and jump into the example folder: ```console $ git clone https://github.com/ipfs/kubo.git $ cd kubo/docs/examples/kubo-as-a-library ``` ## Running the example as-is To run the example, simply do: ```console $ go run main.go ``` You should see the following as output: ``` -- Getting an IPFS node running -- Spawning Kubo node on a temporary repo IPFS node is running -- Adding and getting back files & directories -- Added file to IPFS with CID /ipfs/QmV9tSDx9UiPeWExXEeH6aoDvmihvx6jD5eLb4jbTaKGps Added directory to IPFS with CID /ipfs/QmdQdu1fkaAUokmkfpWrmPHK78F9Eo9K2nnuWuizUjmhyn Got file back from IPFS (IPFS path: /ipfs/QmV9tSDx9UiPeWExXEeH6aoDvmihvx6jD5eLb4jbTaKGps) and wrote it to ./example-folder/QmV9tSDx9UiPeWExXEeH6aoDvmihvx6jD5eLb4jbTaKGps Got directory back from IPFS (IPFS path: /ipfs/QmdQdu1fkaAUokmkfpWrmPHK78F9Eo9K2nnuWuizUjmhyn) and wrote it to ./example-folder/QmdQdu1fkaAUokmkfpWrmPHK78F9Eo9K2nnuWuizUjmhyn -- Going to connect to a few nodes in the Network as bootstrappers -- Fetching a file from the network with CID QmUaoioqU7bxezBQZkUcgcSyokatMY71sxsALxQmRRrHrj Wrote the file to ./example-folder/QmUaoioqU7bxezBQZkUcgcSyokatMY71sxsALxQmRRrHrj All done! You just finalized your first tutorial on how to use Kubo as a library ``` ## Understanding the example In this example, we add a file and a directory with files; we get them back from IPFS; and then we use the IPFS network to fetch a file that we didn't have in our machines before. Each section below has links to lines of code in the file [main.go](./main.go). The code itself will have comments explaining what is happening for you. ### The `func main() {}` The [main function](./main.go#L202-L331) is where the magic starts, and it is the best place to follow the path of what is happening in the tutorial. ### Part 1: Getting an IPFS node running To get [get a node running](./main.go#L218-L223) as an [ephemeral node](./main.go#L114-L128) (that will cease to exist when the run ends), you will need to: - [Prepare and set up the plugins](./main.go#L30-L47) - [Create an IPFS repo](./main.go#L49-L68) - [Construct the IPFS node instance itself](./main.go#L72-L96) As soon as you construct the IPFS node instance, the node will be running. ### Part 2: Adding a file and a directory to IPFS - [Prepare the file to be added to IPFS](./main.go#L166-L184) - [Add the file to IPFS](./main.go#L240-L243) - [Prepare the directory to be added to IPFS](./main.go#L186-L198) - [Add the directory to IPFS](./main.go#L252-L255) ### Part 3: Getting the file and directory you added back - [Get the file back](./main.go#L265-L268) - [Write the file to your local filesystem](./main.go#L270-L273) - [Get the directory back](./main.go#L277-L280) - [Write the directory to your local filesystem](./main.go#L282-L285) ### Part 4: Getting a file from the IPFS network - [Connect to nodes in the network](./main.go#L293-L310) - [Get the file from the network](./main.go#L318-L321) - [Write the file to your local filesystem](./main.go#L323-L326) ### Bonus: Spawn a daemon on your existing IPFS repo (on the default path ~/.ipfs) As a bonus, you can also find lines that show you how to spawn a node over your default path (~/.ipfs) in case you had already started a node there before. To try it: - [Comment these lines](./main.go#L219-L223) - [Uncomment these lines](./main.go#L209-L216) ## Voilá! You are now a Kubo hacker You've learned how to spawn a Kubo node using the Core API. There are many more [methods to experiment next](https://godoc.org/github.com/ipfs/interface-go-ipfs-core). Happy hacking! ================================================ FILE: docs/examples/kubo-as-a-library/go.mod ================================================ module github.com/ipfs/kubo/examples/kubo-as-a-library go 1.25.7 // Used to keep this in sync with the current version of kubo. You should remove // this if you copy this example. replace github.com/ipfs/kubo => ./../../.. require ( github.com/ipfs/boxo v0.37.1-0.20260317235537-851246983422 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.48.0 github.com/multiformats/go-multiaddr v0.16.1 ) require ( bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5 // indirect filippo.io/keygen v0.0.0-20260114151900-8e2790ea4c5b // indirect github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/DataDog/zstd v1.5.7 // indirect github.com/Jorropo/jsync v1.0.1 // indirect github.com/RaduBerinde/axisds v0.1.0 // indirect github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/caddyserver/certmagic v0.23.0 // indirect github.com/caddyserver/zerossl v0.1.3 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/ceramicnetwork/go-dag-jose v0.1.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect github.com/cockroachdb/errors v1.11.3 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/pebble/v2 v2.1.4 // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf // indirect github.com/cskr/pubsub v1.0.2 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect github.com/dgraph-io/badger v1.6.2 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dunglas/httpsfv v1.1.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/filecoin-project/go-clock v0.1.0 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gammazero/chanqueue v1.1.2 // indirect github.com/gammazero/deque v1.2.1 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.2.5 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect github.com/guillaumemichel/reservedpool v0.3.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/ipfs-shipyard/nopfs v0.0.14 // indirect github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect github.com/ipfs/go-cidutil v0.1.1 // indirect github.com/ipfs/go-datastore v0.9.1 // indirect github.com/ipfs/go-ds-badger v0.3.4 // indirect github.com/ipfs/go-ds-flatfs v0.6.0 // indirect github.com/ipfs/go-ds-leveldb v0.5.2 // indirect github.com/ipfs/go-ds-measure v0.2.2 // indirect github.com/ipfs/go-ds-pebble v0.5.9 // indirect github.com/ipfs/go-dsqueue v0.2.0 // indirect github.com/ipfs/go-fs-lock v0.1.1 // indirect github.com/ipfs/go-ipfs-cmds v0.16.0 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect github.com/ipfs/go-ipfs-pq v0.0.4 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect github.com/ipfs/go-ipld-cbor v0.2.1 // indirect github.com/ipfs/go-ipld-format v0.6.3 // indirect github.com/ipfs/go-ipld-git v0.1.1 // indirect github.com/ipfs/go-ipld-legacy v0.3.0 // indirect github.com/ipfs/go-libdht v0.5.0 // indirect github.com/ipfs/go-log/v2 v2.9.1 // indirect github.com/ipfs/go-metrics-interface v0.3.0 // indirect github.com/ipfs/go-peertaskqueue v0.8.3 // indirect github.com/ipfs/go-test v0.2.3 // indirect github.com/ipfs/go-unixfsnode v1.10.3 // indirect github.com/ipld/go-car/v2 v2.16.0 // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect github.com/ipld/go-ipld-prime v0.22.0 // indirect github.com/ipshipyard/p2p-forge v0.7.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/koron/go-ssdp v0.0.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/libdns/libdns v1.0.0-beta.1 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-doh-resolver v0.5.0 // indirect github.com/libp2p/go-flow-metrics v0.3.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect github.com/libp2p/go-libp2p-kad-dht v0.39.0 // indirect github.com/libp2p/go-libp2p-kbucket v0.8.0 // indirect github.com/libp2p/go-libp2p-pubsub v0.15.0 // indirect github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect github.com/libp2p/go-libp2p-record v0.3.1 // indirect github.com/libp2p/go-libp2p-routing-helpers v0.7.5 // indirect github.com/libp2p/go-libp2p-xor v0.1.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-netroute v0.4.0 // indirect github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v5 v5.0.1 // indirect github.com/libp2p/zeroconf/v2 v2.2.0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mholt/acmez/v3 v3.1.2 // indirect github.com/miekg/dns v1.1.72 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr-dns v0.5.0 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multicodec v0.10.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.6.1 // indirect github.com/multiformats/go-varint v0.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pion/datachannel v1.5.10 // indirect github.com/pion/dtls/v3 v3.1.2 // indirect github.com/pion/ice/v4 v4.0.10 // indirect github.com/pion/interceptor v0.1.40 // indirect github.com/pion/logging v0.2.4 // indirect github.com/pion/mdns/v2 v2.0.7 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/rtcp v1.2.16 // indirect github.com/pion/rtp v1.8.19 // indirect github.com/pion/sctp v1.8.39 // indirect github.com/pion/sdp/v3 v3.0.18 // indirect github.com/pion/srtp/v3 v3.0.6 // indirect github.com/pion/stun/v3 v3.1.1 // indirect github.com/pion/transport/v3 v3.0.7 // indirect github.com/pion/transport/v4 v4.0.1 // indirect github.com/pion/turn/v4 v4.0.2 // indirect github.com/pion/webrtc/v4 v4.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.20.1 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect github.com/quic-go/webtransport-go v0.10.0 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.3.1 // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect github.com/wlynxg/anet v0.0.5 // indirect github.com/zeebo/blake3 v0.2.4 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect go.opentelemetry.io/otel v1.42.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.40.0 // indirect go.opentelemetry.io/otel/metric v1.42.0 // indirect go.opentelemetry.io/otel/sdk v1.40.0 // indirect go.opentelemetry.io/otel/trace v1.42.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/dig v1.19.0 // indirect go.uber.org/fx v1.24.0 // indirect go.uber.org/mock v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.uber.org/zap/exp v0.3.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 // indirect golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c // indirect golang.org/x/text v0.35.0 // indirect golang.org/x/time v0.12.0 // indirect golang.org/x/tools v0.43.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gonum.org/v1/gonum v0.17.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/grpc v1.78.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.4.1 // indirect ) ================================================ FILE: docs/examples/kubo-as-a-library/go.sum ================================================ bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc h1:utDghgcjE8u+EBjHOgYT+dJPcnDF05KqWMBcjuJy510= bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5 h1:JA0fFr+kxpqTdxR9LOBiTWpGNchqmkcsgmdeJZRclZ0= filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI= filippo.io/keygen v0.0.0-20260114151900-8e2790ea4c5b h1:REI1FbdW71yO56Are4XAxD+OS/e+BQsB3gE4mZRQEXY= filippo.io/keygen v0.0.0-20260114151900-8e2790ea4c5b/go.mod h1:9nnw1SlYHYuPSo/3wjQzNjSbeHlq2NsKo5iEtfJPWP0= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/RaduBerinde/axisds v0.1.0 h1:YItk/RmU5nvlsv/awo2Fjx97Mfpt4JfgtEVAGPrLdz8= github.com/RaduBerinde/axisds v0.1.0/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y= github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk= github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM= github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/caddyserver/certmagic v0.23.0 h1:CfpZ/50jMfG4+1J/u2LV6piJq4HOfO6ppOnOf7DkFEU= github.com/caddyserver/certmagic v0.23.0/go.mod h1:9mEZIWqqWoI+Gf+4Trh04MOVPD0tGSxtqsxg87hAIH4= github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA= github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4= github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3 h1:oe6fCvaEpkhyW3qAicT0TnGtyht/UrgvOwMcEgLb7Aw= github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3/go.mod h1:qdP0gaj0QtgX2RUZhnlVrceJ+Qln8aSlDyJwelLLFeM= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/ceramicnetwork/go-dag-jose v0.1.1 h1:7pObs22egc14vSS3AfCFfS1VmaL4lQUsAK7OGC3PlKk= github.com/ceramicnetwork/go-dag-jose v0.1.1/go.mod h1:8ptnYwY2Z2y/s5oJnNBn/UCxLg6CpramNJ2ZXF/5aNY= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= github.com/cockroachdb/pebble/v2 v2.1.4 h1:j9wPgMDbkErFdAKYFGhsoCcvzcjR+6zrJ4jhKtJ6bOk= github.com/cockroachdb/pebble/v2 v2.1.4/go.mod h1:Reo1RTniv1UjVTAu/Fv74y5i3kJ5gmVrPhO9UtFiKn8= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b h1:VXvSNzmr8hMj8XTuY0PT9Ane9qZGul/p67vGYwl9BFI= github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf h1:dwGgBWn84wUS1pVikGiruW+x5XM4amhjaZO20vCjay4= github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dunglas/httpsfv v1.1.0 h1:Jw76nAyKWKZKFrpMMcL76y35tOpYHqQPzHQiwDvpe54= github.com/dunglas/httpsfv v1.1.0/go.mod h1:zID2mqw9mFsnt7YC3vYQ9/cjq30q41W+1AnDwH8TiMg= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs= github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gammazero/chanqueue v1.1.2 h1:dZEsxlyANZMyeTRemABqZF8QM9BnE4NBI43Oh3y5fIU= github.com/gammazero/chanqueue v1.1.2/go.mod h1:XDN1X/jjAbmSceNFOQbtKToeSkxtdVdpKu90LiEdBEE= github.com/gammazero/deque v1.2.1 h1:9fnQVFCCZ9/NOc7ccTNqzoKd1tCWOqeI05/lPqFPMGQ= github.com/gammazero/deque v1.2.1/go.mod h1:5nSFkzVm+afG9+gy0VIowlqVAW4N8zNcMne+CMQVD2g= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 h1:r5GgOLGbza2wVHRzK7aAj6lWZjfbAwiu/RDCVOKjRyM= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e h1:4bw4WeyTYPp0smaXiJZCNnLrvVBqirQVreixayXezGc= github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII= github.com/guillaumemichel/reservedpool v0.3.0 h1:eqqO/QvTllLBrit7LVtVJBqw4cD0WdV9ajUe7WNTajw= github.com/guillaumemichel/reservedpool v0.3.0/go.mod h1:sXSDIaef81TFdAJglsCFCMfgF5E5Z5xK1tFhjDhvbUc= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs-shipyard/nopfs v0.0.14 h1:HFepJt/MxhZ3/GsLZkkAPzIPdNYKaLO1Qb7YmPbWIKk= github.com/ipfs-shipyard/nopfs v0.0.14/go.mod h1:mQyd0BElYI2gB/kq/Oue97obP4B3os4eBmgfPZ+hnrE= github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcdHUd7SDsUOY= github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.37.1-0.20260317235537-851246983422 h1:yY3ot/DU1bqTzHDBARACM76Tbx9s4xzcRbzifG1e/es= github.com/ipfs/boxo v0.37.1-0.20260317235537-851246983422/go.mod h1:8yyiRn54F2CsW13n0zwXEPrVsZix/gFj9SYIRYMZ6KE= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= github.com/ipfs/go-block-format v0.2.3/go.mod h1:WJaQmPAKhD3LspLixqlqNFxiZ3BZ3xgqxxoSR/76pnA= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.6.0 h1:DlOReBV1xhHBhhfy/gBNNTSyfOM6rLiIx9J7A4DGf30= github.com/ipfs/go-cid v0.6.0/go.mod h1:NC4kS1LZjzfhK40UGmpXv5/qD2kcMzACYJNntCUiDhQ= github.com/ipfs/go-cidutil v0.1.1 h1:COuby6H8C2ml0alvHYX3WdbFM4F07YtbY0UlT5j+sgI= github.com/ipfs/go-cidutil v0.1.1/go.mod h1:SCoUftGEUgoXe5Hjeyw5CiLZF8cwYn/TbtpFQXJCP6k= github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.9.1 h1:67Po2epre/o0UxrmkzdS9ZTe2GFGODgTd2odx8Wh6Yo= github.com/ipfs/go-datastore v0.9.1/go.mod h1:zi07Nvrpq1bQwSkEnx3bfjz+SQZbdbWyCNvyxMh9pN0= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.3.4 h1:MmqFicftE0KrwMC77WjXTrPuoUxhwyFsjKONSeWrlOo= github.com/ipfs/go-ds-badger v0.3.4/go.mod h1:HfqsKJcNnIr9ZhZ+rkwS1J5PpaWjJjg6Ipmxd7KPfZ8= github.com/ipfs/go-ds-flatfs v0.6.0 h1:olAEnDNBK1VMoTRZvfzgo90H5kBP4qIZPpYMtNlBBws= github.com/ipfs/go-ds-flatfs v0.6.0/go.mod h1:p8a/YhmAFYyuonxDbvuIANlDCgS69uqVv+iH5f8fAxY= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-ds-measure v0.2.2 h1:4kwvBGbbSXNYe4ANlg7qTIYoZU6mNlqzQHdVqICkqGI= github.com/ipfs/go-ds-measure v0.2.2/go.mod h1:b/87ak0jMgH9Ylt7oH0+XGy4P8jHx9KG09Qz+pOeTIs= github.com/ipfs/go-ds-pebble v0.5.9 h1:D1FEuMxjbEmDADNqsyT74n9QHVAn12nv9i9Qa15AFYc= github.com/ipfs/go-ds-pebble v0.5.9/go.mod h1:XmUBN05l6B+tMg7mpMS75ZcKW/CX01uZMhhWw85imQA= github.com/ipfs/go-dsqueue v0.2.0 h1:MBi9w3oSiX98Xc+Y7NuJ9G8MI6mAT4IGdO9dHEMCZzU= github.com/ipfs/go-dsqueue v0.2.0/go.mod h1:8FfNQC4DMF/KkzBXRNB9Rb3MKDW0Sh98HMtXYl1mLQE= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= github.com/ipfs/go-fs-lock v0.1.1/go.mod h1:2goSXMCw7QfscHmSe09oXiR34DQeUdm+ei+dhonqly0= github.com/ipfs/go-ipfs-cmds v0.16.0 h1:Oq39Gzz3pWrPwP25SjbfBQugFjKyjFdsHlAMIXdzYzM= github.com/ipfs/go-ipfs-cmds v0.16.0/go.mod h1:iRNtY9ipM/vH0Yr+/0FY+JfT+trZDQIztDoesmmoTo4= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw= github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo= github.com/ipfs/go-ipfs-pq v0.0.4 h1:U7jjENWJd1jhcrR8X/xHTaph14PTAK9O+yaLJbjqgOw= github.com/ipfs/go-ipfs-pq v0.0.4/go.mod h1:9UdLOIIb99IFrgT0Fc53pvbvlJBhpUb4GJuAQf3+O2A= github.com/ipfs/go-ipfs-redirects-file v0.1.2 h1:QCK7VtL91FH17KROVVy5KrzDx2hu68QvB2FTWk08ZQk= github.com/ipfs/go-ipfs-redirects-file v0.1.2/go.mod h1:yIiTlLcDEM/8lS6T3FlCEXZktPPqSOyuY6dEzVqw7Fw= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= github.com/ipfs/go-ipld-cbor v0.2.1 h1:H05yEJbK/hxg0uf2AJhyerBDbjOuHX4yi+1U/ogRa7E= github.com/ipfs/go-ipld-cbor v0.2.1/go.mod h1:x9Zbeq8CoE5R2WicYgBMcr/9mnkQ0lHddYWJP2sMV3A= github.com/ipfs/go-ipld-format v0.6.3 h1:9/lurLDTotJpZSuL++gh3sTdmcFhVkCwsgx2+rAh4j8= github.com/ipfs/go-ipld-format v0.6.3/go.mod h1:74ilVN12NXVMIV+SrBAyC05UJRk0jVvGqdmrcYZvCBk= github.com/ipfs/go-ipld-git v0.1.1 h1:TWGnZjS0htmEmlMFEkA3ogrNCqWjIxwr16x1OsdhG+Y= github.com/ipfs/go-ipld-git v0.1.1/go.mod h1:+VyMqF5lMcJh4rwEppV0e6g4nCCHXThLYYDpKUkJubI= github.com/ipfs/go-ipld-legacy v0.3.0 h1:7XhFKkRyCvP5upOlQfKUFIqL3S5DEZnbUE4bQmQ/tNE= github.com/ipfs/go-ipld-legacy v0.3.0/go.mod h1:Ukef9ARQiX+RVetwH2XiReLgJvQDEXcUPszrZ1KRjKI= github.com/ipfs/go-libdht v0.5.0 h1:ZN+eCqwahZvUeT0e4DsIxRtm78Mc9UR5tmZUiMsrGjQ= github.com/ipfs/go-libdht v0.5.0/go.mod h1:L3YiuFXecLeZZFuuVRM0hjg1GgVhARzUdahFsuqSa7w= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log/v2 v2.9.1 h1:3JXwHWU31dsCpvQ+7asz6/QsFJHqFr4gLgQ0FWteujk= github.com/ipfs/go-log/v2 v2.9.1/go.mod h1:evFx7sBiohUN3AG12mXlZBw5hacBQld3ZPHrowlJYoo= github.com/ipfs/go-metrics-interface v0.3.0 h1:YwG7/Cy4R94mYDUuwsBfeziJCVm9pBMJ6q/JR9V40TU= github.com/ipfs/go-metrics-interface v0.3.0/go.mod h1:OxxQjZDGocXVdyTPocns6cOLwHieqej/jos7H4POwoY= github.com/ipfs/go-peertaskqueue v0.8.3 h1:tBPpGJy+A92RqtRFq5amJn0Uuj8Pw8tXi0X3eHfHM8w= github.com/ipfs/go-peertaskqueue v0.8.3/go.mod h1:OqVync4kPOcXEGdj/LKvox9DCB5mkSBeXsPczCxLtYA= github.com/ipfs/go-test v0.2.3 h1:Z/jXNAReQFtCYyn7bsv/ZqUwS6E7iIcSpJ2CuzCvnrc= github.com/ipfs/go-test v0.2.3/go.mod h1:QW8vSKkwYvWFwIZQLGQXdkt9Ud76eQXRQ9Ao2H+cA1o= github.com/ipfs/go-unixfsnode v1.10.3 h1:c8sJjuGNkxXAQH75P+f5ngPda/9T+DrboVA0TcDGvGI= github.com/ipfs/go-unixfsnode v1.10.3/go.mod h1:2Jlc7DoEwr12W+7l8Hr6C7XF4NHST3gIkqSArLhGSxU= github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco= github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.22.0 h1:YJhDhjEOvOYaqshd3b4atIWUoRg/rKrgmwCyUHwlbuY= github.com/ipld/go-ipld-prime v0.22.0/go.mod h1:ol7vKxOOVgEh0iAPuiDalM+0gScXVMA5ZZa4DVrTnEA= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20250821084354-a425e60cd714 h1:cqNk8PEwHnK0vqWln+U/YZhQc9h2NB3KjUjDPZo5Q2s= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20250821084354-a425e60cd714/go.mod h1:ZEUdra3CoqRVRYgAX/jAJO9aZGz6SKtKEG628fHHktY= github.com/ipshipyard/p2p-forge v0.7.0 h1:PQayexxZC1FR2Vx0XOSbmZ6wDPliidS48I+xXWuF+YU= github.com/ipshipyard/p2p-forge v0.7.0/go.mod h1:i2wg0p7WmHGyo5vYaK9COZBp8BN5Drncfu3WoQNZlQY= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= github.com/koron/go-ssdp v0.0.6/go.mod h1:0R9LfRJGek1zWTjN3JUNlm5INCDYGpRDfAptnct63fI= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/libdns/libdns v1.0.0-beta.1 h1:KIf4wLfsrEpXpZ3vmc/poM8zCATXT2klbdPe6hyOBjQ= github.com/libdns/libdns v1.0.0-beta.1/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-doh-resolver v0.5.0 h1:4h7plVVW+XTS+oUBw2+8KfoM1jF6w8XmO7+skhePFdE= github.com/libp2p/go-doh-resolver v0.5.0/go.mod h1:aPDxfiD2hNURgd13+hfo29z9IC22fv30ee5iM31RzxU= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.3.0 h1:q31zcHUvHnwDO0SHaukewPYgwOBSxtt830uJtUx6784= github.com/libp2p/go-flow-metrics v0.3.0/go.mod h1:nuhlreIwEguM1IvHAew3ij7A8BMlyHQJ279ao24eZZo= github.com/libp2p/go-libp2p v0.48.0 h1:h2BrLAgrj7X8bEN05K7qmrjpNHYA+6tnsGRdprjTnvo= github.com/libp2p/go-libp2p v0.48.0/go.mod h1:Q1fBZNdmC2Hf82husCTfkKJVfHm2we5zk+NWmOGEmWk= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw= github.com/libp2p/go-libp2p-kad-dht v0.39.0 h1:mww38eBYiUvdsu+Xl/GLlBC0Aa8M+5HAwvafkFOygAM= github.com/libp2p/go-libp2p-kad-dht v0.39.0/go.mod h1:Po2JugFEkDq9Vig/JXtc153ntOi0q58o4j7IuITCOVs= github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio= github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s= github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4= github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs= github.com/libp2p/go-libp2p-pubsub v0.15.0 h1:cG7Cng2BT82WttmPFMi50gDNV+58K626m/wR00vGL1o= github.com/libp2p/go-libp2p-pubsub v0.15.0/go.mod h1:lr4oE8bFgQaifRcoc2uWhWWiK6tPdOEKpUuR408GFN4= github.com/libp2p/go-libp2p-pubsub-router v0.6.0 h1:D30iKdlqDt5ZmLEYhHELCMRj8b4sFAqrUcshIUvVP/s= github.com/libp2p/go-libp2p-pubsub-router v0.6.0/go.mod h1:FY/q0/RBTKsLA7l4vqC2cbRbOvyDotg8PJQ7j8FDudE= github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= github.com/libp2p/go-libp2p-routing-helpers v0.7.5 h1:HdwZj9NKovMx0vqq6YNPTh6aaNzey5zHD7HeLJtq6fI= github.com/libp2p/go-libp2p-routing-helpers v0.7.5/go.mod h1:3YaxrwP0OBPDD7my3D0KxfR89FlcX/IEbxDEDfAmj98= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= github.com/libp2p/go-libp2p-xor v0.1.0 h1:hhQwT4uGrBcuAkUGXADuPltalOdpf9aag9kaYNT2tLA= github.com/libp2p/go-libp2p-xor v0.1.0/go.mod h1:LSTM5yRnjGZbWNTA/hRwq2gGFrvRIbQJscoIL/u6InY= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-netroute v0.4.0 h1:sZZx9hyANYUx9PZyqcgE/E1GUG3iEtTZHUEvdtXT7/Q= github.com/libp2p/go-netroute v0.4.0/go.mod h1:Nkd5ShYgSMS5MUKy/MU2T57xFoOKvvLR92Lic48LEyA= github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= github.com/libp2p/go-yamux/v5 v5.0.1 h1:f0WoX/bEF2E8SbE4c/k1Mo+/9z0O4oC/hWEA+nfYRSg= github.com/libp2p/go-yamux/v5 v5.0.1/go.mod h1:en+3cdX51U0ZslwRdRLrvQsdayFt3TSUKvBGErzpWbU= github.com/libp2p/zeroconf/v2 v2.2.0 h1:Cup06Jv6u81HLhIj1KasuNM/RHHrJ8T7wOTS4+Tv53Q= github.com/libp2p/zeroconf/v2 v2.2.0/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marcopolo/simnet v0.0.4 h1:50Kx4hS9kFGSRIbrt9xUS3NJX33EyPqHVmpXvaKLqrY= github.com/marcopolo/simnet v0.0.4/go.mod h1:tfQF1u2DmaB6WHODMtQaLtClEf3a296CKQLq5gAsIS0= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mholt/acmez/v3 v3.1.2 h1:auob8J/0FhmdClQicvJvuDavgd5ezwLBfKuYmynhYzc= github.com/mholt/acmez/v3 v3.1.2/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= github.com/multiformats/go-multiaddr v0.16.1 h1:fgJ0Pitow+wWXzN9do+1b8Pyjmo8m5WhGfzpL82MpCw= github.com/multiformats/go-multiaddr v0.16.1/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= github.com/multiformats/go-multiaddr-dns v0.5.0 h1:p/FTyHKX0nl59f+S+dEUe8HRK+i5Ow/QHMw8Nh3gPCo= github.com/multiformats/go-multiaddr-dns v0.5.0/go.mod h1:yJ349b8TPIAANUyuOzn1oz9o22tV9f+06L+cCeMxC14= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multicodec v0.10.0 h1:UpP223cig/Cx8J76jWt91njpK3GTAO1w02sdcjZDSuc= github.com/multiformats/go-multicodec v0.10.0/go.mod h1:wg88pM+s2kZJEQfRCKBNU+g32F5aWBEjyFHXvZLTcLI= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.6.1 h1:4aoX5v6T+yWmc2raBHsTvzmFhOI8WVOer28DeBBEYdQ= github.com/multiformats/go-multistream v0.6.1/go.mod h1:ksQf6kqHAb6zIsyw7Zm+gAuVo57Qbq84E27YlYqavqw= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.1.0 h1:i2wqFp4sdl3IcIxfAonHQV9qU5OsZ4Ts9IOoETFs5dI= github.com/multiformats/go-varint v0.1.0/go.mod h1:5KVAVXegtfmNQQm/lCY+ATvDzvJJhSkUlGQV9wgObdI= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.36.3 h1:hID7cr8t3Wp26+cYnfcjR6HpJ00fdogN6dqZ1t6IylU= github.com/onsi/gomega v1.36.3/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o= github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M= github.com/pion/dtls/v3 v3.1.2 h1:gqEdOUXLtCGW+afsBLO0LtDD8GnuBBjEy6HRtyofZTc= github.com/pion/dtls/v3 v3.1.2/go.mod h1:Hw/igcX4pdY69z1Hgv5x7wJFrUkdgHwAn/Q/uo7YHRo= github.com/pion/ice/v4 v4.0.10 h1:P59w1iauC/wPk9PdY8Vjl4fOFL5B+USq1+xbDcN6gT4= github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw= github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4= github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic= github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM= github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.16 h1:fk1B1dNW4hsI78XUCljZJlC4kZOPk67mNRuQ0fcEkSo= github.com/pion/rtcp v1.2.16/go.mod h1:/as7VKfYbs5NIb4h6muQ35kQF/J0ZVNz2Z3xKoCBYOo= github.com/pion/rtp v1.8.19 h1:jhdO/3XhL/aKm/wARFVmvTfq0lC/CvN1xwYKmduly3c= github.com/pion/rtp v1.8.19/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk= github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE= github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE= github.com/pion/sdp/v3 v3.0.18 h1:l0bAXazKHpepazVdp+tPYnrsy9dfh7ZbT8DxesH5ZnI= github.com/pion/sdp/v3 v3.0.18/go.mod h1:ZREGo6A9ZygQ9XkqAj5xYCQtQpif0i6Pa81HOiAdqQ8= github.com/pion/srtp/v3 v3.0.6 h1:E2gyj1f5X10sB/qILUGIkL4C2CqK269Xq167PbGCc/4= github.com/pion/srtp/v3 v3.0.6/go.mod h1:BxvziG3v/armJHAaJ87euvkhHqWe9I7iiOy50K2QkhY= github.com/pion/stun/v3 v3.1.1 h1:CkQxveJ4xGQjulGSROXbXq94TAWu8gIX2dT+ePhUkqw= github.com/pion/stun/v3 v3.1.1/go.mod h1:qC1DfmcCTQjl9PBaMa5wSn3x9IPmKxSdcCsxBcDBndM= github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0= github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= github.com/pion/transport/v4 v4.0.1 h1:sdROELU6BZ63Ab7FrOLn13M6YdJLY20wldXW2Cu2k8o= github.com/pion/transport/v4 v4.0.1/go.mod h1:nEuEA4AD5lPdcIegQDpVLgNoDGreqM/YqmEx3ovP4jM= github.com/pion/turn/v4 v4.0.2 h1:ZqgQ3+MjP32ug30xAbD6Mn+/K4Sxi3SdNOTFf+7mpps= github.com/pion/turn/v4 v4.0.2/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs= github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54= github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a h1:cgqrm0F3zwf9IPzca7xN4w+Zy6MC9ZkPvAC8QEWa/iQ= github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a/go.mod h1:ocZfO/tLSHqfScRDNTJbAJR1by4D1lewauX9OwTaPuY= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/quic-go/webtransport-go v0.10.0 h1:LqXXPOXuETY5Xe8ITdGisBzTYmUOy5eSj+9n4hLTjHI= github.com/quic-go/webtransport-go v0.10.0/go.mod h1:LeGIXr5BQKE3UsynwVBeQrU1TPrbh73MGoC6jd+V7ow= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/slok/go-http-metrics v0.13.0 h1:lQDyJJx9wKhmbliyUsZ2l6peGnXRHjsjoqPt5VYzcP8= github.com/slok/go-http-metrics v0.13.0/go.mod h1:HIr7t/HbN2sJaunvnt9wKP9xoBBVZFo1/KiHU3b0w+4= github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.3.1 h1:82ioxmhEYut7LBVGhGq8xoRkXPLElVuh5mV67AFfdv0= github.com/whyrusleeping/cbor-gen v0.3.1/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI= github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 h1:wVZXIWjQSeSmMoxF74LzAnpVQOAFDo3pPji9Y4SOFKc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0/go.mod h1:khvBS2IggMFNwZK/6lEeHg/W57h/IX6J4URh57fuI40= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.40.0 h1:MzfofMZN8ulNqobCmCAVbqVL5syHw+eB2qPRkCMA/fQ= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.40.0/go.mod h1:E73G9UFtKRXrxhBsHtG00TB5WxX57lpsQzogDkqBTz8= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 h1:jiDhWWeC7jfWqR9c/uplMOqJ0sbNlNWv0UkzE0vX1MA= golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90/go.mod h1:xE1HEv6b+1SCZ5/uscMRjUBKtIxworgEcEi+/n9NQDQ= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c h1:6a8FdnNk6bTXBjR4AGKFgUKuo+7GnR3FX5L7CbveeZc= golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw= pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= ================================================ FILE: docs/examples/kubo-as-a-library/main.go ================================================ package main import ( "context" "flag" "fmt" "io" "log" "os" "path/filepath" "strings" "sync" "time" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/path" icore "github.com/ipfs/kubo/core/coreiface" ma "github.com/multiformats/go-multiaddr" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/coreapi" "github.com/ipfs/kubo/core/node/libp2p" "github.com/ipfs/kubo/plugin/loader" // This package is needed so that all the preloaded plugins are loaded automatically "github.com/ipfs/kubo/repo/fsrepo" "github.com/libp2p/go-libp2p/core/peer" ) /// ------ Setting up the IPFS Repo func setupPlugins(externalPluginsPath string) error { // Load any external plugins if available on externalPluginsPath plugins, err := loader.NewPluginLoader(filepath.Join(externalPluginsPath, "plugins")) if err != nil { return fmt.Errorf("error loading plugins: %s", err) } // Load preloaded and external plugins if err := plugins.Initialize(); err != nil { return fmt.Errorf("error initializing plugins: %s", err) } if err := plugins.Inject(); err != nil { return fmt.Errorf("error initializing plugins: %s", err) } return nil } func createTempRepo() (string, error) { repoPath, err := os.MkdirTemp("", "ipfs-shell") if err != nil { return "", fmt.Errorf("failed to get temp dir: %s", err) } // Create a config with default options and a 2048 bit key cfg, err := config.Init(io.Discard, 2048) if err != nil { return "", err } // Use TCP-only on loopback with random port for reliable local testing. // This matches what kubo's test harness uses (test/cli/transports_test.go). // QUIC/UDP transports are avoided because they may be throttled on CI. cfg.Addresses.Swarm = []string{ "/ip4/127.0.0.1/tcp/0", } // Explicitly disable non-TCP transports for reliability. cfg.Swarm.Transports.Network.QUIC = config.False cfg.Swarm.Transports.Network.Relay = config.False cfg.Swarm.Transports.Network.WebTransport = config.False cfg.Swarm.Transports.Network.WebRTCDirect = config.False cfg.Swarm.Transports.Network.Websocket = config.False cfg.AutoTLS.Enabled = config.False // Disable routing - we don't need DHT for direct peer connections. // Bitswap works with directly connected peers without needing DHT lookups. cfg.Routing.Type = config.NewOptionalString("none") // Disable bootstrap for this example - we manually connect only the peers we need. cfg.Bootstrap = []string{} // When creating the repository, you can define custom settings on the repository, such as enabling experimental // features (See experimental-features.md) or customizing the gateway endpoint. // To do such things, you should modify the variable `cfg`. For example: if *flagExp { // https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-filestore cfg.Experimental.FilestoreEnabled = true // https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-urlstore cfg.Experimental.UrlstoreEnabled = true // https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-p2p cfg.Experimental.Libp2pStreamMounting = true // https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#p2p-http-proxy cfg.Experimental.P2pHttpProxy = true // See also: https://github.com/ipfs/kubo/blob/master/docs/config.md // And: https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md } // Create the repo with the config err = fsrepo.Init(repoPath, cfg) if err != nil { return "", fmt.Errorf("failed to init ephemeral node: %s", err) } return repoPath, nil } /// ------ Spawning the node // Creates an IPFS node and returns its coreAPI. func createNode(ctx context.Context, repoPath string) (*core.IpfsNode, error) { // Open the repo repo, err := fsrepo.Open(repoPath) if err != nil { return nil, err } // Construct the node nodeOptions := &core.BuildCfg{ Online: true, // For this example, we use NilRouterOption (no routing) since we connect peers directly. // Bitswap works with directly connected peers without needing DHT lookups. // In production, you would typically use: // Routing: libp2p.DHTOption, // Full DHT node (stores and fetches records) // Routing: libp2p.DHTClientOption, // DHT client (only fetches records) Routing: libp2p.NilRouterOption, Repo: repo, } return core.NewNode(ctx, nodeOptions) } var loadPluginsOnce sync.Once // Spawns a node to be used just for this run (i.e. creates a tmp repo). func spawnEphemeral(ctx context.Context) (icore.CoreAPI, *core.IpfsNode, error) { var onceErr error loadPluginsOnce.Do(func() { onceErr = setupPlugins("") }) if onceErr != nil { return nil, nil, onceErr } // Create a Temporary Repo repoPath, err := createTempRepo() if err != nil { return nil, nil, fmt.Errorf("failed to create temp repo: %s", err) } node, err := createNode(ctx, repoPath) if err != nil { return nil, nil, err } api, err := coreapi.NewCoreAPI(node) return api, node, err } func connectToPeers(ctx context.Context, ipfs icore.CoreAPI, peers []string) error { var wg sync.WaitGroup peerInfos := make(map[peer.ID]*peer.AddrInfo, len(peers)) for _, addrStr := range peers { addr, err := ma.NewMultiaddr(addrStr) if err != nil { return err } pii, err := peer.AddrInfoFromP2pAddr(addr) if err != nil { return err } pi, ok := peerInfos[pii.ID] if !ok { pi = &peer.AddrInfo{ID: pii.ID} peerInfos[pi.ID] = pi } pi.Addrs = append(pi.Addrs, pii.Addrs...) } wg.Add(len(peerInfos)) for _, peerInfo := range peerInfos { go func(peerInfo *peer.AddrInfo) { defer wg.Done() err := ipfs.Swarm().Connect(ctx, *peerInfo) if err != nil { log.Printf("failed to connect to %s: %s", peerInfo.ID, err) } }(peerInfo) } wg.Wait() return nil } func getUnixfsNode(path string) (files.Node, error) { st, err := os.Stat(path) if err != nil { return nil, err } f, err := files.NewSerialFile(path, false, st) if err != nil { return nil, err } return f, nil } /// ------- var flagExp = flag.Bool("experimental", false, "enable experimental features") func main() { flag.Parse() /// --- Part I: Getting a IPFS node running fmt.Println("-- Getting an IPFS node running -- ") ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() // Spawn a local peer using a temporary path, for testing purposes ipfsA, nodeA, err := spawnEphemeral(ctx) if err != nil { panic(fmt.Errorf("failed to spawn peer node: %s", err)) } peerCidFile, err := ipfsA.Unixfs().Add(ctx, files.NewBytesFile([]byte("hello from ipfs 101 in Kubo"))) if err != nil { panic(fmt.Errorf("could not add File: %s", err)) } fmt.Printf("Added file to peer with CID %s\n", peerCidFile.String()) // Spawn a node using a temporary path, creating a temporary repo for the run fmt.Println("Spawning Kubo node on a temporary repo") ipfsB, _, err := spawnEphemeral(ctx) if err != nil { panic(fmt.Errorf("failed to spawn ephemeral node: %s", err)) } fmt.Println("IPFS node is running") /// --- Part II: Adding a file and a directory to IPFS fmt.Println("\n-- Adding and getting back files & directories --") inputBasePath := "../example-folder/" inputPathFile := inputBasePath + "ipfs.paper.draft3.pdf" inputPathDirectory := inputBasePath + "test-dir" someFile, err := getUnixfsNode(inputPathFile) if err != nil { panic(fmt.Errorf("could not get File: %s", err)) } cidFile, err := ipfsB.Unixfs().Add(ctx, someFile) if err != nil { panic(fmt.Errorf("could not add File: %s", err)) } fmt.Printf("Added file to IPFS with CID %s\n", cidFile.String()) someDirectory, err := getUnixfsNode(inputPathDirectory) if err != nil { panic(fmt.Errorf("could not get File: %s", err)) } cidDirectory, err := ipfsB.Unixfs().Add(ctx, someDirectory) if err != nil { panic(fmt.Errorf("could not add Directory: %s", err)) } fmt.Printf("Added directory to IPFS with CID %s\n", cidDirectory.String()) /// --- Part III: Getting the file and directory you added back outputBasePath, err := os.MkdirTemp("", "example") if err != nil { panic(fmt.Errorf("could not create output dir (%v)", err)) } fmt.Printf("output folder: %s\n", outputBasePath) outputPathFile := outputBasePath + strings.Split(cidFile.String(), "/")[2] outputPathDirectory := outputBasePath + strings.Split(cidDirectory.String(), "/")[2] rootNodeFile, err := ipfsB.Unixfs().Get(ctx, cidFile) if err != nil { panic(fmt.Errorf("could not get file with CID: %s", err)) } err = files.WriteTo(rootNodeFile, outputPathFile) if err != nil { panic(fmt.Errorf("could not write out the fetched CID: %s", err)) } fmt.Printf("got file back from IPFS (IPFS path: %s) and wrote it to %s\n", cidFile.String(), outputPathFile) rootNodeDirectory, err := ipfsB.Unixfs().Get(ctx, cidDirectory) if err != nil { panic(fmt.Errorf("could not get file with CID: %s", err)) } err = files.WriteTo(rootNodeDirectory, outputPathDirectory) if err != nil { panic(fmt.Errorf("could not write out the fetched CID: %s", err)) } fmt.Printf("Got directory back from IPFS (IPFS path: %s) and wrote it to %s\n", cidDirectory.String(), outputPathDirectory) /// --- Part IV: Getting a file from another IPFS node fmt.Println("\n-- Connecting to nodeA and fetching content via bitswap --") // Get nodeA's actual listening address dynamically. // We configured TCP-only on 127.0.0.1 with random port, so this will be a TCP address. peerAddrs, err := ipfsA.Swarm().LocalAddrs(ctx) if err != nil { panic(fmt.Errorf("could not get peer addresses: %s", err)) } peerMa := peerAddrs[0].String() + "/p2p/" + nodeA.Identity.String() bootstrapNodes := []string{ // In production, use real bootstrap peers like: // "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", // For this example, we only connect to nodeA which has our test content. peerMa, } fmt.Println("Connecting to peer...") err = connectToPeers(ctx, ipfsB, bootstrapNodes) if err != nil { panic(fmt.Errorf("failed to connect to peers: %s", err)) } fmt.Println("Connected to peer") exampleCIDStr := peerCidFile.RootCid().String() fmt.Printf("Fetching a file from the network with CID %s\n", exampleCIDStr) outputPath := outputBasePath + exampleCIDStr testCID := path.FromCid(peerCidFile.RootCid()) rootNode, err := ipfsB.Unixfs().Get(ctx, testCID) if err != nil { panic(fmt.Errorf("could not get file with CID: %s", err)) } err = files.WriteTo(rootNode, outputPath) if err != nil { panic(fmt.Errorf("could not write out the fetched CID: %s", err)) } fmt.Printf("Wrote the file to %s\n", outputPath) fmt.Println("\nAll done! You just finalized your first tutorial on how to use Kubo as a library") } ================================================ FILE: docs/examples/kubo-as-a-library/main_test.go ================================================ package main import ( "bytes" "io" "os" "os/exec" "strings" "testing" "time" ) func TestExample(t *testing.T) { t.Log("Starting go run main.go...") start := time.Now() cmd := exec.Command("go", "run", "main.go") cmd.Env = append(os.Environ(), "GOLOG_LOG_LEVEL=error") // reduce libp2p noise // Stream output to both test log and capture buffer for verification // This ensures we see progress even if the process is killed var buf bytes.Buffer cmd.Stdout = io.MultiWriter(os.Stdout, &buf) cmd.Stderr = io.MultiWriter(os.Stderr, &buf) err := cmd.Run() elapsed := time.Since(start) t.Logf("Command completed in %v", elapsed) out := buf.String() if err != nil { t.Fatalf("running example (%v):\n%s", err, out) } if !strings.Contains(out, "All done!") { t.Errorf("example did not complete successfully, output:\n%s", out) } } ================================================ FILE: docs/experimental-features.md ================================================ # Experimental features of Kubo This document contains a list of experimental features in Kubo. These features, commands, and APIs aren't mature, and you shouldn't rely on them. Once they reach maturity, there's going to be mention in the changelog and release posts. If they don't reach maturity, the same applies, and their code is removed. Subscribe to https://github.com/ipfs/kubo/issues/3397 to get updates. When you add a new experimental feature to kubo or change an experimental feature, you MUST please make a PR updating this document, and link the PR in the above issue. - [Raw leaves for unixfs files](#raw-leaves-for-unixfs-files) - [ipfs filestore](#ipfs-filestore) - [ipfs urlstore](#ipfs-urlstore) - [Private Networks](#private-networks) - [ipfs p2p](#ipfs-p2p) - [p2p http proxy](#p2p-http-proxy) - [FUSE](#fuse) - [Plugins](#plugins) - [Directory Sharding / HAMT](#directory-sharding--hamt) - [IPNS PubSub](#ipns-pubsub) - [AutoRelay](#autorelay) - [Strategic Providing](#strategic-providing) - [Graphsync](#graphsync) - [Noise](#noise) - [Optimistic Provide](#optimistic-provide) - [HTTP Gateway over Libp2p](#http-gateway-over-libp2p) --- ## Raw Leaves for unixfs files Allows files to be added with no formatting in the leaf nodes of the graph. ### State Stable but not used by default. ### In Version 0.4.5 ### How to enable Use `--raw-leaves` flag when calling `ipfs add`. This will save some space when adding files. ### Road to being a real feature Enabling this feature _by default_ will change the CIDs (hashes) of all newly imported files and will prevent newly imported files from deduplicating against previously imported files. While we do intend on enabling this by default, we plan on doing so once we have a large batch of "hash-changing" features we can enable all at once. ## ipfs filestore Allows files to be added without duplicating the space they take up on disk. ### State Experimental. ### In Version 0.4.7 ### How to enable > [!WARNING] > **SECURITY CONSIDERATION** > > This feature provides the IPFS [`add` command](https://docs.ipfs.tech/reference/kubo/cli/#ipfs-add) with access to > the local filesystem. Consequently, any user with access to CLI or the HTTP [`/v0/add` RPC API](https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-add) can read > files from the local filesystem with the same permissions as the Kubo daemon. > If you enable this, secure your RPC API using [`API.Authorizations`](https://github.com/ipfs/kubo/blob/master/docs/config.md#apiauthorizations) or custom auth middleware. Modify your ipfs config: ``` ipfs config --json Experimental.FilestoreEnabled true ``` Then restart your IPFS node to reload your config. Finally, when adding files with ipfs add, pass the --nocopy flag to use the filestore instead of copying the files into your local IPFS repo. ### Road to being a real feature - [ ] Needs more people to use and report on how well it works. - [ ] Need to address error states and failure conditions - [ ] cleanup of broken filesystem references (if file is deleted) - [ ] tests that confirm ability to override preexisting filesystem links (allowing user to fix broken link) - [ ] support for a single block having more than one sources in filesystem (blocks can be shared by unrelated files, and not be broken when some files are unpinned / gc'd) - [ ] [other known issues](https://github.com/ipfs/kubo/issues/7161) - [ ] Need to write docs on usage, advantages, disadvantages - [ ] Need to merge utility commands to aid in maintenance and repair of filestore ## ipfs urlstore Allows ipfs to retrieve blocks contents via a URL instead of storing it in the datastore ### State Experimental. ### In Version v0.4.17 ### How to enable > [!WARNING] > **SECURITY CONSIDERATION** > > This feature provides the IPFS [`add` CLI command](https://docs.ipfs.tech/reference/kubo/cli/#ipfs-add) with access to > the local filesystem. Consequently, any user with access to the CLI or HTTP [`/v0/add` RPC API](https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-add) can read > files from the local filesystem with the same permissions as the Kubo daemon. > If you enable this, secure your RPC API using [`API.Authorizations`](https://github.com/ipfs/kubo/blob/master/docs/config.md#apiauthorizations) or custom auth middleware. Modify your ipfs config: ``` ipfs config --json Experimental.UrlstoreEnabled true ``` And then add a file at a specific URL using `ipfs urlstore add ` ### Road to being a real feature - [ ] Needs more people to use and report on how well it works. - [ ] Need to address error states and failure conditions - [ ] cleanup of broken URL+range references (if URL starts returning 404 or error) - [ ] tests that confirm ability to override preexisting URL+range links (allowing user to fix broken link) - [ ] support for a single block having more than one URL+range (blocks can be shared by unrelated URLs) - [ ] Need to write docs on usage, advantages, disadvantages - [ ] Need to implement caching - [ ] Need to add metrics to monitor performance ## Private Networks It allows ipfs to only connect to other peers who have a shared secret key. ### State Stable but not quite ready for prime-time. > [!WARNING] > Limited to TCP transport, comes with overhead of double-encryption. See details below. ### In Version 0.4.7 ### How to enable Generate a pre-shared-key using [ipfs-swarm-key-gen](https://github.com/Kubuxu/go-ipfs-swarm-key-gen)): ``` go install github.com/Kubuxu/go-ipfs-swarm-key-gen/ipfs-swarm-key-gen@latest ipfs-swarm-key-gen > ~/.ipfs/swarm.key ``` To join a given private network, get the key file from someone in the network and save it to `~/.ipfs/swarm.key` (If you are using a custom `$IPFS_PATH`, put it in there instead). When using this feature, you will not be able to connect to the default bootstrap nodes (Since we aren't part of your private network) so you will need to set up your own bootstrap nodes. First, to prevent your node from even trying to connect to the default bootstrap nodes, run: ```bash ipfs bootstrap rm --all ``` Then add your own bootstrap peers with: ```bash ipfs bootstrap add ``` For example: ``` ipfs bootstrap add /ip4/104.236.76.40/tcp/4001/p2p/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64 ``` Bootstrap nodes are no different from all other nodes in the network apart from the function they serve. To be extra cautious, You can also set the `LIBP2P_FORCE_PNET` environment variable to `1` to force the usage of private networks. If no private network is configured, the daemon will fail to start. ### Road to being a real feature - [x] Needs more people to use and report on how well it works - [ ] More documentation - [ ] Improve / future proof libp2p support (see [libp2p/specs#489](https://github.com/libp2p/specs/issues/489)) - [ ] Currently limited to TCP-only, and double-encrypts all data sent on TCP. This is slow. - [ ] Does not work with QUIC: [go-libp2p#1432](https://github.com/libp2p/go-libp2p/issues/1432) - [ ] Needs better tooling/UX - [ ] Detect lack of peers when swarm key is present and prompt user to set up bootstrappers/peering - [ ] ipfs-webui will not load unless blocks are present in private swarm. Detect it and prompt user to import CAR with webui. ## ipfs p2p Allows tunneling of TCP connections through libp2p streams, similar to SSH port forwarding (`ssh -L`). ### State Experimental, will be stabilized in 0.6.0 ### In Version 0.4.10 ### How to enable > [!WARNING] > **SECURITY CONSIDERATION** > > This feature provides CLI and HTTP RPC user with ability to set up port forwarding for all localhost and LAN ports. > If you enable this and plan to expose CLI or HTTP RPC to other users or machines, > secure RPC API using [`API.Authorizations`](https://github.com/ipfs/kubo/blob/master/docs/config.md#apiauthorizations) or custom auth middleware. ```sh > ipfs config --json Experimental.Libp2pStreamMounting true ``` ### How to use See [docs/p2p-tunnels.md](p2p-tunnels.md) for usage examples, foreground mode, and systemd integration. ### Road to being a real feature - [x] More documentation - [x] `ipfs p2p forward` mode - [ ] Ability to define tunnels via JSON config, similar to [`Peering.Peers`](https://github.com/ipfs/kubo/blob/master/docs/config.md#peeringpeers), see [kubo#5460](https://github.com/ipfs/kubo/issues/5460) ## p2p http proxy Allows proxying of HTTP requests over p2p streams. This allows serving any standard HTTP app over p2p streams. ### State Experimental ### In Version 0.4.19 ### How to enable > [!WARNING] > **SECURITY CONSIDERATION** > > This feature provides CLI and HTTP RPC user with ability to set up HTTP forwarding for all localhost and LAN ports. > If you enable this and plan to expose CLI or HTTP RPC to other users or machines, > secure RPC API using [`API.Authorizations`](https://github.com/ipfs/kubo/blob/master/docs/config.md#apiauthorizations) or custom auth middleware. The `p2p` command needs to be enabled in the config: ```sh > ipfs config --json Experimental.Libp2pStreamMounting true ``` On the client, the p2p HTTP proxy needs to be enabled in the config: ```sh > ipfs config --json Experimental.P2pHttpProxy true ``` ### How to use **Netcat example:** First, pick a protocol name for your application. Think of the protocol name as a port number, just significantly more user-friendly. In this example, we're going to use `/http`. ***Setup:*** 1. A "server" node with peer ID `$SERVER_ID` 2. A "client" node. ***On the "server" node:*** First, start your application and have it listen for TCP connections on port `$APP_PORT`. Then, configure the p2p listener by running: ```sh > ipfs p2p listen --allow-custom-protocol /http /ip4/127.0.0.1/tcp/$APP_PORT ``` This will configure IPFS to forward all incoming `/http` streams to `127.0.0.1:$APP_PORT` (opening a new connection to `127.0.0.1:$APP_PORT` per incoming stream. ***On the "client" node:*** Next, have your application make a http request to `127.0.0.1:8080/p2p/$SERVER_ID/http/$FORWARDED_PATH`. This connection will be forwarded to the service running on `127.0.0.1:$APP_PORT` on the remote machine (which needs to be a http server!) with path `$FORWARDED_PATH`. You can test it with netcat: ***On "server" node:*** ```sh > echo -e "HTTP/1.1 200\nContent-length: 11\n\nIPFS rocks!" | nc -l -p $APP_PORT ``` ***On "client" node:*** ```sh > curl http://localhost:8080/p2p/$SERVER_ID/http/ ``` You should now see the resulting HTTP response: IPFS rocks! ### Custom protocol names We also support the use of protocol names of the form /x/$NAME/http where $NAME doesn't contain any "/"'s ### Road to being a real feature - [ ] Needs p2p streams to graduate from experiments - [ ] Needs more people to use and report on how well it works and fits use cases - [ ] More documentation - [ ] Need better integration with the subdomain gateway feature. ## FUSE FUSE makes it possible to mount `/ipfs`, `/ipns` and `/mfs` namespaces in your OS, allowing arbitrary apps access to IPFS using a subset of filesystem abstractions. It is considered EXPERIMENTAL due to limited (and buggy) support on some platforms. See [fuse.md](./fuse.md) for more details. ## Plugins ### In Version 0.4.11 ### State Experimental Plugins allow adding functionality without the need to recompile the daemon. ### Basic Usage: See [Plugin docs](./plugins.md) ### Road to being a real feature - [x] More plugins and plugin types - [ ] A way to reliably build and distribute plugins. - [ ] Better support for platforms other than Linux & MacOS - [ ] Feedback on stability ## Directory Sharding / HAMT ### In Version - 0.4.8: - Introduced `Experimental.ShardingEnabled` which enabled sharding globally. - All-or-nothing, unnecessary sharding of small directories. - 0.11.0 : - Removed support for `Experimental.ShardingEnabled` - Replaced with automatic sharding based on the block size ### State Replaced by autosharding. The `Experimental.ShardingEnabled` config field is no longer used, please remove it from your configs. kubo now automatically shards when directory block is bigger than 256KB, ensuring every block is small enough to be exchanged with other peers ## IPNS pubsub Specification: [IPNS PubSub Router](https://specs.ipfs.tech/ipns/ipns-pubsub-router/) ### In Version 0.4.14 : - Introduced 0.5.0 : - No longer needs to use the DHT for the first resolution - When discovering PubSub peers via the DHT, the DHT key is different from previous versions - This leads to 0.5 IPNS pubsub peers and 0.4 IPNS pubsub peers not being able to find each other in the DHT - Robustness improvements 0.11.0 : - Can be enabled via `Ipns.UsePubsub` flag in config 0.40.0 : - Persistent message sequence number validation to prevent message cycles in large networks ### State Experimental, default-disabled. Utilizes pubsub for publishing IPNS records in real time. When it is enabled: - IPNS publishers push records to a name-specific pubsub topic, in addition to publishing to the DHT. - IPNS resolvers subscribe to the name-specific topic on first resolution and receive subsequently published records through pubsub in real time. This makes subsequent resolutions instant, as they are resolved through the local cache. Both the publisher and the resolver nodes need to have the feature enabled for it to work effectively. ### How to enable Run your daemon with the `--enable-namesys-pubsub` flag or modify your ipfs config and restart the daemon: ``` ipfs config --json Ipns.UsePubsub true ``` NOTE: - This feature implicitly enables pubsub. - Passing `--enable-namesys-pubsub` CLI flag overrides `Ipns.UsePubsub` config. ### Road to being a real feature - [ ] Needs more people to use and report on how well it works ## AutoRelay ### In Version - 0.4.19 : - Introduced Circuit Relay v1 - 0.11.0 : - Deprecated v1 - Introduced [Circuit Relay v2](https://github.com/libp2p/specs/blob/master/relay/circuit-v2.md) ### State Experimental, disabled by default. Automatically discovers relays and advertises relay addresses when the node is behind an impenetrable NAT. ### How to enable Modify your ipfs config: ``` ipfs config --json Swarm.RelayClient.Enabled true ``` ### Road to being a real feature - [ ] needs testing - [ ] needs to be automatically enabled when AutoNAT detects node is behind an impenetrable NAT. ## Strategic Providing ### State `Experimental.StrategicProviding` was removed in Kubo v0.35. Replaced by [`Provide.Enabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#provideenabled) and [`Provide.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providestrategy). ## GraphSync ### State Removed, no plans to reintegrate either as experimental or stable feature. [Trustless Gateway over Libp2p](#http-gateway-over-libp2p) should be easier to use for unixfs usecases and support basic wildcard car streams for non unixfs. See https://github.com/ipfs/kubo/pull/9747 for more information. ## Noise ### State Stable, enabled by default [Noise](https://github.com/libp2p/specs/tree/master/noise) libp2p transport based on the [Noise Protocol Framework](https://noiseprotocol.org/noise.html). While TLS remains the default transport in Kubo, Noise is easier to implement and is thus the "interop" transport between IPFS and libp2p implementations. ## Optimistic Provide ### In Version 0.20.0 ### State Experimental, disabled by default. When the Amino DHT client tries to store a provider in the DHT, it typically searches for the 20 peers that are closest to the target key. However, this process can be time-consuming, as the search terminates only after no closer peers are found among the three currently (during the query) known closest ones. In cases where these closest peers are slow to respond (which often happens if they are located at the edge of the DHT network), the query gets blocked by the slowest peer. To address this issue, the `OptimisticProvide` feature can be enabled. This feature allows the client to estimate the network size and determine how close a peer _likely_ needs to be to the target key to be within the 20 closest peers. While searching for the closest peers in the DHT, the client will _optimistically_ store the provider record with peers and abort the query completely when the set of currently known 20 closest peers are also _likely_ the actual 20 closest ones. This heuristic approach can significantly speed up the process, resulting in a speed improvement of 2x to >10x. When it is enabled: - Amino DHT provide operations should complete much faster than with it disabled - This can be tested with commands such as `ipfs routing provide` **Tradeoffs** There are now the classic client, the accelerated DHT client, and optimistic provide that improve the provider process. There are different trade-offs with all of them. The accelerated DHT client is still faster to provide large amounts of provider records at the cost of high resource requirements. Optimistic provide doesn't have the high resource requirements but might not choose optimal peers and is not as fast as the accelerated client, but still much faster than the classic client. **Caveats:** 1. Providing optimistically requires a current network size estimation. This estimation is calculated through routing table refresh queries and is only available after the daemon has been running for some time. If there is no network size estimation available the client will transparently fall back to the classic approach. 2. The chosen peers to store the provider records might not be the actual closest ones. Measurements showed that this is not a problem. 3. The optimistic provide process returns already after 15 out of the 20 provider records were stored with peers. The reasoning here is that one out of the remaining 5 peers are very likely to time out and delay the whole process. To limit the number of in-flight async requests there is the second `OptimisticProvideJobsPoolSize` setting. Currently, this is set to 60. This means that at most 60 parallel background requests are allowed to be in-flight. If this limit is exceeded optimistic provide will block until all 20 provider records are written. This is still 2x faster than the classic approach but not as fast as returning early which yields >10x speed-ups. 4. Since the in-flight background requests are likely to time out, they are not consuming many resources and the job pool size could probably be much higher. For more information, see: - Project doc: https://protocollabs.notion.site/Optimistic-Provide-2c79745820fa45649d48de038516b814 - go-libp2p-kad-dht: https://github.com/libp2p/go-libp2p-kad-dht/pull/783 ### Configuring To enable: ``` ipfs config --json Experimental.OptimisticProvide true ``` If you want to change the `OptimisticProvideJobsPoolSize` setting from its default of 60: ``` ipfs config --json Experimental.OptimisticProvideJobsPoolSize 120 ``` ### Road to being a real feature - [ ] Needs more people to use and report on how well it works - [ ] Should prove at least equivalent availability of provider records as the classic approach ## HTTP Gateway over Libp2p ### In Version 0.23.0 ### State Experimental, disabled by default. Enables serving a subset of the [IPFS HTTP Gateway](https://specs.ipfs.tech/http-gateways/) semantics over libp2p `/http/1.1` protocol. Notes: - This feature only about serving verifiable gateway requests over libp2p: - Deserialized responses are not supported. - Only operate on `/ipfs` resources (no `/ipns` atm) - Only support requests for `application/vnd.ipld.raw` and `application/vnd.ipld.car` (from [Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/), where data integrity can be verified). - Only serve data that is already local to the node (i.e. similar to a [`Gateway.NoFetch`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaynofetch)) - While Kubo currently mounts the gateway API at the root (i.e. `/`) of the libp2p `/http/1.1` protocol, that is subject to change. - The way to reliably discover where a given HTTP protocol is mounted on a libp2p endpoint is via the `.well-known/libp2p` resource specified in the [http+libp2p specification](https://github.com/libp2p/specs/pull/508) - The identifier of the protocol mount point under `/http/1.1` listener is `/ipfs/gateway`, as noted in [ipfs/specs#434](https://github.com/ipfs/specs/pull/434). ### How to enable Modify your ipfs config: ``` ipfs config --json Experimental.GatewayOverLibp2p true ``` ### Road to being a real feature - [ ] Needs more people to use and report on how well it works - [ ] Needs UX work for exposing non-recursive "HTTP transport" (NoFetch) over both libp2p and plain TCP (and sharing the configuration) - [ ] Needs a mechanism for HTTP handler to signal supported features ([IPIP-425](https://github.com/ipfs/specs/pull/425)) - [ ] Needs an option for Kubo to detect peers that have it enabled and prefer HTTP transport before falling back to bitswap (and use CAR if peer supports dag-scope=entity from [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/)) ## Accelerated DHT Client This feature now lives at [`Routing.AcceleratedDHTClient`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient). ================================================ FILE: docs/file-transfer.md ================================================ # Transferring a file with ipfs This document is a guide to help troubleshoot transferring a file between two machines using ipfs. ## A file transfer To start, make sure that ipfs is running on both machines. To verify, run `ipfs id` on each machine and check if the `Addresses` field has anything in it. If it says `null`, then your node is not online and you will need to run `ipfs daemon`. Now, lets call the node with the file you want to transfer node 'A' and the node you want to get the file to node 'B'. On node A, add the file to ipfs using the `ipfs add` command. This will print out the multihash of the content you added. Now, on node B, you can fetch the content using `ipfs get `. ``` # On A > ipfs add myfile.txt added QmZJ1xT1T9KYkHhgRhbv8D7mYrbemaXwYUkg7CeHdrk1Ye myfile.txt # On B > ipfs get QmZJ1xT1T9KYkHhgRhbv8D7mYrbemaXwYUkg7CeHdrk1Ye Saving file(s) to QmZJ1xT1T9KYkHhgRhbv8D7mYrbemaXwYUkg7CeHdrk1Ye 13 B / 13 B [=====================================================] 100.00% 1s ``` If that worked, and downloaded the file, then congratulations! You just used ipfs to move files across the internet! But, if that `ipfs get` command is hanging, with no output, read onwards. ## Troubleshooting So your ipfs file transfer appears to not be working. The primary reason this happens is because node B cannot figure out how to connect to node A, or node B doesn't even know it has to connect to node A. ### Checking for existing connections The first thing to do is to double-check that both nodes are in fact running and online. To do this, run `ipfs id` on each machine. If both nodes show some addresses (like the example below), then your nodes are online. ```json { "ID": "QmTNwsFkLAed15kQEC1ZJWPfoNbBQnMFojfJKQ9sZj1dk8", "PublicKey": "CAASpgIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDZb6znj3LQZKP1+X81exf+vbnqNCMtHjZ5RKTCm7Fytnfe+AI1fhs9YbZdkgFkM1HLxmIOLQj2bMXPIGxUM+EnewN8tWurx4B3+lR/LWNwNYcCFL+jF2ltc6SE6BC8kMLEZd4zidOLPZ8lIRpd0x3qmsjhGefuRwrKeKlR4tQ3C76ziOms47uLdiVVkl5LyJ5+mn4rXOjNKt/oy2O4m1St7X7/yNt8qQgYsPfe/hCOywxCEIHEkqmil+vn7bu4RpAtsUzCcBDoLUIWuU3i6qfytD05hP8Clo+at+l//ctjMxylf3IQ5qyP+yfvazk+WHcsB0tWueEmiU5P2nfUUIR3AgMBAAE=", "Addresses": [ "/ip4/127.0.0.1/tcp/4001/p2p/QmTNwsFkLAed15kQEC1ZJWPfoNbBQnMFojfJKQ9sZj1dk8", "/ip4/127.0.0.1/udp/4001/quic-v1/p2p/QmTNwsFkLAed15kQEC1ZJWPfoNbBQnMFojfJKQ9sZj1dk8", "/ip4/192.168.2.131/tcp/4001/p2p/QmTNwsFkLAed15kQEC1ZJWPfoNbBQnMFojfJKQ9sZj1dk8", "/ip4/192.168.2.131/udp/4001/quic-v1/p2p/QmTNwsFkLAed15kQEC1ZJWPfoNbBQnMFojfJKQ9sZj1dk8", ], "AgentVersion": "go-ipfs/0.4.11-dev/", "ProtocolVersion": "ipfs/0.1.0" } ``` Next, check to see if the nodes have a connection to each other. You can do this by running `ipfs swarm peers` on one node, and checking for the other nodes peer ID in the output. If the two nodes *are* connected, and the `ipfs get` command is still hanging, then something unexpected is going on, and I recommend filing an issue about it. If they are not connected, then let's try and debug why. (Note: you can skip to 'Manually connecting node A to node B' if you just want things to work. Going through the debugging process and reporting what happened to the ipfs team on IRC is helpful to us to understand common pitfalls that people run into) ### Checking providers When requesting content on ipfs, nodes search the DHT for 'provider records' to see who has what content. Let's manually do that on node B to make sure that node B is able to determine that node A has the data. Run `ipfs routing findprovs `. We expect to see the peer ID of node A printed out. If this command returns nothing (or returns IDs that are not node A), then no record of A having the data exists on the network. This can happen if the data is added while node A does not have a daemon running. If this happens, you can run `ipfs routing provide ` on node A to announce to the network that you have that hash. Then if you restart the `ipfs get` command, node B should now be able to tell that node A has the content it wants. If node A's peer ID showed up in the initial `findprovs` call, or manually providing the hash didn't resolve the problem, then it's likely that node B is unable to make a connection to node A. ### Checking addresses In the case where node B simply cannot form a connection to node A, despite knowing that it needs to, the likely culprit is a bad NAT. When node B learns that it needs to connect to node A, it checks the DHT for addresses for node A, and then starts trying to connect to them. We can check those addresses by running `ipfs routing findpeer ` on node B. This command should return a list of addresses for node A. If it doesn't return any addresses, then you should try running the manual providing command from the previous steps. Example output of addresses might look something like this: ``` /ip4/127.0.0.1/tcp/4001 /ip4/127.0.0.1/udp/4001/quic-v1 /ip4/192.168.2.133/tcp/4001 /ip4/192.168.2.133/udp/4001/quic-v1 /ip4/88.157.217.196/tcp/63674 /ip4/88.157.217.196/udp/63674/quic-v1 ``` In this case, we can see a localhost (127.0.0.1) address, a LAN address (the 192.168.*.* one) and another address. If this third address matches your external IP, then the network knows a valid external address for your node. At this point, its safe to assume that your node has a difficult to traverse NAT situation. If this is the case, you can try to enable UPnP or NAT-PMP on the router of node A and retry the process. Otherwise, you can try manually connecting node A to node B. ### Manually connecting node A to B On node B run `ipfs id` and take one of the multiaddrs that contains its public ip address, and then on node A run `ipfs swarm connect `. You can also try using a relayed connection, for more information [read this doc](./experimental-features.md#circuit-relay). If that *still* doesn't work, then you should either join IRC and ask for help there, or file an issue on github. If this manual step *did* work, then you likely have an issue with NAT traversal, and ipfs cannot figure out how to make it through. Please report situations like this to us so we can work on fixing them. ================================================ FILE: docs/fuse.md ================================================ # FUSE **EXPERIMENTAL:** FUSE support is limited, YMMV. Kubo makes it possible to mount `/ipfs`, `/ipns` and `/mfs` namespaces in your OS, allowing arbitrary apps access to IPFS. ## Install FUSE You will need to install and configure fuse before you can mount IPFS #### Linux Note: while this guide should work for most distributions, you may need to refer to your distribution manual to get things working. Install `fuse` with your favorite package manager: ``` sudo apt-get install fuse3 ``` On some older Linux distributions, you may need to add yourself to the `fuse` group. (If no such group exists, you can probably skip this step) ```sh sudo usermod -a -G fuse ``` Restart user session, if active, for the change to apply, either by restarting ssh connection or by re-logging to the system. #### Mac OSX -- OSXFUSE It has been discovered that versions of `osxfuse` prior to `2.7.0` will cause a kernel panic. For everyone's sake, please upgrade (latest at time of writing is `2.7.4`). The installer can be found at https://osxfuse.github.io/. There is also a homebrew formula (`brew cask install osxfuse`) but users report best results installing from the official OSXFUSE installer package. Note that `ipfs` attempts an automatic version check on `osxfuse` to prevent you from shooting yourself in the foot if you have pre `2.7.0`. Since checking the OSXFUSE version [is more complicated than it should be], running `ipfs mount` may require you to install another binary: ```sh go get github.com/jbenet/go-fuse-version/fuse-version ``` If you run into any problems installing FUSE or mounting IPFS, hop on IRC and speak with us, or if you figure something new out, please add to this document! #### FreeBSD ```sh sudo pkg install fusefs-ext2 ``` Load the fuse kernel module: ```sh sudo kldload fusefs ``` To load automatically on boot: ```sh sudo echo fusefs_load="YES" >> /boot/loader.conf ``` ## Prepare mountpoints By default ipfs uses `/ipfs`, `/ipns` and `/mfs` directories for mounting, this can be changed in config. You will have to create the `/ipfs`, `/ipns` and `/mfs` directories explicitly. Note that modifying root requires sudo permissions. ```sh # make the directories sudo mkdir /ipfs sudo mkdir /ipns sudo mkdir /mfs # chown them so ipfs can use them without root permissions sudo chown /ipfs sudo chown /ipns sudo chown /mfs ``` Depending on whether you are using OSX or Linux, follow the proceeding instructions. ## Make sure IPFS daemon is not running You'll need to stop the IPFS daemon if you have it started, otherwise the mount will complain. ``` # Check to see if IPFS daemon is running ps aux | grep ipfs # Kill the IPFS daemon pkill -f ipfs # Verify that it has been killed ``` ## Mounting IPFS ```sh ipfs daemon --mount ``` If you wish to allow other users to use the mount points, edit `/etc/fuse.conf` to enable non-root users, i.e.: ```sh # /etc/fuse.conf - Configuration file for Filesystem in Userspace (FUSE) # Set the maximum number of FUSE mounts allowed to non-root users. # The default is 1000. #mount_max = 1000 # Allow non-root users to specify the allow_other or allow_root mount options. user_allow_other ``` Next set `Mounts.FuseAllowOther` config option to `true`: ```sh ipfs config --json Mounts.FuseAllowOther true ipfs daemon --mount ``` If using FreeBSD, it is necessary to run `ipfs` as root: ```sh sudo HOME=$HOME ipfs daemon --mount ``` ## MFS mountpoint Kubo v0.35.0 and later supports mounting the MFS (Mutable File System) root as a FUSE filesystem, enabling manipulation of content-addressed data like regular files. The CID for any file or directory is retrievable via the `ipfs_cid` extended attribute. ```sh getfattr -n ipfs_cid /mfs/welcome-to-IPFS.jpg getfattr: Removing leading '/' from absolute path names # file: mfs/welcome-to-IPFS.jpg ipfs_cid="QmaeXDdwpUeKQcMy7d5SFBfVB4y7LtREbhm5KizawPsBSH" ``` Please note that the operations supported by the MFS FUSE mountpoint are limited. Since the MFS wasn't designed to store file attributes like ownership information, permissions and creation date, some applications like `vim` and `sed` may misbehave due to missing functionality. ## Troubleshooting #### `Permission denied` or `fusermount: user has no write access to mountpoint` error in Linux Verify that the config file can be read by your user: ```sh sudo ls -l /etc/fuse.conf -rw-r----- 1 root fuse 216 Jan 2 2013 /etc/fuse.conf ``` In most distributions, the group named `fuse` will be created during fuse installation. You can check this with: ```sh sudo grep -q fuse /etc/group && echo fuse_group_present || echo fuse_group_missing ``` If the group is present, just add your regular user to the `fuse` group: ```sh sudo usermod -G fuse -a ``` If the group didn't exist, create `fuse` group (add your regular user to it) and set necessary permissions, for example: ```sh sudo chgrp fuse /etc/fuse.conf sudo chmod g+r /etc/fuse.conf ``` Note that the use of `fuse` group is optional and may depend on your operating system. It is okay to use a different group as long as proper permissions are set for user running `ipfs mount` command. #### Mount command crashes and mountpoint gets stuck ``` sudo umount /ipfs sudo umount /ipns sudo umount /mfs ``` #### Mounting fails with "error mounting: could not resolve name" Make sure your node's IPNS address has a directory published: ``` $ mkdir hello/; echo 'hello' > hello/hello.txt; ipfs add -rQ ./hello/ QmU5PLEGqjetW4RAmXgHpEFL7nVCL3vFnEyrCKUfRk4MSq $ ipfs name publish QmU5PLEGqjetW4RAmXgHpEFL7nVCL3vFnEyrCKUfRk4MSq ``` If you manage to mount on other systems (or followed an alternative path to one above), please contribute to these docs :D ================================================ FILE: docs/gateway.md ================================================ # Gateway An IPFS Gateway acts as a bridge between traditional web browsers and IPFS. Through the gateway, users can browse files and websites stored in IPFS as if they were stored in a traditional web server. [More about Gateways](https://docs.ipfs.tech/concepts/ipfs-gateway/) and [addressing IPFS on the web](https://docs.ipfs.tech/how-to/address-ipfs-on-web/). Kubo's Gateway implementation follows [IPFS Gateway Specifications](https://specs.ipfs.tech/http-gateways/) and is tested with [Gateway Conformance Test Suite](https://github.com/ipfs/gateway-conformance). ### Local gateway By default, Kubo nodes run a [path gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#path-gateway) at `http://127.0.0.1:8080/` and a [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://localhost:8080/`. > [!CAUTION] > **For browsing websites, web apps, and dapps in a browser, use the subdomain > gateway** (`localhost`). Each content root gets its own > [web origin](https://developer.mozilla.org/en-US/docs/Web/Security/Same-origin_policy), > isolating localStorage, cookies, and session data between sites. > > **For file retrieval, use the path gateway** (`127.0.0.1`). Path gateways are > suited for downloading files or fetching [verifiable](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval) > content, but lack origin isolation (all content shares the same origin). Additional listening addresses and gateway behaviors can be set in the [config](#configuration) file. ### Public gateways IPFS Foundation [provides public gateways](https://docs.ipfs.tech/concepts/public-utilities/) at `https://ipfs.io` ([path](https://specs.ipfs.tech/http-gateways/path-gateway/)), `https://dweb.link` ([subdomain](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway)), and `https://trustless-gateway.link` ([trustless](https://specs.ipfs.tech/http-gateways/trustless-gateway/) only). If you've ever seen a link in the form `https://ipfs.io/ipfs/Qm...`, that's being served from a *public goods* gateway. There is a list of third-party public gateways provided by the IPFS community at https://ipfs.github.io/public-gateway-checker/ ## Configuration The `Gateway.*` configuration options are (briefly) described in the [config](https://github.com/ipfs/kubo/blob/master/docs/config.md#gateway) documentation, including a list of common [gateway recipes](https://github.com/ipfs/kubo/blob/master/docs/config.md#gateway-recipes). ### Debug The gateway's log level can be changed with this command: ``` > ipfs log level core/server debug ``` ## Running in Production When deploying Kubo's gateway in production, be aware of these important considerations: > [!IMPORTANT] > **Reverse Proxy:** When running Kubo behind a reverse proxy (such as nginx), > the original `Host` header **must** be forwarded to Kubo for > [`Gateway.PublicGateways`](config.md#gatewaypublicgateways) to work. > Kubo uses the `Host` header to match configured hostnames and detect > subdomain gateway patterns like `{cid}.ipfs.example.org` or DNSLink hostnames. > > If the `Host` header is not forwarded correctly, Kubo will not recognize > the configured gateway hostnames and requests may be handled incorrectly. > > If `X-Forwarded-Proto` is not set, redirects over HTTPS will use wrong protocol > and DNSLink names will not be inlined for subdomain gateways. > > Example: minimal nginx configuration for `example.org` > > ```nginx > server { > listen 80; > listen [::]:80; > > # IMPORTANT: Include wildcard to match subdomain gateway requests. > # The dot prefix matches both apex domain and all subdomains. > server_name .example.org; > > location / { > proxy_pass http://127.0.0.1:8080; > > # IMPORTANT: Forward the original Host header to Kubo. > # Without this, PublicGateways configuration will not work. > proxy_set_header Host $host; > > # IMPORTANT: X-Forwarded-Proto is required for correct behavior: > # - Redirects will use https:// URLs when set to "https" > # - DNSLink names will be inlined for subdomain gateways > # (e.g., /ipns/en.wikipedia-on-ipfs.org → en-wikipedia--on--ipfs-org.ipns.example.org) > proxy_set_header X-Forwarded-Proto $scheme; > proxy_set_header X-Forwarded-Host $host; > } > } > ``` > > Common mistakes to avoid: > > - **Missing wildcard in `server_name`:** Using only `server_name example.org;` > will not match subdomain requests like `{cid}.ipfs.example.org`. Always > include `*.example.org` or use the dot prefix `.example.org`. > > - **Wrong `Host` header value:** Using `proxy_set_header Host $proxy_host;` > sends the backend's hostname (e.g., `127.0.0.1:8080`) instead of the > original `Host` header. Always use `$host` or `$http_host`. > > - **Missing `Host` header entirely:** If `proxy_set_header Host` is not > specified, nginx defaults to `$proxy_host`, which breaks gateway routing. > [!IMPORTANT] > **Timeouts:** Configure [`Gateway.RetrievalTimeout`](config.md#gatewayretrievaltimeout) > to terminate stalled transfers (resets on each data write, catches unresponsive operations), > and [`Gateway.MaxRequestDuration`](config.md#gatewaymaxrequestduration) as a fallback > deadline (default: 1 hour, catches cases when other timeouts are misconfigured or fail to fire). > [!IMPORTANT] > **Rate Limiting:** Use [`Gateway.MaxConcurrentRequests`](config.md#gatewaymaxconcurrentrequests) > to protect against traffic spikes. > [!IMPORTANT] > **CDN/Cloudflare:** If using Cloudflare or other CDNs with > [deserialized responses](config.md#gatewaydeserializedresponses) enabled, review > [`Gateway.MaxRangeRequestFileSize`](config.md#gatewaymaxrangerequestfilesize) to avoid > excess bandwidth billing from range request bugs. Cloudflare users may need additional > protection via [Cloudflare Snippets](https://github.com/ipfs/boxo/issues/856#issuecomment-3523944976). ## Directories For convenience, the gateway (mostly) acts like a normal web-server when serving a directory: 1. If the directory contains an `index.html` file: 1. If the path does not end in a `/`, append a `/` and redirect. This helps avoid serving duplicate content from different paths.† 2. Otherwise, serve the `index.html` file. 2. Dynamically build and serve a listing of the contents of the directory. †This redirect is skipped if the query string contains a `go-get=1` parameter. See [PR#3963](https://github.com/ipfs/kubo/pull/3963) for details ## Static Websites You can use an IPFS gateway to serve static websites at a custom domain using [DNSLink](https://docs.ipfs.tech/concepts/glossary/#dnslink). See [Example: IPFS Gateway](https://dnslink.dev/#example-ipfs-gateway) for instructions. ## Filenames When downloading files, browsers will usually guess a file's filename by looking at the last component of the path. Unfortunately, when linking *directly* to a file (with no containing directory), the final component is just a CID (`bafy..` or `Qm...`). This isn't exactly user-friendly. To work around this issue, you can add a `filename=some_filename` parameter to your query string to explicitly specify the filename. For example: > https://ipfs.io/ipfs/QmfM2r8seH2GiRaC4esTjeraXEachRt8ZsSeGaWTPLyMoG?filename=hello_world.txt When you try to save above page, you browser will use passed `filename` instead of a CID. ## Downloads It is possible to skip browser rendering of supported filetypes (plain text, images, audio, video, PDF) and trigger immediate "save as" dialog by appending `&download=true`: > https://ipfs.io/ipfs/QmfM2r8seH2GiRaC4esTjeraXEachRt8ZsSeGaWTPLyMoG?filename=hello_world.txt&download=true ## Response Format An explicit response format can be requested using `?format=raw|car|..` URL parameter, or by sending `Accept: application/vnd.ipld.{format}` HTTP header with one of supported content types. ## Content-Types Majority of resources can be retrieved trustlessly by requesting specific content type via `Accept` header or `?format=raw|car|ipns-record` URL query parameter. See [trustless gateway specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/) and [verifiable retrieval documentation](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval) for more details. ### `application/vnd.ipld.raw` Returns a byte array for a single `raw` block. Sending such requests for `/ipfs/{cid}` allows for efficient fetch of blocks with data encoded in custom format, without the need for deserialization and traversal on the gateway. This is equivalent of `ipfs block get`. ### `application/vnd.ipld.car` Returns a [CAR](https://ipld.io/specs/transport/car/) stream for a DAG or a subset of it. The `dag-scope` parameter controls which blocks are included: `all` (default, entire DAG), `entity` (logical unit like a file), or `block` (single block). For [UnixFS](https://specs.ipfs.tech/unixfs/) files, `entity-bytes` enables byte range requests. See [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/) for details. This is a rough equivalent of `ipfs dag export`. ### `application/vnd.ipfs.ipns-record` Only works on `/ipns/{ipns-name}` content paths that use cryptographically signed [IPNS Records](https://specs.ipfs.tech/ipns/ipns-record/). Returns [IPNS Record in Protobuf Serialization Format](https://specs.ipfs.tech/ipns/ipns-record/#record-serialization-format) which can be verified on end client, without trusting gateway. ================================================ FILE: docs/http-rpc-clients.md ================================================ # HTTP/RPC Clients Kubo provides official HTTP RPC (`/api/v0`) clients for selected languages: | Language | Package Name | Github Repository | |:--------:|:-------------------:|--------------------------------------------| | JS | kubo-rpc-client | https://github.com/ipfs/js-kubo-rpc-client | | Go | `rpc` | [`../client/rpc`](../client/rpc) | There are community-maintained libraries for other languages, but the Kubo team does provide support for them, YMMV: - https://docs.ipfs.tech/reference/kubo-rpc-cli/ ================================================ FILE: docs/implement-api-bindings.md ================================================ # IPFS API Implementation Doc This short document aims to give a quick guide to anyone implementing API bindings for IPFS implementations-- in particular kubo. Sections: - IPFS Types - API Transports - API Commands - Implementing bindings for the HTTP API ## IPFS Types IPFS uses a set of value type that is useful to enumerate up front: - `` is unix-style path, beginning with `/ipfs//...` or `/ipns//...` or `/ipns//...`. - `` is a base58 encoded [multihash](https://github.com/multiformats/multihash) - `cid` is a [multibase](https://github.com/multiformats/multibase) encoded [CID](https://github.com/ipld/cid) - a self-describing content-addressing identifier A note on streams: IPFS is a streaming protocol. Everything about it can be streamed. When importing files, API requests should aim to stream the data in, and handle back-pressure correctly, so that the IPFS node can handle it sequentially without too much memory pressure. (If using HTTP, this is typically handled for you by writes to the request body blocking.) ## API Transports Like with everything else, IPFS aims to be flexible regarding the API transports. Currently, the [kubo](https://github.com/ipfs/kubo) implementation supports both an in-process API and an HTTP API. More can be added easily, by mapping the API functions over a transport. (This is similar to how gRPC is also _mapped on top of transports_, like HTTP). Mapping to a transport involves leveraging the transport's features to express function calls. For example: #### CLI API Transport In the commandline, IPFS uses a traditional flag and arg-based mapping, where: - the first arguments select the command, as in git - e.g. `ipfs dag get` - the flags specify options - e.g. `--enc=protobuf -q` - the rest are positional arguments - e.g. `ipfs key rename ` - files are specified by filename, or through stdin (NOTE: When kubo runs the daemon, the CLI API is converted to HTTP calls. otherwise, they execute in the same process) #### HTTP API Transport In HTTP, our API layering uses a REST-like mapping, where: - the URL path selects the command - e.g `/object/get` - the URL query string implements option arguments - e.g. `&enc=protobuf&q=true` - the URL query also implements positional arguments - e.g. `&arg=&arg=add-link&arg=foo&arg=` - the request body streams file data - reads files or stdin - multiple streams are muxed with multipart (todo: add tar stream support) ## API Commands There is a "standard IPFS API" which is currently defined as "all the commands exposed by the kubo implementation". There are auto-generated [API Docs](https://ipfs.io/docs/api/). You can Also see [a listing here](https://github.com/ipfs/kubo/blob/94b832df861728c65e912935641d08880c341e0a/core/commands/root.go#L96-L130), or get a list of commands by running `ipfs commands` locally. ## Implementing bindings for the HTTP API As mentioned above, the API commands map to HTTP with: - the URL path selects the command - e.g `/object/get` - the URL query string implements option arguments - e.g. `&enc=protobuf&q=true` - the URL query also implements positional arguments - e.g. `&arg=&arg=add-link&arg=foo&arg=` - the request body streams file data - reads files or stdin - multiple streams are muxed with multipart (todo: add tar stream support) You can see the latest [list of our HTTP RPC clients here](http-rpc-clients.md) The Go implementation is good to answer harder questions, like how is multipart handled, or what headers should be set in edge conditions. But the javascript implementation is very concise, and easy to follow. ## Note on multipart + inspecting requests Despite all the generalization spoken about above, the IPFS API is actually very simple. You can inspect all the requests made with `nc` and the `--api` option (as of [this PR](https://github.com/ipfs/kubo/pull/1598), or `0.3.8`): ```sh > nc -l 5002 & > ipfs --api /ip4/127.0.0.1/tcp/5002 swarm addrs local --enc=json POST /api/v0/version?enc=json&stream-channels=true HTTP/1.1 Host: 127.0.0.1:5002 User-Agent: /kubo/0.14.0/ Content-Length: 0 Content-Type: application/octet-stream Accept-Encoding: gzip ``` The only hard part is getting the file streaming right. It is (now) fairly easy to stream files to kubo using multipart. Basically, we end up with HTTP requests like this: ```sh > nc -l 5002 & > ipfs --api /ip4/127.0.0.1/tcp/5002 add -r ~/demo/basic/test POST /api/v0/add?encoding=json&progress=true&r=true&stream-channels=true HTTP/1.1 Host: 127.0.0.1:5002 User-Agent: /kubo/0.14.0/ Transfer-Encoding: chunked Content-Disposition: form-data: name="files" Content-Type: multipart/form-data; boundary=2186ef15d8f2c4f100af72d6d345afe36a4d17ef11264ec5b8ec4436447f Accept-Encoding: gzip 1 - e5 -2186ef15d8f2c4f100af72d6d345afe36a4d17ef11264ec5b8ec4436447f Content-Disposition: form-data; name="file"; filename="test" Content-Type: multipart/mixed; boundary=acdb172fe12f25e8ffae9981ce6f4580abdefb0cae3ceebe464d802866be 9c --acdb172fe12f25e8ffae9981ce6f4580abdefb0cae3ceebe464d802866be Content-Disposition: file; filename="test%2Fbar" Content-Type: application/octet-stream 4 bar dc --acdb172fe12f25e8ffae9981ce6f4580abdefb0cae3ceebe464d802866be Content-Disposition: file; filename="test%2Fbaz" Content-Type: multipart/mixed; boundary=2799ac77a72ef7b8a0281945806b9f9a28f7681145aa8e91b052d599b2dd a0 --2799ac77a72ef7b8a0281945806b9f9a28f7681145aa8e91b052d599b2dd Content-Type: application/octet-stream Content-Disposition: file; filename="test%2Fbaz%2Fb" 4 bar a2 --2799ac77a72ef7b8a0281945806b9f9a28f7681145aa8e91b052d599b2dd Content-Disposition: file; filename="test%2Fbaz%2Ff" Content-Type: application/octet-stream 4 foo 44 --2799ac77a72ef7b8a0281945806b9f9a28f7681145aa8e91b052d599b2dd-- 9e --acdb172fe12f25e8ffae9981ce6f4580abdefb0cae3ceebe464d802866be Content-Disposition: file; filename="test%2Ffoo" Content-Type: application/octet-stream 4 foo 44 --acdb172fe12f25e8ffae9981ce6f4580abdefb0cae3ceebe464d802866be-- 44 --2186ef15d8f2c4f100af72d6d345afe36a4d17ef11264ec5b8ec4436447f-- 0 ``` Which produces: http://gateway.ipfs.io/ipfs/QmNtpA5TBNqHrKf3cLQ1AiUKXiE4JmUodbG5gXrajg8wdv ================================================ FILE: docs/libp2p-resource-management.md ================================================ # libp2p Network Resource Manager (`Swarm.ResourceMgr`) ## Purpose The purpose of this document is to provide more information about the [libp2p Network Resource Manager](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#readme) and how it's integrated into Kubo so that Kubo users can understand and configure it appropriately. ## 🙋 Help! The resource manager is protecting my node but I want to understand more The resource manager is generally a *feature* to bound libp2p's resources, whether from bugs, unintentionally misbehaving peers, or intentional Denial of Service attacks. Good places to start are: 1. Understand [how the resource manager is configured](#levels-of-configuration). 2. Understand [how to read the log message](#what-do-these-protected-from-exceeding-resource-limits-log-messages-mean) 3. Understand [how to inspect and change limits](#user-supplied-override-limits) ## Table of Contents - [Purpose](#purpose) - [🙋 Help! The resource manager is protecting my node but I want to understand more](#-help--the-resource-manager-is-protecting-my-node-but-i-want-to-understand-more) - [Table of Contents](#table-of-contents) - [Levels of Configuration](#levels-of-configuration) - [Approach](#approach) - [Computed Default Limits](#computed-default-limits) - [User Supplied Override Limits](#user-supplied-override-limits) - [FAQ](#faq) - [What do these "Protected from exceeding resource limits" log messages mean?](#what-do-these-protected-from-exceeding-resource-limits-log-messages-mean) - [How does one see the Active Limits?](#how-does-one-see-the-active-limits) - [How does one see the Computed Default Limits?](#how-does-one-see-the-computed-default-limits) - [How does one monitor libp2p resource usage?](#how-does-one-monitor-libp2p-resource-usage) - [How does the resource manager (ResourceMgr) relate to the connection manager (ConnMgr)?](#how-does-the-resource-manager-resourcemgr-relate-to-the-connection-manager-connmgr) - [What are the "Application error 0x0 (remote) ... cannot reserve ..." messages?](#what-are-the-application-error-0x0-remote--cannot-reserve--messages) - [History](#history) ## Levels of Configuration See also the [`Swarm.ResourceMgr` config docs](./config.md#swarmresourcemgr). ### Approach libp2p's resource manager provides tremendous flexibility but also adds complexity. There are these levels of limit configuration for resource management protection: 1. "The user who does nothing" - In this case Kubo attempts to give some sane defaults discussed below based on the amount of memory and file descriptors their system has. This should protect the node from many attacks. 2. "Slightly more advanced user" - Where the defaults aren't good enough, a good set of higher-level "knobs" are exposed to satisfy most use cases without requiring users to wade into all the intricacies of libp2p's resource manager. The "knobs"/inputs are `Swarm.ResourceMgr.MaxMemory` and `Swarm.ResourceMgr.MaxFileDescriptors` as described below. 3. "Power user" - They [specify override limits](#user-supplied-override-limits) and own their own destiny without Kubo getting in the way. ### Computed Default Limits With the `Swarm.ResourceMgr.MaxMemory` and `Swarm.ResourceMgr.MaxFileDescriptors` inputs defined, [resource manager limits](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#limits) are created at the [system](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#the-system-scope), [transient](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#the-transient-scope), and [peer](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#peer-scopes) scopes. Other scopes are ignored (by being set to "unlimited"). The reason these scopes are chosen is because: - `system` - This gives us the coarse-grained control we want so we can reason about the system as a whole. It is the backstop, and allows us to reason about resource consumption more easily since don't have think about the interaction of many other scopes. - `transient` - Limiting connections that are in process of being established provides backpressure so not too much work queues up. - `peer` - The peer scope doesn't protect us against intentional DoS attacks. It's just as easy for an attacker to send 100 requests/second with 1 peerId vs. 10 requests/second with 10 peers. We are reliant on the system scope for protection here in the malicious case. The reason for having a peer scope is to protect against unintentional DoS attacks (e.g., bug in a peer which is causing it to "misbehave"). In the unintentional case, we want to make sure a "misbehaving" node doesn't consume more resources than necessary. Within these scopes, limits are set on: 1. [memory](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#memory) 2. [file descriptors (FD)](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#file-descriptors) 3. [*inbound* connections](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#connections). Limits are set based on the `Swarm.ResourceMgr.MaxMemory` and `Swarm.ResourceMgr.MaxFileDescriptors` inputs above. There are also some special cases where minimum values are enforced. For example, Kubo maintainers have found in practice that it's a footgun to have too low of a value for `System.ConnsInbound` and a default minimum is used. (See [core/node/libp2p/rcmgr_defaults.go](https://github.com/ipfs/kubo/blob/master/core/node/libp2p/rcmgr_defaults.go) for specifics.) We trust this node to behave properly and thus don't limit *outbound* connection/stream limits. We apply any limits that libp2p has for its protocols/services since we assume libp2p knows best here. Source: [core/node/libp2p/rcmgr_defaults.go](https://github.com/ipfs/kubo/blob/master/core/node/libp2p/rcmgr_defaults.go) ### User Supplied Override Limits A user who wants fine control over the limits used by the go-libp2p resource manager can specify overrides to the [computed default limits](#computed-default-limits). This is done by defining limits in ``$IPFS_PATH/libp2p-resource-limit-overrides.json``. These values trump anything else and are parsed directly by go-libp2p. (See the [go-libp2p Resource Manager README](https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/README.md) for formatting.) ## FAQ ### What do these "Protected from exceeding resource limits" log messages mean? "Protected from exceeding resource limits" log messages denote that the resource manager is working and that it prevented additional resources from being used beyond the set limits. Per [libp2p code](https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/scope.go), these messages take the form of "$scope: cannot reserve $limitKey". As an example: > Protected from exceeding resource limits 2 times: "system: cannot reserve inbound connection: resource limit exceeded" This means that there were 2 recent occurrences where the libp2p resource manager prevented an inbound connection at the "system" [scope](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#resource-scopes). Specifically the ``System.ConnsInbound`` limit was hit. This can be analyzed by viewing the limit and current usage with `ipfs swarm resources`. `System.ConnsInbound` is likely close or at the limit value. The simplest way to identify all resources across all scopes that are close to exceeding their limit (>90% usage) is with a command like `ipfs swarm resources | egrep "9.\..%"` Sources: * [kubo resource manager logging](https://github.com/ipfs/kubo/blob/master/core/node/libp2p/rcmgr_logging.go) * [libp2p resource manager messages](https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/scope.go) ### How does one see the Active Limits? A dump of what limits are actually being used by the resource manager ([Computed Default Limits](#computed-default-limits) + [User Supplied Override Limits](#user-supplied-override-limits)) can be obtained by `ipfs swarm resources`. ### How does one see the Computed Default Limits? This can be observed [seeing the active limits](#how-does-one-see-the-active-limits) assuming one hasn't detoured into "power user" mode with [User Supplied Override Limits](#user-supplied-override-limits). ### How does one monitor libp2p resource usage? For [monitoring libp2p resource usage](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#monitoring), various `*rcmgr_*` metrics can be accessed as the Prometheus endpoint at `{Addresses.API}/debug/metrics/prometheus` (default: `http://127.0.0.1:5001/debug/metrics/prometheus`). There are also [pre-built Grafana dashboards](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager/obs/grafana-dashboards) that can be added to a Grafana instance. A textual view of current resource usage and a list of services, protocols, and peers can be obtained via `ipfs swarm stats --help` ### How does the resource manager (ResourceMgr) relate to the connection manager (ConnMgr)? As discussed [here](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#connmanager-vs-resource-manager) these are separate systems in go-libp2p. Kubo performs sanity checks to ensure that some of the hard limits of the ResourceMgr are sufficiently greater than the soft limits of the ConnMgr. The soft limit of `Swarm.ConnMgr.HighWater` needs to be less than the resource manager hard limit `System.ConnsInbound` for the configuration to make sense. This ensures the ConnMgr cleans up connections based on connection priorities before the hard limits of the ResourceMgr are applied. If `Swarm.ConnMgr.HighWater` is greater than resource manager's `System.ConnsInbound`, existing low-priority idle connections can prevent new high-priority connections from being established. The ResourceMgr doesn't know that the new connection is high priority and simply blocks it because of the limit its enforcing. To ensure the ConnMgr and ResourceMgr are congruent, the ResourceMgr [computed default limits](#computed-default-limits) are adjusted such that: 1. `System.ConnsInbound` >= `max(Swarm.ConnMgr.HighWater * 2, DefaultResourceMgrMinInboundConns)` AND 2. `System.StreamsInbound` is greater than any new/adjusted `Swarm.ResourceMgr.Limits.System.ConnsInbound` value so that there's enough streams per connection. Source: [core/node/libp2p/rcmgr_defaults.go](https://github.com/ipfs/kubo/blob/master/core/node/libp2p/rcmgr_defaults.go) ### What are the "Application error 0x0 (remote) ... cannot reserve ..." messages? These are messages coming from old (pre go-libp2p 0.26) *remote* go-libp2p peers (likely another older Kubo node) with the resource manager enabled on why it failed to establish a connection. This can be confusing, but these `Application error 0x0 (remote) ... cannot reserve ...` messages can occur even if your local node has the resource manager disabled. You can distinguish resource manager messages originating from your local node if they're from the `resourcemanager` / `libp2p/rcmgr_logging.go` logger or you see the string that is unique to Kubo (and not in go-libp2p): "Protected from exceeding resource limits". See more info in this go-libp2p issue ([#1928](https://github.com/libp2p/go-libp2p/issues/1928)). go-libp2p 0.26 / Kubo 0.19 onwards this confusing error message was removed. ## History Kubo first [exposed this functionality in Kubo 0.13](./changelogs/v0.13.md#-libp2p-network-resource-manager-swarmresourcemgr), but it was disabled by default. It was then enabled by default in [Kubo 0.17](./changelogs/v0.17.md#libp2p-resource-management-enabled-by-default). Until that point, Kubo was vulnerable to unbound resource usage which could bring down nodes. Introducing limits like this by default after the fact is tricky, which is why there have been changes and improvements afterwards. The general trend since 0.17 with (0.18)[./changeloges/v0.18.md#improving-libp2p-resource-management-integration] and 0.19 has been to simplify and provide less options (and footguns!) for users and better documentation. ================================================ FILE: docs/metrics.md ================================================ ## Kubo metrics By default, a Prometheus endpoint is exposed by Kubo at `http://127.0.0.1:5001/debug/metrics/prometheus`. It includes default [Prometheus Go client metrics](https://prometheus.io/docs/guides/go-application/) + Kubo-specific metrics listed below. ### Table of Contents - [DHT RPC](#dht-rpc) - [Inbound RPC metrics](#inbound-rpc-metrics) - [Outbound RPC metrics](#outbound-rpc-metrics) - [Provide](#provide) - [Legacy Provider](#legacy-provider) - [DHT Provider](#dht-provider) - [Gateway (`boxo/gateway`)](#gateway-boxogateway) - [HTTP metrics](#http-metrics) - [Blockstore cache metrics](#blockstore-cache-metrics) - [Backend metrics](#backend-metrics) - [Generic HTTP Servers](#generic-http-servers) - [Core HTTP metrics](#core-http-metrics-ipfs_http_) - [HTTP Server metrics](#http-server-metrics-http_server_) - [OpenTelemetry Metadata](#opentelemetry-metadata) > [!WARNING] > This documentation is incomplete. For an up-to-date list of metrics available at daemon startup, see [test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_measure_profile](https://github.com/ipfs/kubo/blob/master/test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_measure_profile). > > Additional metrics may appear during runtime as some components (like boxo/gateway) register metrics only after their first event occurs (e.g., HTTP request/response). ## DHT RPC Metrics from `go-libp2p-kad-dht` for DHT RPC operations: ### Inbound RPC metrics - `rpc_inbound_messages_total` - Counter: total messages received per RPC - `rpc_inbound_message_errors_total` - Counter: total errors for received messages - `rpc_inbound_bytes_[bucket|sum|count]` - Histogram: distribution of received bytes per RPC - `rpc_inbound_request_latency_[bucket|sum|count]` - Histogram: latency distribution for inbound RPCs ### Outbound RPC metrics - `rpc_outbound_messages_total` - Counter: total messages sent per RPC - `rpc_outbound_message_errors_total` - Counter: total errors for sent messages - `rpc_outbound_requests_total` - Counter: total requests sent - `rpc_outbound_request_errors_total` - Counter: total errors for sent requests - `rpc_outbound_bytes_[bucket|sum|count]` - Histogram: distribution of sent bytes per RPC - `rpc_outbound_request_latency_[bucket|sum|count]` - Histogram: latency distribution for outbound RPCs ## Provide ### Legacy Provider Metrics for the legacy provider system when `Provide.DHT.SweepEnabled=false`: - `provider_reprovider_provide_count` - Counter: total successful provide operations since node startup - `provider_reprovider_reprovide_count` - Counter: total reprovide sweep operations since node startup ### DHT Provider Metrics for the DHT provider system when `Provide.DHT.SweepEnabled=true`: - `provider_provides_total` - Counter: total successful provide operations since node startup (includes both one-time provides and periodic provides done on `Provide.DHT.Interval`) > [!NOTE] > These metrics are exposed by [go-libp2p-kad-dht](https://github.com/libp2p/go-libp2p-kad-dht/). You can enable debug logging for DHT provider activity with `GOLOG_LOG_LEVEL=dht/provider=debug`. ## Gateway (`boxo/gateway`) > [!TIP] > These metrics are limited to [IPFS Gateway](https://specs.ipfs.tech/http-gateways/) endpoints. For general HTTP metrics across all endpoints, consider using a reverse proxy. Gateway metrics appear after the first HTTP request is processed: ### HTTP metrics - `ipfs_http_gw_responses_total{code}` - Counter: total HTTP responses by status code - `ipfs_http_gw_retrieval_timeouts_total{code,truncated}` - Counter: requests that timed out during content retrieval - `ipfs_http_gw_concurrent_requests` - Gauge: number of requests currently being processed ### Blockstore cache metrics - `ipfs_http_blockstore_cache_hit` - Counter: global block cache hits - `ipfs_http_blockstore_cache_requests` - Counter: global block cache requests ### Backend metrics - `ipfs_gw_backend_api_call_duration_seconds_[bucket|sum|count]{backend_method}` - Histogram: time spent in IPFSBackend API calls ## Generic HTTP Servers > [!TIP] > The metrics below are not very useful and exist mostly for historical reasons. If you need non-gateway HTTP metrics, it's better to put a reverse proxy in front of Kubo and use its metrics. ### Core HTTP metrics (`ipfs_http_*`) Prometheus metrics for the HTTP API exposed at port 5001: - `ipfs_http_requests_total{method,code,handler}` - Counter: total HTTP requests (Legacy - new metrics are provided by boxo/gateway for gateway traffic) - `ipfs_http_request_duration_seconds[_sum|_count]{handler}` - Summary: request processing duration - `ipfs_http_request_size_bytes[_sum|_count]{handler}` - Summary: request body sizes - `ipfs_http_response_size_bytes[_sum|_count]{handler}` - Summary: response body sizes ### HTTP Server metrics (`http_server_*`) Additional HTTP instrumentation for all handlers (Gateway, API commands, etc.): - `http_server_request_body_size_bytes_[bucket|count|sum]` - Histogram: distribution of request body sizes - `http_server_request_duration_seconds_[bucket|count|sum]` - Histogram: distribution of request processing times - `http_server_response_body_size_bytes_[bucket|count|sum]` - Histogram: distribution of response body sizes These metrics are automatically added to Gateway handlers, Hostname Gateway, Libp2p Gateway, and API command handlers. > [!NOTE] > The `server_address` label from `otelhttp` is dropped via an OTel SDK View to prevent cardinality explosion on subdomain gateways (where each unique `Host` header creates a new time series). All handlers include a `server_domain` label instead: > > - Gateway and Hostname Gateway handlers group requests by their matching [`Gateway.PublicGateways`](config.md#gatewaypublicgateways) domain suffix (e.g., `dweb.link`, `ipfs.io`). Unmatched hosts are labeled `localhost`, `loopback`, or `other`. > - The RPC API handler uses `api`. > - The Libp2p Gateway handler uses `libp2p`. ## OpenTelemetry Metadata Kubo uses Prometheus for metrics collection for historical reasons, but OpenTelemetry metrics are automatically exposed through the same Prometheus endpoint. These metadata metrics provide context about the instrumentation: - `otel_scope_info` - Information about instrumentation libraries producing metrics - `target_info` - Service metadata including version and instance information ================================================ FILE: docs/p2p-tunnels.md ================================================ # P2P Tunnels Kubo supports tunneling TCP connections through libp2p streams, similar to SSH port forwarding (`ssh -L`). This allows exposing local services to remote peers and forwarding remote services to local ports. - [Why P2P Tunnels?](#why-p2p-tunnels) - [Quick Start](#quick-start) - [Background Mode](#background-mode) - [Foreground Mode](#foreground-mode) - [systemd Integration](#systemd-integration) - [Security Considerations](#security-considerations) - [Troubleshooting](#troubleshooting) ## Why P2P Tunnels? Unlike traditional SSH tunnels, libp2p-based tunnels do not require: - **No public IP or open ports**: The server does not need a static IP address or port forwarding configured on the router. Connectivity to peers behind NAT is facilitated by [Direct Connection Upgrade through Relay (DCUtR)](https://github.com/libp2p/specs/blob/master/relay/DCUtR.md), which enables NAT hole-punching. - **No DNS or IP address management**: All you need is the server's PeerID and an agreed-upon protocol name (e.g., `/x/ssh`). Kubo handles peer discovery and routing via the [Amino DHT](https://specs.ipfs.tech/routing/kad-dht/). - **Simplified firewall rules**: Since connections are established through libp2p's existing swarm connections, no additional firewall configuration is needed beyond what Kubo already requires. This makes p2p tunnels useful for connecting to machines on home networks, behind corporate firewalls, or in environments where traditional port forwarding is not available. ## Quick Start Enable the experimental feature: ```console $ ipfs config --json Experimental.Libp2pStreamMounting true ``` Test with netcat (`nc`) - no services required: **On the server:** ```console $ ipfs p2p listen /x/test /ip4/127.0.0.1/tcp/9999 $ nc -l -p 9999 ``` **On the client:** Replace `$SERVER_ID` with the server's peer ID (get it with `ipfs id -f "\n"` on the server). ```console $ ipfs p2p forward /x/test /ip4/127.0.0.1/tcp/9998 /p2p/$SERVER_ID $ nc 127.0.0.1 9998 ``` Type in either terminal and the text appears in the other. Use Ctrl+C to exit. ## Background Mode By default, `ipfs p2p listen` and `ipfs p2p forward` register the tunnel with the daemon and return immediately. The tunnel persists until explicitly closed with `ipfs p2p close` or the daemon shuts down. This example exposes a local SSH server (listening on `localhost:22`) to a remote peer. The same pattern works for any TCP service. **On the server** (the machine running SSH): Register a p2p listener that forwards incoming connections to the local SSH server. The protocol name `/x/ssh` is an arbitrary identifier that both peers must agree on (the `/x/` prefix is required for custom protocols). ```console $ ipfs p2p listen /x/ssh /ip4/127.0.0.1/tcp/22 ``` **On the client:** Create a local port (`2222`) that tunnels through libp2p to the server's SSH service. ```console $ ipfs p2p forward /x/ssh /ip4/127.0.0.1/tcp/2222 /p2p/$SERVER_ID ``` Now connect to SSH through the tunnel: ```console $ ssh user@127.0.0.1 -p 2222 ``` **Other services:** To tunnel a different service, change the port and protocol name. For example, to expose a web server on port 8080, use `/x/mywebapp` and `/ip4/127.0.0.1/tcp/8080`. ## Foreground Mode Use `--foreground` (`-f`) to block until interrupted. The tunnel is automatically removed when the command exits: ```console $ ipfs p2p listen /x/ssh /ip4/127.0.0.1/tcp/22 --foreground Listening on /x/ssh, forwarding to /ip4/127.0.0.1/tcp/22, waiting for interrupt... ^C Received interrupt, removing listener for /x/ssh ``` The listener/forwarder is automatically removed when: - The command receives Ctrl+C or SIGTERM - `ipfs p2p close` is called - The daemon shuts down This mode is useful for systemd services and scripts that need cleanup on exit. ### systemd Integration The `--foreground` flag enables clean integration with systemd. The examples below show how to run `ipfs p2p listen` as a user service that starts automatically when the IPFS daemon is ready. Ensure IPFS daemon runs as a systemd user service. See [misc/README.md](https://github.com/ipfs/kubo/blob/master/misc/README.md#systemd) for setup instructions and where to place unit files. #### P2P listener with path-based activation Use a `.path` unit to wait for the daemon's RPC API to be ready before starting the p2p listener. **`ipfs-p2p-tunnel.path`**: ```systemd [Unit] Description=Monitor for IPFS daemon startup After=ipfs.service Requires=ipfs.service [Path] PathExists=%h/.ipfs/api Unit=ipfs-p2p-tunnel.service [Install] WantedBy=default.target ``` The `%h` specifier expands to the user's home directory. If you use a custom `IPFS_PATH`, adjust accordingly. **`ipfs-p2p-tunnel.service`**: ```systemd [Unit] Description=IPFS p2p tunnel Requires=ipfs.service [Service] ExecStart=ipfs p2p listen /x/ssh /ip4/127.0.0.1/tcp/22 -f Restart=on-failure RestartSec=10 [Install] WantedBy=default.target ``` #### Enabling the services ```console $ systemctl --user enable ipfs.service $ systemctl --user enable ipfs-p2p-tunnel.path $ systemctl --user start ipfs.service ``` The path unit monitors `~/.ipfs/api` and starts `ipfs-p2p-tunnel.service` once the file exists. ## Security Considerations > [!WARNING] > This feature provides CLI and HTTP RPC users with the ability to set up port > forwarding for localhost and LAN ports. If you enable this and plan to expose > CLI or HTTP RPC to other users or machines, secure the RPC API using > [`API.Authorizations`](https://github.com/ipfs/kubo/blob/master/docs/config.md#apiauthorizations) > or custom auth middleware. ## Troubleshooting ### Foreground listener stops when terminal closes When using `--foreground`, the listener stops if the terminal closes. For persistent foreground listeners, use a systemd service, `nohup`, `tmux`, or `screen`. Without `--foreground`, the listener persists in the daemon regardless of terminal state. ### Connection refused errors Verify: 1. The experimental feature is enabled: `ipfs config Experimental.Libp2pStreamMounting` 2. The listener is active: `ipfs p2p ls` 3. Both peers can connect: `ipfs swarm connect /p2p/$PEER_ID` ### Persistent tunnel configuration There is currently no way to define tunnels in the Kubo JSON config file. Use `--foreground` mode with a systemd service for persistent tunnels. Support for configuring tunnels via JSON config may be added in the future (see [kubo#5460](https://github.com/ipfs/kubo/issues/5460) - PRs welcome!). ================================================ FILE: docs/plugins.md ================================================ # Plugins Since 0.4.11 Kubo has an experimental plugin system that allows augmenting the daemons functionality without recompiling. When an IPFS node is started, it will load plugins from the `$IPFS_PATH/plugins` directory (by default `~/.ipfs/plugins`). **Table of Contents** - [Plugin Types](#plugin-types) - [IPLD](#ipld) - [Datastore](#datastore) - [Available Plugins](#available-plugins) - [Installing Plugins](#installing-plugins) - [External Plugin](#external-plugin) - [In-tree](#in-tree) - [Out-of-tree](#out-of-tree) - [Preloaded Plugins](#preloaded-plugins) - [Creating A Plugin](#creating-a-plugin) ## Plugin Types Plugins can implement one or more plugin types, defined in the [plugin](https://godoc.org/github.com/ipfs/kubo/plugin) package. ### IPLD IPLD plugins add support for additional formats to `ipfs dag` and other IPLD related commands. ### Datastore Datastore plugins add support for additional datastore backends. ### Tracer (experimental) Tracer plugins allow injecting an opentracing backend into Kubo. ### Daemon Daemon plugins are started when the Kubo daemon is started and are given an instance of the CoreAPI. This should make it possible to build an ipfs-based application without IPC and without forking Kubo. Note: We eventually plan to make Kubo usable as a library. However, this plugin type is likely the best interim solution. ### fx (experimental) Fx plugins let you customize the [fx](https://pkg.go.dev/go.uber.org/fx) dependency graph and configuration, by customizing the`fx.Option`s that are passed to `fx` when the Kubo node is initialized. For example, you can override an interface such as [exchange.Interface](https://github.com/ipfs/go-ipfs-exchange-interface) or [pin.Pinner](https://github.com/ipfs/go-ipfs-pinner) with a custom implementation by appending an option like `fx.Decorate(func() exchange.Interface { return customExchange })`. Fx supports some advanced customization. Simple interface replacements like above are unlikely to break in the future, but the more invasive your changes, the more likely they are to break between releases. Kubo cannot guarantee backwards compatibility for `fx` customizations. Fx options are applied across every execution of the `ipfs` binary, including: - Repo initialization - Daemon - Applying migrations - etc. So if you plug in a blockservice that disallows non-allowlisted CIDs, then this may break migrations that fetch migration code over the IPFS network. ### Internal (never stable) Internal plugins are like daemon plugins _except_ that they can access, replace, and modify all internal state. Use this plugin type to extend Kubo in arbitrary ways. However, be aware that your plugin will likely break every time Kubo updated. ## Configuration Plugins can be configured in the `Plugins` section of the config file. Here, plugins can be: 1. Passed an arbitrary config object via the `Config` field. 2. Disabled via the `Disabled` field. Example: ```js { // ... "Plugins": { "Plugins": { // plugin named "plugin-foo" "plugin-foo": { "Config": { /* arbitrary json */ } }, // plugin named "plugin-bar" "plugin-bar": { "Disabled": true // "plugin-bar" will not be loaded } } } } ``` ## Available Plugins | Name | Type | Preloaded | Description | |---------------------------------------------------------------------------------|-----------|-----------|------------------------------------------------| | [git](https://github.com/ipfs/kubo/tree/master/plugin/plugins/git) | IPLD | x | An IPLD format for git objects. | | [badgerds](https://github.com/ipfs/kubo/tree/master/plugin/plugins/badgerds) | Datastore | x | A high performance but experimental datastore. | | [flatfs](https://github.com/ipfs/kubo/tree/master/plugin/plugins/flatfs) | Datastore | x | A stable filesystem-based datastore. | | [levelds](https://github.com/ipfs/kubo/tree/master/plugin/plugins/levelds) | Datastore | x | A stable, flexible datastore backend. | | [jaeger](https://github.com/ipfs/go-jaeger-plugin) | Tracing | | An opentracing backend. | | [telemetry](https://github.com/ipfs/kubo/tree/master/plugin/plugins/telemetry) | Telemetry | x | Collects anonymized usage data for Kubo development. | * **Preloaded** plugins are built into the Kubo binary and do not need to be installed separately. At the moment, all in-tree plugins are preloaded. ## Installing Plugins Kubo supports two types of plugins: External and Preloaded. * External plugins must be installed in `$IPFS_PATH/plugins/` (usually `~/.ipfs/plugins/`). * Preloaded plugins are built-into the Kubo when it's compiled. ### External Plugin The advantage of an external plugin is that it can be built, packaged, and installed independently of Kubo. Unfortunately, this method is only supported on Linux and MacOS at the moment. Users of other operating systems should follow the instructions for preloaded plugins. #### In-tree To build plugins included in [plugin/plugins](https://github.com/ipfs/kubo/tree/master/plugin/plugins), run: ```bash kubo$ make build_plugins kubo$ ls plugin/plugins/*.so ``` To install, copy desired plugins to `$IPFS_PATH/plugins`. For example: ```bash kubo$ mkdir -p ~/.ipfs/plugins/ kubo$ cp plugin/plugins/git.so ~/.ipfs/plugins/ kubo$ chmod +x ~/.ipfs/plugins/git.so # ensure plugin is executable ``` Finally, restart daemon if it is running. #### Out-of-tree To build out-of-tree plugins, use the plugin's Makefile if provided. Otherwise, you can manually build the plugin by running: ```bash myplugin$ go build -buildmode=plugin -o myplugin.so myplugin.go ``` Finally, as with in-tree plugins: 1. Install the plugin in `$IPFS_PATH/plugins`. 2. Mark the plugin as executable (`chmod +x $IPFS_PATH/plugins/myplugin.so`). 3. Restart your IPFS daemon (if running). ### Preloaded Plugins The advantages of preloaded plugins are: 1. They're bundled with the Kubo binary. 2. They work on all platforms. To preload a Kubo plugin: 1. Add the plugin to the preload list: `plugin/loader/preload_list` 2. Build ipfs ```bash kubo$ make build ``` You can also preload an in-tree but disabled-by-default plugin by adding it to the IPFS_PLUGINS variable. For example, to enable plugins foo, bar, and baz: ```bash kubo$ make build IPFS_PLUGINS="foo bar baz" ``` ## Creating A Plugin To create your own out-of-tree plugin, use the [example plugin](https://github.com/ipfs/go-ipfs-example-plugin/) as a starting point. When you're ready, submit a PR adding it to the list of [available plugins](#available-plugins). ================================================ FILE: docs/production/reverse-proxy.md ================================================ # IPFS & Reverse HTTP Proxies When run in production environments, go-ipfs should generally be run behind a reverse HTTP proxy (usually NGINX). You may need a reverse proxy to: * Load balance requests across multiple go-ipfs daemons. * Cache responses. * Buffer requests, only releasing them to go-ipfs when complete. This can help protect go-ipfs from the [slowloris](https://en.wikipedia.org/wiki/Slowloris_(computer_security) attack. * Block content. * Rate limit and timeout requests. * Apply QoS rules (e.g., prioritize traffic for certain important IPFS resources). This document contains a collection of tips, tricks, and pitfalls when running a go-ipfs node behind a reverse HTTP proxy. **WARNING:** Due to [nginx#1293](https://trac.nginx.org/nginx/ticket/1293)/[go-ipfs#6402](https://github.com/ipfs/go-ipfs/issues/6402), parts of the go-ipfs API will not work correctly behind an NGINX reverse proxy as go-ipfs starts sending back a response before it finishes reading the request body. The gateway itself is unaffected. ## Peering Go-ipfs gateways behind a single load balancing reverse proxy should use the [peering](../config.md#peering) subsystem to peer with each other. That way, as long as one go-ipfs daemon has the content being requested, the others will be able to serve it. # Garbage Collection Gateways rarely store content permanently. However, running garbage collection can slow down a go-ipfs node significantly. If you've noticed this issue in production, consider "garbage collecting" by resetting the go-ipfs repo whenever you run out of space, instead of garbage collecting. 1. Initialize your gateways repo to some known-good state (possibly pre-seeding it with some content, a config, etc.). 2. When you start running low on space, for each load-balanced go-ipfs node: 1. Use the nginx API to set one of the upstream go-ipfs node's to "down". 2. Wait a minute to let go-ipfs finish processing any in-progress requests (or the short-lived ones, at least). 3. Take the go-ipfs node down. 4. Rollback the go-ipfs repo to the seed state. 5. Restart the go-ipfs daemon. 6. Update the nginx config, removing the "down" status from the node. This will effectively "garbage collect" without actually running the garbage collector. # Content Blocking TODO: * Filtering requests * Checking the X-IPFS-Path header in responses to filter again after resolving. # Subdomain Gateway TODO: Reverse proxies and the subdomain gateway. # Load balancing TODO: discuss load balancing based on the CID versus the source IP. ================================================ FILE: docs/provide-stats.md ================================================ # Provide Stats The `ipfs provide stat` command gives you statistics about your local provide system. This file provides a detailed explanation of the metrics reported by this command. ## Understanding the Metrics The statistics are organized into three types of measurements: ### Per-worker rates Metrics like "CIDs reprovided/min/worker" measure the throughput of a single worker processing one region. To estimate total system throughput, multiply by the number of active workers of that type (see [Workers stats](#workers-stats)). Example: If "CIDs reprovided/min/worker" shows 100 and you have 10 active periodic workers, your total reprovide throughput is approximately 1,000 CIDs/min. ### Per-region averages Metrics like "Avg CIDs/reprovide" measure properties of the work units (keyspace regions). These represent the average size or characteristics of a region, not a rate. Do NOT multiply these by worker count. Example: "Avg CIDs/reprovide: 250,000" means each region contains an average of 250,000 CIDs that get reprovided together as a batch. ### System totals Metrics like "Total CIDs provided" are cumulative counts since node startup. These aggregate all work across all workers over time. ## Connectivity ### Status Current connectivity status (`online`, `disconnected`, or `offline`) and when it last changed (see [provide connectivity status](./config.md#providedhtofflinedelay)). ## Queues ### Provide queue Number of CIDs waiting for initial provide, and the number of keyspace regions they're grouped into. ### Reprovide queue Number of regions with overdue reprovides. These regions missed their scheduled reprovide time and will be processed as soon as possible. If decreasing, the node is recovering from downtime. If increasing, either the node is offline or the provide system needs more workers (see [`Provide.DHT.MaxWorkers`](./config.md#providedhtmaxworkers) and [`Provide.DHT.DedicatedPeriodicWorkers`](./config.md#providedhtdedicatedperiodicworkers)). ## Schedule ### CIDs scheduled Total CIDs scheduled for reprovide. ### Regions scheduled Number of keyspace regions scheduled for reprovide. Each CID is mapped to a specific region, and all CIDs within the same region are reprovided together as a batch for efficient processing. ### Avg prefix length Average length of binary prefixes identifying the scheduled regions. Each keyspace region is identified by a binary prefix, and this shows the average prefix length across all regions in the schedule. Longer prefixes indicate the keyspace is divided into more regions (because there are more DHT servers in the swarm to distribute records across). ### Next region prefix Keyspace prefix of the next region to be reprovided. ### Next region reprovide When the next region is scheduled to be reprovided. ## Timings ### Uptime How long the provide system has been running since Kubo started, along with the start timestamp. ### Current time offset Elapsed time in the current reprovide cycle, showing cycle progress (e.g., '11h' means 11 hours into a 22-hour cycle, roughly halfway through). ### Cycle started When the current reprovide cycle began. ### Reprovide interval How often each CID is reprovided (the complete cycle duration). ## Network ### Avg record holders Average number of provider records successfully sent for each CID to distinct DHT servers. In practice, this is often lower than the [replication factor](#replication-factor) due to unreachable peers or timeouts. Matching the replication factor would indicate all DHT servers are reachable. Note: this counts successful sends; some DHT servers may have gone offline afterward, so actual availability may be lower. ### Peers swept Number of DHT servers to which we tried to send provider records in the last reprovide cycle (sweep). Excludes peers contacted during initial provides or DHT lookups. ### Full keyspace coverage Whether provider records were sent to all DHT servers in the swarm during the last reprovide cycle. If true, [peers swept](#peers-swept) approximates the total DHT swarm size over the last [reprovide interval](#reprovide-interval). ### Reachable peers Number and percentage of peers to which we successfully sent all provider records assigned to them during the last reprovide cycle. ### Avg region size Average number of DHT servers per keyspace region. ### Replication factor Target number of DHT servers to receive each provider record. ## Operations ### Ongoing provides Number of CIDs and regions currently being provided for the first time. More CIDs than regions indicates efficient batching. Each region provide uses a [burst worker](./config.md#providedhtdedicatedburstworkers). ### Ongoing reprovides Number of CIDs and regions currently being reprovided. Each region reprovide uses a [periodic worker](./config.md#providedhtdedicatedperiodicworkers). ### Total CIDs provided Total number of provide operations since node startup (includes both provides and reprovides). ### Total records provided Total provider records successfully sent to DHT servers since startup (includes reprovides). ### Total provide errors Number of failed region provide/reprovide operations since startup. Failed regions are automatically retried unless the node is offline. ### CIDs provided/min/worker Average rate of initial provides per minute per worker during the last reprovide cycle (excludes reprovides). Each worker handles one keyspace region at a time, providing all CIDs in that region. This measures the throughput of a single worker only. To estimate total system provide throughput, multiply by the number of active burst workers shown in [Workers stats](#workers-stats) (Burst > Active). Note: This rate only counts active time when initial provides are being processed. If workers are idle, actual throughput may be lower. ### CIDs reprovided/min/worker Average rate of reprovides per minute per worker during the last reprovide cycle (excludes initial provides). Each worker handles one keyspace region at a time, reproviding all CIDs in that region. This measures the throughput of a single worker only. To estimate total system reprovide throughput, multiply by the number of active periodic workers shown in [Workers stats](#workers-stats) (Periodic > Active). Example: If this shows 100 CIDs/min and you have 10 active periodic workers, your total reprovide throughput is approximately 1,000 CIDs/min. Note: This rate only counts active time when regions are being reprovided. If workers are idle due to network issues or queue exhaustion, actual throughput may be lower. ### Region reprovide duration Average time to reprovide all CIDs in a region during the last cycle. ### Avg CIDs/reprovide Average number of CIDs per region during the last reprovide cycle. This measures the average size of a region (how many CIDs are batched together), not a throughput rate. Do NOT multiply this by worker count. Combined with [Region reprovide duration](#region-reprovide-duration), this helps estimate per-worker throughput: dividing Avg CIDs/reprovide by Region reprovide duration gives CIDs/min/worker. ### Regions reprovided (last cycle) Number of regions reprovided in the last cycle. > [!NOTE] > (⚠️ 0.39 limitation) If this shows 1 region while using > [`Routing.AcceleratedDHTClient`](./config.md#routingaccelerateddhtclient), sweep mode lost > efficiency gains. Consider disabling the accelerated client. See [caveat 4](./config.md#routingaccelerateddhtclient). ## Workers ### Active workers Number of workers currently processing provide or reprovide operations. ### Free workers Number of idle workers not reserved for periodic or burst tasks. ### Workers stats Breakdown of worker status by type (periodic for scheduled reprovides, burst for initial provides). For each type: - **Active**: Currently processing operations (use this count when calculating total throughput from per-worker rates) - **Dedicated**: Reserved for this type - **Available**: Idle dedicated workers + [free workers](#free-workers) - **Queued**: 0 or 1 (workers acquired only when needed) The number of active workers determines your total system throughput. For example, if you have 10 active periodic workers, multiply [CIDs reprovided/min/worker](#cids-reprovidedminworker) by 10 to estimate total reprovide throughput. See [provide queue](#provide-queue) and [reprovide queue](#reprovide-queue) for regions waiting to be processed. ### Max connections/worker Maximum concurrent DHT server connections per worker when sending provider records for a region. ## Capacity Planning ### Estimating if your system can keep up with the reprovide schedule To check if your provide system has sufficient capacity: 1. Calculate required throughput: - Required CIDs/min = [CIDs scheduled](#cids-scheduled) / ([Reprovide interval](#reprovide-interval) in minutes) - Example: 67M CIDs / (22 hours × 60 min) = 50,758 CIDs/min needed 2. Calculate actual throughput: - Actual CIDs/min = [CIDs reprovided/min/worker](#cids-reprovidedminworker) × Active periodic workers - Example: 100 CIDs/min/worker × 256 active workers = 25,600 CIDs/min 3. Compare: - If actual < required: System is underprovisioned, increase [MaxWorkers](./config.md#providedhtmaxworkers) or [DedicatedPeriodicWorkers](./config.md#providedhtdedicatedperiodicworkers) - If actual > required: System has excess capacity - If [Reprovide queue](#reprovide-queue) is growing: System is falling behind ### Understanding worker utilization - High active workers with growing reprovide queue: Need more workers or network connectivity is limiting throughput - Low active workers with non-empty reprovide queue: Workers may be waiting for network or DHT operations - Check [Reachable peers](#reachable-peers) to diagnose network connectivity issues - (⚠️ 0.39 limitation) If [Regions scheduled](#regions-scheduled) shows 1 while using [`Routing.AcceleratedDHTClient`](./config.md#routingaccelerateddhtclient), consider disabling the accelerated client to restore sweep efficiency. See [caveat 4](./config.md#routingaccelerateddhtclient). ## See Also - [Provide configuration reference](./config.md#provide) - [Provide metrics for Prometheus](./metrics.md#provide) ================================================ FILE: docs/releases.md ================================================ # `kubo` Release Flow # Table of Contents - [`kubo` Release Flow](#kubo-release-flow) - [Table of Contents](#table-of-contents) - [Release Philosophy](#release-philosophy) - [Release Flow](#release-flow) - [Stage 0 - Automated Testing](#stage-0---automated-testing) - [Stage 1 - Internal Testing](#stage-1---internal-testing) - [Stage 2 - Community Dev Testing](#stage-2---community-dev-testing) - [Stage 3 - Community Prod Testing](#stage-3---community-prod-testing) - [Stage 4 - Release](#stage-4---release) - [Release Cycle](#release-cycle) - [Patch Releases](#patch-releases) - [Security Fix Policy](#security-fix-policy) - [Performing a Release](#performing-a-release) - [Release Version Numbers (aka semver)](#release-version-numbers-aka-semver) - [_Footnotes_](#footnotes) ## Release Philosophy `kubo` aims to have a release every six weeks, two releases per quarter. During these 6 week releases, we go through 4 different stages that allow us to test the new version against our test environments (unit, interop, integration), QA in our current production environment, IPFS apps (e.g. Desktop and WebUI) and with our community and _early testers_[1] that have IPFS running in production. We might expand the six-week release schedule in case of: - No new updates to be added - In case of a large community event that takes the core team availability away (e.g. IPFS Conf, Dev Meetings, IPFS Camp, etc.) ## Release Flow `kubo` releases come in 5 stages designed to gradually roll out changes and reduce the impact of any regressions that may have been introduced. If we need to merge non-trivial[2] changes during the process, we start over at stage 0. ![kubo-release-process-illustration](https://user-images.githubusercontent.com/618519/62986422-653fee00-bdf0-11e9-8f61-197117b61da2.png) ### Stage 0 - Automated Testing At this stage, we expect _all_ automated tests (interop, testlab, performance, etc.) to pass. ### Stage 1 - Internal Testing At this stage, we'll: 1. Start a partial-rollout to our own infrastructure. 2. Test against ipfs and ipfs-shipyard applications. **Goals:** 1. Make sure we haven't introduced any obvious regressions. 2. Test the release in an environment we can monitor and easily roll back (i.e. our own infra). ### Stage 2 - Community Dev Testing At this stage, we'll announce the impending release to the community and ask for beta testers. **Goal:** Test the release in as many non-production environments as possible. This is relatively low-risk but gives us a _breadth_ of testing internal testing can't. ### Stage 3 - Community Prod Testing At this stage, we consider the release to be "production-ready" and will ask the community and our early testers to (partially) deploy the release to their production infrastructure. **Goals:** 1. Test the release in some production environments with heavy workloads. 2. Partially roll-out an upgrade to see how it affects the network. 3. Retain the ability to ship last-minute fixes before the final release. ### Stage 4 - Release At this stage, the release is "battle-hardened" and ready for wide deployment. ## Release Cycle A full release process should take about 3 weeks, a week per stage 1-3. We will start a new process every 6 weeks, regardless of when the previous release landed unless it's still ongoing. ### Patch Releases If we encounter a serious bug in the stable latest release, we will create a patch release based on this release. For now, bug fixes will _not_ be backported to previous releases. Patch releases will usually follow a compressed release cycle and should take 2-3 days. In a patch release: 1. Automated and internal testing (stage 0 and 1) will be compressed into a few hours - ideally less than a day. 2. Stage 2 will be skipped. 3. Community production testing will be shortened to 1-2 days of opt-in testing in production (early testers can choose to pass). Some patch releases, especially ones fixing one or more complex bugs, may undergo the full release process. ## Security Fix Policy Any release may contain security fixes. Unless the fix addresses a bug being exploited in the wild, the fix will _not_ be called out in the release notes. Please make sure to update ASAP. By policy, the team will usually wait until about 2 weeks after the final release to announce any fixed security issues. However, depending on the impact and ease of discovery of the issue, the team may wait more or less time. It is important to always update to the latest version ASAP and file issues if you're unable to update for some reason. Finally, unless a security issue is actively being exploited or a significant number of users are unable to update to the latest version (e.g., due to a difficult migration, breaking changes, etc.), security fixes will _not_ be backported to previous releases. ## Performing a Release The release is managed by the `Lead Maintainer` for `kubo`. It starts with the opening of an issue containing the content available on the [RELEASE_ISSUE_TEMPLATE](./RELEASE_ISSUE_TEMPLATE.md) not more than **48 hours** after the previous release. This issue is pinned and labeled ["release"](https://github.com/ipfs/kubo/issues?utf8=%E2%9C%93&q=is%3Aissue+is%3Aopen+label%3Arelease). When the cycle is due to begin the 5 stages will be followed until the release is done. ## Release Version Numbers (aka semver) Until `kubo` 0.4.X, `kubo` was not using semver to communicate the type of release Post `kubo` 0.5.X, `kubo` will use semver. This means that patch releases will not contain any breaking changes nor new features. Minor releases might contain breaking changes and always contain some new feature Post `kubo` 1.X.X (future), `kubo` will use semver. This means that only major releases will contain breaking changes, minors will be reserved for new features and patches for bug fixes. We do not yet retroactively apply fixes to older releases (no Long Term Support releases for now), which means that we always recommend users to update to the latest, whenever possible. ---------------------------- ## _Footnotes_ - **[1]** - _early testers_ is an IPFS programme in which members of the community can self-volunteer to help test `kubo` Release Candidates. You find more info about it at [EARLY_TESTERS.md](./EARLY_TESTERS.md) - **[2]** - A non-trivial change is any change that could potentially introduce an issue not trivially caught by automated testing. This is up to the discretion of the Lead Maintainer but the assumption is that every change is non-trivial unless proven otherwise. ================================================ FILE: docs/releases_thunderdome.md ================================================ # Testing Kubo releases with Thunderdome This document is for running Thunderdome tests by release engineers as part of releasing Kubo. We use Thunderdome to replay ipfs.io gateway traffic in a controlled environment against two different versions of Kubo, and we record metrics and compare them to look for logic or performance regressions before releasing a new Kubo version. For background information about how Thunderdome works, see: https://github.com/ipfs-shipyard/thunderdome ## Prerequisites * Ensure you have access to the "IPFS Stewards" vault in 1Password, which contains the requisite AWS Console and API credentials * Ensure you have Docker and the Docker CLI installed * Checkout the Thunderdome repo locally (or `git pull` to ensure it's up-to-date) * Install AWS CLI v2: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html * Configure the AWS CLI * Configure the credentials as described in the [Thunderdome documentation](https://github.com/ipfs-shipyard/thunderdome/blob/main/cmd/thunderdome/README.md#credentials), using the credentials from 1Password * Make sure the `thunderdome` binary is up-to-date: `go build ./cmd/thunderdome` ## Add & run an experiment Create a new release configuration JSON in the `experiments/` directory, based on the most recent `kubo-release` configuration, and tweak as necessary. Generally we setup the targets to run a commit against the tag of the last release, such as: ```json "targets": [ { "name": "kubo190-4283b9", "description": "kubo 0.19.0-rc1", "build_from_git": { "repo": "https://github.com/ipfs/kubo.git", "commit":"4283b9d98f8438fc8751ccc840d8fc24eeae6f13" } }, { "name": "kubo181", "description": "kubo 0.18.", "build_from_git": { "repo": "https://github.com/ipfs/kubo.git", "tag":"v0.18.1" } } ] ``` Run the experiment (where `$EXPERIMENT_CONFIG_JSON` is a path to the config JSON created above): ```shell AWS_PROFILE=thunderdome ./thunderdome deploy --verbose --duration 120 $EXPERIMENT_CONFIG_JSON ``` This will build the Docker images, upload them to ECR, and then launch the experiment in Thunderdome. Once the experiment starts, the CLI will exit and the experiment will continue to run for the duration. ## Analyze Results Add a log entry in https://www.notion.so/ceb2047e79f2498494077a2739a6c493 and link to it from the release issue, so that experiment results are publicly visible. The `deploy` command will output a link to the Grafana dashboard for the experiment. We don't currently have rigorous acceptance criteria, so you should look for anomalies or changes in the metrics and make sure they are tolerable and explainable. Unexplainable anomalies should be noted in the log with a screenshot, and then root caused. ## Open a PR to merge the experiment config into Thunderdome This is important for both posterity, and so that someone else can sanity-check the test parameters. ================================================ FILE: docs/specifications/keystore.md ================================================ # ![](https://img.shields.io/badge/status-wip-orange.svg?style=flat-square) Keystore **Authors(s):** - [whyrusleeping](github.com/whyrusleeping) - [Hector Sanjuan](github.com/hsanjuan) **Abstract** This spec provides definitions and operations for the keystore feature in IPFS. # Table of Contents - [Goals](#goals) - [Planned Implementation](#planned-implementation) - [Key storage](#key-storage) - [Interface](#interface) - [Code changes and additions](#code-changes-and-additions) - [Structures](#structures) ## Goals To have a secure, simple and user-friendly way of storing and managing keys for use by ipfs. As well as the ability to share these keys, encrypt, decrypt, sign and verify data. ## Planned Implementation ### Key storage Storage layout and format is defined in the [`repository_fs`](repository_fs.md) part of the spec. ### Interface #### ipfs key ``` USAGE ipfs key - Create and list IPNS name keypairs ipfs key 'ipfs key gen' generates a new keypair for usage with IPNS and 'ipfs name publish'. > ipfs key gen --type=rsa --size=2048 mykey > ipfs name publish --key=mykey QmSomeHash 'ipfs key list' lists the available keys. > ipfs key list self mykey SUBCOMMANDS ipfs key export - Export a keypair ipfs key gen - Create a new keypair ipfs key import - Import a key and prints imported key id ipfs key list - List all local keypairs. ipfs key rename - Rename a keypair. ipfs key rm ... - Remove a keypair. ipfs key rotate - Rotates the IPFS identity. For more information about each command, use: 'ipfs key --help' ``` #### ipfs crypt **NOTE:** as of 2023 Q4, `ipfs crypt` commands are not implemented yet. ``` ipfs crypt - Perform cryptographic operations using ipfs keypairs SUBCOMMANDS: ipfs crypt sign - Generates a signature for the given data with a specified key ipfs crypt verify - Verify that the given data and signature match ipfs crypt encrypt - Encrypt the given data ipfs crypt decrypt - Decrypt the given data DESCRIPTION: `ipfs crypt` is a command used to perform various cryptographic operations using ipfs keypairs, including: signing, verifying, encrypting and decrypting. ``` #### Some subcommands: ##### ipfs key Gen ``` USAGE ipfs key gen - Create a new keypair SYNOPSIS ipfs key gen [--type= | -t] [--size= | -s] [--ipns-base=] [--] ARGUMENTS - name of key to create OPTIONS -t, --type string - type of the key to create: rsa, ed25519. Default: ed25519. -s, --size int - size of the key to generate. --ipns-base string - Encoding used for keys: Can either be a multibase encoded CID or a base58btc encoded multihash. Takes {b58mh|base36|k|base32|b...}. Default: base36. ``` * * * ##### Key Send ``` USAGE ipfs key - Create and list IPNS name keypairs SYNOPSIS ipfs key DESCRIPTION 'ipfs key gen' generates a new keypair for usage with IPNS and 'ipfs name publish'. > ipfs key gen --type=rsa --size=2048 mykey > ipfs name publish --key=mykey QmSomeHash 'ipfs key list' lists the available keys. > ipfs key list self mykey SUBCOMMANDS ipfs key export - Export a keypair ipfs key gen - Create a new keypair ipfs key import - Import a key and prints imported key id ipfs key list - List all local keypairs. ipfs key rename - Rename a keypair. ipfs key rm ... - Remove a keypair. ipfs key rotate - Rotates the IPFS identity. For more information about each command, use: 'ipfs key --help' ``` ##### Comments: Ensure that the user knows the implications of sending a key. * * * ##### Crypt Encrypt ``` ipfs crypt encrypt - Encrypt the given data with a specified key ARGUMENTS: data - The filename of the data to be encrypted ("-" for stdin) OPTIONS: -k, -key string - The name of the key to use for encryption (default: localkey) -o, -output string - The name of the output file (default: stdout) -c, -cipher string - The cipher to use for the operation -m, -mode string - The block cipher mode to use for the operation DESCRIPTION: 'ipfs crypt encrypt' is a command used to encrypt data so that only holders of a certain key can read it. ``` ##### Comments: This should probably just operate on raw data and not on DAGs. * * * ##### Other Interface Changes We will also need to make additions to support keys in other commands, these changes are as follows: - `ipfs add` - Support for a `-encrypt-key` option, for block encrypting the file being added with the key - also adds an 'encrypted' node above the root unixfs node - Support for a `-sign-key` option to attach a signature node above the root unixfs node - `ipfs block put` - Support for a `-encrypt-key` option, for encrypting the block before hashing and storing - `ipfs object put` - Support for a `-encrypt-key` option, for encrypting the object before hashing and storing - `ipfs name publish` - Support for a `-key` option to select which keyspace to publish to ### Code changes and additions This sections outlines code organization around this feature. #### Keystore package The fsrepo carries a `keystore` that can be used to load/store keys. The keystore is implemented following this interface: ```go // Keystore provides a key management interface type Keystore interface { // Has returns whether or not a key exist in the Keystore Has(string) (bool, error) // Put stores a key in the Keystore, if a key with the same name already exists, returns ErrKeyExists Put(string, ci.PrivKey) error // Get retrieves a key from the Keystore if it exists, and returns ErrNoSuchKey // otherwise. Get(string) (ci.PrivKey, error) // Delete removes a key from the Keystore Delete(string) error // List returns a list of key identifier List() ([]string, error) } ``` Note: Never store passwords as strings, strings cannot be zeroed out after they are used. using a byte array allows you to write zeroes over the memory so that the users password does not linger in memory. #### Unixfs - new node types, 'encrypted' and 'signed', probably shouldn't be in unixfs, just understood by it - if new node types are not unixfs nodes, special consideration must be given to the interop - DagReader needs to be able to access keystore to seamlessly stream encrypted data we have keys for - also needs to be able to verify signatures #### Importer - DagBuilderHelper needs to be able to encrypt blocks - Dag Nodes should be generated like normal, then encrypted, and their parents should link to the hash of the encrypted node - DagBuilderParams should have extra parameters to accommodate creating a DBH that encrypts the blocks #### New 'Encrypt' package Should contain code for crypto operations on dags. Encryption of dags should work by first generating a symmetric key, and using that key to encrypt all the data. That key should then be encrypted with the public key chosen and stored in the Encrypted DAG structure. Note: One option is to simply add it to the key interface. ### Structures Some tentative mockups (in json) of the new DAG structures for signing and encrypting Signed DAG: ``` { "Links" : [ { "Name":"@content", "Hash":"QmTheContent", } ], "Data": protobuf{ "Type":"Signed DAG", "Signature": "thesignature", "PubKeyID": "QmPubKeyHash", } } ``` Encrypted DAG: ``` { "Links" : [ { "Name":"@content", "Hash":"QmRawEncryptedDag", } ], "Data": protobuf{ "Type":"Encrypted DAG", "PubKeyID": "QmPubKeyHash", "Key": "ephemeral symmetric key, encrypted with public key", } } ``` ================================================ FILE: docs/specifications/repository.md ================================================ # ![](https://img.shields.io/badge/status-wip-orange.svg?style=flat-square) IPFS Repo Spec **Author(s)**: - [Juan Benet](github.com/jbenet) **Abstract** This spec defines an IPFS Repo, its contents, and its interface. It does not specify how the repo data is actually stored, as that is done via swappable implementations. # Table of Contents - [Definition](#definition) - [Repo Contents](#repo-contents) - [version](#version) - [datastore](#datastore) - [keystore](#keystore) - [config (state)](#config-state) - [locks](#locks) - [datastore\_spec](#datastore_spec) - [hooks (TODO)](#hooks-todo) - [Notes](#notes) ## Definition A `repo` is the storage repository of an IPFS node. It is the subsystem that actually stores the data IPFS nodes use. All IPFS objects are stored in a repo (similar to git). There are many possible repo implementations, depending on the storage media used. Most commonly, IPFS nodes use an [fs-repo](repository_fs.md). Repo Implementations: - [fs-repo](repository_fs.md) - stored in the os filesystem - mem-repo - stored in process memory - s3-repo - stored in amazon s3 ## Repo Contents The Repo stores a collection of [IPLD](https://github.com/ipld/specs#readme) objects that represent: - **config** - node configuration and settings - **datastore** - content stored locally, and indexing data - **keystore** - cryptographic keys, including node's identity - **hooks** - scripts to run at predefined times (not yet implemented) Note that the IPLD objects a repo stores are divided into: - **state** (system, control plane) used for the node's internal state - **content** (userland, data plane) which represent the user's cached and pinned data. Additionally, the repo state must determine the following. These need not be IPLD objects, though it is of course encouraged: - **version** - the repo version, required for safe migrations - **locks** - process semaphores for correct concurrent access - **datastore_spec** - array of mounting points and their properties Finally, the repo also stores the blocks with blobs containing binary data. ![](./ipfs-repo-contents.png) ### version Repo implementations may change over time, thus they MUST include a `version` recognizable across versions. Meaning that a tool MUST be able to read the `version` of a given repo type. For example, the `fs-repo` simply includes a `version` file with the version number. This way, the repo contents can evolve over time but the version remains readable the same way across versions. ### datastore IPFS nodes store some IPLD objects locally. These are either (a) **state objects** required for local operation -- such as the `config` and `keys` -- or (b) **content objects** used to represent data locally available. **Content objects** are either _pinned_ (stored until they are unpinned) or _cached_ (stored until the next repo garbage collection). The name "datastore" comes from [go-datastore](https://github.com/jbenet/go-datastore), a library for swappable key-value stores. Like its name-sake, some repo implementations feature swappable datastores, for example: - an fs-repo with a leveldb datastore - an fs-repo with a boltdb datastore - an fs-repo with a union fs and leveldb datastore - an fs-repo with an s3 datastore - an s3-repo with a cached fs and s3 datastore This makes it easy to change properties or performance characteristics of a repo without an entirely new implementation. ### keystore A Repo typically holds the keys a node has access to, for signing and for encryption. Details on operation and storage of the keystore can be found in [`repository_fs.md`](repository_fs.md) and [`keystore.md`](keystore.md). ### config (state) The node's `config` (configuration) is a tree of variables, used to configure various aspects of operation. For example: - the set of bootstrap peers IPFS uses to connect to the network - the Swarm, API, and Gateway network listen addresses - the Datastore configuration regarding the construction and operation of the on-disk storage system. There is a set of properties, which are mandatory for the repo usage. Those are `Addresses`, `Discovery`, `Bootstrap`, `Identity`, `Datastore` and `Keychain`. It is recommended that `config` files avoid identifying information, so that they may be re-shared across multiple nodes. **CHANGES**: today, implementations like js-ipfs and go-ipfs store the peer-id and private key directly in the config. These will be removed and moved out. ### locks IPFS implementations may use multiple processes, or may disallow multiple processes from using the same repo simultaneously. Others may disallow using the same repo but may allow sharing _datastores_ simultaneously. This synchronization is accomplished via _locks_. All repos contain the following standard locks: - `repo.lock` - prevents concurrent access to the repo. Must be held to _read_ or _write_. ### datastore_spec This file is created according to the Datastore configuration specified in the `config` file. It contains an array with all the mounting points that the repo is using, as well as its properties. This way, the `datastore_spec` file must have the same mounting points as defined in the Datastore configuration. It is important pointing out that the `Datastore` in config must have a `Spec` property, which defines the structure of the ipfs datastore. It is a composable structure, where each datastore is represented by a json object. ### hooks (TODO) Like git, IPFS nodes will allow `hooks`, a set of user configurable scripts to run at predefined moments in IPFS operations. This makes it easy to customize the behavior of IPFS nodes without changing the implementations themselves. ## Notes #### A Repo uniquely identifies an IPFS Node A repository uniquely identifies a node. Running two different IPFS programs with identical repositories -- and thus identical identities -- WILL cause problems. Datastores MAY be shared -- with proper synchronization -- though note that sharing datastore access MAY erode privacy. #### Repo implementation changes MUST include migrations **DO NOT BREAK USERS' DATA.** This is critical. Thus, any changes to a repo's implementation **MUST** be accompanied by a **SAFE** migration tool. See https://github.com/jbenet/go-ipfs/issues/537 and https://github.com/jbenet/random-ideas/issues/33 #### Repo Versioning A repo version is a single incrementing integer. All versions are considered non-compatible. Repos of different versions MUST be run through the appropriate migration tools before use. ================================================ FILE: docs/specifications/repository_fs.md ================================================ # ![](https://img.shields.io/badge/status-wip-orange.svg?style=flat-square) fs-repo **Author(s)**: - [Juan Benet](github.com/jbenet) - [David Dias](github.com/daviddias) - [Hector Sanjuan](github.com/hsanjuan) **Abstract** This spec defines `fs-repo` version `1`, its formats, and semantics. # Table of Contents - [Definition](#definition) - [Contents](#contents) - [api](#api) - [blocks/](#blocks) - [config](#config) - [hooks/](#hooks) - [keystore/](#keystore) - [datastore/](#datastore) - [logs/](#logs) - [repo.lock](#repolock) - [version](#version) - [Datastore](#datastore-1) - [Notes](#notes) - [Location](#location) - [blocks/ with an fs-datastore](#blocks-with-an-fs-datastore) - [Reading without the `repo.lock`](#reading-without-the-repolock) ## Definition `fs-repo` is a filesystem implementation of the IPFS [repo](repository.md). ## Contents ![](img/ipfs-repo-contents.png?) ``` .ipfs/ ├── api <--- running daemon api addr ├── blocks/ <--- objects stored directly on disk │ └── aa <--- prefix namespacing like git │ └── aa <--- N tiers ├── config <--- config file (json or toml) ├── hooks/ <--- hook scripts ├── keystore/ <--- cryptographic keys │ ├── key_b32name <--- private key with base32-encoded name ├── datastore/ <--- datastore ├── logs/ <--- 1 or more files (log rotate) │ └── events.log <--- can be tailed ├── repo.lock <--- mutex for repo └── version <--- version file ``` ### api `./api` is a file that exists to denote an API endpoint to listen to. - It MAY exist even if the endpoint is no longer live (i.e. it is a _stale_ or left-over `./api` file). In the presence of an `./api` file, ipfs tools (e.g. go-ipfs `ipfs daemon`) MUST attempt to delegate to the endpoint, and MAY remove the file if reasonably certain the file is stale. (e.g. endpoint is local, but no process is live) The `./api` file is used in conjunction with the `repo.lock`. Clients may opt to use the api service, or wait until the process holding `repo.lock` exits. The file's content is the api endpoint as a [multiaddr](https://github.com/jbenet/multiaddr) ``` > cat .ipfs/api /ip4/127.0.0.1/tcp/5001 ``` Notes: - The API server must remove the api file before releasing the `repo.lock`. - It is not enough to use the `config` file, as the API addr of a daemon may have been overridden via ENV or flag. #### api file for remote control One use case of the `api` file is to have a repo directory like: ``` > tree $IPFS_PATH /Users/jbenet/.ipfs └── api 0 directories, 1 files > cat $IPFS_PATH/api /ip4/1.2.3.4/tcp/5001 ``` In go-ipfs, this has the same effect as: ``` ipfs --api /ip4/1.2.3.4/tcp/5001 ``` Meaning that it makes ipfs tools use an ipfs node at the given endpoint, instead of the local directory as a repo. In this use case, the rest of the `$IPFS_PATH` may be completely empty, and no other information is necessary. It cannot be said it is a _repo_ per-se. (TODO: come up with a good name for this). ### blocks/ The `block/` component contains the raw data representing all IPFS objects stored locally, whether pinned or cached. This component is controlled by the ` datastore`. For example, it may be stored within a leveldb instance in ` datastore/`, or it may be stored entirely with independent files, like git. In the default case, the user uses fs-datastore for all `/blocks` so the objects are stored in individual files. In other cases, `/blocks` may even be stored remotely - [blocks/ with an fs-datastore](#blocks-with-an-fs-datastore) ### config The `config` file is a JSON or TOML file that contains the tree of configuration variables. It MUST only be changed while holding the `repo.lock`, or potentially lose edits. ### hooks/ The `hooks` directory contains executable scripts to be called on specific events to alter ipfs node behavior. Currently available hooks: ``` none ``` ### keystore/ The `keystore` directory holds additional private keys that the node has access to (the public keys can be derived from them). The keystore repository should have `0700` permissions (readable, writable by the owner only). The key files are named as `key_base32encodedNameNoPadding` where `key_` is a fixed prefix followed by a base32 encoded identifier, **without padding and downcased**. The identifier usually corresponds to a human-friendly name given by the user. The key files should have '0400' permissions (read-only, by the owner only). The `self` key identifier is reserved for the peer's main key, and therefore key named `key_onswyzq` is allowed in this folder. The key files themselves contain a serialized representation of the keys as defined in the [libp2p specification](https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#keys). ### datastore/ The `datastore` directory contains the data for a leveldb instance used to store operation data for the IPFS node. If the user uses a `boltdb` datastore instead, the directory will be named `boltdb`. Thus the data files of each database will not clash. TODO: consider whether all should just be named `leveldb/` ### logs/ IPFS implementations put event log files inside the `logs/` directory. The latest log file is `logs/events`. Others, rotated out may exist, with a timestamp of their creation. For example: ### repo.lock `repo.lock` prevents concurrent access to the repo. Its content SHOULD BE the PID of the process currently holding the lock. This allows clients to detect a failed lock and cleanup. ``` > cat .ipfs/repo.lock 42 > ps | grep "ipfs daemon" 42 ttys000 79:05.83 ipfs daemon ``` **TODO, ADDRESS DISCREPANCY:** the go-ipfs implementation does not currently store the PID in the file, which in some systems causes failures after a failure or a teardown. This SHOULD NOT require any manual intervention-- a present lock should give new processes enough information to recover. Doing this correctly in a portable, safe way, with good UX is very tricky. We must be careful with TOCTTOU bugs, and multiple concurrent processes capable of running at any moment. The goal is for all processes to operate safely, to avoid bothering the user, and for the repo to always remain in a correct, consistent state. ### version The `version` file contains the repo implementation name and version. This format has changed over time: ``` # in version 0 > cat $repo-at-version-0/version cat: /Users/jbenet/.ipfs/version: No such file or directory # in versions 1 and 2 > cat $repo-at-version-1/version 1 > cat $repo-at-version-2/version 2 # in versions >3 > cat $repo-at-version-3/version fs-repo/3 ``` _Any_ fs-repo implementation of _any_ versions `>0` MUST be able to read the `version` file. It MUST NOT change format between versions. The sole exception is version 0, which had no file. **TODO: ADDRESS DISCREPANCY:** versions 1 and 2 of the go-ipfs implementation use just the integer number. It SHOULD have used `fs-repo/`. We could either change the spec and always just use the int, or change go-ipfs in version `>3`. we will have to be backwards compatible. ## Datastore Both the `/blocks` and `/datastore` directories are controlled by the `datastore` component of the repo. ## Notes ### Location The `fs-repo` can be located anywhere on the filesystem. By default clients should search for a repo in: ``` ~/.ipfs ``` Users can tell IPFS programs to look elsewhere with the env var: ``` IPFS_PATH=/path/to/repo ``` ### blocks/ with an fs-datastore ![](fs-datastore.png) Each object is stored in its own file. The filename is the hash of the object. The files are nested in directories whose names are prefixes of the hash, as in `.git/objects`. For example: ```sh # multihashes 1220fe389b55ea958590769f9046b0f7268bca90a92e4a9f45cbb30930f4bf89269d # sha2 1114f623e0ec7f8719fb14a18838d2a3ef4e550b5e53 # sha1 # locations of the blocks .ipfs/blocks/1114/f6/23/e0ec7f8719fb14a18838d2a3ef4e550b5e53 .ipfs/blocks/1220/fe/38/9b55ea958590769f9046b0f7268bca90a92e4a9f45cbb30930f4bf89269d ``` **Important Notes:** - the hashes are encoded in hex, not the usual base58, because some filesystems are case insensitive. - the multihash prefix is two bytes, which would waste two directory levels, thus these are combined into one. - the git `idx` and `pack` file formats could be used to coalesce objects **TODO: ADDRESS DISCREPANCY:** the go-ipfs fs-repo in version 2 uses a different `blocks/` dir layout: ``` /Users/jbenet/.ipfs/blocks ├── 12200007 │ └── 12200007d4e3a319cd8c7c9979280e150fc5dbaae1ce54e790f84ae5fd3c3c1a0475.data ├── 1220000f │ └── 1220000fadd95a98f3a47c1ba54a26c77e15c1a175a975d88cf198cc505a06295b12.data ``` We MUST address whether we should change the fs-repo spec to match go-ipfs in version 2, or we should change go-ipfs to match the fs-repo spec (more tiers). We MUST also address whether the levels are a repo version parameter or a config parameter. There are filesystems in which a different fanout will have wildly different performance. These are mostly networked and legacy filesystems. ### Reading without the `repo.lock` Programs MUST hold the `repo.lock` while reading and writing most files in the repo. The only two exceptions are: - `repo.lock` - so clients may check for it - `api` - so clients may use the API ================================================ FILE: docs/telemetry.md ================================================ # Telemetry Plugin Documentation The **Telemetry plugin** is a feature in Kubo that collects **anonymized usage data** to help the development team better understand how the software is used, identify areas for improvement, and guide future feature development. This data is not personally identifiable and is used solely for the purpose of improving the Kubo project. --- ## 🛡️ How to Control Telemetry The behavior of the Telemetry plugin is controlled via the environment variable [`IPFS_TELEMETRY`](environment-variables.md#ipfs_telemetry) and optionally via the `Plugins.Plugins.telemetry.Config.Mode` in the IPFS config file. ### Available Modes | Mode | Description | |----------|-----------------------------------------------------------------------------| | `on` | **Default**. Telemetry is enabled. Data is sent periodically. | | `off` | Telemetry is disabled. No data is sent. Any existing telemetry UUID file is removed. | | `auto` | Like `on`, but logs an informative message about the telemetry and gives user 15 minutes to opt-out before first collection. This mode is automatically used on the first run when `IPFS_TELEMETRY` is not set and telemetry UUID is not found (not generated yet). The informative message is only shown once. | You can set the mode in your environment: ```bash export IPFS_TELEMETRY="off" ``` Or in your IPFS config file: ```json { "Plugins": { "Plugins": { "telemetry": { "Config": { "Mode": "off" } } } } } ``` --- ## 📦 What Data is Collected? The telemetry plugin collects the following anonymized data: ### General Information - **UUID**: Anonymous identifier for this node - **Agent version**: Kubo version string - **Private network**: Whether running in a private IPFS network - **Repository size**: Categorized into privacy-preserving buckets (1GB, 5GB, 10GB, 100GB, 500GB, 1TB, 10TB, >10TB) - **Uptime**: Categorized into privacy-preserving buckets (1d, 2d, 3d, 7d, 14d, 30d, >30d) ### Routing & Discovery - **Custom bootstrap peers**: Whether custom `Bootstrap` peers are configured - **Routing type**: The `Routing.Type` configured for the node - **Accelerated DHT client**: Whether `Routing.AcceleratedDHTClient` is enabled - **Delegated routing count**: Number of `Routing.DelegatedRouters` configured - **AutoConf enabled**: Whether `AutoConf.Enabled` is set - **Custom AutoConf URL**: Whether custom `AutoConf.URL` is configured - **mDNS**: Whether `Discovery.MDNS.Enabled` is set ### Content Providing - **Provide and Reprovide strategy**: The `Provide.Strategy` configured - **Sweep-based provider**: Whether `Provide.DHT.SweepEnabled` is set - **Custom Interval**: Whether custom `Provide.DHT.Interval` is configured - **Custom MaxWorkers**: Whether custom `Provide.DHT.MaxWorkers` is configured ### Network Configuration - **AutoNAT service mode**: The `AutoNAT.ServiceMode` configured - **AutoNAT reachability**: Current reachability status determined by AutoNAT - **Hole punching**: Whether `Swarm.EnableHolePunching` is enabled - **Circuit relay addresses**: Whether the node advertises circuit relay addresses - **Public IPv4 addresses**: Whether the node has public IPv4 addresses - **Public IPv6 addresses**: Whether the node has public IPv6 addresses - **AutoWSS**: Whether `AutoTLS.AutoWSS` is enabled - **Custom domain suffix**: Whether custom `AutoTLS.DomainSuffix` is configured ### Platform Information - **Operating system**: The OS the node is running on - **CPU architecture**: The architecture the node is running on - **Container detection**: Whether the node is running inside a container - **VM detection**: Whether the node is running inside a virtual machine ### Code Reference Data is organized in the `LogEvent` struct at [`plugin/plugins/telemetry/telemetry.go`](https://github.com/ipfs/kubo/blob/master/plugin/plugins/telemetry/telemetry.go). This struct is the authoritative source of truth for all telemetry data, including privacy-preserving buckets for repository size and uptime. Note that this documentation may not always be up-to-date - refer to the code for the current implementation. --- ## 🧑‍🤝‍🧑 Privacy and Anonymization All data collected is: - **Anonymized**: No personally identifiable information (PII) is sent. - **Optional**: Users can choose to opt out at any time. - **Secure**: Data is sent over HTTPS to a trusted endpoint. The telemetry UUID is stored in the IPFS repo folder and is used to identify the node across runs, but it does not contain any personal information. When you opt-out, this UUID file is automatically removed to ensure complete privacy. --- ## 📦 Contributing to the Project By enabling telemetry, you are helping the Kubo team improve the software for the entire community. The data is used to: - Prioritize feature development - Identify performance bottlenecks - Improve user experience You can always disable telemetry at any time if you change your mind. --- ## 🧪 Testing Telemetry If you're testing telemetry locally, you can change the endpoint by setting the `Endpoint` field in the config: ```json { "Plugins": { "Plugins": { "telemetry": { "Config": { "Mode": "on", "Endpoint": "http://localhost:8080" } } } } } ``` This allows you to capture and inspect telemetry data locally. --- ## 📦 Further Reading For more information, see: - [IPFS Environment Variables](docs/environment-variables.md) - [IPFS Plugins](docs/plugins.md) - [IPFS Configuration](docs/config.md) ================================================ FILE: docs/transports.md ================================================ ## /ws and /wss -- websockets If you want browsers to connect to e.g. `/dns4/example.com/tcp/443/wss/p2p/QmFoo` - [ ] An SSL cert matching the `/dns4` or `/dns6` name - [ ] Kubo listening on `/ip4/127.0.0.1/tcp/8081/ws` - 8081 is just an example - note that it's `/ws` here, not `/wss` -- Kubo can't currently do SSL, see the next point - [ ] nginx - configured with the SSL cert - listening on port 443 - forwarding to 127.0.0.1:8081 ================================================ FILE: docs/windows.md ================================================ # Building on Windows ![](https://ipfs.io/ipfs/QmccXW7JSZMVXidSc7tHsU6aktuaiV923q4yBGHUsdymYo/build.gif) If you just want to install kubo, please download it from https://dist.ipfs.tech/#kubo. This document explains how to build it from source. ## Install Go `kubo` is built on Golang and thus depends on it for all building methods. https://golang.org/doc/install The `GOPATH` environment variable must be set as well. https://golang.org/doc/code.html#GOPATH ## Choose the way you want to proceed `kubo` utilizes `make` to automate builds and run tests, but can be built without it using only `git` and `go`. No matter which method you choose, if you encounter issues, please see the [Troubleshooting](#troubleshooting) section. **Using `make`:** MSYS2 and Cygwin provide the Unix tools we need to build `kubo`. You may use either, but if you don't already have one installed, we recommend MSYS2. [MSYS2→](#msys2) [Cygwin→](#cygwin) **Using build tools manually:** This section assumes you have a working version of `go` and `git` already setup. You may want to build this way if your environment restricts installing additional software, or if you're integrating IPFS into your own build system. [Minimal→](#minimal) ## MSYS2 1. Install msys2 (http://www.msys2.org) 2. Run the following inside a normal `cmd` prompt (Not the MSYS2 prompt, we only need MSYS2's tools). An explanation of this block is below. ``` SET PATH=%PATH%;\msys64\usr\bin pacman --noconfirm -S git make unzip go get -u github.com/ipfs/kubo cd %GOPATH%\src\github.com\ipfs\kubo make install %GOPATH%\bin\ipfs.exe version --all ``` If there were no errors, the final command should output version information similar to "`ipfs version 0.4.14-dev-XXXXXXX`" where "XXXXXXX" should match the current short-hash of the `kubo` repo. You can retrieve said hash via this command: `git rev-parse --short HEAD`. If `ipfs.exe` executes and the version string matches, then building was successful. |Command|Explanation| | ---: | :--- | |`SET PATH=%PATH%;\msys64\usr\bin` |Add msys2's tools to our [`PATH`](https://ss64.com/nt/path.html); Defaults to: (\msys64\usr\bin)| |`pacman --noconfirm -S git make unzip` |Install `kubo` build dependencies| |`go get -u github.com/ipfs/kubo` |Fetch / Update `kubo` source| |`cd %GOPATH%\src\github.com\ipfs\kubo` |Change to `kubo` source directory| |`make install` |Build and install to `%GOPATH%\bin\ipfs.exe`| |`%GOPATH%\bin\ipfs.exe version --all` |Test the built binary| To build again after making changes to the source, run: ``` SET PATH=%PATH%;\msys64\usr\bin cd %GOPATH%\src\github.com\ipfs\kubo make install ``` **Tip:** To avoid setting `PATH` every time (`SET PATH=%PATH%;\msys64\usr\bin`), you can lock it in permanently using `setx` after it's been set once: ``` SETX PATH %PATH% ``` ## Cygwin 1. Install Cygwin (https://www.cygwin.com) 2. During the install, select the following packages. (If you already have Cygwin installed, run the setup file again to install additional packages.) A fresh install should look something like [this reference image](https://ipfs.io/ipfs/QmaYFSQa4iHDafcebiKjm1WwuKhosoXr45HPpfaeMbCRpb/cygwin%20-%20install.png). - devel packages - `git` - `make` - archive packages - `unzip` - net packages - `curl` 3. Run the following inside a normal `cmd` prompt (Not the Cygwin prompt, we only need Cygwin's tools) An explanation of this block is below. ``` SET PATH=%PATH%;\cygwin64\bin mkdir %GOPATH%\src\github.com\ipfs cd %GOPATH%\src\github.com\ipfs git clone https://github.com/ipfs/kubo.git cd %GOPATH%\src\github.com\ipfs\kubo make install %GOPATH%\bin\ipfs.exe version --all ``` If there were no errors, the final command should output version information similar to "`ipfs version 0.4.14-dev-XXXXXXX`" where "XXXXXXX" should match the current short-hash of the `kubo` repo. You can retrieve said hash via this command: `git rev-parse --short HEAD`. If `ipfs.exe` executes and the version string matches, then building was successful. |Command|Explanation| | ---: | :--- | |`SET PATH=%PATH%;\cygwin64\bin` |Add Cygwin's tools to our [`PATH`](https://ss64.com/nt/path.html); Defaults to: (\cygwin64\bin)| |`mkdir %GOPATH%\src\github.com\ipfs`
`cd %GOPATH%\src\github.com\ipfs`
`git clone https://github.com/ipfs/kubo.git` |Fetch / Update `kubo` source| |`cd %GOPATH%\src\github.com\ipfs\kubo` |Change to `kubo` source directory| |`make install` |Build and install to `%GOPATH%\bin\ipfs.exe`| |`%GOPATH%\bin\ipfs.exe version --all` |Test the built binary| To build again after making changes to the source, run: ``` SET PATH=%PATH%;\cygwin64\bin cd %GOPATH%\src\github.com\ipfs\kubo make install ``` **Tip:** To avoid setting `PATH` every time (`SET PATH=%PATH%;\cygwin64\bin`), you can lock it in permanently using `setx` after it's been set once: ``` SETX PATH %PATH% ``` ## Minimal While it's possible to build `kubo` with `go` alone, we'll be using `git` to fetch the source. You can use whichever version of `git` you wish but we recommend the Windows builds at . `git` must be in your [`PATH`](https://ss64.com/nt/path.html) for `go get` to recognize and use it. ### kubo Clone and change directory to the source code, if you haven't already: CMD: ```bat git clone https://github.com/ipfs/kubo %GOPATH%/src/github.com/ipfs/kubo cd %GOPATH%/src/github.com/ipfs/kubo/cmd/ipfs ``` PowerShell: ```powershell git clone https://github.com/ipfs/kubo $env:GOPATH/src/github.com/ipfs/kubo cd $env:GOPATH/src/github.com/ipfs/kubo/cmd/ipfs ``` We need the `git` commit hash to be included in our build so that in the extremely rare event a bug is found, we have a reference point later for tracking it. We'll ask `git` for it and store it in a variable. The syntax for the next command is different depending on whether you're using the interactive command line or writing a batch file. Use the one that applies to you. - interactive: `FOR /F %V IN ('git rev-parse --short HEAD') do set SHA=%V` - interpreter: `FOR /F %%V IN ('git rev-parse --short HEAD') do set SHA=%%V` Finally, we'll build and test `ipfs` itself. CMD: ```bat go install -ldflags="-X "github.com/ipfs/kubo".CurrentCommit=%SHA%" %GOPATH%\bin\ipfs.exe version --all ``` PowerShell: ```powershell go install -ldflags="-X "github.com/ipfs/kubo".CurrentCommit=$env:SHA" cp ./ipfs.exe $env:GOPATH/bin/ipfs.exe -force . $env:GOPATH/bin/ipfs.exe version --all ``` You can check that the ipfs output versions match with `go version` and `git rev-parse --short HEAD`. If `ipfs.exe` executes and everything matches, then building was successful. ## Troubleshooting - **Git auth** If you get authentication problems with Git, you might want to take a look at https://help.github.com/articles/caching-your-github-password-in-git/ and use the suggested solution: `git config --global credential.helper wincred` - **Anything else** Please search [https://discuss.ipfs.tech](https://discuss.ipfs.tech/search?q=windows%20category%3A13) for any additional issues you may encounter. If you can't find any existing resolution, feel free to post a question asking for help. If you encounter a bug with `kubo` itself (not related to building) please use the [issue tracker](https://github.com/ipfs/kubo/issues) to report it. ================================================ FILE: fuse/ipns/common.go ================================================ package ipns import ( "context" ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/boxo/namesys" "github.com/ipfs/boxo/path" "github.com/ipfs/kubo/core" ci "github.com/libp2p/go-libp2p/core/crypto" ) // InitializeKeyspace sets the ipns record for the given key to // point to an empty directory. func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error { ctx, cancel := context.WithCancel(n.Context()) defer cancel() emptyDir := ft.EmptyDirNode() err := n.Pinning.Pin(ctx, emptyDir, false, "") if err != nil { return err } err = n.Pinning.Flush(ctx) if err != nil { return err } pub := namesys.NewIPNSPublisher(n.Routing, n.Repo.Datastore()) return pub.Publish(ctx, key, path.FromCid(emptyDir.Cid())) } ================================================ FILE: fuse/ipns/ipns_test.go ================================================ //go:build !nofuse && !openbsd && !netbsd && !plan9 package ipns import ( "bytes" "context" "fmt" "io" mrand "math/rand" "os" "sync" "testing" "bazil.org/fuse" core "github.com/ipfs/kubo/core" coreapi "github.com/ipfs/kubo/core/coreapi" fstest "bazil.org/fuse/fs/fstestutil" racedet "github.com/ipfs/go-detect-race" "github.com/ipfs/go-test/random" ci "github.com/libp2p/go-libp2p-testing/ci" ) func maybeSkipFuseTests(t *testing.T) { if ci.NoFuse() { t.Skip("Skipping FUSE tests") } } func randBytes(size int) []byte { b := make([]byte, size) _, err := io.ReadFull(random.NewRand(), b) if err != nil { panic(err) } return b } func mkdir(t *testing.T, path string) { err := os.Mkdir(path, os.ModeDir) if err != nil { t.Fatal(err) } } func writeFileOrFail(t *testing.T, size int, path string) []byte { data, err := writeFile(size, path) if err != nil { t.Fatal(err) } return data } func writeFile(size int, path string) ([]byte, error) { data := randBytes(size) err := os.WriteFile(path, data, 0o666) return data, err } func verifyFile(t *testing.T, path string, wantData []byte) { isData, err := os.ReadFile(path) if err != nil { t.Fatal(err) } if len(isData) != len(wantData) { t.Fatal("Data not equal - length check failed") } if !bytes.Equal(isData, wantData) { t.Fatal("Data not equal") } } func checkExists(t *testing.T, path string) { _, err := os.Stat(path) if err != nil { t.Fatal(err) } } func closeMount(mnt *mountWrap) { if err := recover(); err != nil { log.Error("Recovered panic") log.Error(err) } mnt.Close() } type mountWrap struct { *fstest.Mount Fs *FileSystem } func (m *mountWrap) Close() error { m.Fs.Destroy() m.Mount.Close() return nil } func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *mountWrap) { t.Helper() maybeSkipFuseTests(t) var err error if node == nil { node, err = core.NewNode(context.Background(), &core.BuildCfg{}) if err != nil { t.Fatal(err) } err = InitializeKeyspace(node, node.PrivateKey) if err != nil { t.Fatal(err) } } coreAPI, err := coreapi.NewCoreAPI(node) if err != nil { t.Fatal(err) } fs, err := NewFileSystem(node.Context(), coreAPI, "", "") if err != nil { t.Fatal(err) } mnt, err := fstest.MountedT(t, fs, nil) if err == fuse.ErrOSXFUSENotFound { t.Skip(err) } if err != nil { t.Fatalf("error mounting at temporary directory: %v", err) } return node, &mountWrap{ Mount: mnt, Fs: fs, } } func TestIpnsLocalLink(t *testing.T) { nd, mnt := setupIpnsTest(t, nil) defer mnt.Close() name := mnt.Dir + "/local" checkExists(t, name) linksto, err := os.Readlink(name) if err != nil { t.Fatal(err) } if linksto != nd.Identity.String() { t.Fatal("Link invalid") } } // Test writing a file and reading it back. func TestIpnsBasicIO(t *testing.T) { if testing.Short() { t.SkipNow() } nd, mnt := setupIpnsTest(t, nil) defer closeMount(mnt) fname := mnt.Dir + "/local/testfile" data := writeFileOrFail(t, 10, fname) rbuf, err := os.ReadFile(fname) if err != nil { t.Fatal(err) } if !bytes.Equal(rbuf, data) { t.Fatal("Incorrect Read!") } fname2 := mnt.Dir + "/" + nd.Identity.String() + "/testfile" rbuf, err = os.ReadFile(fname2) if err != nil { t.Fatal(err) } if !bytes.Equal(rbuf, data) { t.Fatal("Incorrect Read!") } } // Test to make sure file changes persist over mounts of ipns. func TestFilePersistence(t *testing.T) { if testing.Short() { t.SkipNow() } node, mnt := setupIpnsTest(t, nil) fname := "/local/atestfile" data := writeFileOrFail(t, 127, mnt.Dir+fname) mnt.Close() t.Log("Closed, opening new fs") _, mnt = setupIpnsTest(t, node) defer mnt.Close() rbuf, err := os.ReadFile(mnt.Dir + fname) if err != nil { t.Fatal(err) } if !bytes.Equal(rbuf, data) { t.Fatalf("File data changed between mounts! sizes differ: %d != %d", len(data), len(rbuf)) } } func TestMultipleDirs(t *testing.T) { node, mnt := setupIpnsTest(t, nil) t.Log("make a top level dir") dir1 := "/local/test1" mkdir(t, mnt.Dir+dir1) checkExists(t, mnt.Dir+dir1) t.Log("write a file in it") data1 := writeFileOrFail(t, 4000, mnt.Dir+dir1+"/file1") verifyFile(t, mnt.Dir+dir1+"/file1", data1) t.Log("sub directory") mkdir(t, mnt.Dir+dir1+"/dir2") checkExists(t, mnt.Dir+dir1+"/dir2") t.Log("file in that subdirectory") data2 := writeFileOrFail(t, 5000, mnt.Dir+dir1+"/dir2/file2") verifyFile(t, mnt.Dir+dir1+"/dir2/file2", data2) mnt.Close() t.Log("closing mount, then restarting") _, mnt = setupIpnsTest(t, node) checkExists(t, mnt.Dir+dir1) verifyFile(t, mnt.Dir+dir1+"/file1", data1) verifyFile(t, mnt.Dir+dir1+"/dir2/file2", data2) mnt.Close() } // Test to make sure the filesystem reports file sizes correctly. func TestFileSizeReporting(t *testing.T) { if testing.Short() { t.SkipNow() } _, mnt := setupIpnsTest(t, nil) defer mnt.Close() fname := mnt.Dir + "/local/sizecheck" data := writeFileOrFail(t, 5555, fname) finfo, err := os.Stat(fname) if err != nil { t.Fatal(err) } if finfo.Size() != int64(len(data)) { t.Fatal("Read incorrect size from stat!") } } // Test to make sure you can't create multiple entries with the same name. func TestDoubleEntryFailure(t *testing.T) { if testing.Short() { t.SkipNow() } _, mnt := setupIpnsTest(t, nil) defer mnt.Close() dname := mnt.Dir + "/local/thisisadir" err := os.Mkdir(dname, 0o777) if err != nil { t.Fatal(err) } err = os.Mkdir(dname, 0o777) if err == nil { t.Fatal("Should have gotten error one creating new directory.") } } func TestAppendFile(t *testing.T) { if testing.Short() { t.SkipNow() } _, mnt := setupIpnsTest(t, nil) defer mnt.Close() fname := mnt.Dir + "/local/file" data := writeFileOrFail(t, 1300, fname) fi, err := os.OpenFile(fname, os.O_RDWR|os.O_APPEND, 0o666) if err != nil { t.Fatal(err) } nudata := randBytes(500) n, err := fi.Write(nudata) if err != nil { t.Fatal(err) } err = fi.Close() if err != nil { t.Fatal(err) } if n != len(nudata) { t.Fatal("Failed to write enough bytes.") } data = append(data, nudata...) rbuf, err := os.ReadFile(fname) if err != nil { t.Fatal(err) } if !bytes.Equal(rbuf, data) { t.Fatal("Data inconsistent!") } } func TestConcurrentWrites(t *testing.T) { if testing.Short() { t.SkipNow() } _, mnt := setupIpnsTest(t, nil) defer mnt.Close() nactors := 4 filesPerActor := 400 fileSize := 2000 data := make([][][]byte, nactors) if racedet.WithRace() { nactors = 2 filesPerActor = 50 } wg := sync.WaitGroup{} for i := 0; i < nactors; i++ { data[i] = make([][]byte, filesPerActor) wg.Add(1) go func(n int) { defer wg.Done() for j := 0; j < filesPerActor; j++ { out, err := writeFile(fileSize, mnt.Dir+fmt.Sprintf("/local/%dFILE%d", n, j)) if err != nil { t.Error(err) continue } data[n][j] = out } }(i) } wg.Wait() for i := 0; i < nactors; i++ { for j := 0; j < filesPerActor; j++ { if data[i][j] == nil { // Error already reported. continue } verifyFile(t, mnt.Dir+fmt.Sprintf("/local/%dFILE%d", i, j), data[i][j]) } } } func TestFSThrash(t *testing.T) { files := make(map[string][]byte) if testing.Short() { t.SkipNow() } _, mnt := setupIpnsTest(t, nil) defer mnt.Close() base := mnt.Dir + "/local" dirs := []string{base} dirlock := sync.RWMutex{} filelock := sync.Mutex{} ndirWorkers := 2 nfileWorkers := 2 ndirs := 100 nfiles := 200 wg := sync.WaitGroup{} // Spawn off workers to make directories for i := range ndirWorkers { wg.Add(1) go func(worker int) { defer wg.Done() for j := range ndirs { dirlock.RLock() n := mrand.Intn(len(dirs)) dir := dirs[n] dirlock.RUnlock() newDir := fmt.Sprintf("%s/dir%d-%d", dir, worker, j) err := os.Mkdir(newDir, os.ModeDir) if err != nil { t.Error(err) continue } dirlock.Lock() dirs = append(dirs, newDir) dirlock.Unlock() } }(i) } // Spawn off workers to make files for i := range nfileWorkers { wg.Add(1) go func(worker int) { defer wg.Done() for j := range nfiles { dirlock.RLock() n := mrand.Intn(len(dirs)) dir := dirs[n] dirlock.RUnlock() newFileName := fmt.Sprintf("%s/file%d-%d", dir, worker, j) data, err := writeFile(2000+mrand.Intn(5000), newFileName) if err != nil { t.Error(err) continue } filelock.Lock() files[newFileName] = data filelock.Unlock() } }(i) } wg.Wait() for name, data := range files { out, err := os.ReadFile(name) if err != nil { t.Error(err) } if !bytes.Equal(data, out) { t.Errorf("Data didn't match in %s: expected %v, got %v", name, data, out) } } } // Test writing a medium sized file one byte at a time. func TestMultiWrite(t *testing.T) { if testing.Short() { t.SkipNow() } _, mnt := setupIpnsTest(t, nil) defer mnt.Close() fpath := mnt.Dir + "/local/file" fi, err := os.Create(fpath) if err != nil { t.Fatal(err) } data := randBytes(1001) for i := range data { n, err := fi.Write(data[i : i+1]) if err != nil { t.Fatal(err) } if n != 1 { t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)") } } fi.Close() rbuf, err := os.ReadFile(fpath) if err != nil { t.Fatal(err) } if !bytes.Equal(rbuf, data) { t.Fatal("File on disk did not match bytes written") } } ================================================ FILE: fuse/ipns/ipns_unix.go ================================================ //go:build !nofuse && !openbsd && !netbsd && !plan9 // package fuse/ipns implements a fuse filesystem that interfaces // with ipns, the naming system for ipfs. package ipns import ( "context" "errors" "fmt" "io" "os" "strings" "syscall" dag "github.com/ipfs/boxo/ipld/merkledag" ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/boxo/namesys" "github.com/ipfs/boxo/path" fuse "bazil.org/fuse" fs "bazil.org/fuse/fs" mfs "github.com/ipfs/boxo/mfs" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" iface "github.com/ipfs/kubo/core/coreiface" options "github.com/ipfs/kubo/core/coreiface/options" ) func init() { if os.Getenv("IPFS_FUSE_DEBUG") != "" { fuse.Debug = func(msg any) { fmt.Println(msg) } } } var log = logging.Logger("fuse/ipns") // FileSystem is the readwrite IPNS Fuse Filesystem. type FileSystem struct { Ipfs iface.CoreAPI RootNode *Root } // NewFileSystem constructs new fs using given core.IpfsNode instance. func NewFileSystem(ctx context.Context, ipfs iface.CoreAPI, ipfspath, ipnspath string) (*FileSystem, error) { key, err := ipfs.Key().Self(ctx) if err != nil { return nil, err } root, err := CreateRoot(ctx, ipfs, map[string]iface.Key{"local": key}, ipfspath, ipnspath) if err != nil { return nil, err } return &FileSystem{Ipfs: ipfs, RootNode: root}, nil } // Root constructs the Root of the filesystem, a Root object. func (f *FileSystem) Root() (fs.Node, error) { log.Debug("filesystem, get root") return f.RootNode, nil } func (f *FileSystem) Destroy() { err := f.RootNode.Close() if err != nil { log.Errorf("Error Shutting Down Filesystem: %s\n", err) } } // Root is the root object of the filesystem tree. type Root struct { Ipfs iface.CoreAPI Keys map[string]iface.Key // Used for symlinking into ipfs IpfsRoot string IpnsRoot string LocalDirs map[string]fs.Node Roots map[string]*mfs.Root LocalLinks map[string]*Link } func ipnsPubFunc(ipfs iface.CoreAPI, key iface.Key) mfs.PubFunc { return func(ctx context.Context, c cid.Cid) error { _, err := ipfs.Name().Publish(ctx, path.FromCid(c), options.Name.Key(key.Name())) return err } } func loadRoot(ctx context.Context, ipfs iface.CoreAPI, key iface.Key) (*mfs.Root, fs.Node, error) { node, err := ipfs.ResolveNode(ctx, key.Path()) switch err { case nil: case namesys.ErrResolveFailed: node = ft.EmptyDirNode() default: log.Errorf("looking up %s: %s", key.Path(), err) return nil, nil, err } pbnode, ok := node.(*dag.ProtoNode) if !ok { return nil, nil, dag.ErrNotProtobuf } // We have no access to provider.System from the CoreAPI. The Routing // part offers Provide through the router so it may be slow/risky // to give that here to MFS. Therefore we leave as nil. root, err := mfs.NewRoot(ctx, ipfs.Dag(), pbnode, ipnsPubFunc(ipfs, key), nil) if err != nil { return nil, nil, err } return root, &Directory{dir: root.GetDirectory()}, nil } func CreateRoot(ctx context.Context, ipfs iface.CoreAPI, keys map[string]iface.Key, ipfspath, ipnspath string) (*Root, error) { ldirs := make(map[string]fs.Node) roots := make(map[string]*mfs.Root) links := make(map[string]*Link) for alias, k := range keys { root, fsn, err := loadRoot(ctx, ipfs, k) if err != nil { return nil, err } name := k.ID().String() roots[name] = root ldirs[name] = fsn // set up alias symlink links[alias] = &Link{ Target: name, } } return &Root{ Ipfs: ipfs, IpfsRoot: ipfspath, IpnsRoot: ipnspath, Keys: keys, LocalDirs: ldirs, LocalLinks: links, Roots: roots, }, nil } // Attr returns file attributes. func (r *Root) Attr(ctx context.Context, a *fuse.Attr) error { log.Debug("Root Attr") a.Mode = os.ModeDir | 0o111 // -rw+x return nil } // Lookup performs a lookup under this node. func (r *Root) Lookup(ctx context.Context, name string) (fs.Node, error) { switch name { case "mach_kernel", ".hidden", "._.": // Just quiet some log noise on OS X. return nil, syscall.Errno(syscall.ENOENT) } if lnk, ok := r.LocalLinks[name]; ok { return lnk, nil } nd, ok := r.LocalDirs[name] if ok { switch nd := nd.(type) { case *Directory: return nd, nil case *FileNode: return nd, nil default: return nil, syscall.Errno(syscall.EIO) } } // other links go through ipns resolution and are symlinked into the ipfs mountpoint ipnsName := "/ipns/" + name resolved, err := r.Ipfs.Name().Resolve(ctx, ipnsName) if err != nil { log.Warnf("ipns: namesys resolve error: %s", err) return nil, syscall.Errno(syscall.ENOENT) } if resolved.Namespace() != path.IPFSNamespace { return nil, errors.New("invalid path from ipns record") } return &Link{r.IpfsRoot + "/" + strings.TrimPrefix(resolved.String(), "/ipfs/")}, nil } func (r *Root) Close() error { for _, mr := range r.Roots { err := mr.Close() if err != nil { return err } } return nil } // Forget is called when the filesystem is unmounted. probably. // see comments here: http://godoc.org/bazil.org/fuse/fs#FSDestroyer func (r *Root) Forget() { err := r.Close() if err != nil { log.Error(err) } } // ReadDirAll reads a particular directory. Will show locally available keys // as well as a symlink to the peerID key. func (r *Root) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { log.Debug("Root ReadDirAll") listing := make([]fuse.Dirent, 0, len(r.Keys)*2) for alias, k := range r.Keys { ent := fuse.Dirent{ Name: k.ID().String(), Type: fuse.DT_Dir, } link := fuse.Dirent{ Name: alias, Type: fuse.DT_Link, } listing = append(listing, ent, link) } return listing, nil } // Directory is wrapper over an mfs directory to satisfy the fuse fs interface. type Directory struct { dir *mfs.Directory } type FileNode struct { fi *mfs.File } // File is wrapper over an mfs file to satisfy the fuse fs interface. type File struct { fi mfs.FileDescriptor } // Attr returns the attributes of a given node. func (d *Directory) Attr(ctx context.Context, a *fuse.Attr) error { log.Debug("Directory Attr") a.Mode = os.ModeDir | 0o555 a.Uid = uint32(os.Getuid()) a.Gid = uint32(os.Getgid()) return nil } // Attr returns the attributes of a given node. func (fi *FileNode) Attr(ctx context.Context, a *fuse.Attr) error { log.Debug("File Attr") size, err := fi.fi.Size() if err != nil { // In this case, the dag node in question may not be unixfs return fmt.Errorf("fuse/ipns: failed to get file.Size(): %s", err) } a.Mode = os.FileMode(0o666) a.Size = uint64(size) a.Uid = uint32(os.Getuid()) a.Gid = uint32(os.Getgid()) return nil } // Lookup performs a lookup under this node. func (d *Directory) Lookup(ctx context.Context, name string) (fs.Node, error) { child, err := d.dir.Child(name) if err != nil { // todo: make this error more versatile. return nil, syscall.Errno(syscall.ENOENT) } switch child := child.(type) { case *mfs.Directory: return &Directory{dir: child}, nil case *mfs.File: return &FileNode{fi: child}, nil default: // NB: if this happens, we do not want to continue, unpredictable behaviour // may occur. panic("invalid type found under directory. programmer error.") } } // ReadDirAll reads the link structure as directory entries. func (d *Directory) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { listing, err := d.dir.List(ctx) if err != nil { return nil, err } entries := make([]fuse.Dirent, len(listing)) for i, entry := range listing { dirent := fuse.Dirent{Name: entry.Name} switch mfs.NodeType(entry.Type) { case mfs.TDir: dirent.Type = fuse.DT_Dir case mfs.TFile: dirent.Type = fuse.DT_File } entries[i] = dirent } if len(entries) > 0 { return entries, nil } return nil, syscall.Errno(syscall.ENOENT) } func (fi *File) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { _, err := fi.fi.Seek(req.Offset, io.SeekStart) if err != nil { return err } fisize, err := fi.fi.Size() if err != nil { return err } select { case <-ctx.Done(): return ctx.Err() default: } readsize := min(req.Size, int(fisize-req.Offset)) n, err := fi.fi.CtxReadFull(ctx, resp.Data[:readsize]) resp.Data = resp.Data[:n] return err } func (fi *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { // TODO: at some point, ensure that WriteAt here respects the context wrote, err := fi.fi.WriteAt(req.Data, req.Offset) if err != nil { return err } resp.Size = wrote return nil } func (fi *File) Flush(ctx context.Context, req *fuse.FlushRequest) error { errs := make(chan error, 1) go func() { errs <- fi.fi.Flush() }() select { case err := <-errs: return err case <-ctx.Done(): return ctx.Err() } } func (fi *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error { if req.Valid.Size() { cursize, err := fi.fi.Size() if err != nil { return err } if cursize != int64(req.Size) { err := fi.fi.Truncate(int64(req.Size)) if err != nil { return err } } } return nil } // Fsync flushes the content in the file to disk. func (fi *FileNode) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { // This needs to perform a *full* flush because, in MFS, a write isn't // persisted until the root is updated. errs := make(chan error, 1) go func() { errs <- fi.fi.Flush() }() select { case err := <-errs: return err case <-ctx.Done(): return ctx.Err() } } func (fi *File) Forget() { // TODO(steb): this seems like a place where we should be *uncaching*, not flushing. err := fi.fi.Flush() if err != nil { log.Debug("forget file error: ", err) } } func (d *Directory) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { child, err := d.dir.Mkdir(req.Name) if err != nil { return nil, err } return &Directory{dir: child}, nil } func (fi *FileNode) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { fd, err := fi.fi.Open(mfs.Flags{ Read: req.Flags.IsReadOnly() || req.Flags.IsReadWrite(), Write: req.Flags.IsWriteOnly() || req.Flags.IsReadWrite(), Sync: true, }) if err != nil { return nil, err } if req.Flags&fuse.OpenTruncate != 0 { if req.Flags.IsReadOnly() { log.Error("tried to open a readonly file with truncate") return nil, syscall.Errno(syscall.ENOTSUP) } log.Info("Need to truncate file!") err := fd.Truncate(0) if err != nil { return nil, err } } else if req.Flags&fuse.OpenAppend != 0 { log.Info("Need to append to file!") if req.Flags.IsReadOnly() { log.Error("tried to open a readonly file with append") return nil, syscall.Errno(syscall.ENOTSUP) } _, err := fd.Seek(0, io.SeekEnd) if err != nil { log.Error("seek reset failed: ", err) return nil, err } } return &File{fi: fd}, nil } func (fi *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error { return fi.fi.Close() } func (d *Directory) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { // New 'empty' file nd := dag.NodeWithData(ft.FilePBData(nil, 0)) err := d.dir.AddChild(req.Name, nd) if err != nil { return nil, nil, err } child, err := d.dir.Child(req.Name) if err != nil { return nil, nil, err } fi, ok := child.(*mfs.File) if !ok { return nil, nil, errors.New("child creation failed") } nodechild := &FileNode{fi: fi} fd, err := fi.Open(mfs.Flags{ Read: req.Flags.IsReadOnly() || req.Flags.IsReadWrite(), Write: req.Flags.IsWriteOnly() || req.Flags.IsReadWrite(), Sync: true, }) if err != nil { return nil, nil, err } return nodechild, &File{fi: fd}, nil } func (d *Directory) Remove(ctx context.Context, req *fuse.RemoveRequest) error { err := d.dir.Unlink(req.Name) if err != nil { return syscall.Errno(syscall.ENOENT) } return nil } // Rename implements NodeRenamer. func (d *Directory) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fs.Node) error { cur, err := d.dir.Child(req.OldName) if err != nil { return err } err = d.dir.Unlink(req.OldName) if err != nil { return err } switch newDir := newDir.(type) { case *Directory: nd, err := cur.GetNode() if err != nil { return err } err = newDir.dir.AddChild(req.NewName, nd) if err != nil { return err } case *FileNode: log.Error("Cannot move node into a file!") return syscall.Errno(syscall.EPERM) default: log.Error("Unknown node type for rename target dir!") return errors.New("unknown fs node type") } return nil } // to check that out Node implements all the interfaces we want. type ipnsRoot interface { fs.Node fs.HandleReadDirAller fs.NodeStringLookuper } var _ ipnsRoot = (*Root)(nil) type ipnsDirectory interface { fs.HandleReadDirAller fs.Node fs.NodeCreater fs.NodeMkdirer fs.NodeRemover fs.NodeRenamer fs.NodeStringLookuper } var _ ipnsDirectory = (*Directory)(nil) type ipnsFile interface { fs.HandleFlusher fs.HandleReader fs.HandleWriter fs.HandleReleaser } type ipnsFileNode interface { fs.Node fs.NodeFsyncer fs.NodeOpener } var ( _ ipnsFileNode = (*FileNode)(nil) _ ipnsFile = (*File)(nil) ) ================================================ FILE: fuse/ipns/link_unix.go ================================================ //go:build !nofuse && !openbsd && !netbsd && !plan9 package ipns import ( "context" "os" "bazil.org/fuse" "bazil.org/fuse/fs" ) type Link struct { Target string } func (l *Link) Attr(ctx context.Context, a *fuse.Attr) error { log.Debug("Link attr.") a.Mode = os.ModeSymlink | 0o555 return nil } func (l *Link) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { log.Debugf("ReadLink: %s", l.Target) return l.Target, nil } var _ fs.NodeReadlinker = (*Link)(nil) ================================================ FILE: fuse/ipns/mount_unix.go ================================================ //go:build (linux || darwin || freebsd || netbsd || openbsd) && !nofuse package ipns import ( core "github.com/ipfs/kubo/core" coreapi "github.com/ipfs/kubo/core/coreapi" mount "github.com/ipfs/kubo/fuse/mount" ) // Mount mounts ipns at a given location, and returns a mount.Mount instance. func Mount(ipfs *core.IpfsNode, ipnsmp, ipfsmp string) (mount.Mount, error) { coreAPI, err := coreapi.NewCoreAPI(ipfs) if err != nil { return nil, err } cfg, err := ipfs.Repo.Config() if err != nil { return nil, err } allowOther := cfg.Mounts.FuseAllowOther fsys, err := NewFileSystem(ipfs.Context(), coreAPI, ipfsmp, ipnsmp) if err != nil { return nil, err } return mount.NewMount(fsys, ipnsmp, allowOther) } ================================================ FILE: fuse/mfs/mfs_test.go ================================================ //go:build !nofuse && !openbsd && !netbsd && !plan9 package mfs import ( "bytes" "context" "crypto/rand" "errors" iofs "io/fs" "os" "slices" "strconv" "testing" "time" "bazil.org/fuse" "bazil.org/fuse/fs" "bazil.org/fuse/fs/fstestutil" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/node" "github.com/libp2p/go-libp2p-testing/ci" ) // Create an Ipfs.Node, a filesystem and a mount point. func setUp(t *testing.T, ipfs *core.IpfsNode) (fs.FS, *fstestutil.Mount) { if ci.NoFuse() { t.Skip("Skipping FUSE tests") } if ipfs == nil { var err error ipfs, err = core.NewNode(context.Background(), &node.BuildCfg{}) if err != nil { t.Fatal(err) } } fs := NewFileSystem(ipfs) mnt, err := fstestutil.MountedT(t, fs, nil) if err == fuse.ErrOSXFUSENotFound { t.Skip(err) } if err != nil { t.Fatal(err) } return fs, mnt } // Test reading and writing a file. func TestReadWrite(t *testing.T) { _, mnt := setUp(t, nil) defer mnt.Close() path := mnt.Dir + "/testrw" content := make([]byte, 8196) _, err := rand.Read(content) if err != nil { t.Fatal(err) } t.Run("write", func(t *testing.T) { f, err := os.Create(path) if err != nil { t.Fatal(err) } defer f.Close() _, err = f.Write(content) if err != nil { t.Fatal(err) } }) t.Run("read", func(t *testing.T) { f, err := os.Open(path) if err != nil { t.Fatal(err) } defer f.Close() buf := make([]byte, 8196) l, err := f.Read(buf) if err != nil { t.Fatal(err) } if bytes.Equal(content, buf[:l]) != true { t.Fatal("read and write not equal") } }) } // Test creating a directory. func TestMkdir(t *testing.T) { _, mnt := setUp(t, nil) defer mnt.Close() path := mnt.Dir + "/foo/bar/baz/qux/quux" t.Run("write", func(t *testing.T) { err := os.MkdirAll(path, iofs.ModeDir) if err != nil { t.Fatal(err) } }) t.Run("read", func(t *testing.T) { stat, err := os.Stat(path) if err != nil { t.Fatal(err) } if !stat.IsDir() { t.Fatal("not dir") } }) } // Test file persistence across mounts. func TestPersistence(t *testing.T) { ipfs, err := core.NewNode(context.Background(), &node.BuildCfg{}) if err != nil { t.Fatal(err) } content := make([]byte, 8196) _, err = rand.Read(content) if err != nil { t.Fatal(err) } t.Run("write", func(t *testing.T) { _, mnt := setUp(t, ipfs) defer mnt.Close() path := mnt.Dir + "/testpersistence" f, err := os.Create(path) if err != nil { t.Fatal(err) } defer f.Close() _, err = f.Write(content) if err != nil { t.Fatal(err) } }) t.Run("read", func(t *testing.T) { _, mnt := setUp(t, ipfs) defer mnt.Close() path := mnt.Dir + "/testpersistence" f, err := os.Open(path) if err != nil { t.Fatal(err) } defer f.Close() buf := make([]byte, 8196) l, err := f.Read(buf) if err != nil { t.Fatal(err) } if bytes.Equal(content, buf[:l]) != true { t.Fatal("read and write not equal") } }) } // Test getting the file attributes. func TestAttr(t *testing.T) { _, mnt := setUp(t, nil) defer mnt.Close() path := mnt.Dir + "/testattr" content := make([]byte, 8196) _, err := rand.Read(content) if err != nil { t.Fatal(err) } t.Run("write", func(t *testing.T) { f, err := os.Create(path) if err != nil { t.Fatal(err) } defer f.Close() _, err = f.Write(content) if err != nil { t.Fatal(err) } }) t.Run("read", func(t *testing.T) { fi, err := os.Stat(path) if err != nil { t.Fatal(err) } if fi.IsDir() { t.Fatal("file is a directory") } if fi.ModTime().After(time.Now()) { t.Fatal("future modtime") } if time.Since(fi.ModTime()) > time.Second { t.Fatal("past modtime") } if fi.Name() != "testattr" { t.Fatal("invalid filename") } if fi.Size() != 8196 { t.Fatal("invalid size") } }) } // Test concurrent access to the filesystem. func TestConcurrentRW(t *testing.T) { _, mnt := setUp(t, nil) defer mnt.Close() files := 5 fileWorkers := 5 path := mnt.Dir + "/testconcurrent" content := make([][]byte, files) for i := range content { content[i] = make([]byte, 8196) _, err := rand.Read(content[i]) if err != nil { t.Fatal(err) } } t.Run("write", func(t *testing.T) { errs := make(chan (error), 1) for i := range files { go func() { var err error defer func() { errs <- err }() f, err := os.Create(path + strconv.Itoa(i)) if err != nil { return } defer f.Close() _, err = f.Write(content[i]) if err != nil { return } }() } for range files { err := <-errs if err != nil { t.Fatal(err) } } }) t.Run("read", func(t *testing.T) { errs := make(chan (error), 1) for i := 0; i < files*fileWorkers; i++ { go func() { var err error defer func() { errs <- err }() f, err := os.Open(path + strconv.Itoa(i/fileWorkers)) if err != nil { return } defer f.Close() buf := make([]byte, 8196) l, err := f.Read(buf) if err != nil { return } if bytes.Equal(content[i/fileWorkers], buf[:l]) != true { err = errors.New("read and write not equal") return } }() } for range files { err := <-errs if err != nil { t.Fatal(err) } } }) } // Test ipfs_cid extended attribute func TestMFSRootXattr(t *testing.T) { ipfs, err := core.NewNode(context.Background(), &node.BuildCfg{}) if err != nil { t.Fatal(err) } fs, mnt := setUp(t, ipfs) defer mnt.Close() node, err := fs.Root() if err != nil { t.Fatal(err) } root := node.(*Dir) listReq := fuse.ListxattrRequest{} listRes := fuse.ListxattrResponse{} err = root.Listxattr(context.Background(), &listReq, &listRes) if err != nil { t.Fatal(err) } if slices.Compare(listRes.Xattr, []byte("ipfs_cid\x00")) != 0 { t.Fatal("list xattr returns invalid value") } getReq := fuse.GetxattrRequest{ Name: "ipfs_cid", } getRes := fuse.GetxattrResponse{} err = root.Getxattr(context.Background(), &getReq, &getRes) if err != nil { t.Fatal(err) } ipldNode, err := ipfs.FilesRoot.GetDirectory().GetNode() if err != nil { t.Fatal(err) } if slices.Compare(getRes.Xattr, []byte(ipldNode.Cid().String())) != 0 { t.Fatal("xattr cid not equal to mfs root cid") } } ================================================ FILE: fuse/mfs/mfs_unix.go ================================================ //go:build (linux || darwin || freebsd || netbsd || openbsd) && !nofuse package mfs import ( "context" "io" "os" "sync" "syscall" "time" "bazil.org/fuse" "bazil.org/fuse/fs" dag "github.com/ipfs/boxo/ipld/merkledag" ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/boxo/mfs" "github.com/ipfs/kubo/core" ) const ( ipfsCIDXattr = "ipfs_cid" mfsDirMode = os.ModeDir | 0755 mfsFileMode = 0644 blockSize = 512 dirSize = 8 ) // FUSE filesystem mounted at /mfs. type FileSystem struct { root Dir } // Get filesystem root. func (fs *FileSystem) Root() (fs.Node, error) { return &fs.root, nil } // FUSE Adapter for MFS directories. type Dir struct { mfsDir *mfs.Directory } // Directory attributes (stat). func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error { attr.Mode = mfsDirMode attr.Size = dirSize * blockSize attr.Blocks = dirSize return nil } // Access files in a directory. func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) { mfsNode, err := dir.mfsDir.Child(req.Name) switch err { case os.ErrNotExist: return nil, syscall.Errno(syscall.ENOENT) case nil: default: return nil, err } switch mfsNode.Type() { case mfs.TDir: result := Dir{ mfsDir: mfsNode.(*mfs.Directory), } return &result, nil case mfs.TFile: result := File{ mfsFile: mfsNode.(*mfs.File), } return &result, nil } return nil, syscall.Errno(syscall.ENOENT) } // List (ls) MFS directory. func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { var res []fuse.Dirent nodes, err := dir.mfsDir.List(ctx) if err != nil { return nil, err } for _, node := range nodes { nodeType := fuse.DT_File if node.Type == 1 { nodeType = fuse.DT_Dir } res = append(res, fuse.Dirent{ Type: nodeType, Name: node.Name, }) } return res, nil } // Mkdir (mkdir) in MFS. func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { mfsDir, err := dir.mfsDir.Mkdir(req.Name) if err != nil { return nil, err } return &Dir{ mfsDir: mfsDir, }, nil } // Remove (rm/rmdir) an MFS file. func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { // Check for empty directory. if req.Dir { targetNode, err := dir.mfsDir.Child(req.Name) if err != nil { return err } target := targetNode.(*mfs.Directory) children, err := target.ListNames(ctx) if err != nil { return err } if len(children) > 0 { return os.ErrExist } } err := dir.mfsDir.Unlink(req.Name) if err != nil { return err } return dir.mfsDir.Flush() } // Move (mv) an MFS file. func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fs.Node) error { file, err := dir.mfsDir.Child(req.OldName) if err != nil { return err } node, err := file.GetNode() if err != nil { return err } targetDir := newDir.(*Dir) // Remove file if exists err = targetDir.mfsDir.Unlink(req.NewName) if err != nil && err != os.ErrNotExist { return err } err = targetDir.mfsDir.AddChild(req.NewName, node) if err != nil { return err } err = dir.mfsDir.Unlink(req.OldName) if err != nil { return err } return dir.mfsDir.Flush() } // Create (touch) an MFS file. func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { node := dag.NodeWithData(ft.FilePBData(nil, 0)) if err := node.SetCidBuilder(dir.mfsDir.GetCidBuilder()); err != nil { return nil, nil, err } if err := dir.mfsDir.AddChild(req.Name, node); err != nil { return nil, nil, err } if err := dir.mfsDir.Flush(); err != nil { return nil, nil, err } mfsNode, err := dir.mfsDir.Child(req.Name) if err != nil { return nil, nil, err } if err := mfsNode.SetModTime(time.Now()); err != nil { return nil, nil, err } mfsFile := mfsNode.(*mfs.File) file := File{ mfsFile: mfsFile, } // Read access flags and create a handler. accessMode := req.Flags & fuse.OpenAccessModeMask flags := mfs.Flags{ Read: accessMode == fuse.OpenReadOnly || accessMode == fuse.OpenReadWrite, Write: accessMode == fuse.OpenWriteOnly || accessMode == fuse.OpenReadWrite, Sync: req.Flags|fuse.OpenSync > 0, } fd, err := mfsFile.Open(flags) if err != nil { return nil, nil, err } handler := FileHandler{ mfsFD: fd, } return &file, &handler, nil } // List dir xattr. func (dir *Dir) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { resp.Append(ipfsCIDXattr) return nil } // Get dir xattr. func (dir *Dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { switch req.Name { case ipfsCIDXattr: node, err := dir.mfsDir.GetNode() if err != nil { return err } resp.Xattr = []byte(node.Cid().String()) return nil default: return fuse.ErrNoXattr } } // FUSE adapter for MFS files. type File struct { mfsFile *mfs.File } // File attributes. func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error { size, _ := file.mfsFile.Size() attr.Size = uint64(size) if size%blockSize == 0 { attr.Blocks = uint64(size / blockSize) } else { attr.Blocks = uint64(size/blockSize + 1) } mtime, _ := file.mfsFile.ModTime() attr.Mtime = mtime attr.Mode = mfsFileMode return nil } // Open an MFS file. func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { accessMode := req.Flags & fuse.OpenAccessModeMask flags := mfs.Flags{ Read: accessMode == fuse.OpenReadOnly || accessMode == fuse.OpenReadWrite, Write: accessMode == fuse.OpenWriteOnly || accessMode == fuse.OpenReadWrite, Sync: req.Flags|fuse.OpenSync > 0, } fd, err := file.mfsFile.Open(flags) if err != nil { return nil, err } if flags.Write { if err := file.mfsFile.SetModTime(time.Now()); err != nil { return nil, err } } return &FileHandler{ mfsFD: fd, }, nil } // Sync the file's contents to MFS. func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { return file.mfsFile.Sync() } // List file xattr. func (file *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { resp.Append(ipfsCIDXattr) return nil } // Get file xattr. func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { switch req.Name { case ipfsCIDXattr: node, err := file.mfsFile.GetNode() if err != nil { return err } resp.Xattr = []byte(node.Cid().String()) return nil default: return fuse.ErrNoXattr } } // Wrapper for MFS's file descriptor that conforms to the FUSE fs.Handler // interface. type FileHandler struct { mfsFD mfs.FileDescriptor mu sync.Mutex } // Read a opened MFS file. func (fh *FileHandler) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { fh.mu.Lock() defer fh.mu.Unlock() _, err := fh.mfsFD.Seek(req.Offset, io.SeekStart) if err != nil { return err } buf := make([]byte, req.Size) l, err := fh.mfsFD.Read(buf) resp.Data = buf[:l] switch err { case nil, io.EOF, io.ErrUnexpectedEOF: return nil default: return err } } // Write writes to an opened MFS file. func (fh *FileHandler) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { fh.mu.Lock() defer fh.mu.Unlock() l, err := fh.mfsFD.WriteAt(req.Data, req.Offset) if err != nil { return err } resp.Size = l return nil } // Flushes the file's buffer. func (fh *FileHandler) Flush(ctx context.Context, req *fuse.FlushRequest) error { fh.mu.Lock() defer fh.mu.Unlock() return fh.mfsFD.Flush() } // Closes the file. func (fh *FileHandler) Release(ctx context.Context, req *fuse.ReleaseRequest) error { fh.mu.Lock() defer fh.mu.Unlock() return fh.mfsFD.Close() } // Create new filesystem. func NewFileSystem(ipfs *core.IpfsNode) fs.FS { return &FileSystem{ root: Dir{ mfsDir: ipfs.FilesRoot.GetDirectory(), }, } } // Check that our structs implement all the interfaces we want. type mfsDir interface { fs.Node fs.NodeGetxattrer fs.NodeListxattrer fs.HandleReadDirAller fs.NodeRequestLookuper fs.NodeMkdirer fs.NodeRenamer fs.NodeRemover fs.NodeCreater } var _ mfsDir = (*Dir)(nil) type mfsFile interface { fs.Node fs.NodeGetxattrer fs.NodeListxattrer fs.NodeOpener fs.NodeFsyncer } var _ mfsFile = (*File)(nil) type mfsHandler interface { fs.Handle fs.HandleReader fs.HandleWriter fs.HandleFlusher fs.HandleReleaser } var _ mfsHandler = (*FileHandler)(nil) ================================================ FILE: fuse/mfs/mount_unix.go ================================================ //go:build (linux || darwin || freebsd || netbsd || openbsd) && !nofuse package mfs import ( core "github.com/ipfs/kubo/core" mount "github.com/ipfs/kubo/fuse/mount" ) // Mount mounts MFS at a given location, and returns a mount.Mount instance. func Mount(ipfs *core.IpfsNode, mountpoint string) (mount.Mount, error) { cfg, err := ipfs.Repo.Config() if err != nil { return nil, err } allowOther := cfg.Mounts.FuseAllowOther fsys := NewFileSystem(ipfs) return mount.NewMount(fsys, mountpoint, allowOther) } ================================================ FILE: fuse/mount/fuse.go ================================================ //go:build !nofuse && !windows && !openbsd && !netbsd && !plan9 package mount import ( "errors" "fmt" "sync" "time" "bazil.org/fuse" "bazil.org/fuse/fs" ) var ErrNotMounted = errors.New("not mounted") // mount implements go-ipfs/fuse/mount. type mount struct { mpoint string filesys fs.FS fuseConn *fuse.Conn active bool activeLock *sync.RWMutex unmountOnce sync.Once } // Mount mounts a fuse fs.FS at a given location, and returns a Mount instance. // ctx is parent is a ContextGroup to bind the mount's ContextGroup to. func NewMount(fsys fs.FS, mountpoint string, allowOther bool) (Mount, error) { var conn *fuse.Conn var err error mountOpts := []fuse.MountOption{ fuse.MaxReadahead(64 * 1024 * 1024), fuse.AsyncRead(), } if allowOther { mountOpts = append(mountOpts, fuse.AllowOther()) } conn, err = fuse.Mount(mountpoint, mountOpts...) if err != nil { return nil, err } m := &mount{ mpoint: mountpoint, fuseConn: conn, filesys: fsys, active: false, activeLock: &sync.RWMutex{}, } // launch the mounting process. if err = m.mount(); err != nil { _ = m.Unmount() // just in case. return nil, err } return m, nil } func (m *mount) mount() error { log.Infof("Mounting %s", m.MountPoint()) errs := make(chan error, 1) go func() { // fs.Serve blocks until the filesystem is unmounted. err := fs.Serve(m.fuseConn, m.filesys) log.Debugf("%s is unmounted", m.MountPoint()) if err != nil { log.Debugf("fs.Serve returned (%s)", err) errs <- err } m.setActive(false) }() // wait for the mount process to be done, or timed out. select { case <-time.After(MountTimeout): return fmt.Errorf("mounting %s timed out", m.MountPoint()) case err := <-errs: return err case <-m.fuseConn.Ready: } // check if the mount process has an error to report if err := m.fuseConn.MountError; err != nil { return err } m.setActive(true) log.Infof("Mounted %s", m.MountPoint()) return nil } // unmount is called exactly once to unmount this service. // note that closing the connection will not always unmount // properly. If that happens, we bring out the big guns // (mount.ForceUnmountManyTimes, exec unmount). func (m *mount) unmount() error { log.Infof("Unmounting %s", m.MountPoint()) // try unmounting with fuse lib err := fuse.Unmount(m.MountPoint()) if err == nil { m.setActive(false) return nil } log.Warnf("fuse unmount err: %s", err) // try closing the fuseConn err = m.fuseConn.Close() if err == nil { m.setActive(false) return nil } log.Warnf("fuse conn error: %s", err) // try mount.ForceUnmountManyTimes if err := ForceUnmountManyTimes(m, 10); err != nil { return err } log.Infof("Seemingly unmounted %s", m.MountPoint()) m.setActive(false) return nil } func (m *mount) MountPoint() string { return m.mpoint } func (m *mount) Unmount() error { if !m.IsActive() { return ErrNotMounted } var err error m.unmountOnce.Do(func() { err = m.unmount() }) return err } func (m *mount) IsActive() bool { m.activeLock.RLock() defer m.activeLock.RUnlock() return m.active } func (m *mount) setActive(a bool) { m.activeLock.Lock() m.active = a m.activeLock.Unlock() } ================================================ FILE: fuse/mount/mount.go ================================================ // package mount provides a simple abstraction around a mount point package mount import ( "fmt" "io" "os/exec" "runtime" "time" logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("mount") var MountTimeout = time.Second * 5 // Mount represents a filesystem mount. type Mount interface { // MountPoint is the path at which this mount is mounted MountPoint() string // Unmounts the mount Unmount() error // Checks if the mount is still active. IsActive() bool } // ForceUnmount attempts to forcibly unmount a given mount. // It does so by calling diskutil or fusermount directly. func ForceUnmount(m Mount) error { point := m.MountPoint() log.Warnf("Force-Unmounting %s...", point) cmd, err := UnmountCmd(point) if err != nil { return err } errc := make(chan error, 1) go func() { defer close(errc) // try vanilla unmount first. if err := exec.Command("umount", point).Run(); err == nil { return } // retry to unmount with the fallback cmd errc <- cmd.Run() }() select { case <-time.After(7 * time.Second): return fmt.Errorf("umount timeout") case err := <-errc: return err } } // UnmountCmd creates an exec.Cmd that is GOOS-specific // for unmount a FUSE mount. func UnmountCmd(point string) (*exec.Cmd, error) { switch runtime.GOOS { case "darwin": return exec.Command("diskutil", "umount", "force", point), nil case "linux": return exec.Command("fusermount", "-u", point), nil default: return nil, fmt.Errorf("unmount: unimplemented") } } // ForceUnmountManyTimes attempts to forcibly unmount a given mount, // many times. It does so by calling diskutil or fusermount directly. // Attempts a given number of times. func ForceUnmountManyTimes(m Mount, attempts int) error { var err error for range attempts { err = ForceUnmount(m) if err == nil { return err } <-time.After(time.Millisecond * 500) } return fmt.Errorf("unmount %s failed after 10 seconds of trying", m.MountPoint()) } type closer struct { M Mount } func (c *closer) Close() error { log.Warn(" (c *closer) Close(),", c.M.MountPoint()) return c.M.Unmount() } func Closer(m Mount) io.Closer { return &closer{m} } ================================================ FILE: fuse/node/mount_darwin.go ================================================ //go:build !nofuse && darwin package node import ( "bytes" "fmt" "os/exec" "runtime" "strings" core "github.com/ipfs/kubo/core" "github.com/blang/semver/v4" unix "golang.org/x/sys/unix" ) func init() { // this is a hack, but until we need to do it another way, this works. platformFuseChecks = darwinFuseCheckVersion } // dontCheckOSXFUSEConfigKey is a key used to let the user tell us to // skip fuse checks. const dontCheckOSXFUSEConfigKey = "DontCheckOSXFUSE" // fuseVersionPkg is the go pkg url for fuse-version. const fuseVersionPkg = "github.com/jbenet/go-fuse-version/fuse-version" // errStrFuseRequired is returned when we're sure the user does not have fuse. const errStrFuseRequired = `OSXFUSE not found. OSXFUSE is required to mount, please install it. NOTE: Version 2.7.2 or higher required; prior versions are known to kernel panic! It is recommended you install it from the OSXFUSE website: http://osxfuse.github.io/ For more help, see: https://github.com/ipfs/kubo/issues/177 ` // errStrNoFuseHeaders is included in the output of `go get ` if there // are no fuse headers. this means they don't have OSXFUSE installed. var errStrNoFuseHeaders = "no such file or directory: '/usr/local/lib/libosxfuse.dylib'" var errStrUpgradeFuse = `OSXFUSE version %s not supported. OSXFUSE versions <2.7.2 are known to cause kernel panics! Please upgrade to the latest OSXFUSE version. It is recommended you install it from the OSXFUSE website: http://osxfuse.github.io/ For more help, see: https://github.com/ipfs/kubo/issues/177 ` type errNeedFuseVersion struct { cause string } func (me errNeedFuseVersion) Error() string { return fmt.Sprintf(`unable to check fuse version. Dear User, Before mounting, we must check your version of OSXFUSE. We are protecting you from a nasty kernel panic we found in OSXFUSE versions <2.7.2.[1]. To make matters worse, it's harder than it should be to check whether you have the right version installed...[2]. We've automated the process with the help of a little tool. We tried to install it, but something went wrong[3]. Please install it yourself by running: go get %s You can also stop ipfs from running these checks and use whatever OSXFUSE version you have by running: ipfs config --bool %s true [1]: https://github.com/ipfs/kubo/issues/177 [2]: https://github.com/ipfs/kubo/pull/533 [3]: %s `, fuseVersionPkg, dontCheckOSXFUSEConfigKey, me.cause) } var errStrFailedToRunFuseVersion = `unable to check fuse version. Dear User, Before mounting, we must check your version of OSXFUSE. We are protecting you from a nasty kernel panic we found in OSXFUSE versions <2.7.2.[1]. To make matters worse, it's harder than it should be to check whether you have the right version installed...[2]. We've automated the process with the help of a little tool. We tried to run it, but something went wrong[3]. Please, try to run it yourself with: go get %s fuse-version You should see something like this: > fuse-version fuse-version -only agent OSXFUSE.AgentVersion: 2.7.3 Just make sure the number is 2.7.2 or higher. You can then stop ipfs from trying to run these checks with: ipfs config --bool %s true [1]: https://github.com/ipfs/kubo/issues/177 [2]: https://github.com/ipfs/kubo/pull/533 [3]: %s ` var errStrFixConfig = `config key invalid: %s %v You may be able to get this error to go away by setting it again: ipfs config --bool %s true Either way, please tell us at: http://github.com/ipfs/kubo/issues ` func darwinFuseCheckVersion(node *core.IpfsNode) error { // on OSX, check FUSE version. if runtime.GOOS != "darwin" { return nil } ov, errGFV := tryGFV() if errGFV != nil { // if we failed AND the user has told us to ignore the check we // continue. this is in case fuse-version breaks or the user cannot // install it, but is sure their fuse version will work. if skip, err := userAskedToSkipFuseCheck(node); err != nil { return err } else if skip { return nil // user told us not to check version... ok.... } return errGFV } log.Debug("mount: osxfuse version:", ov) min := semver.MustParse("2.7.2") curr, err := semver.Make(ov) if err != nil { return err } if curr.LT(min) { return fmt.Errorf(errStrUpgradeFuse, ov) } return nil } func tryGFV() (string, error) { // first try sysctl. it may work! ov, err := trySysctl() if err == nil { return ov, nil } log.Debug(err) return tryGFVFromFuseVersion() } func trySysctl() (string, error) { v, err := unix.Sysctl("osxfuse.version.number") if err != nil { log.Debug("mount: sysctl osxfuse.version.number:", "failed") return "", err } log.Debug("mount: sysctl osxfuse.version.number:", v) return v, nil } func tryGFVFromFuseVersion() (string, error) { if err := ensureFuseVersionIsInstalled(); err != nil { return "", err } cmd := exec.Command("fuse-version", "-q", "-only", "agent", "-s", "OSXFUSE") out := new(bytes.Buffer) cmd.Stdout = out if err := cmd.Run(); err != nil { return "", fmt.Errorf(errStrFailedToRunFuseVersion, fuseVersionPkg, dontCheckOSXFUSEConfigKey, err) } return out.String(), nil } func ensureFuseVersionIsInstalled() error { // see if fuse-version is there if _, err := exec.LookPath("fuse-version"); err == nil { return nil // got it! } // try installing it... log.Debug("fuse-version: no fuse-version. attempting to install.") cmd := exec.Command("go", "install", "github.com/jbenet/go-fuse-version/fuse-version") cmdout := new(bytes.Buffer) cmd.Stdout = cmdout cmd.Stderr = cmdout if err := cmd.Run(); err != nil { // Ok, install fuse-version failed. is it they don't have fuse? cmdoutstr := cmdout.String() if strings.Contains(cmdoutstr, errStrNoFuseHeaders) { // yes! it is! they don't have fuse! return fmt.Errorf(errStrFuseRequired) } log.Debug("fuse-version: failed to install.") s := err.Error() + "\n" + cmdoutstr return errNeedFuseVersion{s} } // ok, try again... if _, err := exec.LookPath("fuse-version"); err != nil { log.Debug("fuse-version: failed to install?") return errNeedFuseVersion{err.Error()} } log.Debug("fuse-version: install success") return nil } func userAskedToSkipFuseCheck(node *core.IpfsNode) (skip bool, err error) { val, err := node.Repo.GetConfigKey(dontCheckOSXFUSEConfigKey) if err != nil { return false, nil // failed to get config value. don't skip check. } switch val := val.(type) { case string: return val == "true", nil case bool: return val, nil default: // got config value, but it's invalid... don't skip check, ask the user to fix it... return false, fmt.Errorf(errStrFixConfig, dontCheckOSXFUSEConfigKey, val, dontCheckOSXFUSEConfigKey) } } ================================================ FILE: fuse/node/mount_nofuse.go ================================================ //go:build !windows && nofuse package node import ( "errors" core "github.com/ipfs/kubo/core" ) func Mount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { return errors.New("not compiled in") } func Unmount(node *core.IpfsNode) { return } ================================================ FILE: fuse/node/mount_notsupp.go ================================================ //go:build (!nofuse && openbsd) || (!nofuse && netbsd) || (!nofuse && plan9) package node import ( "errors" core "github.com/ipfs/kubo/core" ) func Mount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { return errors.New("FUSE not supported on OpenBSD or NetBSD. See #5334 (https://github.com/ipfs/kubo/issues/5334).") } func Unmount(node *core.IpfsNode) { return } ================================================ FILE: fuse/node/mount_test.go ================================================ //go:build !openbsd && !nofuse && !netbsd && !plan9 package node import ( "context" "os" "strings" "testing" "time" "bazil.org/fuse" core "github.com/ipfs/kubo/core" ipns "github.com/ipfs/kubo/fuse/ipns" mount "github.com/ipfs/kubo/fuse/mount" ci "github.com/libp2p/go-libp2p-testing/ci" ) func maybeSkipFuseTests(t *testing.T) { if ci.NoFuse() { t.Skip("Skipping FUSE tests") } } func mkdir(t *testing.T, path string) { err := os.Mkdir(path, os.ModeDir|os.ModePerm) if err != nil { t.Fatal(err) } } // Test externally unmounting, then trying to unmount in code. func TestExternalUnmount(t *testing.T) { if testing.Short() { t.SkipNow() } // TODO: needed? maybeSkipFuseTests(t) node, err := core.NewNode(context.Background(), &core.BuildCfg{}) if err != nil { t.Fatal(err) } err = ipns.InitializeKeyspace(node, node.PrivateKey) if err != nil { t.Fatal(err) } // get the test dir paths (/tmp/TestExternalUnmount) dir := t.TempDir() ipfsDir := dir + "/ipfs" ipnsDir := dir + "/ipns" mfsDir := dir + "/mfs" mkdir(t, ipfsDir) mkdir(t, ipnsDir) mkdir(t, mfsDir) err = Mount(node, ipfsDir, ipnsDir, mfsDir) if err != nil { if strings.Contains(err.Error(), "unable to check fuse version") || err == fuse.ErrOSXFUSENotFound { t.Skip(err) } } if err != nil { t.Fatalf("error mounting: %v", err) } // Run shell command to externally unmount the directory cmd, err := mount.UnmountCmd(ipfsDir) if err != nil { t.Fatal(err) } if err := cmd.Run(); err != nil { t.Fatal(err) } // TODO(noffle): it takes a moment for the goroutine that's running fs.Serve to be notified and do its cleanup. time.Sleep(time.Millisecond * 100) // Attempt to unmount IPFS; it should unmount successfully. err = node.Mounts.Ipfs.Unmount() if err != mount.ErrNotMounted { t.Fatal("Unmount should have failed") } } ================================================ FILE: fuse/node/mount_unix.go ================================================ //go:build !windows && !openbsd && !netbsd && !plan9 && !nofuse package node import ( "errors" "fmt" "strings" "sync" core "github.com/ipfs/kubo/core" ipns "github.com/ipfs/kubo/fuse/ipns" mfs "github.com/ipfs/kubo/fuse/mfs" mount "github.com/ipfs/kubo/fuse/mount" rofs "github.com/ipfs/kubo/fuse/readonly" logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("node") // fuseNoDirectory used to check the returning fuse error. const fuseNoDirectory = "fusermount: failed to access mountpoint" // fuseExitStatus1 used to check the returning fuse error. const fuseExitStatus1 = "fusermount: exit status 1" // platformFuseChecks can get overridden by arch-specific files // to run fuse checks (like checking the OSXFUSE version). var platformFuseChecks = func(*core.IpfsNode) error { return nil } func Mount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { // check if we already have live mounts. // if the user said "Mount", then there must be something wrong. // so, close them and try again. Unmount(node) if err := platformFuseChecks(node); err != nil { return err } return doMount(node, fsdir, nsdir, mfsdir) } func Unmount(node *core.IpfsNode) { if node.Mounts.Ipfs != nil && node.Mounts.Ipfs.IsActive() { // best effort if err := node.Mounts.Ipfs.Unmount(); err != nil { log.Errorf("error unmounting IPFS: %s", err) } } if node.Mounts.Ipns != nil && node.Mounts.Ipns.IsActive() { // best effort if err := node.Mounts.Ipns.Unmount(); err != nil { log.Errorf("error unmounting IPNS: %s", err) } } if node.Mounts.Mfs != nil && node.Mounts.Mfs.IsActive() { // best effort if err := node.Mounts.Mfs.Unmount(); err != nil { log.Errorf("error unmounting MFS: %s", err) } } } func doMount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { fmtFuseErr := func(err error, mountpoint string) error { s := err.Error() if strings.Contains(s, fuseNoDirectory) { s = strings.Replace(s, `fusermount: "fusermount:`, "", -1) s = strings.Replace(s, `\n", exit status 1`, "", -1) return errors.New(s) } if s == fuseExitStatus1 { s = fmt.Sprintf("fuse failed to access mountpoint %s", mountpoint) return errors.New(s) } return err } // this sync stuff is so that both can be mounted simultaneously. var fsmount, nsmount, mfmount mount.Mount var err1, err2, err3 error var wg sync.WaitGroup wg.Go(func() { fsmount, err1 = rofs.Mount(node, fsdir) }) if node.IsOnline { wg.Go(func() { nsmount, err2 = ipns.Mount(node, nsdir, fsdir) }) } wg.Go(func() { mfmount, err3 = mfs.Mount(node, mfsdir) }) wg.Wait() if err1 != nil { log.Errorf("error mounting IPFS %s: %s", fsdir, err1) } if err2 != nil { log.Errorf("error mounting IPNS %s for IPFS %s: %s", nsdir, fsdir, err2) } if err3 != nil { log.Errorf("error mounting MFS %s: %s", mfsdir, err3) } if err1 != nil || err2 != nil || err3 != nil { if fsmount != nil { _ = fsmount.Unmount() } if nsmount != nil { _ = nsmount.Unmount() } if mfmount != nil { _ = mfmount.Unmount() } if err1 != nil { return fmtFuseErr(err1, fsdir) } if err2 != nil { return fmtFuseErr(err2, nsdir) } return fmtFuseErr(err3, mfsdir) } // setup node state, so that it can be canceled node.Mounts.Ipfs = fsmount node.Mounts.Ipns = nsmount node.Mounts.Mfs = mfmount return nil } ================================================ FILE: fuse/node/mount_windows.go ================================================ package node import ( "github.com/ipfs/kubo/core" ) func Mount(node *core.IpfsNode, fsdir, nsdir, mfsdir string) error { // TODO // currently a no-op, but we don't want to return an error return nil } func Unmount(node *core.IpfsNode) { // TODO // currently a no-op return } ================================================ FILE: fuse/readonly/doc.go ================================================ // package fuse/readonly implements a fuse filesystem to access files // stored inside of ipfs. package readonly ================================================ FILE: fuse/readonly/ipfs_test.go ================================================ //go:build !nofuse && !openbsd && !netbsd && !plan9 package readonly import ( "bytes" "context" "errors" "fmt" "io" "math/rand" "os" gopath "path" "strings" "sync" "testing" "bazil.org/fuse" core "github.com/ipfs/kubo/core" coreapi "github.com/ipfs/kubo/core/coreapi" coremock "github.com/ipfs/kubo/core/mock" fstest "bazil.org/fuse/fs/fstestutil" chunker "github.com/ipfs/boxo/chunker" "github.com/ipfs/boxo/files" dag "github.com/ipfs/boxo/ipld/merkledag" importer "github.com/ipfs/boxo/ipld/unixfs/importer" uio "github.com/ipfs/boxo/ipld/unixfs/io" "github.com/ipfs/boxo/path" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-test/random" ci "github.com/libp2p/go-libp2p-testing/ci" ) func maybeSkipFuseTests(t *testing.T) { if ci.NoFuse() { t.Skip("Skipping FUSE tests") } } func randObj(t *testing.T, nd *core.IpfsNode, size int64) (ipld.Node, []byte) { buf := make([]byte, size) _, err := io.ReadFull(random.NewRand(), buf) if err != nil { t.Fatal(err) } read := bytes.NewReader(buf) obj, err := importer.BuildTrickleDagFromReader(nd.DAG, chunker.DefaultSplitter(read)) if err != nil { t.Fatal(err) } return obj, buf } func setupIpfsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) { t.Helper() maybeSkipFuseTests(t) var err error if node == nil { node, err = coremock.NewMockNode() if err != nil { t.Fatal(err) } } fs := NewFileSystem(node) mnt, err := fstest.MountedT(t, fs, nil) if err == fuse.ErrOSXFUSENotFound { t.Skip(err) } if err != nil { t.Fatalf("error mounting temporary directory: %v", err) } return node, mnt } // Test writing an object and reading it back through fuse. func TestIpfsBasicRead(t *testing.T) { if testing.Short() { t.SkipNow() } nd, mnt := setupIpfsTest(t, nil) defer mnt.Close() fi, data := randObj(t, nd, 10000) k := fi.Cid() fname := gopath.Join(mnt.Dir, k.String()) rbuf, err := os.ReadFile(fname) if err != nil { t.Fatal(err) } if !bytes.Equal(rbuf, data) { t.Fatal("Incorrect Read!") } } func getPaths(t *testing.T, ipfs *core.IpfsNode, name string, n *dag.ProtoNode) []string { if len(n.Links()) == 0 { return []string{name} } var out []string for _, lnk := range n.Links() { child, err := lnk.GetNode(ipfs.Context(), ipfs.DAG) if err != nil { t.Fatal(err) } childpb, ok := child.(*dag.ProtoNode) if !ok { t.Fatal(dag.ErrNotProtobuf) } sub := getPaths(t, ipfs, gopath.Join(name, lnk.Name), childpb) out = append(out, sub...) } return out } // Perform a large number of concurrent reads to stress the system. func TestIpfsStressRead(t *testing.T) { if testing.Short() { t.SkipNow() } nd, mnt := setupIpfsTest(t, nil) defer mnt.Close() api, err := coreapi.NewCoreAPI(nd) if err != nil { t.Fatal(err) } var nodes []ipld.Node var paths []string nobj := 50 ndiriter := 50 // Make a bunch of objects for range nobj { fi, _ := randObj(t, nd, rand.Int63n(50000)) nodes = append(nodes, fi) paths = append(paths, fi.Cid().String()) } // Now make a bunch of dirs for range ndiriter { db, err := uio.NewDirectory(nd.DAG) if err != nil { t.Fatal(err) } for j := 0; j < 1+rand.Intn(10); j++ { name := fmt.Sprintf("child%d", j) err := db.AddChild(nd.Context(), name, nodes[rand.Intn(len(nodes))]) if err != nil { t.Fatal(err) } } newdir, err := db.GetNode() if err != nil { t.Fatal(err) } err = nd.DAG.Add(nd.Context(), newdir) if err != nil { t.Fatal(err) } nodes = append(nodes, newdir) npaths := getPaths(t, nd, newdir.Cid().String(), newdir.(*dag.ProtoNode)) paths = append(paths, npaths...) } // Now read a bunch, concurrently wg := sync.WaitGroup{} errs := make(chan error) for range 4 { wg.Go(func() { for range 2000 { item, err := path.NewPath("/ipfs/" + paths[rand.Intn(len(paths))]) if err != nil { errs <- err continue } relpath := strings.Replace(item.String(), item.Namespace(), "", 1) fname := gopath.Join(mnt.Dir, relpath) rbuf, err := os.ReadFile(fname) if err != nil { errs <- err } // nd.Context() is never closed which leads to // hitting 8128 goroutine limit in go test -race mode ctx, cancelFunc := context.WithCancel(context.Background()) read, err := api.Unixfs().Get(ctx, item) if err != nil { errs <- err } data, err := io.ReadAll(read.(files.File)) if err != nil { errs <- err } cancelFunc() if !bytes.Equal(rbuf, data) { errs <- errors.New("incorrect read") } } }) } go func() { wg.Wait() close(errs) }() for err := range errs { if err != nil { t.Fatal(err) } } } // Test writing a file and reading it back. func TestIpfsBasicDirRead(t *testing.T) { if testing.Short() { t.SkipNow() } nd, mnt := setupIpfsTest(t, nil) defer mnt.Close() // Make a 'file' fi, data := randObj(t, nd, 10000) // Make a directory and put that file in it db, err := uio.NewDirectory(nd.DAG) if err != nil { t.Fatal(err) } err = db.AddChild(nd.Context(), "actual", fi) if err != nil { t.Fatal(err) } d1nd, err := db.GetNode() if err != nil { t.Fatal(err) } err = nd.DAG.Add(nd.Context(), d1nd) if err != nil { t.Fatal(err) } dirname := gopath.Join(mnt.Dir, d1nd.Cid().String()) fname := gopath.Join(dirname, "actual") rbuf, err := os.ReadFile(fname) if err != nil { t.Fatal(err) } dirents, err := os.ReadDir(dirname) if err != nil { t.Fatal(err) } if len(dirents) != 1 { t.Fatal("Bad directory entry count") } if dirents[0].Name() != "actual" { t.Fatal("Bad directory entry") } if !bytes.Equal(rbuf, data) { t.Fatal("Incorrect Read!") } } // Test to make sure the filesystem reports file sizes correctly. func TestFileSizeReporting(t *testing.T) { if testing.Short() { t.SkipNow() } nd, mnt := setupIpfsTest(t, nil) defer mnt.Close() fi, data := randObj(t, nd, 10000) k := fi.Cid() fname := gopath.Join(mnt.Dir, k.String()) finfo, err := os.Stat(fname) if err != nil { t.Fatal(err) } if finfo.Size() != int64(len(data)) { t.Fatal("Read incorrect size from stat!") } } ================================================ FILE: fuse/readonly/mount_unix.go ================================================ //go:build (linux || darwin || freebsd) && !nofuse package readonly import ( core "github.com/ipfs/kubo/core" mount "github.com/ipfs/kubo/fuse/mount" ) // Mount mounts IPFS at a given location, and returns a mount.Mount instance. func Mount(ipfs *core.IpfsNode, mountpoint string) (mount.Mount, error) { cfg, err := ipfs.Repo.Config() if err != nil { return nil, err } allowOther := cfg.Mounts.FuseAllowOther fsys := NewFileSystem(ipfs) return mount.NewMount(fsys, mountpoint, allowOther) } ================================================ FILE: fuse/readonly/readonly_unix.go ================================================ //go:build (linux || darwin || freebsd) && !nofuse package readonly import ( "context" "fmt" "io" "os" "syscall" fuse "bazil.org/fuse" fs "bazil.org/fuse/fs" mdag "github.com/ipfs/boxo/ipld/merkledag" ft "github.com/ipfs/boxo/ipld/unixfs" uio "github.com/ipfs/boxo/ipld/unixfs/io" "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" core "github.com/ipfs/kubo/core" ipldprime "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" ) var log = logging.Logger("fuse/ipfs") // FileSystem is the readonly IPFS Fuse Filesystem. type FileSystem struct { Ipfs *core.IpfsNode } // NewFileSystem constructs new fs using given core.IpfsNode instance. func NewFileSystem(ipfs *core.IpfsNode) *FileSystem { return &FileSystem{Ipfs: ipfs} } // Root constructs the Root of the filesystem, a Root object. func (f FileSystem) Root() (fs.Node, error) { return &Root{Ipfs: f.Ipfs}, nil } // Root is the root object of the filesystem tree. type Root struct { Ipfs *core.IpfsNode } // Attr returns file attributes. func (*Root) Attr(ctx context.Context, a *fuse.Attr) error { a.Mode = os.ModeDir | 0o111 // -rw+x return nil } // Lookup performs a lookup under this node. func (s *Root) Lookup(ctx context.Context, name string) (fs.Node, error) { log.Debugf("Root Lookup: '%s'", name) switch name { case "mach_kernel", ".hidden", "._.": // Just quiet some log noise on OS X. return nil, syscall.Errno(syscall.ENOENT) } p, err := path.NewPath("/ipfs/" + name) if err != nil { log.Debugf("fuse failed to parse path: %q: %s", name, err) return nil, syscall.Errno(syscall.ENOENT) } imPath, err := path.NewImmutablePath(p) if err != nil { log.Debugf("fuse failed to convert path: %q: %s", name, err) return nil, syscall.Errno(syscall.ENOENT) } nd, ndLnk, err := s.Ipfs.UnixFSPathResolver.ResolvePath(ctx, imPath) if err != nil { // todo: make this error more versatile. return nil, syscall.Errno(syscall.ENOENT) } cidLnk, ok := ndLnk.(cidlink.Link) if !ok { log.Debugf("non-cidlink returned from ResolvePath: %v", ndLnk) return nil, syscall.Errno(syscall.ENOENT) } // convert ipld-prime node to universal node blk, err := s.Ipfs.Blockstore.Get(ctx, cidLnk.Cid) if err != nil { log.Debugf("fuse failed to retrieve block: %v: %s", cidLnk, err) return nil, syscall.Errno(syscall.ENOENT) } var fnd ipld.Node switch cidLnk.Cid.Prefix().Codec { case cid.DagProtobuf: adl, ok := nd.(ipldprime.ADL) if ok { substrate := adl.Substrate() fnd, err = mdag.ProtoNodeConverter(blk, substrate) } else { fnd, err = mdag.ProtoNodeConverter(blk, nd) } case cid.Raw: fnd, err = mdag.RawNodeConverter(blk, nd) default: log.Error("fuse node was not a supported type") return nil, syscall.Errno(syscall.ENOTSUP) } if err != nil { log.Errorf("could not convert protobuf or raw node: %s", err) return nil, syscall.Errno(syscall.ENOENT) } return &Node{Ipfs: s.Ipfs, Nd: fnd}, nil } // ReadDirAll reads a particular directory. Disallowed for root. func (*Root) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { log.Debug("read Root") return nil, syscall.Errno(syscall.EPERM) } // Node is the core object representing a filesystem tree node. type Node struct { Ipfs *core.IpfsNode Nd ipld.Node cached *ft.FSNode } func (s *Node) loadData() error { if pbnd, ok := s.Nd.(*mdag.ProtoNode); ok { fsn, err := ft.FSNodeFromBytes(pbnd.Data()) if err != nil { return err } s.cached = fsn } return nil } // Attr returns the attributes of a given node. func (s *Node) Attr(ctx context.Context, a *fuse.Attr) error { log.Debug("Node attr") if rawnd, ok := s.Nd.(*mdag.RawNode); ok { a.Mode = 0o444 a.Size = uint64(len(rawnd.RawData())) a.Blocks = 1 return nil } if s.cached == nil { if err := s.loadData(); err != nil { return fmt.Errorf("readonly: loadData() failed: %s", err) } } switch s.cached.Type() { case ft.TDirectory, ft.THAMTShard: a.Mode = os.ModeDir | 0o555 case ft.TFile: size := s.cached.FileSize() a.Mode = 0o444 a.Size = uint64(size) a.Blocks = uint64(len(s.Nd.Links())) case ft.TRaw: a.Mode = 0o444 a.Size = uint64(len(s.cached.Data())) a.Blocks = uint64(len(s.Nd.Links())) case ft.TSymlink: a.Mode = 0o777 | os.ModeSymlink a.Size = uint64(len(s.cached.Data())) default: return fmt.Errorf("invalid data type - %s", s.cached.Type()) } return nil } // Lookup performs a lookup under this node. func (s *Node) Lookup(ctx context.Context, name string) (fs.Node, error) { log.Debugf("Lookup '%s'", name) link, _, err := uio.ResolveUnixfsOnce(ctx, s.Ipfs.DAG, s.Nd, []string{name}) switch err { case os.ErrNotExist, mdag.ErrLinkNotFound: // todo: make this error more versatile. return nil, syscall.Errno(syscall.ENOENT) case nil: // noop default: log.Errorf("fuse lookup %q: %s", name, err) return nil, syscall.Errno(syscall.EIO) } nd, err := s.Ipfs.DAG.Get(ctx, link.Cid) if err != nil && !ipld.IsNotFound(err) { log.Errorf("fuse lookup %q: %s", name, err) return nil, err } return &Node{Ipfs: s.Ipfs, Nd: nd}, nil } // ReadDirAll reads the link structure as directory entries. func (s *Node) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { log.Debug("Node ReadDir") dir, err := uio.NewDirectoryFromNode(s.Ipfs.DAG, s.Nd) if err != nil { return nil, err } var entries []fuse.Dirent err = dir.ForEachLink(ctx, func(lnk *ipld.Link) error { n := lnk.Name if len(n) == 0 { n = lnk.Cid.String() } nd, err := s.Ipfs.DAG.Get(ctx, lnk.Cid) if err != nil { log.Warn("error fetching directory child node: ", err) } t := fuse.DT_Unknown switch nd := nd.(type) { case *mdag.RawNode: t = fuse.DT_File case *mdag.ProtoNode: if fsn, err := ft.FSNodeFromBytes(nd.Data()); err != nil { log.Warn("failed to unmarshal protonode data field:", err) } else { switch fsn.Type() { case ft.TDirectory, ft.THAMTShard: t = fuse.DT_Dir case ft.TFile, ft.TRaw: t = fuse.DT_File case ft.TSymlink: t = fuse.DT_Link case ft.TMetadata: log.Error("metadata object in fuse should contain its wrapped type") default: log.Error("unrecognized protonode data type: ", fsn.Type()) } } } entries = append(entries, fuse.Dirent{Name: n, Type: t}) return nil }) if err != nil { return nil, err } if len(entries) > 0 { return entries, nil } return nil, syscall.Errno(syscall.ENOENT) } func (s *Node) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { // TODO: is nil the right response for 'bug off, we ain't got none' ? resp.Xattr = nil return nil } func (s *Node) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { if s.cached == nil || s.cached.Type() != ft.TSymlink { return "", fuse.Errno(syscall.EINVAL) } return string(s.cached.Data()), nil } func (s *Node) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { r, err := uio.NewDagReader(ctx, s.Nd, s.Ipfs.DAG) if err != nil { return err } _, err = r.Seek(req.Offset, io.SeekStart) if err != nil { return err } // Data has a capacity of Size buf := resp.Data[:int(req.Size)] n, err := io.ReadFull(r, buf) resp.Data = buf[:n] switch err { case nil, io.EOF, io.ErrUnexpectedEOF: default: return err } resp.Data = resp.Data[:n] return nil // may be non-nil / not succeeded } // to check that our Node implements all the interfaces we want. type roRoot interface { fs.Node fs.HandleReadDirAller fs.NodeStringLookuper } var _ roRoot = (*Root)(nil) type roNode interface { fs.HandleReadDirAller fs.HandleReader fs.Node fs.NodeStringLookuper fs.NodeReadlinker fs.NodeGetxattrer } var _ roNode = (*Node)(nil) ================================================ FILE: gc/gc.go ================================================ // Package gc provides garbage collection for go-ipfs. package gc import ( "context" "errors" "fmt" "strings" bserv "github.com/ipfs/boxo/blockservice" bstore "github.com/ipfs/boxo/blockstore" offline "github.com/ipfs/boxo/exchange/offline" dag "github.com/ipfs/boxo/ipld/merkledag" pin "github.com/ipfs/boxo/pinning/pinner" "github.com/ipfs/boxo/verifcid" cid "github.com/ipfs/go-cid" dstore "github.com/ipfs/go-datastore" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("gc") // Result represents an incremental output from a garbage collection // run. It contains either an error, or the cid of a removed object. type Result struct { KeyRemoved cid.Cid Error error } // converts a set of CIDs with different codecs to a set of CIDs with the raw codec. func toRawCids(set *cid.Set) (*cid.Set, error) { newSet := cid.NewSet() err := set.ForEach(func(c cid.Cid) error { newSet.Add(cid.NewCidV1(cid.Raw, c.Hash())) return nil }) return newSet, err } // GC performs a mark and sweep garbage collection of the blocks in the blockstore // first, it creates a 'marked' set and adds to it the following: // - all recursively pinned blocks, plus all of their descendants (recursively) // - bestEffortRoots, plus all of its descendants (recursively) // - all directly pinned blocks // - all blocks utilized internally by the pinner // // The routine then iterates over every block in the blockstore and // deletes any block that is not found in the marked set. func GC(ctx context.Context, bs bstore.GCBlockstore, dstor dstore.Datastore, pn pin.Pinner, bestEffortRoots []cid.Cid) <-chan Result { ctx, cancel := context.WithCancel(ctx) unlocker := bs.GCLock(ctx) bsrv := bserv.New(bs, offline.Exchange(bs)) ds := dag.NewDAGService(bsrv) output := make(chan Result, 128) go func() { defer cancel() defer close(output) defer unlocker.Unlock(ctx) gcs, err := ColoredSet(ctx, pn, ds, bestEffortRoots, output) if err != nil { select { case output <- Result{Error: err}: case <-ctx.Done(): } return } // The blockstore reports raw blocks. We need to remove the codecs from the CIDs. gcs, err = toRawCids(gcs) if err != nil { select { case output <- Result{Error: err}: case <-ctx.Done(): } return } keychain, err := bs.AllKeysChan(ctx) if err != nil { select { case output <- Result{Error: err}: case <-ctx.Done(): } return } errors := false var removed uint64 loop: for ctx.Err() == nil { // select may not notice that we're "done". select { case k, ok := <-keychain: if !ok { break loop } // NOTE: assumes that all CIDs returned by the keychain are _raw_ CIDv1 CIDs. // This means we keep the block as long as we want it somewhere (CIDv1, CIDv0, Raw, other...). if !gcs.Has(k) { err := bs.DeleteBlock(ctx, k) removed++ if err != nil { errors = true select { case output <- Result{Error: &CannotDeleteBlockError{k, err}}: case <-ctx.Done(): break loop } // continue as error is non-fatal continue loop } select { case output <- Result{KeyRemoved: k}: case <-ctx.Done(): break loop } } case <-ctx.Done(): break loop } } if errors { select { case output <- Result{Error: ErrCannotDeleteSomeBlocks}: case <-ctx.Done(): return } } gds, ok := dstor.(dstore.GCDatastore) if !ok { return } err = gds.CollectGarbage(ctx) if err != nil { select { case output <- Result{Error: err}: case <-ctx.Done(): } return } }() return output } // Descendants recursively finds all the descendants of the given roots and // adds them to the given cid.Set, using the provided dag.GetLinks function // to walk the tree. func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots <-chan pin.StreamedPin) error { verifyGetLinks := func(ctx context.Context, c cid.Cid) ([]*ipld.Link, error) { err := verifcid.ValidateCid(verifcid.DefaultAllowlist, c) if err != nil { return nil, err } return getLinks(ctx, c) } verboseCidError := func(err error) error { if strings.Contains(err.Error(), verifcid.ErrDigestTooSmall.Error()) || strings.Contains(err.Error(), verifcid.ErrPossiblyInsecureHashFunction.Error()) { err = fmt.Errorf("\"%s\"\nPlease run 'ipfs pin verify'"+ // nolint " to list insecure hashes. If you want to read them,"+ " please downgrade your go-ipfs to 0.4.13\n", err) log.Error(err) } return err } for { select { case <-ctx.Done(): return ctx.Err() case wrapper, ok := <-roots: if !ok { return nil } if wrapper.Err != nil { return wrapper.Err } // Walk recursively walks the dag and adds the keys to the given set err := dag.Walk(ctx, verifyGetLinks, wrapper.Pin.Key, func(k cid.Cid) bool { return set.Visit(toCidV1(k)) }, dag.Concurrent()) if err != nil { err = verboseCidError(err) return err } } } } // toCidV1 converts any CIDv0s to CIDv1s. func toCidV1(c cid.Cid) cid.Cid { if c.Version() == 0 { return cid.NewCidV1(c.Type(), c.Hash()) } return c } // ColoredSet computes the set of nodes in the graph that are pinned by the // pins in the given pinner. func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffortRoots []cid.Cid, output chan<- Result) (*cid.Set, error) { // KeySet currently implemented in memory, in the future, may be bloom filter or // disk backed to conserve memory. errors := false gcs := cid.NewSet() getLinks := func(ctx context.Context, cid cid.Cid) ([]*ipld.Link, error) { links, err := ipld.GetLinks(ctx, ng, cid) if err != nil { errors = true select { case output <- Result{Error: &CannotFetchLinksError{cid, err}}: case <-ctx.Done(): return nil, ctx.Err() } } return links, nil } rkeys := pn.RecursiveKeys(ctx, false) err := Descendants(ctx, getLinks, gcs, rkeys) if err != nil { errors = true select { case output <- Result{Error: err}: case <-ctx.Done(): return nil, ctx.Err() } } bestEffortGetLinks := func(ctx context.Context, cid cid.Cid) ([]*ipld.Link, error) { links, err := ipld.GetLinks(ctx, ng, cid) if err != nil && !ipld.IsNotFound(err) { errors = true select { case output <- Result{Error: &CannotFetchLinksError{cid, err}}: case <-ctx.Done(): return nil, ctx.Err() } } return links, nil } bestEffortRootsChan := make(chan pin.StreamedPin) go func() { defer close(bestEffortRootsChan) for _, root := range bestEffortRoots { select { case <-ctx.Done(): return case bestEffortRootsChan <- pin.StreamedPin{Pin: pin.Pinned{Key: root}}: } } }() err = Descendants(ctx, bestEffortGetLinks, gcs, bestEffortRootsChan) if err != nil { errors = true select { case output <- Result{Error: err}: case <-ctx.Done(): return nil, ctx.Err() } } dkeys := pn.DirectKeys(ctx, false) for k := range dkeys { if k.Err != nil { return nil, k.Err } gcs.Add(toCidV1(k.Pin.Key)) } ikeys := pn.InternalPins(ctx, false) err = Descendants(ctx, getLinks, gcs, ikeys) if err != nil { errors = true select { case output <- Result{Error: err}: case <-ctx.Done(): return nil, ctx.Err() } } if errors { return nil, ErrCannotFetchAllLinks } return gcs, nil } // ErrCannotFetchAllLinks is returned as the last Result in the GC output // channel when there was an error creating the marked set because of a // problem when finding descendants. var ErrCannotFetchAllLinks = errors.New("garbage collection aborted: could not retrieve some links") // ErrCannotDeleteSomeBlocks is returned when removing blocks marked for // deletion fails as the last Result in GC output channel. var ErrCannotDeleteSomeBlocks = errors.New("garbage collection incomplete: could not delete some blocks") // CannotFetchLinksError provides detailed information about which links // could not be fetched and can appear as a Result in the GC output channel. type CannotFetchLinksError struct { Key cid.Cid Err error } // Error implements the error interface for this type with a useful // message. func (e *CannotFetchLinksError) Error() string { return fmt.Sprintf("could not retrieve links for %s: %s", e.Key, e.Err) } // CannotDeleteBlockError provides detailed information about which // blocks could not be deleted and can appear as a Result in the GC output // channel. type CannotDeleteBlockError struct { Key cid.Cid Err error } // Error implements the error interface for this type with a // useful message. func (e *CannotDeleteBlockError) Error() string { return fmt.Sprintf("could not remove %s: %s", e.Key, e.Err) } ================================================ FILE: gc/gc_test.go ================================================ package gc import ( "context" "testing" "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/ipld/merkledag" mdutils "github.com/ipfs/boxo/ipld/merkledag/test" pin "github.com/ipfs/boxo/pinning/pinner" "github.com/ipfs/boxo/pinning/pinner/dspinner" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" ) func TestGC(t *testing.T) { ctx := context.Background() ds := dssync.MutexWrap(datastore.NewMapDatastore()) bs := blockstore.NewGCBlockstore(blockstore.NewBlockstore(ds), blockstore.NewGCLocker()) bserv := blockservice.New(bs, offline.Exchange(bs)) dserv := merkledag.NewDAGService(bserv) pinner, err := dspinner.New(ctx, ds, dserv) require.NoError(t, err) daggen := mdutils.NewDAGGenerator() var expectedKept []multihash.Multihash var expectedDiscarded []multihash.Multihash // add some pins for range 5 { // direct root, _, err := daggen.MakeDagNode(dserv.Add, 0, 1) require.NoError(t, err) err = pinner.PinWithMode(ctx, root, pin.Direct, "") require.NoError(t, err) expectedKept = append(expectedKept, root.Hash()) // recursive root, allCids, err := daggen.MakeDagNode(dserv.Add, 5, 2) require.NoError(t, err) err = pinner.PinWithMode(ctx, root, pin.Recursive, "") require.NoError(t, err) expectedKept = append(expectedKept, toMHs(allCids)...) } err = pinner.Flush(ctx) require.NoError(t, err) // add more dags to be GCed for range 5 { _, allCids, err := daggen.MakeDagNode(dserv.Add, 5, 2) require.NoError(t, err) expectedDiscarded = append(expectedDiscarded, toMHs(allCids)...) } // and some other as "best effort roots" var bestEffortRoots []cid.Cid for range 5 { root, allCids, err := daggen.MakeDagNode(dserv.Add, 5, 2) require.NoError(t, err) bestEffortRoots = append(bestEffortRoots, root) expectedKept = append(expectedKept, toMHs(allCids)...) } ch := GC(ctx, bs, ds, pinner, bestEffortRoots) var discarded []multihash.Multihash for res := range ch { require.NoError(t, res.Error) discarded = append(discarded, res.KeyRemoved.Hash()) } allKeys, err := bs.AllKeysChan(ctx) require.NoError(t, err) var kept []multihash.Multihash for key := range allKeys { kept = append(kept, key.Hash()) } require.ElementsMatch(t, expectedDiscarded, discarded) require.ElementsMatch(t, expectedKept, kept) } func toMHs(cids []cid.Cid) []multihash.Multihash { res := make([]multihash.Multihash, len(cids)) for i, c := range cids { res[i] = c.Hash() } return res } ================================================ FILE: go.mod ================================================ module github.com/ipfs/kubo go 1.25.7 require ( bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc contrib.go.opencensus.io/exporter/prometheus v0.4.2 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 github.com/blang/semver/v4 v4.0.0 github.com/caddyserver/certmagic v0.23.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/ceramicnetwork/go-dag-jose v0.1.1 github.com/cheggaaa/pb v1.0.29 github.com/cockroachdb/pebble/v2 v2.1.4 github.com/coreos/go-systemd/v22 v22.7.0 github.com/dustin/go-humanize v1.0.1 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 github.com/fsnotify/fsnotify v1.9.0 github.com/google/uuid v1.6.0 github.com/hashicorp/go-version v1.8.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 github.com/ipfs/boxo v0.37.1-0.20260317235537-851246983422 github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.1 github.com/ipfs/go-datastore v0.9.1 github.com/ipfs/go-detect-race v0.0.1 github.com/ipfs/go-ds-badger v0.3.4 github.com/ipfs/go-ds-flatfs v0.6.0 github.com/ipfs/go-ds-leveldb v0.5.2 github.com/ipfs/go-ds-measure v0.2.2 github.com/ipfs/go-ds-pebble v0.5.9 github.com/ipfs/go-fs-lock v0.1.1 github.com/ipfs/go-ipfs-cmds v0.16.0 github.com/ipfs/go-ipld-cbor v0.2.1 github.com/ipfs/go-ipld-format v0.6.3 github.com/ipfs/go-ipld-git v0.1.1 github.com/ipfs/go-ipld-legacy v0.3.0 github.com/ipfs/go-log/v2 v2.9.1 github.com/ipfs/go-metrics-interface v0.3.0 github.com/ipfs/go-metrics-prometheus v0.1.0 github.com/ipfs/go-test v0.2.3 github.com/ipfs/go-unixfsnode v1.10.3 github.com/ipld/go-car/v2 v2.16.0 github.com/ipld/go-codec-dagpb v1.7.0 github.com/ipld/go-ipld-prime v0.22.0 github.com/ipshipyard/p2p-forge v0.7.0 github.com/jbenet/go-temp-err-catcher v0.1.0 github.com/julienschmidt/httprouter v1.3.0 github.com/libp2p/go-doh-resolver v0.5.0 github.com/libp2p/go-libp2p v0.48.0 github.com/libp2p/go-libp2p-http v0.5.0 github.com/libp2p/go-libp2p-kad-dht v0.39.0 github.com/libp2p/go-libp2p-kbucket v0.8.0 github.com/libp2p/go-libp2p-pubsub v0.15.0 github.com/libp2p/go-libp2p-pubsub-router v0.6.0 github.com/libp2p/go-libp2p-record v0.3.1 github.com/libp2p/go-libp2p-routing-helpers v0.7.5 github.com/libp2p/go-libp2p-testing v0.12.0 github.com/libp2p/go-socket-activation v0.1.1 github.com/miekg/dns v1.1.72 github.com/multiformats/go-multiaddr v0.16.1 github.com/multiformats/go-multiaddr-dns v0.5.0 github.com/multiformats/go-multibase v0.2.0 github.com/multiformats/go-multicodec v0.10.0 github.com/multiformats/go-multihash v0.2.3 github.com/opentracing/opentracing-go v1.2.0 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/probe-lab/go-libdht v0.4.0 github.com/prometheus/client_golang v1.23.2 github.com/stretchr/testify v1.11.1 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 go.opencensus.io v0.24.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 go.opentelemetry.io/contrib/propagators/autoprop v0.46.1 go.opentelemetry.io/otel v1.42.0 go.opentelemetry.io/otel/exporters/prometheus v0.56.0 go.opentelemetry.io/otel/sdk v1.40.0 go.opentelemetry.io/otel/sdk/metric v1.40.0 go.opentelemetry.io/otel/trace v1.42.0 go.uber.org/dig v1.19.0 go.uber.org/fx v1.24.0 go.uber.org/zap v1.27.1 golang.org/x/crypto v0.49.0 golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 golang.org/x/mod v0.34.0 golang.org/x/sync v0.20.0 golang.org/x/sys v0.42.0 google.golang.org/protobuf v1.36.11 ) require ( filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5 // indirect filippo.io/keygen v0.0.0-20260114151900-8e2790ea4c5b // indirect github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/DataDog/zstd v1.5.7 // indirect github.com/Jorropo/jsync v1.0.1 // indirect github.com/RaduBerinde/axisds v0.1.0 // indirect github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/caddyserver/zerossl v0.1.3 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect github.com/cockroachdb/errors v1.11.3 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf // indirect github.com/cskr/pubsub v1.0.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect github.com/dgraph-io/badger v1.6.2 // indirect github.com/dgraph-io/ristretto v0.0.2 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dunglas/httpsfv v1.1.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/filecoin-project/go-clock v0.1.0 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gammazero/chanqueue v1.1.2 // indirect github.com/gammazero/deque v1.2.1 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect github.com/google/gopacket v1.1.19 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect github.com/guillaumemichel/reservedpool v0.3.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-dsqueue v0.2.0 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect github.com/ipfs/go-ipfs-pq v0.0.4 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect github.com/ipfs/go-libdht v0.5.0 // indirect github.com/ipfs/go-peertaskqueue v0.8.3 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/koron/go-ssdp v0.0.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/libdns/libdns v1.0.0-beta.1 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.3.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect github.com/libp2p/go-libp2p-gostream v0.6.0 // indirect github.com/libp2p/go-libp2p-xor v0.1.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-netroute v0.4.0 // indirect github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v5 v5.0.1 // indirect github.com/libp2p/zeroconf/v2 v2.2.0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect github.com/mholt/acmez/v3 v3.1.2 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multistream v0.6.1 // indirect github.com/multiformats/go-varint v0.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/gomega v1.36.3 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pion/datachannel v1.5.10 // indirect github.com/pion/dtls/v3 v3.1.2 // indirect github.com/pion/ice/v4 v4.0.10 // indirect github.com/pion/interceptor v0.1.40 // indirect github.com/pion/logging v0.2.4 // indirect github.com/pion/mdns/v2 v2.0.7 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/rtcp v1.2.16 // indirect github.com/pion/rtp v1.8.19 // indirect github.com/pion/sctp v1.8.39 // indirect github.com/pion/sdp/v3 v3.0.18 // indirect github.com/pion/srtp/v3 v3.0.6 // indirect github.com/pion/stun/v3 v3.1.1 // indirect github.com/pion/transport/v3 v3.0.7 // indirect github.com/pion/transport/v4 v4.0.1 // indirect github.com/pion/turn/v4 v4.0.2 // indirect github.com/pion/webrtc/v4 v4.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.20.1 // indirect github.com/prometheus/statsd_exporter v0.27.1 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect github.com/quic-go/webtransport-go v0.10.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/rs/cors v1.11.1 // indirect github.com/slok/go-http-metrics v0.13.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/texttheater/golang-levenshtein v1.0.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.3.1 // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/wlynxg/anet v0.0.5 // indirect github.com/zeebo/blake3 v0.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/propagators/aws v1.21.1 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.21.1 // indirect go.opentelemetry.io/contrib/propagators/jaeger v1.21.1 // indirect go.opentelemetry.io/contrib/propagators/ot v1.21.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.40.0 // indirect go.opentelemetry.io/otel/metric v1.42.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/mock v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap/exp v0.3.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/oauth2 v0.35.0 // indirect golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c // indirect golang.org/x/term v0.41.0 // indirect golang.org/x/text v0.35.0 // indirect golang.org/x/time v0.12.0 // indirect golang.org/x/tools v0.43.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gonum.org/v1/gonum v0.17.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/grpc v1.78.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.4.1 // indirect ) // Exclude ancient +incompatible versions that confuse Dependabot. // These pre-Go-modules versions reference packages that no longer exist. exclude ( github.com/ipfs/go-ipfs-cmds v2.0.1+incompatible github.com/libp2p/go-libp2p v6.0.23+incompatible ) ================================================ FILE: go.sum ================================================ bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc h1:utDghgcjE8u+EBjHOgYT+dJPcnDF05KqWMBcjuJy510= bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5 h1:JA0fFr+kxpqTdxR9LOBiTWpGNchqmkcsgmdeJZRclZ0= filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI= filippo.io/keygen v0.0.0-20260114151900-8e2790ea4c5b h1:REI1FbdW71yO56Are4XAxD+OS/e+BQsB3gE4mZRQEXY= filippo.io/keygen v0.0.0-20260114151900-8e2790ea4c5b/go.mod h1:9nnw1SlYHYuPSo/3wjQzNjSbeHlq2NsKo5iEtfJPWP0= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/RaduBerinde/axisds v0.1.0 h1:YItk/RmU5nvlsv/awo2Fjx97Mfpt4JfgtEVAGPrLdz8= github.com/RaduBerinde/axisds v0.1.0/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y= github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk= github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM= github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/caddyserver/certmagic v0.23.0 h1:CfpZ/50jMfG4+1J/u2LV6piJq4HOfO6ppOnOf7DkFEU= github.com/caddyserver/certmagic v0.23.0/go.mod h1:9mEZIWqqWoI+Gf+4Trh04MOVPD0tGSxtqsxg87hAIH4= github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA= github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4= github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3 h1:oe6fCvaEpkhyW3qAicT0TnGtyht/UrgvOwMcEgLb7Aw= github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3/go.mod h1:qdP0gaj0QtgX2RUZhnlVrceJ+Qln8aSlDyJwelLLFeM= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/ceramicnetwork/go-dag-jose v0.1.1 h1:7pObs22egc14vSS3AfCFfS1VmaL4lQUsAK7OGC3PlKk= github.com/ceramicnetwork/go-dag-jose v0.1.1/go.mod h1:8ptnYwY2Z2y/s5oJnNBn/UCxLg6CpramNJ2ZXF/5aNY= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= github.com/cockroachdb/pebble/v2 v2.1.4 h1:j9wPgMDbkErFdAKYFGhsoCcvzcjR+6zrJ4jhKtJ6bOk= github.com/cockroachdb/pebble/v2 v2.1.4/go.mod h1:Reo1RTniv1UjVTAu/Fv74y5i3kJ5gmVrPhO9UtFiKn8= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b h1:VXvSNzmr8hMj8XTuY0PT9Ane9qZGul/p67vGYwl9BFI= github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf h1:dwGgBWn84wUS1pVikGiruW+x5XM4amhjaZO20vCjay4= github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dunglas/httpsfv v1.1.0 h1:Jw76nAyKWKZKFrpMMcL76y35tOpYHqQPzHQiwDvpe54= github.com/dunglas/httpsfv v1.1.0/go.mod h1:zID2mqw9mFsnt7YC3vYQ9/cjq30q41W+1AnDwH8TiMg= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 h1:QV0ZrfBLpFc2KDk+a4LJefDczXnonRwrYrQJY/9L4dA= github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302/go.mod h1:qBlWZqWeVx9BjvqBsnC/8RUlAYpIFmPvgROcw0n1scE= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gammazero/chanqueue v1.1.2 h1:dZEsxlyANZMyeTRemABqZF8QM9BnE4NBI43Oh3y5fIU= github.com/gammazero/chanqueue v1.1.2/go.mod h1:XDN1X/jjAbmSceNFOQbtKToeSkxtdVdpKu90LiEdBEE= github.com/gammazero/deque v1.2.1 h1:9fnQVFCCZ9/NOc7ccTNqzoKd1tCWOqeI05/lPqFPMGQ= github.com/gammazero/deque v1.2.1/go.mod h1:5nSFkzVm+afG9+gy0VIowlqVAW4N8zNcMne+CMQVD2g= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 h1:r5GgOLGbza2wVHRzK7aAj6lWZjfbAwiu/RDCVOKjRyM= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e h1:4bw4WeyTYPp0smaXiJZCNnLrvVBqirQVreixayXezGc= github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII= github.com/guillaumemichel/reservedpool v0.3.0 h1:eqqO/QvTllLBrit7LVtVJBqw4cD0WdV9ajUe7WNTajw= github.com/guillaumemichel/reservedpool v0.3.0/go.mod h1:sXSDIaef81TFdAJglsCFCMfgF5E5Z5xK1tFhjDhvbUc= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs-shipyard/nopfs v0.0.14 h1:HFepJt/MxhZ3/GsLZkkAPzIPdNYKaLO1Qb7YmPbWIKk= github.com/ipfs-shipyard/nopfs v0.0.14/go.mod h1:mQyd0BElYI2gB/kq/Oue97obP4B3os4eBmgfPZ+hnrE= github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcdHUd7SDsUOY= github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.37.1-0.20260317235537-851246983422 h1:yY3ot/DU1bqTzHDBARACM76Tbx9s4xzcRbzifG1e/es= github.com/ipfs/boxo v0.37.1-0.20260317235537-851246983422/go.mod h1:8yyiRn54F2CsW13n0zwXEPrVsZix/gFj9SYIRYMZ6KE= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= github.com/ipfs/go-block-format v0.2.3/go.mod h1:WJaQmPAKhD3LspLixqlqNFxiZ3BZ3xgqxxoSR/76pnA= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.6.0 h1:DlOReBV1xhHBhhfy/gBNNTSyfOM6rLiIx9J7A4DGf30= github.com/ipfs/go-cid v0.6.0/go.mod h1:NC4kS1LZjzfhK40UGmpXv5/qD2kcMzACYJNntCUiDhQ= github.com/ipfs/go-cidutil v0.1.1 h1:COuby6H8C2ml0alvHYX3WdbFM4F07YtbY0UlT5j+sgI= github.com/ipfs/go-cidutil v0.1.1/go.mod h1:SCoUftGEUgoXe5Hjeyw5CiLZF8cwYn/TbtpFQXJCP6k= github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.9.1 h1:67Po2epre/o0UxrmkzdS9ZTe2GFGODgTd2odx8Wh6Yo= github.com/ipfs/go-datastore v0.9.1/go.mod h1:zi07Nvrpq1bQwSkEnx3bfjz+SQZbdbWyCNvyxMh9pN0= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.3.4 h1:MmqFicftE0KrwMC77WjXTrPuoUxhwyFsjKONSeWrlOo= github.com/ipfs/go-ds-badger v0.3.4/go.mod h1:HfqsKJcNnIr9ZhZ+rkwS1J5PpaWjJjg6Ipmxd7KPfZ8= github.com/ipfs/go-ds-flatfs v0.6.0 h1:olAEnDNBK1VMoTRZvfzgo90H5kBP4qIZPpYMtNlBBws= github.com/ipfs/go-ds-flatfs v0.6.0/go.mod h1:p8a/YhmAFYyuonxDbvuIANlDCgS69uqVv+iH5f8fAxY= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-ds-measure v0.2.2 h1:4kwvBGbbSXNYe4ANlg7qTIYoZU6mNlqzQHdVqICkqGI= github.com/ipfs/go-ds-measure v0.2.2/go.mod h1:b/87ak0jMgH9Ylt7oH0+XGy4P8jHx9KG09Qz+pOeTIs= github.com/ipfs/go-ds-pebble v0.5.9 h1:D1FEuMxjbEmDADNqsyT74n9QHVAn12nv9i9Qa15AFYc= github.com/ipfs/go-ds-pebble v0.5.9/go.mod h1:XmUBN05l6B+tMg7mpMS75ZcKW/CX01uZMhhWw85imQA= github.com/ipfs/go-dsqueue v0.2.0 h1:MBi9w3oSiX98Xc+Y7NuJ9G8MI6mAT4IGdO9dHEMCZzU= github.com/ipfs/go-dsqueue v0.2.0/go.mod h1:8FfNQC4DMF/KkzBXRNB9Rb3MKDW0Sh98HMtXYl1mLQE= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= github.com/ipfs/go-fs-lock v0.1.1/go.mod h1:2goSXMCw7QfscHmSe09oXiR34DQeUdm+ei+dhonqly0= github.com/ipfs/go-ipfs-cmds v0.16.0 h1:Oq39Gzz3pWrPwP25SjbfBQugFjKyjFdsHlAMIXdzYzM= github.com/ipfs/go-ipfs-cmds v0.16.0/go.mod h1:iRNtY9ipM/vH0Yr+/0FY+JfT+trZDQIztDoesmmoTo4= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw= github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo= github.com/ipfs/go-ipfs-pq v0.0.4 h1:U7jjENWJd1jhcrR8X/xHTaph14PTAK9O+yaLJbjqgOw= github.com/ipfs/go-ipfs-pq v0.0.4/go.mod h1:9UdLOIIb99IFrgT0Fc53pvbvlJBhpUb4GJuAQf3+O2A= github.com/ipfs/go-ipfs-redirects-file v0.1.2 h1:QCK7VtL91FH17KROVVy5KrzDx2hu68QvB2FTWk08ZQk= github.com/ipfs/go-ipfs-redirects-file v0.1.2/go.mod h1:yIiTlLcDEM/8lS6T3FlCEXZktPPqSOyuY6dEzVqw7Fw= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= github.com/ipfs/go-ipld-cbor v0.2.1 h1:H05yEJbK/hxg0uf2AJhyerBDbjOuHX4yi+1U/ogRa7E= github.com/ipfs/go-ipld-cbor v0.2.1/go.mod h1:x9Zbeq8CoE5R2WicYgBMcr/9mnkQ0lHddYWJP2sMV3A= github.com/ipfs/go-ipld-format v0.6.3 h1:9/lurLDTotJpZSuL++gh3sTdmcFhVkCwsgx2+rAh4j8= github.com/ipfs/go-ipld-format v0.6.3/go.mod h1:74ilVN12NXVMIV+SrBAyC05UJRk0jVvGqdmrcYZvCBk= github.com/ipfs/go-ipld-git v0.1.1 h1:TWGnZjS0htmEmlMFEkA3ogrNCqWjIxwr16x1OsdhG+Y= github.com/ipfs/go-ipld-git v0.1.1/go.mod h1:+VyMqF5lMcJh4rwEppV0e6g4nCCHXThLYYDpKUkJubI= github.com/ipfs/go-ipld-legacy v0.3.0 h1:7XhFKkRyCvP5upOlQfKUFIqL3S5DEZnbUE4bQmQ/tNE= github.com/ipfs/go-ipld-legacy v0.3.0/go.mod h1:Ukef9ARQiX+RVetwH2XiReLgJvQDEXcUPszrZ1KRjKI= github.com/ipfs/go-libdht v0.5.0 h1:ZN+eCqwahZvUeT0e4DsIxRtm78Mc9UR5tmZUiMsrGjQ= github.com/ipfs/go-libdht v0.5.0/go.mod h1:L3YiuFXecLeZZFuuVRM0hjg1GgVhARzUdahFsuqSa7w= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log/v2 v2.9.1 h1:3JXwHWU31dsCpvQ+7asz6/QsFJHqFr4gLgQ0FWteujk= github.com/ipfs/go-log/v2 v2.9.1/go.mod h1:evFx7sBiohUN3AG12mXlZBw5hacBQld3ZPHrowlJYoo= github.com/ipfs/go-metrics-interface v0.3.0 h1:YwG7/Cy4R94mYDUuwsBfeziJCVm9pBMJ6q/JR9V40TU= github.com/ipfs/go-metrics-interface v0.3.0/go.mod h1:OxxQjZDGocXVdyTPocns6cOLwHieqej/jos7H4POwoY= github.com/ipfs/go-metrics-prometheus v0.1.0 h1:bApWOHkrH3VTBHzTHrZSfq4n4weOZDzZFxUXv+HyKcA= github.com/ipfs/go-metrics-prometheus v0.1.0/go.mod h1:2GtL525C/4yxtvSXpRJ4dnE45mCX9AS0XRa03vHx7G0= github.com/ipfs/go-peertaskqueue v0.8.3 h1:tBPpGJy+A92RqtRFq5amJn0Uuj8Pw8tXi0X3eHfHM8w= github.com/ipfs/go-peertaskqueue v0.8.3/go.mod h1:OqVync4kPOcXEGdj/LKvox9DCB5mkSBeXsPczCxLtYA= github.com/ipfs/go-test v0.2.3 h1:Z/jXNAReQFtCYyn7bsv/ZqUwS6E7iIcSpJ2CuzCvnrc= github.com/ipfs/go-test v0.2.3/go.mod h1:QW8vSKkwYvWFwIZQLGQXdkt9Ud76eQXRQ9Ao2H+cA1o= github.com/ipfs/go-unixfsnode v1.10.3 h1:c8sJjuGNkxXAQH75P+f5ngPda/9T+DrboVA0TcDGvGI= github.com/ipfs/go-unixfsnode v1.10.3/go.mod h1:2Jlc7DoEwr12W+7l8Hr6C7XF4NHST3gIkqSArLhGSxU= github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco= github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.22.0 h1:YJhDhjEOvOYaqshd3b4atIWUoRg/rKrgmwCyUHwlbuY= github.com/ipld/go-ipld-prime v0.22.0/go.mod h1:ol7vKxOOVgEh0iAPuiDalM+0gScXVMA5ZZa4DVrTnEA= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20250821084354-a425e60cd714 h1:cqNk8PEwHnK0vqWln+U/YZhQc9h2NB3KjUjDPZo5Q2s= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20250821084354-a425e60cd714/go.mod h1:ZEUdra3CoqRVRYgAX/jAJO9aZGz6SKtKEG628fHHktY= github.com/ipshipyard/p2p-forge v0.7.0 h1:PQayexxZC1FR2Vx0XOSbmZ6wDPliidS48I+xXWuF+YU= github.com/ipshipyard/p2p-forge v0.7.0/go.mod h1:i2wg0p7WmHGyo5vYaK9COZBp8BN5Drncfu3WoQNZlQY= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= github.com/koron/go-ssdp v0.0.6/go.mod h1:0R9LfRJGek1zWTjN3JUNlm5INCDYGpRDfAptnct63fI= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/libdns/libdns v1.0.0-beta.1 h1:KIf4wLfsrEpXpZ3vmc/poM8zCATXT2klbdPe6hyOBjQ= github.com/libdns/libdns v1.0.0-beta.1/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-doh-resolver v0.5.0 h1:4h7plVVW+XTS+oUBw2+8KfoM1jF6w8XmO7+skhePFdE= github.com/libp2p/go-doh-resolver v0.5.0/go.mod h1:aPDxfiD2hNURgd13+hfo29z9IC22fv30ee5iM31RzxU= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.3.0 h1:q31zcHUvHnwDO0SHaukewPYgwOBSxtt830uJtUx6784= github.com/libp2p/go-flow-metrics v0.3.0/go.mod h1:nuhlreIwEguM1IvHAew3ij7A8BMlyHQJ279ao24eZZo= github.com/libp2p/go-libp2p v0.48.0 h1:h2BrLAgrj7X8bEN05K7qmrjpNHYA+6tnsGRdprjTnvo= github.com/libp2p/go-libp2p v0.48.0/go.mod h1:Q1fBZNdmC2Hf82husCTfkKJVfHm2we5zk+NWmOGEmWk= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw= github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU= github.com/libp2p/go-libp2p-gostream v0.6.0/go.mod h1:Nywu0gYZwfj7Jc91PQvbGU8dIpqbQQkjWgDuOrFaRdA= github.com/libp2p/go-libp2p-http v0.5.0 h1:+x0AbLaUuLBArHubbbNRTsgWz0RjNTy6DJLOxQ3/QBc= github.com/libp2p/go-libp2p-http v0.5.0/go.mod h1:glh87nZ35XCQyFsdzZps6+F4HYI6DctVFY5u1fehwSg= github.com/libp2p/go-libp2p-kad-dht v0.39.0 h1:mww38eBYiUvdsu+Xl/GLlBC0Aa8M+5HAwvafkFOygAM= github.com/libp2p/go-libp2p-kad-dht v0.39.0/go.mod h1:Po2JugFEkDq9Vig/JXtc153ntOi0q58o4j7IuITCOVs= github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio= github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s= github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4= github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs= github.com/libp2p/go-libp2p-pubsub v0.15.0 h1:cG7Cng2BT82WttmPFMi50gDNV+58K626m/wR00vGL1o= github.com/libp2p/go-libp2p-pubsub v0.15.0/go.mod h1:lr4oE8bFgQaifRcoc2uWhWWiK6tPdOEKpUuR408GFN4= github.com/libp2p/go-libp2p-pubsub-router v0.6.0 h1:D30iKdlqDt5ZmLEYhHELCMRj8b4sFAqrUcshIUvVP/s= github.com/libp2p/go-libp2p-pubsub-router v0.6.0/go.mod h1:FY/q0/RBTKsLA7l4vqC2cbRbOvyDotg8PJQ7j8FDudE= github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= github.com/libp2p/go-libp2p-routing-helpers v0.7.5 h1:HdwZj9NKovMx0vqq6YNPTh6aaNzey5zHD7HeLJtq6fI= github.com/libp2p/go-libp2p-routing-helpers v0.7.5/go.mod h1:3YaxrwP0OBPDD7my3D0KxfR89FlcX/IEbxDEDfAmj98= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= github.com/libp2p/go-libp2p-xor v0.1.0 h1:hhQwT4uGrBcuAkUGXADuPltalOdpf9aag9kaYNT2tLA= github.com/libp2p/go-libp2p-xor v0.1.0/go.mod h1:LSTM5yRnjGZbWNTA/hRwq2gGFrvRIbQJscoIL/u6InY= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-netroute v0.4.0 h1:sZZx9hyANYUx9PZyqcgE/E1GUG3iEtTZHUEvdtXT7/Q= github.com/libp2p/go-netroute v0.4.0/go.mod h1:Nkd5ShYgSMS5MUKy/MU2T57xFoOKvvLR92Lic48LEyA= github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= github.com/libp2p/go-socket-activation v0.1.1 h1:wkLBj6RqKffjt7BI794ewoSt241UV52NKYvIbpzhn4Q= github.com/libp2p/go-socket-activation v0.1.1/go.mod h1:NBfVUPXTRL/FU6UmSOM+1O7/vJkpS523sQiriw0Qln8= github.com/libp2p/go-yamux/v5 v5.0.1 h1:f0WoX/bEF2E8SbE4c/k1Mo+/9z0O4oC/hWEA+nfYRSg= github.com/libp2p/go-yamux/v5 v5.0.1/go.mod h1:en+3cdX51U0ZslwRdRLrvQsdayFt3TSUKvBGErzpWbU= github.com/libp2p/zeroconf/v2 v2.2.0 h1:Cup06Jv6u81HLhIj1KasuNM/RHHrJ8T7wOTS4+Tv53Q= github.com/libp2p/zeroconf/v2 v2.2.0/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marcopolo/simnet v0.0.4 h1:50Kx4hS9kFGSRIbrt9xUS3NJX33EyPqHVmpXvaKLqrY= github.com/marcopolo/simnet v0.0.4/go.mod h1:tfQF1u2DmaB6WHODMtQaLtClEf3a296CKQLq5gAsIS0= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mholt/acmez/v3 v3.1.2 h1:auob8J/0FhmdClQicvJvuDavgd5ezwLBfKuYmynhYzc= github.com/mholt/acmez/v3 v3.1.2/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= github.com/multiformats/go-multiaddr v0.16.1 h1:fgJ0Pitow+wWXzN9do+1b8Pyjmo8m5WhGfzpL82MpCw= github.com/multiformats/go-multiaddr v0.16.1/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= github.com/multiformats/go-multiaddr-dns v0.5.0 h1:p/FTyHKX0nl59f+S+dEUe8HRK+i5Ow/QHMw8Nh3gPCo= github.com/multiformats/go-multiaddr-dns v0.5.0/go.mod h1:yJ349b8TPIAANUyuOzn1oz9o22tV9f+06L+cCeMxC14= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multicodec v0.10.0 h1:UpP223cig/Cx8J76jWt91njpK3GTAO1w02sdcjZDSuc= github.com/multiformats/go-multicodec v0.10.0/go.mod h1:wg88pM+s2kZJEQfRCKBNU+g32F5aWBEjyFHXvZLTcLI= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.6.1 h1:4aoX5v6T+yWmc2raBHsTvzmFhOI8WVOer28DeBBEYdQ= github.com/multiformats/go-multistream v0.6.1/go.mod h1:ksQf6kqHAb6zIsyw7Zm+gAuVo57Qbq84E27YlYqavqw= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.1.0 h1:i2wqFp4sdl3IcIxfAonHQV9qU5OsZ4Ts9IOoETFs5dI= github.com/multiformats/go-varint v0.1.0/go.mod h1:5KVAVXegtfmNQQm/lCY+ATvDzvJJhSkUlGQV9wgObdI= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.36.3 h1:hID7cr8t3Wp26+cYnfcjR6HpJ00fdogN6dqZ1t6IylU= github.com/onsi/gomega v1.36.3/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o= github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M= github.com/pion/dtls/v3 v3.1.2 h1:gqEdOUXLtCGW+afsBLO0LtDD8GnuBBjEy6HRtyofZTc= github.com/pion/dtls/v3 v3.1.2/go.mod h1:Hw/igcX4pdY69z1Hgv5x7wJFrUkdgHwAn/Q/uo7YHRo= github.com/pion/ice/v4 v4.0.10 h1:P59w1iauC/wPk9PdY8Vjl4fOFL5B+USq1+xbDcN6gT4= github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw= github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4= github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic= github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM= github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.16 h1:fk1B1dNW4hsI78XUCljZJlC4kZOPk67mNRuQ0fcEkSo= github.com/pion/rtcp v1.2.16/go.mod h1:/as7VKfYbs5NIb4h6muQ35kQF/J0ZVNz2Z3xKoCBYOo= github.com/pion/rtp v1.8.19 h1:jhdO/3XhL/aKm/wARFVmvTfq0lC/CvN1xwYKmduly3c= github.com/pion/rtp v1.8.19/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk= github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE= github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE= github.com/pion/sdp/v3 v3.0.18 h1:l0bAXazKHpepazVdp+tPYnrsy9dfh7ZbT8DxesH5ZnI= github.com/pion/sdp/v3 v3.0.18/go.mod h1:ZREGo6A9ZygQ9XkqAj5xYCQtQpif0i6Pa81HOiAdqQ8= github.com/pion/srtp/v3 v3.0.6 h1:E2gyj1f5X10sB/qILUGIkL4C2CqK269Xq167PbGCc/4= github.com/pion/srtp/v3 v3.0.6/go.mod h1:BxvziG3v/armJHAaJ87euvkhHqWe9I7iiOy50K2QkhY= github.com/pion/stun/v3 v3.1.1 h1:CkQxveJ4xGQjulGSROXbXq94TAWu8gIX2dT+ePhUkqw= github.com/pion/stun/v3 v3.1.1/go.mod h1:qC1DfmcCTQjl9PBaMa5wSn3x9IPmKxSdcCsxBcDBndM= github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0= github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= github.com/pion/transport/v4 v4.0.1 h1:sdROELU6BZ63Ab7FrOLn13M6YdJLY20wldXW2Cu2k8o= github.com/pion/transport/v4 v4.0.1/go.mod h1:nEuEA4AD5lPdcIegQDpVLgNoDGreqM/YqmEx3ovP4jM= github.com/pion/turn/v4 v4.0.2 h1:ZqgQ3+MjP32ug30xAbD6Mn+/K4Sxi3SdNOTFf+7mpps= github.com/pion/turn/v4 v4.0.2/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs= github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54= github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a h1:cgqrm0F3zwf9IPzca7xN4w+Zy6MC9ZkPvAC8QEWa/iQ= github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a/go.mod h1:ocZfO/tLSHqfScRDNTJbAJR1by4D1lewauX9OwTaPuY= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/probe-lab/go-libdht v0.4.0 h1:LAqHuko/owRW6+0cs5wmJXbHzg09EUMJEh5DI37yXqo= github.com/probe-lab/go-libdht v0.4.0/go.mod h1:hamw22kI6YkPQFGy5P6BrWWDrgE9ety5Si8iWAyuDvc= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= github.com/prometheus/statsd_exporter v0.27.1 h1:tcRJOmwlA83HPfWzosAgr2+zEN5XDFv+M2mn/uYkn5Y= github.com/prometheus/statsd_exporter v0.27.1/go.mod h1:vA6ryDfsN7py/3JApEst6nLTJboq66XsNcJGNmC88NQ= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/quic-go/webtransport-go v0.10.0 h1:LqXXPOXuETY5Xe8ITdGisBzTYmUOy5eSj+9n4hLTjHI= github.com/quic-go/webtransport-go v0.10.0/go.mod h1:LeGIXr5BQKE3UsynwVBeQrU1TPrbh73MGoC6jd+V7ow= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/slok/go-http-metrics v0.13.0 h1:lQDyJJx9wKhmbliyUsZ2l6peGnXRHjsjoqPt5VYzcP8= github.com/slok/go-http-metrics v0.13.0/go.mod h1:HIr7t/HbN2sJaunvnt9wKP9xoBBVZFo1/KiHU3b0w+4= github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/texttheater/golang-levenshtein v1.0.1 h1:+cRNoVrfiwufQPhoMzB6N0Yf/Mqajr6t1lOv8GyGE2U= github.com/texttheater/golang-levenshtein v1.0.1/go.mod h1:PYAKrbF5sAiq9wd+H82hs7gNaen0CplQ9uvm6+enD/8= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.3.1 h1:82ioxmhEYut7LBVGhGq8xoRkXPLElVuh5mV67AFfdv0= github.com/whyrusleeping/cbor-gen v0.3.1/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1 h1:ctS9Anw/KozviCCtK6VWMz5kPL9nbQzbQY4yfqlIV4M= github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1/go.mod h1:tKH72zYNt/exx6/5IQO6L9LoQ0rEjd5SbbWaDTs9Zso= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI= github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= go.opentelemetry.io/contrib/propagators/autoprop v0.46.1 h1:cXTYcMjY0dsYokAuo8LbNBQxpF8VgTHdiHJJ1zlIXl4= go.opentelemetry.io/contrib/propagators/autoprop v0.46.1/go.mod h1:WZxgny1/6+j67B1s72PLJ4bGjidoWFzSmLNfJKVt2bo= go.opentelemetry.io/contrib/propagators/aws v1.21.1 h1:uQIQIDWb0gzyvon2ICnghpLAf9w7ADOCUiIiwCQgR2o= go.opentelemetry.io/contrib/propagators/aws v1.21.1/go.mod h1:kCcto3ACQxm+VrkQX/NK/TkDmAd99MQhvffzyTKhzL4= go.opentelemetry.io/contrib/propagators/b3 v1.21.1 h1:WPYiUgmw3+b7b3sQ1bFBFAf0q+Di9dvNc3AtYfnT4RQ= go.opentelemetry.io/contrib/propagators/b3 v1.21.1/go.mod h1:EmzokPoSqsYMBVK4nRnhsfm5mbn8J1eDuz/U1UaQaWg= go.opentelemetry.io/contrib/propagators/jaeger v1.21.1 h1:f4beMGDKiVzg9IcX7/VuWVy+oGdjx3dNJ72YehmtY5k= go.opentelemetry.io/contrib/propagators/jaeger v1.21.1/go.mod h1:U9jhkEl8d1LL+QXY7q3kneJWJugiN3kZJV2OWz3hkBY= go.opentelemetry.io/contrib/propagators/ot v1.21.1 h1:3TN5vkXjKYWp0YdMcnUEC/A+pBPvqz9V3nCS2xmcurk= go.opentelemetry.io/contrib/propagators/ot v1.21.1/go.mod h1:oy0MYCbS/b3cqUDW37wBWtlwBIsutngS++Lklpgh+fc= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 h1:wVZXIWjQSeSmMoxF74LzAnpVQOAFDo3pPji9Y4SOFKc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0/go.mod h1:khvBS2IggMFNwZK/6lEeHg/W57h/IX6J4URh57fuI40= go.opentelemetry.io/otel/exporters/prometheus v0.56.0 h1:GnCIi0QyG0yy2MrJLzVrIM7laaJstj//flf1zEJCG+E= go.opentelemetry.io/otel/exporters/prometheus v0.56.0/go.mod h1:JQcVZtbIIPM+7SWBB+T6FK+xunlyidwLp++fN0sUaOk= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.40.0 h1:MzfofMZN8ulNqobCmCAVbqVL5syHw+eB2qPRkCMA/fQ= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.40.0/go.mod h1:E73G9UFtKRXrxhBsHtG00TB5WxX57lpsQzogDkqBTz8= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 h1:jiDhWWeC7jfWqR9c/uplMOqJ0sbNlNWv0UkzE0vX1MA= golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90/go.mod h1:xE1HEv6b+1SCZ5/uscMRjUBKtIxworgEcEi+/n9NQDQ= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c h1:6a8FdnNk6bTXBjR4AGKFgUKuo+7GnR3FX5L7CbveeZc= golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw= pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= ================================================ FILE: misc/README.md ================================================ ## init system integration go-ipfs can be started by your operating system's native init system. - [systemd](#systemd) - [LSB init script](#initd) - [Upstart/startup job](#upstart) - [launchd](#launchd) ### systemd For `systemd`, the best approach is to run the daemon in a user session. Here is a sample service file: ```systemd [Unit] Description=IPFS daemon [Service] # Environment="IPFS_PATH=/data/ipfs" # optional path to ipfs init directory if not default ($HOME/.ipfs) ExecStart=/usr/local/bin/ipfs daemon Restart=on-failure [Install] WantedBy=default.target ``` To run this in your user session, save it as `~/.config/systemd/user/ipfs.service` (creating directories as necessary). Once you run `ipfs init` to create your IPFS settings, you can control the daemon using the following commands: * `systemctl --user start ipfs` - start the daemon * `systemctl --user stop ipfs` - stop the daemon * `systemctl --user status ipfs` - get status of the daemon * `systemctl --user enable ipfs` - enable starting the daemon at boot * `systemctl --user disable ipfs` - disable starting the daemon at boot *Note:* If you want this `--user` service to run at system boot, you must [`enable-linger`](http://www.freedesktop.org/software/systemd/man/loginctl.html) on the account that runs the service: ``` # loginctl enable-linger [user] ``` Read more about `--user` services here: [wiki.archlinux.org:Systemd ](https://wiki.archlinux.org/index.php/Systemd/User#Automatic_start-up_of_systemd_user_instances) #### P2P tunnel services For running `ipfs p2p listen` or `ipfs p2p forward` as systemd services, see [docs/p2p-tunnels.md](../docs/p2p-tunnels.md) for examples using the `--foreground` flag and path-based activation. ### initd - Here is a full-featured sample service file: https://github.com/dylanPowers/ipfs-linux-service/blob/master/init.d/ipfs - Use `service` or your distribution's equivalent to control the service. ## upstart - And below is a very basic sample upstart job. **Note the username jbenet**. ``` cat /etc/init/ipfs.conf ``` ``` description "ipfs: interplanetary filesystem" start on (local-filesystems and net-device-up IFACE!=lo) stop on runlevel [!2345] limit nofile 524288 1048576 limit nproc 524288 1048576 setuid jbenet chdir /home/jbenet respawn exec ipfs daemon ``` Another version is available here: ```sh ipfs cat /ipfs/QmbYCwVeA23vz6mzAiVQhJNa2JSiRH4ebef1v2e5EkDEZS/ipfs.conf >/etc/init/ipfs.conf ``` For both, edit to replace occurrences of `jbenet` with whatever user you want it to run as: ```sh sed -i s/jbenet// /etc/init/ipfs.conf ``` Once you run `ipfs init` to create your IPFS settings, you can control the daemon using the `init.d` commands: ```sh sudo service ipfs start sudo service ipfs stop sudo service ipfs restart ... ``` ## launchd Similar to `systemd`, on macOS you can run `go-ipfs` via a user LaunchAgent. - Create `~/Library/LaunchAgents/io.ipfs.go-ipfs.plist`: ```xml KeepAlive Label io.ipfs.go-ipfs ProcessType Background ProgramArguments /bin/sh -c ~/go/bin/ipfs daemon RunAtLoad ``` The reason for running `ipfs` under a shell is to avoid needing to hard-code the user's home directory in the job. - To start the job, run `launchctl load ~/Library/LaunchAgents/io.ipfs.go-ipfs.plist` Notes: - To check that the job is running, run `launchctl list | grep ipfs`. - IPFS should now start whenever you log in (and exit when you log out). - [LaunchControl](http://www.soma-zone.com/LaunchControl/) is a GUI tool which simplifies management of LaunchAgents. ================================================ FILE: misc/fsutil/fsutil.go ================================================ package fsutil import ( "errors" "fmt" "io/fs" "os" "path/filepath" ) // DirWritable checks if a directory is writable. If the directory does // not exist it is created with writable permission. func DirWritable(dir string) error { if dir == "" { return errors.New("directory not specified") } var err error dir, err = ExpandHome(dir) if err != nil { return err } fi, err := os.Stat(dir) if err != nil { if errors.Is(err, fs.ErrNotExist) { // Directory does not exist, so create it. err = os.Mkdir(dir, 0775) if err == nil { return nil } } if errors.Is(err, fs.ErrPermission) { err = fs.ErrPermission } return fmt.Errorf("directory not writable: %s: %w", dir, err) } if !fi.IsDir() { return fmt.Errorf("not a directory: %s", dir) } // Directory exists, check that a file can be written. file, err := os.CreateTemp(dir, "writetest") if err != nil { if errors.Is(err, fs.ErrPermission) { err = fs.ErrPermission } return fmt.Errorf("directory not writable: %s: %w", dir, err) } file.Close() return os.Remove(file.Name()) } // ExpandHome expands the path to include the home directory if the path is // prefixed with `~`. If it isn't prefixed with `~`, the path is returned // as-is. func ExpandHome(path string) (string, error) { if path == "" { return path, nil } if path[0] != '~' { return path, nil } if len(path) > 1 && path[1] != '/' && path[1] != '\\' { return "", errors.New("cannot expand user-specific home dir") } dir, err := os.UserHomeDir() if err != nil { return "", err } return filepath.Join(dir, path[1:]), nil } // FileExists return true if the file exists func FileExists(filename string) bool { _, err := os.Lstat(filename) return !errors.Is(err, os.ErrNotExist) } ================================================ FILE: misc/fsutil/fsutil_test.go ================================================ package fsutil_test import ( "io/fs" "os" "path/filepath" "runtime" "testing" "github.com/ipfs/kubo/misc/fsutil" "github.com/stretchr/testify/require" ) func TestDirWritable(t *testing.T) { err := fsutil.DirWritable("") require.Error(t, err) err = fsutil.DirWritable("~nosuchuser/tmp") require.Error(t, err) tmpDir := t.TempDir() wrDir := filepath.Join(tmpDir, "readwrite") err = fsutil.DirWritable(wrDir) require.NoError(t, err) // Check that DirWritable created directory. fi, err := os.Stat(wrDir) require.NoError(t, err) require.True(t, fi.IsDir()) err = fsutil.DirWritable(wrDir) require.NoError(t, err) // If running on Windows, skip read-only directory tests. if runtime.GOOS == "windows" { t.SkipNow() } roDir := filepath.Join(tmpDir, "readonly") require.NoError(t, os.Mkdir(roDir, 0500)) err = fsutil.DirWritable(roDir) require.ErrorIs(t, err, fs.ErrPermission) roChild := filepath.Join(roDir, "child") err = fsutil.DirWritable(roChild) require.ErrorIs(t, err, fs.ErrPermission) } func TestFileExists(t *testing.T) { fileName := filepath.Join(t.TempDir(), "somefile") require.False(t, fsutil.FileExists(fileName)) file, err := os.Create(fileName) require.NoError(t, err) file.Close() require.True(t, fsutil.FileExists(fileName)) } func TestExpandHome(t *testing.T) { dir, err := fsutil.ExpandHome("") require.NoError(t, err) require.Equal(t, "", dir) origDir := filepath.Join("somedir", "somesub") dir, err = fsutil.ExpandHome(origDir) require.NoError(t, err) require.Equal(t, origDir, dir) _, err = fsutil.ExpandHome(filepath.FromSlash("~nosuchuser/somedir")) require.Error(t, err) homeEnv := "HOME" if runtime.GOOS == "windows" { homeEnv = "USERPROFILE" } origHome := os.Getenv(homeEnv) defer os.Setenv(homeEnv, origHome) homeDir := filepath.Join(t.TempDir(), "testhome") os.Setenv(homeEnv, homeDir) const subDir = "mytmp" origDir = filepath.Join("~", subDir) dir, err = fsutil.ExpandHome(origDir) require.NoError(t, err) require.Equal(t, filepath.Join(homeDir, subDir), dir) os.Unsetenv(homeEnv) _, err = fsutil.ExpandHome(origDir) require.Error(t, err) } ================================================ FILE: misc/launchd/README.md ================================================ # ipfs launchd agent A bare-bones launchd agent file for ipfs. To have launchd automatically run the ipfs daemon for you, run `./misc/launchd/install.sh` ================================================ FILE: misc/launchd/install.sh ================================================ #!/bin/bash src_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) plist=io.ipfs.ipfs-daemon.plist dest_dir="$HOME/Library/LaunchAgents" IPFS_PATH="${IPFS_PATH:-$HOME/.ipfs}" escaped_ipfs_path=$(echo $IPFS_PATH|sed 's/\//\\\//g') IPFS_BIN=$(which ipfs || echo ipfs) escaped_ipfs_bin=$(echo $IPFS_BIN|sed 's/\//\\\//g') mkdir -p "$dest_dir" sed -e 's/{{IPFS_PATH}}/'"$escaped_ipfs_path"'/g' \ -e 's/{{IPFS_BIN}}/'"$escaped_ipfs_bin"'/g' \ "$src_dir/$plist" \ > "$dest_dir/$plist" launchctl list | grep ipfs-daemon >/dev/null if [ $? ]; then echo Unloading existing ipfs-daemon launchctl unload "$dest_dir/$plist" fi echo Loading ipfs-daemon if (( `sw_vers -productVersion | cut -d'.' -f2` > 9 )); then sudo chown root "$dest_dir/$plist" sudo launchctl bootstrap system "$dest_dir/$plist" else launchctl load "$dest_dir/$plist" fi launchctl list | grep ipfs-daemon ================================================ FILE: misc/launchd/io.ipfs.ipfs-daemon.plist ================================================ KeepAlive Label io.ipfs.ipfs-daemon ProgramArguments {{IPFS_BIN}} daemon EnvironmentVariables IPFS_PATH {{IPFS_PATH}} RunAtLoad ================================================ FILE: misc/systemd/ipfs-api.socket ================================================ # Enabling this will *completely override* any API listeners configured in your # config. [Unit] Description=IPFS API Socket [Socket] Service=ipfs.service FileDescriptorName=io.ipfs.api BindIPv6Only=true ListenStream=127.0.0.1:5001 ListenStream=[::1]:5001 [Install] WantedBy=sockets.target ================================================ FILE: misc/systemd/ipfs-gateway.socket ================================================ # Enabling this will *completely override* any Gateway listeners configured in # your config. [Unit] Description=IPFS Gateway Socket [Socket] Service=ipfs.service FileDescriptorName=io.ipfs.gateway BindIPv6Only=true ListenStream=127.0.0.1:8080 ListenStream=[::1]:8080 [Install] WantedBy=sockets.target ================================================ FILE: misc/systemd/ipfs-hardened.service ================================================ # This file will be overwritten on package upgrades, avoid customizations here. # # To make persistent changes, create file in # "/etc/systemd/system/ipfs.service.d/overwrite.conf" with # `systemctl edit ipfs.service`. This file will be parsed after this # file has been parsed. # # To overwrite a variable, like ExecStart you have to specify it once # blank and a second time with a new value, like: # ExecStart= # ExecStart=/usr/local/bin/ipfs daemon --flag1 --flag2 # # For more info about custom unit files see systemd.unit(5). # This service file enables systemd-hardening features compatible with IPFS, # while breaking compatibility with the fuse-mount function. Use this one only # if you don't need the fuse-mount functionality. [Unit] Description=InterPlanetary File System (IPFS) daemon Documentation=https://docs.ipfs.tech/ After=network.target [Service] # hardening ReadWritePaths="/var/lib/ipfs/" NoNewPrivileges=true ProtectSystem=strict ProtectKernelTunables=true ProtectKernelModules=true ProtectKernelLogs=true PrivateDevices=true DevicePolicy=closed ProtectControlGroups=true RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK ProtectHostname=true PrivateTmp=true ProtectClock=true LockPersonality=true RestrictNamespaces=true RestrictRealtime=true MemoryDenyWriteExecute=true SystemCallArchitectures=native SystemCallFilter=@system-service SystemCallFilter=~@privileged ProtectHome=true RemoveIPC=true RestrictSUIDSGID=true CapabilityBoundingSet=CAP_NET_BIND_SERVICE # enable for 1-1024 port listening #AmbientCapabilities=CAP_NET_BIND_SERVICE # enable to specify a custom path see docs/environment-variables.md for further documentations #Environment=IPFS_PATH=/custom/ipfs/path # enable to specify a higher limit for open files/connections #LimitNOFILE=1000000 #don't use swap MemorySwapMax=0 # Don't timeout on startup. Opening the IPFS repo can take a long time in some cases (e.g., when # badger is recovering) and migrations can delay startup. # # Ideally, we'd be a bit smarter about this but there's no good way to do that without hooking # systemd dependencies deeper into go-ipfs. TimeoutStartSec=infinity Type=notify User=ipfs Group=ipfs StateDirectory=ipfs Environment=IPFS_PATH="${HOME}" ExecStart=/usr/local/bin/ipfs daemon --init --migrate Restart=on-failure KillSignal=SIGINT [Install] WantedBy=default.target ================================================ FILE: misc/systemd/ipfs-sysusers.conf ================================================ u ipfs - "IPFS daemon" /var/lib/ipfs g ipfs - m ipfs ipfs ================================================ FILE: misc/systemd/ipfs.service ================================================ # This file will be overwritten on package upgrades, avoid customizations here. # # To make persistent changes, create file in # "/etc/systemd/system/ipfs.service.d/overwrite.conf" with # `systemctl edit ipfs.service`. This file will be parsed after this # file has been parsed. # # To overwrite a variable, like ExecStart you have to specify it once # blank and a second time with a new value, like: # ExecStart= # ExecStart=/usr/local/bin/ipfs daemon --flag1 --flag2 # # For more info about custom unit files see systemd.unit(5). [Unit] Description=InterPlanetary File System (IPFS) daemon Documentation=https://docs.ipfs.tech/ After=network.target [Service] # enable for 1-1024 port listening #AmbientCapabilities=CAP_NET_BIND_SERVICE # enable to specify a custom path see docs/environment-variables.md for further documentations #Environment=IPFS_PATH=/custom/ipfs/path # enable to specify a higher limit for open files/connections #LimitNOFILE=1000000 #don't use swap MemorySwapMax=0 # Don't timeout on startup. Opening the IPFS repo can take a long time in some cases (e.g., when # badger is recovering) and migrations can delay startup. # # Ideally, we'd be a bit smarter about this but there's no good way to do that without hooking # systemd dependencies deeper into go-ipfs. TimeoutStartSec=infinity Type=notify User=ipfs Group=ipfs StateDirectory=ipfs Environment=IPFS_PATH="${HOME}" ExecStart=/usr/local/bin/ipfs daemon --init --migrate Restart=on-failure KillSignal=SIGINT [Install] WantedBy=default.target ================================================ FILE: mk/footer.mk ================================================ # standard NR-make boilerplate, to be included at the end of a file d := $(dirstack_$(sp)) sp := $(basename $(sp)) ================================================ FILE: mk/git.mk ================================================ # First try to "describe" the state. This tells us if the state is dirty. # If that fails (e.g., we're building a docker image and have an empty objects # directory), assume the source isn't dirty and build anyways. git-hash:=$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null) # Detect if HEAD is a clean, tagged release. Used to omit redundant commit # hash from the libp2p user agent (the version number suffices). ifeq ($(findstring dirty,$(git-hash)),) git-tag:=$(shell git tag --points-at HEAD 2>/dev/null | grep '^v' | head -1) else git-tag:= endif ================================================ FILE: mk/golang.mk ================================================ # golang utilities export GO111MODULE=on # pre-definitions GOCC ?= go GOTAGS ?= GOTFLAGS ?= # Unexport GOFLAGS so we only apply it where we actually want it. unexport GOFLAGS # Override so we can combine with the user's go flags. # Try to make building as reproducible as possible by stripping the go path. override GOFLAGS += "-trimpath" ifeq ($(tarball-is),1) GOFLAGS += -mod=vendor endif # match Go's default GOPATH behaviour export GOPATH ?= $(shell $(GOCC) env GOPATH) DEPS_GO := TEST_GO := TEST_GO_BUILD := CHECK_GO := go-pkg-name=$(shell GOFLAGS=-buildvcs=false $(GOCC) list $(go-tags) github.com/ipfs/kubo/$(1)) go-main-name=$(notdir $(call go-pkg-name,$(1)))$(?exe) go-curr-pkg-tgt=$(d)/$(call go-main-name,$(d)) go-pkgs=$(shell GOFLAGS=-buildvcs=false $(GOCC) list github.com/ipfs/kubo/...) go-tags=$(if $(GOTAGS), -tags="$(call join-with,$(space),$(GOTAGS))") go-flags-with-tags=$(GOFLAGS)$(go-tags) define go-build-relative $(GOCC) build $(go-flags-with-tags) -o "$@" "$(call go-pkg-name,$<)" endef define go-build $(GOCC) build $(go-flags-with-tags) -o "$@" "$(1)" endef # Only disable colors when running in CI (non-interactive terminal) GOTESTSUM_NOCOLOR := $(if $(CI),--no-color,) # Packages excluded from coverage (test code and examples are not production code) COVERPKG_EXCLUDE := /(test|docs/examples)/ # Packages excluded from unit tests: coverage exclusions + client/rpc (tested by test_cli) UNIT_EXCLUDE := /(test|docs/examples)/|/client/rpc$$ # Unit tests with coverage # Produces JSON for CI reporting and coverage profile for Codecov test_unit: test/bin/gotestsum $$(DEPS_GO) mkdir -p test/unit coverage rm -f test/unit/gotest.json coverage/unit_tests.coverprofile gotestsum $(GOTESTSUM_NOCOLOR) --jsonfile test/unit/gotest.json -- $(go-flags-with-tags) $(GOTFLAGS) -covermode=atomic -coverprofile=coverage/unit_tests.coverprofile -coverpkg=$$($(GOCC) list $(go-tags) ./... | grep -vE '$(COVERPKG_EXCLUDE)' | tr '\n' ',' | sed 's/,$$//') $$($(GOCC) list $(go-tags) ./... | grep -vE '$(UNIT_EXCLUDE)') .PHONY: test_unit # CLI/integration tests (requires built binary in PATH) # Includes test/cli, test/integration, and client/rpc # Produces JSON for CI reporting # Override TEST_CLI_TIMEOUT for local development: make test_cli TEST_CLI_TIMEOUT=5m TEST_CLI_TIMEOUT ?= 10m test_cli: cmd/ipfs/ipfs test/bin/gotestsum $$(DEPS_GO) mkdir -p test/cli rm -f test/cli/cli-tests.json PATH="$(CURDIR)/cmd/ipfs:$(CURDIR)/test/bin:$$PATH" gotestsum $(GOTESTSUM_NOCOLOR) --jsonfile test/cli/cli-tests.json -- -v -timeout=$(TEST_CLI_TIMEOUT) ./test/cli/... ./test/integration/... ./client/rpc/... .PHONY: test_cli # Example tests (docs/examples/kubo-as-a-library) # Tests against both published and current kubo versions # Uses timeout to ensure CI gets output before job-level timeout kills everything TEST_EXAMPLES_TIMEOUT ?= 2m test_examples: cd docs/examples/kubo-as-a-library && go test -v -timeout=$(TEST_EXAMPLES_TIMEOUT) ./... && cp go.mod go.mod.bak && cp go.sum go.sum.bak && (go mod edit -replace github.com/ipfs/kubo=./../../.. && go mod tidy && go test -v -timeout=$(TEST_EXAMPLES_TIMEOUT) ./...; ret=$$?; mv go.mod.bak go.mod; mv go.sum.bak go.sum; exit $$ret) .PHONY: test_examples # Build kubo for all platforms from .github/build-platforms.yml test_go_build: bin/test-go-build-platforms .PHONY: test_go_build # Check Go source formatting test_go_fmt: bin/test-go-fmt .PHONY: test_go_fmt # Run golangci-lint (used by CI) test_go_lint: test/bin/golangci-lint golangci-lint run --timeout=3m ./... .PHONY: test_go_lint TEST_GO := test_go_fmt test_unit test_cli test_examples TEST += $(TEST_GO) TEST_SHORT += test_go_fmt test_unit ================================================ FILE: mk/header.mk ================================================ # keep track of dirs # standard NR-make boilerplate, to be included at the beginning of a file p := $(sp).x dirstack_$(sp) := $(d) d := $(dir) ================================================ FILE: mk/tarball.mk ================================================ ifeq (,$(wildcard .tarball)) tarball-is:=0 else tarball-is:=1 # override git hash git-hash:=$(shell cat .tarball) endif GOCC ?= go go-ipfs-source.tar.gz: distclean GOCC=$(GOCC) bin/maketarball.sh $@ kubo-source.tar.gz: distclean GOCC=$(GOCC) bin/maketarball.sh $@ ================================================ FILE: mk/util.mk ================================================ # util functions OS ?= $(shell sh -c 'uname -s 2>/dev/null || echo not') ifeq ($(OS),Windows_NT) WINDOWS :=1 ?exe :=.exe # windows compat PATH_SEP :=; else ?exe := PATH_SEP :=: endif # Platforms are now defined in .github/build-platforms.yml # The cmd/ipfs-try-build target is deprecated in favor of GitHub Actions # Use 'make supported' to see the list of platforms space:=$() $() comma:=, join-with=$(subst $(space),$1,$(strip $2)) # debug target, prints variable. Example: `make print-GOFLAGS` print-%: @echo $*=$($*) # phony target that will mean that recipe is always executed ALWAYS: .PHONY: ALWAYS ================================================ FILE: p2p/listener.go ================================================ package p2p import ( "errors" "sync" p2phost "github.com/libp2p/go-libp2p/core/host" net "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/protocol" ma "github.com/multiformats/go-multiaddr" ) // Listener listens for connections and proxies them to a target. type Listener interface { Protocol() protocol.ID ListenAddress() ma.Multiaddr TargetAddress() ma.Multiaddr key() protocol.ID // close closes the listener. Does not affect child streams close() // Done returns a channel that is closed when the listener is closed. // This allows callers to detect when a listener has been removed. Done() <-chan struct{} } // Listeners manages a group of Listener implementations, // checking for conflicts and optionally dispatching connections. type Listeners struct { sync.RWMutex Listeners map[protocol.ID]Listener } func newListenersLocal() *Listeners { return &Listeners{ Listeners: map[protocol.ID]Listener{}, } } func newListenersP2P(host p2phost.Host) *Listeners { reg := &Listeners{ Listeners: map[protocol.ID]Listener{}, } host.SetStreamHandlerMatch("/x/", func(p protocol.ID) bool { reg.RLock() defer reg.RUnlock() _, ok := reg.Listeners[p] return ok }, func(stream net.Stream) { reg.RLock() defer reg.RUnlock() l := reg.Listeners[stream.Protocol()] if l != nil { go l.(*remoteListener).handleStream(stream) } }) return reg } // Register registers listenerInfo into this registry and starts it. func (r *Listeners) Register(l Listener) error { r.Lock() defer r.Unlock() if _, ok := r.Listeners[l.key()]; ok { return errors.New("listener already registered") } r.Listeners[l.key()] = l return nil } // Close removes and closes all listeners for which matchFunc returns true. // Returns the number of listeners closed. func (r *Listeners) Close(matchFunc func(listener Listener) bool) int { var todo []Listener r.Lock() for _, l := range r.Listeners { if matchFunc(l) { delete(r.Listeners, l.key()) todo = append(todo, l) } } r.Unlock() for _, l := range todo { l.close() } return len(todo) } ================================================ FILE: p2p/local.go ================================================ package p2p import ( "context" "time" tec "github.com/jbenet/go-temp-err-catcher" net "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) // localListener manet streams and proxies them to libp2p services. type localListener struct { ctx context.Context p2p *P2P proto protocol.ID laddr ma.Multiaddr peer peer.ID listener manet.Listener done chan struct{} } // ForwardLocal creates new P2P stream to a remote listener. func (p2p *P2P) ForwardLocal(ctx context.Context, peer peer.ID, proto protocol.ID, bindAddr ma.Multiaddr) (Listener, error) { listener := &localListener{ ctx: ctx, p2p: p2p, proto: proto, peer: peer, done: make(chan struct{}), } maListener, err := manet.Listen(bindAddr) if err != nil { return nil, err } listener.listener = maListener listener.laddr = maListener.Multiaddr() if err := p2p.ListenersLocal.Register(listener); err != nil { return nil, err } go listener.acceptConns() return listener, nil } func (l *localListener) dial(ctx context.Context) (net.Stream, error) { cctx, cancel := context.WithTimeout(ctx, time.Second*30) // TODO: configurable? defer cancel() return l.p2p.peerHost.NewStream(cctx, l.peer, l.proto) } func (l *localListener) acceptConns() { for { local, err := l.listener.Accept() if err != nil { if tec.ErrIsTemporary(err) { continue } return } go l.setupStream(local) } } func (l *localListener) setupStream(local manet.Conn) { remote, err := l.dial(l.ctx) if err != nil { local.Close() log.Warnf("failed to dial to remote %s/%s", l.peer, l.proto) return } stream := &Stream{ Protocol: l.proto, OriginAddr: local.RemoteMultiaddr(), TargetAddr: l.TargetAddress(), peer: l.peer, Local: local, Remote: remote, Registry: l.p2p.Streams, } l.p2p.Streams.Register(stream) } func (l *localListener) close() { l.listener.Close() close(l.done) } func (l *localListener) Done() <-chan struct{} { return l.done } func (l *localListener) Protocol() protocol.ID { return l.proto } func (l *localListener) ListenAddress() ma.Multiaddr { return l.laddr } func (l *localListener) TargetAddress() ma.Multiaddr { addr, err := ma.NewMultiaddr(maPrefix + l.peer.String()) if err != nil { panic(err) } return addr } func (l *localListener) key() protocol.ID { return protocol.ID(l.ListenAddress().String()) } ================================================ FILE: p2p/p2p.go ================================================ package p2p import ( logging "github.com/ipfs/go-log/v2" p2phost "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" pstore "github.com/libp2p/go-libp2p/core/peerstore" "github.com/libp2p/go-libp2p/core/protocol" ) var log = logging.Logger("p2p-mount") // P2P structure holds information on currently running streams/Listeners. type P2P struct { ListenersLocal *Listeners ListenersP2P *Listeners Streams *StreamRegistry identity peer.ID peerHost p2phost.Host peerstore pstore.Peerstore } // New creates new P2P struct. func New(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) *P2P { return &P2P{ identity: identity, peerHost: peerHost, peerstore: peerstore, ListenersLocal: newListenersLocal(), ListenersP2P: newListenersP2P(peerHost), Streams: &StreamRegistry{ Streams: map[uint64]*Stream{}, ConnManager: peerHost.ConnManager(), conns: map[peer.ID]int{}, }, } } // CheckProtoExists checks whether a proto handler is registered to // mux handler. func (p2p *P2P) CheckProtoExists(proto protocol.ID) bool { protos := p2p.peerHost.Mux().Protocols() for _, p := range protos { if p != proto { continue } return true } return false } ================================================ FILE: p2p/remote.go ================================================ package p2p import ( "context" "fmt" net "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/protocol" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) var maPrefix = "/" + ma.ProtocolWithCode(ma.P_IPFS).Name + "/" // remoteListener accepts libp2p streams and proxies them to a manet host. type remoteListener struct { p2p *P2P // Application proto identifier. proto protocol.ID // Address to proxy the incoming connections to addr ma.Multiaddr // reportRemote if set to true makes the handler send '\n' // to target before any data is forwarded reportRemote bool done chan struct{} } // ForwardRemote creates new p2p listener. func (p2p *P2P) ForwardRemote(ctx context.Context, proto protocol.ID, addr ma.Multiaddr, reportRemote bool) (Listener, error) { listener := &remoteListener{ p2p: p2p, proto: proto, addr: addr, reportRemote: reportRemote, done: make(chan struct{}), } if err := p2p.ListenersP2P.Register(listener); err != nil { return nil, err } return listener, nil } func (l *remoteListener) handleStream(remote net.Stream) { local, err := manet.Dial(l.addr) if err != nil { _ = remote.Reset() return } peer := remote.Conn().RemotePeer() if l.reportRemote { if _, err := fmt.Fprintf(local, "%s\n", peer); err != nil { _ = remote.Reset() return } } peerMa, err := ma.NewMultiaddr(maPrefix + peer.String()) if err != nil { _ = remote.Reset() return } stream := &Stream{ Protocol: l.proto, OriginAddr: peerMa, TargetAddr: l.addr, peer: peer, Local: local, Remote: remote, Registry: l.p2p.Streams, } l.p2p.Streams.Register(stream) } func (l *remoteListener) Protocol() protocol.ID { return l.proto } func (l *remoteListener) ListenAddress() ma.Multiaddr { addr, err := ma.NewMultiaddr(maPrefix + l.p2p.identity.String()) if err != nil { panic(err) } return addr } func (l *remoteListener) TargetAddress() ma.Multiaddr { return l.addr } func (l *remoteListener) close() { close(l.done) } func (l *remoteListener) Done() <-chan struct{} { return l.done } func (l *remoteListener) key() protocol.ID { return l.proto } ================================================ FILE: p2p/stream.go ================================================ package p2p import ( "io" "sync" ifconnmgr "github.com/libp2p/go-libp2p/core/connmgr" net "github.com/libp2p/go-libp2p/core/network" peer "github.com/libp2p/go-libp2p/core/peer" protocol "github.com/libp2p/go-libp2p/core/protocol" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) const cmgrTag = "stream-fwd" // Stream holds information on active incoming and outgoing p2p streams. type Stream struct { id uint64 Protocol protocol.ID OriginAddr ma.Multiaddr TargetAddr ma.Multiaddr peer peer.ID Local manet.Conn Remote net.Stream Registry *StreamRegistry } // close stream endpoints and deregister it. func (s *Stream) close() { s.Registry.Close(s) } // reset closes stream endpoints and deregisters it. func (s *Stream) reset() { s.Registry.Reset(s) } func (s *Stream) startStreaming() { go func() { _, err := io.Copy(s.Local, s.Remote) if err != nil { s.reset() } else { s.close() } }() go func() { _, err := io.Copy(s.Remote, s.Local) if err != nil { s.reset() } else { s.close() } }() } // StreamRegistry is a collection of active incoming and outgoing proto app streams. type StreamRegistry struct { sync.Mutex Streams map[uint64]*Stream conns map[peer.ID]int nextID uint64 ifconnmgr.ConnManager } // Register registers a stream to the registry. func (r *StreamRegistry) Register(streamInfo *Stream) { r.Lock() defer r.Unlock() r.ConnManager.TagPeer(streamInfo.peer, cmgrTag, 20) r.conns[streamInfo.peer]++ streamInfo.id = r.nextID r.Streams[r.nextID] = streamInfo r.nextID++ streamInfo.startStreaming() } // Deregister deregisters stream from the registry. func (r *StreamRegistry) Deregister(streamID uint64) { r.Lock() defer r.Unlock() s, ok := r.Streams[streamID] if !ok { return } p := s.peer r.conns[p]-- if r.conns[p] < 1 { delete(r.conns, p) r.ConnManager.UntagPeer(p, cmgrTag) } delete(r.Streams, streamID) } // Close stream endpoints and deregister it. func (r *StreamRegistry) Close(s *Stream) { _ = s.Local.Close() _ = s.Remote.Close() s.Registry.Deregister(s.id) } // Reset closes stream endpoints and deregisters it. func (r *StreamRegistry) Reset(s *Stream) { _ = s.Local.Close() _ = s.Remote.Reset() s.Registry.Deregister(s.id) } ================================================ FILE: plugin/Rules.mk ================================================ include mk/header.mk dir := $(d)/loader include $(dir)/Rules.mk dir := $(d)/plugins include $(dir)/Rules.mk include mk/footer.mk ================================================ FILE: plugin/daemon.go ================================================ package plugin import ( coreiface "github.com/ipfs/kubo/core/coreiface" ) // PluginDaemon is an interface for daemon plugins. These plugins will be run on // the daemon and will be given access to an implementation of the CoreAPI. type PluginDaemon interface { Plugin Start(coreiface.CoreAPI) error } ================================================ FILE: plugin/daemoninternal.go ================================================ package plugin import "github.com/ipfs/kubo/core" // PluginDaemonInternal is an interface for daemon plugins. These plugins will be run on // the daemon and will be given a direct access to the IpfsNode. // // Note: PluginDaemonInternal is considered internal and no guarantee is made concerning // the stability of its API. If you can, use PluginAPI instead. type PluginDaemonInternal interface { Plugin Start(*core.IpfsNode) error } ================================================ FILE: plugin/datastore.go ================================================ package plugin import ( "github.com/ipfs/kubo/repo/fsrepo" ) // PluginDatastore is an interface that can be implemented to add handlers for // for different datastores. type PluginDatastore interface { Plugin DatastoreTypeName() string DatastoreConfigParser() fsrepo.ConfigFromMap } ================================================ FILE: plugin/fx.go ================================================ package plugin import ( "github.com/ipfs/kubo/core" "go.uber.org/fx" ) // PluginFx can be used to customize the fx options passed to the go-ipfs app when it is initialized. // // This is invasive and depends on internal details such as the structure of the dependency graph, // so breaking changes might occur between releases. // So it's recommended to keep this as simple as possible, and stick to overriding interfaces // with fx.Replace() or fx.Decorate(). // // The returned options become the complete array of options passed to fx. // Generally you'll want to append additional options to NodeInfo.FXOptions and return that. type PluginFx interface { Plugin Options(core.FXNodeInfo) ([]fx.Option, error) } ================================================ FILE: plugin/ipld.go ================================================ package plugin import ( multicodec "github.com/ipld/go-ipld-prime/multicodec" ) // PluginIPLD is an interface that can be implemented to add handlers for // for different IPLD codecs. type PluginIPLD interface { Plugin Register(multicodec.Registry) error } ================================================ FILE: plugin/loader/Rules.mk ================================================ include mk/header.mk IPFS_PLUGINS ?= export IPFS_PLUGINS $(d)/preload.go: d:=$(d) $(d)/preload.go: $(d)/preload_list $(d)/preload.sh ALWAYS $(d)/preload.sh > $@ go fmt $@ >/dev/null DEPS_GO += $(d)/preload.go include mk/footer.mk ================================================ FILE: plugin/loader/load_nocgo.go ================================================ //go:build !cgo && !noplugin && (linux || darwin || freebsd) package loader import ( "errors" iplugin "github.com/ipfs/kubo/plugin" ) func init() { loadPluginFunc = nocgoLoadPlugin } func nocgoLoadPlugin(fi string) ([]iplugin.Plugin, error) { return nil, errors.New("not built with cgo support") } ================================================ FILE: plugin/loader/load_noplugin.go ================================================ //go:build noplugin package loader import ( "errors" iplugin "github.com/ipfs/kubo/plugin" ) func init() { loadPluginFunc = nopluginLoadPlugin } func nopluginLoadPlugin(string) ([]iplugin.Plugin, error) { return nil, errors.New("not built with plugin support") } ================================================ FILE: plugin/loader/load_unix.go ================================================ //go:build cgo && !noplugin && (linux || darwin || freebsd) package loader import ( "errors" "plugin" iplugin "github.com/ipfs/kubo/plugin" ) func init() { loadPluginFunc = unixLoadPlugin } func unixLoadPlugin(fi string) ([]iplugin.Plugin, error) { pl, err := plugin.Open(fi) if err != nil { return nil, err } pls, err := pl.Lookup("Plugins") if err != nil { return nil, err } typePls, ok := pls.(*[]iplugin.Plugin) if !ok { return nil, errors.New("filed 'Plugins' didn't contain correct type") } return *typePls, nil } ================================================ FILE: plugin/loader/loader.go ================================================ package loader import ( "encoding/json" "errors" "fmt" "io" "os" "path/filepath" "runtime" "strings" config "github.com/ipfs/kubo/config" "github.com/ipld/go-ipld-prime/multicodec" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/coreapi" plugin "github.com/ipfs/kubo/plugin" fsrepo "github.com/ipfs/kubo/repo/fsrepo" logging "github.com/ipfs/go-log/v2" opentracing "github.com/opentracing/opentracing-go" ) var preloadPlugins []plugin.Plugin // Preload adds one or more plugins to the preload list. This should _only_ be called during init. func Preload(plugins ...plugin.Plugin) { preloadPlugins = append(preloadPlugins, plugins...) } var log = logging.Logger("plugin/loader") var loadPluginFunc = func(string) ([]plugin.Plugin, error) { return nil, fmt.Errorf("unsupported platform %s", runtime.GOOS) } type loaderState int const ( loaderLoading loaderState = iota loaderInitializing loaderInitialized loaderInjecting loaderInjected loaderStarting loaderStarted loaderClosing loaderClosed loaderFailed ) func (ls loaderState) String() string { switch ls { case loaderLoading: return "Loading" case loaderInitializing: return "Initializing" case loaderInitialized: return "Initialized" case loaderInjecting: return "Injecting" case loaderInjected: return "Injected" case loaderStarting: return "Starting" case loaderStarted: return "Started" case loaderClosing: return "Closing" case loaderClosed: return "Closed" case loaderFailed: return "Failed" default: return "Unknown" } } // PluginLoader keeps track of loaded plugins. // // To use: // 1. Load any desired plugins with Load and LoadDirectory. Preloaded plugins // will automatically be loaded. // 2. Call Initialize to run all initialization logic. // 3. Call Inject to register the plugins. // 4. Optionally call Start to start plugins. // 5. Call Close to close all plugins. type PluginLoader struct { state loaderState plugins []plugin.Plugin started []plugin.Plugin config config.Plugins repo string } // NewPluginLoader creates new plugin loader. func NewPluginLoader(repo string) (*PluginLoader, error) { loader := &PluginLoader{plugins: make([]plugin.Plugin, 0, len(preloadPlugins)), repo: repo} if repo != "" { switch plugins, err := readPluginsConfig(repo, config.DefaultConfigFile); { case err == nil: loader.config = plugins case os.IsNotExist(err): default: return nil, err } } for _, v := range preloadPlugins { if err := loader.Load(v); err != nil { return nil, err } } if err := loader.LoadDirectory(filepath.Join(repo, "plugins")); err != nil { return nil, err } return loader, nil } // readPluginsConfig reads the Plugins section of the IPFS config, avoiding // reading anything other than the Plugin section. That way, we're free to // make arbitrary changes to all _other_ sections in migrations. func readPluginsConfig(repoRoot string, userConfigFile string) (config.Plugins, error) { var cfg struct { Plugins config.Plugins } cfgPath, err := config.Filename(repoRoot, userConfigFile) if err != nil { return config.Plugins{}, err } cfgFile, err := os.Open(cfgPath) if err != nil { return config.Plugins{}, err } defer cfgFile.Close() err = json.NewDecoder(cfgFile).Decode(&cfg) if err != nil { return config.Plugins{}, err } return cfg.Plugins, nil } func (loader *PluginLoader) assertState(state loaderState) error { if loader.state != state { return fmt.Errorf("loader state must be %s, was %s", state, loader.state) } return nil } func (loader *PluginLoader) transition(from, to loaderState) error { if err := loader.assertState(from); err != nil { return err } loader.state = to return nil } // Load loads a plugin into the plugin loader. func (loader *PluginLoader) Load(pl plugin.Plugin) error { if err := loader.assertState(loaderLoading); err != nil { return err } name := pl.Name() for _, p := range loader.plugins { if p.Name() == name { // plugin is already loaded return fmt.Errorf( "plugin: %s, is duplicated in version: %s, "+ "while trying to load dynamically: %s", name, p.Version(), pl.Version()) } } if loader.config.Plugins[name].Disabled { log.Infof("not loading disabled plugin %s", name) return nil } loader.plugins = append(loader.plugins, pl) return nil } // LoadDirectory loads a directory of plugins into the plugin loader. func (loader *PluginLoader) LoadDirectory(pluginDir string) error { if err := loader.assertState(loaderLoading); err != nil { return err } newPls, err := loadDynamicPlugins(pluginDir) if err != nil { return err } for _, pl := range newPls { if err := loader.Load(pl); err != nil { return err } } return nil } func loadDynamicPlugins(pluginDir string) ([]plugin.Plugin, error) { _, err := os.Stat(pluginDir) if os.IsNotExist(err) { return nil, nil } if err != nil { return nil, err } var plugins []plugin.Plugin err = filepath.Walk(pluginDir, func(fi string, info os.FileInfo, err error) error { if err != nil { return err } if info.IsDir() { if fi != pluginDir { log.Warnf("found directory inside plugins directory: %s", fi) } return nil } if info.Mode().Perm()&0o111 == 0 { // file is not executable let's not load it // this is to prevent loading plugins from for example non-executable // mounts, some /tmp mounts are marked as such for security log.Errorf("non-executable file in plugins directory: %s", fi) return nil } if newPlugins, err := loadPluginFunc(fi); err == nil { plugins = append(plugins, newPlugins...) } else { return fmt.Errorf("loading plugin %s: %s", fi, err) } return nil }) return plugins, err } // Initialize initializes all loaded plugins. func (loader *PluginLoader) Initialize() error { if err := loader.transition(loaderLoading, loaderInitializing); err != nil { return err } for _, p := range loader.plugins { err := p.Init(&plugin.Environment{ Repo: loader.repo, Config: loader.config.Plugins[p.Name()].Config, }) if err != nil { loader.state = loaderFailed return err } } return loader.transition(loaderInitializing, loaderInitialized) } // Inject hooks all the plugins into the appropriate subsystems. func (loader *PluginLoader) Inject() error { if err := loader.transition(loaderInitialized, loaderInjecting); err != nil { return err } for _, pl := range loader.plugins { if pl, ok := pl.(plugin.PluginIPLD); ok { err := injectIPLDPlugin(pl) if err != nil { loader.state = loaderFailed return err } } if pl, ok := pl.(plugin.PluginTracer); ok { err := injectTracerPlugin(pl) if err != nil { loader.state = loaderFailed return err } } if pl, ok := pl.(plugin.PluginDatastore); ok { err := injectDatastorePlugin(pl) if err != nil { loader.state = loaderFailed return err } } if pl, ok := pl.(plugin.PluginFx); ok { err := injectFxPlugin(pl) if err != nil { loader.state = loaderFailed return err } } } return loader.transition(loaderInjecting, loaderInjected) } // Start starts all long-running plugins. func (loader *PluginLoader) Start(node *core.IpfsNode) error { if err := loader.transition(loaderInjected, loaderStarting); err != nil { return err } iface, err := coreapi.NewCoreAPI(node) if err != nil { return err } for _, pl := range loader.plugins { if pl, ok := pl.(plugin.PluginDaemon); ok { err := pl.Start(iface) if err != nil { _ = loader.Close() return err } loader.started = append(loader.started, pl) } if pl, ok := pl.(plugin.PluginDaemonInternal); ok { err := pl.Start(node) if err != nil { _ = loader.Close() return err } loader.started = append(loader.started, pl) } } return loader.transition(loaderStarting, loaderStarted) } // Close stops all long-running plugins. func (loader *PluginLoader) Close() error { switch loader.state { case loaderClosing, loaderFailed, loaderClosed: // nothing to do. return nil } loader.state = loaderClosing var errs []string started := loader.started loader.started = nil for _, pl := range started { if closer, ok := pl.(io.Closer); ok { err := closer.Close() if err != nil { errs = append(errs, fmt.Sprintf( "error closing plugin %s: %s", pl.Name(), err.Error(), )) } } } if errs != nil { loader.state = loaderFailed return errors.New(strings.Join(errs, "\n")) } loader.state = loaderClosed return nil } func injectDatastorePlugin(pl plugin.PluginDatastore) error { return fsrepo.AddDatastoreConfigHandler(pl.DatastoreTypeName(), pl.DatastoreConfigParser()) } func injectIPLDPlugin(pl plugin.PluginIPLD) error { return pl.Register(multicodec.DefaultRegistry) } func injectTracerPlugin(pl plugin.PluginTracer) error { log.Warn("Tracer plugins are deprecated, it's recommended to configure an OpenTelemetry collector instead.") tracer, err := pl.InitTracer() if err != nil { return err } opentracing.SetGlobalTracer(tracer) return nil } func injectFxPlugin(pl plugin.PluginFx) error { core.RegisterFXOptionFunc(pl.Options) return nil } ================================================ FILE: plugin/loader/preload.go ================================================ package loader import ( pluginbadgerds "github.com/ipfs/kubo/plugin/plugins/badgerds" pluginiplddagjose "github.com/ipfs/kubo/plugin/plugins/dagjose" pluginflatfs "github.com/ipfs/kubo/plugin/plugins/flatfs" pluginfxtest "github.com/ipfs/kubo/plugin/plugins/fxtest" pluginipldgit "github.com/ipfs/kubo/plugin/plugins/git" pluginlevelds "github.com/ipfs/kubo/plugin/plugins/levelds" pluginnopfs "github.com/ipfs/kubo/plugin/plugins/nopfs" pluginpebbleds "github.com/ipfs/kubo/plugin/plugins/pebbleds" pluginpeerlog "github.com/ipfs/kubo/plugin/plugins/peerlog" plugintelemetry "github.com/ipfs/kubo/plugin/plugins/telemetry" ) // DO NOT EDIT THIS FILE // This file is being generated as part of plugin build process // To change it, modify the plugin/loader/preload.sh func init() { Preload(pluginipldgit.Plugins...) Preload(pluginiplddagjose.Plugins...) Preload(pluginbadgerds.Plugins...) Preload(pluginflatfs.Plugins...) Preload(pluginlevelds.Plugins...) Preload(pluginpebbleds.Plugins...) Preload(pluginpeerlog.Plugins...) Preload(pluginfxtest.Plugins...) Preload(pluginnopfs.Plugins...) Preload(plugintelemetry.Plugins...) } ================================================ FILE: plugin/loader/preload.sh ================================================ #!/usr/bin/env bash DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" LIST="$DIR/preload_list" to_preload() { awk 'NF' $LIST | sed '/^#/d' if [[ -n "$IPFS_PLUGINS" ]]; then for plugin in $IPFS_PLUGINS; do echo "$plugin github.com/ipfs/kubo/plugin/plugins/$plugin *" done fi } cat </dev/null $($(d)_plugins_so): %.so : %/main/main.go $($(d)_plugins_so): $$(DEPS_GO) ALWAYS $(GOCC) build -buildmode=plugin -pkgdir "$(GOPATH)/pkg/linux_amd64_dynlink" $(go-flags-with-tags) -o "$@" "$(call go-pkg-name,$(basename $@))/main" chmod +x "$@" CLEAN += $($(d)_plugins_so) CLEAN += $(foreach main_dir,$($(d)_plugins_main),$(dir $(main_dir))) build_plugins: $($(d)_plugins_so) include mk/footer.mk ================================================ FILE: plugin/plugins/badgerds/badgerds.go ================================================ package badgerds import ( "fmt" "os" "path/filepath" logging "github.com/ipfs/go-log/v2" "github.com/ipfs/kubo/plugin" "github.com/ipfs/kubo/repo" "github.com/ipfs/kubo/repo/fsrepo" humanize "github.com/dustin/go-humanize" badgerds "github.com/ipfs/go-ds-badger" ) var log = logging.Logger("plugin/badgerds") // Plugins is exported list of plugins that will be loaded. var Plugins = []plugin.Plugin{ &badgerdsPlugin{}, } type badgerdsPlugin struct{} var _ plugin.PluginDatastore = (*badgerdsPlugin)(nil) func (*badgerdsPlugin) Name() string { return "ds-badgerds" } func (*badgerdsPlugin) Version() string { return "0.1.0" } func (*badgerdsPlugin) Init(_ *plugin.Environment) error { return nil } func (*badgerdsPlugin) DatastoreTypeName() string { return "badgerds" } type datastoreConfig struct { path string syncWrites bool truncate bool vlogFileSize int64 } // BadgerdsDatastoreConfig returns a configuration stub for a badger datastore // from the given parameters. func (*badgerdsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap { return func(params map[string]any) (fsrepo.DatastoreConfig, error) { var c datastoreConfig var ok bool c.path, ok = params["path"].(string) if !ok { return nil, fmt.Errorf("'path' field is missing or not string") } sw, ok := params["syncWrites"] if !ok { c.syncWrites = false } else { if swb, ok := sw.(bool); ok { c.syncWrites = swb } else { return nil, fmt.Errorf("'syncWrites' field was not a boolean") } } truncate, ok := params["truncate"] if !ok { c.truncate = true } else { if truncate, ok := truncate.(bool); ok { c.truncate = truncate } else { return nil, fmt.Errorf("'truncate' field was not a boolean") } } vls, ok := params["vlogFileSize"] if !ok { // default to 1GiB c.vlogFileSize = badgerds.DefaultOptions.ValueLogFileSize } else { if vlogSize, ok := vls.(string); ok { s, err := humanize.ParseBytes(vlogSize) if err != nil { return nil, err } c.vlogFileSize = int64(s) } else { return nil, fmt.Errorf("'vlogFileSize' field was not a string") } } return &c, nil } } func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec { return map[string]any{ "type": "badgerds", "path": c.path, } } func (c *datastoreConfig) Create(path string) (repo.Datastore, error) { log.Error("badger v1 datastore is deprecated and will be removed later in 2026, migrate to flatfs or experimental pebbleds: https://github.com/ipfs/kubo/issues/11186") fmt.Fprintf(os.Stderr, ` ╔════════════════════════════════════════════════════════════════════════════╗ ║ ║ ║ ERROR: BADGER v1 DATASTORE IS DEPRECATED ║ ║ ║ ║ This datastore is based on badger 1.x which has not been maintained ║ ║ by its upstream maintainers for years and has known bugs (startup ║ ║ timeouts, shutdown hangs, file descriptor exhaustion, and more). ║ ║ ║ ║ Badger v1 support will be REMOVED later in 2026. ║ ║ ║ ║ To migrate: ║ ║ 1. Create a new IPFS_PATH with flatfs (or experimental pebbleds ║ ║ if flatfs does not serve your use case): ║ ║ export IPFS_PATH=/path/to/new/repo ║ ║ ipfs init --profile=flatfs ║ ║ 2. Move pinned data via ipfs dag export/import ║ ║ or ipfs pin ls -t recursive|add ║ ║ 3. Decommission the old badger-based node ║ ║ ║ ║ See https://github.com/ipfs/kubo/blob/master/docs/datastores.md ║ ║ https://github.com/ipfs/kubo/issues/11186 ║ ║ ║ ╚════════════════════════════════════════════════════════════════════════════╝ `) p := c.path if !filepath.IsAbs(p) { p = filepath.Join(path, p) } err := os.MkdirAll(p, 0o755) if err != nil { return nil, err } defopts := badgerds.DefaultOptions defopts.SyncWrites = c.syncWrites defopts.Truncate = c.truncate defopts.ValueLogFileSize = c.vlogFileSize return badgerds.NewDatastore(p, &defopts) } ================================================ FILE: plugin/plugins/dagjose/dagjose.go ================================================ package dagjose import ( "github.com/ipfs/kubo/plugin" "github.com/ceramicnetwork/go-dag-jose/dagjose" "github.com/ipld/go-ipld-prime/multicodec" mc "github.com/multiformats/go-multicodec" ) // Plugins is exported list of plugins that will be loaded. var Plugins = []plugin.Plugin{ &dagjosePlugin{}, } type dagjosePlugin struct{} var _ plugin.PluginIPLD = (*dagjosePlugin)(nil) func (*dagjosePlugin) Name() string { return "ipld-codec-dagjose" } func (*dagjosePlugin) Version() string { return "0.0.1" } func (*dagjosePlugin) Init(_ *plugin.Environment) error { return nil } func (*dagjosePlugin) Register(reg multicodec.Registry) error { reg.RegisterEncoder(uint64(mc.DagJose), dagjose.Encode) reg.RegisterDecoder(uint64(mc.DagJose), dagjose.Decode) return nil } ================================================ FILE: plugin/plugins/flatfs/flatfs.go ================================================ package flatfs import ( "fmt" "path/filepath" "github.com/ipfs/kubo/plugin" "github.com/ipfs/kubo/repo" "github.com/ipfs/kubo/repo/fsrepo" flatfs "github.com/ipfs/go-ds-flatfs" ) // Plugins is exported list of plugins that will be loaded. var Plugins = []plugin.Plugin{ &flatfsPlugin{}, } type flatfsPlugin struct{} var _ plugin.PluginDatastore = (*flatfsPlugin)(nil) func (*flatfsPlugin) Name() string { return "ds-flatfs" } func (*flatfsPlugin) Version() string { return "0.1.0" } func (*flatfsPlugin) Init(_ *plugin.Environment) error { return nil } func (*flatfsPlugin) DatastoreTypeName() string { return "flatfs" } type datastoreConfig struct { path string shardFun *flatfs.ShardIdV1 syncField bool } // DatastoreConfigParser returns a configuration stub for a flatfs datastore // from the given parameters. func (*flatfsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap { return func(params map[string]any) (fsrepo.DatastoreConfig, error) { var c datastoreConfig var ok bool var err error c.path, ok = params["path"].(string) if !ok { return nil, fmt.Errorf("'path' field is missing or not boolean") } sshardFun, ok := params["shardFunc"].(string) if !ok { return nil, fmt.Errorf("'shardFunc' field is missing or not a string") } c.shardFun, err = flatfs.ParseShardFunc(sshardFun) if err != nil { return nil, err } c.syncField, ok = params["sync"].(bool) if !ok { return nil, fmt.Errorf("'sync' field is missing or not boolean") } return &c, nil } } func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec { return map[string]any{ "type": "flatfs", "path": c.path, "shardFunc": c.shardFun.String(), } } func (c *datastoreConfig) Create(path string) (repo.Datastore, error) { p := c.path if !filepath.IsAbs(p) { p = filepath.Join(path, p) } return flatfs.CreateOrOpen(p, c.shardFun, c.syncField) } ================================================ FILE: plugin/plugins/fxtest/fxtest.go ================================================ package fxtest import ( "os" logging "github.com/ipfs/go-log/v2" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/plugin" "go.uber.org/fx" ) var log = logging.Logger("fxtestplugin") var Plugins = []plugin.Plugin{ &fxtestPlugin{}, } // fxtestPlugin is used for testing the fx plugin. // It merely adds an fx option that logs a debug statement, so we can verify that it works in tests. type fxtestPlugin struct{} var _ plugin.PluginFx = (*fxtestPlugin)(nil) func (p *fxtestPlugin) Name() string { return "fx-test" } func (p *fxtestPlugin) Version() string { return "0.1.0" } func (p *fxtestPlugin) Init(env *plugin.Environment) error { return nil } func (p *fxtestPlugin) Options(info core.FXNodeInfo) ([]fx.Option, error) { opts := info.FXOptions if os.Getenv("TEST_FX_PLUGIN") != "" { opts = append(opts, fx.Invoke(func() { log.Debug("invoked test fx function") })) } return opts, nil } ================================================ FILE: plugin/plugins/gen_main.sh ================================================ #!/usr/bin/env bash dir=${1:?first parameter with dir to work in is required} pkg=${2:?second parameter with full name of the package is required} main_pkg="$dir/main" shortpkg="uniquepkgname" mkdir -p "$main_pkg" cat > "$main_pkg/main.go" < 0 { busyCounter++ // sleep a bit to give the system a chance to catch up with logging. select { case <-time.After(time.Duration(busyCounter) * time.Second): case <-ctx.Done(): return } // drain 1/8th of the backlog backlog so we // don't immediately run into this situation // again. loop: for range busyDropAmount { select { case <-pl.events: dropped++ default: break loop } } // Add in any events we've dropped in the mean-time. dropped += atomic.SwapUint64(&pl.droppedCount, 0) // Report that we've dropped events. dlog.Error("dropped events", zap.Uint64("count", dropped)) } else { busyCounter = 0 } var e plEvent select { case <-ctx.Done(): return case e = <-pl.events: } peerID := zap.String("peer", e.peer.String()) switch e.kind { case eventConnect: dlog.Info("connected", peerID) case eventIdentify: agent, err := node.Peerstore.Get(e.peer, "AgentVersion") switch err { case nil: case peerstore.ErrNotFound: continue default: dlog.Error("failed to get agent version", zap.Error(err)) continue } agentS, ok := agent.(string) if !ok { continue } dlog.Info("identified", peerID, zap.String("agent", agentS)) } } } func (pl *peerLogPlugin) emit(evt eventType, p peer.ID) { select { case pl.events <- plEvent{kind: evt, peer: p}: default: atomic.AddUint64(&pl.droppedCount, 1) } } func (pl *peerLogPlugin) Start(node *core.IpfsNode) error { if !pl.enabled { return nil } // Ensure logs from this plugin get printed regardless of global GOLOG_LOG_LEVEL value if err := logging.SetLogLevel("plugin/peerlog", "info"); err != nil { return fmt.Errorf("failed to set log level: %w", err) } sub, err := node.PeerHost.EventBus().Subscribe(new(event.EvtPeerIdentificationCompleted)) if err != nil { return fmt.Errorf("failed to subscribe to identify notifications") } var notifee network.NotifyBundle notifee.ConnectedF = func(net network.Network, conn network.Conn) { pl.emit(eventConnect, conn.RemotePeer()) } node.PeerHost.Network().Notify(¬ifee) go func() { defer sub.Close() for e := range sub.Out() { switch e := e.(type) { case event.EvtPeerIdentificationCompleted: pl.emit(eventIdentify, e.Peer) } } }() go pl.collectEvents(node) return nil } func (*peerLogPlugin) Close() error { return nil } ================================================ FILE: plugin/plugins/peerlog/peerlog_test.go ================================================ package peerlog import "testing" func TestExtractEnabled(t *testing.T) { for _, c := range []struct { name string config any expected bool }{ { name: "nil config returns false", config: nil, expected: false, }, { name: "returns false when config is not a string map", config: 1, expected: false, }, { name: "returns false when config has no Enabled field", config: map[string]any{}, expected: false, }, { name: "returns false when config has a null Enabled field", config: map[string]any{"Enabled": nil}, expected: false, }, { name: "returns false when config has a non-boolean Enabled field", config: map[string]any{"Enabled": 1}, expected: false, }, { name: "returns the value of the Enabled field", config: map[string]any{"Enabled": true}, expected: true, }, } { t.Run(c.name, func(t *testing.T) { isEnabled := extractEnabled(c.config) if isEnabled != c.expected { t.Fatalf("expected %v, got %v", c.expected, isEnabled) } }) } } ================================================ FILE: plugin/plugins/telemetry/telemetry.go ================================================ package telemetry import ( "bytes" "context" "encoding/json" "fmt" "net/http" "os" "path" "runtime" "slices" "strings" "sync" "time" "github.com/google/uuid" logging "github.com/ipfs/go-log/v2" ipfs "github.com/ipfs/kubo" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/corerepo" "github.com/ipfs/kubo/plugin" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/pnet" multiaddr "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) var log = logging.Logger("telemetry") // Caching for virtualization detection - these values never change during process lifetime var ( containerDetectionOnce sync.Once vmDetectionOnce sync.Once isContainerCached bool isVMCached bool ) const ( modeEnvVar = "IPFS_TELEMETRY" uuidFilename = "telemetry_uuid" endpoint = "https://telemetry.ipshipyard.dev" sendDelay = 15 * time.Minute // delay before first telemetry collection after daemon start sendInterval = 24 * time.Hour // interval between telemetry collections after the first one httpTimeout = 30 * time.Second // timeout for telemetry HTTP requests ) type pluginMode int const ( modeAuto pluginMode = iota modeOn modeOff ) // repoSizeBuckets defines size thresholds for categorizing repository sizes. // Each value represents the upper limit of a bucket in bytes (except the last) var repoSizeBuckets = []uint64{ 1 << 30, // 1 GB 5 << 30, // 5 GB 10 << 30, // 10 GB 100 << 30, // 100 GB 500 << 30, // 500 GB 1 << 40, // 1 TB 10 << 40, // 10 TB 11 << 40, // + anything more than 10TB falls here. } var uptimeBuckets = []time.Duration{ 1 * 24 * time.Hour, 2 * 24 * time.Hour, 3 * 24 * time.Hour, 7 * 24 * time.Hour, 14 * 24 * time.Hour, 30 * 24 * time.Hour, 31 * 24 * time.Hour, // + anything more than 30 days falls here. } // A LogEvent is the object sent to the telemetry endpoint. // See https://github.com/ipfs/kubo/blob/master/docs/telemetry.md for details. type LogEvent struct { UUID string `json:"uuid"` AgentVersion string `json:"agent_version"` PrivateNetwork bool `json:"private_network"` BootstrappersCustom bool `json:"bootstrappers_custom"` RepoSizeBucket uint64 `json:"repo_size_bucket"` UptimeBucket time.Duration `json:"uptime_bucket"` ReproviderStrategy string `json:"reprovider_strategy"` ProvideDHTSweepEnabled bool `json:"provide_dht_sweep_enabled"` ProvideDHTIntervalCustom bool `json:"provide_dht_interval_custom"` ProvideDHTMaxWorkersCustom bool `json:"provide_dht_max_workers_custom"` RoutingType string `json:"routing_type"` RoutingAcceleratedDHTClient bool `json:"routing_accelerated_dht_client"` RoutingDelegatedCount int `json:"routing_delegated_count"` AutoNATServiceMode string `json:"autonat_service_mode"` AutoNATReachability string `json:"autonat_reachability"` AutoConf bool `json:"autoconf"` AutoConfCustom bool `json:"autoconf_custom"` SwarmEnableHolePunching bool `json:"swarm_enable_hole_punching"` SwarmCircuitAddresses bool `json:"swarm_circuit_addresses"` SwarmIPv4PublicAddresses bool `json:"swarm_ipv4_public_addresses"` SwarmIPv6PublicAddresses bool `json:"swarm_ipv6_public_addresses"` AutoTLSAutoWSS bool `json:"auto_tls_auto_wss"` AutoTLSDomainSuffixCustom bool `json:"auto_tls_domain_suffix_custom"` DiscoveryMDNSEnabled bool `json:"discovery_mdns_enabled"` PlatformOS string `json:"platform_os"` PlatformArch string `json:"platform_arch"` PlatformContainerized bool `json:"platform_containerized"` PlatformVM bool `json:"platform_vm"` } var Plugins = []plugin.Plugin{ &telemetryPlugin{}, } type telemetryPlugin struct { uuidFilename string mode pluginMode endpoint string runOnce bool // test-only flag: when true, sends telemetry immediately without delay sendDelay time.Duration node *core.IpfsNode config *config.Config event *LogEvent startTime time.Time } func (p *telemetryPlugin) Name() string { return "telemetry" } func (p *telemetryPlugin) Version() string { return "0.0.1" } func readFromConfig(cfg any, key string) string { if cfg == nil { return "" } pcfg, ok := cfg.(map[string]any) if !ok { return "" } val, ok := pcfg[key].(string) if !ok { return "" } return val } func (p *telemetryPlugin) Init(env *plugin.Environment) error { // logging.SetLogLevel("telemetry", "DEBUG") log.Debug("telemetry plugin Init()") p.event = &LogEvent{} p.startTime = time.Now() repoPath := env.Repo p.uuidFilename = path.Join(repoPath, uuidFilename) v := os.Getenv(modeEnvVar) if v != "" { log.Debug("mode set from env-var") } else if pmode := readFromConfig(env.Config, "Mode"); pmode != "" { v = pmode log.Debug("mode set from config") } // read "Delay" from the config. Parse as duration. Set p.sendDelay to it // or set default. if delayStr := readFromConfig(env.Config, "Delay"); delayStr != "" { delay, err := time.ParseDuration(delayStr) if err != nil { log.Debug("sendDelay set from default") p.sendDelay = sendDelay } else { log.Debug("sendDelay set from config") p.sendDelay = delay } } else { log.Debug("sendDelay set from default") p.sendDelay = sendDelay } p.endpoint = endpoint if ep := readFromConfig(env.Config, "Endpoint"); ep != "" { log.Debug("endpoint set from config", ep) p.endpoint = ep } switch v { case "off": p.mode = modeOff log.Debug("telemetry disabled via opt-out") // Remove UUID file if it exists when user opts out if _, err := os.Stat(p.uuidFilename); err == nil { if err := os.Remove(p.uuidFilename); err != nil { log.Debugf("failed to remove telemetry UUID file: %s", err) } else { log.Debug("removed existing telemetry UUID file due to opt-out") } } return nil case "auto": p.mode = modeAuto default: p.mode = modeOn } log.Debug("telemetry mode: ", p.mode) return nil } func (p *telemetryPlugin) loadUUID() error { // Generate or read our UUID from disk b, err := os.ReadFile(p.uuidFilename) if err != nil { if !os.IsNotExist(err) { log.Errorf("error reading telemetry uuid from disk: %s", err) return err } uid, err := uuid.NewRandom() if err != nil { log.Errorf("cannot generate telemetry uuid: %s", err) return err } p.event.UUID = uid.String() p.mode = modeAuto log.Debugf("new telemetry UUID %s. Mode set to Auto", uid) // Write the UUID to disk if err := os.WriteFile(p.uuidFilename, []byte(p.event.UUID), 0600); err != nil { log.Errorf("cannot write telemetry uuid: %s", err) return err } return nil } v := string(b) v = strings.TrimSpace(v) uid, err := uuid.Parse(v) if err != nil { log.Errorf("cannot parse telemetry uuid: %s", err) return err } log.Debugf("uuid read from disk %s", uid) p.event.UUID = uid.String() return nil } func (p *telemetryPlugin) hasDefaultBootstrapPeers() bool { // With autoconf, default bootstrap is represented as ["auto"] currentPeers := p.config.Bootstrap return len(currentPeers) == 1 && currentPeers[0] == "auto" } func (p *telemetryPlugin) showInfo() { fmt.Printf(` ℹ️ Anonymous telemetry will be enabled in %s Kubo will collect anonymous usage data to help improve the software: • What: Feature usage and configuration (no personal data) Use GOLOG_LOG_LEVEL="telemetry=debug" to inspect collected data • When: First collection in %s, then every 24h • How: HTTP POST to %s Anonymous ID: %s No data sent yet. To opt-out before collection starts: • Set environment: %s=off • Or run: ipfs config Plugins.Plugins.telemetry.Config.Mode off • Then restart daemon This message is shown only once. Learn more: https://github.com/ipfs/kubo/blob/master/docs/telemetry.md `, p.sendDelay, p.sendDelay, endpoint, p.event.UUID, modeEnvVar) } // Start finishes telemetry initialization once the IpfsNode is ready, // collects telemetry data and sends it to the endpoint. func (p *telemetryPlugin) Start(n *core.IpfsNode) error { // We should not be crashing the daemon due to problems with telemetry // so this is always going to return nil and panics are going to be // handled. defer func() { if r := recover(); r != nil { log.Errorf("telemetry plugin panicked: %v", r) } }() p.node = n cfg, err := n.Repo.Config() if err != nil { log.Error("error getting the repo.Config: %s", err) return nil } p.config = cfg if p.mode == modeOff { log.Debug("telemetry collection skipped: opted out") return nil } if !n.IsDaemon || !n.IsOnline { log.Debugf("skipping telemetry. Daemon: %t. Online: %t", n.IsDaemon, n.IsOnline) return nil } // loadUUID might switch to modeAuto when generating a new uuid if err := p.loadUUID(); err != nil { p.mode = modeOff return nil } if p.mode == modeAuto { p.showInfo() } // runOnce is only used in tests to send telemetry immediately. // In production, this is always false, ensuring users get the 15-minute delay. if p.runOnce { p.prepareEvent() return p.sendTelemetry() } go func() { timer := time.NewTimer(p.sendDelay) for range timer.C { p.prepareEvent() if err := p.sendTelemetry(); err != nil { log.Warnf("telemetry submission failed: %s (will retry in %s)", err, sendInterval) } timer.Reset(sendInterval) } }() return nil } func (p *telemetryPlugin) prepareEvent() { p.collectBasicInfo() p.collectRoutingInfo() p.collectProvideInfo() p.collectAutoNATInfo() p.collectAutoConfInfo() p.collectSwarmInfo() p.collectAutoTLSInfo() p.collectDiscoveryInfo() p.collectPlatformInfo() } func (p *telemetryPlugin) collectBasicInfo() { p.event.AgentVersion = ipfs.GetUserAgentVersion() privNet := false if pnet.ForcePrivateNetwork { privNet = true } else if key, _ := p.node.Repo.SwarmKey(); key != nil { privNet = true } p.event.PrivateNetwork = privNet p.event.BootstrappersCustom = !p.hasDefaultBootstrapPeers() repoSizeBucket := repoSizeBuckets[len(repoSizeBuckets)-1] sizeStat, err := corerepo.RepoSize(context.Background(), p.node) if err == nil { for _, b := range repoSizeBuckets { if sizeStat.RepoSize > b { continue } repoSizeBucket = b break } p.event.RepoSizeBucket = repoSizeBucket } else { log.Debugf("error setting sizeStat: %s", err) } uptime := time.Since(p.startTime) uptimeBucket := uptimeBuckets[len(uptimeBuckets)-1] for _, bucket := range uptimeBuckets { if uptime > bucket { continue } uptimeBucket = bucket break } p.event.UptimeBucket = uptimeBucket } func (p *telemetryPlugin) collectRoutingInfo() { p.event.RoutingType = p.config.Routing.Type.WithDefault("auto") p.event.RoutingAcceleratedDHTClient = p.config.Routing.AcceleratedDHTClient.WithDefault(false) p.event.RoutingDelegatedCount = len(p.config.Routing.DelegatedRouters) } func (p *telemetryPlugin) collectProvideInfo() { p.event.ReproviderStrategy = p.config.Provide.Strategy.WithDefault(config.DefaultProvideStrategy) p.event.ProvideDHTSweepEnabled = p.config.Provide.DHT.SweepEnabled.WithDefault(config.DefaultProvideDHTSweepEnabled) p.event.ProvideDHTIntervalCustom = !p.config.Provide.DHT.Interval.IsDefault() p.event.ProvideDHTMaxWorkersCustom = !p.config.Provide.DHT.MaxWorkers.IsDefault() } type reachabilityHost interface { Reachability() network.Reachability } func (p *telemetryPlugin) collectAutoNATInfo() { autonat := p.config.AutoNAT.ServiceMode if autonat == config.AutoNATServiceUnset { autonat = config.AutoNATServiceEnabled } autoNATSvcModeB, err := autonat.MarshalText() if err == nil { autoNATSvcMode := string(autoNATSvcModeB) if autoNATSvcMode == "" { autoNATSvcMode = "unset" } p.event.AutoNATServiceMode = autoNATSvcMode } h := p.node.PeerHost reachHost, ok := h.(reachabilityHost) if ok { p.event.AutoNATReachability = reachHost.Reachability().String() } } func (p *telemetryPlugin) collectSwarmInfo() { p.event.SwarmEnableHolePunching = p.config.Swarm.EnableHolePunching.WithDefault(true) var circuitAddrs, publicIP4Addrs, publicIP6Addrs bool for _, addr := range p.node.PeerHost.Addrs() { if manet.IsPublicAddr(addr) { if _, err := addr.ValueForProtocol(multiaddr.P_IP4); err == nil { publicIP4Addrs = true } else if _, err := addr.ValueForProtocol(multiaddr.P_IP6); err == nil { publicIP6Addrs = true } } if _, err := addr.ValueForProtocol(multiaddr.P_CIRCUIT); err == nil { circuitAddrs = true } } p.event.SwarmCircuitAddresses = circuitAddrs p.event.SwarmIPv4PublicAddresses = publicIP4Addrs p.event.SwarmIPv6PublicAddresses = publicIP6Addrs } func (p *telemetryPlugin) collectAutoTLSInfo() { p.event.AutoTLSAutoWSS = p.config.AutoTLS.AutoWSS.WithDefault(config.DefaultAutoWSS) domainSuffix := p.config.AutoTLS.DomainSuffix.WithDefault(config.DefaultDomainSuffix) p.event.AutoTLSDomainSuffixCustom = domainSuffix != config.DefaultDomainSuffix } func (p *telemetryPlugin) collectAutoConfInfo() { p.event.AutoConf = p.config.AutoConf.Enabled.WithDefault(config.DefaultAutoConfEnabled) p.event.AutoConfCustom = p.config.AutoConf.URL.WithDefault(config.DefaultAutoConfURL) != config.DefaultAutoConfURL } func (p *telemetryPlugin) collectDiscoveryInfo() { p.event.DiscoveryMDNSEnabled = p.config.Discovery.MDNS.Enabled } func (p *telemetryPlugin) collectPlatformInfo() { p.event.PlatformOS = runtime.GOOS p.event.PlatformArch = runtime.GOARCH p.event.PlatformContainerized = isRunningInContainer() p.event.PlatformVM = isRunningInVM() } func isRunningInContainer() bool { containerDetectionOnce.Do(func() { isContainerCached = detectContainer() }) return isContainerCached } func detectContainer() bool { // Docker creates /.dockerenv inside containers if _, err := os.Stat("/.dockerenv"); err == nil { return true } // Kubernetes mounts service account tokens inside pods if _, err := os.Stat("/var/run/secrets/kubernetes.io"); err == nil { return true } // systemd-nspawn creates this file inside containers if _, err := os.Stat("/run/systemd/container"); err == nil { return true } // Check if our process is running inside a container cgroup // Look for container-specific patterns in the cgroup path after "::/" if content, err := os.ReadFile("/proc/self/cgroup"); err == nil { for line := range strings.Lines(string(content)) { // cgroup lines format: "ID:subsystem:/path" // We want to check the path part after the last ":" parts := strings.SplitN(line, ":", 3) if len(parts) == 3 { cgroupPath := parts[2] // Check for container-specific paths containerIndicators := []string{ "/docker/", // Docker containers "/containerd/", // containerd runtime "/cri-o/", // CRI-O runtime "/lxc/", // LXC containers "/podman/", // Podman containers "/kubepods/", // Kubernetes pods } for _, indicator := range containerIndicators { if strings.Contains(cgroupPath, indicator) { return true } } } } } // WSL is technically a container-like environment if runtime.GOOS == "linux" { if content, err := os.ReadFile("/proc/sys/kernel/osrelease"); err == nil { osrelease := strings.ToLower(string(content)) if strings.Contains(osrelease, "microsoft") || strings.Contains(osrelease, "wsl") { return true } } } // LXC sets container environment variable if content, err := os.ReadFile("/proc/1/environ"); err == nil { if strings.Contains(string(content), "container=lxc") { return true } } // Additional check: In containers, PID 1 is often not systemd/init if content, err := os.ReadFile("/proc/1/comm"); err == nil { pid1 := strings.TrimSpace(string(content)) // Common container init processes containerInits := []string{"tini", "dumb-init", "s6-svscan", "runit"} if slices.Contains(containerInits, pid1) { return true } } return false } func isRunningInVM() bool { vmDetectionOnce.Do(func() { isVMCached = detectVM() }) return isVMCached } func detectVM() bool { // Check for VM-specific files and drivers that only exist inside VMs vmIndicators := []string{ "/proc/xen", // Xen hypervisor guest "/sys/hypervisor/uuid", // KVM/Xen hypervisor guest "/dev/vboxguest", // VirtualBox guest additions "/sys/module/vmw_balloon", // VMware balloon driver (guest only) "/sys/module/hv_vmbus", // Hyper-V VM bus driver (guest only) } for _, path := range vmIndicators { if _, err := os.Stat(path); err == nil { return true } } // Check DMI for VM vendors - these strings only appear inside VMs // DMI (Desktop Management Interface) is populated by the hypervisor dmiFiles := map[string][]string{ "/sys/class/dmi/id/sys_vendor": { "qemu", "kvm", "vmware", "virtualbox", "xen", "parallels", // Parallels Desktop // Note: Removed "microsoft corporation" as it can match Surface devices }, "/sys/class/dmi/id/product_name": { "virtualbox", "vmware", "kvm", "qemu", "hvm domu", // Xen HVM guest // Note: Removed generic "virtual machine" to avoid false positives }, "/sys/class/dmi/id/chassis_vendor": { "qemu", "oracle", // Oracle for VirtualBox }, } for path, signatures := range dmiFiles { if content, err := os.ReadFile(path); err == nil { contentStr := strings.ToLower(strings.TrimSpace(string(content))) for _, sig := range signatures { if strings.Contains(contentStr, sig) { return true } } } } return false } func (p *telemetryPlugin) sendTelemetry() error { data, err := json.MarshalIndent(p.event, "", " ") if err != nil { return err } log.Debugf("sending telemetry:\n %s", data) req, err := http.NewRequest("POST", p.endpoint, bytes.NewBuffer(data)) if err != nil { return err } req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", ipfs.GetUserAgentVersion()) req.Close = true // Use client with timeout to prevent hanging client := &http.Client{ Timeout: httpTimeout, } resp, err := client.Do(req) if err != nil { log.Debugf("failed to send telemetry: %s", err) return err } defer resp.Body.Close() if resp.StatusCode >= 400 { err := fmt.Errorf("telemetry endpoint returned HTTP %d", resp.StatusCode) log.Debug(err) return err } log.Debugf("telemetry sent successfully (%d)", resp.StatusCode) return nil } ================================================ FILE: plugin/plugins/telemetry/telemetry_test.go ================================================ package telemetry import ( "context" "encoding/json" "io" "net/http" "net/http/httptest" "os" "testing" "github.com/cockroachdb/pebble/v2" logging "github.com/ipfs/go-log/v2" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/node/libp2p" "github.com/ipfs/kubo/plugin" "github.com/ipfs/kubo/plugin/plugins/pebbleds" "github.com/ipfs/kubo/repo/fsrepo" ) func mockServer(t *testing.T) (*httptest.Server, func() LogEvent) { t.Helper() var e LogEvent // Create a mock HTTP test server return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Check if the request is POST to the correct endpoint if r.Method != "POST" || r.URL.Path != "/" { t.Log("invalid request") http.Error(w, "invalid request", http.StatusBadRequest) return } // Check content type if r.Header.Get("Content-Type") != "application/json" { t.Log("invalid content type") http.Error(w, "invalid content type", http.StatusBadRequest) return } // Check if the body is not empty if r.Body == nil { t.Log("empty body") http.Error(w, "empty body", http.StatusBadRequest) return } // Read the body body, _ := io.ReadAll(r.Body) if len(body) == 0 { t.Log("zero-length body") http.Error(w, "empty body", http.StatusBadRequest) return } t.Logf("Received telemetry:\n %s", string(body)) err := json.Unmarshal(body, &e) if err != nil { t.Log("error unmarshaling event", err) http.Error(w, err.Error(), http.StatusBadRequest) return } // Return success w.WriteHeader(http.StatusOK) })), func() LogEvent { return e } } func makeNode(t *testing.T) (node *core.IpfsNode, repopath string) { t.Helper() // Create a Temporary Repo repoPath, err := os.MkdirTemp("", "ipfs-shell") if err != nil { t.Fatal(err) } pebbledspli := pebbleds.Plugins[0] pebbledspl, ok := pebbledspli.(plugin.PluginDatastore) if !ok { t.Fatal("bad datastore plugin") } err = fsrepo.AddDatastoreConfigHandler(pebbledspl.DatastoreTypeName(), pebbledspl.DatastoreConfigParser()) if err != nil { t.Fatal(err) } // Create a config with default options and a 2048 bit key cfg, err := config.Init(io.Discard, 2048) if err != nil { t.Fatal(err) } cfg.Datastore.Spec = map[string]any{ "type": "pebbleds", "prefix": "pebble.datastore", "path": "pebbleds", "formatMajorVersion": int(pebble.FormatNewest), } // Create the repo with the config err = fsrepo.Init(repoPath, cfg) if err != nil { t.Fatal(err) } // Open the repo repo, err := fsrepo.Open(repoPath) if err != nil { t.Fatal(err) } // Construct the node nodeOptions := &core.BuildCfg{ Online: true, Routing: libp2p.NilRouterOption, Repo: repo, } node, err = core.NewNode(context.Background(), nodeOptions) if err != nil { t.Fatal(err) } node.IsDaemon = true return } func TestSendTelemetry(t *testing.T) { if err := logging.SetLogLevel("telemetry", "DEBUG"); err != nil { t.Fatal(err) } ts, eventGetter := mockServer(t) defer ts.Close() node, repoPath := makeNode(t) // Create a plugin instance p := &telemetryPlugin{ runOnce: true, } // Initialize the plugin pe := &plugin.Environment{ Repo: repoPath, Config: nil, } err := p.Init(pe) if err != nil { t.Fatalf("Init() failed: %v", err) } p.endpoint = ts.URL // Start the plugin err = p.Start(node) if err != nil { t.Fatalf("Start() failed: %v", err) } e := eventGetter() if e.UUID != p.event.UUID { t.Fatal("uuid mismatch") } } ================================================ FILE: plugin/plugins/telemetry/telemetry_uuid ================================================ 289ffed8-c770-49ae-922f-b020c8f776f2 ================================================ FILE: plugin/tracer.go ================================================ package plugin import ( "github.com/opentracing/opentracing-go" ) // PluginTracer is an interface that can be implemented to add a tracer. type PluginTracer interface { Plugin InitTracer() (opentracing.Tracer, error) } ================================================ FILE: profile/goroutines.go ================================================ package profile import ( "io" "runtime" ) // WriteAllGoroutineStacks writes a stack trace to the given writer. // This is distinct from the Go-provided method because it does not truncate after 64 MB. func WriteAllGoroutineStacks(w io.Writer) error { // this is based on pprof.writeGoroutineStacks, and removes the 64 MB limit buf := make([]byte, 1<<20) for i := 0; ; i++ { n := runtime.Stack(buf, true) if n < len(buf) { buf = buf[:n] break } // if len(buf) >= 64<<20 { // // Filled 64 MB - stop there. // break // } buf = make([]byte, 2*len(buf)) } _, err := w.Write(buf) return err } ================================================ FILE: profile/profile.go ================================================ package profile import ( "archive/zip" "bytes" "context" "encoding/json" "fmt" "io" "os" "runtime" "runtime/pprof" "runtime/trace" "sync" "time" "github.com/ipfs/go-log/v2" version "github.com/ipfs/kubo" ) const ( CollectorGoroutinesStack = "goroutines-stack" CollectorGoroutinesPprof = "goroutines-pprof" CollectorVersion = "version" CollectorHeap = "heap" CollectorAllocs = "allocs" CollectorBin = "bin" CollectorCPU = "cpu" CollectorMutex = "mutex" CollectorBlock = "block" CollectorTrace = "trace" ) var ( logger = log.Logger("profile") goos = runtime.GOOS ) type collector struct { outputFile string isExecutable bool collectFunc func(ctx context.Context, opts Options, writer io.Writer) error enabledFunc func(opts Options) bool } func (p *collector) outputFileName() string { fName := p.outputFile if p.isExecutable { if goos == "windows" { fName += ".exe" } } return fName } var collectors = map[string]collector{ CollectorGoroutinesStack: { outputFile: "goroutines.stacks", collectFunc: goroutineStacksText, enabledFunc: func(opts Options) bool { return true }, }, CollectorGoroutinesPprof: { outputFile: "goroutines.pprof", collectFunc: goroutineStacksProto, enabledFunc: func(opts Options) bool { return true }, }, CollectorVersion: { outputFile: "version.json", collectFunc: versionInfo, enabledFunc: func(opts Options) bool { return true }, }, CollectorHeap: { outputFile: "heap.pprof", collectFunc: heapProfile, enabledFunc: func(opts Options) bool { return true }, }, CollectorAllocs: { outputFile: "allocs.pprof", collectFunc: allocsProfile, enabledFunc: func(opts Options) bool { return true }, }, CollectorBin: { outputFile: "ipfs", isExecutable: true, collectFunc: binary, enabledFunc: func(opts Options) bool { return true }, }, CollectorCPU: { outputFile: "cpu.pprof", collectFunc: profileCPU, enabledFunc: func(opts Options) bool { return opts.ProfileDuration > 0 }, }, CollectorMutex: { outputFile: "mutex.pprof", collectFunc: mutexProfile, enabledFunc: func(opts Options) bool { return opts.ProfileDuration > 0 && opts.MutexProfileFraction > 0 }, }, CollectorBlock: { outputFile: "block.pprof", collectFunc: blockProfile, enabledFunc: func(opts Options) bool { return opts.ProfileDuration > 0 && opts.BlockProfileRate > 0 }, }, CollectorTrace: { outputFile: "trace", collectFunc: captureTrace, enabledFunc: func(opts Options) bool { return opts.ProfileDuration > 0 }, }, } type Options struct { Collectors []string ProfileDuration time.Duration MutexProfileFraction int BlockProfileRate time.Duration } func WriteProfiles(ctx context.Context, archive *zip.Writer, opts Options) error { p := profiler{ archive: archive, opts: opts, } return p.runProfile(ctx) } // profiler runs the collectors concurrently and writes the results to the zip archive. type profiler struct { archive *zip.Writer opts Options } func (p *profiler) runProfile(ctx context.Context) error { type profileResult struct { fName string buf *bytes.Buffer err error } ctx, cancelFn := context.WithCancel(ctx) defer cancelFn() collectorsToRun := make([]collector, len(p.opts.Collectors)) for i, name := range p.opts.Collectors { c, ok := collectors[name] if !ok { return fmt.Errorf("unknown collector '%s'", name) } collectorsToRun[i] = c } results := make(chan profileResult, len(p.opts.Collectors)) wg := sync.WaitGroup{} for _, c := range collectorsToRun { if !c.enabledFunc(p.opts) { continue } fName := c.outputFileName() wg.Add(1) go func(c collector) { defer wg.Done() logger.Infow("collecting profile", "File", fName) defer logger.Infow("profile done", "File", fName) b := bytes.Buffer{} err := c.collectFunc(ctx, p.opts, &b) if err != nil { select { case results <- profileResult{err: fmt.Errorf("generating profile data for %q: %w", fName, err)}: case <-ctx.Done(): return } } select { case results <- profileResult{buf: &b, fName: fName}: case <-ctx.Done(): } }(c) } go func() { wg.Wait() close(results) }() for res := range results { if res.err != nil { return res.err } out, err := p.archive.Create(res.fName) if err != nil { return fmt.Errorf("creating output file %q: %w", res.fName, err) } _, err = io.Copy(out, res.buf) if err != nil { return fmt.Errorf("compressing result %q: %w", res.fName, err) } } return nil } func goroutineStacksText(ctx context.Context, _ Options, w io.Writer) error { return WriteAllGoroutineStacks(w) } func goroutineStacksProto(ctx context.Context, _ Options, w io.Writer) error { return pprof.Lookup("goroutine").WriteTo(w, 0) } func heapProfile(ctx context.Context, _ Options, w io.Writer) error { return pprof.Lookup("heap").WriteTo(w, 0) } func allocsProfile(ctx context.Context, _ Options, w io.Writer) error { return pprof.Lookup("allocs").WriteTo(w, 0) } func versionInfo(ctx context.Context, _ Options, w io.Writer) error { return json.NewEncoder(w).Encode(version.GetVersionInfo()) } func binary(ctx context.Context, _ Options, w io.Writer) error { var ( path string err error ) if goos == "linux" { pid := os.Getpid() path = fmt.Sprintf("/proc/%d/exe", pid) } else { path, err = os.Executable() if err != nil { return fmt.Errorf("finding binary path: %w", err) } } fi, err := os.Open(path) if err != nil { return fmt.Errorf("opening binary %q: %w", path, err) } _, err = io.Copy(w, fi) _ = fi.Close() if err != nil { return fmt.Errorf("copying binary %q: %w", path, err) } return nil } func mutexProfile(ctx context.Context, opts Options, w io.Writer) error { prev := runtime.SetMutexProfileFraction(opts.MutexProfileFraction) defer runtime.SetMutexProfileFraction(prev) err := waitOrCancel(ctx, opts.ProfileDuration) if err != nil { return err } return pprof.Lookup("mutex").WriteTo(w, 2) } func blockProfile(ctx context.Context, opts Options, w io.Writer) error { runtime.SetBlockProfileRate(int(opts.BlockProfileRate.Nanoseconds())) defer runtime.SetBlockProfileRate(0) err := waitOrCancel(ctx, opts.ProfileDuration) if err != nil { return err } return pprof.Lookup("block").WriteTo(w, 2) } func profileCPU(ctx context.Context, opts Options, w io.Writer) error { err := pprof.StartCPUProfile(w) if err != nil { return err } defer pprof.StopCPUProfile() return waitOrCancel(ctx, opts.ProfileDuration) } func captureTrace(ctx context.Context, opts Options, w io.Writer) error { err := trace.Start(w) if err != nil { return err } defer trace.Stop() return waitOrCancel(ctx, opts.ProfileDuration) } func waitOrCancel(ctx context.Context, d time.Duration) error { timer := time.NewTimer(d) defer timer.Stop() select { case <-timer.C: return nil case <-ctx.Done(): return ctx.Err() } } ================================================ FILE: profile/profile_test.go ================================================ package profile import ( "archive/zip" "bytes" "context" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestProfiler(t *testing.T) { allCollectors := []string{ CollectorGoroutinesStack, CollectorGoroutinesPprof, CollectorVersion, CollectorHeap, CollectorAllocs, CollectorBin, CollectorCPU, CollectorMutex, CollectorBlock, CollectorTrace, } cases := []struct { name string opts Options goos string expectFiles []string }{ { name: "happy case", opts: Options{ Collectors: allCollectors, ProfileDuration: 1 * time.Millisecond, MutexProfileFraction: 4, BlockProfileRate: 50 * time.Nanosecond, }, expectFiles: []string{ "goroutines.stacks", "goroutines.pprof", "version.json", "heap.pprof", "allocs.pprof", "ipfs", "cpu.pprof", "mutex.pprof", "block.pprof", "trace", }, }, { name: "windows", opts: Options{ Collectors: allCollectors, ProfileDuration: 1 * time.Millisecond, MutexProfileFraction: 4, BlockProfileRate: 50 * time.Nanosecond, }, goos: "windows", expectFiles: []string{ "goroutines.stacks", "goroutines.pprof", "version.json", "heap.pprof", "allocs.pprof", "ipfs.exe", "cpu.pprof", "mutex.pprof", "block.pprof", "trace", }, }, { name: "sampling profiling disabled", opts: Options{ Collectors: allCollectors, MutexProfileFraction: 4, BlockProfileRate: 50 * time.Nanosecond, }, expectFiles: []string{ "goroutines.stacks", "goroutines.pprof", "version.json", "heap.pprof", "allocs.pprof", "ipfs", }, }, { name: "Mutex profiling disabled", opts: Options{ Collectors: allCollectors, ProfileDuration: 1 * time.Millisecond, BlockProfileRate: 50 * time.Nanosecond, }, expectFiles: []string{ "goroutines.stacks", "goroutines.pprof", "version.json", "heap.pprof", "allocs.pprof", "ipfs", "cpu.pprof", "block.pprof", "trace", }, }, { name: "block profiling disabled", opts: Options{ Collectors: allCollectors, ProfileDuration: 1 * time.Millisecond, MutexProfileFraction: 4, BlockProfileRate: 0, }, expectFiles: []string{ "goroutines.stacks", "goroutines.pprof", "version.json", "heap.pprof", "allocs.pprof", "ipfs", "cpu.pprof", "mutex.pprof", "trace", }, }, { name: "single collector", opts: Options{ Collectors: []string{CollectorVersion}, ProfileDuration: 1 * time.Millisecond, MutexProfileFraction: 4, BlockProfileRate: 0, }, expectFiles: []string{ "version.json", }, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { if c.goos != "" { oldGOOS := goos goos = c.goos defer func() { goos = oldGOOS }() } buf := &bytes.Buffer{} archive := zip.NewWriter(buf) err := WriteProfiles(context.Background(), archive, c.opts) require.NoError(t, err) err = archive.Close() require.NoError(t, err) zr, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) require.NoError(t, err) for _, f := range zr.File { logger.Info("zip file: ", f.Name) } require.Equal(t, len(c.expectFiles), len(zr.File)) for _, expectedFile := range c.expectFiles { func() { f, err := zr.Open(expectedFile) require.NoError(t, err) defer f.Close() fi, err := f.Stat() require.NoError(t, err) assert.NotZero(t, fi.Size()) }() } }) } } ================================================ FILE: repo/common/common.go ================================================ package common import ( "fmt" "maps" "strings" ) func MapGetKV(v map[string]any, key string) (any, error) { var ok bool var mcursor map[string]any var cursor any = v parts := strings.Split(key, ".") for i, part := range parts { sofar := strings.Join(parts[:i], ".") mcursor, ok = cursor.(map[string]any) if !ok { return nil, fmt.Errorf("%s key is not a map", sofar) } cursor, ok = mcursor[part] if !ok { // Construct the current path traversed to print a nice error message var path string if len(sofar) > 0 { path += sofar + "." } path += part return nil, fmt.Errorf("%s not found", path) } } return cursor, nil } func MapSetKV(v map[string]any, key string, value any) error { var ok bool var mcursor map[string]any var cursor any = v parts := strings.Split(key, ".") for i, part := range parts { mcursor, ok = cursor.(map[string]any) if !ok { sofar := strings.Join(parts[:i], ".") return fmt.Errorf("%s key is not a map", sofar) } // last part? set here if i == (len(parts) - 1) { mcursor[part] = value break } cursor, ok = mcursor[part] if !ok || cursor == nil { // create map if this is empty or is null mcursor[part] = map[string]any{} cursor = mcursor[part] } } return nil } // MapMergeDeep merges the right map into the left map, recursively traversing // child maps until a non-map value is found. func MapMergeDeep(left, right map[string]any) map[string]any { // We want to alter a copy of the map, not the original result := maps.Clone(left) if result == nil { result = make(map[string]any) } for key, rightVal := range right { // If right value is a map if rightMap, ok := rightVal.(map[string]any); ok { // If key is in left if leftVal, found := result[key]; found { // If left value is also a map if leftMap, ok := leftVal.(map[string]any); ok { // Merge nested map result[key] = MapMergeDeep(leftMap, rightMap) continue } } } // Otherwise set new value to result result[key] = rightVal } return result } ================================================ FILE: repo/common/common_test.go ================================================ package common import ( "testing" "github.com/stretchr/testify/require" ) func TestMapMergeDeepReturnsNew(t *testing.T) { leftMap := make(map[string]any) leftMap["A"] = "Hello World" rightMap := make(map[string]any) rightMap["A"] = "Foo" MapMergeDeep(leftMap, rightMap) require.Equal(t, "Hello World", leftMap["A"], "MapMergeDeep should return a new map instance") } func TestMapMergeDeepNewKey(t *testing.T) { leftMap := make(map[string]any) leftMap["A"] = "Hello World" /* leftMap { A: "Hello World" } */ rightMap := make(map[string]any) rightMap["B"] = "Bar" /* rightMap { B: "Bar" } */ result := MapMergeDeep(leftMap, rightMap) /* expected { A: "Hello World" B: "Bar" } */ require.Equal(t, "Bar", result["B"], "New keys in right map should exist in resulting map") } func TestMapMergeDeepRecursesOnMaps(t *testing.T) { leftMapA := make(map[string]any) leftMapA["B"] = "A value!" leftMapA["C"] = "Another value!" leftMap := make(map[string]any) leftMap["A"] = leftMapA /* leftMap { A: { B: "A value!" C: "Another value!" } } */ rightMapA := make(map[string]any) rightMapA["C"] = "A different value!" rightMap := make(map[string]any) rightMap["A"] = rightMapA /* rightMap { A: { C: "A different value!" } } */ result := MapMergeDeep(leftMap, rightMap) /* expected { A: { B: "A value!" C: "A different value!" } } */ resultA := result["A"].(map[string]any) require.Equal(t, "A value!", resultA["B"], "Unaltered values should not change") require.Equal(t, "A different value!", resultA["C"], "Nested values should be altered") } func TestMapMergeDeepRightNotAMap(t *testing.T) { leftMapA := make(map[string]any) leftMapA["B"] = "A value!" leftMap := make(map[string]any) leftMap["A"] = leftMapA /* origMap { A: { B: "A value!" } } */ rightMap := make(map[string]any) rightMap["A"] = "Not a map!" /* newMap { A: "Not a map!" } */ result := MapMergeDeep(leftMap, rightMap) /* expected { A: "Not a map!" } */ require.Equal(t, "Not a map!", result["A"], "Right values that are not a map should be set on the result") } ================================================ FILE: repo/fsrepo/config_test.go ================================================ package fsrepo_test import ( "encoding/json" "reflect" "testing" "github.com/ipfs/kubo/plugin/loader" "github.com/ipfs/kubo/repo/fsrepo" "github.com/ipfs/kubo/config" ) // note: to test sorting of the mountpoints in the disk spec they are // specified out of order in the test config. var defaultConfig = []byte(`{ "StorageMax": "10GB", "StorageGCWatermark": 90, "GCPeriod": "1h", "Spec": { "mounts": [ { "child": { "compression": "none", "path": "datastore", "type": "levelds" }, "mountpoint": "/", "prefix": "leveldb.datastore", "type": "measure" }, { "child": { "path": "blocks", "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2", "sync": true, "type": "flatfs" }, "mountpoint": "/blocks", "prefix": "flatfs.datastore", "type": "measure" } ], "type": "mount" }, "HashOnRead": false, "BloomFilterSize": 0 }`) var leveldbConfig = []byte(`{ "compression": "none", "path": "datastore", "type": "levelds" }`) var flatfsConfig = []byte(`{ "path": "blocks", "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2", "sync": true, "type": "flatfs" }`) var measureConfig = []byte(`{ "child": { "path": "blocks", "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2", "sync": true, "type": "flatfs" }, "mountpoint": "/blocks", "prefix": "flatfs.datastore", "type": "measure" }`) func TestDefaultDatastoreConfig(t *testing.T) { loader, err := loader.NewPluginLoader("") if err != nil { t.Fatal(err) } err = loader.Initialize() if err != nil { t.Fatal(err) } err = loader.Inject() if err != nil { t.Fatal(err) } dir := t.TempDir() config := new(config.Datastore) err = json.Unmarshal(defaultConfig, config) if err != nil { t.Fatal(err) } dsc, err := fsrepo.AnyDatastoreConfig(config.Spec) if err != nil { t.Fatal(err) } expected := `{"mounts":[{"mountpoint":"/blocks","path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"},{"mountpoint":"/","path":"datastore","type":"levelds"}],"type":"mount"}` if dsc.DiskSpec().String() != expected { t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String()) } ds, err := dsc.Create(dir) if err != nil { t.Fatal(err) } if typ := reflect.TypeOf(ds).String(); typ != "*mount.Datastore" { t.Errorf("expected '*mount.Datastore' got '%s'", typ) } } func TestLevelDbConfig(t *testing.T) { config := new(config.Datastore) err := json.Unmarshal(defaultConfig, config) if err != nil { t.Fatal(err) } dir := t.TempDir() spec := make(map[string]any) err = json.Unmarshal(leveldbConfig, &spec) if err != nil { t.Fatal(err) } dsc, err := fsrepo.AnyDatastoreConfig(spec) if err != nil { t.Fatal(err) } expected := `{"path":"datastore","type":"levelds"}` if dsc.DiskSpec().String() != expected { t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String()) } ds, err := dsc.Create(dir) if err != nil { t.Fatal(err) } if typ := reflect.TypeOf(ds).String(); typ != "*leveldb.Datastore" { t.Errorf("expected '*leveldb.datastore' got '%s'", typ) } } func TestFlatfsConfig(t *testing.T) { config := new(config.Datastore) err := json.Unmarshal(defaultConfig, config) if err != nil { t.Fatal(err) } dir := t.TempDir() spec := make(map[string]any) err = json.Unmarshal(flatfsConfig, &spec) if err != nil { t.Fatal(err) } dsc, err := fsrepo.AnyDatastoreConfig(spec) if err != nil { t.Fatal(err) } expected := `{"path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"}` if dsc.DiskSpec().String() != expected { t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String()) } ds, err := dsc.Create(dir) if err != nil { t.Fatal(err) } if typ := reflect.TypeOf(ds).String(); typ != "*flatfs.Datastore" { t.Errorf("expected '*flatfs.Datastore' got '%s'", typ) } } func TestMeasureConfig(t *testing.T) { config := new(config.Datastore) err := json.Unmarshal(defaultConfig, config) if err != nil { t.Fatal(err) } dir := t.TempDir() spec := make(map[string]any) err = json.Unmarshal(measureConfig, &spec) if err != nil { t.Fatal(err) } dsc, err := fsrepo.AnyDatastoreConfig(spec) if err != nil { t.Fatal(err) } expected := `{"path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"}` if dsc.DiskSpec().String() != expected { t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String()) } ds, err := dsc.Create(dir) if err != nil { t.Fatal(err) } if typ := reflect.TypeOf(ds).String(); typ != "*measure.measure" { t.Errorf("expected '*measure.measure' got '%s'", typ) } } ================================================ FILE: repo/fsrepo/datastores.go ================================================ package fsrepo import ( "bytes" "encoding/json" "fmt" "sort" "github.com/ipfs/kubo/repo" ds "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/mount" dssync "github.com/ipfs/go-datastore/sync" "github.com/ipfs/go-ds-measure" ) // ConfigFromMap creates a new datastore config from a map. type ConfigFromMap func(map[string]any) (DatastoreConfig, error) // DatastoreConfig is an abstraction of a datastore config. A "spec" is first // converted to a DatastoreConfig and then Create() is called to instantiate a // new datastore. type DatastoreConfig interface { // DiskSpec returns a minimal configuration of the datastore representing // what is stored on disk. Run time values are excluded. DiskSpec() DiskSpec // Create instantiates a new datastore from this config. Create(path string) (repo.Datastore, error) } // DiskSpec is a minimal representation of the characteristic values of the // datastore. If two diskspecs are the same, the loader assumes that they refer // to exactly the same datastore. If they differ at all, it is assumed they are // completely different datastores and a migration will be performed. Runtime // values such as cache options or concurrency options should not be added // here. type DiskSpec map[string]any // Bytes returns a minimal JSON encoding of the DiskSpec. func (spec DiskSpec) Bytes() []byte { b, err := json.Marshal(spec) if err != nil { // should not happen panic(err) } return bytes.TrimSpace(b) } // String returns a minimal JSON encoding of the DiskSpec. func (spec DiskSpec) String() string { return string(spec.Bytes()) } var datastores map[string]ConfigFromMap func init() { datastores = map[string]ConfigFromMap{ "mount": MountDatastoreConfig, "mem": MemDatastoreConfig, "log": LogDatastoreConfig, "measure": MeasureDatastoreConfig, } } func AddDatastoreConfigHandler(name string, dsc ConfigFromMap) error { _, ok := datastores[name] if ok { return fmt.Errorf("already have a datastore named %q", name) } datastores[name] = dsc return nil } // AnyDatastoreConfig returns a DatastoreConfig from a spec based on // the "type" parameter. func AnyDatastoreConfig(params map[string]any) (DatastoreConfig, error) { which, ok := params["type"].(string) if !ok { return nil, fmt.Errorf("'type' field missing or not a string") } fun, ok := datastores[which] if !ok { return nil, fmt.Errorf("unknown datastore type: %s", which) } return fun(params) } type mountDatastoreConfig struct { mounts []premount } type premount struct { ds DatastoreConfig prefix ds.Key } // MountDatastoreConfig returns a mount DatastoreConfig from a spec. func MountDatastoreConfig(params map[string]any) (DatastoreConfig, error) { var res mountDatastoreConfig mounts, ok := params["mounts"].([]any) if !ok { return nil, fmt.Errorf("'mounts' field is missing or not an array") } for _, iface := range mounts { cfg, ok := iface.(map[string]any) if !ok { return nil, fmt.Errorf("expected map for mountpoint") } child, err := AnyDatastoreConfig(cfg) if err != nil { return nil, err } prefix, found := cfg["mountpoint"] if !found { return nil, fmt.Errorf("no 'mountpoint' on mount") } res.mounts = append(res.mounts, premount{ ds: child, prefix: ds.NewKey(prefix.(string)), }) } sort.Slice(res.mounts, func(i, j int) bool { return res.mounts[i].prefix.String() > res.mounts[j].prefix.String() }) return &res, nil } func (c *mountDatastoreConfig) DiskSpec() DiskSpec { cfg := map[string]any{"type": "mount"} mounts := make([]any, len(c.mounts)) for i, m := range c.mounts { c := m.ds.DiskSpec() if c == nil { c = make(map[string]any) } c["mountpoint"] = m.prefix.String() mounts[i] = c } cfg["mounts"] = mounts return cfg } func (c *mountDatastoreConfig) Create(path string) (repo.Datastore, error) { mounts := make([]mount.Mount, len(c.mounts)) for i, m := range c.mounts { ds, err := m.ds.Create(path) if err != nil { return nil, err } mounts[i].Datastore = ds mounts[i].Prefix = m.prefix } return mount.New(mounts), nil } type memDatastoreConfig struct { cfg map[string]any } // MemDatastoreConfig returns a memory DatastoreConfig from a spec. func MemDatastoreConfig(params map[string]any) (DatastoreConfig, error) { return &memDatastoreConfig{params}, nil } func (c *memDatastoreConfig) DiskSpec() DiskSpec { return nil } func (c *memDatastoreConfig) Create(string) (repo.Datastore, error) { return dssync.MutexWrap(ds.NewMapDatastore()), nil } type logDatastoreConfig struct { child DatastoreConfig name string } // LogDatastoreConfig returns a log DatastoreConfig from a spec. func LogDatastoreConfig(params map[string]any) (DatastoreConfig, error) { childField, ok := params["child"].(map[string]any) if !ok { return nil, fmt.Errorf("'child' field is missing or not a map") } child, err := AnyDatastoreConfig(childField) if err != nil { return nil, err } name, ok := params["name"].(string) if !ok { return nil, fmt.Errorf("'name' field was missing or not a string") } return &logDatastoreConfig{child, name}, nil } func (c *logDatastoreConfig) Create(path string) (repo.Datastore, error) { child, err := c.child.Create(path) if err != nil { return nil, err } return ds.NewLogDatastore(child, c.name), nil } func (c *logDatastoreConfig) DiskSpec() DiskSpec { return c.child.DiskSpec() } type measureDatastoreConfig struct { child DatastoreConfig prefix string } // MeasureDatastoreConfig returns a measure DatastoreConfig from a spec. func MeasureDatastoreConfig(params map[string]any) (DatastoreConfig, error) { childField, ok := params["child"].(map[string]any) if !ok { return nil, fmt.Errorf("'child' field is missing or not a map") } child, err := AnyDatastoreConfig(childField) if err != nil { return nil, err } prefix, ok := params["prefix"].(string) if !ok { return nil, fmt.Errorf("'prefix' field was missing or not a string") } return &measureDatastoreConfig{child, prefix}, nil } func (c *measureDatastoreConfig) DiskSpec() DiskSpec { return c.child.DiskSpec() } func (c measureDatastoreConfig) Create(path string) (repo.Datastore, error) { child, err := c.child.Create(path) if err != nil { return nil, err } return measure.New(c.prefix, child), nil } ================================================ FILE: repo/fsrepo/doc.go ================================================ // package fsrepo // // TODO explain the package roadmap... // // .ipfs/ // ├── client/ // | ├── client.lock <------ protects client/ + signals its own pid // │ ├── ipfs-client.cpuprof // │ └── ipfs-client.memprof // ├── config // ├── daemon/ // │ ├── daemon.lock <------ protects daemon/ + signals its own address // │ ├── ipfs-daemon.cpuprof // │ └── ipfs-daemon.memprof // ├── datastore/ // ├── repo.lock <------ protects datastore/ and config // └── version package fsrepo // TODO prevent multiple daemons from running ================================================ FILE: repo/fsrepo/fsrepo.go ================================================ package fsrepo import ( "context" "errors" "fmt" "io" "net" "os" "path/filepath" "strings" "sync" "time" filestore "github.com/ipfs/boxo/filestore" keystore "github.com/ipfs/boxo/keystore" version "github.com/ipfs/kubo" repo "github.com/ipfs/kubo/repo" "github.com/ipfs/kubo/repo/common" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" ds "github.com/ipfs/go-datastore" measure "github.com/ipfs/go-ds-measure" lockfile "github.com/ipfs/go-fs-lock" logging "github.com/ipfs/go-log/v2" config "github.com/ipfs/kubo/config" serialize "github.com/ipfs/kubo/config/serialize" "github.com/ipfs/kubo/misc/fsutil" "github.com/ipfs/kubo/repo/fsrepo/migrations" ma "github.com/multiformats/go-multiaddr" ) // LockFile is the filename of the repo lock, relative to config dir // TODO rename repo lock and hide name. const LockFile = "repo.lock" var log = logging.Logger("fsrepo") // RepoVersion is the version number that we are currently expecting to see. var RepoVersion = version.RepoVersion var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md Sorry for the inconvenience. In the future, these will run automatically.` var programTooLowMessage = `Your programs version (%d) is lower than your repos (%d). Please update ipfs to a version that supports the existing repo, or run a migration in reverse. See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md for details.` var ( ErrNoVersion = errors.New("no version file found, please run 0-to-1 migration tool.\n" + migrationInstructions) ErrOldRepo = errors.New("ipfs repo found in old '~/.go-ipfs' location, please run migration tool.\n" + migrationInstructions) ErrNeedMigration = errors.New("ipfs repo needs migration, please run migration tool.\n" + migrationInstructions) ) type NoRepoError struct { Path string } var _ error = NoRepoError{} func (err NoRepoError) Error() string { return fmt.Sprintf("no IPFS repo found in %s.\nplease run: 'ipfs init'", err.Path) } const ( apiFile = "api" gatewayFile = "gateway" swarmKeyFile = "swarm.key" ) const specFn = "datastore_spec" var ( // packageLock must be held to while performing any operation that modifies an // FSRepo's state field. This includes Init, Open, Close, and Remove. packageLock sync.Mutex // onlyOne keeps track of open FSRepo instances. // // TODO: once command Context / Repo integration is cleaned up, // this can be removed. Right now, this makes ConfigCmd.Run // function try to open the repo twice: // // $ ipfs daemon & // $ ipfs config foo // // The reason for the above is that in standalone mode without the // daemon, `ipfs config` tries to save work by not building the // full IpfsNode, but accessing the Repo directly. onlyOne repo.OnlyOne ) // FSRepo represents an IPFS FileSystem Repo. It is safe for use by multiple // callers. type FSRepo struct { // has Close been called already closed bool // path is the file-system path path string // Path to the configuration file that may or may not be inside the FSRepo // path (see config.Filename for more details). configFilePath string // lockfile is the file system lock to prevent others from opening // the same fsrepo path concurrently lockfile io.Closer config *config.Config userResourceOverrides rcmgr.PartialLimitConfig ds repo.Datastore keystore keystore.Keystore filemgr *filestore.FileManager } var _ repo.Repo = (*FSRepo)(nil) // Open the FSRepo at path. Returns an error if the repo is not // initialized. func Open(repoPath string) (repo.Repo, error) { fn := func() (repo.Repo, error) { return open(repoPath, "") } return onlyOne.Open(repoPath, fn) } // OpenWithUserConfig is the equivalent to the Open function above but with the // option to set the configuration file path instead of using the default. func OpenWithUserConfig(repoPath string, userConfigFilePath string) (repo.Repo, error) { fn := func() (repo.Repo, error) { return open(repoPath, userConfigFilePath) } return onlyOne.Open(repoPath, fn) } func open(repoPath string, userConfigFilePath string) (repo.Repo, error) { packageLock.Lock() defer packageLock.Unlock() r, err := newFSRepo(repoPath, userConfigFilePath) if err != nil { return nil, err } // Check if its initialized if err := checkInitialized(r.path); err != nil { return nil, err } text := os.Getenv("IPFS_WAIT_REPO_LOCK") if text != "" { var lockWaitTime time.Duration lockWaitTime, err = time.ParseDuration(text) if err != nil { log.Errorw("Cannot parse value of IPFS_WAIT_REPO_LOCK as duration, not waiting for repo lock", "err", err, "value", text) r.lockfile, err = lockfile.Lock(r.path, LockFile) } else if lockWaitTime <= 0 { r.lockfile, err = lockfile.WaitLock(context.Background(), r.path, LockFile) } else { ctx, cancel := context.WithTimeout(context.Background(), lockWaitTime) r.lockfile, err = lockfile.WaitLock(ctx, r.path, LockFile) cancel() } } else { r.lockfile, err = lockfile.Lock(r.path, LockFile) } if err != nil { return nil, err } keepLocked := false defer func() { // unlock on error, leave it locked on success if !keepLocked { r.lockfile.Close() } }() // Check version, and error out if not matching ver, err := migrations.RepoVersion(r.path) if err != nil { if os.IsNotExist(err) { return nil, ErrNoVersion } return nil, err } if RepoVersion > ver { return nil, ErrNeedMigration } else if ver > RepoVersion { // program version too low for existing repo return nil, fmt.Errorf(programTooLowMessage, RepoVersion, ver) } // check repo path, then check all constituent parts. if err := fsutil.DirWritable(r.path); err != nil { return nil, err } if err := r.openConfig(); err != nil { return nil, err } if err := r.openUserResourceOverrides(); err != nil { return nil, err } if err := r.openDatastore(); err != nil { return nil, err } if err := r.openKeystore(); err != nil { return nil, err } if r.config.Experimental.FilestoreEnabled || r.config.Experimental.UrlstoreEnabled { r.filemgr = filestore.NewFileManager(r.ds, filepath.Dir(r.path)) r.filemgr.AllowFiles = r.config.Experimental.FilestoreEnabled r.filemgr.AllowUrls = r.config.Experimental.UrlstoreEnabled } keepLocked = true return r, nil } func newFSRepo(rpath string, userConfigFilePath string) (*FSRepo, error) { expPath, err := fsutil.ExpandHome(filepath.Clean(rpath)) if err != nil { return nil, err } configFilePath, err := config.Filename(rpath, userConfigFilePath) if err != nil { // FIXME: Personalize this when the user config path is "". return nil, fmt.Errorf("finding config filepath from repo %s and user config %s: %w", rpath, userConfigFilePath, err) } return &FSRepo{path: expPath, configFilePath: configFilePath}, nil } func checkInitialized(path string) error { if !isInitializedUnsynced(path) { alt := strings.Replace(path, ".ipfs", ".go-ipfs", 1) if isInitializedUnsynced(alt) { return ErrOldRepo } return NoRepoError{Path: path} } return nil } // configIsInitialized returns true if the repo is initialized at // provided |path|. func configIsInitialized(path string) bool { configFilename, err := config.Filename(path, "") if err != nil { return false } if !fsutil.FileExists(configFilename) { return false } return true } func initConfig(path string, conf *config.Config) error { if configIsInitialized(path) { return nil } configFilename, err := config.Filename(path, "") if err != nil { return err } // initialization is the one time when it's okay to write to the config // without reading the config from disk and merging any user-provided keys // that may exist. if err := serialize.WriteConfigFile(configFilename, conf); err != nil { return err } return nil } func initSpec(path string, conf map[string]any) error { fn, err := config.Path(path, specFn) if err != nil { return err } if fsutil.FileExists(fn) { return nil } dsc, err := AnyDatastoreConfig(conf) if err != nil { return err } bytes := dsc.DiskSpec().Bytes() return os.WriteFile(fn, bytes, 0o600) } // Init initializes a new FSRepo at the given path with the provided config. // TODO add support for custom datastores. func Init(repoPath string, conf *config.Config) error { // packageLock must be held to ensure that the repo is not initialized more // than once. packageLock.Lock() defer packageLock.Unlock() if isInitializedUnsynced(repoPath) { return nil } if err := initConfig(repoPath, conf); err != nil { return err } if err := initSpec(repoPath, conf.Datastore.Spec); err != nil { return err } if err := migrations.WriteRepoVersion(repoPath, RepoVersion); err != nil { return err } return nil } // LockedByOtherProcess returns true if the FSRepo is locked by another // process. If true, then the repo cannot be opened by this process. func LockedByOtherProcess(repoPath string) (bool, error) { repoPath = filepath.Clean(repoPath) locked, err := lockfile.Locked(repoPath, LockFile) if locked { log.Debugf("(%t)<->Lock is held at %s", locked, repoPath) } return locked, err } // APIAddr returns the registered API addr, according to the api file // in the fsrepo. This is a concurrent operation, meaning that any // process may read this file. modifying this file, therefore, should // use "mv" to replace the whole file and avoid interleaved read/writes. func APIAddr(repoPath string) (ma.Multiaddr, error) { repoPath = filepath.Clean(repoPath) apiFilePath := filepath.Join(repoPath, apiFile) // if there is no file, assume there is no api addr. f, err := os.Open(apiFilePath) if err != nil { if os.IsNotExist(err) { return nil, repo.ErrApiNotRunning } return nil, err } defer f.Close() // read up to 2048 bytes. io.ReadAll is a vulnerability, as // someone could hose the process by putting a massive file there. // // NOTE(@stebalien): @jbenet probably wasn't thinking straight when he // wrote that comment but I'm leaving the limit here in case there was // some hidden wisdom. However, I'm fixing it such that: // 1. We don't read too little. // 2. We don't truncate and succeed. buf, err := io.ReadAll(io.LimitReader(f, 2048)) if err != nil { return nil, err } if len(buf) == 2048 { return nil, fmt.Errorf("API file too large, must be <2048 bytes long: %s", apiFilePath) } s := string(buf) s = strings.TrimSpace(s) return ma.NewMultiaddr(s) } func (r *FSRepo) Keystore() keystore.Keystore { return r.keystore } func (r *FSRepo) Path() string { return r.path } // SetAPIAddr writes the API Addr to the /api file. func (r *FSRepo) SetAPIAddr(addr ma.Multiaddr) error { // Create a temp file to write the address, so that we don't leave empty file when the // program crashes after creating the file. f, err := os.Create(filepath.Join(r.path, "."+apiFile+".tmp")) if err != nil { return err } if _, err = f.WriteString(addr.String()); err != nil { f.Close() return err } if err = f.Close(); err != nil { return err } // Atomically rename the temp file to the correct file name. if err = os.Rename(filepath.Join(r.path, "."+apiFile+".tmp"), filepath.Join(r.path, apiFile)); err == nil { return nil } // Remove the temp file when rename return error if err1 := os.Remove(filepath.Join(r.path, "."+apiFile+".tmp")); err1 != nil { return fmt.Errorf("file Rename error: %s, file remove error: %s", err.Error(), err1.Error()) } return err } // SetGatewayAddr writes the Gateway Addr to the /gateway file. func (r *FSRepo) SetGatewayAddr(addr net.Addr) error { // Create a temp file to write the address, so that we don't leave empty file when the // program crashes after creating the file. tmpPath := filepath.Join(r.path, "."+gatewayFile+".tmp") f, err := os.Create(tmpPath) if err != nil { return err } var good bool // Silently remove as worst last case with defers. defer func() { if !good { os.Remove(tmpPath) } }() defer f.Close() if _, err := fmt.Fprintf(f, "http://%s", addr.String()); err != nil { return err } if err := f.Close(); err != nil { return err } // Atomically rename the temp file to the correct file name. err = os.Rename(tmpPath, filepath.Join(r.path, gatewayFile)) good = err == nil if good { return nil } // Remove the temp file when rename return error if err1 := os.Remove(tmpPath); err1 != nil { return fmt.Errorf("file Rename error: %w, file remove error: %s", err, err1.Error()) } return err } // openConfig returns an error if the config file is not present. func (r *FSRepo) openConfig() error { conf, err := serialize.Load(r.configFilePath) if err != nil { return err } r.config = conf return nil } // openUserResourceOverrides will remove all overrides if the file is not present. // It will error if the decoding fails. func (r *FSRepo) openUserResourceOverrides() error { // This filepath is documented in docs/libp2p-resource-management.md and be kept in sync. err := serialize.ReadConfigFile(filepath.Join(r.path, "libp2p-resource-limit-overrides.json"), &r.userResourceOverrides) if errors.Is(err, serialize.ErrNotInitialized) { err = nil } return err } func (r *FSRepo) openKeystore() error { ksp := filepath.Join(r.path, "keystore") ks, err := keystore.NewFSKeystore(ksp) if err != nil { return err } r.keystore = ks return nil } // openDatastore returns an error if the config file is not present. func (r *FSRepo) openDatastore() error { if r.config.Datastore.Type != "" || r.config.Datastore.Path != "" { return fmt.Errorf("old style datatstore config detected") } else if r.config.Datastore.Spec == nil { return fmt.Errorf("required Datastore.Spec entry missing from config file") } if r.config.Datastore.NoSync { log.Warn("NoSync is now deprecated in favor of datastore specific settings. If you want to disable fsync on flatfs set 'sync' to false. See https://github.com/ipfs/kubo/blob/master/docs/datastores.md#flatfs.") } dsc, err := AnyDatastoreConfig(r.config.Datastore.Spec) if err != nil { return err } spec := dsc.DiskSpec() oldSpec, err := r.readSpec() if err != nil { return err } if oldSpec != spec.String() { return fmt.Errorf("datastore configuration of '%s' does not match what is on disk '%s'", oldSpec, spec.String()) } d, err := dsc.Create(r.path) if err != nil { return err } r.ds = d // Wrap it with metrics gathering prefix := "ipfs.fsrepo.datastore" r.ds = measure.New(prefix, r.ds) return nil } func (r *FSRepo) readSpec() (string, error) { fn, err := config.Path(r.path, specFn) if err != nil { return "", err } b, err := os.ReadFile(fn) if err != nil { return "", err } return strings.TrimSpace(string(b)), nil } // Close closes the FSRepo, releasing held resources. func (r *FSRepo) Close() error { packageLock.Lock() defer packageLock.Unlock() if r.closed { return errors.New("repo is closed") } err := os.Remove(filepath.Join(r.path, apiFile)) if err != nil && !os.IsNotExist(err) { log.Warn("error removing api file: ", err) } err = os.Remove(filepath.Join(r.path, gatewayFile)) if err != nil && !os.IsNotExist(err) { log.Warn("error removing gateway file: ", err) } if err := r.ds.Close(); err != nil { return err } // This code existed in the previous versions, but // EventlogComponent.Close was never called. Preserving here // pending further discussion. // // TODO It isn't part of the current contract, but callers may like for us // to disable logging once the component is closed. // logging.Configure(logging.Output(os.Stderr)) r.closed = true return r.lockfile.Close() } // Config the current config. This function DOES NOT copy the config. The caller // MUST NOT modify it without first calling `Clone`. // // Result when not Open is undefined. The method may panic if it pleases. func (r *FSRepo) Config() (*config.Config, error) { // It is not necessary to hold the package lock since the repo is in an // opened state. The package lock is _not_ meant to ensure that the repo is // thread-safe. The package lock is only meant to guard against removal and // coordinate the lockfile. However, we provide thread-safety to keep // things simple. packageLock.Lock() defer packageLock.Unlock() if r.closed { return nil, errors.New("cannot access config, repo not open") } return r.config, nil } func (r *FSRepo) UserResourceOverrides() (rcmgr.PartialLimitConfig, error) { // It is not necessary to hold the package lock since the repo is in an // opened state. The package lock is _not_ meant to ensure that the repo is // thread-safe. The package lock is only meant to guard against removal and // coordinate the lockfile. However, we provide thread-safety to keep // things simple. packageLock.Lock() defer packageLock.Unlock() if r.closed { return rcmgr.PartialLimitConfig{}, errors.New("cannot access config, repo not open") } return r.userResourceOverrides, nil } func (r *FSRepo) FileManager() *filestore.FileManager { return r.filemgr } func (r *FSRepo) BackupConfig(prefix string) (string, error) { temp, err := os.CreateTemp(r.path, "config-"+prefix) if err != nil { return "", err } defer temp.Close() orig, err := os.OpenFile(r.configFilePath, os.O_RDONLY, 0o600) if err != nil { return "", err } defer orig.Close() _, err = io.Copy(temp, orig) if err != nil { return "", err } return orig.Name(), nil } // SetConfig updates the FSRepo's config. The user must not modify the config // object after calling this method. // FIXME: There is an inherent contradiction with storing non-user-generated // Go config.Config structures as user-generated JSON nested maps. This is // evidenced by the issue of `omitempty` property of fields that aren't defined // by the user and Go still needs to initialize them to its default (which // is not reflected in the repo's config file, see // https://github.com/ipfs/kubo/issues/8088 for more details). // In general we should call this API with a JSON nested maps as argument // (`map[string]interface{}`). Many calls to this function are forced to // synthesize the config.Config struct from their available JSON map just to // satisfy this (causing incompatibilities like the `omitempty` one above). // We need to comb SetConfig calls and replace them when possible with a // JSON map variant. func (r *FSRepo) SetConfig(updated *config.Config) error { // packageLock is held to provide thread-safety. packageLock.Lock() defer packageLock.Unlock() // to avoid clobbering user-provided keys, must read the config from disk // as a map, write the updated struct values to the map and write the map // to disk. var mapconf map[string]any if err := serialize.ReadConfigFile(r.configFilePath, &mapconf); err != nil { return err } m, err := config.ToMap(updated) if err != nil { return err } mergedMap := common.MapMergeDeep(mapconf, m) if err := serialize.WriteConfigFile(r.configFilePath, mergedMap); err != nil { return err } // Do not use `*r.config = ...`. This will modify the *shared* config // returned by `r.Config`. r.config = updated return nil } // GetConfigKey retrieves only the value of a particular key. func (r *FSRepo) GetConfigKey(key string) (any, error) { packageLock.Lock() defer packageLock.Unlock() if r.closed { return nil, errors.New("repo is closed") } var cfg map[string]any if err := serialize.ReadConfigFile(r.configFilePath, &cfg); err != nil { return nil, err } return common.MapGetKV(cfg, key) } // SetConfigKey writes the value of a particular key. func (r *FSRepo) SetConfigKey(key string, value any) error { packageLock.Lock() defer packageLock.Unlock() if r.closed { return errors.New("repo is closed") } // Validate the key's presence in the config structure. err := config.CheckKey(key) if err != nil { return err } // Load into a map so we don't end up writing any additional defaults to the config file. var mapconf map[string]any if err := serialize.ReadConfigFile(r.configFilePath, &mapconf); err != nil { return err } // Load private key to guard against it being overwritten. // NOTE: this is a temporary measure to secure this field until we move // keys out of the config file. pkval, err := common.MapGetKV(mapconf, config.PrivKeySelector) if err != nil { return err } // Set the key in the map. if err := common.MapSetKV(mapconf, key, value); err != nil { return err } // replace private key, in case it was overwritten. if err := common.MapSetKV(mapconf, config.PrivKeySelector, pkval); err != nil { return err } // This step doubles as to validate the map against the struct // before serialization conf, err := config.FromMap(mapconf) if err != nil { return err } r.config = conf if err := serialize.WriteConfigFile(r.configFilePath, mapconf); err != nil { return err } return nil } // Datastore returns a repo-owned datastore. If FSRepo is Closed, return value // is undefined. func (r *FSRepo) Datastore() repo.Datastore { packageLock.Lock() d := r.ds packageLock.Unlock() return d } // GetStorageUsage computes the storage space taken by the repo in bytes. func (r *FSRepo) GetStorageUsage(ctx context.Context) (uint64, error) { return ds.DiskUsage(ctx, r.Datastore()) } func (r *FSRepo) SwarmKey() ([]byte, error) { repoPath := filepath.Clean(r.path) spath := filepath.Join(repoPath, swarmKeyFile) f, err := os.Open(spath) if err != nil { if os.IsNotExist(err) { err = nil } return nil, err } defer f.Close() return io.ReadAll(f) } var ( _ io.Closer = &FSRepo{} _ repo.Repo = &FSRepo{} ) // IsInitialized returns true if the repo is initialized at provided |path|. func IsInitialized(path string) bool { // packageLock is held to ensure that another caller doesn't attempt to // Init or Remove the repo while this call is in progress. packageLock.Lock() defer packageLock.Unlock() return isInitializedUnsynced(path) } // private methods below this point. NB: packageLock must held by caller. // isInitializedUnsynced reports whether the repo is initialized. Caller must // hold the packageLock. func isInitializedUnsynced(repoPath string) bool { return configIsInitialized(repoPath) } ================================================ FILE: repo/fsrepo/fsrepo_test.go ================================================ package fsrepo import ( "bytes" "context" "os" "path/filepath" "testing" datastore "github.com/ipfs/go-datastore" config "github.com/ipfs/kubo/config" "github.com/stretchr/testify/require" ) func TestInitIdempotence(t *testing.T) { t.Parallel() path := t.TempDir() for range 10 { require.NoError(t, Init(path, &config.Config{Datastore: config.DefaultDatastoreConfig()}), "multiple calls to init should succeed") } } func Remove(repoPath string) error { repoPath = filepath.Clean(repoPath) return os.RemoveAll(repoPath) } func TestCanManageReposIndependently(t *testing.T) { t.Parallel() pathA := t.TempDir() pathB := t.TempDir() t.Log("initialize two repos") require.NoError(t, Init(pathA, &config.Config{Datastore: config.DefaultDatastoreConfig()}), "a", "should initialize successfully") require.NoError(t, Init(pathB, &config.Config{Datastore: config.DefaultDatastoreConfig()}), "b", "should initialize successfully") t.Log("ensure repos initialized") require.True(t, IsInitialized(pathA), "a should be initialized") require.True(t, IsInitialized(pathB), "b should be initialized") t.Log("open the two repos") repoA, err := Open(pathA) require.NoError(t, err, "a") repoB, err := Open(pathB) require.NoError(t, err, "b") t.Log("close and remove b while a is open") require.NoError(t, repoB.Close(), "close b") require.NoError(t, Remove(pathB), "remove b") t.Log("close and remove a") require.NoError(t, repoA.Close()) require.NoError(t, Remove(pathA)) } func TestDatastoreGetNotAllowedAfterClose(t *testing.T) { t.Parallel() path := t.TempDir() require.False(t, IsInitialized(path), "should NOT be initialized") require.NoError(t, Init(path, &config.Config{Datastore: config.DefaultDatastoreConfig()}), "should initialize successfully") r, err := Open(path) require.NoError(t, err, "should open successfully") k := "key" data := []byte(k) require.NoError(t, r.Datastore().Put(context.Background(), datastore.NewKey(k), data), "Put should be successful") require.NoError(t, r.Close()) _, err = r.Datastore().Get(context.Background(), datastore.NewKey(k)) require.Error(t, err, "after closer, Get should be fail") } func TestDatastorePersistsFromRepoToRepo(t *testing.T) { t.Parallel() path := t.TempDir() require.NoError(t, Init(path, &config.Config{Datastore: config.DefaultDatastoreConfig()})) r1, err := Open(path) require.NoError(t, err) k := "key" expected := []byte(k) require.NoError(t, r1.Datastore().Put(context.Background(), datastore.NewKey(k), expected), "using first repo, Put should be successful") require.NoError(t, r1.Close()) r2, err := Open(path) require.NoError(t, err) actual, err := r2.Datastore().Get(context.Background(), datastore.NewKey(k)) require.NoError(t, err, "using second repo, Get should be successful") require.NoError(t, r2.Close()) require.True(t, bytes.Equal(expected, actual), "data should match") } func TestOpenMoreThanOnceInSameProcess(t *testing.T) { t.Parallel() path := t.TempDir() require.NoError(t, Init(path, &config.Config{Datastore: config.DefaultDatastoreConfig()})) r1, err := Open(path) require.NoError(t, err, "first repo should open successfully") r2, err := Open(path) require.NoError(t, err, "second repo should open successfully") require.Equal(t, r1, r2, "second open returns same value") require.NoError(t, r1.Close()) require.NoError(t, r2.Close()) } ================================================ FILE: repo/fsrepo/migrations/README.md ================================================ # IPFS Repository Migrations This directory contains the migration system for IPFS repositories, handling both embedded and external migrations. ## Migration System Overview ### Embedded vs External Migrations Starting from **repo version 17**, Kubo uses **embedded migrations** that are built into the binary, eliminating the need to download external migration tools. - **Repo versions <17**: Use external binary migrations downloaded from fs-repo-migrations - **Repo version 17+**: Use embedded migrations built into Kubo ### Migration Functions #### `migrations.RunEmbeddedMigrations()` - **Purpose**: Runs migrations that are embedded directly in the Kubo binary - **Scope**: Handles repo version 17+ migrations - **Performance**: Fast execution, no network downloads required - **Dependencies**: Self-contained, uses only Kubo's internal dependencies - **Usage**: Primary migration method for modern repo versions **Parameters**: - `ctx`: Context for cancellation and timeouts - `targetVersion`: Target repository version to migrate to - `repoPath`: Path to the IPFS repository directory - `allowDowngrade`: Whether to allow downgrade migrations ```go err = migrations.RunEmbeddedMigrations(ctx, targetVersion, repoPath, allowDowngrade) if err != nil { // Handle migration failure, may fall back to external migrations } ``` #### `migrations.RunMigration()` with `migrations.ReadMigrationConfig()` - **Purpose**: Runs external binary migrations downloaded from fs-repo-migrations - **Scope**: Handles legacy repo versions <17 and serves as fallback - **Performance**: Slower due to network downloads and external process execution - **Dependencies**: Requires fs-repo-migrations binaries and network access - **Usage**: Fallback method for legacy migrations ```go // Read migration configuration for external migrations migrationCfg, err := migrations.ReadMigrationConfig(repoPath, configFile) fetcher, err := migrations.GetMigrationFetcher(migrationCfg.DownloadSources, ...) err = migrations.RunMigration(ctx, fetcher, targetVersion, repoPath, allowDowngrade) ``` ## Migration Flow in Daemon Startup 1. **Primary**: Try embedded migrations first (`RunEmbeddedMigrations`) 2. **Fallback**: If embedded migration fails, fall back to external migrations (`RunMigration`) 3. **Legacy Support**: External migrations ensure compatibility with older repo versions ## Directory Structure ``` repo/fsrepo/migrations/ ├── README.md # This file ├── embedded.go # Embedded migration system ├── embedded_test.go # Tests for embedded migrations ├── migrations.go # External migration system ├── fs-repo-16-to-17/ # First embedded migration (16→17) │ ├── migration/ │ │ ├── migration.go # Migration logic │ │ └── migration_test.go # Migration tests │ ├── atomicfile/ │ │ └── atomicfile.go # Atomic file operations │ ├── main.go # Standalone migration binary │ └── README.md # Migration-specific documentation └── [other migration utilities] ``` ## Adding New Embedded Migrations To add a new embedded migration (e.g., fs-repo-17-to-18): 1. **Create migration package**: `fs-repo-17-to-18/migration/migration.go` 2. **Implement interface**: Ensure your migration implements the `EmbeddedMigration` interface 3. **Register migration**: Add to `embeddedMigrations` map in `embedded.go` 4. **Add tests**: Create comprehensive tests for your migration logic 5. **Update repo version**: Increment `RepoVersion` in `fsrepo.go` ```go // In embedded.go var embeddedMigrations = map[string]EmbeddedMigration{ "fs-repo-16-to-17": &mg16.Migration{}, "fs-repo-17-to-18": &mg17.Migration{}, // Add new migration } ``` ## Migration Requirements Each embedded migration must: - Implement the `EmbeddedMigration` interface - Be reversible with proper backup handling - Use atomic file operations to prevent corruption - Preserve user customizations - Include comprehensive tests - Follow the established naming pattern ## External Migration Support External migrations are maintained for: - **Backward compatibility** with repo versions <17 - **Fallback mechanism** if embedded migrations fail - **Legacy installations** that cannot be upgraded directly The external migration system will continue to work but is not the preferred method for new migrations. ## Security and Safety All migrations (embedded and external) include: - **Atomic operations**: Prevent repository corruption - **Backup creation**: Allow rollback if migration fails - **Version validation**: Ensure migrations run on correct repo versions - **Error handling**: Graceful failure with informative messages - **User preservation**: Maintain custom configurations during migration ## Testing Test both embedded and external migration systems: ```bash # Test embedded migrations go test ./repo/fsrepo/migrations/ -run TestEmbedded # Test specific migration go test ./repo/fsrepo/migrations/fs-repo-16-to-17/migration/ # Test migration registration go test ./repo/fsrepo/migrations/ -run TestHasEmbedded ``` ================================================ FILE: repo/fsrepo/migrations/atomicfile/atomicfile.go ================================================ package atomicfile import ( "fmt" "io" "os" "path/filepath" ) // File represents an atomic file writer type File struct { *os.File path string } // New creates a new atomic file writer func New(path string, mode os.FileMode) (*File, error) { dir := filepath.Dir(path) tempFile, err := os.CreateTemp(dir, ".tmp-"+filepath.Base(path)) if err != nil { return nil, err } if err := tempFile.Chmod(mode); err != nil { tempFile.Close() os.Remove(tempFile.Name()) return nil, err } return &File{ File: tempFile, path: path, }, nil } // Close atomically replaces the target file with the temporary file func (f *File) Close() error { closeErr := f.File.Close() if closeErr != nil { // Try to cleanup temp file, but prioritize close error _ = os.Remove(f.File.Name()) return closeErr } return os.Rename(f.File.Name(), f.path) } // Abort removes the temporary file without replacing the target func (f *File) Abort() error { closeErr := f.File.Close() removeErr := os.Remove(f.File.Name()) if closeErr != nil && removeErr != nil { return fmt.Errorf("abort failed: close: %w, remove: %v", closeErr, removeErr) } if closeErr != nil { return closeErr } return removeErr } // ReadFrom reads from the given reader into the atomic file func (f *File) ReadFrom(r io.Reader) (int64, error) { return io.Copy(f.File, r) } ================================================ FILE: repo/fsrepo/migrations/atomicfile/atomicfile_test.go ================================================ package atomicfile import ( "bytes" "fmt" "os" "path/filepath" "runtime" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // TestNew_Success verifies atomic file creation func TestNew_Success(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "test.txt") af, err := New(path, 0644) require.NoError(t, err) defer func() { _ = af.Abort() }() // Verify temp file exists assert.FileExists(t, af.File.Name()) // Verify temp file is in same directory assert.Equal(t, dir, filepath.Dir(af.File.Name())) } // TestClose_Success verifies atomic replacement func TestClose_Success(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "test.txt") af, err := New(path, 0644) require.NoError(t, err) content := []byte("test content") _, err = af.Write(content) require.NoError(t, err) tempName := af.File.Name() require.NoError(t, af.Close()) // Verify target file exists with correct content data, err := os.ReadFile(path) require.NoError(t, err) assert.Equal(t, content, data) // Verify temp file removed assert.NoFileExists(t, tempName) } // TestAbort_Success verifies cleanup func TestAbort_Success(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "test.txt") af, err := New(path, 0644) require.NoError(t, err) tempName := af.File.Name() require.NoError(t, af.Abort()) // Verify temp file removed assert.NoFileExists(t, tempName) // Verify target not created assert.NoFileExists(t, path) } // TestAbort_ErrorHandling tests error capture func TestAbort_ErrorHandling(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "test.txt") af, err := New(path, 0644) require.NoError(t, err) // Close file to force close error af.File.Close() // Remove temp file to force remove error os.Remove(af.File.Name()) err = af.Abort() // Should get both errors require.Error(t, err) assert.Contains(t, err.Error(), "abort failed") } // TestClose_CloseError verifies cleanup on close failure func TestClose_CloseError(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "test.txt") af, err := New(path, 0644) require.NoError(t, err) tempName := af.File.Name() // Close file to force close error af.File.Close() err = af.Close() require.Error(t, err) // Verify temp file cleaned up even on error assert.NoFileExists(t, tempName) } // TestReadFrom verifies io.Copy integration func TestReadFrom(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "test.txt") af, err := New(path, 0644) require.NoError(t, err) defer func() { _ = af.Abort() }() content := []byte("test content from reader") n, err := af.ReadFrom(bytes.NewReader(content)) require.NoError(t, err) assert.Equal(t, int64(len(content)), n) } // TestFilePermissions verifies mode is set correctly func TestFilePermissions(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "test.txt") af, err := New(path, 0600) require.NoError(t, err) _, err = af.Write([]byte("test")) require.NoError(t, err) require.NoError(t, af.Close()) info, err := os.Stat(path) require.NoError(t, err) // On Unix, check exact permissions if runtime.GOOS != "windows" { mode := info.Mode().Perm() assert.Equal(t, os.FileMode(0600), mode) } } // TestMultipleAbortsSafe verifies calling Abort multiple times is safe func TestMultipleAbortsSafe(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "test.txt") af, err := New(path, 0644) require.NoError(t, err) tempName := af.File.Name() // First abort should succeed require.NoError(t, af.Abort()) assert.NoFileExists(t, tempName, "temp file should be removed after first abort") // Second abort should handle gracefully (file already gone) err = af.Abort() // Error is acceptable since file is already removed, but it should not panic t.Logf("Second Abort() returned: %v", err) } // TestNoTempFilesAfterOperations verifies no .tmp-* files remain after operations func TestNoTempFilesAfterOperations(t *testing.T) { const testIterations = 5 tests := []struct { name string operation func(*File) error }{ {"close", (*File).Close}, {"abort", (*File).Abort}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { dir := t.TempDir() // Perform multiple operations for i := range testIterations { path := filepath.Join(dir, fmt.Sprintf("test%d.txt", i)) af, err := New(path, 0644) require.NoError(t, err) _, err = af.Write([]byte("test data")) require.NoError(t, err) require.NoError(t, tt.operation(af)) } // Check for any .tmp-* files tmpFiles, err := filepath.Glob(filepath.Join(dir, ".tmp-*")) require.NoError(t, err) assert.Empty(t, tmpFiles, "should be no temp files after %s", tt.name) }) } } ================================================ FILE: repo/fsrepo/migrations/common/base.go ================================================ package common import ( "fmt" "io" "path/filepath" ) // BaseMigration provides common functionality for migrations type BaseMigration struct { FromVersion string ToVersion string Description string Convert func(in io.ReadSeeker, out io.Writer) error } // Versions returns the version string for this migration func (m *BaseMigration) Versions() string { return fmt.Sprintf("%s-to-%s", m.FromVersion, m.ToVersion) } // configBackupSuffix returns the backup suffix for the config file // e.g. ".16-to-17.bak" results in "config.16-to-17.bak" func (m *BaseMigration) configBackupSuffix() string { return fmt.Sprintf(".%s-to-%s.bak", m.FromVersion, m.ToVersion) } // Reversible returns true as we keep backups func (m *BaseMigration) Reversible() bool { return true } // Apply performs the migration func (m *BaseMigration) Apply(opts Options) error { if opts.Verbose { fmt.Printf("applying %s repo migration\n", m.Versions()) if m.Description != "" { fmt.Printf("> %s\n", m.Description) } } // Check version if err := CheckVersion(opts.Path, m.FromVersion); err != nil { return err } configPath := filepath.Join(opts.Path, "config") // Perform migration with backup if err := WithBackup(configPath, m.configBackupSuffix(), m.Convert); err != nil { return err } // Update version if err := WriteVersion(opts.Path, m.ToVersion); err != nil { if opts.Verbose { fmt.Printf("failed to update version file to %s\n", m.ToVersion) } return err } if opts.Verbose { fmt.Println("updated version file") fmt.Printf("Migration %s succeeded\n", m.Versions()) } return nil } // Revert reverts the migration func (m *BaseMigration) Revert(opts Options) error { if opts.Verbose { fmt.Println("reverting migration") } // Check we're at the expected version if err := CheckVersion(opts.Path, m.ToVersion); err != nil { return err } // Restore backup configPath := filepath.Join(opts.Path, "config") if err := RevertBackup(configPath, m.configBackupSuffix()); err != nil { return err } // Revert version if err := WriteVersion(opts.Path, m.FromVersion); err != nil { return err } if opts.Verbose { fmt.Printf("lowered version number to %s\n", m.FromVersion) } return nil } ================================================ FILE: repo/fsrepo/migrations/common/config_helpers.go ================================================ package common import ( "fmt" "maps" "slices" "strings" ) // GetField retrieves a field from a nested config structure using a dot-separated path // Example: GetField(config, "DNS.Resolvers") returns config["DNS"]["Resolvers"] func GetField(config map[string]any, path string) (any, bool) { parts := strings.Split(path, ".") current := config for i, part := range parts { // Last part - return the value if i == len(parts)-1 { val, exists := current[part] return val, exists } // Navigate deeper next, exists := current[part] if !exists { return nil, false } // Ensure it's a map nextMap, ok := next.(map[string]any) if !ok { return nil, false } current = nextMap } return nil, false } // SetField sets a field in a nested config structure using a dot-separated path // It creates intermediate maps as needed func SetField(config map[string]any, path string, value any) { parts := strings.Split(path, ".") current := config for i, part := range parts { // Last part - set the value if i == len(parts)-1 { current[part] = value return } // Navigate or create intermediate maps next, exists := current[part] if !exists { // Create new intermediate map newMap := make(map[string]any) current[part] = newMap current = newMap } else { // Ensure it's a map nextMap, ok := next.(map[string]any) if !ok { // Can't navigate further, replace with new map newMap := make(map[string]any) current[part] = newMap current = newMap } else { current = nextMap } } } } // DeleteField removes a field from a nested config structure func DeleteField(config map[string]any, path string) bool { parts := strings.Split(path, ".") // Handle simple case if len(parts) == 1 { _, exists := config[parts[0]] delete(config, parts[0]) return exists } // Navigate to parent parentPath := strings.Join(parts[:len(parts)-1], ".") parent, exists := GetField(config, parentPath) if !exists { return false } parentMap, ok := parent.(map[string]any) if !ok { return false } fieldName := parts[len(parts)-1] _, exists = parentMap[fieldName] delete(parentMap, fieldName) return exists } // MoveField moves a field from one location to another func MoveField(config map[string]any, from, to string) error { value, exists := GetField(config, from) if !exists { return fmt.Errorf("source field %s does not exist", from) } SetField(config, to, value) DeleteField(config, from) return nil } // RenameField renames a field within the same parent func RenameField(config map[string]any, path, oldName, newName string) error { var parent map[string]any if path == "" { parent = config } else { p, exists := GetField(config, path) if !exists { return fmt.Errorf("parent path %s does not exist", path) } var ok bool parent, ok = p.(map[string]any) if !ok { return fmt.Errorf("parent path %s is not a map", path) } } value, exists := parent[oldName] if !exists { return fmt.Errorf("field %s does not exist", oldName) } parent[newName] = value delete(parent, oldName) return nil } // SetDefault sets a field value only if it doesn't already exist func SetDefault(config map[string]any, path string, value any) { if _, exists := GetField(config, path); !exists { SetField(config, path, value) } } // TransformField applies a transformation function to a field value func TransformField(config map[string]any, path string, transformer func(any) any) error { value, exists := GetField(config, path) if !exists { return fmt.Errorf("field %s does not exist", path) } newValue := transformer(value) SetField(config, path, newValue) return nil } // EnsureFieldIs checks if a field equals expected value, sets it if missing func EnsureFieldIs(config map[string]any, path string, expected any) { current, exists := GetField(config, path) if !exists || current != expected { SetField(config, path, expected) } } // MergeInto merges multiple source fields into a destination map func MergeInto(config map[string]any, destination string, sources ...string) { var destMap map[string]any // Get existing destination if it exists if existing, exists := GetField(config, destination); exists { if m, ok := existing.(map[string]any); ok { destMap = m } } // Merge each source for _, source := range sources { if value, exists := GetField(config, source); exists { if sourceMap, ok := value.(map[string]any); ok { if destMap == nil { destMap = make(map[string]any) } maps.Copy(destMap, sourceMap) } } } if destMap != nil { SetField(config, destination, destMap) } } // CopyField copies a field value to a new location (keeps original) func CopyField(config map[string]any, from, to string) error { value, exists := GetField(config, from) if !exists { return fmt.Errorf("source field %s does not exist", from) } SetField(config, to, value) return nil } // ConvertInterfaceSlice converts []interface{} to []string func ConvertInterfaceSlice(slice []any) []string { result := make([]string, 0, len(slice)) for _, item := range slice { if str, ok := item.(string); ok { result = append(result, str) } } return result } // GetOrCreateSection gets or creates a map section in config func GetOrCreateSection(config map[string]any, path string) map[string]any { existing, exists := GetField(config, path) if exists { if section, ok := existing.(map[string]any); ok { return section } } // Create new section section := make(map[string]any) SetField(config, path, section) return section } // SafeCastMap safely casts to map[string]any with fallback to empty map func SafeCastMap(value any) map[string]any { if m, ok := value.(map[string]any); ok { return m } return make(map[string]any) } // SafeCastSlice safely casts to []interface{} with fallback to empty slice func SafeCastSlice(value any) []any { if s, ok := value.([]any); ok { return s } return []any{} } // ReplaceDefaultsWithAuto replaces default values with "auto" in a map func ReplaceDefaultsWithAuto(values map[string]any, defaults map[string]string) map[string]string { result := make(map[string]string) for k, v := range values { if vStr, ok := v.(string); ok { if replacement, isDefault := defaults[vStr]; isDefault { result[k] = replacement } else { result[k] = vStr } } } return result } // EnsureSliceContains ensures a slice field contains a value func EnsureSliceContains(config map[string]any, path string, value string) { existing, exists := GetField(config, path) if !exists { SetField(config, path, []string{value}) return } if slice, ok := existing.([]any); ok { // Check if value already exists for _, item := range slice { if str, ok := item.(string); ok && str == value { return // Already contains value } } // Add value SetField(config, path, append(slice, value)) } else if strSlice, ok := existing.([]string); ok { if !slices.Contains(strSlice, value) { SetField(config, path, append(strSlice, value)) } } else { // Replace with new slice containing value SetField(config, path, []string{value}) } } // ReplaceInSlice replaces old values with new in a slice field func ReplaceInSlice(config map[string]any, path string, oldValue, newValue string) { existing, exists := GetField(config, path) if !exists { return } if slice, ok := existing.([]any); ok { result := make([]string, 0, len(slice)) for _, item := range slice { if str, ok := item.(string); ok { if str == oldValue { result = append(result, newValue) } else { result = append(result, str) } } } SetField(config, path, result) } } // GetMapSection gets a map section with error handling func GetMapSection(config map[string]any, path string) (map[string]any, error) { value, exists := GetField(config, path) if !exists { return nil, fmt.Errorf("section %s does not exist", path) } section, ok := value.(map[string]any) if !ok { return nil, fmt.Errorf("section %s is not a map", path) } return section, nil } // CloneStringMap clones a map[string]any to map[string]string func CloneStringMap(m map[string]any) map[string]string { result := make(map[string]string, len(m)) for k, v := range m { if str, ok := v.(string); ok { result[k] = str } } return result } // IsEmptySlice checks if a value is an empty slice func IsEmptySlice(value any) bool { if value == nil { return true } if slice, ok := value.([]any); ok { return len(slice) == 0 } if slice, ok := value.([]string); ok { return len(slice) == 0 } return false } ================================================ FILE: repo/fsrepo/migrations/common/migration.go ================================================ // Package common contains common types and interfaces for file system repository migrations package common // Options contains migration options for embedded migrations type Options struct { Path string Verbose bool } // Migration is the interface that all migrations must implement type Migration interface { Versions() string Apply(opts Options) error Revert(opts Options) error Reversible() bool } ================================================ FILE: repo/fsrepo/migrations/common/testing_helpers.go ================================================ package common import ( "bytes" "encoding/json" "fmt" "maps" "os" "path/filepath" "reflect" "testing" ) // TestCase represents a single migration test case type TestCase struct { Name string InputConfig map[string]any Assertions []ConfigAssertion } // ConfigAssertion represents an assertion about the migrated config type ConfigAssertion struct { Path string Expected any } // RunMigrationTest runs a migration test with the given test case func RunMigrationTest(t *testing.T, migration Migration, tc TestCase) { t.Helper() // Convert input to JSON inputJSON, err := json.MarshalIndent(tc.InputConfig, "", " ") if err != nil { t.Fatalf("failed to marshal input config: %v", err) } // Run the migration's convert function var output bytes.Buffer if baseMig, ok := migration.(*BaseMigration); ok { err = baseMig.Convert(bytes.NewReader(inputJSON), &output) if err != nil { t.Fatalf("migration failed: %v", err) } } else { t.Skip("migration is not a BaseMigration") } // Parse output var result map[string]any err = json.Unmarshal(output.Bytes(), &result) if err != nil { t.Fatalf("failed to unmarshal output: %v", err) } // Run assertions for _, assertion := range tc.Assertions { AssertConfigField(t, result, assertion.Path, assertion.Expected) } } // AssertConfigField asserts that a field in the config has the expected value func AssertConfigField(t *testing.T, config map[string]any, path string, expected any) { t.Helper() actual, exists := GetField(config, path) if expected == nil { if exists { t.Errorf("expected field %s to not exist, but it has value: %v", path, actual) } return } if !exists { t.Errorf("expected field %s to exist with value %v, but it doesn't exist", path, expected) return } // Handle different types of comparisons switch exp := expected.(type) { case []string: actualSlice, ok := actual.([]any) if !ok { t.Errorf("field %s: expected []string, got %T", path, actual) return } if len(exp) != len(actualSlice) { t.Errorf("field %s: expected slice of length %d, got %d", path, len(exp), len(actualSlice)) return } for i, expVal := range exp { if actualSlice[i] != expVal { t.Errorf("field %s[%d]: expected %v, got %v", path, i, expVal, actualSlice[i]) } } case map[string]string: actualMap, ok := actual.(map[string]any) if !ok { t.Errorf("field %s: expected map, got %T", path, actual) return } for k, v := range exp { if actualMap[k] != v { t.Errorf("field %s[%s]: expected %v, got %v", path, k, v, actualMap[k]) } } default: if actual != expected { t.Errorf("field %s: expected %v, got %v", path, expected, actual) } } } // GenerateTestConfig creates a basic test config with the given fields func GenerateTestConfig(fields map[string]any) map[string]any { // Start with a minimal valid config config := map[string]any{ "Identity": map[string]any{ "PeerID": "QmTest", }, } // Merge in the provided fields maps.Copy(config, fields) return config } // CreateTestRepo creates a temporary test repository with the given version and config func CreateTestRepo(t *testing.T, version int, config map[string]any) string { t.Helper() tempDir := t.TempDir() // Write version file versionPath := filepath.Join(tempDir, "version") err := os.WriteFile(versionPath, fmt.Appendf(nil, "%d", version), 0644) if err != nil { t.Fatalf("failed to write version file: %v", err) } // Write config file configPath := filepath.Join(tempDir, "config") configData, err := json.MarshalIndent(config, "", " ") if err != nil { t.Fatalf("failed to marshal config: %v", err) } err = os.WriteFile(configPath, configData, 0644) if err != nil { t.Fatalf("failed to write config file: %v", err) } return tempDir } // AssertMigrationSuccess runs a full migration and checks that it succeeds func AssertMigrationSuccess(t *testing.T, migration Migration, fromVersion, toVersion int, inputConfig map[string]any) map[string]any { t.Helper() // Create test repo repoPath := CreateTestRepo(t, fromVersion, inputConfig) // Run migration opts := Options{ Path: repoPath, Verbose: false, } err := migration.Apply(opts) if err != nil { t.Fatalf("migration failed: %v", err) } // Check version was updated versionBytes, err := os.ReadFile(filepath.Join(repoPath, "version")) if err != nil { t.Fatalf("failed to read version file: %v", err) } actualVersion := string(versionBytes) if actualVersion != fmt.Sprintf("%d", toVersion) { t.Errorf("expected version %d, got %s", toVersion, actualVersion) } // Read and return the migrated config configBytes, err := os.ReadFile(filepath.Join(repoPath, "config")) if err != nil { t.Fatalf("failed to read config file: %v", err) } var result map[string]any err = json.Unmarshal(configBytes, &result) if err != nil { t.Fatalf("failed to unmarshal config: %v", err) } return result } // AssertMigrationReversible checks that a migration can be reverted func AssertMigrationReversible(t *testing.T, migration Migration, fromVersion, toVersion int, inputConfig map[string]any) { t.Helper() // Create test repo at target version repoPath := CreateTestRepo(t, toVersion, inputConfig) // Create backup file (simulating a previous migration) backupPath := filepath.Join(repoPath, fmt.Sprintf("config.%d-to-%d.bak", fromVersion, toVersion)) originalConfig, err := json.MarshalIndent(inputConfig, "", " ") if err != nil { t.Fatalf("failed to marshal original config: %v", err) } if err := os.WriteFile(backupPath, originalConfig, 0644); err != nil { t.Fatalf("failed to write backup file: %v", err) } // Run revert if err := migration.Revert(Options{Path: repoPath}); err != nil { t.Fatalf("revert failed: %v", err) } // Verify version was reverted versionBytes, err := os.ReadFile(filepath.Join(repoPath, "version")) if err != nil { t.Fatalf("failed to read version file: %v", err) } if actualVersion := string(versionBytes); actualVersion != fmt.Sprintf("%d", fromVersion) { t.Errorf("expected version %d after revert, got %s", fromVersion, actualVersion) } // Verify config was reverted configBytes, err := os.ReadFile(filepath.Join(repoPath, "config")) if err != nil { t.Fatalf("failed to read reverted config file: %v", err) } var revertedConfig map[string]any if err := json.Unmarshal(configBytes, &revertedConfig); err != nil { t.Fatalf("failed to unmarshal reverted config: %v", err) } // Compare reverted config with original compareConfigs(t, inputConfig, revertedConfig, "") } // compareConfigs recursively compares two config maps and reports differences func compareConfigs(t *testing.T, expected, actual map[string]any, path string) { t.Helper() // Build current path helper buildPath := func(key string) string { if path == "" { return key } return path + "." + key } // Check all expected fields exist and match for key, expectedValue := range expected { currentPath := buildPath(key) actualValue, exists := actual[key] if !exists { t.Errorf("reverted config missing field %s", currentPath) continue } switch exp := expectedValue.(type) { case map[string]any: act, ok := actualValue.(map[string]any) if !ok { t.Errorf("field %s: expected map, got %T", currentPath, actualValue) continue } compareConfigs(t, exp, act, currentPath) default: if !reflect.DeepEqual(expectedValue, actualValue) { t.Errorf("field %s: expected %v, got %v after revert", currentPath, expectedValue, actualValue) } } } // Check for unexpected fields using maps.Keys (Go 1.23+) for key := range actual { if _, exists := expected[key]; !exists { t.Errorf("reverted config has unexpected field %s", buildPath(key)) } } } ================================================ FILE: repo/fsrepo/migrations/common/utils.go ================================================ package common import ( "bytes" "encoding/json" "fmt" "io" "os" "path/filepath" "strings" "github.com/ipfs/kubo/repo/fsrepo/migrations/atomicfile" ) // CheckVersion verifies the repo is at the expected version func CheckVersion(repoPath string, expectedVersion string) error { versionPath := filepath.Join(repoPath, "version") versionBytes, err := os.ReadFile(versionPath) if err != nil { return fmt.Errorf("could not read version file: %w", err) } version := strings.TrimSpace(string(versionBytes)) if version != expectedVersion { return fmt.Errorf("expected version %s, got %s", expectedVersion, version) } return nil } // WriteVersion writes the version to the repo func WriteVersion(repoPath string, version string) error { versionPath := filepath.Join(repoPath, "version") return os.WriteFile(versionPath, []byte(version), 0644) } // Must panics if the error is not nil. Use only for errors that cannot be handled gracefully. func Must(err error) { if err != nil { panic(fmt.Errorf("error can't be dealt with transactionally: %w", err)) } } // WithBackup performs a config file operation with automatic backup and rollback on error func WithBackup(configPath string, backupSuffix string, fn func(in io.ReadSeeker, out io.Writer) error) error { // Read the entire file into memory first // This allows us to close the file before doing atomic operations, // which is necessary on Windows where open files can't be renamed data, err := os.ReadFile(configPath) if err != nil { return fmt.Errorf("failed to read config file %s: %w", configPath, err) } // Create an in-memory reader for the data in := bytes.NewReader(data) // Create backup atomically to prevent partial backup on interruption backupPath := configPath + backupSuffix backup, err := atomicfile.New(backupPath, 0600) if err != nil { return fmt.Errorf("failed to create backup file for %s: %w", backupPath, err) } if _, err := backup.Write(data); err != nil { Must(backup.Abort()) return fmt.Errorf("failed to write backup data: %w", err) } if err := backup.Close(); err != nil { Must(backup.Abort()) return fmt.Errorf("failed to finalize backup: %w", err) } // Create output file atomically out, err := atomicfile.New(configPath, 0600) if err != nil { // Clean up backup on error os.Remove(backupPath) return fmt.Errorf("failed to create atomic file for %s: %w", configPath, err) } // Run the conversion function if err := fn(in, out); err != nil { Must(out.Abort()) // Clean up backup on error os.Remove(backupPath) return fmt.Errorf("config conversion failed: %w", err) } // Close the output file atomically Must(out.Close()) // Backup remains for potential revert return nil } // RevertBackup restores a backup file func RevertBackup(configPath string, backupSuffix string) error { return os.Rename(configPath+backupSuffix, configPath) } // ReadConfig reads and unmarshals a JSON config file into a map func ReadConfig(r io.Reader) (map[string]any, error) { confMap := make(map[string]any) if err := json.NewDecoder(r).Decode(&confMap); err != nil { return nil, err } return confMap, nil } // WriteConfig marshals and writes a config map as indented JSON func WriteConfig(w io.Writer, config map[string]any) error { enc := json.NewEncoder(w) enc.SetIndent("", " ") return enc.Encode(config) } ================================================ FILE: repo/fsrepo/migrations/embedded.go ================================================ package migrations import ( "context" "fmt" "log" "os" lockfile "github.com/ipfs/go-fs-lock" "github.com/ipfs/kubo/repo/fsrepo/migrations/common" mg16 "github.com/ipfs/kubo/repo/fsrepo/migrations/fs-repo-16-to-17/migration" mg17 "github.com/ipfs/kubo/repo/fsrepo/migrations/fs-repo-17-to-18/migration" ) // embeddedMigrations contains all embedded migrations // Using a slice to maintain order and allow for future range-based operations var embeddedMigrations = []common.Migration{ mg16.Migration, mg17.Migration, } // migrationsByName provides quick lookup by name var migrationsByName = make(map[string]common.Migration) func init() { for _, m := range embeddedMigrations { migrationsByName["fs-repo-"+m.Versions()] = m } } // RunEmbeddedMigration runs an embedded migration if available func RunEmbeddedMigration(ctx context.Context, migrationName string, ipfsDir string, revert bool) error { migration, exists := migrationsByName[migrationName] if !exists { return fmt.Errorf("embedded migration %s not found", migrationName) } if revert && !migration.Reversible() { return fmt.Errorf("migration %s is not reversible", migrationName) } logger := log.New(os.Stdout, "", 0) logger.Printf("Running embedded migration %s...", migrationName) opts := common.Options{ Path: ipfsDir, Verbose: true, } var err error if revert { err = migration.Revert(opts) } else { err = migration.Apply(opts) } if err != nil { return fmt.Errorf("embedded migration %s failed: %w", migrationName, err) } logger.Printf("Embedded migration %s completed successfully", migrationName) return nil } // HasEmbeddedMigration checks if a migration is available as embedded func HasEmbeddedMigration(migrationName string) bool { _, exists := migrationsByName[migrationName] return exists } // RunEmbeddedMigrations runs all needed embedded migrations from current version to target version. // // This function migrates an IPFS repository using embedded migrations that are built into the Kubo binary. // Embedded migrations are available for repo version 17+ and provide fast, network-free migration execution. // // Parameters: // - ctx: Context for cancellation and deadlines // - targetVer: Target repository version to migrate to // - ipfsDir: Path to the IPFS repository directory // - allowDowngrade: Whether to allow downgrade migrations (reduces target version) // // Returns: // - nil on successful migration // - error if migration fails, repo path is invalid, or no embedded migrations are available // // Behavior: // - Validates that ipfsDir contains a valid IPFS repository // - Determines current repository version automatically // - Returns immediately if already at target version // - Prevents downgrades unless allowDowngrade is true // - Runs all necessary migrations in sequence (e.g., 16→17→18 if going from 16 to 18) // - Creates backups and uses atomic operations to prevent corruption // // Error conditions: // - Repository path is invalid or inaccessible // - Current version cannot be determined // - Downgrade attempted with allowDowngrade=false // - No embedded migrations available for the version range // - Individual migration fails during execution // // Example: // // err := RunEmbeddedMigrations(ctx, 17, "/path/to/.ipfs", false) // if err != nil { // // Handle migration failure, may need to fall back to external migrations // } func RunEmbeddedMigrations(ctx context.Context, targetVer int, ipfsDir string, allowDowngrade bool) error { ipfsDir, err := CheckIpfsDir(ipfsDir) if err != nil { return err } // Acquire lock once for all embedded migrations to prevent concurrent access lk, err := lockfile.Lock(ipfsDir, "repo.lock") if err != nil { return fmt.Errorf("failed to acquire repo lock: %w", err) } defer lk.Close() fromVer, err := RepoVersion(ipfsDir) if err != nil { return fmt.Errorf("could not get repo version: %w", err) } if fromVer == targetVer { return nil } revert := fromVer > targetVer if revert && !allowDowngrade { return fmt.Errorf("downgrade not allowed from %d to %d", fromVer, targetVer) } logger := log.New(os.Stdout, "", 0) logger.Print("Looking for embedded migrations.") migrations, _, err := findMigrations(ctx, fromVer, targetVer) if err != nil { return err } embeddedCount := 0 for _, migrationName := range migrations { if HasEmbeddedMigration(migrationName) { err = RunEmbeddedMigration(ctx, migrationName, ipfsDir, revert) if err != nil { return err } embeddedCount++ } } if embeddedCount == 0 { return fmt.Errorf("no embedded migrations found for version %d to %d", fromVer, targetVer) } logger.Printf("Success: fs-repo migrated to version %d using embedded migrations.\n", targetVer) return nil } ================================================ FILE: repo/fsrepo/migrations/embedded_test.go ================================================ package migrations import ( "context" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestHasEmbeddedMigration(t *testing.T) { // Test that the 16-to-17 migration is registered assert.True(t, HasEmbeddedMigration("fs-repo-16-to-17"), "fs-repo-16-to-17 migration should be registered") // Test that a non-existent migration is not found assert.False(t, HasEmbeddedMigration("fs-repo-99-to-100"), "fs-repo-99-to-100 migration should not be registered") } func TestEmbeddedMigrations(t *testing.T) { // Test that we have at least one embedded migration assert.NotEmpty(t, embeddedMigrations, "No embedded migrations found") // Test that all registered migrations implement the interface for name, migration := range embeddedMigrations { assert.NotEmpty(t, migration.Versions(), "Migration %s has empty versions", name) } } func TestRunEmbeddedMigration(t *testing.T) { // Test that running a non-existent migration returns an error err := RunEmbeddedMigration(context.Background(), "non-existent", "/tmp", false) require.Error(t, err, "Expected error for non-existent migration") } ================================================ FILE: repo/fsrepo/migrations/fetch.go ================================================ package migrations import ( "bufio" "bytes" "context" "fmt" "io" "os" "os/exec" "path/filepath" "runtime" "strings" ) // DownloadDirectory can be set as the location for FetchBinary to save the // downloaded archive file in. If not set, then FetchBinary saves the archive // in a temporary directory that is removed after the contents of the archive // is extracted. var DownloadDirectory string // FetchBinary downloads an archive from the distribution site and unpacks it. // // The base name of the binary inside the archive may differ from the base // archive name. If it does, then specify binName. For example, the following // is needed because the archive "go-ipfs_v0.7.0_linux-amd64.tar.gz" contains a // binary named "ipfs" // // FetchBinary(ctx, fetcher, "go-ipfs", "v0.7.0", "ipfs", tmpDir) // // If out is a directory, then the binary is written to that directory with the // same name it has inside the archive. Otherwise, the binary file is written // to the file named by out. func FetchBinary(ctx context.Context, fetcher Fetcher, dist, ver, binName, out string) (string, error) { // The archive file name is the base of dist. This is to support a possible subdir in // dist, for example: "ipfs-repo-migrations/fs-repo-11-to-12" arcName := filepath.Base(dist) // If binary base name is not specified, then it is same as archive base name. if binName == "" { binName = arcName } // Name of binary that exists inside archive binName = ExeName(binName) // Return error if file exists or stat fails for reason other than not // exists. If out is a directory, then write extracted binary to that dir. fi, err := os.Stat(out) if !os.IsNotExist(err) { if err != nil { return "", err } if !fi.IsDir() { return "", &os.PathError{ Op: "FetchBinary", Path: out, Err: os.ErrExist, } } // out exists and is a directory, so compose final name out = filepath.Join(out, binName) // Check if the binary already exists in the directory _, err = os.Stat(out) if !os.IsNotExist(err) { if err != nil { return "", err } return "", &os.PathError{ Op: "FetchBinary", Path: out, Err: os.ErrExist, } } } tmpDir := DownloadDirectory if tmpDir != "" { fi, err = os.Stat(tmpDir) if err != nil { return "", err } if !fi.IsDir() { return "", &os.PathError{ Op: "FetchBinary", Path: tmpDir, Err: os.ErrExist, } } } else { // Create temp directory to store download tmpDir, err = os.MkdirTemp("", arcName) if err != nil { return "", err } defer os.RemoveAll(tmpDir) } atype := "tar.gz" if runtime.GOOS == "windows" { atype = "zip" } arcDistPath, arcFullName := makeArchivePath(dist, arcName, ver, atype) // Create a file to write the archive data to arcPath := filepath.Join(tmpDir, arcFullName) arcFile, err := os.Create(arcPath) if err != nil { return "", err } defer arcFile.Close() // Open connection to download archive from ipfs path and write to file arcBytes, err := fetcher.Fetch(ctx, arcDistPath) if err != nil { return "", err } // Write download data _, err = io.Copy(arcFile, bytes.NewReader(arcBytes)) if err != nil { return "", err } arcFile.Close() // Unpack the archive and write binary to out err = unpackArchive(arcPath, atype, dist, binName, out) if err != nil { return "", err } // Set mode of binary to executable err = os.Chmod(out, 0o755) if err != nil { return "", err } return out, nil } // osWithVariant returns the OS name with optional variant. // Currently returns either runtime.GOOS, or "linux-musl". func osWithVariant() (string, error) { if runtime.GOOS != "linux" { return runtime.GOOS, nil } // ldd outputs the system's kind of libc. // - on standard ubuntu: ldd (Ubuntu GLIBC 2.23-0ubuntu5) 2.23 // - on alpine: musl libc (x86_64) // // we use the combined stdout+stderr, // because ldd --version prints differently on different OSes. // - on standard ubuntu: stdout // - on alpine: stderr (it probably doesn't know the --version flag) // // we suppress non-zero exit codes (see last point about alpine). out, err := exec.Command("sh", "-c", "ldd --version || true").CombinedOutput() if err != nil { return "", err } // now just see if we can find "musl" somewhere in the output scan := bufio.NewScanner(bytes.NewBuffer(out)) for scan.Scan() { if strings.Contains(scan.Text(), "musl") { return "linux-musl", nil } } return "linux", nil } // makeArchivePath composes the path, relative to the distribution site, from which to // download a binary. The path returned does not contain the distribution site path, // e.g. "/ipns/dist.ipfs.tech/", since that is know to the fetcher. // // Returns the archive path and the base name. // // The ipfs path format is: distribution/version/archiveName // - distribution is the name of a distribution, such as "go-ipfs" // - version is the version to fetch, such as "v0.8.0-rc2" // - archiveName is formatted as name_version_osv-GOARCH.atype, such as // "go-ipfs_v0.8.0-rc2_linux-amd64.tar.gz" // // This would form the path: // go-ipfs/v0.8.0/go-ipfs_v0.8.0_linux-amd64.tar.gz func makeArchivePath(dist, name, ver, atype string) (string, string) { arcName := fmt.Sprintf("%s_%s_%s-%s.%s", name, ver, runtime.GOOS, runtime.GOARCH, atype) return fmt.Sprintf("%s/%s/%s", dist, ver, arcName), arcName } ================================================ FILE: repo/fsrepo/migrations/fetch_test.go ================================================ package migrations import ( "bufio" "bytes" "fmt" "os" "path/filepath" "runtime" "strings" "testing" ) func TestGetDistPath(t *testing.T) { os.Unsetenv(envIpfsDistPath) distPath := GetDistPathEnv("") if distPath != LatestIpfsDist { t.Error("did not set default dist path") } testDist := "/unit/test/dist" t.Setenv(envIpfsDistPath, testDist) defer func() { os.Unsetenv(envIpfsDistPath) }() distPath = GetDistPathEnv("") if distPath != testDist { t.Error("did not set dist path from environ") } distPath = GetDistPathEnv("ignored") if distPath != testDist { t.Error("did not set dist path from environ") } testDist = "/unit/test/dist2" fetcher := NewHttpFetcher(testDist, "", "", 0) if fetcher.distPath != testDist { t.Error("did not set dist path") } } func TestHttpFetch(t *testing.T) { ctx := t.Context() fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) out, err := fetcher.Fetch(ctx, "/kubo/versions") if err != nil { t.Fatal(err) } var lines []string scan := bufio.NewScanner(bytes.NewReader(out)) for scan.Scan() { lines = append(lines, scan.Text()) } err = scan.Err() if err != nil { t.Fatal("could not read versions:", err) } if len(lines) < 6 { t.Fatal("do not get all expected data") } if lines[0] != "v1.0.0" { t.Fatal("expected v1.0.0 as first line, got", lines[0]) } // Check not found _, err = fetcher.Fetch(ctx, "/no_such_file") if err == nil || !strings.Contains(err.Error(), "no link") { t.Fatal("expected error 404") } } func TestFetchBinary(t *testing.T) { tmpDir := t.TempDir() ctx := t.Context() fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) vers, err := DistVersions(ctx, fetcher, distFSRM, false) if err != nil { t.Fatal(err) } t.Log("latest version of", distFSRM, "is", vers[len(vers)-1]) bin, err := FetchBinary(ctx, fetcher, distFSRM, vers[0], "", tmpDir) if err != nil { t.Fatal(err) } fi, err := os.Stat(bin) if os.IsNotExist(err) { t.Error("expected file to exist:", bin) } t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name()) bin, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir) if err != nil { t.Fatal(err) } fi, err = os.Stat(bin) if os.IsNotExist(err) { t.Error("expected file to exist:", bin) } t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name()) // Check error is destination already exists and is not directory _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", bin) if !os.IsExist(err) { t.Fatal("expected 'exists' error, got", err) } _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir) if !os.IsExist(err) { t.Error("expected 'exists' error, got:", err) } os.Remove(filepath.Join(tmpDir, ExeName("ipfs"))) // Check error creating temp download directory // // Windows doesn't have read-only directories https://github.com/golang/go/issues/35042 this would need to be // tested another way if runtime.GOOS != "windows" { err = os.Chmod(tmpDir, 0o555) if err != nil { panic(err) } t.Setenv("TMPDIR", tmpDir) _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir) if !os.IsPermission(err) { t.Error("expected 'permission' error, got:", err) } t.Setenv("TMPDIR", "/tmp") err = os.Chmod(tmpDir, 0o755) if err != nil { panic(err) } } // Check error if failure to fetch due to bad dist _, err = FetchBinary(ctx, fetcher, "not-here", "v1.0.0", "ipfs", tmpDir) if err == nil || !strings.Contains(err.Error(), "no link") { t.Error("expected 'Not Found' error, got:", err) } // Check error if failure to unpack archive _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "not-such-bin", tmpDir) if err == nil || err.Error() != "no binary found in archive" { t.Error("expected 'no binary found in archive' error") } } func TestMultiFetcher(t *testing.T) { ctx := t.Context() badFetcher := NewHttpFetcher("", "bad-url", "", 0) fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) mf := NewMultiFetcher(badFetcher, fetcher) vers, err := mf.Fetch(ctx, "/kubo/versions") if err != nil { t.Fatal(err) } if len(vers) < 45 { fmt.Println("unexpected more data") } } ================================================ FILE: repo/fsrepo/migrations/fetcher.go ================================================ package migrations import ( "context" "errors" "fmt" "io" "os" ) const ( // Current distribution to fetch migrations from. CurrentIpfsDist = "/ipfs/QmRzRGJEjYDfbHHaALnHBuhzzrkXGdwcPMrgd5fgM7hqbe" // fs-repo-15-to-16 v1.0.1 // Latest distribution path. Default for fetchers. LatestIpfsDist = "/ipns/dist.ipfs.tech" // Distribution environ variable. envIpfsDistPath = "IPFS_DIST_PATH" ) type Fetcher interface { // Fetch attempts to fetch the file at the given ipfs path. Fetch(ctx context.Context, filePath string) ([]byte, error) // Close performs any cleanup after the fetcher is not longer needed. Close() error } // MultiFetcher holds multiple Fetchers and provides a Fetch that tries each // until one succeeds. type MultiFetcher struct { fetchers []Fetcher } type limitReadCloser struct { io.Reader io.Closer } // NewMultiFetcher creates a MultiFetcher with the given Fetchers. The // Fetchers are tried in order, then passed to this function. func NewMultiFetcher(f ...Fetcher) *MultiFetcher { mf := &MultiFetcher{ fetchers: make([]Fetcher, len(f)), } copy(mf.fetchers, f) return mf } // Fetch attempts to fetch the file at each of its fetchers until one succeeds. func (f *MultiFetcher) Fetch(ctx context.Context, ipfsPath string) ([]byte, error) { var errs []error for _, fetcher := range f.fetchers { out, err := fetcher.Fetch(ctx, ipfsPath) if err == nil { return out, nil } fmt.Printf("Error fetching: %s\n", err.Error()) errs = append(errs, err) } return nil, errors.Join(errs...) } func (f *MultiFetcher) Close() error { var errs error for _, fetcher := range f.fetchers { if err := fetcher.Close(); err != nil { errs = errors.Join(errs, err) } } return errs } func (f *MultiFetcher) Len() int { return len(f.fetchers) } func (f *MultiFetcher) Fetchers() []Fetcher { return f.fetchers } // NewLimitReadCloser returns a new io.ReadCloser with the reader wrapped in a // io.LimitedReader limited to reading the amount specified. func NewLimitReadCloser(rc io.ReadCloser, limit int64) io.ReadCloser { return limitReadCloser{ Reader: io.LimitReader(rc, limit), Closer: rc, } } // GetDistPathEnv returns the IPFS path to the distribution site, using // the value of environ variable specified by envIpfsDistPath. If the environ // variable is not set, then returns the provided distPath, and if that is not set // then returns the IPNS path. // // To get the IPFS path of the latest distribution, if not overridden by the // environ variable: GetDistPathEnv(CurrentIpfsDist). func GetDistPathEnv(distPath string) string { if dist := os.Getenv(envIpfsDistPath); dist != "" { return dist } if distPath == "" { return LatestIpfsDist } return distPath } ================================================ FILE: repo/fsrepo/migrations/fs-repo-16-to-17/main.go ================================================ // Package main implements fs-repo-16-to-17 migration for IPFS repositories. // // This migration transitions repositories from version 16 to 17, introducing // the AutoConf system that replaces hardcoded network defaults with dynamic // configuration fetched from autoconf.json. // // Changes made: // - Enables AutoConf system with default settings // - Migrates default bootstrap peers to "auto" sentinel value // - Sets DNS.Resolvers["."] to "auto" for dynamic DNS resolver configuration // - Migrates Routing.DelegatedRouters to ["auto"] // - Migrates Ipns.DelegatedPublishers to ["auto"] // - Preserves user customizations (custom bootstrap peers, DNS resolvers) // // The migration is reversible and creates config.16-to-17.bak for rollback. // // Usage: // // fs-repo-16-to-17 -path /path/to/ipfs/repo [-verbose] [-revert] // // This migration is embedded in Kubo starting from version 0.37 and runs // automatically during daemon startup. This standalone binary is provided // for manual migration scenarios. package main import ( "flag" "fmt" "os" "github.com/ipfs/kubo/repo/fsrepo/migrations/common" mg16 "github.com/ipfs/kubo/repo/fsrepo/migrations/fs-repo-16-to-17/migration" ) func main() { var path = flag.String("path", "", "Path to IPFS repository") var verbose = flag.Bool("verbose", false, "Enable verbose output") var revert = flag.Bool("revert", false, "Revert migration") flag.Parse() if *path == "" { fmt.Fprintf(os.Stderr, "Error: -path flag is required\n") flag.Usage() os.Exit(1) } opts := common.Options{ Path: *path, Verbose: *verbose, } var err error if *revert { err = mg16.Migration.Revert(opts) } else { err = mg16.Migration.Apply(opts) } if err != nil { fmt.Fprintf(os.Stderr, "Migration failed: %v\n", err) os.Exit(1) } } ================================================ FILE: repo/fsrepo/migrations/fs-repo-16-to-17/migration/migration.go ================================================ // package mg16 contains the code to perform 16-17 repository migration in Kubo. // This handles the following: // - Migrate default bootstrap peers to "auto" // - Migrate DNS resolvers to use "auto" for "." eTLD // - Enable AutoConf system with default settings // - Increment repo version to 17 package mg16 import ( "io" "slices" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/repo/fsrepo/migrations/common" ) // DefaultBootstrapAddresses are the hardcoded bootstrap addresses from Kubo 0.36 // for IPFS. they are nodes run by the IPFS team. docs on these later. // As with all p2p networks, bootstrap is an important security concern. // This list is used during migration to detect which peers are defaults vs custom. var DefaultBootstrapAddresses = []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", // rust-libp2p-server "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", "/dnsaddr/va1.bootstrap.libp2p.io/p2p/12D3KooWKnDdG3iXw9eTFijk3EWSunZcFi54Zka4wmtqtt6rPxc8", // js-libp2p-amino-dht-bootstrapper "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", // mars.i.ipfs.io "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", // mars.i.ipfs.io } // Migration is the main exported migration for 16-to-17 var Migration = &common.BaseMigration{ FromVersion: "16", ToVersion: "17", Description: "Upgrading config to use AutoConf system", Convert: convert, } // NewMigration creates a new migration instance (for compatibility) func NewMigration() common.Migration { return Migration } // convert converts the config from version 16 to 17 func convert(in io.ReadSeeker, out io.Writer) error { confMap, err := common.ReadConfig(in) if err != nil { return err } // Enable AutoConf system if err := enableAutoConf(confMap); err != nil { return err } // Migrate Bootstrap peers if err := migrateBootstrap(confMap); err != nil { return err } // Migrate DNS resolvers if err := migrateDNSResolvers(confMap); err != nil { return err } // Migrate DelegatedRouters if err := migrateDelegatedRouters(confMap); err != nil { return err } // Migrate DelegatedPublishers if err := migrateDelegatedPublishers(confMap); err != nil { return err } // Save new config return common.WriteConfig(out, confMap) } // enableAutoConf adds AutoConf section to config func enableAutoConf(confMap map[string]any) error { // Add empty AutoConf section if it doesn't exist - all fields will use implicit defaults: // - Enabled defaults to true (via DefaultAutoConfEnabled) // - URL defaults to mainnet URL (via DefaultAutoConfURL) // - RefreshInterval defaults to 24h (via DefaultAutoConfRefreshInterval) // - TLSInsecureSkipVerify defaults to false (no WithDefault, but false is zero value) common.SetDefault(confMap, "AutoConf", map[string]any{}) return nil } // migrateBootstrap migrates bootstrap peers to use "auto" func migrateBootstrap(confMap map[string]any) error { bootstrap, exists := confMap["Bootstrap"] if !exists { // No bootstrap section, add "auto" confMap["Bootstrap"] = []string{config.AutoPlaceholder} return nil } // Convert to string slice using helper bootstrapPeers := common.ConvertInterfaceSlice(common.SafeCastSlice(bootstrap)) if len(bootstrapPeers) == 0 && bootstrap != nil { // Invalid bootstrap format, replace with "auto" confMap["Bootstrap"] = []string{config.AutoPlaceholder} return nil } // Process bootstrap peers according to migration rules newBootstrap := processBootstrapPeers(bootstrapPeers) confMap["Bootstrap"] = newBootstrap return nil } // processBootstrapPeers processes bootstrap peers according to migration rules func processBootstrapPeers(peers []string) []string { // If empty, use "auto" if len(peers) == 0 { return []string{config.AutoPlaceholder} } // Filter out default peers to get only custom ones customPeers := slices.DeleteFunc(slices.Clone(peers), func(peer string) bool { return slices.Contains(DefaultBootstrapAddresses, peer) }) // Check if any default peers were removed hasDefaultPeers := len(customPeers) < len(peers) // If we have default peers, replace them with "auto" if hasDefaultPeers { return append([]string{config.AutoPlaceholder}, customPeers...) } // No default peers found, keep as is return peers } // migrateDNSResolvers migrates DNS resolvers to use "auto" for "." eTLD func migrateDNSResolvers(confMap map[string]any) error { // Get or create DNS section dns := common.GetOrCreateSection(confMap, "DNS") // Get existing resolvers or create empty map resolvers := common.SafeCastMap(dns["Resolvers"]) // Define default resolvers that should be replaced with "auto" defaultResolvers := map[string]string{ "https://dns.eth.limo/dns-query": config.AutoPlaceholder, "https://dns.eth.link/dns-query": config.AutoPlaceholder, "https://resolver.cloudflare-eth.com/dns-query": config.AutoPlaceholder, } // Replace default resolvers with "auto" stringResolvers := common.ReplaceDefaultsWithAuto(resolvers, defaultResolvers) // Ensure "." is set to "auto" if not already set if _, exists := stringResolvers["."]; !exists { stringResolvers["."] = config.AutoPlaceholder } dns["Resolvers"] = stringResolvers return nil } // migrateDelegatedRouters migrates DelegatedRouters to use "auto" func migrateDelegatedRouters(confMap map[string]any) error { // Get or create Routing section routing := common.GetOrCreateSection(confMap, "Routing") // Get existing delegated routers delegatedRouters, exists := routing["DelegatedRouters"] // Check if it's empty or nil if !exists || common.IsEmptySlice(delegatedRouters) { routing["DelegatedRouters"] = []string{config.AutoPlaceholder} return nil } // Process the list to replace cid.contact with "auto" and preserve others routers := common.ConvertInterfaceSlice(common.SafeCastSlice(delegatedRouters)) var newRouters []string hasAuto := false for _, router := range routers { if router == "https://cid.contact" { if !hasAuto { newRouters = append(newRouters, config.AutoPlaceholder) hasAuto = true } } else { newRouters = append(newRouters, router) } } // If empty after processing, add "auto" if len(newRouters) == 0 { newRouters = []string{config.AutoPlaceholder} } routing["DelegatedRouters"] = newRouters return nil } // migrateDelegatedPublishers migrates DelegatedPublishers to use "auto" func migrateDelegatedPublishers(confMap map[string]any) error { // Get or create Ipns section ipns := common.GetOrCreateSection(confMap, "Ipns") // Get existing delegated publishers delegatedPublishers, exists := ipns["DelegatedPublishers"] // Check if it's empty or nil - only then replace with "auto" // Otherwise preserve custom publishers if !exists || common.IsEmptySlice(delegatedPublishers) { ipns["DelegatedPublishers"] = []string{config.AutoPlaceholder} } // If there are custom publishers, leave them as is return nil } ================================================ FILE: repo/fsrepo/migrations/fs-repo-16-to-17/migration/migration_test.go ================================================ package mg16 import ( "bytes" "encoding/json" "maps" "os" "path/filepath" "testing" "github.com/ipfs/kubo/repo/fsrepo/migrations/common" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // Helper function to run migration on JSON input and return result func runMigrationOnJSON(t *testing.T, input string) map[string]any { t.Helper() var output bytes.Buffer err := convert(bytes.NewReader([]byte(input)), &output) require.NoError(t, err) var result map[string]any err = json.Unmarshal(output.Bytes(), &result) require.NoError(t, err) return result } // Helper function to assert nested map key has expected value func assertMapKeyEquals(t *testing.T, result map[string]any, path []string, key string, expected any) { t.Helper() current := result for _, p := range path { section, exists := current[p] require.True(t, exists, "Section %s not found in path %v", p, path) current = section.(map[string]any) } assert.Equal(t, expected, current[key], "Expected %s to be %v", key, expected) } // Helper function to assert slice contains expected values func assertSliceEquals(t *testing.T, result map[string]any, path []string, expected []string) { t.Helper() current := result for i, p := range path[:len(path)-1] { section, exists := current[p] require.True(t, exists, "Section %s not found in path %v at index %d", p, path, i) current = section.(map[string]any) } sliceKey := path[len(path)-1] slice, exists := current[sliceKey] require.True(t, exists, "Slice %s not found", sliceKey) actualSlice := slice.([]any) require.Equal(t, len(expected), len(actualSlice), "Expected slice length %d, got %d", len(expected), len(actualSlice)) for i, exp := range expected { assert.Equal(t, exp, actualSlice[i], "Expected slice[%d] to be %s", i, exp) } } // Helper to build test config JSON with specified fields func buildTestConfig(fields map[string]any) string { config := map[string]any{ "Identity": map[string]any{"PeerID": "QmTest"}, } maps.Copy(config, fields) data, _ := json.MarshalIndent(config, "", " ") return string(data) } // Helper to run migration and get DNS resolvers func runMigrationAndGetDNSResolvers(t *testing.T, input string) map[string]any { t.Helper() result := runMigrationOnJSON(t, input) dns := result["DNS"].(map[string]any) return dns["Resolvers"].(map[string]any) } // Helper to assert multiple resolver values func assertResolvers(t *testing.T, resolvers map[string]any, expected map[string]string) { t.Helper() for key, expectedValue := range expected { assert.Equal(t, expectedValue, resolvers[key], "Expected %s resolver to be %v", key, expectedValue) } } // ============================================================================= // End-to-End Migration Tests // ============================================================================= func TestMigration(t *testing.T) { // Create a temporary directory for testing tempDir, err := os.MkdirTemp("", "migration-test-16-to-17") require.NoError(t, err) defer os.RemoveAll(tempDir) // Create a test config with default bootstrap peers testConfig := map[string]any{ "Bootstrap": []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", "/ip4/192.168.1.1/tcp/4001/p2p/QmCustomPeer", // Custom peer }, "DNS": map[string]any{ "Resolvers": map[string]string{}, }, "Routing": map[string]any{ "DelegatedRouters": []string{}, }, "Ipns": map[string]any{ "ResolveCacheSize": 128, }, "Identity": map[string]any{ "PeerID": "QmTest", }, "Version": map[string]any{ "Current": "0.36.0", }, } // Write test config configPath := filepath.Join(tempDir, "config") configData, err := json.MarshalIndent(testConfig, "", " ") require.NoError(t, err) err = os.WriteFile(configPath, configData, 0644) require.NoError(t, err) // Create version file versionPath := filepath.Join(tempDir, "version") err = os.WriteFile(versionPath, []byte("16"), 0644) require.NoError(t, err) // Run migration opts := common.Options{ Path: tempDir, Verbose: true, } err = Migration.Apply(opts) require.NoError(t, err) // Verify version was updated versionData, err := os.ReadFile(versionPath) require.NoError(t, err) assert.Equal(t, "17", string(versionData), "Expected version 17") // Verify config was updated configData, err = os.ReadFile(configPath) require.NoError(t, err) var updatedConfig map[string]any err = json.Unmarshal(configData, &updatedConfig) require.NoError(t, err) // Check AutoConf was added autoConf, exists := updatedConfig["AutoConf"] assert.True(t, exists, "AutoConf section not added") autoConfMap := autoConf.(map[string]any) // URL is not set explicitly in migration (uses implicit default) _, hasURL := autoConfMap["URL"] assert.False(t, hasURL, "AutoConf URL should not be explicitly set in migration") // Check Bootstrap was updated bootstrap := updatedConfig["Bootstrap"].([]any) assert.Equal(t, 2, len(bootstrap), "Expected 2 bootstrap entries") assert.Equal(t, "auto", bootstrap[0], "Expected first bootstrap entry to be 'auto'") assert.Equal(t, "/ip4/192.168.1.1/tcp/4001/p2p/QmCustomPeer", bootstrap[1], "Expected custom peer to be preserved") // Check DNS.Resolvers was updated dns := updatedConfig["DNS"].(map[string]any) resolvers := dns["Resolvers"].(map[string]any) assert.Equal(t, "auto", resolvers["."], "Expected DNS resolver for '.' to be 'auto'") // Check Routing.DelegatedRouters was updated routing := updatedConfig["Routing"].(map[string]any) delegatedRouters := routing["DelegatedRouters"].([]any) assert.Equal(t, 1, len(delegatedRouters)) assert.Equal(t, "auto", delegatedRouters[0], "Expected DelegatedRouters to be ['auto']") // Check Ipns.DelegatedPublishers was updated ipns := updatedConfig["Ipns"].(map[string]any) delegatedPublishers := ipns["DelegatedPublishers"].([]any) assert.Equal(t, 1, len(delegatedPublishers)) assert.Equal(t, "auto", delegatedPublishers[0], "Expected DelegatedPublishers to be ['auto']") // Test revert err = Migration.Revert(opts) require.NoError(t, err) // Verify version was reverted versionData, err = os.ReadFile(versionPath) require.NoError(t, err) assert.Equal(t, "16", string(versionData), "Expected version 16 after revert") } func TestConvert(t *testing.T) { t.Parallel() input := buildTestConfig(map[string]any{ "Bootstrap": []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", }, }) result := runMigrationOnJSON(t, input) // Check that AutoConf section was added but is empty (using implicit defaults) autoConf, exists := result["AutoConf"] require.True(t, exists, "AutoConf section should exist") autoConfMap, ok := autoConf.(map[string]any) require.True(t, ok, "AutoConf should be a map") require.Empty(t, autoConfMap, "AutoConf should be empty (using implicit defaults)") // Check that Bootstrap was updated to "auto" assertSliceEquals(t, result, []string{"Bootstrap"}, []string{"auto"}) } // ============================================================================= // Bootstrap Migration Tests // ============================================================================= func TestBootstrapMigration(t *testing.T) { t.Parallel() t.Run("process bootstrap peers logic verification", func(t *testing.T) { t.Parallel() tests := []struct { name string peers []string expected []string }{ { name: "empty peers", peers: []string{}, expected: []string{"auto"}, }, { name: "only default peers", peers: []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", }, expected: []string{"auto"}, }, { name: "mixed default and custom peers", peers: []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/ip4/192.168.1.1/tcp/4001/p2p/QmCustomPeer", }, expected: []string{"auto", "/ip4/192.168.1.1/tcp/4001/p2p/QmCustomPeer"}, }, { name: "only custom peers", peers: []string{ "/ip4/192.168.1.1/tcp/4001/p2p/QmCustomPeer1", "/ip4/192.168.1.2/tcp/4001/p2p/QmCustomPeer2", }, expected: []string{ "/ip4/192.168.1.1/tcp/4001/p2p/QmCustomPeer1", "/ip4/192.168.1.2/tcp/4001/p2p/QmCustomPeer2", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() result := processBootstrapPeers(tt.peers) require.Equal(t, len(tt.expected), len(result), "Expected %d peers, got %d", len(tt.expected), len(result)) for i, expected := range tt.expected { assert.Equal(t, expected, result[i], "Expected peer %d to be %s", i, expected) } }) } }) t.Run("replaces all old default bootstrapper peers with auto entry", func(t *testing.T) { t.Parallel() input := buildTestConfig(map[string]any{ "Bootstrap": []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", "/dnsaddr/va1.bootstrap.libp2p.io/p2p/12D3KooWKnDdG3iXw9eTFijk3EWSunZcFi54Zka4wmtqtt6rPxc8", "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", }, }) result := runMigrationOnJSON(t, input) assertSliceEquals(t, result, []string{"Bootstrap"}, []string{"auto"}) }) t.Run("creates Bootstrap section with auto when missing", func(t *testing.T) { t.Parallel() input := `{"Identity": {"PeerID": "QmTest"}}` result := runMigrationOnJSON(t, input) assertSliceEquals(t, result, []string{"Bootstrap"}, []string{"auto"}) }) } // ============================================================================= // DNS Migration Tests // ============================================================================= func TestDNSMigration(t *testing.T) { t.Parallel() t.Run("creates DNS section with auto resolver when missing", func(t *testing.T) { t.Parallel() input := `{"Identity": {"PeerID": "QmTest"}}` result := runMigrationOnJSON(t, input) assertMapKeyEquals(t, result, []string{"DNS", "Resolvers"}, ".", "auto") }) t.Run("preserves all custom DNS resolvers unchanged", func(t *testing.T) { t.Parallel() input := buildTestConfig(map[string]any{ "DNS": map[string]any{ "Resolvers": map[string]string{ ".": "https://my-custom-resolver.com", ".eth": "https://eth.resolver", }, }, }) resolvers := runMigrationAndGetDNSResolvers(t, input) assertResolvers(t, resolvers, map[string]string{ ".": "https://my-custom-resolver.com", ".eth": "https://eth.resolver", }) }) t.Run("preserves custom dot and eth resolvers unchanged", func(t *testing.T) { t.Parallel() input := buildTestConfig(map[string]any{ "DNS": map[string]any{ "Resolvers": map[string]string{ ".": "https://cloudflare-dns.com/dns-query", ".eth": "https://example.com/dns-query", }, }, }) resolvers := runMigrationAndGetDNSResolvers(t, input) assertResolvers(t, resolvers, map[string]string{ ".": "https://cloudflare-dns.com/dns-query", ".eth": "https://example.com/dns-query", }) }) t.Run("replaces old default eth resolver with auto", func(t *testing.T) { t.Parallel() input := buildTestConfig(map[string]any{ "DNS": map[string]any{ "Resolvers": map[string]string{ ".": "https://cloudflare-dns.com/dns-query", ".eth": "https://dns.eth.limo/dns-query", // should be replaced ".crypto": "https://resolver.cloudflare-eth.com/dns-query", // should be replaced ".link": "https://dns.eth.link/dns-query", // should be replaced }, }, }) resolvers := runMigrationAndGetDNSResolvers(t, input) assertResolvers(t, resolvers, map[string]string{ ".": "https://cloudflare-dns.com/dns-query", // preserved ".eth": "auto", // replaced ".crypto": "auto", // replaced ".link": "auto", // replaced }) }) } // ============================================================================= // Routing Migration Tests // ============================================================================= func TestRoutingMigration(t *testing.T) { t.Parallel() t.Run("creates Routing section with auto DelegatedRouters when missing", func(t *testing.T) { t.Parallel() input := `{"Identity": {"PeerID": "QmTest"}}` result := runMigrationOnJSON(t, input) assertSliceEquals(t, result, []string{"Routing", "DelegatedRouters"}, []string{"auto"}) }) t.Run("replaces cid.contact with auto while preserving custom routers added by user", func(t *testing.T) { t.Parallel() input := buildTestConfig(map[string]any{ "Routing": map[string]any{ "DelegatedRouters": []string{ "https://cid.contact", "https://my-custom-router.com", }, }, }) result := runMigrationOnJSON(t, input) assertSliceEquals(t, result, []string{"Routing", "DelegatedRouters"}, []string{"auto", "https://my-custom-router.com"}) }) } // ============================================================================= // IPNS Migration Tests // ============================================================================= func TestIpnsMigration(t *testing.T) { t.Parallel() t.Run("creates Ipns section with auto DelegatedPublishers when missing", func(t *testing.T) { t.Parallel() input := `{"Identity": {"PeerID": "QmTest"}}` result := runMigrationOnJSON(t, input) assertSliceEquals(t, result, []string{"Ipns", "DelegatedPublishers"}, []string{"auto"}) }) t.Run("preserves existing custom DelegatedPublishers unchanged", func(t *testing.T) { t.Parallel() input := buildTestConfig(map[string]any{ "Ipns": map[string]any{ "DelegatedPublishers": []string{ "https://my-publisher.com", "https://another-publisher.com", }, }, }) result := runMigrationOnJSON(t, input) assertSliceEquals(t, result, []string{"Ipns", "DelegatedPublishers"}, []string{"https://my-publisher.com", "https://another-publisher.com"}) }) t.Run("adds auto DelegatedPublishers to existing Ipns section", func(t *testing.T) { t.Parallel() input := buildTestConfig(map[string]any{ "Ipns": map[string]any{ "ResolveCacheSize": 128, }, }) result := runMigrationOnJSON(t, input) assertMapKeyEquals(t, result, []string{"Ipns"}, "ResolveCacheSize", float64(128)) assertSliceEquals(t, result, []string{"Ipns", "DelegatedPublishers"}, []string{"auto"}) }) } // ============================================================================= // AutoConf Migration Tests // ============================================================================= func TestAutoConfMigration(t *testing.T) { t.Parallel() t.Run("preserves existing AutoConf fields unchanged", func(t *testing.T) { t.Parallel() input := buildTestConfig(map[string]any{ "AutoConf": map[string]any{ "URL": "https://custom.example.com/autoconf.json", "Enabled": false, "CustomField": "preserved", }, }) result := runMigrationOnJSON(t, input) assertMapKeyEquals(t, result, []string{"AutoConf"}, "URL", "https://custom.example.com/autoconf.json") assertMapKeyEquals(t, result, []string{"AutoConf"}, "Enabled", false) assertMapKeyEquals(t, result, []string{"AutoConf"}, "CustomField", "preserved") }) } ================================================ FILE: repo/fsrepo/migrations/fs-repo-17-to-18/main.go ================================================ // Package main implements fs-repo-17-to-18 migration for IPFS repositories. // // This migration consolidates the Provider and Reprovider configurations into // a unified Provide configuration section. // // Changes made: // - Migrates Provider.Enabled to Provide.Enabled // - Migrates Provider.WorkerCount to Provide.DHT.MaxWorkers // - Migrates Reprovider.Strategy to Provide.Strategy (converts "flat" to "all") // - Migrates Reprovider.Interval to Provide.DHT.Interval // - Removes deprecated Provider and Reprovider sections // // The migration is reversible and creates config.17-to-18.bak for rollback. // // Usage: // // fs-repo-17-to-18 -path /path/to/ipfs/repo [-verbose] [-revert] // // This migration is embedded in Kubo and runs automatically during daemon startup. // This standalone binary is provided for manual migration scenarios. package main import ( "flag" "fmt" "os" "github.com/ipfs/kubo/repo/fsrepo/migrations/common" mg17 "github.com/ipfs/kubo/repo/fsrepo/migrations/fs-repo-17-to-18/migration" ) func main() { var path = flag.String("path", "", "Path to IPFS repository") var verbose = flag.Bool("verbose", false, "Enable verbose output") var revert = flag.Bool("revert", false, "Revert migration") flag.Parse() if *path == "" { fmt.Fprintf(os.Stderr, "Error: -path flag is required\n") flag.Usage() os.Exit(1) } opts := common.Options{ Path: *path, Verbose: *verbose, } var err error if *revert { err = mg17.Migration.Revert(opts) } else { err = mg17.Migration.Apply(opts) } if err != nil { fmt.Fprintf(os.Stderr, "Migration failed: %v\n", err) os.Exit(1) } } ================================================ FILE: repo/fsrepo/migrations/fs-repo-17-to-18/migration/migration.go ================================================ // package mg17 contains the code to perform 17-18 repository migration in Kubo. // This handles the following: // - Migrate Provider and Reprovider configs to unified Provide config // - Clear deprecated Provider and Reprovider fields // - Increment repo version to 18 package mg17 import ( "fmt" "io" "github.com/ipfs/kubo/repo/fsrepo/migrations/common" ) // Migration is the main exported migration for 17-to-18 var Migration = &common.BaseMigration{ FromVersion: "17", ToVersion: "18", Description: "Migrating Provider and Reprovider configuration to unified Provide configuration", Convert: convert, } // NewMigration creates a new migration instance (for compatibility) func NewMigration() common.Migration { return Migration } // convert performs the actual configuration transformation func convert(in io.ReadSeeker, out io.Writer) error { // Read the configuration confMap, err := common.ReadConfig(in) if err != nil { return err } // Create new Provide section with DHT subsection from Provider and Reprovider provide := make(map[string]any) dht := make(map[string]any) hasNonDefaultValues := false // Migrate Provider fields if they exist provider := common.SafeCastMap(confMap["Provider"]) if enabled, exists := provider["Enabled"]; exists { provide["Enabled"] = enabled // Log migration for non-default values if enabledBool, ok := enabled.(bool); ok && !enabledBool { fmt.Printf(" Migrated Provider.Enabled=%v to Provide.Enabled=%v\n", enabledBool, enabledBool) hasNonDefaultValues = true } } if workerCount, exists := provider["WorkerCount"]; exists { dht["MaxWorkers"] = workerCount // Log migration for all worker count values if count, ok := workerCount.(float64); ok { fmt.Printf(" Migrated Provider.WorkerCount=%v to Provide.DHT.MaxWorkers=%v\n", int(count), int(count)) hasNonDefaultValues = true // Additional guidance for high WorkerCount if count > 5 { fmt.Printf(" ⚠️ For better resource utilization, consider enabling Provide.DHT.SweepEnabled=true\n") fmt.Printf(" and adjusting Provide.DHT.DedicatedBurstWorkers if announcement of new CIDs\n") fmt.Printf(" should take priority over periodic reprovide interval.\n") } } } // Note: Skip Provider.Strategy as it was unused // Migrate Reprovider fields if they exist reprovider := common.SafeCastMap(confMap["Reprovider"]) if strategy, exists := reprovider["Strategy"]; exists { if strategyStr, ok := strategy.(string); ok { // Convert deprecated "flat" strategy to "all" if strategyStr == "flat" { provide["Strategy"] = "all" fmt.Printf(" Migrated deprecated Reprovider.Strategy=\"flat\" to Provide.Strategy=\"all\"\n") } else { // Migrate any other strategy value as-is provide["Strategy"] = strategyStr fmt.Printf(" Migrated Reprovider.Strategy=\"%s\" to Provide.Strategy=\"%s\"\n", strategyStr, strategyStr) } hasNonDefaultValues = true } else { // Not a string, set to default "all" to ensure valid config provide["Strategy"] = "all" fmt.Printf(" Warning: Reprovider.Strategy was not a string, setting Provide.Strategy=\"all\"\n") hasNonDefaultValues = true } } if interval, exists := reprovider["Interval"]; exists { dht["Interval"] = interval // Log migration for non-default intervals if intervalStr, ok := interval.(string); ok && intervalStr != "22h" && intervalStr != "" { fmt.Printf(" Migrated Reprovider.Interval=\"%s\" to Provide.DHT.Interval=\"%s\"\n", intervalStr, intervalStr) hasNonDefaultValues = true } } // Note: Sweep is a new field introduced in v0.38, not present in v0.37 // So we don't need to migrate it from Reprovider // Set the DHT section if we have any DHT fields to migrate if len(dht) > 0 { provide["DHT"] = dht } // Set the new Provide section if we have any fields to migrate if len(provide) > 0 { confMap["Provide"] = provide } // Clear old Provider and Reprovider sections delete(confMap, "Provider") delete(confMap, "Reprovider") // Print documentation link if we migrated any non-default values if hasNonDefaultValues { fmt.Printf(" See: https://github.com/ipfs/kubo/blob/master/docs/config.md#provide\n") } // Write the updated config return common.WriteConfig(out, confMap) } ================================================ FILE: repo/fsrepo/migrations/fs-repo-17-to-18/migration/migration_test.go ================================================ package mg17 import ( "testing" "github.com/ipfs/kubo/repo/fsrepo/migrations/common" ) func TestMigration17to18(t *testing.T) { migration := NewMigration() testCases := []common.TestCase{ { Name: "Migrate Provider and Reprovider to Provide", InputConfig: common.GenerateTestConfig(map[string]any{ "Provider": map[string]any{ "Enabled": true, "WorkerCount": 8, "Strategy": "unused", // This field was unused and should be ignored }, "Reprovider": map[string]any{ "Strategy": "pinned", "Interval": "12h", }, }), Assertions: []common.ConfigAssertion{ {Path: "Provide.Enabled", Expected: true}, {Path: "Provide.DHT.MaxWorkers", Expected: float64(8)}, // JSON unmarshals to float64 {Path: "Provide.Strategy", Expected: "pinned"}, {Path: "Provide.DHT.Interval", Expected: "12h"}, {Path: "Provider", Expected: nil}, // Should be deleted {Path: "Reprovider", Expected: nil}, // Should be deleted }, }, { Name: "Convert flat strategy to all", InputConfig: common.GenerateTestConfig(map[string]any{ "Provider": map[string]any{ "Enabled": false, }, "Reprovider": map[string]any{ "Strategy": "flat", // Deprecated, should be converted to "all" "Interval": "24h", }, }), Assertions: []common.ConfigAssertion{ {Path: "Provide.Enabled", Expected: false}, {Path: "Provide.Strategy", Expected: "all"}, // "flat" converted to "all" {Path: "Provide.DHT.Interval", Expected: "24h"}, {Path: "Provider", Expected: nil}, {Path: "Reprovider", Expected: nil}, }, }, { Name: "Handle missing Provider section", InputConfig: common.GenerateTestConfig(map[string]any{ "Reprovider": map[string]any{ "Strategy": "roots", "Interval": "6h", }, }), Assertions: []common.ConfigAssertion{ {Path: "Provide.Strategy", Expected: "roots"}, {Path: "Provide.DHT.Interval", Expected: "6h"}, {Path: "Provider", Expected: nil}, {Path: "Reprovider", Expected: nil}, }, }, { Name: "Handle missing Reprovider section", InputConfig: common.GenerateTestConfig(map[string]any{ "Provider": map[string]any{ "Enabled": true, "WorkerCount": 16, }, }), Assertions: []common.ConfigAssertion{ {Path: "Provide.Enabled", Expected: true}, {Path: "Provide.DHT.MaxWorkers", Expected: float64(16)}, {Path: "Provider", Expected: nil}, {Path: "Reprovider", Expected: nil}, }, }, { Name: "Handle empty Provider and Reprovider sections", InputConfig: common.GenerateTestConfig(map[string]any{ "Provider": map[string]any{}, "Reprovider": map[string]any{}, }), Assertions: []common.ConfigAssertion{ {Path: "Provide", Expected: nil}, // No fields to migrate {Path: "Provider", Expected: nil}, {Path: "Reprovider", Expected: nil}, }, }, { Name: "Handle missing both sections", InputConfig: common.GenerateTestConfig(map[string]any{ "Datastore": map[string]any{ "StorageMax": "10GB", }, }), Assertions: []common.ConfigAssertion{ {Path: "Provide", Expected: nil}, // No Provider/Reprovider to migrate {Path: "Provider", Expected: nil}, {Path: "Reprovider", Expected: nil}, {Path: "Datastore.StorageMax", Expected: "10GB"}, // Other config preserved }, }, { Name: "Preserve other config sections", InputConfig: common.GenerateTestConfig(map[string]any{ "Provider": map[string]any{ "Enabled": true, }, "Reprovider": map[string]any{ "Strategy": "all", }, "Swarm": map[string]any{ "ConnMgr": map[string]any{ "Type": "basic", }, }, }), Assertions: []common.ConfigAssertion{ {Path: "Provide.Enabled", Expected: true}, {Path: "Provide.Strategy", Expected: "all"}, {Path: "Swarm.ConnMgr.Type", Expected: "basic"}, // Other config preserved {Path: "Provider", Expected: nil}, {Path: "Reprovider", Expected: nil}, }, }, } for _, tc := range testCases { t.Run(tc.Name, func(t *testing.T) { common.RunMigrationTest(t, migration, tc) }) } } func TestMigration17to18Reversible(t *testing.T) { migration := NewMigration() // Test that migration is reversible inputConfig := common.GenerateTestConfig(map[string]any{ "Provide": map[string]any{ "Enabled": true, "WorkerCount": 8, "Strategy": "pinned", "Interval": "12h", }, }) // Test full migration and revert migratedConfig := common.AssertMigrationSuccess(t, migration, 17, 18, inputConfig) // Check that Provide section exists after migration common.AssertConfigField(t, migratedConfig, "Provide.Enabled", true) // Test revert common.AssertMigrationReversible(t, migration, 17, 18, migratedConfig) } func TestMigration17to18Integration(t *testing.T) { migration := NewMigration() // Test that the migration properly integrates with the common framework if migration.Versions() != "17-to-18" { t.Errorf("expected versions '17-to-18', got '%s'", migration.Versions()) } if !migration.Reversible() { t.Error("migration should be reversible") } } ================================================ FILE: repo/fsrepo/migrations/httpfetcher.go ================================================ package migrations import ( "context" "errors" "fmt" "io" "net/http" gopath "path" "strings" "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/exchange/offline" bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" files "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/ipld/merkledag" unixfile "github.com/ipfs/boxo/ipld/unixfs/file" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/namesys" "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/path/resolver" "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" "github.com/ipfs/go-unixfsnode" gocarv2 "github.com/ipld/go-car/v2" dagpb "github.com/ipld/go-codec-dagpb" madns "github.com/multiformats/go-multiaddr-dns" ) const ( // default is different name than ipfs.io which is being blocked by some ISPs defaultGatewayURL = "https://trustless-gateway.link" // Default maximum download size. defaultFetchLimit = 1024 * 1024 * 512 ) // HttpFetcher fetches files over HTTP using verifiable CAR archives. type HttpFetcher struct { //nolint distPath string gateway string limit int64 userAgent string } var _ Fetcher = (*HttpFetcher)(nil) // NewHttpFetcher creates a new [HttpFetcher]. // // Specifying "" for distPath sets the default IPNS path. // Specifying "" for gateway sets the default. // Specifying 0 for fetchLimit sets the default, -1 means no limit. func NewHttpFetcher(distPath, gateway, userAgent string, fetchLimit int64) *HttpFetcher { //nolint f := &HttpFetcher{ distPath: LatestIpfsDist, gateway: defaultGatewayURL, limit: defaultFetchLimit, } if distPath != "" { if !strings.HasPrefix(distPath, "/") { distPath = "/" + distPath } f.distPath = distPath } if gateway != "" { f.gateway = strings.TrimRight(gateway, "/") } if fetchLimit != 0 { if fetchLimit < 0 { fetchLimit = 0 } f.limit = fetchLimit } return f } // Fetch attempts to fetch the file at the given path, from the distribution // site configured for this HttpFetcher. func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error) { imPath, err := f.resolvePath(ctx, gopath.Join(f.distPath, filePath)) if err != nil { return nil, fmt.Errorf("path could not be resolved: %w", err) } rc, err := f.httpRequest(ctx, imPath, "application/vnd.ipld.car") if err != nil { return nil, fmt.Errorf("failed to fetch CAR: %w", err) } return carStreamToFileBytes(ctx, rc, imPath) } func (f *HttpFetcher) Close() error { return nil } func (f *HttpFetcher) resolvePath(ctx context.Context, pathStr string) (path.ImmutablePath, error) { p, err := path.NewPath(pathStr) if err != nil { return path.ImmutablePath{}, fmt.Errorf("path is invalid: %w", err) } for p.Mutable() { // Download IPNS record and verify through the gateway, or resolve the // DNSLink with the default DNS resolver. name, err := ipns.NameFromString(p.Segments()[1]) if err == nil { p, err = f.resolveIPNS(ctx, name) } else { p, err = f.resolveDNSLink(ctx, p) } if err != nil { return path.ImmutablePath{}, err } } return path.NewImmutablePath(p) } func (f *HttpFetcher) resolveIPNS(ctx context.Context, name ipns.Name) (path.Path, error) { rc, err := f.httpRequest(ctx, name.AsPath(), "application/vnd.ipfs.ipns-record") if err != nil { return path.ImmutablePath{}, err } rc = NewLimitReadCloser(rc, int64(ipns.MaxRecordSize)) rawRecord, err := io.ReadAll(rc) if err != nil { return path.ImmutablePath{}, err } rec, err := ipns.UnmarshalRecord(rawRecord) if err != nil { return path.ImmutablePath{}, err } err = ipns.ValidateWithName(rec, name) if err != nil { return path.ImmutablePath{}, err } return rec.Value() } func (f *HttpFetcher) resolveDNSLink(ctx context.Context, p path.Path) (path.Path, error) { dnsResolver := namesys.NewDNSResolver(madns.DefaultResolver.LookupTXT) res, err := dnsResolver.Resolve(ctx, p) if err != nil { return nil, err } return res.Path, nil } func (f *HttpFetcher) httpRequest(ctx context.Context, p path.Path, accept string) (io.ReadCloser, error) { url := f.gateway + p.String() fmt.Printf("Fetching with HTTP: %q\n", url) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, fmt.Errorf("http.NewRequest error: %w", err) } req.Header.Set("Accept", accept) if f.userAgent != "" { req.Header.Set("User-Agent", f.userAgent) } resp, err := http.DefaultClient.Do(req) if err != nil { return nil, fmt.Errorf("http.DefaultClient.Do error: %w", err) } if resp.StatusCode >= 400 { defer resp.Body.Close() mes, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("error reading error body: %w", err) } return nil, fmt.Errorf("GET %s error: %s: %s", url, resp.Status, string(mes)) } var rc io.ReadCloser if f.limit != 0 { rc = NewLimitReadCloser(resp.Body, f.limit) } else { rc = resp.Body } return rc, nil } func carStreamToFileBytes(ctx context.Context, r io.ReadCloser, imPath path.ImmutablePath) ([]byte, error) { defer r.Close() // Create temporary block datastore and dag service. dataStore := dssync.MutexWrap(datastore.NewMapDatastore()) blockStore := blockstore.NewBlockstore(dataStore) blockService := blockservice.New(blockStore, offline.Exchange(blockStore)) dagService := merkledag.NewDAGService(blockService) defer dagService.Blocks.Close() defer dataStore.Close() // Create CAR reader car, err := gocarv2.NewBlockReader(r) if err != nil { fmt.Println(err) return nil, fmt.Errorf("error creating car reader: %s", err) } // Add all blocks to the blockstore. for { block, err := car.Next() if err != nil && err != io.EOF { return nil, fmt.Errorf("error reading block from car: %s", err) } else if block == nil { break } err = blockStore.Put(ctx, block) if err != nil { return nil, fmt.Errorf("error putting block in blockstore: %s", err) } } fetcherCfg := bsfetcher.NewFetcherConfig(blockService) fetcherCfg.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser) fetcher := fetcherCfg.WithReifier(unixfsnode.Reify) resolver := resolver.NewBasicResolver(fetcher) cid, _, err := resolver.ResolveToLastNode(ctx, imPath) if err != nil { return nil, fmt.Errorf("failed to resolve: %w", err) } nd, err := dagService.Get(ctx, cid) if err != nil { return nil, fmt.Errorf("failed to resolve: %w", err) } // Make UnixFS file out of the node. uf, err := unixfile.NewUnixfsFile(ctx, dagService, nd) if err != nil { return nil, fmt.Errorf("error building unixfs file: %s", err) } // Check if it's a file and return. if f, ok := uf.(files.File); ok { return io.ReadAll(f) } return nil, errors.New("unexpected unixfs node type") } ================================================ FILE: repo/fsrepo/migrations/ipfsdir.go ================================================ package migrations import ( "errors" "fmt" "os" "path/filepath" "strconv" "strings" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/misc/fsutil" ) const ( versionFile = "version" ) // IpfsDir returns the path of the ipfs directory. If dir specified, then // returns the expanded version dir. If dir is "", then return the directory // set by IPFS_PATH, or if IPFS_PATH is not set, then return the default // location in the home directory. func IpfsDir(dir string) (string, error) { var err error if dir == "" { dir, err = config.PathRoot() if err != nil { return "", err } } dir, err = fsutil.ExpandHome(dir) if err != nil { return "", err } return dir, nil } // CheckIpfsDir gets the ipfs directory and checks that the directory exists. func CheckIpfsDir(dir string) (string, error) { var err error dir, err = IpfsDir(dir) if err != nil { return "", err } _, err = os.Stat(dir) if err != nil { return "", err } return dir, nil } // RepoVersion returns the version of the repo in the ipfs directory. If the // ipfs directory is not specified then the default location is used. func RepoVersion(ipfsDir string) (int, error) { ipfsDir, err := CheckIpfsDir(ipfsDir) if err != nil { return 0, err } return repoVersion(ipfsDir) } // WriteRepoVersion writes the specified repo version to the repo located in // ipfsDir. If ipfsDir is not specified, then the default location is used. func WriteRepoVersion(ipfsDir string, version int) error { ipfsDir, err := IpfsDir(ipfsDir) if err != nil { return err } vFilePath := filepath.Join(ipfsDir, versionFile) return os.WriteFile(vFilePath, fmt.Appendf(nil, "%d\n", version), 0o644) } func repoVersion(ipfsDir string) (int, error) { c, err := os.ReadFile(filepath.Join(ipfsDir, versionFile)) if err != nil { return 0, err } ver, err := strconv.Atoi(strings.TrimSpace(string(c))) if err != nil { return 0, errors.New("invalid data in repo version file") } return ver, nil } ================================================ FILE: repo/fsrepo/migrations/ipfsdir_test.go ================================================ package migrations import ( "os" "path/filepath" "testing" "github.com/ipfs/kubo/config" ) func TestRepoDir(t *testing.T) { fakeHome := t.TempDir() t.Setenv("HOME", fakeHome) // On Windows, os.UserHomeDir() uses USERPROFILE, not HOME t.Setenv("USERPROFILE", fakeHome) fakeIpfs := filepath.Join(fakeHome, ".ipfs") t.Setenv(config.EnvDir, fakeIpfs) t.Run("testIpfsDir", func(t *testing.T) { testIpfsDir(t, fakeIpfs) }) t.Run("testCheckIpfsDir", func(t *testing.T) { testCheckIpfsDir(t, fakeIpfs) }) t.Run("testRepoVersion", func(t *testing.T) { testRepoVersion(t, fakeIpfs) }) } func testIpfsDir(t *testing.T, fakeIpfs string) { _, err := CheckIpfsDir("") if err == nil { t.Fatal("expected error when no .ipfs directory to find") } err = os.Mkdir(fakeIpfs, os.ModePerm) if err != nil { panic(err) } dir, err := IpfsDir("") if err != nil { t.Fatal(err) } if dir != fakeIpfs { t.Fatalf("wrong ipfs directory: got %s, expected %s", dir, fakeIpfs) } t.Setenv(config.EnvDir, "~/.ipfs") dir, err = IpfsDir("") if err != nil { t.Fatal(err) } if dir != fakeIpfs { t.Fatalf("wrong ipfs directory: got %s, expected %s", dir, fakeIpfs) } _, err = IpfsDir("~somesuer/foo") if err == nil { t.Fatal("expected error with user-specific home dir") } t.Setenv(config.EnvDir, "~somesuer/foo") _, err = IpfsDir("~somesuer/foo") if err == nil { t.Fatal("expected error with user-specific home dir") } err = os.Unsetenv(config.EnvDir) if err != nil { panic(err) } dir, err = IpfsDir("~/.ipfs") if err != nil { t.Fatal(err) } if dir != fakeIpfs { t.Fatalf("wrong ipfs directory: got %s, expected %s", dir, fakeIpfs) } _, err = IpfsDir("") if err != nil { t.Fatal(err) } } func testCheckIpfsDir(t *testing.T, fakeIpfs string) { _, err := CheckIpfsDir("~somesuer/foo") if err == nil { t.Fatal("expected error with user-specific home dir") } _, err = CheckIpfsDir("~/no_such_dir") if err == nil { t.Fatal("expected error from nonexistent directory") } dir, err := CheckIpfsDir("~/.ipfs") if err != nil { t.Fatal(err) } if dir != fakeIpfs { t.Fatal("wrong ipfs directory:", dir) } } func testRepoVersion(t *testing.T, fakeIpfs string) { badDir := "~somesuer/foo" _, err := RepoVersion(badDir) if err == nil { t.Fatal("expected error with user-specific home dir") } _, err = RepoVersion(fakeIpfs) if !os.IsNotExist(err) { t.Fatal("expected not-exist error") } testVer := 42 err = WriteRepoVersion(fakeIpfs, testVer) if err != nil { t.Fatal(err) } var ver int ver, err = RepoVersion(fakeIpfs) if err != nil { t.Fatal(err) } if ver != testVer { t.Fatalf("expected version %d, got %d", testVer, ver) } err = WriteRepoVersion(badDir, testVer) if err == nil { t.Fatal("expected error with user-specific home dir") } ipfsDir, err := IpfsDir(fakeIpfs) if err != nil { t.Fatal(err) } vFilePath := filepath.Join(ipfsDir, versionFile) err = os.WriteFile(vFilePath, []byte("bad-version-data\n"), 0o644) if err != nil { panic(err) } _, err = RepoVersion(fakeIpfs) if err == nil || err.Error() != "invalid data in repo version file" { t.Fatal("expected 'invalid data' error") } err = WriteRepoVersion(fakeIpfs, testVer) if err != nil { t.Fatal(err) } } ================================================ FILE: repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go ================================================ package ipfsfetcher import ( "context" "encoding/json" "fmt" "io" "net/url" "os" gopath "path" "strings" "sync" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/path" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/coreapi" iface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/core/coreiface/options" "github.com/ipfs/kubo/core/node/libp2p" "github.com/ipfs/kubo/repo/fsrepo" "github.com/ipfs/kubo/repo/fsrepo/migrations" peer "github.com/libp2p/go-libp2p/core/peer" ) const ( // Default maximum download size. defaultFetchLimit = 1024 * 1024 * 512 tempNodeTCPAddr = "/ip4/127.0.0.1/tcp/0" ) type IpfsFetcher struct { distPath string limit int64 repoRoot *string userConfigFile string openOnce sync.Once openErr error closeOnce sync.Once closeErr error ipfs iface.CoreAPI ipfsTmpDir string ipfsStopFunc func() fetched []path.Path mutex sync.Mutex addrInfo peer.AddrInfo } var _ migrations.Fetcher = (*IpfsFetcher)(nil) // NewIpfsFetcher creates a new IpfsFetcher // // Specifying "" for distPath sets the default IPNS path. // Specifying 0 for fetchLimit sets the default, -1 means no limit. // // Bootstrap and peer information in read from the IPFS config file in // repoRoot, unless repoRoot is nil. If repoRoot is empty (""), then read the // config from the default IPFS directory. func NewIpfsFetcher(distPath string, fetchLimit int64, repoRoot *string, userConfigFile string) *IpfsFetcher { f := &IpfsFetcher{ limit: defaultFetchLimit, distPath: migrations.LatestIpfsDist, repoRoot: repoRoot, userConfigFile: userConfigFile, } if distPath != "" { if !strings.HasPrefix(distPath, "/") { distPath = "/" + distPath } f.distPath = distPath } if fetchLimit != 0 { if fetchLimit < 0 { fetchLimit = 0 } f.limit = fetchLimit } return f } // Fetch attempts to fetch the file at the given path, from the distribution // site configured for this HttpFetcher. func (f *IpfsFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error) { // Initialize and start IPFS node on first call to Fetch, since the fetcher // may be created by not used. f.openOnce.Do(func() { bootstrap, peers := readIpfsConfig(f.repoRoot, f.userConfigFile) f.ipfsTmpDir, f.openErr = initTempNode(ctx, bootstrap, peers) if f.openErr != nil { return } f.openErr = f.startTempNode(ctx) }) fmt.Printf("Fetching with IPFS: %q\n", filePath) if f.openErr != nil { return nil, f.openErr } iPath, err := parsePath(gopath.Join(f.distPath, filePath)) if err != nil { return nil, err } nd, err := f.ipfs.Unixfs().Get(ctx, iPath) if err != nil { return nil, err } f.recordFetched(iPath) fileNode, ok := nd.(files.File) if !ok { return nil, fmt.Errorf("%q is not a file", filePath) } var rc io.ReadCloser if f.limit != 0 { rc = migrations.NewLimitReadCloser(fileNode, f.limit) } else { rc = fileNode } defer rc.Close() return io.ReadAll(rc) } func (f *IpfsFetcher) Close() error { f.closeOnce.Do(func() { if f.ipfsStopFunc != nil { // Tell ipfs node to stop and wait for it to stop f.ipfsStopFunc() } if f.ipfsTmpDir != "" { // Remove the temp ipfs dir f.closeErr = os.RemoveAll(f.ipfsTmpDir) } }) return f.closeErr } func (f *IpfsFetcher) AddrInfo() peer.AddrInfo { return f.addrInfo } // FetchedPaths returns the IPFS paths of all items fetched by this fetcher. func (f *IpfsFetcher) FetchedPaths() []path.Path { f.mutex.Lock() defer f.mutex.Unlock() return f.fetched } func (f *IpfsFetcher) recordFetched(fetchedPath path.Path) { // Mutex protects against update by concurrent calls to Fetch f.mutex.Lock() defer f.mutex.Unlock() f.fetched = append(f.fetched, fetchedPath) } func initTempNode(ctx context.Context, bootstrap []string, peers []peer.AddrInfo) (string, error) { identity, err := config.CreateIdentity(io.Discard, []options.KeyGenerateOption{ options.Key.Type(options.Ed25519Key), }) if err != nil { return "", err } cfg, err := config.InitWithIdentity(identity) if err != nil { return "", err } // create temporary ipfs directory dir, err := os.MkdirTemp("", "ipfs-temp") if err != nil { return "", fmt.Errorf("failed to get temp dir: %s", err) } // configure the temporary node cfg.Routing.Type = config.NewOptionalString("dhtclient") // Disable listening for inbound connections cfg.Addresses.Gateway = []string{} cfg.Addresses.API = []string{} cfg.Addresses.Swarm = []string{tempNodeTCPAddr} if len(bootstrap) != 0 { cfg.Bootstrap = bootstrap } if len(peers) != 0 { cfg.Peering.Peers = peers } // Assumes that repo plugins are already loaded err = fsrepo.Init(dir, cfg) if err != nil { os.RemoveAll(dir) return "", fmt.Errorf("failed to initialize ephemeral node: %s", err) } return dir, nil } func (f *IpfsFetcher) startTempNode(ctx context.Context) error { // Open the repo r, err := fsrepo.Open(f.ipfsTmpDir) if err != nil { return err } // Create a new lifetime context that is used to stop the temp ipfs node ctxIpfsLife, cancel := context.WithCancel(context.Background()) // Construct the node node, err := core.NewNode(ctxIpfsLife, &core.BuildCfg{ Online: true, Routing: libp2p.DHTClientOption, Repo: r, }) if err != nil { cancel() r.Close() return err } ipfs, err := coreapi.NewCoreAPI(node) if err != nil { cancel() return err } stopFunc := func() { // Tell ipfs to stop cancel() // Wait until ipfs is stopped <-node.Context().Done() } addrs, err := ipfs.Swarm().LocalAddrs(ctx) if err != nil { // Failure to get the local swarm address only means that the // downloaded migrations cannot be fetched through the temporary node. // So, print the error message and keep going. fmt.Fprintln(os.Stderr, "cannot get local swarm address:", err) } f.addrInfo = peer.AddrInfo{ ID: node.Identity, Addrs: addrs, } f.ipfs = ipfs f.ipfsStopFunc = stopFunc return nil } func parsePath(fetchPath string) (path.Path, error) { if ipfsPath, err := path.NewPath(fetchPath); err == nil { return ipfsPath, nil } u, err := url.Parse(fetchPath) if err != nil { return nil, fmt.Errorf("%q could not be parsed: %s", fetchPath, err) } switch proto := u.Scheme; proto { case "ipfs", "ipld", "ipns": return path.NewPath(gopath.Join("/", proto, u.Host, u.Path)) default: return nil, fmt.Errorf("%q is not an IPFS path", fetchPath) } } func readIpfsConfig(repoRoot *string, userConfigFile string) (bootstrap []string, peers []peer.AddrInfo) { if repoRoot == nil { return } cfgPath, err := config.Filename(*repoRoot, userConfigFile) if err != nil { fmt.Fprintln(os.Stderr, err) return } cfgFile, err := os.Open(cfgPath) if err != nil { fmt.Fprintln(os.Stderr, err) return } defer cfgFile.Close() // Attempt to read bootstrap addresses var bootstrapCfg struct { Bootstrap []string } err = json.NewDecoder(cfgFile).Decode(&bootstrapCfg) if err != nil { fmt.Fprintln(os.Stderr, "cannot read bootstrap peers from config") } else { bootstrap = bootstrapCfg.Bootstrap } if _, err = cfgFile.Seek(0, 0); err != nil { // If Seek fails, only log the error and continue on to try to read the // peering config anyway as it might still be readable fmt.Fprintln(os.Stderr, err) } // Attempt to read peers var peeringCfg struct { Peering config.Peering } err = json.NewDecoder(cfgFile).Decode(&peeringCfg) if err != nil { fmt.Fprintln(os.Stderr, "cannot read peering from config") } else { peers = peeringCfg.Peering.Peers } return } ================================================ FILE: repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher_test.go ================================================ package ipfsfetcher import ( "bufio" "bytes" "fmt" "os" "path/filepath" "testing" "github.com/ipfs/kubo/plugin/loader" "github.com/ipfs/kubo/repo/fsrepo/migrations" ) func init() { err := setupPlugins() if err != nil { panic(err) } } func TestIpfsFetcher(t *testing.T) { skipUnlessEpic(t) ctx := t.Context() fetcher := NewIpfsFetcher("", 0, nil, "") defer fetcher.Close() out, err := fetcher.Fetch(ctx, "go-ipfs/versions") if err != nil { t.Fatal(err) } var lines []string scan := bufio.NewScanner(bytes.NewReader(out)) for scan.Scan() { lines = append(lines, scan.Text()) } err = scan.Err() if err != nil { t.Fatal("could not read versions:", err) } if len(lines) < 6 { t.Fatal("do not get all expected data") } if lines[0] != "v0.3.2" { t.Fatal("expected v1.0.0 as first line, got", lines[0]) } // Check not found if _, err = fetcher.Fetch(ctx, "/no_such_file"); err == nil { t.Fatal("expected error 404") } } func TestInitIpfsFetcher(t *testing.T) { ctx := t.Context() f := NewIpfsFetcher("", 0, nil, "") defer f.Close() // Init ipfs repo f.ipfsTmpDir, f.openErr = initTempNode(ctx, nil, nil) if f.openErr != nil { t.Fatalf("failed to initialize ipfs node: %s", f.openErr) } // Start ipfs node f.openErr = f.startTempNode(ctx) if f.openErr != nil { t.Errorf("failed to start ipfs node: %s", f.openErr) return } var stopFuncCalled bool stopFunc := f.ipfsStopFunc f.ipfsStopFunc = func() { stopFuncCalled = true stopFunc() } addrInfo := f.AddrInfo() if string(addrInfo.ID) == "" { t.Error("AddInfo ID not set") } if len(addrInfo.Addrs) == 0 { t.Error("AddInfo Addrs not set") } t.Log("Temp node listening on:", addrInfo.Addrs) err := f.Close() if err != nil { t.Fatalf("failed to close fetcher: %s", err) } if stopFunc != nil && !stopFuncCalled { t.Error("Close did not call stop function") } err = f.Close() if err != nil { t.Fatalf("failed to close fetcher 2nd time: %s", err) } } func TestReadIpfsConfig(t *testing.T) { testConfig := ` { "Bootstrap": [ "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" ], "Migration": { "DownloadSources": ["IPFS", "HTTP", "127.0.0.1", "https://127.0.1.1"], "Keep": "cache" }, "Peering": { "Peers": [ { "ID": "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5", "Addrs": ["/ip4/127.0.0.1/tcp/4001", "/ip4/127.0.0.1/udp/4001/quic"] } ] } } ` noSuchDir := "no_such_dir-5953aa51-1145-4efd-afd1-a069075fcf76" bootstrap, peers := readIpfsConfig(&noSuchDir, "") if bootstrap != nil { t.Error("expected nil bootstrap") } if peers != nil { t.Error("expected nil peers") } tmpDir := makeConfig(t, testConfig) bootstrap, peers = readIpfsConfig(nil, "") if bootstrap != nil || peers != nil { t.Fatal("expected nil ipfs config items") } bootstrap, peers = readIpfsConfig(&tmpDir, "") if len(bootstrap) != 2 { t.Fatal("wrong number of bootstrap addresses") } if bootstrap[0] != "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt" { t.Fatal("wrong bootstrap address") } if len(peers) != 1 { t.Fatal("wrong number of peers") } peer := peers[0] if peer.ID.String() != "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5" { t.Errorf("wrong ID for first peer") } if len(peer.Addrs) != 2 { t.Error("wrong number of addrs for first peer") } } func TestBadBootstrappingIpfsConfig(t *testing.T) { const configBadBootstrap = ` { "Bootstrap": "unreadable", "Migration": { "DownloadSources": ["IPFS", "HTTP", "127.0.0.1"], "Keep": "cache" }, "Peering": { "Peers": [ { "ID": "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5", "Addrs": ["/ip4/127.0.0.1/tcp/4001", "/ip4/127.0.0.1/udp/4001/quic"] } ] } } ` tmpDir := makeConfig(t, configBadBootstrap) bootstrap, peers := readIpfsConfig(&tmpDir, "") if bootstrap != nil { t.Fatal("expected nil bootstrap") } if len(peers) != 1 { t.Fatal("wrong number of peers") } if len(peers[0].Addrs) != 2 { t.Error("wrong number of addrs for first peer") } os.RemoveAll(tmpDir) } func TestBadPeersIpfsConfig(t *testing.T) { const configBadPeers = ` { "Bootstrap": [ "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" ], "Migration": { "DownloadSources": ["IPFS", "HTTP", "127.0.0.1"], "Keep": "cache" }, "Peering": "Unreadable-data" } ` tmpDir := makeConfig(t, configBadPeers) bootstrap, peers := readIpfsConfig(&tmpDir, "") if peers != nil { t.Fatal("expected nil peers") } if len(bootstrap) != 2 { t.Fatal("wrong number of bootstrap addresses") } if bootstrap[0] != "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt" { t.Fatal("wrong bootstrap address") } } func makeConfig(t *testing.T, configData string) string { tmpDir := t.TempDir() cfgFile, err := os.Create(filepath.Join(tmpDir, "config")) if err != nil { t.Fatal(err) } if _, err = cfgFile.Write([]byte(configData)); err != nil { t.Fatal(err) } if err = cfgFile.Close(); err != nil { t.Fatal(err) } return tmpDir } func skipUnlessEpic(t *testing.T) { if os.Getenv("IPFS_EPIC_TEST") == "" { t.SkipNow() } } func setupPlugins() error { defaultPath, err := migrations.IpfsDir("") if err != nil { return err } // Load plugins. This will skip the repo if not available. plugins, err := loader.NewPluginLoader(filepath.Join(defaultPath, "plugins")) if err != nil { return fmt.Errorf("error loading plugins: %w", err) } if err := plugins.Initialize(); err != nil { // Need to ignore errors here because plugins may already be loaded when // run from ipfs daemon. return fmt.Errorf("error initializing plugins: %w", err) } if err := plugins.Inject(); err != nil { // Need to ignore errors here because plugins may already be loaded when // run from ipfs daemon. return fmt.Errorf("error injecting plugins: %w", err) } return nil } ================================================ FILE: repo/fsrepo/migrations/migrations.go ================================================ package migrations import ( "context" "encoding/json" "errors" "fmt" "log" "net/url" "os" "os/exec" "path" "runtime" "strings" "sync" config "github.com/ipfs/kubo/config" ) const ( // Migrations subdirectory in distribution. Empty for root (no subdir). distMigsRoot = "" distFSRM = "fs-repo-migrations" ) // RunMigration finds, downloads, and runs the individual migrations needed to // migrate the repo from its current version to the target version. // // Deprecated: This function downloads migration binaries from the internet and will be removed // in a future version. Use RunHybridMigrations for modern migrations with embedded support, // or RunEmbeddedMigrations for repo versions ≥16. func RunMigration(ctx context.Context, fetcher Fetcher, targetVer int, ipfsDir string, allowDowngrade bool) error { ipfsDir, err := CheckIpfsDir(ipfsDir) if err != nil { return err } fromVer, err := RepoVersion(ipfsDir) if err != nil { return fmt.Errorf("could not get repo version: %w", err) } if fromVer == targetVer { // repo already at target version number return nil } if fromVer > targetVer && !allowDowngrade { return fmt.Errorf("downgrade not allowed from %d to %d", fromVer, targetVer) } logger := log.New(os.Stdout, "", 0) logger.Print("Looking for suitable migration binaries.") migrations, binPaths, err := findMigrations(ctx, fromVer, targetVer) if err != nil { return err } // Download migrations that were not found if len(binPaths) < len(migrations) { missing := make([]string, 0, len(migrations)-len(binPaths)) for _, mig := range migrations { if _, ok := binPaths[mig]; !ok { missing = append(missing, mig) } } logger.Println("Need", len(missing), "migrations, downloading.") tmpDir, err := os.MkdirTemp("", "migrations") if err != nil { return err } defer os.RemoveAll(tmpDir) fetched, err := fetchMigrations(ctx, fetcher, missing, tmpDir, logger) if err != nil { logger.Print("Failed to download migrations.") return err } for i := range missing { binPaths[missing[i]] = fetched[i] } } var revert bool if fromVer > targetVer { revert = true } for _, migration := range migrations { logger.Println("Running migration", migration, "...") err = runMigration(ctx, binPaths[migration], ipfsDir, revert, logger) if err != nil { return fmt.Errorf("migration %s failed: %w", migration, err) } } logger.Printf("Success: fs-repo migrated to version %d.\n", targetVer) return nil } func NeedMigration(target int) (bool, error) { vnum, err := RepoVersion("") if err != nil { return false, fmt.Errorf("could not get repo version: %w", err) } return vnum != target, nil } func ExeName(name string) string { if runtime.GOOS == "windows" { return name + ".exe" } return name } // ReadMigrationConfig reads the Migration section of the IPFS config, avoiding // reading anything other than the Migration section. That way, we're free to // make arbitrary changes to all _other_ sections in migrations. // // Deprecated: This function is used by legacy migration downloads and will be removed // in a future version. Use RunHybridMigrations or RunEmbeddedMigrations instead. func ReadMigrationConfig(repoRoot string, userConfigFile string) (*config.Migration, error) { var cfg struct { Migration config.Migration } cfgPath, err := config.Filename(repoRoot, userConfigFile) if err != nil { return nil, err } cfgFile, err := os.Open(cfgPath) if err != nil { return nil, err } defer cfgFile.Close() err = json.NewDecoder(cfgFile).Decode(&cfg) if err != nil { return nil, err } switch cfg.Migration.Keep { case "": cfg.Migration.Keep = config.DefaultMigrationKeep case "discard", "cache", "keep": default: return nil, errors.New("unknown config value, Migrations.Keep must be 'cache', 'pin', or 'discard'") } if len(cfg.Migration.DownloadSources) == 0 { cfg.Migration.DownloadSources = config.DefaultMigrationDownloadSources } return &cfg.Migration, nil } // GetMigrationFetcher creates one or more fetchers according to // downloadSources. // // Deprecated: This function is used by legacy migration downloads and will be removed // in a future version. Use RunHybridMigrations or RunEmbeddedMigrations instead. func GetMigrationFetcher(downloadSources []string, distPath string, newIpfsFetcher func(string) Fetcher) (Fetcher, error) { const httpUserAgent = "kubo/migration" const numTriesPerHTTP = 3 var fetchers []Fetcher for _, src := range downloadSources { src := strings.TrimSpace(src) switch src { case "HTTPS", "https", "HTTP", "http": fetchers = append(fetchers, &RetryFetcher{NewHttpFetcher(distPath, "", httpUserAgent, 0), numTriesPerHTTP}) case "IPFS", "ipfs": return nil, errors.New("IPFS downloads are not supported for legacy migrations (repo versions <16). Please use only HTTPS in Migration.DownloadSources") case "": // Ignore empty string default: u, err := url.Parse(src) if err != nil { return nil, fmt.Errorf("bad gateway address: %w", err) } switch u.Scheme { case "": u.Scheme = "https" case "https", "http": default: return nil, errors.New("bad gateway address: url scheme must be http or https") } fetchers = append(fetchers, &RetryFetcher{NewHttpFetcher(distPath, u.String(), httpUserAgent, 0), numTriesPerHTTP}) } } switch len(fetchers) { case 0: return nil, errors.New("no sources specified") case 1: return fetchers[0], nil } // Wrap fetchers in a MultiFetcher to try them in order return NewMultiFetcher(fetchers...), nil } func migrationName(from, to int) string { return fmt.Sprintf("fs-repo-%d-to-%d", from, to) } // findMigrations returns a list of migrations, ordered from first to last // migration to apply, and a map of locations of migration binaries of any // migrations that were found. // // Deprecated: This function is used by legacy migration downloads and will be removed // in a future version. func findMigrations(ctx context.Context, from, to int) ([]string, map[string]string, error) { step := 1 count := to - from if from > to { step = -1 count = from - to } migrations := make([]string, 0, count) binPaths := make(map[string]string, count) for cur := from; cur != to; cur += step { if ctx.Err() != nil { return nil, nil, ctx.Err() } var migName string if step == -1 { migName = migrationName(cur+step, cur) } else { migName = migrationName(cur, cur+step) } migrations = append(migrations, migName) bin, err := exec.LookPath(migName) if err != nil { continue } binPaths[migName] = bin } return migrations, binPaths, nil } func runMigration(ctx context.Context, binPath, ipfsDir string, revert bool, logger *log.Logger) error { pathArg := fmt.Sprintf("-path=%s", ipfsDir) var cmd *exec.Cmd if revert { logger.Println(" => Running:", binPath, pathArg, "-verbose=true -revert") cmd = exec.CommandContext(ctx, binPath, pathArg, "-verbose=true", "-revert") } else { logger.Println(" => Running:", binPath, pathArg, "-verbose=true") cmd = exec.CommandContext(ctx, binPath, pathArg, "-verbose=true") } cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() } // fetchMigrations downloads the requested migrations, and returns a slice with // the paths of each binary, in the same order specified by needed. // // Deprecated: This function downloads migration binaries from the internet and will be removed // in a future version. Use RunHybridMigrations or RunEmbeddedMigrations instead. func fetchMigrations(ctx context.Context, fetcher Fetcher, needed []string, destDir string, logger *log.Logger) ([]string, error) { osv, err := osWithVariant() if err != nil { return nil, err } if osv == "linux-musl" { return nil, fmt.Errorf("linux-musl not supported, you must build the binary from source for your platform") } var wg sync.WaitGroup wg.Add(len(needed)) bins := make([]string, len(needed)) // Download and unpack all requested migrations concurrently. for i, name := range needed { logger.Printf("Downloading migration: %s...", name) go func(i int, name string) { defer wg.Done() dist := path.Join(distMigsRoot, name) ver, err := LatestDistVersion(ctx, fetcher, dist, false) if err != nil { logger.Printf("could not get latest version of migration %s: %s", name, err) return } loc, err := FetchBinary(ctx, fetcher, dist, ver, name, destDir) if err != nil { logger.Printf("could not download %s: %s", name, err) return } logger.Printf("Downloaded and unpacked migration: %s (%s)", loc, ver) bins[i] = loc }(i, name) } wg.Wait() var fails []string for i := range bins { if bins[i] == "" { fails = append(fails, needed[i]) } } if len(fails) != 0 { err = fmt.Errorf("failed to download migrations: %s", strings.Join(fails, " ")) if ctx.Err() != nil { err = fmt.Errorf("%s, %w", ctx.Err(), err) } return nil, err } return bins, nil } // RunHybridMigrations intelligently runs migrations using external tools for legacy versions // and embedded migrations for modern versions. This handles the transition from external // fs-repo-migrations binaries (for repo versions <16) to embedded migrations (for repo versions ≥16). // // The function automatically: // 1. Uses external migrations to get from current version to v16 (if needed) // 2. Uses embedded migrations for v16+ steps // 3. Handles pure external, pure embedded, or mixed migration scenarios // // Legacy external migrations (repo versions <16) only support HTTPS downloads. // // Parameters: // - ctx: Context for cancellation and timeouts // - targetVer: Target repository version to migrate to // - ipfsDir: Path to the IPFS repository directory // - allowDowngrade: Whether to allow downgrade migrations // // Returns error if migration fails at any step. func RunHybridMigrations(ctx context.Context, targetVer int, ipfsDir string, allowDowngrade bool) error { const embeddedMigrationsMinVersion = 16 // Get current repo version currentVer, err := RepoVersion(ipfsDir) if err != nil { return fmt.Errorf("could not get current repo version: %w", err) } var logger = log.New(os.Stdout, "", 0) // Check if migration is needed if currentVer == targetVer { logger.Printf("Repository is already at version %d", targetVer) return nil } // Validate downgrade request if targetVer < currentVer && !allowDowngrade { return fmt.Errorf("downgrade from version %d to %d requires allowDowngrade=true", currentVer, targetVer) } // Determine migration strategy based on version ranges needsExternal := currentVer < embeddedMigrationsMinVersion needsEmbedded := targetVer >= embeddedMigrationsMinVersion // Case 1: Pure embedded migration (both current and target ≥ 16) if !needsExternal && needsEmbedded { return RunEmbeddedMigrations(ctx, targetVer, ipfsDir, allowDowngrade) } // For cases requiring external migrations, we check if migration binaries // are available in PATH before attempting network downloads // Case 2: Pure external migration (target < 16) if needsExternal && !needsEmbedded { // Check for migration binaries in PATH first (for testing/local development) migrations, binPaths, err := findMigrations(ctx, currentVer, targetVer) if err != nil { return fmt.Errorf("could not determine migration paths: %w", err) } foundAll := true for _, migName := range migrations { if _, exists := binPaths[migName]; !exists { foundAll = false break } } if foundAll { return runMigrationsFromPath(ctx, migrations, binPaths, ipfsDir, logger, false) } // Fall back to network download (original behavior) migrationCfg, err := ReadMigrationConfig(ipfsDir, "") if err != nil { return fmt.Errorf("could not read migration config: %w", err) } // Use existing RunMigration which handles network downloads properly (HTTPS only for legacy migrations) fetcher, err := GetMigrationFetcher(migrationCfg.DownloadSources, GetDistPathEnv(CurrentIpfsDist), nil) if err != nil { return fmt.Errorf("failed to get migration fetcher: %w", err) } defer fetcher.Close() return RunMigration(ctx, fetcher, targetVer, ipfsDir, allowDowngrade) } // Case 3: Hybrid migration (current < 16, target ≥ 16) if needsExternal && needsEmbedded { logger.Printf("Starting hybrid migration from version %d to %d", currentVer, targetVer) logger.Print("Using hybrid migration strategy: external to v16, then embedded") // Phase 1: Use external migrations to get to v16 logger.Printf("Phase 1: External migration from v%d to v%d", currentVer, embeddedMigrationsMinVersion) // Check for external migration binaries in PATH first migrations, binPaths, err := findMigrations(ctx, currentVer, embeddedMigrationsMinVersion) if err != nil { return fmt.Errorf("could not determine external migration paths: %w", err) } foundAll := true for _, migName := range migrations { if _, exists := binPaths[migName]; !exists { foundAll = false break } } if foundAll { if err = runMigrationsFromPath(ctx, migrations, binPaths, ipfsDir, logger, false); err != nil { return fmt.Errorf("external migration phase failed: %w", err) } } else { migrationCfg, err := ReadMigrationConfig(ipfsDir, "") if err != nil { return fmt.Errorf("could not read migration config: %w", err) } // Legacy migrations only support HTTPS downloads fetcher, err := GetMigrationFetcher(migrationCfg.DownloadSources, GetDistPathEnv(CurrentIpfsDist), nil) if err != nil { return fmt.Errorf("failed to get migration fetcher: %w", err) } defer fetcher.Close() if err = RunMigration(ctx, fetcher, embeddedMigrationsMinVersion, ipfsDir, allowDowngrade); err != nil { return fmt.Errorf("external migration phase failed: %w", err) } } // Phase 2: Use embedded migrations for v16+ logger.Printf("Phase 2: Embedded migration from v%d to v%d", embeddedMigrationsMinVersion, targetVer) err = RunEmbeddedMigrations(ctx, targetVer, ipfsDir, allowDowngrade) if err != nil { return fmt.Errorf("embedded migration phase failed: %w", err) } logger.Printf("Hybrid migration completed successfully: v%d → v%d", currentVer, targetVer) return nil } // Case 4: Reverse hybrid migration (≥16 to <16) // Use embedded migrations for ≥16 steps, then external migrations for <16 steps logger.Printf("Starting reverse hybrid migration from version %d to %d", currentVer, targetVer) logger.Print("Using reverse hybrid migration strategy: embedded to v16, then external") // Phase 1: Use embedded migrations from current version down to v16 (if needed) if currentVer > embeddedMigrationsMinVersion { logger.Printf("Phase 1: Embedded downgrade from v%d to v%d", currentVer, embeddedMigrationsMinVersion) err = RunEmbeddedMigrations(ctx, embeddedMigrationsMinVersion, ipfsDir, allowDowngrade) if err != nil { return fmt.Errorf("embedded downgrade phase failed: %w", err) } } // Phase 2: Use external migrations from v16 to target (if needed) if embeddedMigrationsMinVersion > targetVer { logger.Printf("Phase 2: External downgrade from v%d to v%d", embeddedMigrationsMinVersion, targetVer) // Check for external migration binaries in PATH first migrations, binPaths, err := findMigrations(ctx, embeddedMigrationsMinVersion, targetVer) if err != nil { return fmt.Errorf("could not determine external migration paths: %w", err) } foundAll := true for _, migName := range migrations { if _, exists := binPaths[migName]; !exists { foundAll = false break } } if foundAll { if err = runMigrationsFromPath(ctx, migrations, binPaths, ipfsDir, logger, true); err != nil { return fmt.Errorf("external downgrade phase failed: %w", err) } } else { migrationCfg, err := ReadMigrationConfig(ipfsDir, "") if err != nil { return fmt.Errorf("could not read migration config: %w", err) } // Legacy migrations only support HTTPS downloads fetcher, err := GetMigrationFetcher(migrationCfg.DownloadSources, GetDistPathEnv(CurrentIpfsDist), nil) if err != nil { return fmt.Errorf("failed to get migration fetcher: %w", err) } defer fetcher.Close() if err = RunMigration(ctx, fetcher, targetVer, ipfsDir, allowDowngrade); err != nil { return fmt.Errorf("external downgrade phase failed: %w", err) } } } logger.Printf("Reverse hybrid migration completed successfully: v%d → v%d", currentVer, targetVer) return nil } // runMigrationsFromPath runs migrations using binaries found in PATH func runMigrationsFromPath(ctx context.Context, migrations []string, binPaths map[string]string, ipfsDir string, logger *log.Logger, revert bool) error { for _, migName := range migrations { binPath, exists := binPaths[migName] if !exists { return fmt.Errorf("migration binary %s not found in PATH", migName) } logger.Printf("Running migration %s using binary from PATH: %s", migName, binPath) // Run the migration binary directly err := runMigration(ctx, binPath, ipfsDir, revert, logger) if err != nil { return fmt.Errorf("migration %s failed: %w", migName, err) } } return nil } ================================================ FILE: repo/fsrepo/migrations/migrations_test.go ================================================ package migrations import ( "context" "fmt" "log" "os" "path/filepath" "strings" "testing" config "github.com/ipfs/kubo/config" ) func TestFindMigrations(t *testing.T) { tmpDir := t.TempDir() ctx := t.Context() migs, bins, err := findMigrations(ctx, 0, 5) if err != nil { t.Fatal(err) } if len(migs) != 5 { t.Fatal("expected 5 migrations") } if len(bins) != 0 { t.Fatal("should not have found migrations") } for i := 1; i < 6; i++ { createFakeBin(i-1, i, tmpDir) } t.Setenv("PATH", tmpDir) migs, bins, err = findMigrations(ctx, 0, 5) if err != nil { t.Fatal(err) } if len(migs) != 5 { t.Fatal("expected 5 migrations") } if len(bins) != len(migs) { t.Fatal("missing", len(migs)-len(bins), "migrations") } os.Remove(bins[migs[2]]) migs, bins, err = findMigrations(ctx, 0, 5) if err != nil { t.Fatal(err) } if len(bins) != len(migs)-1 { t.Fatal("should be missing one migration bin") } } func TestFindMigrationsReverse(t *testing.T) { tmpDir := t.TempDir() ctx := t.Context() migs, bins, err := findMigrations(ctx, 5, 0) if err != nil { t.Fatal(err) } if len(migs) != 5 { t.Fatal("expected 5 migrations") } if len(bins) != 0 { t.Fatal("should not have found migrations") } for i := 1; i < 6; i++ { createFakeBin(i-1, i, tmpDir) } t.Setenv("PATH", tmpDir) migs, bins, err = findMigrations(ctx, 5, 0) if err != nil { t.Fatal(err) } if len(migs) != 5 { t.Fatal("expected 5 migrations") } if len(bins) != len(migs) { t.Fatal("missing", len(migs)-len(bins), "migrations:", migs) } os.Remove(bins[migs[2]]) migs, bins, err = findMigrations(ctx, 5, 0) if err != nil { t.Fatal(err) } if len(bins) != len(migs)-1 { t.Fatal("should be missing one migration bin") } } func TestFetchMigrations(t *testing.T) { ctx := t.Context() fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) tmpDir := t.TempDir() needed := []string{"fs-repo-1-to-2", "fs-repo-2-to-3"} buf := new(strings.Builder) buf.Grow(256) logger := log.New(buf, "", 0) fetched, err := fetchMigrations(ctx, fetcher, needed, tmpDir, logger) if err != nil { t.Fatal(err) } for _, bin := range fetched { _, err = os.Stat(bin) if os.IsNotExist(err) { t.Error("expected file to exist:", bin) } } // Check expected log output for _, mig := range needed { logOut := fmt.Sprintf("Downloading migration: %s", mig) if !strings.Contains(buf.String(), logOut) { t.Fatalf("did not find expected log output %q", logOut) } logOut = fmt.Sprintf("Downloaded and unpacked migration: %s", filepath.Join(tmpDir, mig)) if !strings.Contains(buf.String(), logOut) { t.Fatalf("did not find expected log output %q", logOut) } } } func TestRunMigrations(t *testing.T) { fakeIpfs := filepath.Join(t.TempDir(), ".ipfs") t.Setenv(config.EnvDir, fakeIpfs) err := os.Mkdir(fakeIpfs, os.ModePerm) if err != nil { panic(err) } testVer := 11 err = WriteRepoVersion(fakeIpfs, testVer) if err != nil { t.Fatal(err) } fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) ctx := t.Context() targetVer := 9 err = RunMigration(ctx, fetcher, targetVer, fakeIpfs, false) if err == nil || !strings.HasPrefix(err.Error(), "downgrade not allowed") { t.Fatal("expected 'downgrade not allowed' error") } err = RunMigration(ctx, fetcher, targetVer, fakeIpfs, true) if err != nil { if !strings.HasPrefix(err.Error(), "migration fs-repo-10-to-11 failed") { t.Fatal(err) } } } func createFakeBin(from, to int, tmpDir string) { migPath := filepath.Join(tmpDir, ExeName(migrationName(from, to))) emptyFile, err := os.Create(migPath) if err != nil { panic(err) } emptyFile.Close() err = os.Chmod(migPath, 0o755) if err != nil { panic(err) } } var testConfig = ` { "Bootstrap": [ "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" ], "Migration": { "DownloadSources": ["IPFS", "HTTP", "127.0.0.1", "https://127.0.1.1"], "Keep": "cache" }, "Peering": { "Peers": [ { "ID": "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5", "Addrs": ["/ip4/127.0.0.1/tcp/4001", "/ip4/127.0.0.1/udp/4001/quic"] } ] } } ` func TestReadMigrationConfigDefaults(t *testing.T) { tmpDir := makeConfig(t, "{}") cfg, err := ReadMigrationConfig(tmpDir, "") if err != nil { t.Fatal(err) } if cfg.Keep != config.DefaultMigrationKeep { t.Error("expected default value for Keep") } if len(cfg.DownloadSources) != len(config.DefaultMigrationDownloadSources) { t.Fatal("expected default number of download sources") } for i, src := range config.DefaultMigrationDownloadSources { if cfg.DownloadSources[i] != src { t.Errorf("wrong DownloadSource: %s", cfg.DownloadSources[i]) } } } func TestReadMigrationConfigErrors(t *testing.T) { tmpDir := makeConfig(t, `{"Migration": {"Keep": "badvalue"}}`) _, err := ReadMigrationConfig(tmpDir, "") if err == nil { t.Fatal("expected error") } if !strings.HasPrefix(err.Error(), "unknown") { t.Fatal("did not get expected error:", err) } os.RemoveAll(tmpDir) _, err = ReadMigrationConfig(tmpDir, "") if err == nil { t.Fatal("expected error") } tmpDir = makeConfig(t, `}{`) _, err = ReadMigrationConfig(tmpDir, "") if err == nil { t.Fatal("expected error") } } func TestReadMigrationConfig(t *testing.T) { tmpDir := makeConfig(t, testConfig) cfg, err := ReadMigrationConfig(tmpDir, "") if err != nil { t.Fatal(err) } if len(cfg.DownloadSources) != 4 { t.Fatal("wrong number of DownloadSources") } expect := []string{"IPFS", "HTTP", "127.0.0.1", "https://127.0.1.1"} for i := range expect { if cfg.DownloadSources[i] != expect[i] { t.Errorf("wrong DownloadSource at %d", i) } } if cfg.Keep != "cache" { t.Error("wrong value for Keep") } } type mockIpfsFetcher struct{} var _ Fetcher = (*mockIpfsFetcher)(nil) func (m *mockIpfsFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error) { return nil, nil } func (m *mockIpfsFetcher) Close() error { return nil } func TestGetMigrationFetcher(t *testing.T) { var f Fetcher var err error newIpfsFetcher := func(distPath string) Fetcher { return &mockIpfsFetcher{} } downloadSources := []string{"ftp://bad.gateway.io"} _, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher) if err == nil || !strings.HasPrefix(err.Error(), "bad gateway addr") { t.Fatal("Expected bad gateway address error, got:", err) } downloadSources = []string{"::bad.gateway.io"} _, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher) if err == nil || !strings.HasPrefix(err.Error(), "bad gateway addr") { t.Fatal("Expected bad gateway address error, got:", err) } downloadSources = []string{"http://localhost"} f, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher) if err != nil { t.Fatal(err) } if rf, ok := f.(*RetryFetcher); !ok { t.Fatal("expected RetryFetcher") } else if _, ok := rf.Fetcher.(*HttpFetcher); !ok { t.Fatal("expected HttpFetcher") } downloadSources = []string{"ipfs"} _, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher) if err == nil || !strings.Contains(err.Error(), "IPFS downloads are not supported for legacy migrations") { t.Fatal("Expected IPFS downloads error, got:", err) } downloadSources = []string{"http"} f, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher) if err != nil { t.Fatal(err) } if rf, ok := f.(*RetryFetcher); !ok { t.Fatal("expected RetryFetcher") } else if _, ok := rf.Fetcher.(*HttpFetcher); !ok { t.Fatal("expected HttpFetcher") } downloadSources = []string{"IPFS", "HTTPS"} _, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher) if err == nil || !strings.Contains(err.Error(), "IPFS downloads are not supported for legacy migrations") { t.Fatal("Expected IPFS downloads error, got:", err) } downloadSources = []string{"https", "some.domain.io"} f, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher) if err != nil { t.Fatal(err) } mf, ok := f.(*MultiFetcher) if !ok { t.Fatal("expected MultiFetcher") } if mf.Len() != 2 { t.Fatal("expected 2 fetchers in MultiFetcher") } downloadSources = nil _, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher) if err == nil { t.Fatal("expected error when no sources specified") } downloadSources = []string{"", ""} _, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher) if err == nil { t.Fatal("expected error when empty string fetchers specified") } } func makeConfig(t *testing.T, configData string) string { tmpDir := t.TempDir() cfgFile, err := os.Create(filepath.Join(tmpDir, "config")) if err != nil { t.Fatal(err) } if _, err = cfgFile.Write([]byte(configData)); err != nil { t.Fatal(err) } if err = cfgFile.Close(); err != nil { t.Fatal(err) } return tmpDir } ================================================ FILE: repo/fsrepo/migrations/retryfetcher.go ================================================ package migrations import ( "context" "fmt" ) type RetryFetcher struct { Fetcher MaxTries int } var _ Fetcher = (*RetryFetcher)(nil) func (r *RetryFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error) { var lastErr error for i := 0; i < r.MaxTries; i++ { out, err := r.Fetcher.Fetch(ctx, filePath) if err == nil { return out, nil } if ctx.Err() != nil { return nil, ctx.Err() } lastErr = err } return nil, fmt.Errorf("exceeded number of retries. last error was %w", lastErr) } func (r *RetryFetcher) Close() error { return r.Fetcher.Close() } ================================================ FILE: repo/fsrepo/migrations/setup_test.go ================================================ package migrations import ( "bytes" "context" "fmt" "io" "net/http/httptest" "os" "path" "path/filepath" "strings" "testing" "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/gateway" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-unixfsnode/data/builder" "github.com/ipld/go-car/v2" carblockstore "github.com/ipld/go-car/v2/blockstore" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/multiformats/go-multicodec" "github.com/multiformats/go-multihash" ) var ( testIpfsDist string testServer *httptest.Server ) func TestMain(m *testing.M) { t := &testing.T{} // Setup test data testDataDir := makeTestData(t) testCar := makeTestCar(testDataDir) defer os.RemoveAll(testCar) // Setup test gateway fd := setupTestGateway(testCar) defer fd.Close() // Run tests os.Exit(m.Run()) } func makeTestData(t testing.TB) string { tempDir := t.TempDir() versions := []string{"v1.0.0", "v1.1.0", "v1.1.2", "v2.0.0-rc1", "2.0.0", "v2.0.1"} packages := []string{"kubo", "go-ipfs", "fs-repo-migrations", "fs-repo-1-to-2", "fs-repo-2-to-3", "fs-repo-9-to-10", "fs-repo-10-to-11"} // Generate fake data for _, name := range packages { err := os.MkdirAll(filepath.Join(tempDir, name), 0777) if err != nil { panic(err) } err = os.WriteFile(filepath.Join(tempDir, name, "versions"), []byte(strings.Join(versions, "\n")+"\n"), 0666) if err != nil { panic(err) } for _, version := range versions { filename, archName := makeArchivePath(name, name, version, "tar.gz") createFakeArchive(filepath.Join(tempDir, filename), archName, false) filename, archName = makeArchivePath(name, name, version, "zip") createFakeArchive(filepath.Join(tempDir, filename), archName, true) } } return tempDir } func createFakeArchive(archName, name string, archZip bool) { err := os.MkdirAll(filepath.Dir(archName), 0777) if err != nil { panic(err) } fileName := strings.Split(path.Base(name), "_")[0] root := fileName // Simulate fetching go-ipfs, which has "ipfs" as the name in the archive. if fileName == "go-ipfs" || fileName == "kubo" { fileName = "ipfs" } fileName = ExeName(fileName) if archZip { err = writeZipFile(archName, root, fileName, "FAKE DATA") } else { err = writeTarGzipFile(archName, root, fileName, "FAKE DATA") } if err != nil { panic(err) } } // makeTestCar makes a CAR file with the directory [testData]. This code is mostly // sourced from https://github.com/ipld/go-car/blob/1e2f0bd2c44ee31f48a8f602b25b5671cc0c4687/cmd/car/create.go func makeTestCar(testData string) string { // make a cid with the right length that we eventually will patch with the root. hasher, err := multihash.GetHasher(multihash.SHA2_256) if err != nil { panic(err) } digest := hasher.Sum([]byte{}) hash, err := multihash.Encode(digest, multihash.SHA2_256) if err != nil { panic(err) } proxyRoot := cid.NewCidV1(uint64(multicodec.DagPb), hash) // Make CAR file fd, err := os.CreateTemp("", "kubo-migrations-test-*.car") if err != nil { panic(err) } defer fd.Close() filename := fd.Name() rw, err := carblockstore.OpenReadWriteFile(fd, []cid.Cid{proxyRoot}, carblockstore.WriteAsCarV1(true)) if err != nil { panic(err) } defer rw.Close() ctx := context.Background() ls := cidlink.DefaultLinkSystem() ls.TrustedStorage = true ls.StorageReadOpener = func(_ ipld.LinkContext, l ipld.Link) (io.Reader, error) { cl, ok := l.(cidlink.Link) if !ok { return nil, fmt.Errorf("not a cidlink") } blk, err := rw.Get(ctx, cl.Cid) if err != nil { return nil, err } return bytes.NewBuffer(blk.RawData()), nil } ls.StorageWriteOpener = func(_ ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) { buf := bytes.NewBuffer(nil) return buf, func(l ipld.Link) error { cl, ok := l.(cidlink.Link) if !ok { return fmt.Errorf("not a cidlink") } blk, err := blocks.NewBlockWithCid(buf.Bytes(), cl.Cid) if err != nil { return err } return rw.Put(ctx, blk) }, nil } l, _, err := builder.BuildUnixFSRecursive(testData, &ls) if err != nil { panic(err) } rcl, ok := l.(cidlink.Link) if !ok { panic(fmt.Errorf("could not interpret %s", l)) } if err := rw.Finalize(); err != nil { panic(err) } // re-open/finalize with the final root. err = car.ReplaceRootsInFile(filename, []cid.Cid{rcl.Cid}) if err != nil { panic(err) } return filename } func setupTestGateway(testCar string) io.Closer { blockService, roots, fd, err := newBlockServiceFromCAR(testCar) if err != nil { panic(err) } if len(roots) != 1 { panic("expected car with 1 root") } backend, err := gateway.NewBlocksBackend(blockService) if err != nil { panic(err) } conf := gateway.Config{ NoDNSLink: false, DeserializedResponses: false, } testIpfsDist = "/ipfs/" + roots[0].String() testServer = httptest.NewServer(gateway.NewHandler(conf, backend)) return fd } func newBlockServiceFromCAR(filepath string) (blockservice.BlockService, []cid.Cid, io.Closer, error) { r, err := os.Open(filepath) if err != nil { return nil, nil, nil, err } bs, err := carblockstore.NewReadOnly(r, nil) if err != nil { _ = r.Close() return nil, nil, nil, err } roots, err := bs.Roots() if err != nil { return nil, nil, nil, err } blockService := blockservice.New(bs, offline.Exchange(bs)) return blockService, roots, r, nil } ================================================ FILE: repo/fsrepo/migrations/unpack.go ================================================ package migrations import ( "archive/tar" "archive/zip" "compress/gzip" "errors" "fmt" "io" "os" ) func unpackArchive(arcPath, atype, root, name, out string) error { var err error switch atype { case "tar.gz": err = unpackTgz(arcPath, root, name, out) case "zip": err = unpackZip(arcPath, root, name, out) default: err = fmt.Errorf("unrecognized archive type: %s", atype) } if err != nil { return err } return nil } func unpackTgz(arcPath, root, name, out string) error { fi, err := os.Open(arcPath) if err != nil { return fmt.Errorf("cannot open archive file: %w", err) } defer fi.Close() gzr, err := gzip.NewReader(fi) if err != nil { return fmt.Errorf("error opening gzip reader: %w", err) } defer gzr.Close() var bin io.Reader tarr := tar.NewReader(gzr) lookFor := root + "/" + name for { th, err := tarr.Next() if err != nil { if err == io.EOF { break } return fmt.Errorf("cannot read archive: %w", err) } if th.Name == lookFor { bin = tarr break } } if bin == nil { return errors.New("no binary found in archive") } return writeToPath(bin, out) } func unpackZip(arcPath, root, name, out string) error { zipr, err := zip.OpenReader(arcPath) if err != nil { return fmt.Errorf("error opening zip reader: %w", err) } defer zipr.Close() lookFor := root + "/" + name var bin io.ReadCloser for _, fis := range zipr.File { if fis.Name == lookFor { rc, err := fis.Open() if err != nil { return fmt.Errorf("error extracting binary from archive: %w", err) } bin = rc break } } if bin == nil { return errors.New("no binary found in archive") } return writeToPath(bin, out) } func writeToPath(rc io.Reader, out string) error { binfi, err := os.Create(out) if err != nil { return fmt.Errorf("error creating output file '%s': %w", out, err) } defer binfi.Close() _, err = io.Copy(binfi, rc) return err } ================================================ FILE: repo/fsrepo/migrations/unpack_test.go ================================================ package migrations import ( "archive/tar" "archive/zip" "bufio" "compress/gzip" "io" "os" "path" "path/filepath" "strings" "testing" ) func TestUnpackArchive(t *testing.T) { // Check unrecognized archive type err := unpackArchive("", "no-arch-type", "", "", "") if err == nil || err.Error() != "unrecognized archive type: no-arch-type" { t.Fatal("expected 'unrecognized archive type' error") } // Test cannot open errors err = unpackArchive("no-archive", "tar.gz", "", "", "") if err == nil || !strings.HasPrefix(err.Error(), "cannot open archive file") { t.Fatal("expected 'cannot open' error, got:", err) } err = unpackArchive("no-archive", "zip", "", "", "") if err == nil || !strings.HasPrefix(err.Error(), "error opening zip reader") { t.Fatal("expected 'cannot open' error, got:", err) } } func TestUnpackTgz(t *testing.T) { tmpDir := t.TempDir() badTarGzip := filepath.Join(tmpDir, "bad.tar.gz") err := os.WriteFile(badTarGzip, []byte("bad-data\n"), 0o644) if err != nil { panic(err) } err = unpackTgz(badTarGzip, "", "abc", "abc") if err == nil || !strings.HasPrefix(err.Error(), "error opening gzip reader") { t.Fatal("expected error opening gzip reader, got:", err) } testTarGzip := filepath.Join(tmpDir, "test.tar.gz") testData := "some data" err = writeTarGzipFile(testTarGzip, "testroot", "testfile", testData) if err != nil { panic(err) } out := filepath.Join(tmpDir, "out.txt") // Test looking for file that is not in archive err = unpackTgz(testTarGzip, "testroot", "abc", out) if err == nil || err.Error() != "no binary found in archive" { t.Fatal("expected 'no binary found in archive' error, got:", err) } // Test that unpack works. err = unpackTgz(testTarGzip, "testroot", "testfile", out) if err != nil { t.Fatal(err) } fi, err := os.Stat(out) if err != nil { t.Fatal(err) } if fi.Size() != int64(len(testData)) { t.Fatal("unpacked file size is", fi.Size(), "expected", len(testData)) } } func TestUnpackZip(t *testing.T) { tmpDir := t.TempDir() badZip := filepath.Join(tmpDir, "bad.zip") err := os.WriteFile(badZip, []byte("bad-data\n"), 0o644) if err != nil { panic(err) } err = unpackZip(badZip, "", "abc", "abc") if err == nil || !strings.HasPrefix(err.Error(), "error opening zip reader") { t.Fatal("expected error opening zip reader, got:", err) } testZip := filepath.Join(tmpDir, "test.zip") testData := "some data" err = writeZipFile(testZip, "testroot", "testfile", testData) if err != nil { panic(err) } out := filepath.Join(tmpDir, "out.txt") // Test looking for file that is not in archive err = unpackZip(testZip, "testroot", "abc", out) if err == nil || err.Error() != "no binary found in archive" { t.Fatal("expected 'no binary found in archive' error, got:", err) } // Test that unpack works. err = unpackZip(testZip, "testroot", "testfile", out) if err != nil { t.Fatal(err) } fi, err := os.Stat(out) if err != nil { t.Fatal(err) } if fi.Size() != int64(len(testData)) { t.Fatal("unpacked file size is", fi.Size(), "expected", len(testData)) } } func writeTarGzipFile(archName, root, fileName, data string) error { archFile, err := os.Create(archName) if err != nil { return err } defer archFile.Close() w := bufio.NewWriter(archFile) err = writeTarGzip(root, fileName, data, w) if err != nil { return err } // Flush buffered data to file if err = w.Flush(); err != nil { return err } // Close tar file if err = archFile.Close(); err != nil { return err } return nil } func writeTarGzip(root, fileName, data string, w io.Writer) error { // gzip writer writes to buffer gzw := gzip.NewWriter(w) defer gzw.Close() // tar writer writes to gzip tw := tar.NewWriter(gzw) defer tw.Close() var err error if fileName != "" { hdr := &tar.Header{ Name: path.Join(root, fileName), Mode: 0o600, Size: int64(len(data)), } // Write header if err = tw.WriteHeader(hdr); err != nil { return err } // Write file body if _, err := tw.Write([]byte(data)); err != nil { return err } } if err = tw.Close(); err != nil { return err } // Close gzip writer; finish writing gzip data to buffer if err = gzw.Close(); err != nil { return err } return nil } func writeZipFile(archName, root, fileName, data string) error { archFile, err := os.Create(archName) if err != nil { return err } defer archFile.Close() w := bufio.NewWriter(archFile) err = writeZip(root, fileName, data, w) if err != nil { return err } // Flush buffered data to file if err = w.Flush(); err != nil { return err } // Close zip file if err = archFile.Close(); err != nil { return err } return nil } func writeZip(root, fileName, data string, w io.Writer) error { zw := zip.NewWriter(w) defer zw.Close() // Write file name f, err := zw.Create(path.Join(root, fileName)) if err != nil { return err } // Write file data _, err = f.Write([]byte(data)) if err != nil { return err } // Close zip writer if err = zw.Close(); err != nil { return err } return nil } ================================================ FILE: repo/fsrepo/migrations/versions.go ================================================ package migrations import ( "bufio" "bytes" "context" "errors" "fmt" "path" "sort" "strings" "github.com/blang/semver/v4" ) const distVersions = "versions" // LatestDistVersion returns the latest version, of the specified distribution, // that is available on the distribution site. func LatestDistVersion(ctx context.Context, fetcher Fetcher, dist string, stableOnly bool) (string, error) { vs, err := DistVersions(ctx, fetcher, dist, false) if err != nil { return "", err } for i := len(vs) - 1; i >= 0; i-- { ver := vs[i] if stableOnly && strings.Contains(ver, "-rc") { continue } if strings.Contains(ver, "-dev") { continue } return ver, nil } return "", errors.New("could not find a non dev version") } // DistVersions returns all versions of the specified distribution, that are // available on the distriburion site. List is in ascending order, unless // sortDesc is true. func DistVersions(ctx context.Context, fetcher Fetcher, dist string, sortDesc bool) ([]string, error) { versionBytes, err := fetcher.Fetch(ctx, path.Join(dist, distVersions)) if err != nil { return nil, err } prefix := "v" var vers []semver.Version scan := bufio.NewScanner(bytes.NewReader(versionBytes)) for scan.Scan() { ver, err := semver.Make(strings.TrimLeft(scan.Text(), prefix)) if err != nil { continue } vers = append(vers, ver) } if scan.Err() != nil { return nil, fmt.Errorf("could not read versions: %w", scan.Err()) } if sortDesc { sort.Sort(sort.Reverse(semver.Versions(vers))) } else { sort.Sort(semver.Versions(vers)) } out := make([]string, len(vers)) for i := range vers { out[i] = prefix + vers[i].String() } return out, nil } ================================================ FILE: repo/fsrepo/migrations/versions_test.go ================================================ package migrations import ( "testing" "github.com/blang/semver/v4" ) const testDist = "go-ipfs" func TestDistVersions(t *testing.T) { ctx := t.Context() fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) vers, err := DistVersions(ctx, fetcher, testDist, true) if err != nil { t.Fatal(err) } if len(vers) == 0 { t.Fatal("no versions of", testDist) } t.Log("There are", len(vers), "versions of", testDist) t.Log("Latest 5 are:", vers[:5]) } func TestLatestDistVersion(t *testing.T) { ctx := t.Context() fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) latest, err := LatestDistVersion(ctx, fetcher, testDist, false) if err != nil { t.Fatal(err) } if len(latest) < 6 { t.Fatal("latest version string too short", latest) } _, err = semver.New(latest[1:]) if err != nil { t.Fatal("latest version has invalid format:", latest) } t.Log("Latest version of", testDist, "is", latest) } ================================================ FILE: repo/fsrepo/misc.go ================================================ package fsrepo import ( "os" config "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/misc/fsutil" ) // BestKnownPath returns the best known fsrepo path. If the ENV override is // present, this function returns that value. Otherwise, it returns the default // repo path. func BestKnownPath() (string, error) { ipfsPath := config.DefaultPathRoot if os.Getenv(config.EnvDir) != "" { ipfsPath = os.Getenv(config.EnvDir) } ipfsPath, err := fsutil.ExpandHome(ipfsPath) if err != nil { return "", err } return ipfsPath, nil } ================================================ FILE: repo/mock.go ================================================ package repo import ( "context" "errors" "net" filestore "github.com/ipfs/boxo/filestore" keystore "github.com/ipfs/boxo/keystore" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" config "github.com/ipfs/kubo/config" ma "github.com/multiformats/go-multiaddr" ) var errTODO = errors.New("TODO: mock repo") // Mock is not thread-safe. type Mock struct { C config.Config D Datastore K keystore.Keystore F *filestore.FileManager } func (m *Mock) Config() (*config.Config, error) { return &m.C, nil // FIXME threadsafety } func (m *Mock) Path() string { return "" } func (m *Mock) UserResourceOverrides() (rcmgr.PartialLimitConfig, error) { return rcmgr.PartialLimitConfig{}, nil } func (m *Mock) SetConfig(updated *config.Config) error { m.C = *updated // FIXME threadsafety return nil } func (m *Mock) BackupConfig(prefix string) (string, error) { return "", errTODO } func (m *Mock) SetConfigKey(key string, value any) error { return errTODO } func (m *Mock) GetConfigKey(key string) (any, error) { return nil, errTODO } func (m *Mock) Datastore() Datastore { return m.D } func (m *Mock) GetStorageUsage(_ context.Context) (uint64, error) { return 0, nil } func (m *Mock) Close() error { return m.D.Close() } func (m *Mock) SetAPIAddr(addr ma.Multiaddr) error { return errTODO } func (m *Mock) SetGatewayAddr(addr net.Addr) error { return errTODO } func (m *Mock) Keystore() keystore.Keystore { return m.K } func (m *Mock) SwarmKey() ([]byte, error) { return nil, nil } func (m *Mock) FileManager() *filestore.FileManager { return m.F } ================================================ FILE: repo/onlyone.go ================================================ package repo import ( "sync" ) // OnlyOne tracks open Repos by arbitrary key and returns the already // open one. type OnlyOne struct { mu sync.Mutex active map[any]*ref } // Open a Repo identified by key. If Repo is not already open, the // open function is called, and the result is remembered for further // use. // // Key must be comparable, or Open will panic. Make sure to pick keys // that are unique across different concrete Repo implementations, // e.g. by creating a local type: // // type repoKey string // r, err := o.Open(repoKey(path), open) // // Call Repo.Close when done. func (o *OnlyOne) Open(key any, open func() (Repo, error)) (Repo, error) { o.mu.Lock() defer o.mu.Unlock() if o.active == nil { o.active = make(map[any]*ref) } item, found := o.active[key] if !found { repo, err := open() if err != nil { return nil, err } item = &ref{ parent: o, key: key, Repo: repo, } o.active[key] = item } item.refs++ return item, nil } type ref struct { parent *OnlyOne key any refs uint32 Repo } var _ Repo = (*ref)(nil) func (r *ref) Close() error { r.parent.mu.Lock() defer r.parent.mu.Unlock() r.refs-- if r.refs > 0 { // others are holding it open return nil } // last one delete(r.parent.active, r.key) return r.Repo.Close() } ================================================ FILE: repo/repo.go ================================================ package repo import ( "context" "errors" "io" "net" filestore "github.com/ipfs/boxo/filestore" keystore "github.com/ipfs/boxo/keystore" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" ds "github.com/ipfs/go-datastore" config "github.com/ipfs/kubo/config" ma "github.com/multiformats/go-multiaddr" ) var ErrApiNotRunning = errors.New("api not running") //nolint // Repo represents all persistent data of a given ipfs node. type Repo interface { // Config returns the ipfs configuration file from the repo. Changes made // to the returned config are not automatically persisted. Config() (*config.Config, error) // Path is the repo file-system path Path() string // UserResourceOverrides returns optional user resource overrides for the // libp2p resource manager. UserResourceOverrides() (rcmgr.PartialLimitConfig, error) // BackupConfig creates a backup of the current configuration file using // the given prefix for naming. BackupConfig(prefix string) (string, error) // SetConfig persists the given configuration struct to storage. SetConfig(*config.Config) error // SetConfigKey sets the given key-value pair within the config and persists it to storage. SetConfigKey(key string, value any) error // GetConfigKey reads the value for the given key from the configuration in storage. GetConfigKey(key string) (any, error) // Datastore returns a reference to the configured data storage backend. Datastore() Datastore // GetStorageUsage returns the number of bytes stored. GetStorageUsage(context.Context) (uint64, error) // Keystore returns a reference to the key management interface. Keystore() keystore.Keystore // FileManager returns a reference to the filestore file manager. FileManager() *filestore.FileManager // SetAPIAddr sets the API address in the repo. SetAPIAddr(addr ma.Multiaddr) error // SetGatewayAddr sets the Gateway address in the repo. SetGatewayAddr(addr net.Addr) error // SwarmKey returns the configured shared symmetric key for the private networks feature. SwarmKey() ([]byte, error) io.Closer } // Datastore is the interface required from a datastore to be // acceptable to FSRepo. type Datastore interface { ds.Batching // must be thread-safe } ================================================ FILE: routing/composer.go ================================================ package routing import ( "context" "errors" "github.com/ipfs/go-cid" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multihash" ) var ( _ routinghelpers.ProvideManyRouter = &Composer{} _ routing.Routing = &Composer{} ) type Composer struct { GetValueRouter routing.Routing PutValueRouter routing.Routing FindPeersRouter routing.Routing FindProvidersRouter routing.Routing ProvideRouter routing.Routing } func (c *Composer) Provide(ctx context.Context, cid cid.Cid, provide bool) error { log.Debug("composer: calling provide: ", cid) err := c.ProvideRouter.Provide(ctx, cid, provide) if err != nil { log.Debug("composer: calling provide: ", cid, " error: ", err) } return err } func (c *Composer) ProvideMany(ctx context.Context, keys []multihash.Multihash) error { log.Debug("composer: calling provide many: ", len(keys)) pmr, ok := c.ProvideRouter.(routinghelpers.ProvideManyRouter) if !ok { log.Debug("composer: provide many is not implemented on the actual router") return nil } err := pmr.ProvideMany(ctx, keys) if err != nil { log.Debug("composer: calling provide many error: ", err) } return err } func (c *Composer) Ready() bool { log.Debug("composer: calling ready") pmr, ok := c.ProvideRouter.(routinghelpers.ReadyAbleRouter) if !ok { return true } ready := pmr.Ready() log.Debug("composer: calling ready result: ", ready) return ready } func (c *Composer) FindProvidersAsync(ctx context.Context, cid cid.Cid, count int) <-chan peer.AddrInfo { log.Debug("composer: calling findProvidersAsync: ", cid) return c.FindProvidersRouter.FindProvidersAsync(ctx, cid, count) } func (c *Composer) FindPeer(ctx context.Context, pid peer.ID) (peer.AddrInfo, error) { log.Debug("composer: calling findPeer: ", pid) addr, err := c.FindPeersRouter.FindPeer(ctx, pid) if err != nil { log.Debug("composer: calling findPeer error: ", pid, addr.String(), err) } return addr, err } func (c *Composer) PutValue(ctx context.Context, key string, val []byte, opts ...routing.Option) error { log.Debug("composer: calling putValue: ", key, len(val)) err := c.PutValueRouter.PutValue(ctx, key, val, opts...) if err != nil { log.Debug("composer: calling putValue error: ", key, len(val), err) } return err } func (c *Composer) GetValue(ctx context.Context, key string, opts ...routing.Option) ([]byte, error) { log.Debug("composer: calling getValue: ", key) val, err := c.GetValueRouter.GetValue(ctx, key, opts...) if err != nil { log.Debug("composer: calling getValue error: ", key, len(val), err) } return val, err } func (c *Composer) SearchValue(ctx context.Context, key string, opts ...routing.Option) (<-chan []byte, error) { log.Debug("composer: calling searchValue: ", key) ch, err := c.GetValueRouter.SearchValue(ctx, key, opts...) // avoid nil channels on implementations not supporting SearchValue method. if errors.Is(err, routing.ErrNotFound) && ch == nil { out := make(chan []byte) close(out) return out, err } if err != nil { log.Debug("composer: calling searchValue error: ", key, err) } return ch, err } func (c *Composer) Bootstrap(ctx context.Context) error { log.Debug("composer: calling bootstrap") errfp := c.FindPeersRouter.Bootstrap(ctx) errfps := c.FindProvidersRouter.Bootstrap(ctx) errgv := c.GetValueRouter.Bootstrap(ctx) errpv := c.PutValueRouter.Bootstrap(ctx) errp := c.ProvideRouter.Bootstrap(ctx) err := errors.Join(errfp, errfps, errgv, errpv, errp) if err != nil { log.Debug("composer: calling bootstrap error: ", err) } return err } ================================================ FILE: routing/delegated.go ================================================ package routing import ( "context" "encoding/base64" "errors" "fmt" "net/http" "path" "strings" drclient "github.com/ipfs/boxo/routing/http/client" "github.com/ipfs/boxo/routing/http/contentrouter" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" version "github.com/ipfs/kubo" "github.com/ipfs/kubo/config" dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p-kad-dht/dual" "github.com/libp2p/go-libp2p-kad-dht/fullrt" record "github.com/libp2p/go-libp2p-record" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" ic "github.com/libp2p/go-libp2p/core/crypto" host "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" ma "github.com/multiformats/go-multiaddr" "go.opencensus.io/stats/view" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) var log = logging.Logger("routing/delegated") // Parse creates a composed router from the custom routing configuration. // // EXPERIMENTAL: Custom routing (Routing.Type=custom with Routing.Routers and // Routing.Methods) is for research and testing only, not production use. // The configuration format and behavior may change without notice between // releases. HTTP-only configurations cannot reliably provide content. // See docs/delegated-routing.md for limitations. func Parse(routers config.Routers, methods config.Methods, extraDHT *ExtraDHTParams, extraHTTP *ExtraHTTPParams) (routing.Routing, error) { if err := methods.Check(); err != nil { return nil, err } createdRouters := make(map[string]routing.Routing) finalRouter := &Composer{} // Create all needed routers from method names for mn, m := range methods { router, err := parse(make(map[string]bool), createdRouters, m.RouterName, routers, extraDHT, extraHTTP) if err != nil { return nil, err } switch mn { case config.MethodNamePutIPNS: finalRouter.PutValueRouter = router case config.MethodNameGetIPNS: finalRouter.GetValueRouter = router case config.MethodNameFindPeers: finalRouter.FindPeersRouter = router case config.MethodNameFindProviders: finalRouter.FindProvidersRouter = router case config.MethodNameProvide: finalRouter.ProvideRouter = router } log.Info("using method ", mn, " with router ", m.RouterName) } return finalRouter, nil } func parse(visited map[string]bool, createdRouters map[string]routing.Routing, routerName string, routersCfg config.Routers, extraDHT *ExtraDHTParams, extraHTTP *ExtraHTTPParams, ) (routing.Routing, error) { // check if we already created it r, ok := createdRouters[routerName] if ok { return r, nil } // check if we are in a dep loop if visited[routerName] { return nil, fmt.Errorf("dependency loop creating router with name %q", routerName) } // set node as visited visited[routerName] = true cfg, ok := routersCfg[routerName] if !ok { return nil, fmt.Errorf("config for router with name %q not found", routerName) } var router routing.Routing var err error switch cfg.Type { case config.RouterTypeHTTP: router, err = httpRoutingFromConfig(cfg.Router, extraHTTP) case config.RouterTypeDHT: router, err = dhtRoutingFromConfig(cfg.Router, extraDHT) case config.RouterTypeParallel: crp := cfg.Parameters.(*config.ComposableRouterParams) var pr []*routinghelpers.ParallelRouter for _, cr := range crp.Routers { ri, err := parse(visited, createdRouters, cr.RouterName, routersCfg, extraDHT, extraHTTP) if err != nil { return nil, err } pr = append(pr, &routinghelpers.ParallelRouter{ Router: ri, IgnoreError: cr.IgnoreErrors, DoNotWaitForSearchValue: true, Timeout: cr.Timeout.Duration, ExecuteAfter: cr.ExecuteAfter.WithDefault(0), }) } router = routinghelpers.NewComposableParallel(pr) case config.RouterTypeSequential: crp := cfg.Parameters.(*config.ComposableRouterParams) var sr []*routinghelpers.SequentialRouter for _, cr := range crp.Routers { ri, err := parse(visited, createdRouters, cr.RouterName, routersCfg, extraDHT, extraHTTP) if err != nil { return nil, err } sr = append(sr, &routinghelpers.SequentialRouter{ Router: ri, IgnoreError: cr.IgnoreErrors, Timeout: cr.Timeout.Duration, }) } router = routinghelpers.NewComposableSequential(sr) default: return nil, fmt.Errorf("unknown router type %q", cfg.Type) } if err != nil { return nil, err } createdRouters[routerName] = router log.Info("created router ", routerName, " with params ", cfg.Parameters) return router, nil } type ExtraHTTPParams struct { PeerID string AddrFunc func() []ma.Multiaddr // dynamic address resolver for provider records PrivKeyB64 string HTTPRetrieval bool } func ConstructHTTPRouter(endpoint string, peerID string, addrFunc func() []ma.Multiaddr, privKey string, httpRetrieval bool) (routing.Routing, error) { return httpRoutingFromConfig( config.Router{ Type: "http", Parameters: &config.HTTPRouterParams{ Endpoint: endpoint, }, }, &ExtraHTTPParams{ PeerID: peerID, AddrFunc: addrFunc, PrivKeyB64: privKey, HTTPRetrieval: httpRetrieval, }, ) } func httpRoutingFromConfig(conf config.Router, extraHTTP *ExtraHTTPParams) (routing.Routing, error) { params := conf.Parameters.(*config.HTTPRouterParams) if params.Endpoint == "" { return nil, NewParamNeededErr("Endpoint", conf.Type) } params.FillDefaults() // Increase per-host connection pool since we are making lots of concurrent requests. transport := http.DefaultTransport.(*http.Transport).Clone() transport.MaxIdleConns = 500 transport.MaxIdleConnsPerHost = 100 delegateHTTPClient := &http.Client{ Transport: &drclient.ResponseBodyLimitedTransport{ RoundTripper: otelhttp.NewTransport(transport, otelhttp.WithSpanNameFormatter(func(operation string, req *http.Request) string { if req.Method == http.MethodGet { switch { case strings.HasPrefix(req.URL.Path, "/routing/v1/providers"): return "DelegatedHTTPClient.FindProviders" case strings.HasPrefix(req.URL.Path, "/routing/v1/peers"): return "DelegatedHTTPClient.FindPeers" case strings.HasPrefix(req.URL.Path, "/routing/v1/ipns"): return "DelegatedHTTPClient.GetIPNS" } } else if req.Method == http.MethodPut { switch { case strings.HasPrefix(req.URL.Path, "/routing/v1/ipns"): return "DelegatedHTTPClient.PutIPNS" } } return "DelegatedHTTPClient." + path.Dir(req.URL.Path) }), ), LimitBytes: 1 << 20, }, } key, err := decodePrivKey(extraHTTP.PrivKeyB64) if err != nil { return nil, err } protocols := config.DefaultHTTPRoutersFilterProtocols if extraHTTP.HTTPRetrieval { protocols = append(protocols, "transport-ipfs-gateway-http") } peerID, err := peer.Decode(extraHTTP.PeerID) if err != nil { return nil, err } var providerInfoOpt drclient.Option if extraHTTP.AddrFunc != nil { providerInfoOpt = drclient.WithProviderInfoFunc(peerID, extraHTTP.AddrFunc) } else { providerInfoOpt = drclient.WithProviderInfo(peerID, nil) } cli, err := drclient.New( params.Endpoint, drclient.WithHTTPClient(delegateHTTPClient), drclient.WithIdentity(key), providerInfoOpt, drclient.WithUserAgent(version.GetUserAgentVersion()), drclient.WithProtocolFilter(protocols), drclient.WithStreamResultsRequired(), // https://specs.ipfs.tech/routing/http-routing-v1/#streaming drclient.WithDisabledLocalFiltering(false), // force local filtering in case remote server does not support IPIP-484 ) if err != nil { return nil, err } cr := contentrouter.NewContentRoutingClient( cli, contentrouter.WithMaxProvideBatchSize(params.MaxProvideBatchSize), contentrouter.WithMaxProvideConcurrency(params.MaxProvideConcurrency), ) err = view.Register(drclient.OpenCensusViews...) if err != nil { return nil, fmt.Errorf("registering HTTP delegated routing views: %w", err) } return &httpRoutingWrapper{ ContentRouting: cr, PeerRouting: cr, ValueStore: cr, ProvideManyRouter: cr, }, nil } func decodePrivKey(keyB64 string) (ic.PrivKey, error) { pk, err := base64.StdEncoding.DecodeString(keyB64) if err != nil { return nil, err } return ic.UnmarshalPrivateKey(pk) } type ExtraDHTParams struct { BootstrapPeers []peer.AddrInfo Host host.Host Validator record.Validator Datastore datastore.Batching Context context.Context } func dhtRoutingFromConfig(conf config.Router, extra *ExtraDHTParams) (routing.Routing, error) { params, ok := conf.Parameters.(*config.DHTRouterParams) if !ok { return nil, errors.New("incorrect params for DHT router") } if params.AcceleratedDHTClient { return createFullRT(extra) } var mode dht.ModeOpt switch params.Mode { case config.DHTModeAuto: mode = dht.ModeAuto case config.DHTModeClient: mode = dht.ModeClient case config.DHTModeServer: mode = dht.ModeServer default: return nil, fmt.Errorf("invalid DHT mode: %q", params.Mode) } return createDHT(extra, params.PublicIPNetwork, mode) } func createDHT(params *ExtraDHTParams, public bool, mode dht.ModeOpt) (routing.Routing, error) { var opts []dht.Option if public { opts = append(opts, dht.QueryFilter(dht.PublicQueryFilter), dht.RoutingTableFilter(dht.PublicRoutingTableFilter), dht.RoutingTablePeerDiversityFilter(dht.NewRTPeerDiversityFilter(params.Host, 2, 3))) } else { opts = append(opts, dht.ProtocolExtension(dual.LanExtension), dht.QueryFilter(dht.PrivateQueryFilter), dht.RoutingTableFilter(dht.PrivateRoutingTableFilter)) } opts = append(opts, dht.Concurrency(10), dht.Mode(mode), dht.Datastore(params.Datastore), dht.Validator(params.Validator), dht.BootstrapPeers(params.BootstrapPeers...)) return dht.New( params.Context, params.Host, opts..., ) } func createFullRT(params *ExtraDHTParams) (routing.Routing, error) { return fullrt.NewFullRT(params.Host, dht.DefaultPrefix, fullrt.DHTOption( dht.Validator(params.Validator), dht.Datastore(params.Datastore), dht.BootstrapPeers(params.BootstrapPeers...), dht.BucketSize(20), ), ) } ================================================ FILE: routing/delegated_test.go ================================================ package routing import ( "crypto/rand" "encoding/base64" "testing" "github.com/ipfs/kubo/config" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" ) func TestParser(t *testing.T) { require := require.New(t) pid, sk, err := generatePeerID() require.NoError(err) router, err := Parse(config.Routers{ "r1": config.RouterParser{ Router: config.Router{ Type: config.RouterTypeHTTP, Parameters: &config.HTTPRouterParams{ Endpoint: "http://testEndpoint", }, }, }, "r2": config.RouterParser{ Router: config.Router{ Type: config.RouterTypeSequential, Parameters: &config.ComposableRouterParams{ Routers: []config.ConfigRouter{ { RouterName: "r1", }, }, }, }, }, }, config.Methods{ config.MethodNameFindPeers: config.Method{ RouterName: "r1", }, config.MethodNameFindProviders: config.Method{ RouterName: "r1", }, config.MethodNameGetIPNS: config.Method{ RouterName: "r1", }, config.MethodNamePutIPNS: config.Method{ RouterName: "r2", }, config.MethodNameProvide: config.Method{ RouterName: "r2", }, }, &ExtraDHTParams{}, &ExtraHTTPParams{ PeerID: string(pid), PrivKeyB64: sk, }) require.NoError(err) comp, ok := router.(*Composer) require.True(ok) require.Equal(comp.FindPeersRouter, comp.FindProvidersRouter) require.Equal(comp.ProvideRouter, comp.PutValueRouter) } func TestParserRecursive(t *testing.T) { require := require.New(t) pid, sk, err := generatePeerID() require.NoError(err) router, err := Parse(config.Routers{ "http1": config.RouterParser{ Router: config.Router{ Type: config.RouterTypeHTTP, Parameters: &config.HTTPRouterParams{ Endpoint: "http://testEndpoint1", }, }, }, "http2": config.RouterParser{ Router: config.Router{ Type: config.RouterTypeHTTP, Parameters: &config.HTTPRouterParams{ Endpoint: "http://testEndpoint2", }, }, }, "http3": config.RouterParser{ Router: config.Router{ Type: config.RouterTypeHTTP, Parameters: &config.HTTPRouterParams{ Endpoint: "http://testEndpoint3", }, }, }, "composable1": config.RouterParser{ Router: config.Router{ Type: config.RouterTypeSequential, Parameters: &config.ComposableRouterParams{ Routers: []config.ConfigRouter{ { RouterName: "http1", }, { RouterName: "http2", }, }, }, }, }, "composable2": config.RouterParser{ Router: config.Router{ Type: config.RouterTypeParallel, Parameters: &config.ComposableRouterParams{ Routers: []config.ConfigRouter{ { RouterName: "composable1", }, { RouterName: "http3", }, }, }, }, }, }, config.Methods{ config.MethodNameFindPeers: config.Method{ RouterName: "composable2", }, config.MethodNameFindProviders: config.Method{ RouterName: "composable2", }, config.MethodNameGetIPNS: config.Method{ RouterName: "composable2", }, config.MethodNamePutIPNS: config.Method{ RouterName: "composable2", }, config.MethodNameProvide: config.Method{ RouterName: "composable2", }, }, &ExtraDHTParams{}, &ExtraHTTPParams{ PeerID: string(pid), PrivKeyB64: sk, }) require.NoError(err) _, ok := router.(*Composer) require.True(ok) } func TestParserRecursiveLoop(t *testing.T) { require := require.New(t) _, err := Parse(config.Routers{ "composable1": config.RouterParser{ Router: config.Router{ Type: config.RouterTypeSequential, Parameters: &config.ComposableRouterParams{ Routers: []config.ConfigRouter{ { RouterName: "composable2", }, }, }, }, }, "composable2": config.RouterParser{ Router: config.Router{ Type: config.RouterTypeParallel, Parameters: &config.ComposableRouterParams{ Routers: []config.ConfigRouter{ { RouterName: "composable1", }, }, }, }, }, }, config.Methods{ config.MethodNameFindPeers: config.Method{ RouterName: "composable2", }, config.MethodNameFindProviders: config.Method{ RouterName: "composable2", }, config.MethodNameGetIPNS: config.Method{ RouterName: "composable2", }, config.MethodNamePutIPNS: config.Method{ RouterName: "composable2", }, config.MethodNameProvide: config.Method{ RouterName: "composable2", }, }, &ExtraDHTParams{}, nil) require.ErrorContains(err, "dependency loop creating router with name \"composable2\"") } func generatePeerID() (string, string, error) { sk, pk, err := crypto.GenerateEd25519Key(rand.Reader) if err != nil { return "", "", err } bytes, err := crypto.MarshalPrivateKey(sk) if err != nil { return "", "", err } enc := base64.StdEncoding.EncodeToString(bytes) if err != nil { return "", "", err } pid, err := peer.IDFromPublicKey(pk) return pid.String(), enc, err } ================================================ FILE: routing/error.go ================================================ package routing import ( "fmt" "github.com/ipfs/kubo/config" ) type ParamNeededError struct { ParamName string RouterType config.RouterType } func NewParamNeededErr(param string, routing config.RouterType) error { return &ParamNeededError{ ParamName: param, RouterType: routing, } } func (e *ParamNeededError) Error() string { return fmt.Sprintf("configuration param '%v' is needed for %v delegated routing types", e.ParamName, e.RouterType) } ================================================ FILE: routing/wrapper.go ================================================ package routing import ( "context" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/routing" ) type ProvideManyRouter interface { routinghelpers.ProvideManyRouter routing.Routing } var ( _ routing.Routing = &httpRoutingWrapper{} _ routinghelpers.ProvideManyRouter = &httpRoutingWrapper{} ) // httpRoutingWrapper is a wrapper needed to construct the routing.Routing interface from // http delegated routing. type httpRoutingWrapper struct { routing.ContentRouting routing.PeerRouting routing.ValueStore routinghelpers.ProvideManyRouter } func (c *httpRoutingWrapper) Bootstrap(ctx context.Context) error { return nil } ================================================ FILE: test/.gitignore ================================================ IPFS-BUILD-OPTIONS ================================================ FILE: test/3nodetest/GNUmakefile ================================================ IMAGE_NAME = ipfs-test-latest IPFS_ROOT = ../.. test: clean setup ./run-test-on-img.sh $(IMAGE_NAME) setup: docker_ipfs_image data/filetiny data/filerand save_logs: sh bin/save_logs.sh save_profiling_data: sh bin/save_profiling_data.sh data/filetiny: Makefile cp Makefile ./data/filetiny # simple data/filerand: ../bin/random ../bin/random 50000000 > ./data/filerand ../bin/random: make -C ./../../ test/bin/random # just build it every time... this part isn't # even the lengthy part, and it decreases pain. docker_ipfs_image: docker build -t $(IMAGE_NAME) -f Dockerfile . docker images | grep $(IMAGE_NAME) clean: sh bin/clean.sh fig stop fig rm -v --force rm -f bin/random rm -f data/filetiny rm -f data/filerand rm -rf build/* docker rmi $(docker images | grep "^" | awk "{print \$3}") -f || true ================================================ FILE: test/3nodetest/README.md ================================================ this is an ipfs integration test **requirements** * Docker * fig * Go * ipfs image named "zaqwsx_ipfs-test-img" ``` make setup fig build fig up ``` ================================================ FILE: test/3nodetest/bin/.gitignore ================================================ random ================================================ FILE: test/3nodetest/bin/clean.sh ================================================ docker rm -f $( docker ps -q -a -f status=exited ) || true ================================================ FILE: test/3nodetest/bin/save_logs.sh ================================================ # STRIP strips color from terminal output STRIP="perl -pe 's/\e\[?.*?[\@-~]//g'" # TODO use a for loop like a grownup docker logs 3nodetest_bootstrap_1 2>&1 | eval $STRIP > ./build/bootstrap.log docker logs 3nodetest_client_1 2>&1 | eval $STRIP > ./build/client.log docker logs 3nodetest_data_1 2>&1 | eval $STRIP > ./build/data.log docker logs 3nodetest_server_1 2>&1 | eval $STRIP > ./build/server.log ================================================ FILE: test/3nodetest/bin/save_profiling_data.sh ================================================ #!/bin/sh for container in 3nodetest_bootstrap_1 3nodetest_client_1 3nodetest_server_1; do # ipfs binary is required by `go tool pprof` docker cp $container:/go/bin/ipfs build/profiling_data_$container done # since the nodes are executed with the --debug flag, profiling data is written # to the working dir. by default, the working dir is /go. for container in 3nodetest_bootstrap_1 3nodetest_client_1 3nodetest_server_1; do docker cp $container:/go/ipfs.cpuprof build/profiling_data_$container done # TODO get memprof from client (client daemon isn't terminated, so memprof isn't retrieved) for container in 3nodetest_bootstrap_1 3nodetest_server_1; do docker cp $container:/go/ipfs.memprof build/profiling_data_$container done ================================================ FILE: test/3nodetest/bootstrap/Dockerfile ================================================ FROM zaqwsx_ipfs-test-img RUN ipfs init -b=2048 ADD . /tmp/id RUN mv -f /tmp/id/config /root/.ipfs/config RUN ipfs id ENV IPFS_PROF true ENV GOLOG_LOG_FMT nocolor EXPOSE 4011 4012/udp ================================================ FILE: test/3nodetest/bootstrap/README.md ================================================ this is a bootstrap peer with an empty bootstrap list it listens on 4011 and 4012 ================================================ FILE: test/3nodetest/bootstrap/config ================================================ { "Identity": { "PeerID": "QmNXuBh8HFsWq68Fid8dMbGNQTh7eG6hV9rr1fQyfmfomE", "PrivKey": "CAAS4gQwggJeAgEAAoGBAL+E7A0fcQS9+CHO3YAHj+JzHnWyVA7qqtiAIYbTnp9UvHBb2VFj2Q8eeyKFZD5wHoq3AtOqmIb4TUOMEtWYqXnE8o0T9np8vyCRK5dPn5SVoUw9uax6o2X7OxO1HqTcXHNHGbracawJUdwsk4yuZUpzXLez03yocWwneR0JpVJPAgMBAAECgYAXsa4ygW1OFOKZ7CnjKQxYC738+a8EmWvBlTiQoaXCOI2HqRVdyGiWQkMhpjccsmpU5wdmgHiWWinU7YN3AYgV3cP3qAjyNLBFoxy2dKsS9AOWVwRuuRP12tD05kCCjG4rJAX0JEOClOOtzvQ7/bXarMc3/tMHW7TMLNV8MzcYwQJBAOP9aYSHp8VnsO5j32Ju5SjOQorSdcCweqeUxwlAnXz50KdbNSCMypP3TOt7VeiXTuSITtN44yh+eogF5c4ehycCQQDXDHVmPeBN7uqqqZxZwW5pdeJWvx+REiXXCLE6KEPWlcxbw1D9ublpCCFLYuM68rjq1sjsIVGtiV1tYoMdHJSZAkEA0ddMZ070fB0UHFaQJGktQoGVfXB4MQI94kBtcXanfX/xLBgmre7oBYh4o8TBLXMWigFri/iYG41N+iRzf2NZwQJBAIh8rMpufT2ZZLFaoxRIc4ZVvojmFufhR8j6CFnsElpQivq2tWHEDcx+z3rkUWopgXnzRmSwJQHqTDTPsH26lQkCQQCehmxqaQEwE/QKAI8L8YVolgY2cjUGi6qF/awnI584lDSCuJRU3R/c6nc9R8qljtlJYTtp9iUr8vuN+jt48j+6" }, "Datastore": { "Type": "leveldb", "Path": "/root/.ipfs/datastore" }, "Addresses": { "Swarm": [ "/ip4/0.0.0.0/tcp/4011" ], "API": "/ip4/127.0.0.1/tcp/5001" }, "Mounts": { "IPFS": "/ipfs", "IPNS": "/ipns", "MFS": "/mfs" }, "Version": { "Current": "0.1.7", "Check": "error", "CheckDate": "0001-01-01T00:00:00Z", "CheckPeriod": "172800000000000", "AutoUpdate": "minor" }, "Bootstrap": [ ] } ================================================ FILE: test/3nodetest/build/.gitignore ================================================ .built_img *.log profiling_data* ================================================ FILE: test/3nodetest/build/.gitkeep ================================================ ================================================ FILE: test/3nodetest/client/Dockerfile ================================================ FROM zaqwsx_ipfs-test-img RUN ipfs init -b=2048 ADD . /tmp/id RUN mv -f /tmp/id/config /root/.ipfs/config RUN ipfs id EXPOSE 4031 4032/udp ENV IPFS_PROF true ENV GOLOG_LOG_FMT nocolor ENTRYPOINT ["/bin/bash"] CMD ["/tmp/id/run.sh"] ================================================ FILE: test/3nodetest/client/config ================================================ { "Addresses": { "API": "/ip4/127.0.0.1/tcp/5001", "Swarm": [ "/ip4/0.0.0.0/tcp/4031" ] }, "Bootstrap": [ ], "Datastore": { "Path": "/root/.ipfs/datastore", "Type": "leveldb" }, "Identity": { "PeerID": "Qmbtc35vdjVh5o9w2AaT2SgcWwigsaoZUpRfq2FSrFD6mb", "PrivKey": "CAAS4AQwggJcAgEAAoGBANlJUjOCbPXgYUfo1Pr6nlIjJDPNwN81ACamhaoEZ9VRHXI3fPe7RVAaaXrWLHb892mRqFi1ScE2lcMTLc7WGfyc7dwPqBOZqkVvT0KpCx3Mg246+WvnG8I3HCbWyjSP9tJflOBQxVq6qT2yZSXjNTtDdO4skd4PsPqBco53guYTAgMBAAECgYEAtIcYhrdMNBSSfp5RpZxnwbJ0t52xK0HruDEOSK2UX0Ufg+/aIjEza1QmYupi0xFltg5QojMs7hyd3Q+oNXro5tKsYVeiqrLsUh9jMjaQofzSlV9Oc+bhkkl48YWvF6Y8qx88UYAX+oJqB627H4S1gxLdNEJhPjEAD6n/jql3zUECQQDmHP75wJ7nC4TlxT1SHim5syMAqWNs/SOHnvX8yLrFV9FrMRzsD5qMlIEGBrAjaESzEck6XpbqkyxB8KKGo7OjAkEA8brtEh/AMoQ/yoSWdYT2MRbJxCAn+KG2c6Hi9AMMmJ+K779HxywpUIDYIa22hzLKYumYIuRa1X++1glOAFGq0QJAPQgXwFoMSy9M8jwcBXmmi3AtqnFCw5doIwJQL9l1X/3ot0txZlLFJOAGUHjZoqp2/h+LhYWs9U5PgLW4BYnJjQJAPydY/J0y93+5ss1FCdr8/wI3IHhOORT2t+sZgiqxxcYY5F4TAKQ2/wNKdDIQN+47FfB1gNgsKw8+6mhv6oFroQJACBF2yssNVXiXa2Na/a9tKYutGvxbm3lXzOvmpkW3FukbsObKYS344J1vdg0nzM6EWQCaiBweSA5TQ27iNW6BzQ==" }, "Mounts": { "IPFS": "/ipfs", "IPNS": "/ipns", "MFS": "/mfs" }, "Version": { "AutoUpdate": "minor", "Check": "error", "CheckDate": "0001-01-01T00:00:00Z", "CheckPeriod": "172800000000000", "Current": "0.1.7" } } ================================================ FILE: test/3nodetest/client/run.sh ================================================ ipfs bootstrap add /ip4/$BOOTSTRAP_PORT_4011_TCP_ADDR/tcp/$BOOTSTRAP_PORT_4011_TCP_PORT/p2p/QmNXuBh8HFsWq68Fid8dMbGNQTh7eG6hV9rr1fQyfmfomE ipfs bootstrap # list bootstrap nodes for debugging echo "3nodetest> starting client daemon" ipfs daemon --debug & sleep 3 # switch dirs so ipfs client profiling data doesn't overwrite the ipfs daemon # profiling data cd /tmp while [ ! -f /data/idtiny ] do echo "3nodetest> waiting for server to add the file..." sleep 1 done echo "3nodetest> client found file with hash:" $(cat /data/idtiny) ipfs cat $(cat /data/idtiny) > filetiny cat filetiny diff -u filetiny /data/filetiny if (($? > 0)); then printf '%s\n' 'files did not match' >&2 exit 1 fi while [ ! -f /data/idrand ] do echo "3nodetest> waiting for server to add the file..." sleep 1 done echo "3nodetest> client found file with hash:" $(cat /data/idrand) cat /data/idrand ipfs cat $(cat /data/idrand) > filerand if (($? > 0)); then printf '%s\n' 'ipfs cat failed' >&2 exit 1 fi diff -u filerand /data/filerand if (($? > 0)); then printf '%s\n' 'files did not match' >&2 exit 1 fi echo "3nodetest> success" ================================================ FILE: test/3nodetest/data/.gitignore ================================================ file* ================================================ FILE: test/3nodetest/data/Dockerfile ================================================ FROM ubuntu ADD filetiny /data/filetiny ADD filerand /data/filerand VOLUME ["/data"] ================================================ FILE: test/3nodetest/fig.yml ================================================ data: build: ./data volumes: - /data command: sleep 1000000 bootstrap: build: ./bootstrap command: daemon --debug --init expose: - "4011" - "4012/udp" environment: GOLOG_LOG_LEVEL: debug server: build: ./server links: - bootstrap volumes_from: - data expose: - "4021" - "4022/udp" environment: GOLOG_LOG_LEVEL: debug client: build: ./client links: - bootstrap volumes_from: - data expose: - "4031" - "4032/udp" environment: GOLOG_LOG_LEVEL: debug ================================================ FILE: test/3nodetest/run-test-on-img.sh ================================================ #!/bin/sh if [ "$#" -ne 1 ]; then echo "usage: $0 " echo "runs this test on image matching " exit 1 fi # this tag is used by the dockerfiles in # {data, server, client, bootstrap} tag=zaqwsx_ipfs-test-img # could use set -v, but i don't want to see the comments... img=$(docker images | grep $1 | awk '{print $3}') echo "using docker image: $img ($1)" echo docker tag -f $img $tag docker tag -f $img $tag echo "fig build --no-cache" fig build --no-cache echo "fig up --no-color | tee build/fig.log" fig up --no-color | tee build/fig.log # save the ipfs logs for inspection echo "make save_logs" make save_logs || true # don't fail # save the ipfs logs for inspection echo "make save_profiling_data" make save_profiling_data || true # don't fail # fig up won't report the error using an error code, so we grep the # fig.log file to find out whether the call succeeded echo 'tail build/fig.log | grep "exited with code 0"' tail build/fig.log | grep "exited with code 0" ================================================ FILE: test/3nodetest/server/Dockerfile ================================================ FROM zaqwsx_ipfs-test-img RUN ipfs init -b=2048 ADD . /tmp/test RUN mv -f /tmp/test/config /root/.ipfs/config RUN ipfs id RUN chmod +x /tmp/test/run.sh EXPOSE 4021 4022/udp ENV IPFS_PROF true ENV GOLOG_LOG_FMT nocolor ENTRYPOINT ["/bin/bash"] CMD ["/tmp/test/run.sh"] ================================================ FILE: test/3nodetest/server/README.md ================================================ **requirements** * docker container with bootstrap node linked as "bootstrap" with ID QmNXuBh8HFsWq68Fid8dMbGNQTh7eG6hV9rr1fQyfmfomE * file in data volume, internally mapped to /data/file ================================================ FILE: test/3nodetest/server/config ================================================ { "Addresses": { "API": "/ip4/127.0.0.1/tcp/5001", "Swarm": [ "/ip4/0.0.0.0/tcp/4021" ] }, "Bootstrap": [ ], "Datastore": { "Path": "/root/.ipfs/datastore", "Type": "leveldb" }, "Identity": { "PeerID": "Qmbtc2C7rqmAfdeMTM7FX4YF8CeBumMCfk5Z1GBCMbMTfY", "PrivKey": "CAAS4AQwggJcAgEAAoGBANW3mJMmDSJbdRyykO0Ze5t6WL6jeTtpOhklxePBIkJL/Uil78Va/tODx6Mvv3GMCkbGvzWslTZXpaHa9vBmjE3MVZSmd5fLRybKT0zZ3juABKcx+WIVNw8JlkpEORihJdwb+5tRUC5pUcMzxqHSmGX+d6e9KZqLnv7piNKg2+r7AgMBAAECgYAqc6+w+wv82SHoM2gqULeG6MScCajZLkvGFwS5+vEtLh7/wUZhc3PO3AxZ0/A5Q9H+wRfWN5PkGYDjJ7WJhzUzGfTbrQ821JV6B3IUR4UHo2IgJkZO4EUB5L9KBUqvYxDJigtGBopgQh0EeDSS+9X8vaGmit5l4zcAfi+UGYPgMQJBAOCJQU8N2HW5SawBo2QX0bnCAAnu5Ilk2QaqwDZbDQaM5JWFcpRpGnjBhsZihHwVWvKCbnq83JhAGRQvKAEepMUCQQDzqjvIyM+Au42nP7SFDHoMjEnHW8Nimvz8zPbyrSUEHe4l9/yS4+BeRPxpwI5xgzp8g1wEYfNeXt08buYwCsy/AkBXWg5mSuSjJ+pZWGnQTtPwiGCrfJy8NteXmGYev11Z5wYmhTwGML1zrRZZp4oTG9u97LA+X6sSMB2RlKbjiKBhAkEAgl/hoSshK+YugwCpHE9ytmgRyeOlhYscNj+NGofeOHezRwmLUSUwlgAfdo4bKU1n69t1TrsCNspXYdCMxcPhjQJAMNxkJ8t2tFMpucCQfWJ09wvFKZSHX1/iD9GKWL0Qk2FcMCg3NXiqei5NL3NYqCWpdC/IfjsAEGCJrTFwp/OoUw==" }, "Mounts": { "IPFS": "/ipfs", "IPNS": "/ipns", "MFS": "/mfs" }, "Version": { "AutoUpdate": "minor", "Check": "error", "CheckDate": "0001-01-01T00:00:00Z", "CheckPeriod": "172800000000000", "Current": "0.1.7" } } ================================================ FILE: test/3nodetest/server/run.sh ================================================ # must be connected to bootstrap node ipfs bootstrap add /ip4/$BOOTSTRAP_PORT_4011_TCP_ADDR/tcp/$BOOTSTRAP_PORT_4011_TCP_PORT/p2p/QmNXuBh8HFsWq68Fid8dMbGNQTh7eG6hV9rr1fQyfmfomE ipfs bootstrap # list bootstrap nodes for debugging # wait for daemon to start/bootstrap # alternatively use ipfs swarm connect echo "3nodetest> starting server daemon" # run daemon in debug mode to collect profiling data ipfs daemon --debug & sleep 3 # TODO instead of bootstrapping: ipfs swarm connect /ip4/$BOOTSTRAP_PORT_4011_TCP_ADDR/tcp/$BOOTSTRAP_PORT_4011_TCP_PORT/p2p/QmNXuBh8HFsWq68Fid8dMbGNQTh7eG6hV9rr1fQyfmfomE # change dir before running add commands so ipfs client profiling data doesn't # overwrite the daemon profiling data cd /tmp # must mount this volume from data container ipfs add -q /data/filetiny > tmptiny mv tmptiny /data/idtiny echo "3nodetest> added tiny file. hash is" $(cat /data/idtiny) ipfs add -q /data/filerand > tmprand mv tmprand /data/idrand echo "3nodetest> added rand file. hash is" $(cat /data/idrand) # allow ample time for the client to pull the data sleep 10000000 ================================================ FILE: test/README.md ================================================ ## Sharness test command coverage Module | Online Test | Offline Test | -----------|-----------------|-----------------| object | t0051 | t0051 ls | t0045 | t0045 cat | t0040 | dht | | bitswap | | block | | t0050 daemon | t0030 | N/A init | N/A | t0020 add | t0040 | config | t0021 | t0021 version | t0060 | t0010 ping | | diag | | mount | t0030 | name | t0110 | t0100 pin | t0080 | get | t0090 | t0090 refs | t0080 | repo gc | t0080 | id | | bootstrap | t0120 | t0120 swarm | | update | | commands | | ================================================ FILE: test/Rules.mk ================================================ include mk/header.mk dir := $(d)/bin include $(dir)/Rules.mk dir := $(d)/sharness include $(dir)/Rules.mk dir := $(d)/unit include $(dir)/Rules.mk include mk/footer.mk ================================================ FILE: test/api-startup/main.go ================================================ package main import ( "fmt" "log" "net/http" "sync" "time" ) func main() { when := make(chan time.Time, 2) var wg sync.WaitGroup wg.Add(2) for _, port := range []string{"5001", "8080"} { go func(port string) { defer wg.Done() for { r, err := http.Get(fmt.Sprintf("http://127.0.0.1:%s", port)) if err != nil { continue } t := time.Now() when <- t log.Println(port, t, r.StatusCode) break } }(port) } wg.Wait() first := <-when second := <-when log.Println(second.Sub(first)) } ================================================ FILE: test/bench/bench_cli_ipfs_add/main.go ================================================ package main import ( "flag" "fmt" "io" "log" "os" "os/exec" "path" "testing" "github.com/ipfs/kubo/thirdparty/unit" random "github.com/ipfs/go-test/random" config "github.com/ipfs/kubo/config" ) var ( debug = flag.Bool("debug", false, "direct ipfs output to console") online = flag.Bool("online", false, "run the benchmarks with a running daemon") ) func main() { flag.Parse() if err := compareResults(); err != nil { log.Fatal(err) } } func compareResults() error { var amount unit.Information for amount = 10 * unit.MB; amount > 0; amount = amount * 2 { if results, err := benchmarkAdd(int64(amount)); err != nil { // TODO compare return err } else { log.Println(amount, "\t", results) } } return nil } func benchmarkAdd(amount int64) (*testing.BenchmarkResult, error) { var benchmarkError error results := testing.Benchmark(func(b *testing.B) { b.SetBytes(amount) for i := 0; i < b.N; i++ { b.StopTimer() tmpDir := b.TempDir() env := append( []string{fmt.Sprintf("%s=%s", config.EnvDir, path.Join(tmpDir, config.DefaultPathName))}, // first in order to override os.Environ()..., ) setupCmd := func(cmd *exec.Cmd) { cmd.Env = env if *debug { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr } } initCmd := exec.Command("ipfs", "init") setupCmd(initCmd) if err := initCmd.Run(); err != nil { benchmarkError = err b.Fatal(err) } const seed = 1 f, err := os.CreateTemp("", "") if err != nil { benchmarkError = err b.Fatal(err) } defer os.Remove(f.Name()) randReader := &io.LimitedReader{ R: random.NewSeededRand(seed), N: amount, } if _, err := io.Copy(f, randReader); err != nil { benchmarkError = err b.Fatal(err) } if err := f.Close(); err != nil { benchmarkError = err b.Fatal(err) } func() { // FIXME online mode isn't working. client complains that it cannot open leveldb if *online { daemonCmd := exec.Command("ipfs", "daemon") setupCmd(daemonCmd) if err := daemonCmd.Start(); err != nil { benchmarkError = err b.Fatal(err) } defer func() { _ = daemonCmd.Process.Signal(os.Interrupt) _ = daemonCmd.Wait() }() } b.StartTimer() addCmd := exec.Command("ipfs", "add", f.Name()) setupCmd(addCmd) if err := addCmd.Run(); err != nil { benchmarkError = err b.Fatal(err) } b.StopTimer() }() } }) if benchmarkError != nil { return nil, benchmarkError } return &results, nil } ================================================ FILE: test/bench/offline_add/main.go ================================================ package main import ( "fmt" "io" "log" "os" "os/exec" "path" "testing" "github.com/ipfs/kubo/thirdparty/unit" random "github.com/ipfs/go-test/random" config "github.com/ipfs/kubo/config" ) func main() { if err := compareResults(); err != nil { log.Fatal(err) } } func compareResults() error { var amount unit.Information for amount = 10 * unit.MB; amount > 0; amount = amount * 2 { if results, err := benchmarkAdd(int64(amount)); err != nil { // TODO compare return err } else { log.Println(amount, "\t", results) } } return nil } func benchmarkAdd(amount int64) (*testing.BenchmarkResult, error) { results := testing.Benchmark(func(b *testing.B) { b.SetBytes(amount) for i := 0; i < b.N; i++ { b.StopTimer() tmpDir := b.TempDir() env := append(os.Environ(), fmt.Sprintf("%s=%s", config.EnvDir, path.Join(tmpDir, config.DefaultPathName))) setupCmd := func(cmd *exec.Cmd) { cmd.Env = env } cmd := exec.Command("ipfs", "init") setupCmd(cmd) if err := cmd.Run(); err != nil { b.Fatal(err) } const seed = 1 f, err := os.CreateTemp("", "") if err != nil { b.Fatal(err) } defer os.Remove(f.Name()) randReader := &io.LimitedReader{ R: random.NewSeededRand(seed), N: amount, } _, err = io.Copy(f, randReader) if err != nil { b.Fatal(err) } if err := f.Close(); err != nil { b.Fatal(err) } b.StartTimer() cmd = exec.Command("ipfs", "add", f.Name()) setupCmd(cmd) if err := cmd.Run(); err != nil { b.Fatal(err) } b.StopTimer() } }) return &results, nil } ================================================ FILE: test/bin/.gitignore ================================================ # Ignore everything in this directory by default /** # Do not ignore this file itself !.gitignore # Do not ignore the following special scripts !checkflags !continueyn !verify-go-fmt.sh !Rules.mk ================================================ FILE: test/bin/Rules.mk ================================================ include mk/header.mk TGTS_$(d) := define go-build-testdep OUT="$(CURDIR)/$@" ; \ cd "test/dependencies" ; \ $(GOCC) build $(go-flags-with-tags) -o "$${OUT}" "$<" 2>&1 endef .PHONY: github.com/ipfs/kubo/test/dependencies/pollEndpoint $(d)/pollEndpoint: github.com/ipfs/kubo/test/dependencies/pollEndpoint $(go-build-testdep) TGTS_$(d) += $(d)/pollEndpoint .PHONY: github.com/ipfs/kubo/test/dependencies/go-sleep $(d)/go-sleep: github.com/ipfs/kubo/test/dependencies/go-sleep $(go-build-testdep) TGTS_$(d) += $(d)/go-sleep .PHONY: github.com/ipfs/kubo/test/dependencies/go-timeout $(d)/go-timeout: github.com/ipfs/kubo/test/dependencies/go-timeout $(go-build-testdep) TGTS_$(d) += $(d)/go-timeout .PHONY: github.com/ipfs/kubo/test/dependencies/iptb $(d)/iptb: github.com/ipfs/kubo/test/dependencies/iptb $(go-build-testdep) TGTS_$(d) += $(d)/iptb .PHONY: github.com/ipfs/kubo/test/dependencies/ma-pipe-unidir $(d)/ma-pipe-unidir: github.com/ipfs/kubo/test/dependencies/ma-pipe-unidir $(go-build-testdep) TGTS_$(d) += $(d)/ma-pipe-unidir .PHONY: github.com/ipfs/kubo/test/dependencies/json-to-junit $(d)/json-to-junit: github.com/ipfs/kubo/test/dependencies/json-to-junit $(go-build-testdep) TGTS_$(d) += $(d)/json-to-junit .PHONY: gotest.tools/gotestsum $(d)/gotestsum: gotest.tools/gotestsum $(go-build-testdep) TGTS_$(d) += $(d)/gotestsum .PHONY: github.com/ipfs/hang-fds $(d)/hang-fds: github.com/ipfs/hang-fds $(go-build-testdep) TGTS_$(d) += $(d)/hang-fds .PHONY: github.com/multiformats/go-multihash/multihash $(d)/multihash: github.com/multiformats/go-multihash/multihash $(go-build-testdep) TGTS_$(d) += $(d)/multihash .PHONY: github.com/ipfs/go-cidutil/cid-fmt $(d)/cid-fmt: github.com/ipfs/go-cidutil/cid-fmt $(go-build-testdep) TGTS_$(d) += $(d)/cid-fmt .PHONY: github.com/ipfs/go-test/cli/random-data $(d)/random-data: github.com/ipfs/go-test/cli/random-data $(go-build-testdep) TGTS_$(d) += $(d)/random-data .PHONY: github.com/ipfs/go-test/cli/random-files $(d)/random-files: github.com/ipfs/go-test/cli/random-files $(go-build-testdep) TGTS_$(d) += $(d)/random-files .PHONY: github.com/Kubuxu/gocovmerge $(d)/gocovmerge: github.com/Kubuxu/gocovmerge $(go-build-testdep) TGTS_$(d) += $(d)/gocovmerge .PHONY: github.com/golangci/golangci-lint/cmd/golangci-lint $(d)/golangci-lint: github.com/golangci/golangci-lint/cmd/golangci-lint $(go-build-testdep) TGTS_$(d) += $(d)/golangci-lint $(TGTS_$(d)): $$(DEPS_GO) CLEAN += $(TGTS_$(d)) PATH := $(realpath $(d)):$(PATH) include mk/footer.mk ================================================ FILE: test/bin/checkflags ================================================ #!/bin/sh # Author: Christian Couder # MIT LICENSED if test "$#" -lt 3 then echo >&2 "usage $0 FILE VALUES MSG..." exit 1 fi FLAG_FILE="$1" FLAG_VALS="$2" shift shift FLAG_MSGS="$@" test -f $FLAG_FILE || touch $FLAG_FILE # Use x in front of tested values as flags could be # interpreted by "test" to be for itself. if test x"$FLAG_VALS" != x"$(cat "$FLAG_FILE")" then echo "$FLAG_MSGS" echo "$FLAG_VALS" >"$FLAG_FILE" fi ================================================ FILE: test/bin/continueyn ================================================ #!/bin/sh # Author: Juan Batiz-Benet # MIT LICENSED # if not a terminal, exit 0 (yes!) test -t 1 || exit 0 read -p "continue? [y/N] " REPLY echo case "$REPLY" in [Yy]*) exit 0 ;; *) exit 1 ;; esac ================================================ FILE: test/cli/add_test.go ================================================ package cli import ( "io" "os" "path/filepath" "strings" "testing" "time" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // waitForLogMessage polls a buffer for a log message, waiting up to timeout duration. // Returns true if message found, false if timeout reached. func waitForLogMessage(buffer *harness.Buffer, message string, timeout time.Duration) bool { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { if strings.Contains(buffer.String(), message) { return true } time.Sleep(100 * time.Millisecond) } return false } func TestAdd(t *testing.T) { t.Parallel() var ( shortString = "hello world" shortStringCidV0 = "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD" // cidv0 - dag-pb - sha2-256 shortStringCidV1 = "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e" // cidv1 - raw - sha2-256 shortStringCidV1NoRawLeaves = "bafybeihykld7uyxzogax6vgyvag42y7464eywpf55gxi5qpoisibh3c5wa" // cidv1 - dag-pb - sha2-256 shortStringCidV1Sha512 = "bafkrgqbqt3gerhas23vuzrapkdeqf4vu2dwxp3srdj6hvg6nhsug2tgyn6mj3u23yx7utftq3i2ckw2fwdh5qmhid5qf3t35yvkc5e5ottlw6" ) t.Run("produced cid version: implicit default (CIDv0)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() cidStr := node.IPFSAddStr(shortString) require.Equal(t, shortStringCidV0, cidStr) }) t.Run("produced cid version: follows user-set configuration Import.CidVersion=0", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.CidVersion = *config.NewOptionalInteger(0) }) node.StartDaemon() defer node.StopDaemon() cidStr := node.IPFSAddStr(shortString) require.Equal(t, shortStringCidV0, cidStr) }) t.Run("produced cid multihash: follows user-set configuration in Import.HashFunction", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.HashFunction = *config.NewOptionalString("sha2-512") }) node.StartDaemon() defer node.StopDaemon() cidStr := node.IPFSAddStr(shortString) require.Equal(t, shortStringCidV1Sha512, cidStr) }) t.Run("produced cid version: follows user-set configuration Import.CidVersion=1", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.CidVersion = *config.NewOptionalInteger(1) }) node.StartDaemon() defer node.StopDaemon() cidStr := node.IPFSAddStr(shortString) require.Equal(t, shortStringCidV1, cidStr) }) t.Run("produced cid version: command flag overrides configuration in Import.CidVersion", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.CidVersion = *config.NewOptionalInteger(1) }) node.StartDaemon() defer node.StopDaemon() cidStr := node.IPFSAddStr(shortString, "--cid-version", "0") require.Equal(t, shortStringCidV0, cidStr) }) t.Run("produced unixfs raw leaves: follows user-set configuration Import.UnixFSRawLeaves", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { // CIDv1 defaults to raw-leaves=true cfg.Import.CidVersion = *config.NewOptionalInteger(1) // disable manually cfg.Import.UnixFSRawLeaves = config.False }) node.StartDaemon() defer node.StopDaemon() cidStr := node.IPFSAddStr(shortString) require.Equal(t, shortStringCidV1NoRawLeaves, cidStr) }) t.Run("ipfs add --pin-name=foo", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() pinName := "test-pin-name" cidStr := node.IPFSAddStr(shortString, "--pin-name", pinName) require.Equal(t, shortStringCidV0, cidStr) pinList := node.IPFS("pin", "ls", "--names").Stdout.Trimmed() require.Contains(t, pinList, shortStringCidV0) require.Contains(t, pinList, pinName) }) t.Run("ipfs add --pin=false --pin-name=foo returns an error", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Use RunIPFS to allow for errors without assertion result := node.RunIPFS("add", "--pin=false", "--pin-name=foo") require.Error(t, result.Err, "Expected an error due to incompatible --pin and --pin-name") require.Contains(t, result.Stderr.String(), "pin-name option requires pin to be set") }) t.Run("ipfs add --pin-name without value should fail", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // When --pin-name is passed without any value, it should fail result := node.RunIPFS("add", "--pin-name") require.Error(t, result.Err, "Expected an error when --pin-name has no value") require.Contains(t, result.Stderr.String(), "missing argument for option \"pin-name\"") }) t.Run("produced unixfs max file links: command flag --max-file-links overrides configuration in Import.UnixFSFileMaxLinks", func(t *testing.T) { t.Parallel() // // UnixFSChunker=size-262144 (256KiB) // Import.UnixFSFileMaxLinks=174 node := harness.NewT(t).NewNode().Init("--profile=unixfs-v0-2015") // unixfs-v0-2015 for determinism across all params node.UpdateConfig(func(cfg *config.Config) { cfg.Import.UnixFSChunker = *config.NewOptionalString("size-262144") // 256 KiB chunks cfg.Import.UnixFSFileMaxLinks = *config.NewOptionalInteger(174) // max 174 per level }) node.StartDaemon() defer node.StopDaemon() // Add 174MiB file: // 1024 * 256KiB should fit in single layer seed := shortString cidStr := node.IPFSAddDeterministic("262144KiB", seed, "--max-file-links", "1024") root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) // Expect 1024 links due to cli parameter raising link limit from 174 to 1024 require.Equal(t, 1024, len(root.Links)) // expect same CID every time require.Equal(t, "QmbBftNHWmjSWKLC49dMVrfnY8pjrJYntiAXirFJ7oJrNk", cidStr) }) // Profile-specific threshold tests are in cid_profiles_test.go (TestCIDProfiles). // Tests here cover general ipfs add behavior not tied to specific profiles. t.Run("ipfs add --hidden", func(t *testing.T) { t.Parallel() // Helper to create test directory with hidden file setupTestDir := func(t *testing.T, node *harness.Node) string { testDir, err := os.MkdirTemp(node.Dir, "hidden-test") require.NoError(t, err) require.NoError(t, os.WriteFile(filepath.Join(testDir, "visible.txt"), []byte("visible"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(testDir, ".hidden"), []byte("hidden"), 0o644)) return testDir } t.Run("default excludes hidden files", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := setupTestDir(t, node) cidStr := node.IPFS("add", "-r", "-Q", testDir).Stdout.Trimmed() lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed() require.Contains(t, lsOutput, "visible.txt") require.NotContains(t, lsOutput, ".hidden") }) t.Run("--hidden includes hidden files", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := setupTestDir(t, node) cidStr := node.IPFS("add", "-r", "-Q", "--hidden", testDir).Stdout.Trimmed() lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed() require.Contains(t, lsOutput, "visible.txt") require.Contains(t, lsOutput, ".hidden") }) t.Run("-H includes hidden files", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := setupTestDir(t, node) cidStr := node.IPFS("add", "-r", "-Q", "-H", testDir).Stdout.Trimmed() lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed() require.Contains(t, lsOutput, "visible.txt") require.Contains(t, lsOutput, ".hidden") }) }) t.Run("ipfs add --empty-dirs", func(t *testing.T) { t.Parallel() // Helper to create test directory with empty subdirectory setupTestDir := func(t *testing.T, node *harness.Node) string { testDir, err := os.MkdirTemp(node.Dir, "empty-dirs-test") require.NoError(t, err) require.NoError(t, os.Mkdir(filepath.Join(testDir, "empty-subdir"), 0o755)) require.NoError(t, os.WriteFile(filepath.Join(testDir, "file.txt"), []byte("content"), 0o644)) return testDir } t.Run("default includes empty directories", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := setupTestDir(t, node) cidStr := node.IPFS("add", "-r", "-Q", testDir).Stdout.Trimmed() require.Contains(t, node.IPFS("ls", cidStr).Stdout.Trimmed(), "empty-subdir") }) t.Run("--empty-dirs=true includes empty directories", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := setupTestDir(t, node) cidStr := node.IPFS("add", "-r", "-Q", "--empty-dirs=true", testDir).Stdout.Trimmed() require.Contains(t, node.IPFS("ls", cidStr).Stdout.Trimmed(), "empty-subdir") }) t.Run("--empty-dirs=false excludes empty directories", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := setupTestDir(t, node) cidStr := node.IPFS("add", "-r", "-Q", "--empty-dirs=false", testDir).Stdout.Trimmed() lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed() require.NotContains(t, lsOutput, "empty-subdir") require.Contains(t, lsOutput, "file.txt") }) }) t.Run("ipfs add symlink handling", func(t *testing.T) { t.Parallel() // Helper to create test directory structure: // testDir/ // target.txt (file with "target content") // link.txt -> target.txt (symlink at top level) // subdir/ // subsubdir/ // nested-target.txt (file with "nested content") // nested-link.txt -> nested-target.txt (symlink in sub-sub directory) setupTestDir := func(t *testing.T, node *harness.Node) string { testDir, err := os.MkdirTemp(node.Dir, "deref-symlinks-test") require.NoError(t, err) // Top-level file and symlink targetFile := filepath.Join(testDir, "target.txt") require.NoError(t, os.WriteFile(targetFile, []byte("target content"), 0o644)) require.NoError(t, os.Symlink("target.txt", filepath.Join(testDir, "link.txt"))) // Nested file and symlink in sub-sub directory subsubdir := filepath.Join(testDir, "subdir", "subsubdir") require.NoError(t, os.MkdirAll(subsubdir, 0o755)) nestedTarget := filepath.Join(subsubdir, "nested-target.txt") require.NoError(t, os.WriteFile(nestedTarget, []byte("nested content"), 0o644)) require.NoError(t, os.Symlink("nested-target.txt", filepath.Join(subsubdir, "nested-link.txt"))) return testDir } t.Run("default preserves symlinks", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := setupTestDir(t, node) // Add directory with symlink (default: preserve) dirCID := node.IPFS("add", "-r", "-Q", testDir).Stdout.Trimmed() // Get and verify symlinks are preserved outDir, err := os.MkdirTemp(node.Dir, "symlink-get-out") require.NoError(t, err) node.IPFS("get", "-o", outDir, dirCID) // Check top-level symlink is preserved linkPath := filepath.Join(outDir, "link.txt") fi, err := os.Lstat(linkPath) require.NoError(t, err) require.True(t, fi.Mode()&os.ModeSymlink != 0, "link.txt should be a symlink") target, err := os.Readlink(linkPath) require.NoError(t, err) require.Equal(t, "target.txt", target) // Check nested symlink is preserved nestedLinkPath := filepath.Join(outDir, "subdir", "subsubdir", "nested-link.txt") fi, err = os.Lstat(nestedLinkPath) require.NoError(t, err) require.True(t, fi.Mode()&os.ModeSymlink != 0, "nested-link.txt should be a symlink") }) // --dereference-args is deprecated but still works for backwards compatibility. // It only resolves symlinks passed as CLI arguments, NOT symlinks found // during directory traversal. Use --dereference-symlinks instead. t.Run("--dereference-args resolves CLI args only", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := setupTestDir(t, node) symlinkPath := filepath.Join(testDir, "link.txt") targetPath := filepath.Join(testDir, "target.txt") symlinkCID := node.IPFS("add", "-Q", "--dereference-args", symlinkPath).Stdout.Trimmed() targetCID := node.IPFS("add", "-Q", targetPath).Stdout.Trimmed() // CIDs should match because --dereference-args resolves the symlink require.Equal(t, targetCID, symlinkCID, "--dereference-args should resolve CLI arg symlink to target content") // Now add the directory recursively with --dereference-args // Nested symlinks should NOT be resolved (only CLI args are resolved) dirCID := node.IPFS("add", "-r", "-Q", "--dereference-args", testDir).Stdout.Trimmed() outDir, err := os.MkdirTemp(node.Dir, "deref-args-out") require.NoError(t, err) node.IPFS("get", "-o", outDir, dirCID) // Nested symlink should still be a symlink (not dereferenced) nestedLinkPath := filepath.Join(outDir, "subdir", "subsubdir", "nested-link.txt") fi, err := os.Lstat(nestedLinkPath) require.NoError(t, err) require.True(t, fi.Mode()&os.ModeSymlink != 0, "--dereference-args should NOT resolve nested symlinks, only CLI args") }) // --dereference-symlinks resolves ALL symlinks: both CLI arguments AND // symlinks found during directory traversal. This is a superset of // the deprecated --dereference-args behavior. t.Run("--dereference-symlinks resolves all symlinks", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := setupTestDir(t, node) symlinkPath := filepath.Join(testDir, "link.txt") targetPath := filepath.Join(testDir, "target.txt") symlinkCID := node.IPFS("add", "-Q", "--dereference-symlinks", symlinkPath).Stdout.Trimmed() targetCID := node.IPFS("add", "-Q", targetPath).Stdout.Trimmed() require.Equal(t, targetCID, symlinkCID, "--dereference-symlinks should resolve CLI arg symlink (like --dereference-args)") // Test 2: Nested symlinks in sub-sub directory are ALSO resolved dirCID := node.IPFS("add", "-r", "-Q", "--dereference-symlinks", testDir).Stdout.Trimmed() outDir, err := os.MkdirTemp(node.Dir, "deref-symlinks-out") require.NoError(t, err) node.IPFS("get", "-o", outDir, dirCID) // Top-level symlink should be dereferenced to regular file linkPath := filepath.Join(outDir, "link.txt") fi, err := os.Lstat(linkPath) require.NoError(t, err) require.False(t, fi.Mode()&os.ModeSymlink != 0, "link.txt should be dereferenced to regular file") content, err := os.ReadFile(linkPath) require.NoError(t, err) require.Equal(t, "target content", string(content)) // Nested symlink in sub-sub directory should ALSO be dereferenced nestedLinkPath := filepath.Join(outDir, "subdir", "subsubdir", "nested-link.txt") fi, err = os.Lstat(nestedLinkPath) require.NoError(t, err) require.False(t, fi.Mode()&os.ModeSymlink != 0, "nested-link.txt should be dereferenced (--dereference-symlinks resolves ALL symlinks)") nestedContent, err := os.ReadFile(nestedLinkPath) require.NoError(t, err) require.Equal(t, "nested content", string(nestedContent)) }) }) } func TestAddFastProvide(t *testing.T) { t.Parallel() const ( shortString = "hello world" shortStringCidV0 = "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD" // cidv0 - dag-pb - sha2-256 ) t.Run("fast-provide-root disabled via config: verify skipped in logs", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.FastProvideRoot = config.False }) // Start daemon with debug logging node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithEnv(map[string]string{ "GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug", }), }, }, "") defer node.StopDaemon() cidStr := node.IPFSAddStr(shortString) require.Equal(t, shortStringCidV0, cidStr) // Verify fast-provide-root was disabled daemonLog := node.Daemon.Stderr.String() require.Contains(t, daemonLog, "fast-provide-root: skipped") }) t.Run("fast-provide-root enabled with wait=false: verify async provide", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Use default config (FastProvideRoot=true, FastProvideWait=false) node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithEnv(map[string]string{ "GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug", }), }, }, "") defer node.StopDaemon() cidStr := node.IPFSAddStr(shortString) require.Equal(t, shortStringCidV0, cidStr) daemonLog := node.Daemon.Stderr // Should see async mode started require.Contains(t, daemonLog.String(), "fast-provide-root: enabled") require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously") // Wait for async completion or failure (up to 11 seconds - slightly more than fastProvideTimeout) // In test environment with no DHT peers, this will fail with "failed to find any peer in table" completedOrFailed := waitForLogMessage(daemonLog, "async provide completed", 11*time.Second) || waitForLogMessage(daemonLog, "async provide failed", 11*time.Second) require.True(t, completedOrFailed, "async provide should complete or fail within timeout") }) t.Run("fast-provide-root enabled with wait=true: verify sync provide", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.FastProvideWait = config.True }) node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithEnv(map[string]string{ "GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug", }), }, }, "") defer node.StopDaemon() // Use Runner.Run with stdin to allow for expected errors res := node.Runner.Run(harness.RunRequest{ Path: node.IPFSBin, Args: []string{"add", "-q"}, CmdOpts: []harness.CmdOpt{ harness.RunWithStdin(strings.NewReader(shortString)), }, }) // In sync mode (wait=true), provide errors propagate and fail the command. // Test environment uses 'test' profile with no bootstrappers, and CI has // insufficient peers for proper DHT puts, so we expect this to fail with // "failed to find any peer in table" error from the DHT. require.Equal(t, 1, res.ExitCode()) require.Contains(t, res.Stderr.String(), "Error: fast-provide: failed to find any peer in table") daemonLog := node.Daemon.Stderr.String() // Should see sync mode started require.Contains(t, daemonLog, "fast-provide-root: enabled") require.Contains(t, daemonLog, "fast-provide-root: providing synchronously") require.Contains(t, daemonLog, "sync provide failed") // Verify the failure was logged }) t.Run("fast-provide-wait ignored when root disabled", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.FastProvideRoot = config.False cfg.Import.FastProvideWait = config.True }) node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithEnv(map[string]string{ "GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug", }), }, }, "") defer node.StopDaemon() cidStr := node.IPFSAddStr(shortString) require.Equal(t, shortStringCidV0, cidStr) daemonLog := node.Daemon.Stderr.String() require.Contains(t, daemonLog, "fast-provide-root: skipped") require.Contains(t, daemonLog, "wait-flag-ignored") }) t.Run("CLI flag overrides config: flag=true overrides config=false", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.FastProvideRoot = config.False }) node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithEnv(map[string]string{ "GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug", }), }, }, "") defer node.StopDaemon() cidStr := node.IPFSAddStr(shortString, "--fast-provide-root=true") require.Equal(t, shortStringCidV0, cidStr) daemonLog := node.Daemon.Stderr // Flag should enable it despite config saying false require.Contains(t, daemonLog.String(), "fast-provide-root: enabled") require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously") }) t.Run("CLI flag overrides config: flag=false overrides config=true", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.FastProvideRoot = config.True }) node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithEnv(map[string]string{ "GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug", }), }, }, "") defer node.StopDaemon() cidStr := node.IPFSAddStr(shortString, "--fast-provide-root=false") require.Equal(t, shortStringCidV0, cidStr) daemonLog := node.Daemon.Stderr.String() // Flag should disable it despite config saying true require.Contains(t, daemonLog, "fast-provide-root: skipped") }) } // createDirectoryForHAMTLinksEstimation creates a directory with the specified number // of files for testing links-based size estimation (size = sum of nameLen + cidLen). // Used by legacy profiles (unixfs-v0-2015). // // The lastNameLen parameter allows the last file to have a different name length, // enabling exact +1 byte threshold tests. func createDirectoryForHAMTLinksEstimation(dirPath string, numFiles, nameLen, lastNameLen int, seed string) error { return createDeterministicFiles(dirPath, numFiles, nameLen, lastNameLen, seed) } // createDirectoryForHAMTBlockEstimation creates a directory with the specified number // of files for testing block-based size estimation (LinkSerializedSize with protobuf overhead). // Used by modern profiles (unixfs-v1-2025). // // The lastNameLen parameter allows the last file to have a different name length, // enabling exact +1 byte threshold tests. func createDirectoryForHAMTBlockEstimation(dirPath string, numFiles, nameLen, lastNameLen int, seed string) error { return createDeterministicFiles(dirPath, numFiles, nameLen, lastNameLen, seed) } // createDeterministicFiles creates numFiles files with deterministic names. // Files 0 to numFiles-2 have nameLen characters, and the last file has lastNameLen characters. // Each file contains "x" (1 byte) for non-zero tsize in directory links. func createDeterministicFiles(dirPath string, numFiles, nameLen, lastNameLen int, seed string) error { alphabetLen := len(testutils.AlphabetEasy) // Deterministic pseudo-random bytes for static filenames drand, err := testutils.DeterministicRandomReader("1MiB", seed) if err != nil { return err } for i := range numFiles { // Use lastNameLen for the final file currentNameLen := nameLen if i == numFiles-1 { currentNameLen = lastNameLen } buf := make([]byte, currentNameLen) _, err := io.ReadFull(drand, buf) if err != nil { return err } // Convert deterministic pseudo-random bytes to ASCII var sb strings.Builder for _, b := range buf { char := testutils.AlphabetEasy[int(b)%alphabetLen] sb.WriteRune(char) } filename := sb.String()[:currentNameLen] filePath := filepath.Join(dirPath, filename) // Create file with 1-byte content for non-zero tsize if err := os.WriteFile(filePath, []byte("x"), 0o644); err != nil { return err } } return nil } ================================================ FILE: test/cli/agent_version_unicode_test.go ================================================ package cli import ( "strings" "testing" "github.com/ipfs/kubo/core/commands/cmdutils" "github.com/stretchr/testify/assert" ) func TestCleanAndTrimUnicode(t *testing.T) { tests := []struct { name string input string expected string }{ { name: "Basic ASCII", input: "kubo/1.0.0", expected: "kubo/1.0.0", }, { name: "Polish characters preserved", input: "test-ąęćłńóśźż", expected: "test-ąęćłńóśźż", }, { name: "Chinese characters preserved", input: "版本-中文测试", expected: "版本-中文测试", }, { name: "Arabic text preserved", input: "اختبار-العربية", expected: "اختبار-العربية", }, { name: "Emojis preserved", input: "version-1.0-🚀-🎉", expected: "version-1.0-🚀-🎉", }, { name: "Complex Unicode with combining marks preserved", input: "h̸̢̢̢̢̢̢̢̢̢̢e̵̵̵̵̵̵̵̵̵̵l̷̷̷̷̷̷̷̷̷̷l̶̶̶̶̶̶̶̶̶̶o̴̴̴̴̴̴̴̴̴̴", expected: "h̸̢̢̢̢̢̢̢̢̢̢e̵̵̵̵̵̵̵̵̵̵l̷̷̷̷̷̷̷̷̷̷l̶̶̶̶̶̶̶̶̶̶o̴̴̴̴̴̴̴̴̴̴", // Preserved as-is (only 50 runes) }, { name: "Long text with combining marks truncated at 128", input: strings.Repeat("ẽ̸̢̛̖̬͈͉͖͇͈̭̥́̓̌̾͊̊̂̄̍̅̂͌́", 10), // Very long text (260 runes) expected: "ẽ̸̢̛̖̬͈͉͖͇͈̭̥́̓̌̾͊̊̂̄̍̅̂͌́ẽ̸̢̛̖̬͈͉͖͇͈̭̥́̓̌̾͊̊̂̄̍̅̂͌́ẽ̸̢̛̖̬͈͉͖͇͈̭̥́̓̌̾͊̊̂̄̍̅̂͌́ẽ̸̢̛̖̬͈͉͖͇͈̭̥́̓̌̾͊̊̂̄̍̅̂͌́ẽ̸̢̛̖̬͈͉͖͇͈̭̥́̓̌̾͊̊̂̄̍̅̂", // Truncated at 128 runes }, { name: "Zero-width characters replaced with U+FFFD", input: "test\u200Bzero\u200Cwidth\u200D\uFEFFchars", expected: "test�zero�width��chars", }, { name: "RTL/LTR override replaced with U+FFFD", input: "test\u202Drtl\u202Eltr\u202Aoverride", expected: "test�rtl�ltr�override", }, { name: "Bidi isolates replaced with U+FFFD", input: "test\u2066bidi\u2067isolate\u2068text\u2069end", expected: "test�bidi�isolate�text�end", }, { name: "Control characters replaced with U+FFFD", input: "test\x00null\x1Fescape\x7Fdelete", expected: "test�null�escape�delete", }, { name: "Combining marks preserved", input: "e\u0301\u0302\u0303\u0304\u0305", // e with 5 combining marks expected: "e\u0301\u0302\u0303\u0304\u0305", // All preserved }, { name: "No truncation at 70 characters", input: "123456789012345678901234567890123456789012345678901234567890123456789", expected: "123456789012345678901234567890123456789012345678901234567890123456789", }, { name: "No truncation with Unicode - 70 rockets preserved", input: strings.Repeat("🚀", 70), expected: strings.Repeat("🚀", 70), }, { name: "Empty string", input: "", expected: "", }, { name: "Only whitespace with control chars", input: " \t\n ", expected: "\uFFFD\uFFFD", // Tab and newline become U+FFFD, spaces trimmed }, { name: "Leading and trailing whitespace", input: " test ", expected: "test", }, { name: "Complex mix - invisible chars replaced with U+FFFD, Unicode preserved", input: "kubo/1.0-🚀\u200B h̸̢̏̔ḛ̶̽̀s̵t\u202E-ąęł-中文", expected: "kubo/1.0-🚀� h̸̢̏̔ḛ̶̽̀s̵t�-ąęł-中文", }, { name: "Emoji with skin tone preserved", input: "👍🏽", // Thumbs up with skin tone modifier expected: "👍🏽", // Preserved as-is }, { name: "Mixed scripts preserved", input: "Hello-你好-مرحبا-Здравствуйте", expected: "Hello-你好-مرحبا-Здравствуйте", }, { name: "Format characters replaced with U+FFFD", input: "test\u00ADsoft\u2060word\u206Fnom\u200Ebreak", expected: "test�soft�word�nom�break", // Soft hyphen, word joiner, etc replaced }, { name: "Complex Unicode text with many combining marks (91 runes, no truncation)", input: "ț̸̢͙̞̖̏̔ȩ̶̰͓̪͎̱̠̥̳͔̽̀̃̿̌̾̀͗̕̕͜s̵̢̛̖̬͈͉͖͇͈̭̥̃́̓̌̾͊̊̂̄̍̅̂͌́ͅţ̴̯̹̪͖͓̘̊́̑̄̋̈́͐̈́̔̇̄̂́̎̓͛͠ͅ test", expected: "ț̸̢͙̞̖̏̔ȩ̶̰͓̪͎̱̠̥̳͔̽̀̃̿̌̾̀͗̕̕͜s̵̢̛̖̬͈͉͖͇͈̭̥̃́̓̌̾͊̊̂̄̍̅̂͌́ͅţ̴̯̹̪͖͓̘̊́̑̄̋̈́͐̈́̔̇̄̂́̎̓͛͠ͅ test", // Not truncated (91 < 128) }, { name: "Truncation at 128 characters", input: strings.Repeat("a", 150), expected: strings.Repeat("a", 128), }, { name: "Truncation with Unicode at 128", input: strings.Repeat("🚀", 150), expected: strings.Repeat("🚀", 128), }, { name: "Private use characters preserved (per spec)", input: "test\uE000\uF8FF", // Private use area characters expected: "test\uE000\uF8FF", // Should be preserved }, { name: "U+FFFD replacement for multiple categories", input: "a\x00b\u200Cc\u202Ed", // control, format chars expected: "a\uFFFDb\uFFFDc\uFFFDd", // All replaced with U+FFFD }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := cmdutils.CleanAndTrim(tt.input) assert.Equal(t, tt.expected, result, "CleanAndTrim(%q) = %q, want %q", tt.input, result, tt.expected) }) } } func TestCleanAndTrimIdempotent(t *testing.T) { // Test that applying CleanAndTrim twice gives the same result inputs := []string{ "test-ąęćłńóśźż", "版本-中文测试", "version-1.0-🚀-🎉", "h̸e̵l̷l̶o̴ w̸o̵r̷l̶d̴", "test\u200Bzero\u200Cwidth", } for _, input := range inputs { once := cmdutils.CleanAndTrim(input) twice := cmdutils.CleanAndTrim(once) assert.Equal(t, once, twice, "CleanAndTrim should be idempotent for %q", input) } } func TestCleanAndTrimSecurity(t *testing.T) { // Test that all invisible/dangerous characters are removed tests := []struct { name string input string check func(string) bool }{ { name: "No zero-width spaces", input: "test\u200B\u200C\u200Dtest", check: func(s string) bool { return !strings.Contains(s, "\u200B") && !strings.Contains(s, "\u200C") && !strings.Contains(s, "\u200D") }, }, { name: "No bidi overrides", input: "test\u202A\u202B\u202C\u202D\u202Etest", check: func(s string) bool { for _, r := range []rune{0x202A, 0x202B, 0x202C, 0x202D, 0x202E} { if strings.ContainsRune(s, r) { return false } } return true }, }, { name: "No control characters", input: "test\x00\x01\x02\x1F\x7Ftest", check: func(s string) bool { for _, r := range s { if r < 0x20 || r == 0x7F { return false } } return true }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := cmdutils.CleanAndTrim(tt.input) assert.True(t, tt.check(result), "Security check failed for %q -> %q", tt.input, result) }) } } ================================================ FILE: test/cli/api_file_test.go ================================================ package cli import ( "net/http" "os" "os/exec" "path/filepath" "strings" "testing" "time" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/require" ) // TestAddressFileReady verifies that when address files ($IPFS_PATH/api and // $IPFS_PATH/gateway) are created, the corresponding HTTP servers are ready // to accept connections immediately. This prevents race conditions for tools // like systemd path units that start services when these files appear. func TestAddressFileReady(t *testing.T) { t.Parallel() t.Run("api file", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() // Start daemon in background (don't use StartDaemon which waits for API) res := node.Runner.MustRun(harness.RunRequest{ Path: node.IPFSBin, Args: []string{"daemon"}, RunFunc: (*exec.Cmd).Start, }) node.Daemon = res defer node.StopDaemon() // Poll for api file to appear apiFile := filepath.Join(node.Dir, "api") var fileExists bool for range 100 { if _, err := os.Stat(apiFile); err == nil { fileExists = true break } time.Sleep(100 * time.Millisecond) } require.True(t, fileExists, "api file should be created") // Read the api file to get the address apiAddr, err := node.TryAPIAddr() require.NoError(t, err) // Extract IP and port from multiaddr ip, err := apiAddr.ValueForProtocol(4) // P_IP4 require.NoError(t, err) port, err := apiAddr.ValueForProtocol(6) // P_TCP require.NoError(t, err) // Immediately try to use the API - should work on first attempt url := "http://" + ip + ":" + port + "/api/v0/id" resp, err := http.Post(url, "", nil) require.NoError(t, err, "RPC API should be ready immediately when api file exists") defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) }) t.Run("gateway file", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() // Start daemon in background res := node.Runner.MustRun(harness.RunRequest{ Path: node.IPFSBin, Args: []string{"daemon"}, RunFunc: (*exec.Cmd).Start, }) node.Daemon = res defer node.StopDaemon() // Poll for gateway file to appear gatewayFile := filepath.Join(node.Dir, "gateway") var fileExists bool for range 100 { if _, err := os.Stat(gatewayFile); err == nil { fileExists = true break } time.Sleep(100 * time.Millisecond) } require.True(t, fileExists, "gateway file should be created") // Read the gateway file to get the URL (already includes http:// prefix) gatewayURL, err := os.ReadFile(gatewayFile) require.NoError(t, err) // Immediately try to use the Gateway - should work on first attempt url := strings.TrimSpace(string(gatewayURL)) + "/ipfs/bafkqaaa" // empty file CID resp, err := http.Get(url) require.NoError(t, err, "Gateway should be ready immediately when gateway file exists") defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) }) } ================================================ FILE: test/cli/autoconf/autoconf_test.go ================================================ package autoconf import ( "encoding/json" "fmt" "net/http" "net/http/httptest" "os" "strings" "sync/atomic" "testing" "time" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestAutoConf(t *testing.T) { t.Parallel() t.Run("basic functionality", func(t *testing.T) { t.Parallel() testAutoConfBasicFunctionality(t) }) t.Run("background service updates", func(t *testing.T) { t.Parallel() testAutoConfBackgroundService(t) }) t.Run("HTTP error scenarios", func(t *testing.T) { t.Parallel() testAutoConfHTTPErrors(t) }) t.Run("cache-based config expansion", func(t *testing.T) { t.Parallel() testAutoConfCacheBasedExpansion(t) }) t.Run("disabled autoconf", func(t *testing.T) { t.Parallel() testAutoConfDisabled(t) }) t.Run("bootstrap list shows auto as-is", func(t *testing.T) { t.Parallel() testBootstrapListResolved(t) }) t.Run("daemon uses resolved bootstrap values", func(t *testing.T) { t.Parallel() testDaemonUsesResolvedBootstrap(t) }) t.Run("empty cache uses fallback defaults", func(t *testing.T) { t.Parallel() testEmptyCacheUsesFallbacks(t) }) t.Run("stale cache with unreachable server", func(t *testing.T) { t.Parallel() testStaleCacheWithUnreachableServer(t) }) t.Run("autoconf disabled with auto values", func(t *testing.T) { t.Parallel() testAutoConfDisabledWithAutoValues(t) }) t.Run("network behavior - cached vs refresh", func(t *testing.T) { t.Parallel() testAutoConfNetworkBehavior(t) }) t.Run("HTTPS autoconf server", func(t *testing.T) { t.Parallel() testAutoConfWithHTTPS(t) }) } func testAutoConfBasicFunctionality(t *testing.T) { // Load test autoconf data autoConfData := loadTestData(t, "valid_autoconf.json") // Create HTTP server that serves autoconf.json etag := `"test-etag-123"` requestCount := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requestCount++ t.Logf("AutoConf server request #%d: %s %s", requestCount, r.Method, r.URL.Path) w.Header().Set("Content-Type", "application/json") w.Header().Set("ETag", etag) w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") _, _ = w.Write(autoConfData) })) defer server.Close() // Create IPFS node and configure it to use our test server // Use test profile to avoid autoconf profile being applied by default node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) // Disable background updates to prevent multiple requests node.SetIPFSConfig("AutoConf.RefreshInterval", "24h") // Test with normal bootstrap peers (not "auto") to avoid multiaddr parsing issues // This tests that autoconf fetching works without complex auto replacement node.SetIPFSConfig("Bootstrap", []string{"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN"}) // Start daemon to trigger autoconf fetch node.StartDaemon() defer node.StopDaemon() // Give autoconf some time to fetch time.Sleep(2 * time.Second) // Verify that the autoconf system fetched data from our server t.Logf("Server request count: %d", requestCount) require.GreaterOrEqual(t, requestCount, 1, "AutoConf server should have been called at least once") // Test that daemon is functional result := node.RunIPFS("id") assert.Equal(t, 0, result.ExitCode(), "IPFS daemon should be responsive") assert.Contains(t, result.Stdout.String(), "ID", "IPFS id command should return peer information") // Success! AutoConf system is working: // 1. Server was called (proves fetch works) // 2. Daemon started successfully (proves DNS resolver validation is fixed) // 3. Daemon is functional (proves autoconf doesn't break core functionality) // Note: We skip checking metadata values due to JSON parsing complexity in test harness } func testAutoConfBackgroundService(t *testing.T) { // Test that the startAutoConfUpdater() goroutine makes network requests for background refresh // This is separate from daemon config operations which now use cache-first approach // Load initial and updated test data initialData := loadTestData(t, "valid_autoconf.json") updatedData := loadTestData(t, "updated_autoconf.json") // Track which config is being served currentData := initialData var requestCount atomic.Int32 // Create server that switches payload after first request server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { count := requestCount.Add(1) t.Logf("Background service request #%d from %s", count, r.UserAgent()) w.Header().Set("Content-Type", "application/json") w.Header().Set("ETag", fmt.Sprintf(`"background-test-etag-%d"`, count)) w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat)) if count > 1 { // After first request, serve updated config currentData = updatedData } _, _ = w.Write(currentData) })) defer server.Close() // Create IPFS node with short refresh interval to trigger background service node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("AutoConf.RefreshInterval", "1s") // Very short for testing background service // Use normal bootstrap values to avoid dependency on autoconf during initialization node.SetIPFSConfig("Bootstrap", []string{"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN"}) // Start daemon - this should start the background service via startAutoConfUpdater() node.StartDaemon() defer node.StopDaemon() // Wait for initial request (daemon startup may trigger one) time.Sleep(1 * time.Second) initialCount := requestCount.Load() t.Logf("Initial request count after daemon start: %d", initialCount) // Wait for background service to make additional requests // The background service should make requests at the RefreshInterval (1s) time.Sleep(3 * time.Second) finalCount := requestCount.Load() t.Logf("Final request count after background updates: %d", finalCount) // Background service should have made multiple requests due to 1s refresh interval assert.Greater(t, finalCount, initialCount, "Background service should have made additional requests beyond daemon startup") // Verify that the service is actively making requests (not just relying on cache) assert.GreaterOrEqual(t, finalCount, int32(2), "Should have at least 2 requests total (startup + background refresh)") t.Logf("Successfully verified startAutoConfUpdater() background service makes network requests") } func testAutoConfHTTPErrors(t *testing.T) { tests := []struct { name string statusCode int body string }{ {"404 Not Found", http.StatusNotFound, "Not Found"}, {"500 Internal Server Error", http.StatusInternalServerError, "Internal Server Error"}, {"Invalid JSON", http.StatusOK, "invalid json content"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Create server that returns error server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(tt.statusCode) _, _ = w.Write([]byte(tt.body)) })) defer server.Close() // Create node with failing AutoConf URL // Use test profile to avoid autoconf profile being applied by default node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Start daemon - it should start but autoconf should fail gracefully node.StartDaemon() defer node.StopDaemon() // Daemon should still be functional even with autoconf HTTP errors result := node.RunIPFS("version") assert.Equal(t, 0, result.ExitCode(), "Daemon should start even with HTTP errors in autoconf") }) } } func testAutoConfCacheBasedExpansion(t *testing.T) { // Test that config expansion works correctly with cached autoconf data // without requiring active network requests during expansion operations autoConfData := loadTestData(t, "valid_autoconf.json") // Create server that serves autoconf data server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.Header().Set("ETag", `"cache-test-etag"`) w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") _, _ = w.Write(autoConfData) })) defer server.Close() // Create IPFS node with autoconf enabled node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) // Set configuration with "auto" values to test expansion node.SetIPFSConfig("Bootstrap", []string{"auto"}) node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"}) node.SetIPFSConfig("DNS.Resolvers", map[string]string{"test.": "auto"}) // Populate cache by running a command that triggers autoconf (without daemon) result := node.RunIPFS("bootstrap", "list", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "Initial bootstrap expansion should succeed") expandedBootstrap := result.Stdout.String() assert.NotContains(t, expandedBootstrap, "auto", "Expanded bootstrap should not contain 'auto' literal") assert.Greater(t, len(strings.Fields(expandedBootstrap)), 0, "Should have expanded bootstrap peers") // Test that subsequent config operations work with cached data (no network required) // This simulates the cache-first behavior our architecture now uses // Test Bootstrap expansion result = node.RunIPFS("config", "Bootstrap", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "Cached bootstrap expansion should succeed") var expandedBootstrapList []string err := json.Unmarshal([]byte(result.Stdout.String()), &expandedBootstrapList) require.NoError(t, err) assert.NotContains(t, expandedBootstrapList, "auto", "Expanded bootstrap list should not contain 'auto'") assert.Greater(t, len(expandedBootstrapList), 0, "Should have expanded bootstrap peers from cache") // Test Routing.DelegatedRouters expansion result = node.RunIPFS("config", "Routing.DelegatedRouters", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "Cached router expansion should succeed") var expandedRouters []string err = json.Unmarshal([]byte(result.Stdout.String()), &expandedRouters) require.NoError(t, err) assert.NotContains(t, expandedRouters, "auto", "Expanded routers should not contain 'auto'") // Test DNS.Resolvers expansion result = node.RunIPFS("config", "DNS.Resolvers", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "Cached DNS resolver expansion should succeed") var expandedResolvers map[string]string err = json.Unmarshal([]byte(result.Stdout.String()), &expandedResolvers) require.NoError(t, err) // Should have expanded the "auto" value for test. domain, or removed it if no autoconf data available testResolver, exists := expandedResolvers["test."] if exists { assert.NotEqual(t, "auto", testResolver, "test. resolver should not be literal 'auto'") t.Logf("Found expanded resolver for test.: %s", testResolver) } else { t.Logf("No resolver found for test. domain (autoconf may not have DNS resolver data)") } // Test full config expansion result = node.RunIPFS("config", "show", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "Full config expansion should succeed") expandedConfig := result.Stdout.String() // Should not contain literal "auto" values after expansion assert.NotContains(t, expandedConfig, `"auto"`, "Expanded config should not contain literal 'auto' values") assert.Contains(t, expandedConfig, `"Bootstrap"`, "Should contain Bootstrap section") assert.Contains(t, expandedConfig, `"DNS"`, "Should contain DNS section") t.Logf("Successfully tested cache-based config expansion without active network requests") } func testAutoConfDisabled(t *testing.T) { // Create node with AutoConf disabled but "auto" values // Use test profile to avoid autoconf profile being applied by default node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.Enabled", false) node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Test by trying to list bootstrap - when AutoConf is disabled, it should show literal "auto" result := node.RunIPFS("bootstrap", "list") if result.ExitCode() == 0 { // If command succeeds, it should show literal "auto" (no resolution) output := result.Stdout.String() assert.Contains(t, output, "auto", "Should show literal 'auto' when AutoConf is disabled") } else { // If command fails, error should mention autoconf issue assert.Contains(t, result.Stderr.String(), "auto", "Should mention 'auto' values in error") } } // Helper function to load test data files func loadTestData(t *testing.T, filename string) []byte { t.Helper() data, err := os.ReadFile("testdata/" + filename) require.NoError(t, err, "Failed to read test data file: %s", filename) return data } func testBootstrapListResolved(t *testing.T) { // Test that bootstrap list shows "auto" as-is (not expanded) // Load test autoconf data autoConfData := loadTestData(t, "valid_autoconf.json") // Create HTTP server that serves autoconf.json server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(autoConfData) })) defer server.Close() // Create IPFS node with "auto" bootstrap value node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Test 1: bootstrap list (without --expand-auto) shows "auto" as-is - NO DAEMON NEEDED! result := node.RunIPFS("bootstrap", "list") require.Equal(t, 0, result.ExitCode(), "bootstrap list command should succeed") output := result.Stdout.String() t.Logf("Bootstrap list output: %s", output) assert.Contains(t, output, "auto", "bootstrap list should show 'auto' value as-is") // Should NOT contain expanded bootstrap peers without --expand-auto unexpectedPeers := []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", } for _, peer := range unexpectedPeers { assert.NotContains(t, output, peer, "bootstrap list should not contain expanded peer: %s", peer) } // Test 2: bootstrap list --expand-auto shows expanded values (no daemon needed!) result = node.RunIPFS("bootstrap", "list", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "bootstrap list --expand-auto command should succeed") expandedOutput := result.Stdout.String() t.Logf("Bootstrap list --expand-auto output: %s", expandedOutput) // Should NOT contain "auto" literal when expanded assert.NotContains(t, expandedOutput, "auto", "bootstrap list --expand-auto should not show 'auto' literal") // Should contain at least one expanded bootstrap peer expectedPeers := []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", } foundExpectedPeer := false for _, peer := range expectedPeers { if strings.Contains(expandedOutput, peer) { foundExpectedPeer = true t.Logf("Found expected expanded peer: %s", peer) break } } assert.True(t, foundExpectedPeer, "bootstrap list --expand-auto should contain at least one expanded bootstrap peer") } func testDaemonUsesResolvedBootstrap(t *testing.T) { // Test that daemon actually uses expanded bootstrap values for P2P connections // even though bootstrap list shows "auto" // Step 1: Create bootstrap node (target for connections) bootstrapNode := harness.NewT(t).NewNode().Init("--profile=test") // Set a specific swarm port for the bootstrap node to avoid port 0 issues bootstrapNode.SetIPFSConfig("Addresses.Swarm", []string{"/ip4/127.0.0.1/tcp/14001"}) // Disable routing and discovery to ensure it's only discoverable via explicit multiaddr bootstrapNode.SetIPFSConfig("Routing.Type", "none") bootstrapNode.SetIPFSConfig("Discovery.MDNS.Enabled", false) bootstrapNode.SetIPFSConfig("Bootstrap", []string{}) // No bootstrap peers // Start the bootstrap node first bootstrapNode.StartDaemon() defer bootstrapNode.StopDaemon() // Get bootstrap node's peer ID and swarm address bootstrapPeerID := bootstrapNode.PeerID() // Use the configured swarm address (we set it to a specific port above) bootstrapMultiaddr := fmt.Sprintf("/ip4/127.0.0.1/tcp/14001/p2p/%s", bootstrapPeerID.String()) t.Logf("Bootstrap node configured at: %s", bootstrapMultiaddr) // Step 2: Create autoconf server that returns bootstrap node's address autoConfData := fmt.Sprintf(`{ "AutoConfVersion": 2025072301, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": { "AminoDHT": { "Description": "Test AminoDHT system", "NativeConfig": { "Bootstrap": ["%s"] } } }, "DNSResolvers": {}, "DelegatedEndpoints": {} }`, bootstrapMultiaddr) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(autoConfData)) })) defer server.Close() // Step 3: Create autoconf-enabled node that should connect to bootstrap node autoconfNode := harness.NewT(t).NewNode().Init("--profile=test") autoconfNode.SetIPFSConfig("AutoConf.URL", server.URL) autoconfNode.SetIPFSConfig("AutoConf.Enabled", true) autoconfNode.SetIPFSConfig("Bootstrap", []string{"auto"}) // This should resolve to bootstrap node // Disable other discovery methods to force bootstrap-only connectivity autoconfNode.SetIPFSConfig("Routing.Type", "none") autoconfNode.SetIPFSConfig("Discovery.MDNS.Enabled", false) // Start the autoconf node autoconfNode.StartDaemon() defer autoconfNode.StopDaemon() // Step 4: Give time for autoconf resolution and connection attempts time.Sleep(8 * time.Second) // Step 5: Verify both nodes are responsive result := bootstrapNode.RunIPFS("id") require.Equal(t, 0, result.ExitCode(), "Bootstrap node should be responsive: %s", result.Stderr.String()) result = autoconfNode.RunIPFS("id") require.Equal(t, 0, result.ExitCode(), "AutoConf node should be responsive: %s", result.Stderr.String()) // Step 6: Verify that autoconf node connected to bootstrap node // Check swarm peers on autoconf node - it should show bootstrap node's peer ID result = autoconfNode.RunIPFS("swarm", "peers") if result.ExitCode() == 0 { peerOutput := result.Stdout.String() if strings.Contains(peerOutput, bootstrapPeerID.String()) { t.Logf("SUCCESS: AutoConf node connected to bootstrap peer %s", bootstrapPeerID.String()) } else { t.Logf("No active connection found. Peers output: %s", peerOutput) // This might be OK if connection attempt was made but didn't persist } } else { // If swarm peers fails, try alternative verification via daemon logs t.Logf("Swarm peers command failed, checking daemon logs for connection attempts") daemonOutput := autoconfNode.Daemon.Stderr.String() if strings.Contains(daemonOutput, bootstrapPeerID.String()) { t.Logf("SUCCESS: Found bootstrap peer %s in daemon logs, connection attempted", bootstrapPeerID.String()) } else { t.Logf("Daemon stderr: %s", daemonOutput) } } // Step 7: Verify bootstrap configuration still shows "auto" (not resolved values) result = autoconfNode.RunIPFS("bootstrap", "list") require.Equal(t, 0, result.ExitCode(), "Bootstrap list command should work") assert.Contains(t, result.Stdout.String(), "auto", "Bootstrap list should still show 'auto' even though values were resolved for networking") } func testEmptyCacheUsesFallbacks(t *testing.T) { // Test that daemon uses fallback defaults when no cache exists and server is unreachable // Create IPFS node with auto values and unreachable autoconf server node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", "http://127.0.0.1:9999/nonexistent") node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{"auto"}) node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"}) // Start daemon - should succeed using fallback values node.StartDaemon() defer node.StopDaemon() // Verify daemon started successfully (uses fallback bootstrap) result := node.RunIPFS("id") require.Equal(t, 0, result.ExitCode(), "Daemon should start successfully with fallback values") // Verify config commands still show "auto" result = node.RunIPFS("config", "Bootstrap") require.Equal(t, 0, result.ExitCode()) assert.Contains(t, result.Stdout.String(), "auto", "Bootstrap config should still show 'auto'") result = node.RunIPFS("config", "Routing.DelegatedRouters") require.Equal(t, 0, result.ExitCode()) assert.Contains(t, result.Stdout.String(), "auto", "DelegatedRouters config should still show 'auto'") // Check daemon logs for error about failed autoconf fetch logOutput := node.Daemon.Stderr.String() // The daemon should attempt to fetch autoconf but will use fallbacks on failure // We don't require specific log messages as long as the daemon starts successfully if logOutput != "" { t.Logf("Daemon logs: %s", logOutput) } } func testStaleCacheWithUnreachableServer(t *testing.T) { // Test that daemon uses stale cache when server is unreachable // First create a working autoconf server and cache autoConfData := loadTestData(t, "valid_autoconf.json") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(autoConfData) })) // Create node and fetch autoconf to populate cache node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Start daemon briefly to populate cache node.StartDaemon() time.Sleep(1 * time.Second) // Allow cache population node.StopDaemon() // Close the server to make it unreachable server.Close() // Update config to point to unreachable server node.SetIPFSConfig("AutoConf.URL", "http://127.0.0.1:9999/unreachable") // Start daemon again - should use stale cache node.StartDaemon() defer node.StopDaemon() // Verify daemon started successfully (uses cached autoconf) result := node.RunIPFS("id") require.Equal(t, 0, result.ExitCode(), "Daemon should start successfully with cached autoconf") // Check daemon logs for error about using stale config logOutput := node.Daemon.Stderr.String() // The daemon should use cached config when server is unreachable // We don't require specific log messages as long as the daemon starts successfully if logOutput != "" { t.Logf("Daemon logs: %s", logOutput) } } func testAutoConfDisabledWithAutoValues(t *testing.T) { // Test that daemon fails to start when AutoConf is disabled but "auto" values are present // Create IPFS node with AutoConf disabled but "auto" values configured node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.Enabled", false) node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Test by trying to list bootstrap - when AutoConf is disabled, it should show literal "auto" result := node.RunIPFS("bootstrap", "list") if result.ExitCode() == 0 { // If command succeeds, it should show literal "auto" (no resolution) output := result.Stdout.String() assert.Contains(t, output, "auto", "Should show literal 'auto' when AutoConf is disabled") } else { // If command fails, error should mention autoconf issue logOutput := result.Stderr.String() assert.Contains(t, logOutput, "auto", "Error should mention 'auto' values") // Check that the error message contains information about disabled state assert.True(t, strings.Contains(logOutput, "disabled") || strings.Contains(logOutput, "AutoConf.Enabled=false"), "Error should mention that AutoConf is disabled or show AutoConf.Enabled=false") } } func testAutoConfNetworkBehavior(t *testing.T) { // Test the network behavior differences between MustGetConfigCached and MustGetConfigWithRefresh // This validates that our cache-first architecture works as expected autoConfData := loadTestData(t, "valid_autoconf.json") var requestCount atomic.Int32 // Create server that tracks all requests server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { count := requestCount.Add(1) t.Logf("Network behavior test request #%d: %s %s", count, r.Method, r.URL.Path) w.Header().Set("Content-Type", "application/json") w.Header().Set("ETag", fmt.Sprintf(`"network-test-etag-%d"`, count)) w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat)) _, _ = w.Write(autoConfData) })) defer server.Close() // Create IPFS node with autoconf node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Phase 1: Test cache-first behavior (no network requests expected) t.Logf("=== Phase 1: Testing cache-first behavior ===") initialCount := requestCount.Load() // Multiple config operations should NOT trigger network requests (cache-first) result := node.RunIPFS("config", "Bootstrap") require.Equal(t, 0, result.ExitCode(), "Bootstrap config read should succeed") result = node.RunIPFS("config", "show") require.Equal(t, 0, result.ExitCode(), "Config show should succeed") result = node.RunIPFS("bootstrap", "list") require.Equal(t, 0, result.ExitCode(), "Bootstrap list should succeed") // Check that cache-first operations didn't trigger network requests afterCacheOpsCount := requestCount.Load() cachedRequestDiff := afterCacheOpsCount - initialCount t.Logf("Network requests during cache-first operations: %d", cachedRequestDiff) // Phase 2: Test explicit expansion (may trigger cache population) t.Logf("=== Phase 2: Testing expansion operations ===") beforeExpansionCount := requestCount.Load() // Expansion operations may need to populate cache if empty result = node.RunIPFS("bootstrap", "list", "--expand-auto") if result.ExitCode() == 0 { output := result.Stdout.String() assert.NotContains(t, output, "auto", "Expanded bootstrap should not contain 'auto' literal") t.Logf("Bootstrap expansion succeeded") } else { t.Logf("Bootstrap expansion failed (may be due to network/cache issues): %s", result.Stderr.String()) } result = node.RunIPFS("config", "Bootstrap", "--expand-auto") if result.ExitCode() == 0 { t.Logf("Config Bootstrap expansion succeeded") } else { t.Logf("Config Bootstrap expansion failed: %s", result.Stderr.String()) } afterExpansionCount := requestCount.Load() expansionRequestDiff := afterExpansionCount - beforeExpansionCount t.Logf("Network requests during expansion operations: %d", expansionRequestDiff) // Phase 3: Test background service behavior (if daemon is started) t.Logf("=== Phase 3: Testing background service behavior ===") beforeDaemonCount := requestCount.Load() // Set short refresh interval to test background service node.SetIPFSConfig("AutoConf.RefreshInterval", "1s") // Start daemon - this triggers startAutoConfUpdater() which should make network requests node.StartDaemon() defer node.StopDaemon() // Wait for background service to potentially make requests time.Sleep(2 * time.Second) afterDaemonCount := requestCount.Load() daemonRequestDiff := afterDaemonCount - beforeDaemonCount t.Logf("Network requests from background service: %d", daemonRequestDiff) // Verify expected behavior patterns t.Logf("=== Summary ===") t.Logf("Cache-first operations: %d requests", cachedRequestDiff) t.Logf("Expansion operations: %d requests", expansionRequestDiff) t.Logf("Background service: %d requests", daemonRequestDiff) // Cache-first operations should minimize network requests assert.LessOrEqual(t, cachedRequestDiff, int32(1), "Cache-first config operations should make minimal network requests") // Background service should make requests for refresh if daemonRequestDiff > 0 { t.Logf("✓ Background service is making network requests as expected") } else { t.Logf("⚠ Background service made no requests (may be using existing cache)") } t.Logf("Successfully verified network behavior patterns in autoconf architecture") } func testAutoConfWithHTTPS(t *testing.T) { // Test autoconf with HTTPS server and TLSInsecureSkipVerify enabled autoConfData := loadTestData(t, "valid_autoconf.json") // Create HTTPS server with self-signed certificate server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Logf("HTTPS autoconf request from %s", r.UserAgent()) w.Header().Set("Content-Type", "application/json") w.Header().Set("ETag", `"https-test-etag"`) w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") _, _ = w.Write(autoConfData) })) // Enable HTTP/2 and start with TLS (self-signed certificate) server.EnableHTTP2 = true server.StartTLS() defer server.Close() // Create IPFS node with HTTPS autoconf server and TLS skip verify node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("AutoConf.TLSInsecureSkipVerify", true) // Allow self-signed cert node.SetIPFSConfig("AutoConf.RefreshInterval", "24h") // Disable background updates // Use normal bootstrap peers to test HTTPS fetching without complex auto replacement node.SetIPFSConfig("Bootstrap", []string{"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN"}) // Start daemon to trigger HTTPS autoconf fetch node.StartDaemon() defer node.StopDaemon() // Give autoconf time to fetch over HTTPS time.Sleep(2 * time.Second) // Verify daemon is functional with HTTPS autoconf result := node.RunIPFS("id") assert.Equal(t, 0, result.ExitCode(), "IPFS daemon should be responsive with HTTPS autoconf") assert.Contains(t, result.Stdout.String(), "ID", "IPFS id command should return peer information") // Test that config operations work with HTTPS-fetched autoconf cache result = node.RunIPFS("config", "show") assert.Equal(t, 0, result.ExitCode(), "Config show should work with HTTPS autoconf") // Test bootstrap list functionality result = node.RunIPFS("bootstrap", "list") assert.Equal(t, 0, result.ExitCode(), "Bootstrap list should work with HTTPS autoconf") t.Logf("Successfully tested AutoConf with HTTPS server and TLS skip verify") } ================================================ FILE: test/cli/autoconf/dns_test.go ================================================ package autoconf import ( "encoding/base64" "fmt" "io" "net/http" "net/http/httptest" "strings" "sync" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/miekg/dns" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestAutoConfDNS(t *testing.T) { t.Parallel() t.Run("DNS resolution with auto DoH resolver", func(t *testing.T) { t.Parallel() testDNSResolutionWithAutoDoH(t) }) t.Run("DNS errors are handled properly", func(t *testing.T) { t.Parallel() testDNSErrorHandling(t) }) } // mockDoHServer implements a simple DNS-over-HTTPS server for testing type mockDoHServer struct { t *testing.T server *httptest.Server mu sync.Mutex requests []string responseFunc func(name string) *dns.Msg } func newMockDoHServer(t *testing.T) *mockDoHServer { m := &mockDoHServer{ t: t, requests: []string{}, } // Default response function returns a dnslink TXT record m.responseFunc = func(name string) *dns.Msg { msg := &dns.Msg{} msg.SetReply(&dns.Msg{Question: []dns.Question{{Name: name, Qtype: dns.TypeTXT}}}) if strings.HasPrefix(name, "_dnslink.") { // Return a valid dnslink record rr := &dns.TXT{ Hdr: dns.RR_Header{ Name: name, Rrtype: dns.TypeTXT, Class: dns.ClassINET, Ttl: 300, }, Txt: []string{"dnslink=/ipfs/QmYNQJoKGNHTpPxCBPh9KkDpaExgd2duMa3aF6ytMpHdao"}, } msg.Answer = append(msg.Answer, rr) } return msg } mux := http.NewServeMux() mux.HandleFunc("/dns-query", m.handleDNSQuery) m.server = httptest.NewServer(mux) return m } func (m *mockDoHServer) handleDNSQuery(w http.ResponseWriter, r *http.Request) { m.mu.Lock() defer m.mu.Unlock() var dnsMsg *dns.Msg if r.Method == "GET" { // Handle GET with ?dns= parameter dnsParam := r.URL.Query().Get("dns") if dnsParam == "" { http.Error(w, "missing dns parameter", http.StatusBadRequest) return } data, err := base64.RawURLEncoding.DecodeString(dnsParam) if err != nil { http.Error(w, "invalid base64", http.StatusBadRequest) return } dnsMsg = &dns.Msg{} if err := dnsMsg.Unpack(data); err != nil { http.Error(w, "invalid DNS message", http.StatusBadRequest) return } } else if r.Method == "POST" { // Handle POST with DNS wire format data, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "failed to read body", http.StatusBadRequest) return } dnsMsg = &dns.Msg{} if err := dnsMsg.Unpack(data); err != nil { http.Error(w, "invalid DNS message", http.StatusBadRequest) return } } else { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } // Log the DNS query if len(dnsMsg.Question) > 0 { qname := dnsMsg.Question[0].Name m.requests = append(m.requests, qname) m.t.Logf("DoH server received query for: %s", qname) } // Generate response response := m.responseFunc(dnsMsg.Question[0].Name) responseData, err := response.Pack() if err != nil { http.Error(w, "failed to pack response", http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/dns-message") _, _ = w.Write(responseData) } func (m *mockDoHServer) getRequests() []string { m.mu.Lock() defer m.mu.Unlock() return append([]string{}, m.requests...) } func (m *mockDoHServer) close() { m.server.Close() } func testDNSResolutionWithAutoDoH(t *testing.T) { // Create mock DoH server dohServer := newMockDoHServer(t) defer dohServer.close() // Create autoconf data with DoH resolver for "foo." domain autoConfData := fmt.Sprintf(`{ "AutoConfVersion": 2025072302, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": { "AminoDHT": { "Description": "Test AminoDHT system", "NativeConfig": { "Bootstrap": [] } } }, "DNSResolvers": { "foo.": ["%s/dns-query"] }, "DelegatedEndpoints": {} }`, dohServer.server.URL) // Create autoconf server autoConfServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(autoConfData)) })) defer autoConfServer.Close() // Create IPFS node with auto DNS resolver node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", autoConfServer.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("DNS.Resolvers", map[string]string{"foo.": "auto"}) // Start daemon node.StartDaemon() defer node.StopDaemon() // Verify config still shows "auto" for DNS resolvers result := node.RunIPFS("config", "DNS.Resolvers") require.Equal(t, 0, result.ExitCode()) dnsResolversOutput := result.Stdout.String() assert.Contains(t, dnsResolversOutput, "foo.", "DNS resolvers should contain foo. domain") assert.Contains(t, dnsResolversOutput, "auto", "DNS resolver config should show 'auto'") // Try to resolve a .foo domain result = node.RunIPFS("resolve", "/ipns/example.foo") require.Equal(t, 0, result.ExitCode()) // Should resolve to the IPFS path from our mock DoH server output := strings.TrimSpace(result.Stdout.String()) assert.Equal(t, "/ipfs/QmYNQJoKGNHTpPxCBPh9KkDpaExgd2duMa3aF6ytMpHdao", output, "Should resolve to the path returned by DoH server") // Verify DoH server received the DNS query requests := dohServer.getRequests() require.Greater(t, len(requests), 0, "DoH server should have received at least one request") foundDNSLink := false for _, req := range requests { if strings.Contains(req, "_dnslink.example.foo") { foundDNSLink = true break } } assert.True(t, foundDNSLink, "DoH server should have received query for _dnslink.example.foo") } func testDNSErrorHandling(t *testing.T) { // Create DoH server that returns NXDOMAIN dohServer := newMockDoHServer(t) defer dohServer.close() // Configure to return NXDOMAIN dohServer.responseFunc = func(name string) *dns.Msg { msg := &dns.Msg{} msg.SetReply(&dns.Msg{Question: []dns.Question{{Name: name, Qtype: dns.TypeTXT}}}) msg.Rcode = dns.RcodeNameError // NXDOMAIN return msg } // Create autoconf data with DoH resolver autoConfData := fmt.Sprintf(`{ "AutoConfVersion": 2025072302, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": { "AminoDHT": { "Description": "Test AminoDHT system", "NativeConfig": { "Bootstrap": [] } } }, "DNSResolvers": { "bar.": ["%s/dns-query"] }, "DelegatedEndpoints": {} }`, dohServer.server.URL) // Create autoconf server autoConfServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(autoConfData)) })) defer autoConfServer.Close() // Create IPFS node node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", autoConfServer.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("DNS.Resolvers", map[string]string{"bar.": "auto"}) // Start daemon node.StartDaemon() defer node.StopDaemon() // Try to resolve a non-existent domain result := node.RunIPFS("resolve", "/ipns/nonexistent.bar") require.NotEqual(t, 0, result.ExitCode(), "Resolution should fail for non-existent domain") // Should contain appropriate error message stderr := result.Stderr.String() assert.Contains(t, stderr, "could not resolve name", "Error should indicate DNS resolution failure") // Verify DoH server received the query requests := dohServer.getRequests() foundQuery := false for _, req := range requests { if strings.Contains(req, "_dnslink.nonexistent.bar") { foundQuery = true break } } assert.True(t, foundQuery, "DoH server should have received query even for failed resolution") } ================================================ FILE: test/cli/autoconf/expand_comprehensive_test.go ================================================ // Package autoconf provides comprehensive tests for --expand-auto functionality. // // Test Scenarios: // 1. Tests WITH daemon: Most tests start a daemon to fetch and cache autoconf data, // then test CLI commands that read from that cache using MustGetConfigCached. // 2. Tests WITHOUT daemon: Error condition tests that don't need cached autoconf. // // The daemon setup uses startDaemonAndWaitForAutoConf() helper which: // - Starts the daemon // - Waits for HTTP request to mock server (not arbitrary timeout) // - Returns when autoconf is cached and ready for CLI commands package autoconf import ( "encoding/json" "fmt" "net/http" "net/http/httptest" "os" "strings" "sync/atomic" "testing" "time" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestExpandAutoComprehensive(t *testing.T) { t.Parallel() t.Run("all autoconf fields resolve correctly", func(t *testing.T) { t.Parallel() testAllAutoConfFieldsResolve(t) }) t.Run("bootstrap list --expand-auto matches config Bootstrap --expand-auto", func(t *testing.T) { t.Parallel() testBootstrapCommandConsistency(t) }) t.Run("write operations fail with --expand-auto", func(t *testing.T) { t.Parallel() testWriteOperationsFailWithExpandAuto(t) }) t.Run("config show --expand-auto provides complete expanded view", func(t *testing.T) { t.Parallel() testConfigShowExpandAutoComplete(t) }) t.Run("multiple expand-auto calls use cache (single HTTP request)", func(t *testing.T) { t.Parallel() testMultipleExpandAutoUsesCache(t) }) t.Run("CLI uses cache only while daemon handles background updates", func(t *testing.T) { t.Parallel() testCLIUsesCacheOnlyDaemonUpdatesBackground(t) }) } // testAllAutoConfFieldsResolve verifies that all autoconf fields (Bootstrap, DNS.Resolvers, // Routing.DelegatedRouters, and Ipns.DelegatedPublishers) can be resolved from "auto" values // to their actual configuration using --expand-auto flag with daemon-cached autoconf data. // // This test is critical because: // 1. It validates the core autoconf resolution functionality across all supported fields // 2. It ensures that "auto" placeholders are properly replaced with real configuration values // 3. It verifies that the autoconf JSON structure is correctly parsed and applied // 4. It tests the end-to-end flow from HTTP fetch to config field expansion func testAllAutoConfFieldsResolve(t *testing.T) { // Test scenario: CLI with daemon started and autoconf cached // This validates core autoconf resolution functionality across all supported fields // Track HTTP requests to verify mock server is being used var requestCount atomic.Int32 var autoConfData []byte server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { count := requestCount.Add(1) t.Logf("Mock autoconf server request #%d: %s %s", count, r.Method, r.URL.Path) // Create comprehensive autoconf response matching Schema 4 format // Use server URLs to ensure they're reachable and valid serverURL := fmt.Sprintf("http://%s", r.Host) // Get the server URL from the request autoConf := map[string]any{ "AutoConfVersion": 2025072301, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": map[string]any{ "AminoDHT": map[string]any{ "URL": "https://github.com/ipfs/specs/pull/497", "Description": "Test AminoDHT system", "NativeConfig": map[string]any{ "Bootstrap": []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", }, }, "DelegatedConfig": map[string]any{ "Read": []string{"/routing/v1/providers", "/routing/v1/peers", "/routing/v1/ipns"}, "Write": []string{"/routing/v1/ipns"}, }, }, "IPNI": map[string]any{ "URL": serverURL + "/ipni-system", "Description": "Test IPNI system", "DelegatedConfig": map[string]any{ "Read": []string{"/routing/v1/providers"}, "Write": []string{}, }, }, "CustomIPNS": map[string]any{ "URL": serverURL + "/ipns-system", "Description": "Test IPNS system", "DelegatedConfig": map[string]any{ "Read": []string{"/routing/v1/ipns"}, "Write": []string{"/routing/v1/ipns"}, }, }, }, "DNSResolvers": map[string][]string{ ".": {"https://cloudflare-dns.com/dns-query"}, "eth.": {"https://dns.google/dns-query"}, }, "DelegatedEndpoints": map[string]any{ serverURL: map[string]any{ "Systems": []string{"IPNI", "CustomIPNS"}, // Use non-AminoDHT systems to avoid filtering "Read": []string{"/routing/v1/providers", "/routing/v1/ipns"}, "Write": []string{"/routing/v1/ipns"}, }, }, } var err error autoConfData, err = json.Marshal(autoConf) if err != nil { t.Fatalf("Failed to marshal autoConf: %v", err) } t.Logf("Serving mock autoconf data: %s", string(autoConfData)) w.Header().Set("Content-Type", "application/json") w.Header().Set("ETag", `"test-mock-config"`) w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") _, _ = w.Write(autoConfData) })) defer server.Close() // Create IPFS node with all auto values node := harness.NewT(t).NewNode().Init("--profile=test") // Clear any existing autoconf cache to prevent interference result := node.RunIPFS("config", "show") if result.ExitCode() == 0 { var cfg map[string]any if json.Unmarshal([]byte(result.Stdout.String()), &cfg) == nil { if repoPath, exists := cfg["path"]; exists { if pathStr, ok := repoPath.(string); ok { t.Logf("Clearing autoconf cache from %s/autoconf", pathStr) // Note: We can't directly remove files, but clearing cache via config change should help } } } } node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("AutoConf.RefreshInterval", "1s") // Force fresh fetches for testing node.SetIPFSConfig("Bootstrap", []string{"auto"}) node.SetIPFSConfig("DNS.Resolvers", map[string]string{ ".": "auto", "eth.": "auto", }) node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"}) node.SetIPFSConfig("Ipns.DelegatedPublishers", []string{"auto"}) // Start daemon and wait for autoconf fetch daemon := startDaemonAndWaitForAutoConf(t, node, &requestCount) defer daemon.StopDaemon() // Test 1: Bootstrap resolution result = node.RunIPFS("config", "Bootstrap", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "Bootstrap expansion should succeed") var expandedBootstrap []string var err error err = json.Unmarshal([]byte(result.Stdout.String()), &expandedBootstrap) require.NoError(t, err) assert.NotContains(t, expandedBootstrap, "auto", "Bootstrap should not contain 'auto'") assert.Contains(t, expandedBootstrap, "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN") assert.Contains(t, expandedBootstrap, "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa") t.Logf("Bootstrap expanded to: %v", expandedBootstrap) // Test 2: DNS.Resolvers resolution result = node.RunIPFS("config", "DNS.Resolvers", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "DNS.Resolvers expansion should succeed") var expandedResolvers map[string]string err = json.Unmarshal([]byte(result.Stdout.String()), &expandedResolvers) require.NoError(t, err) assert.NotContains(t, expandedResolvers, "auto", "DNS.Resolvers should not contain 'auto'") assert.Equal(t, "https://cloudflare-dns.com/dns-query", expandedResolvers["."]) assert.Equal(t, "https://dns.google/dns-query", expandedResolvers["eth."]) t.Logf("DNS.Resolvers expanded to: %v", expandedResolvers) // Test 3: Routing.DelegatedRouters resolution result = node.RunIPFS("config", "Routing.DelegatedRouters", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "Routing.DelegatedRouters expansion should succeed") var expandedRouters []string err = json.Unmarshal([]byte(result.Stdout.String()), &expandedRouters) require.NoError(t, err) assert.NotContains(t, expandedRouters, "auto", "DelegatedRouters should not contain 'auto'") // Test should strictly require mock autoconf to work - no fallback acceptance // The mock endpoint has Read paths ["/routing/v1/providers", "/routing/v1/ipns"] // so we expect 2 URLs with those paths expectedMockURLs := []string{ server.URL + "/routing/v1/providers", server.URL + "/routing/v1/ipns", } require.Equal(t, 2, len(expandedRouters), "Should have exactly 2 routers from mock autoconf (one for each Read path). Got %d routers: %v. "+ "This indicates autoconf is not working properly - check if mock server data is being parsed and filtered correctly.", len(expandedRouters), expandedRouters) // Check that both expected URLs are present for _, expectedURL := range expectedMockURLs { assert.Contains(t, expandedRouters, expectedURL, "Should contain mock autoconf endpoint with path %s. Got: %v. "+ "This indicates autoconf endpoint path generation is not working properly.", expectedURL, expandedRouters) } // Test 4: Ipns.DelegatedPublishers resolution result = node.RunIPFS("config", "Ipns.DelegatedPublishers", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "Ipns.DelegatedPublishers expansion should succeed") var expandedPublishers []string err = json.Unmarshal([]byte(result.Stdout.String()), &expandedPublishers) require.NoError(t, err) assert.NotContains(t, expandedPublishers, "auto", "DelegatedPublishers should not contain 'auto'") // Test should require mock autoconf endpoint for IPNS publishing // The mock endpoint supports /routing/v1/ipns write operations, so it should be included with path expectedMockPublisherURL := server.URL + "/routing/v1/ipns" require.Equal(t, 1, len(expandedPublishers), "Should have exactly 1 IPNS publisher from mock autoconf. Got %d publishers: %v. "+ "This indicates autoconf IPNS publisher filtering is not working properly.", len(expandedPublishers), expandedPublishers) assert.Equal(t, expectedMockPublisherURL, expandedPublishers[0], "Should use mock autoconf endpoint %s for IPNS publishing, not fallback. Got: %s. "+ "This indicates autoconf IPNS publisher resolution is not working properly.", expectedMockPublisherURL, expandedPublishers[0]) // CRITICAL: Verify that mock server was actually used finalRequestCount := requestCount.Load() require.Greater(t, finalRequestCount, int32(0), "Mock autoconf server should have been called at least once. Got %d requests. "+ "This indicates the test is using cached or fallback config instead of mock data.", finalRequestCount) t.Logf("Mock server was called %d times - test is using mock data", finalRequestCount) } // testBootstrapCommandConsistency verifies that `ipfs bootstrap list --expand-auto` and // `ipfs config Bootstrap --expand-auto` return identical results when both use autoconf. // // This test is important because: // 1. It ensures consistency between different CLI commands that access the same data // 2. It validates that both the bootstrap-specific command and generic config command // use the same underlying autoconf resolution mechanism // 3. It prevents regression where different commands might resolve "auto" differently // 4. It ensures users get consistent results regardless of which command they use func testBootstrapCommandConsistency(t *testing.T) { // Test scenario: CLI with daemon started and autoconf cached // This ensures both bootstrap commands read from the same cached autoconf data // Load test autoconf data autoConfData := loadTestDataComprehensive(t, "valid_autoconf.json") // Track HTTP requests to verify daemon fetches autoconf var requestCount atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requestCount.Add(1) t.Logf("Bootstrap consistency test request: %s %s", r.Method, r.URL.Path) w.Header().Set("Content-Type", "application/json") _, _ = w.Write(autoConfData) })) defer server.Close() // Create IPFS node with auto bootstrap node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Start daemon and wait for autoconf fetch daemon := startDaemonAndWaitForAutoConf(t, node, &requestCount) defer daemon.StopDaemon() // Get bootstrap via config command configResult := node.RunIPFS("config", "Bootstrap", "--expand-auto") require.Equal(t, 0, configResult.ExitCode(), "config Bootstrap --expand-auto should succeed") // Get bootstrap via bootstrap command bootstrapResult := node.RunIPFS("bootstrap", "list", "--expand-auto") require.Equal(t, 0, bootstrapResult.ExitCode(), "bootstrap list --expand-auto should succeed") // Parse both results var configBootstrap, bootstrapBootstrap []string err := json.Unmarshal([]byte(configResult.Stdout.String()), &configBootstrap) require.NoError(t, err) // Bootstrap command output is line-separated, not JSON bootstrapOutput := strings.TrimSpace(bootstrapResult.Stdout.String()) if bootstrapOutput != "" { bootstrapBootstrap = strings.Split(bootstrapOutput, "\n") } // Results should be equivalent assert.Equal(t, len(configBootstrap), len(bootstrapBootstrap), "Both commands should return same number of peers") // Both should contain same peers (order might differ due to different output formats) for _, peer := range configBootstrap { found := false for _, bsPeer := range bootstrapBootstrap { if strings.TrimSpace(bsPeer) == peer { found = true break } } assert.True(t, found, "Peer %s should be in both results", peer) } t.Logf("Config command result: %v", configBootstrap) t.Logf("Bootstrap command result: %v", bootstrapBootstrap) } // testWriteOperationsFailWithExpandAuto verifies that --expand-auto flag is properly // restricted to read-only operations and fails when used with config write operations. // // This test is essential because: // 1. It enforces the security principle that --expand-auto should only be used for reading // 2. It prevents users from accidentally overwriting config with expanded values // 3. It ensures that "auto" placeholders are preserved in the stored configuration // 4. It validates proper error handling and user guidance when misused // 5. It protects against accidental loss of the "auto" semantic meaning func testWriteOperationsFailWithExpandAuto(t *testing.T) { // Test scenario: CLI without daemon (tests error conditions) // This test doesn't need daemon setup since it's testing that write operations // with --expand-auto should fail with appropriate error messages // Create IPFS node node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Test that setting config with --expand-auto fails testCases := []struct { name string args []string }{ {"config set with expand-auto", []string{"config", "Bootstrap", "[\"test\"]", "--expand-auto"}}, {"config set JSON with expand-auto", []string{"config", "Bootstrap", "[\"test\"]", "--json", "--expand-auto"}}, {"config set bool with expand-auto", []string{"config", "SomeField", "true", "--bool", "--expand-auto"}}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { result := node.RunIPFS(tc.args...) assert.NotEqual(t, 0, result.ExitCode(), "Write operation with --expand-auto should fail") stderr := result.Stderr.String() assert.Contains(t, stderr, "--expand-auto", "Error should mention --expand-auto") assert.Contains(t, stderr, "reading", "Error should mention reading limitation") t.Logf("Expected error: %s", stderr) }) } } // testConfigShowExpandAutoComplete verifies that `ipfs config show --expand-auto` // produces a complete configuration with all "auto" values expanded to their resolved forms. // // This test is important because: // 1. It validates the full-config expansion functionality for comprehensive troubleshooting // 2. It ensures that users can see the complete resolved configuration state // 3. It verifies that all "auto" placeholders are replaced, not just individual fields // 4. It tests that the resulting JSON is valid and well-formed // 5. It provides a way to export/backup the fully expanded configuration func testConfigShowExpandAutoComplete(t *testing.T) { // Test scenario: CLI with daemon started and autoconf cached // Load test autoconf data autoConfData := loadTestDataComprehensive(t, "valid_autoconf.json") // Track HTTP requests to verify daemon fetches autoconf var requestCount atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requestCount.Add(1) t.Logf("Config show test request: %s %s", r.Method, r.URL.Path) w.Header().Set("Content-Type", "application/json") _, _ = w.Write(autoConfData) })) defer server.Close() // Create IPFS node with multiple auto values node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{"auto"}) node.SetIPFSConfig("DNS.Resolvers", map[string]string{".": "auto"}) // Start daemon and wait for autoconf fetch daemon := startDaemonAndWaitForAutoConf(t, node, &requestCount) defer daemon.StopDaemon() // Test config show --expand-auto result := node.RunIPFS("config", "show", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config show --expand-auto should succeed") expandedConfig := result.Stdout.String() // Should not contain any literal "auto" values assert.NotContains(t, expandedConfig, `"auto"`, "Expanded config should not contain literal 'auto' values") // Should contain expected expanded sections assert.Contains(t, expandedConfig, `"Bootstrap"`, "Should contain Bootstrap section") assert.Contains(t, expandedConfig, `"DNS"`, "Should contain DNS section") assert.Contains(t, expandedConfig, `"Resolvers"`, "Should contain Resolvers section") // Should contain expanded peer addresses (not "auto") assert.Contains(t, expandedConfig, "bootstrap.libp2p.io", "Should contain expanded bootstrap peers") // Should be valid JSON var configMap map[string]any err := json.Unmarshal([]byte(expandedConfig), &configMap) require.NoError(t, err, "Expanded config should be valid JSON") // Verify specific fields were expanded if bootstrap, ok := configMap["Bootstrap"].([]any); ok { assert.Greater(t, len(bootstrap), 0, "Bootstrap should have expanded entries") for _, peer := range bootstrap { assert.NotEqual(t, "auto", peer, "Bootstrap entries should not be 'auto'") } } t.Logf("Config show --expand-auto produced %d characters of expanded config", len(expandedConfig)) } // testMultipleExpandAutoUsesCache verifies that multiple consecutive --expand-auto calls // efficiently use cached autoconf data instead of making repeated HTTP requests. // // This test is critical for performance because: // 1. It validates that the caching mechanism works correctly to reduce network overhead // 2. It ensures that users can make multiple config queries without causing excessive HTTP traffic // 3. It verifies that cached data is shared across different config fields and commands // 4. It tests that HTTP headers (ETag/Last-Modified) are properly used for cache validation // 5. It prevents regression where each --expand-auto call would trigger a new HTTP request // 6. It demonstrates the performance benefit: 5 operations with only 1 network request func testMultipleExpandAutoUsesCache(t *testing.T) { // Test scenario: CLI with daemon started and autoconf cached // Create comprehensive autoconf response autoConfData := loadTestDataComprehensive(t, "valid_autoconf.json") // Track HTTP requests to verify caching var requestCount atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { count := requestCount.Add(1) t.Logf("AutoConf cache test request #%d: %s %s", count, r.Method, r.URL.Path) w.Header().Set("Content-Type", "application/json") w.Header().Set("ETag", `"cache-test-123"`) w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") _, _ = w.Write(autoConfData) })) defer server.Close() // Create IPFS node with all auto values node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) // Note: Using default RefreshInterval (24h) to ensure caching - explicit setting would require rebuilt binary // Set up auto values for multiple fields node.SetIPFSConfig("Bootstrap", []string{"auto"}) node.SetIPFSConfig("DNS.Resolvers", map[string]string{"foo.": "auto"}) node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"}) node.SetIPFSConfig("Ipns.DelegatedPublishers", []string{"auto"}) // Start daemon and wait for autoconf fetch daemon := startDaemonAndWaitForAutoConf(t, node, &requestCount) defer daemon.StopDaemon() // Reset counter to only track our expand-auto calls requestCount.Store(0) // Make multiple --expand-auto calls on different fields t.Log("Testing multiple --expand-auto calls should use cache...") // Call 1: Bootstrap --expand-auto (should trigger HTTP request) result1 := node.RunIPFS("config", "Bootstrap", "--expand-auto") require.Equal(t, 0, result1.ExitCode(), "Bootstrap --expand-auto should succeed") var expandedBootstrap []string err := json.Unmarshal([]byte(result1.Stdout.String()), &expandedBootstrap) require.NoError(t, err) assert.NotContains(t, expandedBootstrap, "auto", "Bootstrap should be expanded") assert.Greater(t, len(expandedBootstrap), 0, "Bootstrap should have entries") // Call 2: DNS.Resolvers --expand-auto (should use cache, no HTTP) result2 := node.RunIPFS("config", "DNS.Resolvers", "--expand-auto") require.Equal(t, 0, result2.ExitCode(), "DNS.Resolvers --expand-auto should succeed") var expandedResolvers map[string]string err = json.Unmarshal([]byte(result2.Stdout.String()), &expandedResolvers) require.NoError(t, err) // Call 3: Routing.DelegatedRouters --expand-auto (should use cache, no HTTP) result3 := node.RunIPFS("config", "Routing.DelegatedRouters", "--expand-auto") require.Equal(t, 0, result3.ExitCode(), "Routing.DelegatedRouters --expand-auto should succeed") var expandedRouters []string err = json.Unmarshal([]byte(result3.Stdout.String()), &expandedRouters) require.NoError(t, err) assert.NotContains(t, expandedRouters, "auto", "Routers should be expanded") // Call 4: Ipns.DelegatedPublishers --expand-auto (should use cache, no HTTP) result4 := node.RunIPFS("config", "Ipns.DelegatedPublishers", "--expand-auto") require.Equal(t, 0, result4.ExitCode(), "Ipns.DelegatedPublishers --expand-auto should succeed") var expandedPublishers []string err = json.Unmarshal([]byte(result4.Stdout.String()), &expandedPublishers) require.NoError(t, err) assert.NotContains(t, expandedPublishers, "auto", "Publishers should be expanded") // Call 5: config show --expand-auto (should use cache, no HTTP) result5 := node.RunIPFS("config", "show", "--expand-auto") require.Equal(t, 0, result5.ExitCode(), "config show --expand-auto should succeed") expandedConfig := result5.Stdout.String() assert.NotContains(t, expandedConfig, `"auto"`, "Full config should not contain 'auto' values") // CRITICAL TEST: Verify NO HTTP requests were made for --expand-auto calls (using cache) finalRequestCount := requestCount.Load() assert.Equal(t, int32(0), finalRequestCount, "Multiple --expand-auto calls should result in 0 HTTP requests (using cache). Got %d requests", finalRequestCount) t.Logf("Made 5 --expand-auto calls, resulted in %d HTTP request(s) - cache is being used!", finalRequestCount) // Now simulate a manual cache refresh (what the background updater would do) t.Log("Simulating manual cache refresh...") // Update the mock server to return different data autoConfData2 := loadTestDataComprehensive(t, "updated_autoconf.json") server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { count := requestCount.Add(1) t.Logf("Manual refresh request #%d: %s %s", count, r.Method, r.URL.Path) w.Header().Set("Content-Type", "application/json") w.Header().Set("ETag", `"cache-test-456"`) w.Header().Set("Last-Modified", "Thu, 22 Oct 2015 08:00:00 GMT") _, _ = w.Write(autoConfData2) }) // Note: In the actual daemon, the background updater would call MustGetConfigWithRefresh // For this test, we'll verify that subsequent --expand-auto calls still use cache // and don't trigger additional requests // Reset counter before manual refresh simulation beforeRefresh := requestCount.Load() // Make another --expand-auto call - should still use cache result6 := node.RunIPFS("config", "Bootstrap", "--expand-auto") require.Equal(t, 0, result6.ExitCode(), "Bootstrap --expand-auto after refresh should succeed") afterRefresh := requestCount.Load() assert.Equal(t, beforeRefresh, afterRefresh, "--expand-auto should continue using cache even after server update") t.Logf("Cache continues to be used after server update - background updater pattern confirmed!") } // testCLIUsesCacheOnlyDaemonUpdatesBackground verifies the correct autoconf behavior: // daemon makes exactly one HTTP request during startup to fetch and cache data, then // CLI commands always use cached data without making additional HTTP requests. // // This test is essential for correctness because: // 1. It validates that daemon startup makes exactly one HTTP request to fetch autoconf // 2. It verifies that CLI --expand-auto never makes HTTP requests (uses cache only) // 3. It ensures CLI commands remain fast by always using cached data // 4. It prevents regression where CLI commands might start making HTTP requests // 5. It confirms the correct separation between daemon (network) and CLI (cache-only) behavior func testCLIUsesCacheOnlyDaemonUpdatesBackground(t *testing.T) { // Test scenario: CLI with daemon and long RefreshInterval (no background updates during test) // Create autoconf response autoConfData := loadTestDataComprehensive(t, "valid_autoconf.json") // Track HTTP requests with timestamps var requestCount atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { count := requestCount.Add(1) t.Logf("Cache expiry test request #%d at %s: %s %s", count, time.Now().Format("15:04:05.000"), r.Method, r.URL.Path) w.Header().Set("Content-Type", "application/json") // Use different ETag for each request to ensure we can detect new fetches w.Header().Set("ETag", fmt.Sprintf(`"expiry-test-%d"`, count)) w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat)) _, _ = w.Write(autoConfData) })) defer server.Close() // Create IPFS node with long refresh interval node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) // Set long RefreshInterval to avoid background updates during test node.SetIPFSConfig("AutoConf.RefreshInterval", "1h") node.SetIPFSConfig("Bootstrap", []string{"auto"}) node.SetIPFSConfig("DNS.Resolvers", map[string]string{"test.": "auto"}) // Start daemon and wait for autoconf fetch daemon := startDaemonAndWaitForAutoConf(t, node, &requestCount) defer daemon.StopDaemon() // Confirm only one request was made during daemon startup initialRequestCount := requestCount.Load() assert.Equal(t, int32(1), initialRequestCount, "Expected exactly 1 HTTP request during daemon startup, got: %d", initialRequestCount) t.Logf("Daemon startup made exactly 1 HTTP request") // Test: CLI commands use cache only (no additional HTTP requests) t.Log("Testing that CLI --expand-auto commands use cache only...") // Make several CLI calls - none should trigger HTTP requests result1 := node.RunIPFS("config", "Bootstrap", "--expand-auto") require.Equal(t, 0, result1.ExitCode(), "Bootstrap --expand-auto should succeed") result2 := node.RunIPFS("config", "DNS.Resolvers", "--expand-auto") require.Equal(t, 0, result2.ExitCode(), "DNS.Resolvers --expand-auto should succeed") result3 := node.RunIPFS("config", "Routing.DelegatedRouters", "--expand-auto") require.Equal(t, 0, result3.ExitCode(), "Routing.DelegatedRouters --expand-auto should succeed") // Verify the request count remains at 1 (no additional requests from CLI) finalRequestCount := requestCount.Load() assert.Equal(t, int32(1), finalRequestCount, "Request count should remain at 1 after CLI commands, got: %d", finalRequestCount) t.Log("CLI commands use cache only - request count remains at 1") t.Log("Test completed: Daemon makes 1 startup request, CLI commands use cache only") } // loadTestDataComprehensive is a helper function that loads test autoconf JSON data files. // It locates the test data directory relative to the test file and reads the specified file. // This centralized helper ensures consistent test data loading across all comprehensive tests. func loadTestDataComprehensive(t *testing.T, filename string) []byte { t.Helper() data, err := os.ReadFile("testdata/" + filename) require.NoError(t, err, "Failed to read test data file: %s", filename) return data } // startDaemonAndWaitForAutoConf starts a daemon and waits for it to fetch autoconf data. // It returns the node with daemon running and ensures autoconf has been cached before returning. // This is a DRY helper to avoid repeating daemon setup and request waiting logic in every test. func startDaemonAndWaitForAutoConf(t *testing.T, node *harness.Node, requestCount *atomic.Int32) *harness.Node { t.Helper() // Start daemon to fetch and cache autoconf data t.Log("Starting daemon to fetch and cache autoconf data...") daemon := node.StartDaemon() // StartDaemon returns *Node, no error to check // Wait for daemon to fetch autoconf (wait for HTTP request to mock server) t.Log("Waiting for daemon to fetch autoconf from mock server...") timeout := time.After(10 * time.Second) // Safety timeout ticker := time.NewTicker(10 * time.Millisecond) defer ticker.Stop() for { select { case <-timeout: t.Fatal("Timeout waiting for autoconf fetch") case <-ticker.C: if requestCount.Load() > 0 { t.Logf("Daemon fetched autoconf (%d requests made)", requestCount.Load()) t.Log("AutoConf should now be cached by daemon") return daemon } } } } ================================================ FILE: test/cli/autoconf/expand_fallback_test.go ================================================ package autoconf import ( "encoding/json" "net/http" "net/http/httptest" "os" "slices" "testing" "time" "github.com/ipfs/boxo/autoconf" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestExpandAutoFallbacks(t *testing.T) { t.Parallel() t.Run("expand-auto with unreachable server shows fallbacks", func(t *testing.T) { t.Parallel() testExpandAutoWithUnreachableServer(t) }) t.Run("expand-auto with disabled autoconf shows error", func(t *testing.T) { t.Parallel() testExpandAutoWithDisabledAutoConf(t) }) t.Run("expand-auto with malformed response shows fallbacks", func(t *testing.T) { t.Parallel() testExpandAutoWithMalformedResponse(t) }) t.Run("expand-auto preserves static values in mixed config", func(t *testing.T) { t.Parallel() testExpandAutoMixedConfigPreservesStatic(t) }) t.Run("daemon gracefully handles malformed autoconf and uses fallbacks", func(t *testing.T) { t.Parallel() testDaemonWithMalformedAutoConf(t) }) } func testExpandAutoWithUnreachableServer(t *testing.T) { // Create IPFS node with unreachable AutoConf server node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", "http://127.0.0.1:99999/nonexistent") // Unreachable node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{"auto"}) node.SetIPFSConfig("DNS.Resolvers", map[string]string{"foo.": "auto"}) // Test that --expand-auto falls back to defaults when server is unreachable result := node.RunIPFS("config", "Bootstrap", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Bootstrap --expand-auto should succeed even with unreachable server") var bootstrap []string err := json.Unmarshal([]byte(result.Stdout.String()), &bootstrap) require.NoError(t, err) // Should contain fallback bootstrap peers (not "auto" and not empty) assert.NotContains(t, bootstrap, "auto", "Fallback bootstrap should not contain 'auto'") assert.Greater(t, len(bootstrap), 0, "Fallback bootstrap should not be empty") // Should contain known default bootstrap peers foundDefaultPeer := false for _, peer := range bootstrap { if peer != "" && peer != "auto" { foundDefaultPeer = true t.Logf("Found fallback bootstrap peer: %s", peer) break } } assert.True(t, foundDefaultPeer, "Should contain at least one fallback bootstrap peer") // Test DNS resolvers fallback result = node.RunIPFS("config", "DNS.Resolvers", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config DNS.Resolvers --expand-auto should succeed with unreachable server") var resolvers map[string]string err = json.Unmarshal([]byte(result.Stdout.String()), &resolvers) require.NoError(t, err) // When autoconf server is unreachable, DNS resolvers should fall back to defaults // The "foo." resolver should not exist in fallbacks (only "eth." has fallback) fooResolver, fooExists := resolvers["foo."] if !fooExists { t.Log("DNS resolver for 'foo.' has no fallback - correct behavior (only eth. has fallbacks)") } else { assert.NotEqual(t, "auto", fooResolver, "DNS resolver should not be 'auto' after expansion") t.Logf("Unexpected DNS resolver for foo.: %s", fooResolver) } } func testExpandAutoWithDisabledAutoConf(t *testing.T) { // Create IPFS node with AutoConf disabled node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.Enabled", false) node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Test that --expand-auto with disabled AutoConf returns appropriate error or fallback result := node.RunIPFS("config", "Bootstrap", "--expand-auto") // When AutoConf is disabled, expand-auto should show empty results // since "auto" values are not expanded when AutoConf.Enabled=false var bootstrap []string err := json.Unmarshal([]byte(result.Stdout.String()), &bootstrap) require.NoError(t, err) // With AutoConf disabled, "auto" values are not expanded so we get empty result assert.NotContains(t, bootstrap, "auto", "Should not contain 'auto' after expansion") assert.Equal(t, 0, len(bootstrap), "Should be empty when AutoConf disabled (auto values not expanded)") t.Log("Bootstrap is empty when AutoConf disabled - correct behavior") } func testExpandAutoWithMalformedResponse(t *testing.T) { // Create server that returns malformed JSON server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"invalid": "json", "Bootstrap": [incomplete`)) // Malformed JSON })) defer server.Close() // Create IPFS node with malformed autoconf server node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Test that --expand-auto handles malformed response gracefully result := node.RunIPFS("config", "Bootstrap", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Bootstrap --expand-auto should succeed even with malformed response") var bootstrap []string err := json.Unmarshal([]byte(result.Stdout.String()), &bootstrap) require.NoError(t, err) // Should fall back to defaults, not contain "auto" assert.NotContains(t, bootstrap, "auto", "Should not contain 'auto' after fallback") assert.Greater(t, len(bootstrap), 0, "Should contain fallback peers after malformed response") t.Logf("Bootstrap after malformed response: %v", bootstrap) } func testExpandAutoMixedConfigPreservesStatic(t *testing.T) { // Load valid test autoconf data autoConfData := loadTestDataForFallback(t, "valid_autoconf.json") // Create HTTP server that serves autoconf.json server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(autoConfData) })) defer server.Close() // Create IPFS node with mixed auto and static values node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) // Set mixed configuration: static + auto + static node.SetIPFSConfig("Bootstrap", []string{ "/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWTest", "auto", "/ip4/127.0.0.2/tcp/4001/p2p/12D3KooWTest2", }) // Test that --expand-auto only expands "auto" values, preserves static ones result := node.RunIPFS("config", "Bootstrap", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Bootstrap --expand-auto should succeed") var bootstrap []string err := json.Unmarshal([]byte(result.Stdout.String()), &bootstrap) require.NoError(t, err) // Should not contain literal "auto" anymore assert.NotContains(t, bootstrap, "auto", "Expanded config should not contain literal 'auto'") // Should preserve static values at original positions assert.Contains(t, bootstrap, "/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWTest", "Should preserve first static peer") assert.Contains(t, bootstrap, "/ip4/127.0.0.2/tcp/4001/p2p/12D3KooWTest2", "Should preserve third static peer") // Should have more entries than just the static ones (auto got expanded) assert.Greater(t, len(bootstrap), 2, "Should have more than just the 2 static peers") t.Logf("Mixed config expansion result: %v", bootstrap) // Verify order is preserved: static, expanded auto values, static assert.Equal(t, "/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWTest", bootstrap[0], "First peer should be preserved") lastIndex := len(bootstrap) - 1 assert.Equal(t, "/ip4/127.0.0.2/tcp/4001/p2p/12D3KooWTest2", bootstrap[lastIndex], "Last peer should be preserved") } func testDaemonWithMalformedAutoConf(t *testing.T) { // Test scenario: Daemon starts with AutoConf.URL pointing to server that returns malformed JSON // This tests that daemon gracefully handles malformed responses and falls back to hardcoded defaults // Create server that returns malformed JSON to simulate broken autoconf service server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") // Return malformed JSON that cannot be parsed _, _ = w.Write([]byte(`{"Bootstrap": ["incomplete array", "missing closing bracket"`)) })) defer server.Close() // Create IPFS node with autoconf pointing to malformed server node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{"auto"}) node.SetIPFSConfig("DNS.Resolvers", map[string]string{"foo.": "auto"}) // Start daemon - this will attempt to fetch autoconf from malformed server t.Log("Starting daemon with malformed autoconf server...") daemon := node.StartDaemon() defer daemon.StopDaemon() // Wait for daemon to attempt autoconf fetch and handle the error gracefully time.Sleep(6 * time.Second) // defaultTimeout is 5s, add 1s buffer t.Log("Daemon should have attempted autoconf fetch and fallen back to defaults") // Test that daemon is still running and CLI commands work with fallback values result := node.RunIPFS("config", "Bootstrap", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Bootstrap --expand-auto should succeed with daemon running") var bootstrap []string err := json.Unmarshal([]byte(result.Stdout.String()), &bootstrap) require.NoError(t, err) // Should fall back to hardcoded defaults from GetMainnetFallbackConfig() // NOTE: These values may change if autoconf library updates GetMainnetFallbackConfig() assert.NotContains(t, bootstrap, "auto", "Should not contain 'auto' after fallback") assert.Greater(t, len(bootstrap), 0, "Should contain fallback bootstrap peers") // Verify we got actual fallback bootstrap peers from GetMainnetFallbackConfig() AminoDHT NativeConfig fallbackConfig := autoconf.GetMainnetFallbackConfig() aminoDHTSystem := fallbackConfig.SystemRegistry["AminoDHT"] expectedBootstrapPeers := aminoDHTSystem.NativeConfig.Bootstrap foundFallbackPeers := 0 for _, expectedPeer := range expectedBootstrapPeers { if slices.Contains(bootstrap, expectedPeer) { foundFallbackPeers++ } } assert.Greater(t, foundFallbackPeers, 0, "Should contain bootstrap peers from GetMainnetFallbackConfig() AminoDHT NativeConfig") assert.Equal(t, len(expectedBootstrapPeers), foundFallbackPeers, "Should contain all bootstrap peers from GetMainnetFallbackConfig() AminoDHT NativeConfig") t.Logf("Daemon fallback bootstrap peers after malformed response: %v", bootstrap) // Test DNS resolvers also fall back correctly result = node.RunIPFS("config", "DNS.Resolvers", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config DNS.Resolvers --expand-auto should succeed with daemon running") var resolvers map[string]string err = json.Unmarshal([]byte(result.Stdout.String()), &resolvers) require.NoError(t, err) // Should not contain "auto" and should have fallback DNS resolvers assert.NotEqual(t, "auto", resolvers["foo."], "DNS resolver should not be 'auto' after fallback") if resolvers["foo."] != "" { // If resolver is populated, it should be a valid URL from fallbacks assert.Contains(t, resolvers["foo."], "https://", "Fallback DNS resolver should be HTTPS URL") } t.Logf("Daemon fallback DNS resolvers after malformed response: %v", resolvers) // Verify daemon is still healthy and responsive versionResult := node.RunIPFS("version") require.Equal(t, 0, versionResult.ExitCode(), "daemon should remain healthy after handling malformed autoconf") t.Log("Daemon remains healthy after gracefully handling malformed autoconf response") } // Helper function to load test data files for fallback tests func loadTestDataForFallback(t *testing.T, filename string) []byte { t.Helper() data, err := os.ReadFile("testdata/" + filename) require.NoError(t, err, "Failed to read test data file: %s", filename) return data } ================================================ FILE: test/cli/autoconf/expand_test.go ================================================ package autoconf import ( "encoding/json" "net/http" "net/http/httptest" "os" "testing" "time" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestAutoConfExpand(t *testing.T) { t.Parallel() t.Run("config commands show auto values", func(t *testing.T) { t.Parallel() testConfigCommandsShowAutoValues(t) }) t.Run("mixed configuration preserves both auto and static", func(t *testing.T) { t.Parallel() testMixedConfigurationPreserved(t) }) t.Run("config replace preserves auto values", func(t *testing.T) { t.Parallel() testConfigReplacePreservesAuto(t) }) t.Run("expand-auto filters unsupported URL paths with delegated routing", func(t *testing.T) { t.Parallel() testExpandAutoFiltersUnsupportedPathsDelegated(t) }) t.Run("expand-auto with auto routing uses NewRoutingSystem", func(t *testing.T) { t.Parallel() testExpandAutoWithAutoRouting(t) }) t.Run("expand-auto with auto routing shows AminoDHT native vs IPNI delegated", func(t *testing.T) { t.Parallel() testExpandAutoWithMixedSystems(t) }) t.Run("expand-auto filters paths with NewRoutingSystem and auto routing", func(t *testing.T) { t.Parallel() testExpandAutoWithFiltering(t) }) t.Run("expand-auto falls back to defaults without cache (delegated)", func(t *testing.T) { t.Parallel() testExpandAutoWithoutCacheDelegated(t) }) t.Run("expand-auto with auto routing without cache", func(t *testing.T) { t.Parallel() testExpandAutoWithoutCacheAuto(t) }) } func testConfigCommandsShowAutoValues(t *testing.T) { // Create IPFS node node := harness.NewT(t).NewNode().Init("--profile=test") // Set all fields to "auto" node.SetIPFSConfig("Bootstrap", []string{"auto"}) node.SetIPFSConfig("DNS.Resolvers", map[string]string{"foo.": "auto"}) node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"}) node.SetIPFSConfig("Ipns.DelegatedPublishers", []string{"auto"}) // Test individual field queries t.Run("Bootstrap shows auto", func(t *testing.T) { result := node.RunIPFS("config", "Bootstrap") require.Equal(t, 0, result.ExitCode()) var bootstrap []string err := json.Unmarshal([]byte(result.Stdout.String()), &bootstrap) require.NoError(t, err) assert.Equal(t, []string{"auto"}, bootstrap) }) t.Run("DNS.Resolvers shows auto", func(t *testing.T) { result := node.RunIPFS("config", "DNS.Resolvers") require.Equal(t, 0, result.ExitCode()) var resolvers map[string]string err := json.Unmarshal([]byte(result.Stdout.String()), &resolvers) require.NoError(t, err) assert.Equal(t, map[string]string{"foo.": "auto"}, resolvers) }) t.Run("Routing.DelegatedRouters shows auto", func(t *testing.T) { result := node.RunIPFS("config", "Routing.DelegatedRouters") require.Equal(t, 0, result.ExitCode()) var routers []string err := json.Unmarshal([]byte(result.Stdout.String()), &routers) require.NoError(t, err) assert.Equal(t, []string{"auto"}, routers) }) t.Run("Ipns.DelegatedPublishers shows auto", func(t *testing.T) { result := node.RunIPFS("config", "Ipns.DelegatedPublishers") require.Equal(t, 0, result.ExitCode()) var publishers []string err := json.Unmarshal([]byte(result.Stdout.String()), &publishers) require.NoError(t, err) assert.Equal(t, []string{"auto"}, publishers) }) t.Run("config show contains all auto values", func(t *testing.T) { result := node.RunIPFS("config", "show") require.Equal(t, 0, result.ExitCode()) output := result.Stdout.String() // Check that auto values are present in the full config assert.Contains(t, output, `"Bootstrap": [ "auto" ]`, "Bootstrap should contain auto") assert.Contains(t, output, `"DNS": { "Resolvers": { "foo.": "auto" } }`, "DNS.Resolvers should contain auto") assert.Contains(t, output, `"DelegatedRouters": [ "auto" ]`, "Routing.DelegatedRouters should contain auto") assert.Contains(t, output, `"DelegatedPublishers": [ "auto" ]`, "Ipns.DelegatedPublishers should contain auto") }) // Test with autoconf server for --expand-auto functionality t.Run("config with --expand-auto expands auto values", func(t *testing.T) { // Load test autoconf data autoConfData := loadTestDataExpand(t, "valid_autoconf.json") // Create HTTP server that serves autoconf.json server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(autoConfData) })) defer server.Close() // Configure autoconf for the node node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) // Test Bootstrap field expansion result := node.RunIPFS("config", "Bootstrap", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Bootstrap --expand-auto should succeed") var expandedBootstrap []string err := json.Unmarshal([]byte(result.Stdout.String()), &expandedBootstrap) require.NoError(t, err) assert.NotContains(t, expandedBootstrap, "auto", "Expanded bootstrap should not contain 'auto'") assert.Greater(t, len(expandedBootstrap), 0, "Expanded bootstrap should contain expanded peers") // Test DNS.Resolvers field expansion result = node.RunIPFS("config", "DNS.Resolvers", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config DNS.Resolvers --expand-auto should succeed") var expandedResolvers map[string]string err = json.Unmarshal([]byte(result.Stdout.String()), &expandedResolvers) require.NoError(t, err) assert.NotEqual(t, "auto", expandedResolvers["foo."], "Expanded DNS resolver should not be 'auto'") // Test Routing.DelegatedRouters field expansion result = node.RunIPFS("config", "Routing.DelegatedRouters", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Routing.DelegatedRouters --expand-auto should succeed") var expandedRouters []string err = json.Unmarshal([]byte(result.Stdout.String()), &expandedRouters) require.NoError(t, err) assert.NotContains(t, expandedRouters, "auto", "Expanded routers should not contain 'auto'") // Test Ipns.DelegatedPublishers field expansion result = node.RunIPFS("config", "Ipns.DelegatedPublishers", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Ipns.DelegatedPublishers --expand-auto should succeed") var expandedPublishers []string err = json.Unmarshal([]byte(result.Stdout.String()), &expandedPublishers) require.NoError(t, err) assert.NotContains(t, expandedPublishers, "auto", "Expanded publishers should not contain 'auto'") // Test config show --expand-auto (full config expansion) result = node.RunIPFS("config", "show", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config show --expand-auto should succeed") expandedOutput := result.Stdout.String() t.Logf("Expanded config output contains: %d characters", len(expandedOutput)) // Verify that auto values are expanded in the full config assert.NotContains(t, expandedOutput, `"auto"`, "Expanded config should not contain literal 'auto' values") assert.Contains(t, expandedOutput, `"Bootstrap"`, "Expanded config should contain Bootstrap section") assert.Contains(t, expandedOutput, `"DNS"`, "Expanded config should contain DNS section") }) } func testMixedConfigurationPreserved(t *testing.T) { // Create IPFS node node := harness.NewT(t).NewNode().Init("--profile=test") // Set mixed configuration node.SetIPFSConfig("Bootstrap", []string{ "/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWTest", "auto", "/ip4/127.0.0.2/tcp/4001/p2p/12D3KooWTest2", }) node.SetIPFSConfig("DNS.Resolvers", map[string]string{ "eth.": "https://eth.resolver", "foo.": "auto", "bar.": "https://bar.resolver", }) node.SetIPFSConfig("Routing.DelegatedRouters", []string{ "https://static.router", "auto", }) // Verify Bootstrap preserves order and mixes auto with static result := node.RunIPFS("config", "Bootstrap") require.Equal(t, 0, result.ExitCode()) var bootstrap []string err := json.Unmarshal([]byte(result.Stdout.String()), &bootstrap) require.NoError(t, err) assert.Equal(t, []string{ "/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWTest", "auto", "/ip4/127.0.0.2/tcp/4001/p2p/12D3KooWTest2", }, bootstrap) // Verify DNS.Resolvers preserves both auto and static result = node.RunIPFS("config", "DNS.Resolvers") require.Equal(t, 0, result.ExitCode()) var resolvers map[string]string err = json.Unmarshal([]byte(result.Stdout.String()), &resolvers) require.NoError(t, err) assert.Equal(t, "https://eth.resolver", resolvers["eth."]) assert.Equal(t, "auto", resolvers["foo."]) assert.Equal(t, "https://bar.resolver", resolvers["bar."]) // Verify Routing.DelegatedRouters preserves order result = node.RunIPFS("config", "Routing.DelegatedRouters") require.Equal(t, 0, result.ExitCode()) var routers []string err = json.Unmarshal([]byte(result.Stdout.String()), &routers) require.NoError(t, err) assert.Equal(t, []string{ "https://static.router", "auto", }, routers) } func testConfigReplacePreservesAuto(t *testing.T) { // Create IPFS node h := harness.NewT(t) node := h.NewNode().Init("--profile=test") // Set initial auto values node.SetIPFSConfig("Bootstrap", []string{"auto"}) node.SetIPFSConfig("DNS.Resolvers", map[string]string{"foo.": "auto"}) // Export current config result := node.RunIPFS("config", "show") require.Equal(t, 0, result.ExitCode()) originalConfig := result.Stdout.String() // Verify auto values are in the exported config assert.Contains(t, originalConfig, `"Bootstrap": [ "auto" ]`) assert.Contains(t, originalConfig, `"foo.": "auto"`) // Modify the config string to add a new field but preserve auto values var configMap map[string]any err := json.Unmarshal([]byte(originalConfig), &configMap) require.NoError(t, err) // Add a new field configMap["NewTestField"] = "test-value" // Marshal back to JSON modifiedConfig, err := json.MarshalIndent(configMap, "", " ") require.NoError(t, err) // Write config to file and replace configFile := h.WriteToTemp(string(modifiedConfig)) replaceResult := node.RunIPFS("config", "replace", configFile) if replaceResult.ExitCode() != 0 { t.Logf("Config replace failed: stdout=%s, stderr=%s", replaceResult.Stdout.String(), replaceResult.Stderr.String()) } require.Equal(t, 0, replaceResult.ExitCode()) // Verify auto values are still present after replace result = node.RunIPFS("config", "Bootstrap") require.Equal(t, 0, result.ExitCode()) var bootstrap []string err = json.Unmarshal([]byte(result.Stdout.String()), &bootstrap) require.NoError(t, err) assert.Equal(t, []string{"auto"}, bootstrap, "Bootstrap should still contain auto after config replace") // Verify DNS resolver config is preserved after replace result = node.RunIPFS("config", "DNS.Resolvers") require.Equal(t, 0, result.ExitCode()) var resolvers map[string]string err = json.Unmarshal([]byte(result.Stdout.String()), &resolvers) require.NoError(t, err) assert.Equal(t, "auto", resolvers["foo."], "DNS resolver for foo. should still be auto after config replace") } func testExpandAutoFiltersUnsupportedPathsDelegated(t *testing.T) { // Test scenario: CLI with daemon started and autoconf cached using delegated routing // This tests the production scenario where delegated routing is enabled and // daemon has fetched and cached autoconf data, and CLI commands read from that cache // Create IPFS node node := harness.NewT(t).NewNode().Init("--profile=test") // Configure delegated routing to use autoconf URLs node.SetIPFSConfig("Routing.Type", "delegated") node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"}) node.SetIPFSConfig("Ipns.DelegatedPublishers", []string{"auto"}) // Disable content providing when using delegated routing node.SetIPFSConfig("Provide.Enabled", false) node.SetIPFSConfig("Provide.DHT.Interval", "0") // Load test autoconf data with unsupported paths autoConfData := loadTestDataExpand(t, "autoconf_with_unsupported_paths.json") // Create HTTP server that serves autoconf.json with unsupported paths server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(autoConfData) })) defer server.Close() // Configure autoconf for the node node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) // Verify the autoconf URL is set correctly result := node.RunIPFS("config", "AutoConf.URL") require.Equal(t, 0, result.ExitCode(), "config AutoConf.URL should succeed") t.Logf("AutoConf URL is set to: %s", result.Stdout.String()) assert.Contains(t, result.Stdout.String(), "127.0.0.1", "AutoConf URL should contain the test server address") // Start daemon to fetch and cache autoconf data t.Log("Starting daemon to fetch and cache autoconf data...") daemon := node.StartDaemon() defer daemon.StopDaemon() // Wait for autoconf fetch (use autoconf default timeout + buffer) time.Sleep(6 * time.Second) // defaultTimeout is 5s, add 1s buffer t.Log("AutoConf should now be cached by daemon") // Test Routing.DelegatedRouters field expansion filters unsupported paths result = node.RunIPFS("config", "Routing.DelegatedRouters", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Routing.DelegatedRouters --expand-auto should succeed") var expandedRouters []string err := json.Unmarshal([]byte(result.Stdout.String()), &expandedRouters) require.NoError(t, err) // After cache prewarming, should get URLs from autoconf that have supported paths assert.Contains(t, expandedRouters, "https://supported.example.com/routing/v1/providers", "Should contain supported provider URL") assert.Contains(t, expandedRouters, "https://supported.example.com/routing/v1/peers", "Should contain supported peers URL") assert.Contains(t, expandedRouters, "https://mixed.example.com/routing/v1/providers", "Should contain mixed provider URL") assert.Contains(t, expandedRouters, "https://mixed.example.com/routing/v1/peers", "Should contain mixed peers URL") // Verify unsupported URLs from autoconf are filtered out (not in result) assert.NotContains(t, expandedRouters, "https://unsupported.example.com/example/v0/read", "Should filter out unsupported path /example/v0/read") assert.NotContains(t, expandedRouters, "https://unsupported.example.com/api/v1/custom", "Should filter out unsupported path /api/v1/custom") assert.NotContains(t, expandedRouters, "https://mixed.example.com/unsupported/path", "Should filter out unsupported path /unsupported/path") t.Logf("Filtered routers: %v", expandedRouters) // Test Ipns.DelegatedPublishers field expansion filters unsupported paths result = node.RunIPFS("config", "Ipns.DelegatedPublishers", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Ipns.DelegatedPublishers --expand-auto should succeed") var expandedPublishers []string err = json.Unmarshal([]byte(result.Stdout.String()), &expandedPublishers) require.NoError(t, err) // After cache prewarming, should get URLs from autoconf that have supported paths assert.Contains(t, expandedPublishers, "https://supported.example.com/routing/v1/ipns", "Should contain supported IPNS URL") assert.Contains(t, expandedPublishers, "https://mixed.example.com/routing/v1/ipns", "Should contain mixed IPNS URL") // Verify unsupported URLs from autoconf are filtered out (not in result) assert.NotContains(t, expandedPublishers, "https://unsupported.example.com/example/v0/write", "Should filter out unsupported write path") t.Logf("Filtered publishers: %v", expandedPublishers) } func testExpandAutoWithoutCacheDelegated(t *testing.T) { // Test scenario: CLI without daemon ever starting (no cached autoconf) using delegated routing // This tests the fallback scenario where delegated routing is configured but CLI commands // cannot read from cache and must fall back to hardcoded defaults // Create IPFS node but DO NOT start daemon node := harness.NewT(t).NewNode().Init("--profile=test") // Configure delegated routing to use autoconf URLs (but no daemon to fetch them) node.SetIPFSConfig("Routing.Type", "delegated") node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"}) node.SetIPFSConfig("Ipns.DelegatedPublishers", []string{"auto"}) // Disable content providing when using delegated routing node.SetIPFSConfig("Provide.Enabled", false) node.SetIPFSConfig("Provide.DHT.Interval", "0") // Load test autoconf data with unsupported paths (this won't be used since no daemon) autoConfData := loadTestDataExpand(t, "autoconf_with_unsupported_paths.json") // Create HTTP server that serves autoconf.json with unsupported paths server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(autoConfData) })) defer server.Close() // Configure autoconf for the node (but daemon never starts to fetch it) node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) // Test Routing.DelegatedRouters field expansion without cached autoconf result := node.RunIPFS("config", "Routing.DelegatedRouters", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Routing.DelegatedRouters --expand-auto should succeed") var expandedRouters []string err := json.Unmarshal([]byte(result.Stdout.String()), &expandedRouters) require.NoError(t, err) // Without cached autoconf, should get fallback URLs from GetMainnetFallbackConfig() // NOTE: These values may change if autoconf library updates GetMainnetFallbackConfig() assert.Contains(t, expandedRouters, "https://cid.contact/routing/v1/providers", "Should contain fallback provider URL from GetMainnetFallbackConfig()") t.Logf("Fallback routers (no cache): %v", expandedRouters) // Test Ipns.DelegatedPublishers field expansion without cached autoconf result = node.RunIPFS("config", "Ipns.DelegatedPublishers", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Ipns.DelegatedPublishers --expand-auto should succeed") var expandedPublishers []string err = json.Unmarshal([]byte(result.Stdout.String()), &expandedPublishers) require.NoError(t, err) // Without cached autoconf, should get fallback IPNS publishers from GetMainnetFallbackConfig() // NOTE: These values may change if autoconf library updates GetMainnetFallbackConfig() assert.Contains(t, expandedPublishers, "https://delegated-ipfs.dev/routing/v1/ipns", "Should contain fallback IPNS URL from GetMainnetFallbackConfig()") t.Logf("Fallback publishers (no cache): %v", expandedPublishers) } func testExpandAutoWithAutoRouting(t *testing.T) { // Test scenario: CLI with daemon started using auto routing with NewRoutingSystem // This tests that non-native systems (NewRoutingSystem) ARE delegated even with auto routing // Only native systems like AminoDHT are handled internally with auto routing // Create IPFS node node := harness.NewT(t).NewNode().Init("--profile=test") // Configure auto routing with non-native system node.SetIPFSConfig("Routing.Type", "auto") node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"}) node.SetIPFSConfig("Ipns.DelegatedPublishers", []string{"auto"}) // Load test autoconf data with NewRoutingSystem (non-native, will be delegated) autoConfData := loadTestDataExpand(t, "autoconf_new_routing_system.json") // Create HTTP server that serves autoconf.json server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(autoConfData) })) defer server.Close() // Configure autoconf for the node node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) // Start daemon to fetch and cache autoconf data t.Log("Starting daemon to fetch and cache autoconf data...") daemon := node.StartDaemon() defer daemon.StopDaemon() // Wait for autoconf fetch (use autoconf default timeout + buffer) time.Sleep(6 * time.Second) // defaultTimeout is 5s, add 1s buffer t.Log("AutoConf should now be cached by daemon") // Test Routing.DelegatedRouters field expansion with auto routing result := node.RunIPFS("config", "Routing.DelegatedRouters", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Routing.DelegatedRouters --expand-auto should succeed") var expandedRouters []string err := json.Unmarshal([]byte(result.Stdout.String()), &expandedRouters) require.NoError(t, err) // With auto routing and NewRoutingSystem (non-native), delegated endpoints should be populated assert.Contains(t, expandedRouters, "https://new-routing.example.com/routing/v1/providers", "Should contain NewRoutingSystem provider URL") assert.Contains(t, expandedRouters, "https://new-routing.example.com/routing/v1/peers", "Should contain NewRoutingSystem peers URL") t.Logf("Auto routing routers (NewRoutingSystem delegated): %v", expandedRouters) // Test Ipns.DelegatedPublishers field expansion with auto routing result = node.RunIPFS("config", "Ipns.DelegatedPublishers", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Ipns.DelegatedPublishers --expand-auto should succeed") var expandedPublishers []string err = json.Unmarshal([]byte(result.Stdout.String()), &expandedPublishers) require.NoError(t, err) // With auto routing and NewRoutingSystem (non-native), delegated publishers should be populated assert.Contains(t, expandedPublishers, "https://new-routing.example.com/routing/v1/ipns", "Should contain NewRoutingSystem IPNS URL") t.Logf("Auto routing publishers (NewRoutingSystem delegated): %v", expandedPublishers) } func testExpandAutoWithMixedSystems(t *testing.T) { // Test scenario: Auto routing with both AminoDHT (native) and IPNI (delegated) systems // This explicitly confirms that AminoDHT is NOT delegated but IPNI at cid.contact IS delegated // Create IPFS node node := harness.NewT(t).NewNode().Init("--profile=test") // Configure auto routing node.SetIPFSConfig("Routing.Type", "auto") node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"}) node.SetIPFSConfig("Ipns.DelegatedPublishers", []string{"auto"}) // Load test autoconf data with both AminoDHT and IPNI systems autoConfData := loadTestDataExpand(t, "autoconf_amino_and_ipni.json") // Create HTTP server that serves autoconf.json server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(autoConfData) })) defer server.Close() // Configure autoconf for the node node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) // Start daemon to fetch and cache autoconf data t.Log("Starting daemon to fetch and cache autoconf data...") daemon := node.StartDaemon() defer daemon.StopDaemon() // Wait for autoconf fetch (use autoconf default timeout + buffer) time.Sleep(6 * time.Second) // defaultTimeout is 5s, add 1s buffer t.Log("AutoConf should now be cached by daemon") // Test Routing.DelegatedRouters field expansion result := node.RunIPFS("config", "Routing.DelegatedRouters", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Routing.DelegatedRouters --expand-auto should succeed") var expandedRouters []string err := json.Unmarshal([]byte(result.Stdout.String()), &expandedRouters) require.NoError(t, err) // With auto routing: AminoDHT (native) should NOT be delegated, IPNI should be delegated assert.Contains(t, expandedRouters, "https://cid.contact/routing/v1/providers", "Should contain IPNI provider URL (delegated)") assert.NotContains(t, expandedRouters, "https://amino-dht.example.com", "Should NOT contain AminoDHT URLs (native)") t.Logf("Mixed systems routers (IPNI delegated, AminoDHT native): %v", expandedRouters) // Test Ipns.DelegatedPublishers field expansion result = node.RunIPFS("config", "Ipns.DelegatedPublishers", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Ipns.DelegatedPublishers --expand-auto should succeed") var expandedPublishers []string err = json.Unmarshal([]byte(result.Stdout.String()), &expandedPublishers) require.NoError(t, err) // IPNI system doesn't have write endpoints, so publishers should be empty // (or contain other systems if they have write endpoints) t.Logf("Mixed systems publishers (IPNI has no write endpoints): %v", expandedPublishers) } func testExpandAutoWithFiltering(t *testing.T) { // Test scenario: Auto routing with NewRoutingSystem and path filtering // This tests that path filtering works for delegated systems even with auto routing // Create IPFS node node := harness.NewT(t).NewNode().Init("--profile=test") // Configure auto routing node.SetIPFSConfig("Routing.Type", "auto") node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"}) node.SetIPFSConfig("Ipns.DelegatedPublishers", []string{"auto"}) // Load test autoconf data with NewRoutingSystem and mixed valid/invalid paths autoConfData := loadTestDataExpand(t, "autoconf_new_routing_with_filtering.json") // Create HTTP server that serves autoconf.json server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(autoConfData) })) defer server.Close() // Configure autoconf for the node node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) // Start daemon to fetch and cache autoconf data t.Log("Starting daemon to fetch and cache autoconf data...") daemon := node.StartDaemon() defer daemon.StopDaemon() // Wait for autoconf fetch (use autoconf default timeout + buffer) time.Sleep(6 * time.Second) // defaultTimeout is 5s, add 1s buffer t.Log("AutoConf should now be cached by daemon") // Test Routing.DelegatedRouters field expansion with filtering result := node.RunIPFS("config", "Routing.DelegatedRouters", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Routing.DelegatedRouters --expand-auto should succeed") var expandedRouters []string err := json.Unmarshal([]byte(result.Stdout.String()), &expandedRouters) require.NoError(t, err) // Should contain supported paths from NewRoutingSystem assert.Contains(t, expandedRouters, "https://supported-new.example.com/routing/v1/providers", "Should contain supported provider URL") assert.Contains(t, expandedRouters, "https://supported-new.example.com/routing/v1/peers", "Should contain supported peers URL") assert.Contains(t, expandedRouters, "https://mixed-new.example.com/routing/v1/providers", "Should contain mixed provider URL") assert.Contains(t, expandedRouters, "https://mixed-new.example.com/routing/v1/peers", "Should contain mixed peers URL") // Should NOT contain unsupported paths assert.NotContains(t, expandedRouters, "https://unsupported-new.example.com/custom/v0/read", "Should filter out unsupported path") assert.NotContains(t, expandedRouters, "https://unsupported-new.example.com/api/v1/nonstandard", "Should filter out unsupported path") assert.NotContains(t, expandedRouters, "https://mixed-new.example.com/invalid/path", "Should filter out invalid path from mixed endpoint") t.Logf("Filtered routers (NewRoutingSystem with auto routing): %v", expandedRouters) // Test Ipns.DelegatedPublishers field expansion with filtering result = node.RunIPFS("config", "Ipns.DelegatedPublishers", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Ipns.DelegatedPublishers --expand-auto should succeed") var expandedPublishers []string err = json.Unmarshal([]byte(result.Stdout.String()), &expandedPublishers) require.NoError(t, err) // Should contain supported IPNS paths assert.Contains(t, expandedPublishers, "https://supported-new.example.com/routing/v1/ipns", "Should contain supported IPNS URL") assert.Contains(t, expandedPublishers, "https://mixed-new.example.com/routing/v1/ipns", "Should contain mixed IPNS URL") // Should NOT contain unsupported write paths assert.NotContains(t, expandedPublishers, "https://unsupported-new.example.com/custom/v0/write", "Should filter out unsupported write path") t.Logf("Filtered publishers (NewRoutingSystem with auto routing): %v", expandedPublishers) } func testExpandAutoWithoutCacheAuto(t *testing.T) { // Test scenario: CLI without daemon ever starting using auto routing (default) // This tests the fallback scenario where auto routing is used but doesn't populate delegated config fields // Create IPFS node but DO NOT start daemon node := harness.NewT(t).NewNode().Init("--profile=test") // Configure auto routing - delegated fields are set to "auto" but won't be populated // because auto routing uses different internal mechanisms node.SetIPFSConfig("Routing.Type", "auto") node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"}) node.SetIPFSConfig("Ipns.DelegatedPublishers", []string{"auto"}) // Load test autoconf data (this won't be used since no daemon and auto routing doesn't use these fields) autoConfData := loadTestDataExpand(t, "autoconf_with_unsupported_paths.json") // Create HTTP server (won't be contacted since no daemon) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(autoConfData) })) defer server.Close() // Configure autoconf for the node (but daemon never starts to fetch it) node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) // Test Routing.DelegatedRouters field expansion without cached autoconf result := node.RunIPFS("config", "Routing.DelegatedRouters", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Routing.DelegatedRouters --expand-auto should succeed") var expandedRouters []string err := json.Unmarshal([]byte(result.Stdout.String()), &expandedRouters) require.NoError(t, err) // With auto routing, some fallback URLs are still populated from GetMainnetFallbackConfig() // NOTE: These values may change if autoconf library updates GetMainnetFallbackConfig() assert.Contains(t, expandedRouters, "https://cid.contact/routing/v1/providers", "Should contain fallback provider URL from GetMainnetFallbackConfig()") t.Logf("Auto routing fallback routers (with fallbacks): %v", expandedRouters) // Test Ipns.DelegatedPublishers field expansion without cached autoconf result = node.RunIPFS("config", "Ipns.DelegatedPublishers", "--expand-auto") require.Equal(t, 0, result.ExitCode(), "config Ipns.DelegatedPublishers --expand-auto should succeed") var expandedPublishers []string err = json.Unmarshal([]byte(result.Stdout.String()), &expandedPublishers) require.NoError(t, err) // With auto routing, delegated publishers may be empty for fallback scenario // This can vary based on which systems have write endpoints in the fallback config t.Logf("Auto routing fallback publishers: %v", expandedPublishers) } // Helper function to load test data files func loadTestDataExpand(t *testing.T, filename string) []byte { t.Helper() data, err := os.ReadFile("testdata/" + filename) require.NoError(t, err, "Failed to read test data file: %s", filename) return data } ================================================ FILE: test/cli/autoconf/extensibility_test.go ================================================ package autoconf import ( "encoding/json" "net/http" "net/http/httptest" "slices" "strings" "testing" "time" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/require" ) // TestAutoConfExtensibility_NewSystem verifies that the AutoConf system can be extended // with new routing systems beyond the default AminoDHT and IPNI. // // The test verifies that: // 1. New systems can be added via AutoConf's SystemRegistry // 2. Native vs delegated system filtering works correctly: // - Native systems (AminoDHT) provide bootstrap peers and are used for P2P routing // - Delegated systems (IPNI, NewSystem) provide HTTP endpoints for delegated routing // // 3. The system correctly filters endpoints based on routing type // // Note: Only native systems contribute bootstrap peers. Delegated systems like "NewSystem" // only provide HTTP routing endpoints, not P2P bootstrap peers. func TestAutoConfExtensibility_NewSystem(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode") } // Setup mock autoconf server with NewSystem var mockServer *httptest.Server mockServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Create autoconf.json with NewSystem autoconfData := map[string]any{ "AutoConfVersion": 2025072901, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": map[string]any{ "AminoDHT": map[string]any{ "URL": "https://github.com/ipfs/specs/pull/497", "Description": "Public DHT swarm", "NativeConfig": map[string]any{ "Bootstrap": []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", }, }, "DelegatedConfig": map[string]any{ "Read": []string{"/routing/v1/providers", "/routing/v1/peers", "/routing/v1/ipns"}, "Write": []string{"/routing/v1/ipns"}, }, }, "IPNI": map[string]any{ "URL": "https://ipni.example.com", "Description": "Network Indexer", "DelegatedConfig": map[string]any{ "Read": []string{"/routing/v1/providers"}, "Write": []string{}, }, }, "NewSystem": map[string]any{ "URL": "https://example.com/newsystem", "Description": "Test system for extensibility verification", "NativeConfig": map[string]any{ "Bootstrap": []string{ "/ip4/127.0.0.1/tcp/9999/p2p/12D3KooWPeQ4r3v6CmVmKXoFGtqEqcr3L8P6La9yH5oEWKtoLVVa", }, }, "DelegatedConfig": map[string]any{ "Read": []string{"/routing/v1/providers"}, "Write": []string{}, }, }, }, "DNSResolvers": map[string]any{ "eth.": []string{"https://dns.eth.limo/dns-query"}, }, "DelegatedEndpoints": map[string]any{ "https://ipni.example.com": map[string]any{ "Systems": []string{"IPNI"}, "Read": []string{"/routing/v1/providers"}, "Write": []string{}, }, mockServer.URL + "/newsystem": map[string]any{ "Systems": []string{"NewSystem"}, "Read": []string{"/routing/v1/providers"}, "Write": []string{}, }, }, } w.Header().Set("Content-Type", "application/json") w.Header().Set("Cache-Control", "max-age=300") _ = json.NewEncoder(w).Encode(autoconfData) })) defer mockServer.Close() // NewSystem mock server URL will be dynamically assigned newSystemServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Simple mock server for NewSystem endpoint response := map[string]any{"Providers": []any{}} w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(response) })) defer newSystemServer.Close() // Update the autoconf to point to the correct NewSystem endpoint mockServer.Close() mockServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { autoconfData := map[string]any{ "AutoConfVersion": 2025072901, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": map[string]any{ "AminoDHT": map[string]any{ "URL": "https://github.com/ipfs/specs/pull/497", "Description": "Public DHT swarm", "NativeConfig": map[string]any{ "Bootstrap": []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", }, }, "DelegatedConfig": map[string]any{ "Read": []string{"/routing/v1/providers", "/routing/v1/peers", "/routing/v1/ipns"}, "Write": []string{"/routing/v1/ipns"}, }, }, "IPNI": map[string]any{ "URL": "https://ipni.example.com", "Description": "Network Indexer", "DelegatedConfig": map[string]any{ "Read": []string{"/routing/v1/providers"}, "Write": []string{}, }, }, "NewSystem": map[string]any{ "URL": "https://example.com/newsystem", "Description": "Test system for extensibility verification", "NativeConfig": map[string]any{ "Bootstrap": []string{ "/ip4/127.0.0.1/tcp/9999/p2p/12D3KooWPeQ4r3v6CmVmKXoFGtqEqcr3L8P6La9yH5oEWKtoLVVa", }, }, "DelegatedConfig": map[string]any{ "Read": []string{"/routing/v1/providers"}, "Write": []string{}, }, }, }, "DNSResolvers": map[string]any{ "eth.": []string{"https://dns.eth.limo/dns-query"}, }, "DelegatedEndpoints": map[string]any{ "https://ipni.example.com": map[string]any{ "Systems": []string{"IPNI"}, "Read": []string{"/routing/v1/providers"}, "Write": []string{}, }, newSystemServer.URL: map[string]any{ "Systems": []string{"NewSystem"}, "Read": []string{"/routing/v1/providers"}, "Write": []string{}, }, }, } w.Header().Set("Content-Type", "application/json") w.Header().Set("Cache-Control", "max-age=300") _ = json.NewEncoder(w).Encode(autoconfData) })) defer mockServer.Close() // Create Kubo node with autoconf pointing to mock server h := harness.NewT(t) node := h.NewNode().Init() // Update config to use mock autoconf server node.UpdateConfig(func(cfg *config.Config) { cfg.AutoConf.URL = config.NewOptionalString(mockServer.URL) cfg.AutoConf.Enabled = config.True cfg.AutoConf.RefreshInterval = config.NewOptionalDuration(1 * time.Second) cfg.Routing.Type = config.NewOptionalString("auto") // Should enable native AminoDHT + delegated others cfg.Bootstrap = []string{"auto"} cfg.Routing.DelegatedRouters = []string{"auto"} }) // Start the daemon daemon := node.StartDaemon() defer daemon.StopDaemon() // Give the daemon some time to initialize and make requests time.Sleep(3 * time.Second) // Test 1: Verify bootstrap includes both AminoDHT and NewSystem peers (deduplicated) bootstrapResult := daemon.IPFS("bootstrap", "list", "--expand-auto") bootstrapOutput := bootstrapResult.Stdout.String() t.Logf("Bootstrap output: %s", bootstrapOutput) // Should contain original DHT bootstrap peer (AminoDHT is a native system) require.Contains(t, bootstrapOutput, "QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "Should contain AminoDHT bootstrap peer") // Note: NewSystem bootstrap peers are NOT included because only native systems // (AminoDHT for Routing.Type="auto") contribute bootstrap peers. // Delegated systems like NewSystem only provide HTTP routing endpoints. // Test 2: Verify delegated endpoints are filtered correctly // For Routing.Type=auto, native systems=[AminoDHT], so: // - AminoDHT endpoints should be filtered out // - IPNI and NewSystem endpoints should be included // Get the expanded delegated routers using --expand-auto routerResult := daemon.IPFS("config", "Routing.DelegatedRouters", "--expand-auto") var expandedRouters []string require.NoError(t, json.Unmarshal([]byte(routerResult.Stdout.String()), &expandedRouters)) t.Logf("Expanded delegated routers: %v", expandedRouters) // Verify we got exactly 2 delegated routers: IPNI and NewSystem require.Equal(t, 2, len(expandedRouters), "Should have exactly 2 delegated routers (IPNI and NewSystem). Got %d: %v", len(expandedRouters), expandedRouters) // Convert to URLs for checking routerURLs := expandedRouters // Should contain NewSystem endpoint (not native) - now with routing path foundNewSystem := false expectedNewSystemURL := newSystemServer.URL + "/routing/v1/providers" // Full URL with path, as returned by DelegatedRoutersWithAutoConf if slices.Contains(routerURLs, expectedNewSystemURL) { foundNewSystem = true } require.True(t, foundNewSystem, "Should contain NewSystem endpoint (%s) for delegated routing, got: %v", expectedNewSystemURL, routerURLs) // Should contain ipni.example.com (IPNI is not native) foundIPNI := false for _, url := range routerURLs { if strings.Contains(url, "ipni.example.com") { foundIPNI = true break } } require.True(t, foundIPNI, "Should contain ipni.example.com endpoint for IPNI") // Test passes - we've verified that: // 1. Bootstrap peers are correctly resolved from native systems only // 2. Delegated routers include both IPNI and NewSystem endpoints // 3. URL format is correct (base URLs with paths) // 4. AutoConf extensibility works for unknown systems t.Log("NewSystem extensibility test passed - Kubo successfully discovered and used unknown routing system") } ================================================ FILE: test/cli/autoconf/fuzz_test.go ================================================ package autoconf import ( "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "strings" "testing" "time" "github.com/ipfs/boxo/autoconf" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // testAutoConfWithFallback is a helper function that tests autoconf parsing with fallback detection func testAutoConfWithFallback(t *testing.T, serverURL string, expectError bool, expectErrorMsg string) (*autoconf.Config, bool) { return testAutoConfWithFallbackAndTimeout(t, serverURL, expectError, expectErrorMsg, 10*time.Second) } // testAutoConfWithFallbackAndTimeout is a helper function that tests autoconf parsing with fallback detection and custom timeout func testAutoConfWithFallbackAndTimeout(t *testing.T, serverURL string, expectError bool, expectErrorMsg string, timeout time.Duration) (*autoconf.Config, bool) { // Use fallback detection to test error conditions with MustGetConfigWithRefresh fallbackUsed := false fallbackConfig := &autoconf.Config{ AutoConfVersion: -999, // Special marker to detect fallback usage AutoConfSchema: -999, } client, err := autoconf.NewClient( autoconf.WithUserAgent("test-agent"), autoconf.WithURL(serverURL), autoconf.WithRefreshInterval(autoconf.DefaultRefreshInterval), autoconf.WithFallback(func() *autoconf.Config { fallbackUsed = true return fallbackConfig }), ) require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() result := client.GetCachedOrRefresh(ctx) if expectError { require.True(t, fallbackUsed, expectErrorMsg) require.Equal(t, int64(-999), result.AutoConfVersion, "Should return fallback config for error case") } else { require.False(t, fallbackUsed, "Expected no fallback to be used") require.NotEqual(t, int64(-999), result.AutoConfVersion, "Should return fetched config for success case") } return result, fallbackUsed } func TestAutoConfFuzz(t *testing.T) { t.Parallel() t.Run("fuzz autoconf version", testFuzzAutoConfVersion) t.Run("fuzz bootstrap arrays", testFuzzBootstrapArrays) t.Run("fuzz dns resolvers", testFuzzDNSResolvers) t.Run("fuzz delegated routers", testFuzzDelegatedRouters) t.Run("fuzz delegated publishers", testFuzzDelegatedPublishers) t.Run("fuzz malformed json", testFuzzMalformedJSON) t.Run("fuzz large payloads", testFuzzLargePayloads) } func testFuzzAutoConfVersion(t *testing.T) { testCases := []struct { name string version any expectError bool }{ {"valid version", 2025071801, false}, {"zero version", 0, true}, // Should be invalid {"negative version", -1, false}, // Parser accepts negative versions {"string version", "2025071801", true}, // Should be number {"float version", 2025071801.5, true}, {"very large version", 9999999999999999, false}, // Large but valid int64 {"null version", nil, true}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { config := map[string]any{ "AutoConfVersion": tc.version, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": map[string]any{ "AminoDHT": map[string]any{ "Description": "Test AminoDHT system", "NativeConfig": map[string]any{ "Bootstrap": []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", }, }, }, }, "DNSResolvers": map[string]any{}, "DelegatedEndpoints": map[string]any{}, } jsonData, err := json.Marshal(config) require.NoError(t, err) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(jsonData) })) defer server.Close() // Test that our autoconf parser handles this gracefully _, _ = testAutoConfWithFallback(t, server.URL, tc.expectError, fmt.Sprintf("Expected fallback to be used for %s", tc.name)) }) } } func testFuzzBootstrapArrays(t *testing.T) { type testCase struct { name string bootstrap any expectError bool validate func(*testing.T, *autoconf.Response) } testCases := []testCase{ { name: "valid bootstrap", bootstrap: []string{"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN"}, validate: func(t *testing.T, resp *autoconf.Response) { expected := []string{"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN"} bootstrapPeers := resp.Config.GetBootstrapPeers("AminoDHT") assert.Equal(t, expected, bootstrapPeers, "Bootstrap peers should match configured values") }, }, { name: "empty bootstrap", bootstrap: []string{}, validate: func(t *testing.T, resp *autoconf.Response) { bootstrapPeers := resp.Config.GetBootstrapPeers("AminoDHT") assert.Empty(t, bootstrapPeers, "Empty bootstrap should result in empty peers") }, }, { name: "null bootstrap", bootstrap: nil, validate: func(t *testing.T, resp *autoconf.Response) { bootstrapPeers := resp.Config.GetBootstrapPeers("AminoDHT") assert.Empty(t, bootstrapPeers, "Null bootstrap should result in empty peers") }, }, { name: "invalid multiaddr", bootstrap: []string{"invalid-multiaddr"}, expectError: true, }, { name: "very long multiaddr", bootstrap: []string{"/dnsaddr/" + strings.Repeat("a", 100) + ".com/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN"}, validate: func(t *testing.T, resp *autoconf.Response) { expected := []string{"/dnsaddr/" + strings.Repeat("a", 100) + ".com/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN"} bootstrapPeers := resp.Config.GetBootstrapPeers("AminoDHT") assert.Equal(t, expected, bootstrapPeers, "Very long multiaddr should be preserved") }, }, { name: "bootstrap as string", bootstrap: "/dnsaddr/test", expectError: true, }, { name: "bootstrap as number", bootstrap: 123, expectError: true, }, { name: "mixed types in array", bootstrap: []any{"/dnsaddr/test", 123, nil}, expectError: true, }, { name: "extremely large array", bootstrap: make([]string, 1000), validate: func(t *testing.T, resp *autoconf.Response) { // Array will be filled in the loop below bootstrapPeers := resp.Config.GetBootstrapPeers("AminoDHT") assert.Len(t, bootstrapPeers, 1000, "Large bootstrap array should be preserved") }, }, } // Fill the large array with valid multiaddrs largeArray := testCases[len(testCases)-1].bootstrap.([]string) for i := range largeArray { largeArray[i] = fmt.Sprintf("/dnsaddr/bootstrap%d.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", i) } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { config := map[string]any{ "AutoConfVersion": 2025072301, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": map[string]any{ "AminoDHT": map[string]any{ "Description": "Test AminoDHT system", "NativeConfig": map[string]any{ "Bootstrap": tc.bootstrap, }, }, }, "DNSResolvers": map[string]any{}, "DelegatedEndpoints": map[string]any{}, } jsonData, err := json.Marshal(config) require.NoError(t, err) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(jsonData) })) defer server.Close() autoConf, fallbackUsed := testAutoConfWithFallback(t, server.URL, tc.expectError, fmt.Sprintf("Expected fallback to be used for %s", tc.name)) if !tc.expectError { require.NotNil(t, autoConf, "AutoConf should not be nil for successful parsing") // Verify structure is reasonable bootstrapPeers := autoConf.GetBootstrapPeers("AminoDHT") require.IsType(t, []string{}, bootstrapPeers, "Bootstrap should be []string") // Run test-specific validation if provided (only for non-fallback cases) if tc.validate != nil && !fallbackUsed { // Create a mock Response for compatibility with validation functions mockResponse := &autoconf.Response{Config: autoConf} tc.validate(t, mockResponse) } } }) } } func testFuzzDNSResolvers(t *testing.T) { type testCase struct { name string resolvers any expectError bool validate func(*testing.T, *autoconf.Response) } testCases := []testCase{ { name: "valid resolvers", resolvers: map[string][]string{".": {"https://dns.google/dns-query"}}, validate: func(t *testing.T, resp *autoconf.Response) { expected := map[string][]string{".": {"https://dns.google/dns-query"}} assert.Equal(t, expected, resp.Config.DNSResolvers, "DNS resolvers should match configured values") }, }, { name: "empty resolvers", resolvers: map[string][]string{}, validate: func(t *testing.T, resp *autoconf.Response) { assert.Empty(t, resp.Config.DNSResolvers, "Empty resolvers should result in empty map") }, }, { name: "null resolvers", resolvers: nil, validate: func(t *testing.T, resp *autoconf.Response) { assert.Empty(t, resp.Config.DNSResolvers, "Null resolvers should result in empty map") }, }, { name: "relative URL (missing scheme)", resolvers: map[string][]string{".": {"not-a-url"}}, expectError: true, // Should error due to strict HTTP/HTTPS validation }, { name: "invalid URL format", resolvers: map[string][]string{".": {"://invalid-missing-scheme"}}, expectError: true, // Should error because url.Parse() fails }, { name: "non-HTTP scheme", resolvers: map[string][]string{".": {"ftp://example.com/dns-query"}}, expectError: true, // Should error due to non-HTTP/HTTPS scheme }, { name: "very long domain", resolvers: map[string][]string{strings.Repeat("a", 1000) + ".com": {"https://dns.google/dns-query"}}, validate: func(t *testing.T, resp *autoconf.Response) { expected := map[string][]string{strings.Repeat("a", 1000) + ".com": {"https://dns.google/dns-query"}} assert.Equal(t, expected, resp.Config.DNSResolvers, "Very long domain should be preserved") }, }, { name: "many resolvers", resolvers: generateManyResolvers(100), validate: func(t *testing.T, resp *autoconf.Response) { expected := generateManyResolvers(100) assert.Equal(t, expected, resp.Config.DNSResolvers, "Many resolvers should be preserved") assert.Equal(t, 100, len(resp.Config.DNSResolvers), "Should have 100 resolvers") }, }, { name: "resolvers as array", resolvers: []string{"https://dns.google/dns-query"}, expectError: true, }, { name: "nested invalid structure", resolvers: map[string]any{".": map[string]string{"invalid": "structure"}}, expectError: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { config := map[string]any{ "AutoConfVersion": 2025072301, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": map[string]any{ "AminoDHT": map[string]any{ "Description": "Test AminoDHT system", "NativeConfig": map[string]any{ "Bootstrap": []string{"/dnsaddr/test"}, }, }, }, "DNSResolvers": tc.resolvers, "DelegatedEndpoints": map[string]any{}, } jsonData, err := json.Marshal(config) require.NoError(t, err) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(jsonData) })) defer server.Close() autoConf, fallbackUsed := testAutoConfWithFallback(t, server.URL, tc.expectError, fmt.Sprintf("Expected fallback to be used for %s", tc.name)) if !tc.expectError { require.NotNil(t, autoConf, "AutoConf should not be nil for successful parsing") // Run test-specific validation if provided (only for non-fallback cases) if tc.validate != nil && !fallbackUsed { // Create a mock Response for compatibility with validation functions mockResponse := &autoconf.Response{Config: autoConf} tc.validate(t, mockResponse) } } }) } } func testFuzzDelegatedRouters(t *testing.T) { // Test various malformed delegated router configurations type testCase struct { name string routers any expectError bool validate func(*testing.T, *autoconf.Response) } testCases := []testCase{ { name: "valid endpoints", routers: map[string]any{ "https://ipni.example.com": map[string]any{ "Systems": []string{"IPNI"}, "Read": []string{"/routing/v1/providers"}, "Write": []string{}, }, }, validate: func(t *testing.T, resp *autoconf.Response) { assert.Len(t, resp.Config.DelegatedEndpoints, 1, "Should have 1 delegated endpoint") for url, config := range resp.Config.DelegatedEndpoints { assert.Contains(t, url, "ipni.example.com", "Endpoint URL should contain expected domain") assert.Contains(t, config.Systems, "IPNI", "Endpoint should have IPNI system") assert.Contains(t, config.Read, "/routing/v1/providers", "Endpoint should have providers read path") } }, }, { name: "empty routers", routers: map[string]any{}, validate: func(t *testing.T, resp *autoconf.Response) { assert.Empty(t, resp.Config.DelegatedEndpoints, "Empty routers should result in empty endpoints") }, }, { name: "null routers", routers: nil, validate: func(t *testing.T, resp *autoconf.Response) { assert.Empty(t, resp.Config.DelegatedEndpoints, "Null routers should result in empty endpoints") }, }, { name: "invalid nested structure", routers: map[string]string{"invalid": "structure"}, expectError: true, }, { name: "invalid endpoint URLs", routers: map[string]any{ "not-a-url": map[string]any{ "Systems": []string{"IPNI"}, "Read": []string{"/routing/v1/providers"}, "Write": []string{}, }, }, expectError: true, // Should error due to URL validation }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { config := map[string]any{ "AutoConfVersion": 2025072301, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": map[string]any{ "AminoDHT": map[string]any{ "Description": "Test AminoDHT system", "NativeConfig": map[string]any{ "Bootstrap": []string{"/dnsaddr/test"}, }, }, }, "DNSResolvers": map[string]any{}, "DelegatedEndpoints": tc.routers, } jsonData, err := json.Marshal(config) require.NoError(t, err) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(jsonData) })) defer server.Close() autoConf, fallbackUsed := testAutoConfWithFallback(t, server.URL, tc.expectError, fmt.Sprintf("Expected fallback to be used for %s", tc.name)) if !tc.expectError { require.NotNil(t, autoConf, "AutoConf should not be nil for successful parsing") // Run test-specific validation if provided (only for non-fallback cases) if tc.validate != nil && !fallbackUsed { // Create a mock Response for compatibility with validation functions mockResponse := &autoconf.Response{Config: autoConf} tc.validate(t, mockResponse) } } }) } } func testFuzzDelegatedPublishers(t *testing.T) { // DelegatedPublishers use the same autoclient library validation as DelegatedRouters // Test that URL validation works for delegated publishers type testCase struct { name string urls []string expectErr bool validate func(*testing.T, *autoconf.Response) } testCases := []testCase{ { name: "valid HTTPS URLs", urls: []string{"https://delegated-ipfs.dev", "https://another-publisher.com"}, validate: func(t *testing.T, resp *autoconf.Response) { assert.Len(t, resp.Config.DelegatedEndpoints, 2, "Should have 2 delegated endpoints") foundURLs := make([]string, 0, len(resp.Config.DelegatedEndpoints)) for url := range resp.Config.DelegatedEndpoints { foundURLs = append(foundURLs, url) } expectedURLs := []string{"https://delegated-ipfs.dev", "https://another-publisher.com"} for _, expectedURL := range expectedURLs { assert.Contains(t, foundURLs, expectedURL, "Should contain configured URL: %s", expectedURL) } }, }, { name: "invalid URL", urls: []string{"not-a-url"}, expectErr: true, }, { name: "HTTP URL (accepted during parsing)", urls: []string{"http://insecure-publisher.com"}, validate: func(t *testing.T, resp *autoconf.Response) { assert.Len(t, resp.Config.DelegatedEndpoints, 1, "Should have 1 delegated endpoint") for url := range resp.Config.DelegatedEndpoints { assert.Equal(t, "http://insecure-publisher.com", url, "HTTP URL should be preserved during parsing") } }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { autoConfData := map[string]any{ "AutoConfVersion": 2025072301, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": map[string]any{ "TestSystem": map[string]any{ "Description": "Test system for fuzz testing", "DelegatedConfig": map[string]any{ "Read": []string{"/routing/v1/ipns"}, "Write": []string{"/routing/v1/ipns"}, }, }, }, "DNSResolvers": map[string]any{}, "DelegatedEndpoints": map[string]any{}, } // Add test URLs as delegated endpoints for _, url := range tc.urls { autoConfData["DelegatedEndpoints"].(map[string]any)[url] = map[string]any{ "Systems": []string{"TestSystem"}, "Read": []string{"/routing/v1/ipns"}, "Write": []string{"/routing/v1/ipns"}, } } jsonData, err := json.Marshal(autoConfData) require.NoError(t, err) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(jsonData) })) defer server.Close() // Test that our autoconf parser handles this gracefully autoConf, fallbackUsed := testAutoConfWithFallback(t, server.URL, tc.expectErr, fmt.Sprintf("Expected fallback to be used for %s", tc.name)) if !tc.expectErr { require.NotNil(t, autoConf, "AutoConf should not be nil for successful parsing") // Run test-specific validation if provided (only for non-fallback cases) if tc.validate != nil && !fallbackUsed { // Create a mock Response for compatibility with validation functions mockResponse := &autoconf.Response{Config: autoConf} tc.validate(t, mockResponse) } } }) } } func testFuzzMalformedJSON(t *testing.T) { malformedJSONs := []string{ `{`, // Incomplete JSON `{"AutoConfVersion": }`, // Missing value `{"AutoConfVersion": 123,}`, // Trailing comma `{AutoConfVersion: 123}`, // Unquoted key `{"Bootstrap": [}`, // Incomplete array `{"Bootstrap": ["/test",]}`, // Trailing comma in array `invalid json`, // Not JSON at all `null`, // Just null `[]`, // Array instead of object `""`, // String instead of object } for i, malformedJSON := range malformedJSONs { t.Run(fmt.Sprintf("malformed_%d", i), func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(malformedJSON)) })) defer server.Close() // All malformed JSON should result in fallback usage _, _ = testAutoConfWithFallback(t, server.URL, true, fmt.Sprintf("Expected fallback to be used for malformed JSON: %s", malformedJSON)) }) } } func testFuzzLargePayloads(t *testing.T) { // Test with very large but valid JSON payloads largeBootstrap := make([]string, 10000) for i := range largeBootstrap { largeBootstrap[i] = fmt.Sprintf("/dnsaddr/bootstrap%d.example.com/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", i) } largeDNSResolvers := make(map[string][]string) for i := range 1000 { domain := fmt.Sprintf("domain%d.example.com", i) largeDNSResolvers[domain] = []string{ fmt.Sprintf("https://resolver%d.example.com/dns-query", i), } } config := map[string]any{ "AutoConfVersion": 2025072301, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": map[string]any{ "AminoDHT": map[string]any{ "Description": "Test AminoDHT system", "NativeConfig": map[string]any{ "Bootstrap": largeBootstrap, }, }, }, "DNSResolvers": largeDNSResolvers, "DelegatedEndpoints": map[string]any{}, } jsonData, err := json.Marshal(config) require.NoError(t, err) t.Logf("Large payload size: %d bytes", len(jsonData)) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write(jsonData) })) defer server.Close() // Should handle large payloads gracefully (up to reasonable limits) autoConf, _ := testAutoConfWithFallbackAndTimeout(t, server.URL, false, "Large payload should not trigger fallback", 30*time.Second) require.NotNil(t, autoConf, "Should return valid config") // Verify bootstrap entries were preserved bootstrapPeers := autoConf.GetBootstrapPeers("AminoDHT") require.Len(t, bootstrapPeers, 10000, "Should preserve all bootstrap entries") } // Helper function to generate many DNS resolvers for testing func generateManyResolvers(count int) map[string][]string { resolvers := make(map[string][]string) for i := range count { domain := fmt.Sprintf("domain%d.example.com", i) resolvers[domain] = []string{ fmt.Sprintf("https://resolver%d.example.com/dns-query", i), } } return resolvers } ================================================ FILE: test/cli/autoconf/ipns_test.go ================================================ package autoconf import ( "encoding/json" "fmt" "io" "maps" "net/http" "net/http/httptest" "strings" "sync" "testing" "time" "github.com/ipfs/boxo/autoconf" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // TestAutoConfIPNS tests IPNS publishing with autoconf-resolved delegated publishers func TestAutoConfIPNS(t *testing.T) { t.Parallel() t.Run("PublishingWithWorkingEndpoint", func(t *testing.T) { t.Parallel() testIPNSPublishingWithWorkingEndpoint(t) }) t.Run("PublishingResilience", func(t *testing.T) { t.Parallel() testIPNSPublishingResilience(t) }) } // testIPNSPublishingWithWorkingEndpoint verifies that IPNS delegated publishing works // correctly when the HTTP endpoint is functioning normally and accepts requests. // It also verifies that the PUT payload matches what can be retrieved via routing get. func testIPNSPublishingWithWorkingEndpoint(t *testing.T) { // Create mock IPNS publisher that accepts requests publisher := newMockIPNSPublisher(t) defer publisher.close() // Create node with delegated publisher node := setupNodeWithAutoconf(t, publisher.server.URL, "auto") defer node.StopDaemon() // Wait for daemon to be ready time.Sleep(5 * time.Second) // Get node's peer ID idResult := node.RunIPFS("id", "-f", "") require.Equal(t, 0, idResult.ExitCode()) peerID := strings.TrimSpace(idResult.Stdout.String()) // Get peer ID in base36 format (used for IPNS keys) idBase36Result := node.RunIPFS("id", "--peerid-base", "base36", "-f", "") require.Equal(t, 0, idBase36Result.ExitCode()) peerIDBase36 := strings.TrimSpace(idBase36Result.Stdout.String()) // Verify autoconf resolved "auto" correctly result := node.RunIPFS("config", "Ipns.DelegatedPublishers", "--expand-auto") var resolvedPublishers []string err := json.Unmarshal([]byte(result.Stdout.String()), &resolvedPublishers) require.NoError(t, err) expectedURL := publisher.server.URL + "/routing/v1/ipns" assert.Contains(t, resolvedPublishers, expectedURL, "AutoConf should resolve 'auto' to mock publisher") // Test publishing with --allow-delegated testCID := "bafkqablimvwgy3y" result = node.RunIPFS("name", "publish", "--allow-delegated", "/ipfs/"+testCID) require.Equal(t, 0, result.ExitCode(), "Publishing should succeed") assert.Contains(t, result.Stdout.String(), "Published to") // Wait for async HTTP request to delegated publisher time.Sleep(2 * time.Second) // Verify HTTP PUT was made to delegated publisher publishedKeys := publisher.getPublishedKeys() assert.NotEmpty(t, publishedKeys, "HTTP PUT request should have been made to delegated publisher") // Get the PUT payload that was sent to the delegated publisher putPayload := publisher.getRecordPayload(peerIDBase36) require.NotNil(t, putPayload, "Should have captured PUT payload") require.Greater(t, len(putPayload), 0, "PUT payload should not be empty") // Retrieve the IPNS record using routing get getResult := node.RunIPFS("routing", "get", "/ipns/"+peerID) require.Equal(t, 0, getResult.ExitCode(), "Should be able to retrieve IPNS record") getPayload := getResult.Stdout.Bytes() // Compare the payloads assert.Equal(t, putPayload, getPayload, "PUT payload sent to delegated publisher should match what routing get returns") // Also verify the record points to the expected content assert.Contains(t, getResult.Stdout.String(), testCID, "Retrieved IPNS record should reference the published CID") // Use ipfs name inspect to verify the IPNS record's value matches the published CID // First write the routing get result to a file for inspection node.WriteBytes("ipns-record", getPayload) inspectResult := node.RunIPFS("name", "inspect", "ipns-record") require.Equal(t, 0, inspectResult.ExitCode(), "Should be able to inspect IPNS record") // The inspect output should show the path we published inspectOutput := inspectResult.Stdout.String() assert.Contains(t, inspectOutput, "/ipfs/"+testCID, "IPNS record value should match the published path") // Also verify it's a valid record with proper fields assert.Contains(t, inspectOutput, "Value:", "Should have Value field") assert.Contains(t, inspectOutput, "Validity:", "Should have Validity field") assert.Contains(t, inspectOutput, "Sequence:", "Should have Sequence field") t.Log("Verified: PUT payload to delegated publisher matches routing get result and name inspect confirms correct path") } // testIPNSPublishingResilience verifies that IPNS publishing is resilient by design. // Publishing succeeds as long as local storage works, even when all delegated endpoints fail. // This test documents the intentional resilient behavior, not bugs. func testIPNSPublishingResilience(t *testing.T) { testCases := []struct { name string routingType string // "auto" or "delegated" description string }{ { name: "AutoRouting", routingType: "auto", description: "auto mode uses DHT + HTTP, tolerates HTTP failures", }, { name: "DelegatedRouting", routingType: "delegated", description: "delegated mode uses HTTP only, tolerates HTTP failures", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Create publisher that always fails publisher := newMockIPNSPublisher(t) defer publisher.close() publisher.responseFunc = func(peerID string, record []byte) int { return http.StatusInternalServerError } // Create node with failing endpoint node := setupNodeWithAutoconf(t, publisher.server.URL, tc.routingType) defer node.StopDaemon() // Test different publishing modes - all should succeed due to resilient design testCID := "/ipfs/bafkqablimvwgy3y" // Normal publishing (should succeed despite endpoint failures) result := node.RunIPFS("name", "publish", testCID) assert.Equal(t, 0, result.ExitCode(), "%s: Normal publishing should succeed (local storage works)", tc.description) // Publishing with --allow-offline (local only, no network) result = node.RunIPFS("name", "publish", "--allow-offline", testCID) assert.Equal(t, 0, result.ExitCode(), "--allow-offline should succeed (local only)") // Publishing with --allow-delegated (if using auto routing) if tc.routingType == "auto" { result = node.RunIPFS("name", "publish", "--allow-delegated", testCID) assert.Equal(t, 0, result.ExitCode(), "--allow-delegated should succeed (no DHT required)") } t.Logf("%s: All publishing modes succeeded despite endpoint failures (resilient design)", tc.name) }) } } // ============================================================================ // Helper Functions // ============================================================================ // setupNodeWithAutoconf creates an IPFS node with autoconf-configured delegated publishers func setupNodeWithAutoconf(t *testing.T, publisherURL string, routingType string) *harness.Node { // Create autoconf server with the publisher endpoint autoconfData := createAutoconfJSON(publisherURL) autoconfServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, autoconfData) })) t.Cleanup(func() { autoconfServer.Close() }) // Create and configure node h := harness.NewT(t) node := h.NewNode().Init("--profile=test") // Configure autoconf node.SetIPFSConfig("AutoConf.URL", autoconfServer.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Ipns.DelegatedPublishers", []string{"auto"}) node.SetIPFSConfig("Routing.Type", routingType) // Additional config for delegated routing mode if routingType == "delegated" { node.SetIPFSConfig("Provide.Enabled", false) node.SetIPFSConfig("Provide.DHT.Interval", "0s") } // Add bootstrap peers for connectivity node.SetIPFSConfig("Bootstrap", autoconf.FallbackBootstrapPeers) // Start daemon node.StartDaemon() return node } // createAutoconfJSON generates autoconf configuration with a delegated IPNS publisher func createAutoconfJSON(publisherURL string) string { // Use bootstrap peers from autoconf fallbacks for consistency bootstrapPeers, _ := json.Marshal(autoconf.FallbackBootstrapPeers) return fmt.Sprintf(`{ "AutoConfVersion": 2025072302, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": { "TestSystem": { "Description": "Test system for IPNS publishing", "NativeConfig": { "Bootstrap": %s } } }, "DNSResolvers": {}, "DelegatedEndpoints": { "%s": { "Systems": ["TestSystem"], "Read": ["/routing/v1/ipns"], "Write": ["/routing/v1/ipns"] } } }`, string(bootstrapPeers), publisherURL) } // ============================================================================ // Mock IPNS Publisher // ============================================================================ // mockIPNSPublisher implements a simple IPNS publishing HTTP API server type mockIPNSPublisher struct { t *testing.T server *httptest.Server mu sync.Mutex publishedKeys map[string]string // peerID -> published CID recordPayloads map[string][]byte // peerID -> actual HTTP PUT record payload responseFunc func(peerID string, record []byte) int // returns HTTP status code } func newMockIPNSPublisher(t *testing.T) *mockIPNSPublisher { m := &mockIPNSPublisher{ t: t, publishedKeys: make(map[string]string), recordPayloads: make(map[string][]byte), } // Default response function accepts all publishes m.responseFunc = func(peerID string, record []byte) int { return http.StatusOK } mux := http.NewServeMux() mux.HandleFunc("/routing/v1/ipns/", m.handleIPNS) m.server = httptest.NewServer(mux) return m } func (m *mockIPNSPublisher) handleIPNS(w http.ResponseWriter, r *http.Request) { m.mu.Lock() defer m.mu.Unlock() // Extract peer ID from path parts := strings.Split(r.URL.Path, "/") if len(parts) < 5 { http.Error(w, "invalid path", http.StatusBadRequest) return } peerID := parts[4] if r.Method == "PUT" { // Handle IPNS record publication body, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "failed to read body", http.StatusBadRequest) return } // Get response status from response function status := m.responseFunc(peerID, body) if status == http.StatusOK { if len(body) > 0 { // Store the actual record payload m.recordPayloads[peerID] = make([]byte, len(body)) copy(m.recordPayloads[peerID], body) } // Mark as published m.publishedKeys[peerID] = fmt.Sprintf("published-%d", time.Now().Unix()) } w.WriteHeader(status) if status != http.StatusOK { fmt.Fprint(w, `{"error": "publish failed"}`) } } else if r.Method == "GET" { // Handle IPNS record retrieval if record, exists := m.publishedKeys[peerID]; exists { w.Header().Set("Content-Type", "application/vnd.ipfs.ipns-record") fmt.Fprint(w, record) } else { http.Error(w, "record not found", http.StatusNotFound) } } else { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } } func (m *mockIPNSPublisher) getPublishedKeys() map[string]string { m.mu.Lock() defer m.mu.Unlock() result := make(map[string]string) maps.Copy(result, m.publishedKeys) return result } func (m *mockIPNSPublisher) getRecordPayload(peerID string) []byte { m.mu.Lock() defer m.mu.Unlock() if payload, exists := m.recordPayloads[peerID]; exists { result := make([]byte, len(payload)) copy(result, payload) return result } return nil } func (m *mockIPNSPublisher) close() { m.server.Close() } ================================================ FILE: test/cli/autoconf/routing_test.go ================================================ package autoconf import ( "encoding/json" "fmt" "net/http" "net/http/httptest" "strings" "sync" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestAutoConfDelegatedRouting(t *testing.T) { t.Parallel() t.Run("delegated routing with auto router", func(t *testing.T) { t.Parallel() testDelegatedRoutingWithAuto(t) }) t.Run("routing errors are handled properly", func(t *testing.T) { t.Parallel() testRoutingErrorHandling(t) }) } // mockRoutingServer implements a simple Delegated Routing HTTP API server type mockRoutingServer struct { t *testing.T server *httptest.Server mu sync.Mutex requests []string providerFunc func(cid string) []map[string]any } func newMockRoutingServer(t *testing.T) *mockRoutingServer { m := &mockRoutingServer{ t: t, requests: []string{}, } // Default provider function returns mock provider records m.providerFunc = func(cid string) []map[string]any { return []map[string]any{ { "Protocol": "transport-bitswap", "Schema": "bitswap", "ID": "12D3KooWMockProvider1", "Addrs": []string{"/ip4/192.168.1.100/tcp/4001"}, }, { "Protocol": "transport-bitswap", "Schema": "bitswap", "ID": "12D3KooWMockProvider2", "Addrs": []string{"/ip4/192.168.1.101/tcp/4001"}, }, } } mux := http.NewServeMux() mux.HandleFunc("/routing/v1/providers/", m.handleProviders) m.server = httptest.NewServer(mux) return m } func (m *mockRoutingServer) handleProviders(w http.ResponseWriter, r *http.Request) { m.mu.Lock() defer m.mu.Unlock() // Extract CID from path parts := strings.Split(r.URL.Path, "/") if len(parts) < 5 { http.Error(w, "invalid path", http.StatusBadRequest) return } cid := parts[4] m.requests = append(m.requests, cid) m.t.Logf("Routing server received providers request for CID: %s", cid) // Get provider records providers := m.providerFunc(cid) // Return NDJSON response as per IPIP-378 w.Header().Set("Content-Type", "application/x-ndjson") encoder := json.NewEncoder(w) for _, provider := range providers { if err := encoder.Encode(provider); err != nil { m.t.Logf("Failed to encode provider: %v", err) return } } } func (m *mockRoutingServer) close() { m.server.Close() } func testDelegatedRoutingWithAuto(t *testing.T) { // Create mock routing server routingServer := newMockRoutingServer(t) defer routingServer.close() // Create autoconf data with delegated router autoConfData := fmt.Sprintf(`{ "AutoConfVersion": 2025072302, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": { "AminoDHT": { "Description": "Test AminoDHT system", "NativeConfig": { "Bootstrap": [] } } }, "DNSResolvers": {}, "DelegatedEndpoints": { "%s": { "Systems": ["AminoDHT", "IPNI"], "Read": ["/routing/v1/providers", "/routing/v1/peers", "/routing/v1/ipns"], "Write": [] } } }`, routingServer.server.URL) // Create autoconf server autoConfServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(autoConfData)) })) defer autoConfServer.Close() // Create IPFS node with auto delegated router node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", autoConfServer.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"}) // Test that daemon starts successfully with auto routing configuration // The actual routing functionality requires online mode, but we can test // that the configuration is expanded and daemon starts properly node.StartDaemon("--offline") defer node.StopDaemon() // Verify config still shows "auto" (this tests that auto values are preserved in user-facing config) result := node.RunIPFS("config", "Routing.DelegatedRouters") require.Equal(t, 0, result.ExitCode()) var routers []string err := json.Unmarshal([]byte(result.Stdout.String()), &routers) require.NoError(t, err) assert.Equal(t, []string{"auto"}, routers, "Delegated routers config should show 'auto'") // Test that daemon is running and accepting commands result = node.RunIPFS("version") require.Equal(t, 0, result.ExitCode(), "Daemon should be running and accepting commands") // Test that autoconf server was contacted (indicating successful resolution) // We can't test actual routing in offline mode, but we can verify that // the AutoConf system expanded the "auto" placeholder successfully // by checking that the daemon started without errors t.Log("AutoConf successfully expanded delegated router configuration and daemon started") } func testRoutingErrorHandling(t *testing.T) { // Create routing server that returns no providers routingServer := newMockRoutingServer(t) defer routingServer.close() // Configure to return no providers (empty response) routingServer.providerFunc = func(cid string) []map[string]any { return []map[string]any{} } // Create autoconf data autoConfData := fmt.Sprintf(`{ "AutoConfVersion": 2025072302, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": { "AminoDHT": { "Description": "Test AminoDHT system", "NativeConfig": { "Bootstrap": [] } } }, "DNSResolvers": {}, "DelegatedEndpoints": { "%s": { "Systems": ["AminoDHT", "IPNI"], "Read": ["/routing/v1/providers", "/routing/v1/peers", "/routing/v1/ipns"], "Write": [] } } }`, routingServer.server.URL) // Create autoconf server autoConfServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(autoConfData)) })) defer autoConfServer.Close() // Create IPFS node node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", autoConfServer.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"}) // Test that daemon starts successfully even when no providers are available node.StartDaemon("--offline") defer node.StopDaemon() // Verify config shows "auto" result := node.RunIPFS("config", "Routing.DelegatedRouters") require.Equal(t, 0, result.ExitCode()) var routers []string err := json.Unmarshal([]byte(result.Stdout.String()), &routers) require.NoError(t, err) assert.Equal(t, []string{"auto"}, routers, "Delegated routers config should show 'auto'") // Test that daemon is running and accepting commands result = node.RunIPFS("version") require.Equal(t, 0, result.ExitCode(), "Daemon should be running even with empty routing config") t.Log("AutoConf successfully handled routing configuration with empty providers") } ================================================ FILE: test/cli/autoconf/swarm_connect_test.go ================================================ package autoconf import ( "testing" "time" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // TestSwarmConnectWithAutoConf tests that ipfs swarm connect works properly // when AutoConf is enabled and a daemon is running. // // This is a regression test for the issue where: // - AutoConf disabled: ipfs swarm connect works // - AutoConf enabled: ipfs swarm connect fails with "Error: connect" // // The issue affects CLI command fallback behavior when the HTTP API connection fails. func TestSwarmConnectWithAutoConf(t *testing.T) { t.Parallel() t.Run("AutoConf disabled - should work", func(t *testing.T) { testSwarmConnectWithAutoConfSetting(t, false, true) // expect success }) t.Run("AutoConf enabled - should work", func(t *testing.T) { testSwarmConnectWithAutoConfSetting(t, true, true) // expect success (fix the bug!) }) } func testSwarmConnectWithAutoConfSetting(t *testing.T, autoConfEnabled bool, expectSuccess bool) { // Create IPFS node with test profile node := harness.NewT(t).NewNode().Init("--profile=test") // Configure AutoConf node.SetIPFSConfig("AutoConf.Enabled", autoConfEnabled) // Set up bootstrap peers so the node has something to connect to // Use the same bootstrap peers from boxo/autoconf fallbacks node.SetIPFSConfig("Bootstrap", []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", }) // CRITICAL: Start the daemon first - this is the key requirement // The daemon must be running and working properly node.StartDaemon() defer node.StopDaemon() // Give daemon time to start up completely time.Sleep(3 * time.Second) // Verify daemon is responsive result := node.RunIPFS("id") require.Equal(t, 0, result.ExitCode(), "Daemon should be responsive before testing swarm connect") t.Logf("Daemon is running and responsive. AutoConf enabled: %v", autoConfEnabled) // Now test swarm connect to a bootstrap peer // This should work because: // 1. The daemon is running // 2. The CLI should connect to the daemon via API // 3. The daemon should handle the swarm connect request result = node.RunIPFS("swarm", "connect", "/dnsaddr/bootstrap.libp2p.io") // swarm connect should work regardless of AutoConf setting assert.Equal(t, 0, result.ExitCode(), "swarm connect should succeed with AutoConf=%v. stderr: %s", autoConfEnabled, result.Stderr.String()) // Should contain success message output := result.Stdout.String() assert.Contains(t, output, "success", "swarm connect output should contain 'success' with AutoConf=%v. output: %s", autoConfEnabled, output) // Additional diagnostic: Check if ipfs id shows addresses // Both AutoConf enabled and disabled should show proper addresses result = node.RunIPFS("id") require.Equal(t, 0, result.ExitCode(), "ipfs id should work with AutoConf=%v", autoConfEnabled) idOutput := result.Stdout.String() t.Logf("ipfs id output with AutoConf=%v: %s", autoConfEnabled, idOutput) // Addresses should not be null regardless of AutoConf setting assert.Contains(t, idOutput, `"Addresses"`, "ipfs id should show Addresses field") assert.NotContains(t, idOutput, `"Addresses": null`, "ipfs id should not show null addresses with AutoConf=%v", autoConfEnabled) } ================================================ FILE: test/cli/autoconf/testdata/autoconf_amino_and_ipni.json ================================================ { "AutoConfVersion": 2025072901, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": { "AminoDHT": { "URL": "https://github.com/ipfs/specs/pull/497", "Description": "Public DHT swarm that implements the IPFS Kademlia DHT specification under protocol identifier /ipfs/kad/1.0.0", "NativeConfig": { "Bootstrap": [ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN" ] }, "DelegatedConfig": { "Read": [ "/routing/v1/providers", "/routing/v1/peers", "/routing/v1/ipns" ], "Write": [ "/routing/v1/ipns" ] } }, "IPNI": { "URL": "https://cid.contact", "Description": "Network Indexer - content routing database for large storage providers", "DelegatedConfig": { "Read": [ "/routing/v1/providers" ], "Write": [] } } }, "DNSResolvers": { "eth.": [ "https://dns.eth.limo/dns-query" ] }, "DelegatedEndpoints": { "https://amino-dht.example.com": { "Systems": ["AminoDHT"], "Read": [ "/routing/v1/providers", "/routing/v1/peers" ], "Write": [ "/routing/v1/ipns" ] }, "https://cid.contact": { "Systems": ["IPNI"], "Read": [ "/routing/v1/providers" ], "Write": [] } } } ================================================ FILE: test/cli/autoconf/testdata/autoconf_new_routing_system.json ================================================ { "AutoConfVersion": 2025072901, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": { "NewRoutingSystem": { "URL": "https://new-routing.example.com", "Description": "New routing system for testing delegation with auto routing", "DelegatedConfig": { "Read": [ "/routing/v1/providers", "/routing/v1/peers", "/routing/v1/ipns" ], "Write": [ "/routing/v1/ipns" ] } } }, "DNSResolvers": { "eth.": [ "https://dns.eth.limo/dns-query" ] }, "DelegatedEndpoints": { "https://new-routing.example.com": { "Systems": ["NewRoutingSystem"], "Read": [ "/routing/v1/providers", "/routing/v1/peers" ], "Write": [ "/routing/v1/ipns" ] } } } ================================================ FILE: test/cli/autoconf/testdata/autoconf_new_routing_with_filtering.json ================================================ { "AutoConfVersion": 2025072901, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": { "NewRoutingSystem": { "URL": "https://new-routing.example.com", "Description": "New routing system for testing path filtering with auto routing", "DelegatedConfig": { "Read": [ "/routing/v1/providers", "/routing/v1/peers", "/routing/v1/ipns" ], "Write": [ "/routing/v1/ipns" ] } } }, "DNSResolvers": { "eth.": [ "https://dns.eth.limo/dns-query" ] }, "DelegatedEndpoints": { "https://supported-new.example.com": { "Systems": ["NewRoutingSystem"], "Read": [ "/routing/v1/providers", "/routing/v1/peers" ], "Write": [ "/routing/v1/ipns" ] }, "https://unsupported-new.example.com": { "Systems": ["NewRoutingSystem"], "Read": [ "/custom/v0/read", "/api/v1/nonstandard" ], "Write": [ "/custom/v0/write" ] }, "https://mixed-new.example.com": { "Systems": ["NewRoutingSystem"], "Read": [ "/routing/v1/providers", "/invalid/path", "/routing/v1/peers" ], "Write": [ "/routing/v1/ipns" ] } } } ================================================ FILE: test/cli/autoconf/testdata/autoconf_with_unsupported_paths.json ================================================ { "AutoConfVersion": 2025072901, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": { "AminoDHT": { "URL": "https://github.com/ipfs/specs/pull/497", "Description": "Public DHT swarm that implements the IPFS Kademlia DHT specification under protocol identifier /ipfs/kad/1.0.0", "NativeConfig": { "Bootstrap": [ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN" ] }, "DelegatedConfig": { "Read": [ "/routing/v1/providers", "/routing/v1/peers", "/routing/v1/ipns" ], "Write": [ "/routing/v1/ipns" ] } } }, "DNSResolvers": { "eth.": [ "https://dns.eth.limo/dns-query" ] }, "DelegatedEndpoints": { "https://supported.example.com": { "Systems": ["AminoDHT"], "Read": [ "/routing/v1/providers", "/routing/v1/peers" ], "Write": [ "/routing/v1/ipns" ] }, "https://unsupported.example.com": { "Systems": ["AminoDHT"], "Read": [ "/example/v0/read", "/api/v1/custom" ], "Write": [ "/example/v0/write" ] }, "https://mixed.example.com": { "Systems": ["AminoDHT"], "Read": [ "/routing/v1/providers", "/unsupported/path", "/routing/v1/peers" ], "Write": [ "/routing/v1/ipns" ] } } } ================================================ FILE: test/cli/autoconf/testdata/updated_autoconf.json ================================================ { "AutoConfVersion": 2025072902, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": { "AminoDHT": { "URL": "https://github.com/ipfs/specs/pull/497", "Description": "Public DHT swarm that implements the IPFS Kademlia DHT specification under protocol identifier /ipfs/kad/1.0.0", "NativeConfig": { "Bootstrap": [ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", "/dnsaddr/va1.bootstrap.libp2p.io/p2p/12D3KooWKnDdG3iXw9eTFijk3EWSunZcFi54Zka4wmtqtt6rPxc8", "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" ] }, "DelegatedConfig": { "Read": [ "/routing/v1/providers", "/routing/v1/peers", "/routing/v1/ipns" ], "Write": [ "/routing/v1/ipns" ] } }, "IPNI": { "URL": "https://ipni.example.com", "Description": "Network Indexer - content routing database for large storage providers", "DelegatedConfig": { "Read": [ "/routing/v1/providers" ], "Write": [] } } }, "DNSResolvers": { "eth.": [ "https://dns.eth.limo/dns-query", "https://dns.eth.link/dns-query" ], "test.": [ "https://test.resolver/dns-query" ] }, "DelegatedEndpoints": { "https://ipni.example.com": { "Systems": ["IPNI"], "Read": [ "/routing/v1/providers" ], "Write": [] }, "https://routing.example.com": { "Systems": ["IPNI"], "Read": [ "/routing/v1/providers" ], "Write": [] }, "https://delegated-ipfs.dev": { "Systems": ["AminoDHT", "IPNI"], "Read": [ "/routing/v1/providers", "/routing/v1/peers", "/routing/v1/ipns" ], "Write": [ "/routing/v1/ipns" ] }, "https://ipns.example.com": { "Systems": ["AminoDHT"], "Read": [ "/routing/v1/ipns" ], "Write": [ "/routing/v1/ipns" ] } } } ================================================ FILE: test/cli/autoconf/testdata/valid_autoconf.json ================================================ { "AutoConfVersion": 2025072901, "AutoConfSchema": 1, "AutoConfTTL": 86400, "SystemRegistry": { "AminoDHT": { "URL": "https://github.com/ipfs/specs/pull/497", "Description": "Public DHT swarm that implements the IPFS Kademlia DHT specification under protocol identifier /ipfs/kad/1.0.0", "NativeConfig": { "Bootstrap": [ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", "/dnsaddr/va1.bootstrap.libp2p.io/p2p/12D3KooWKnDdG3iXw9eTFijk3EWSunZcFi54Zka4wmtqtt6rPxc8", "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" ] }, "DelegatedConfig": { "Read": [ "/routing/v1/providers", "/routing/v1/peers", "/routing/v1/ipns" ], "Write": [ "/routing/v1/ipns" ] } }, "IPNI": { "URL": "https://ipni.example.com", "Description": "Network Indexer - content routing database for large storage providers", "DelegatedConfig": { "Read": [ "/routing/v1/providers" ], "Write": [] } } }, "DNSResolvers": { "eth.": [ "https://dns.eth.limo/dns-query", "https://dns.eth.link/dns-query" ] }, "DelegatedEndpoints": { "https://ipni.example.com": { "Systems": ["IPNI"], "Read": [ "/routing/v1/providers" ], "Write": [] }, "https://delegated-ipfs.dev": { "Systems": ["AminoDHT", "IPNI"], "Read": [ "/routing/v1/providers", "/routing/v1/peers", "/routing/v1/ipns" ], "Write": [ "/routing/v1/ipns" ] } } } ================================================ FILE: test/cli/autoconf/validation_test.go ================================================ package autoconf import ( "net/http" "net/http/httptest" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" ) func TestAutoConfValidation(t *testing.T) { t.Parallel() t.Run("invalid autoconf JSON prevents caching", func(t *testing.T) { t.Parallel() testInvalidAutoConfJSONPreventsCaching(t) }) t.Run("malformed multiaddr in autoconf", func(t *testing.T) { t.Parallel() testMalformedMultiaddrInAutoConf(t) }) t.Run("malformed URL in autoconf", func(t *testing.T) { t.Parallel() testMalformedURLInAutoConf(t) }) } func testInvalidAutoConfJSONPreventsCaching(t *testing.T) { // Create server that serves invalid autoconf JSON invalidAutoConfData := `{ "AutoConfVersion": 123, "AutoConfSchema": 1, "SystemRegistry": { "AminoDHT": { "NativeConfig": { "Bootstrap": [ "invalid-multiaddr-that-should-fail" ] } } } }` requestCount := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requestCount++ t.Logf("Invalid autoconf server request #%d: %s %s", requestCount, r.Method, r.URL.Path) w.Header().Set("Content-Type", "application/json") w.Header().Set("ETag", `"invalid-config-123"`) _, _ = w.Write([]byte(invalidAutoConfData)) })) defer server.Close() // Create IPFS node and try to start daemon with invalid autoconf node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Start daemon to trigger autoconf fetch - this should start but log validation errors node.StartDaemon() defer node.StopDaemon() // Give autoconf some time to attempt fetch and fail validation // The daemon should still start but autoconf should fail result := node.RunIPFS("version") assert.Equal(t, 0, result.ExitCode(), "Daemon should start even with invalid autoconf") // Verify server was called (autoconf was attempted even though validation failed) assert.Greater(t, requestCount, 0, "Invalid autoconf server should have been called") } func testMalformedMultiaddrInAutoConf(t *testing.T) { // Create server that serves autoconf with malformed multiaddr invalidAutoConfData := `{ "AutoConfVersion": 456, "AutoConfSchema": 1, "SystemRegistry": { "AminoDHT": { "NativeConfig": { "Bootstrap": [ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "not-a-valid-multiaddr" ] } } } }` server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(invalidAutoConfData)) })) defer server.Close() // Create IPFS node node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Start daemon to trigger autoconf fetch - daemon should start but autoconf validation should fail node.StartDaemon() defer node.StopDaemon() // Daemon should still be functional even with invalid autoconf result := node.RunIPFS("version") assert.Equal(t, 0, result.ExitCode(), "Daemon should start even with invalid autoconf") } func testMalformedURLInAutoConf(t *testing.T) { // Create server that serves autoconf with malformed URL invalidAutoConfData := `{ "AutoConfVersion": 789, "AutoConfSchema": 1, "DNSResolvers": { "eth.": ["https://valid.example.com"], "bad.": ["://malformed-url-missing-scheme"] } }` server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(invalidAutoConfData)) })) defer server.Close() // Create IPFS node node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.URL", server.URL) node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("DNS.Resolvers", map[string]string{"foo.": "auto"}) // Start daemon to trigger autoconf fetch - daemon should start but autoconf validation should fail node.StartDaemon() defer node.StopDaemon() // Daemon should still be functional even with invalid autoconf result := node.RunIPFS("version") assert.Equal(t, 0, result.ExitCode(), "Daemon should start even with invalid autoconf") } ================================================ FILE: test/cli/backup_bootstrap_test.go ================================================ package cli import ( "fmt" "testing" "time" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" ) func TestBackupBootstrapPeers(t *testing.T) { nodes := harness.NewT(t).NewNodes(3).Init() nodes.ForEachPar(func(n *harness.Node) { n.UpdateConfig(func(cfg *config.Config) { cfg.Bootstrap = []string{} cfg.Addresses.Swarm = []string{fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", harness.NewRandPort())} cfg.Discovery.MDNS.Enabled = false cfg.Internal.BackupBootstrapInterval = config.NewOptionalDuration(250 * time.Millisecond) }) }) // Start all nodes and ensure they all have no peers. nodes.StartDaemons() nodes.ForEachPar(func(n *harness.Node) { assert.Len(t, n.Peers(), 0) }) // Connect nodes 0 and 1, ensure they know each other. nodes[0].Connect(nodes[1]) assert.Len(t, nodes[0].Peers(), 1) assert.Len(t, nodes[1].Peers(), 1) assert.Len(t, nodes[2].Peers(), 0) // Wait a bit to ensure that 0 and 1 saved their temporary bootstrap backups. time.Sleep(time.Millisecond * 500) nodes.StopDaemons() // Start 1 and 2. 2 does not know anyone yet. nodes[1].StartDaemon() defer nodes[1].StopDaemon() nodes[2].StartDaemon() defer nodes[2].StopDaemon() assert.Len(t, nodes[1].Peers(), 0) assert.Len(t, nodes[2].Peers(), 0) // Connect 1 and 2, ensure they know each other. nodes[1].Connect(nodes[2]) assert.Len(t, nodes[1].Peers(), 1) assert.Len(t, nodes[2].Peers(), 1) // Start 0, wait a bit. Should connect to 1, and then discover 2 via the // backup bootstrap peers. nodes[0].StartDaemon() defer nodes[0].StopDaemon() time.Sleep(time.Millisecond * 500) // Check if they're all connected. assert.Len(t, nodes[0].Peers(), 2) assert.Len(t, nodes[1].Peers(), 2) assert.Len(t, nodes[2].Peers(), 2) } ================================================ FILE: test/cli/basic_commands_test.go ================================================ package cli import ( "fmt" "regexp" "strings" "testing" "github.com/blang/semver/v4" "github.com/ipfs/kubo/test/cli/harness" . "github.com/ipfs/kubo/test/cli/testutils" "github.com/stretchr/testify/assert" gomod "golang.org/x/mod/module" ) var versionRegexp = regexp.MustCompile(`^ipfs version (.+)$`) func parseVersionOutput(s string) semver.Version { versString := versionRegexp.FindStringSubmatch(s)[1] v, err := semver.Parse(versString) if err != nil { panic(err) } return v } func TestCurDirIsWritable(t *testing.T) { t.Parallel() h := harness.NewT(t) h.WriteFile("test.txt", "It works!") } func TestIPFSVersionCommandMatchesFlag(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() commandVersionStr := node.IPFS("version").Stdout.String() commandVersionStr = strings.TrimSpace(commandVersionStr) commandVersion := parseVersionOutput(commandVersionStr) flagVersionStr := node.IPFS("--version").Stdout.String() flagVersionStr = strings.TrimSpace(flagVersionStr) flagVersion := parseVersionOutput(flagVersionStr) assert.Equal(t, commandVersion, flagVersion) } func TestIPFSVersionAll(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() res := node.IPFS("version", "--all").Stdout.String() res = strings.TrimSpace(res) assert.Contains(t, res, "Kubo version") assert.Contains(t, res, "Repo version") assert.Contains(t, res, "System version") assert.Contains(t, res, "Golang version") } func TestIPFSVersionDeps(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() res := node.IPFS("version", "deps").Stdout.String() res = strings.TrimSpace(res) lines := SplitLines(res) assert.True(t, strings.HasPrefix(lines[0], "github.com/ipfs/kubo@v")) for _, depLine := range lines[1:] { split := strings.SplitSeq(depLine, " => ") for moduleVersion := range split { splitModVers := strings.Split(moduleVersion, "@") modPath := splitModVers[0] modVers := splitModVers[1] // Skip local replace paths (starting with "./") if strings.HasPrefix(modPath, "./") { continue } assert.NoError(t, gomod.Check(modPath, modVers), "path: %s, version: %s", modPath, modVers) } } } func TestIPFSCommands(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() cmds := node.IPFSCommands() assert.Contains(t, cmds, "ipfs add") assert.Contains(t, cmds, "ipfs daemon") assert.Contains(t, cmds, "ipfs update") } func TestAllSubcommandsAcceptHelp(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() for _, cmd := range node.IPFSCommands() { t.Run(fmt.Sprintf("command %q accepts help", cmd), func(t *testing.T) { t.Parallel() splitCmd := strings.Split(cmd, " ")[1:] node.IPFS(StrCat("help", splitCmd)...) node.IPFS(StrCat(splitCmd, "--help")...) }) } } func TestAllRootCommandsAreMentionedInHelpText(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() cmds := node.IPFSCommands() var rootCmds []string for _, cmd := range cmds { splitCmd := strings.Split(cmd, " ") if len(splitCmd) == 2 { rootCmds = append(rootCmds, splitCmd[1]) } } // a few base commands are not expected to be in the help message // but we default to requiring them to be in the help message, so that we // have to make a conscious decision to exclude them notInHelp := map[string]bool{ "object": true, "shutdown": true, } helpMsg := strings.TrimSpace(node.IPFS("--help").Stdout.String()) for _, rootCmd := range rootCmds { if _, ok := notInHelp[rootCmd]; ok { continue } assert.Contains(t, helpMsg, fmt.Sprintf(" %s", rootCmd)) } } func TestCommandDocsWidth(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() // require new commands to explicitly opt in to longer lines allowList := map[string]bool{ "ipfs add": true, "ipfs block put": true, "ipfs daemon": true, "ipfs config profile": true, "ipfs pin remote service": true, "ipfs name pubsub": true, "ipfs object patch": true, "ipfs swarm connect": true, "ipfs p2p forward": true, "ipfs p2p close": true, "ipfs swarm disconnect": true, "ipfs swarm addrs listen": true, "ipfs dag resolve": true, "ipfs dag get": true, "ipfs pin remote add": true, "ipfs config show": true, "ipfs config edit": true, "ipfs pin remote rm": true, "ipfs pin remote ls": true, "ipfs pin verify": true, "ipfs pin remote service add": true, "ipfs pin update": true, "ipfs pin rm": true, "ipfs p2p": true, "ipfs resolve": true, "ipfs dag stat": true, "ipfs name publish": true, "ipfs object diff": true, "ipfs object patch add-link": true, "ipfs name": true, "ipfs diag profile": true, "ipfs diag cmds": true, "ipfs swarm addrs local": true, "ipfs files ls": true, "ipfs stats bw": true, "ipfs swarm peers": true, "ipfs pubsub sub": true, "ipfs files write": true, "ipfs swarm limit": true, "ipfs commands completion fish": true, "ipfs key export": true, "ipfs routing get": true, "ipfs refs": true, "ipfs refs local": true, "ipfs cid base32": true, "ipfs pubsub pub": true, "ipfs repo ls": true, "ipfs routing put": true, "ipfs key import": true, "ipfs swarm peering add": true, "ipfs swarm peering rm": true, "ipfs swarm peering ls": true, "ipfs update": true, "ipfs swarm stats": true, } for _, cmd := range node.IPFSCommands() { if _, ok := allowList[cmd]; ok { continue } t.Run(fmt.Sprintf("command %q conforms to docs width limit", cmd), func(t *testing.T) { splitCmd := strings.Split(cmd, " ") resStr := node.IPFS(StrCat(splitCmd[1:], "--help")...) res := strings.TrimSpace(resStr.Stdout.String()) for _, line := range SplitLines(res) { assert.LessOrEqualf(t, len(line), 80, "expected width %d < 80 for %q", len(line), cmd) } }) } } func TestAllCommandsFailWhenPassedBadFlag(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() for _, cmd := range node.IPFSCommands() { t.Run(fmt.Sprintf("command %q fails when passed a bad flag", cmd), func(t *testing.T) { splitCmd := strings.Split(cmd, " ") res := node.RunIPFS(StrCat(splitCmd, "--badflag")...) assert.Equal(t, 1, res.Cmd.ProcessState.ExitCode()) }) } } func TestCommandsFlags(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() resStr := node.IPFS("commands", "--flags").Stdout.String() assert.Contains(t, resStr, "ipfs pin add --recursive / ipfs pin add -r") assert.Contains(t, resStr, "ipfs id --format / ipfs id -f") assert.Contains(t, resStr, "ipfs repo gc --quiet / ipfs repo gc -q") } ================================================ FILE: test/cli/bitswap_config_test.go ================================================ package cli import ( "strings" "testing" "time" "github.com/ipfs/boxo/bitswap/network/bsnet" "github.com/ipfs/go-test/random" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" ) func TestBitswapConfig(t *testing.T) { t.Parallel() // Create test data that will be shared between nodes testData := random.Bytes(100) t.Run("server enabled (default)", func(t *testing.T) { t.Parallel() h := harness.NewT(t) provider := h.NewNode().Init().StartDaemon() defer provider.StopDaemon() requester := h.NewNode().Init().StartDaemon() defer requester.StopDaemon() hash := provider.IPFSAddStr(string(testData)) requester.Connect(provider) res := requester.IPFS("cat", hash) assert.Equal(t, testData, res.Stdout.Bytes(), "retrieved data should match original") }) t.Run("server disabled", func(t *testing.T) { t.Parallel() h := harness.NewT(t) provider := h.NewNode().Init() provider.SetIPFSConfig("Bitswap.ServerEnabled", false) provider = provider.StartDaemon() defer provider.StopDaemon() requester := h.NewNode().Init().StartDaemon() defer requester.StopDaemon() hash := provider.IPFSAddStr(string(testData)) requester.Connect(provider) // If the data was available, it would be retrieved immediately. // Therefore, after the timeout, we can assume the data is not available // i.e. the server is disabled timeout := time.After(3 * time.Second) dataChan := make(chan []byte) go func() { res := requester.RunIPFS("cat", hash) dataChan <- res.Stdout.Bytes() }() select { case data := <-dataChan: assert.NotEqual(t, testData, data, "retrieved data should not match original") case <-timeout: t.Log("Test passed: operation timed out after 3 seconds as expected") } }) t.Run("client still works when server disabled", func(t *testing.T) { t.Parallel() h := harness.NewT(t) requester := h.NewNode().Init() requester.SetIPFSConfig("Bitswap.ServerEnabled", false) requester.StartDaemon() defer requester.StopDaemon() provider := h.NewNode().Init().StartDaemon() defer provider.StopDaemon() hash := provider.IPFSAddStr(string(testData)) requester.Connect(provider) // Even when the server is disabled, the client should be able to retrieve data res := requester.RunIPFS("cat", hash) assert.Equal(t, testData, res.Stdout.Bytes(), "retrieved data should match original") }) t.Run("bitswap over libp2p disabled", func(t *testing.T) { t.Parallel() h := harness.NewT(t) requester := h.NewNode().Init() requester.UpdateConfig(func(cfg *config.Config) { cfg.Bitswap.Libp2pEnabled = config.False cfg.Bitswap.ServerEnabled = config.False cfg.HTTPRetrieval.Enabled = config.True }) requester.StartDaemon() defer requester.StopDaemon() provider := h.NewNode().Init().StartDaemon() defer provider.StopDaemon() hash := provider.IPFSAddStr(string(testData)) requester.Connect(provider) res := requester.RunIPFS("cat", hash) assert.Equal(t, []byte{}, res.Stdout.Bytes(), "cat should not return any data") assert.Contains(t, res.Stderr.String(), "Error: ipld: could not find") // Verify that basic operations still work with bitswap disabled res = requester.IPFS("id") assert.Equal(t, 0, res.ExitCode(), "basic IPFS operations should work") res = requester.IPFS("bitswap", "stat") assert.Equal(t, 0, res.ExitCode(), "bitswap stat should work even with bitswap disabled") res = requester.IPFS("bitswap", "wantlist") assert.Equal(t, 0, res.ExitCode(), "bitswap wantlist should work even with bitswap disabled") // Verify local operations still work hashNew := requester.IPFSAddStr("random") res = requester.IPFS("cat", hashNew) assert.Equal(t, []byte("random"), res.Stdout.Bytes(), "cat should return the added data") }) // Disabling Bitswap.Libp2pEnabled should remove /ipfs/bitswap* protocols from `ipfs id` t.Run("disabling bitswap over libp2p removes it from identify protocol list", func(t *testing.T) { t.Parallel() h := harness.NewT(t) provider := h.NewNode().Init() provider.UpdateConfig(func(cfg *config.Config) { cfg.Bitswap.Libp2pEnabled = config.False cfg.Bitswap.ServerEnabled = config.False cfg.HTTPRetrieval.Enabled = config.True }) provider = provider.StartDaemon() defer provider.StopDaemon() requester := h.NewNode().Init().StartDaemon() defer requester.StopDaemon() requester.Connect(provider) // read libp2p identify from remote peer, and print protocols res := requester.IPFS("id", "-f", "", provider.PeerID().String()) protocols := strings.SplitSeq(strings.TrimSpace(res.Stdout.String()), "\n") // No bitswap protocols should be present for proto := range protocols { assert.NotContains(t, proto, bsnet.ProtocolBitswap, "bitswap protocol %s should not be advertised when server is disabled", proto) assert.NotContains(t, proto, bsnet.ProtocolBitswapNoVers, "bitswap protocol %s should not be advertised when server is disabled", proto) assert.NotContains(t, proto, bsnet.ProtocolBitswapOneOne, "bitswap protocol %s should not be advertised when server is disabled", proto) assert.NotContains(t, proto, bsnet.ProtocolBitswapOneZero, "bitswap protocol %s should not be advertised when server is disabled", proto) } }) // HTTPRetrieval uses bitswap engine, we need it t.Run("errors when both HTTP and libp2p are disabled", func(t *testing.T) { t.Parallel() // init Kubo repo node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.HTTPRetrieval.Enabled = config.False cfg.Bitswap.Libp2pEnabled = config.False cfg.Bitswap.ServerEnabled = config.Default }) res := node.RunIPFS("daemon") assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Bitswap.Libp2pEnabled and HTTPRetrieval.Enabled are both disabled, unable to initialize Bitswap") assert.Equal(t, 1, res.ExitCode()) }) // HTTPRetrieval uses bitswap engine, we need it t.Run("errors when user set conflicting HTTP and libp2p flags", func(t *testing.T) { t.Parallel() // init Kubo repo node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.HTTPRetrieval.Enabled = config.False cfg.Bitswap.Libp2pEnabled = config.False cfg.Bitswap.ServerEnabled = config.True // bad user config: can't enable server when libp2p is down }) res := node.RunIPFS("daemon") assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Bitswap.Libp2pEnabled and HTTPRetrieval.Enabled are both disabled, unable to initialize Bitswap") assert.Equal(t, 1, res.ExitCode()) }) } ================================================ FILE: test/cli/block_size_test.go ================================================ package cli import ( "bytes" "crypto/rand" "encoding/json" "fmt" "os" "path/filepath" "strings" "testing" "time" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) const ( twoMiB = 2 * 1024 * 1024 // 2097152 - bitswap spec block size limit twoMiBPlus = twoMiB + 1 // 2097153 maxChunkSize = twoMiB - 256 // 2096896 - max chunker value (overhead budget for protobuf framing) overMaxChunk = maxChunkSize + 1 // 2096897 // go-libp2p v0.47.0 network.MessageSizeMax is 4194304 bytes (4MiB). // A bitswap message carrying a single block has a protobuf envelope // whose size depends on the CID used to represent the block. For // CIDv1 with raw codec and SHA2-256 multihash (4-byte CID prefix), // the envelope is 18 bytes: 2 bytes for the empty Wantlist submessage, // 6 bytes for the CID prefix field, 5 bytes for field tags and the // payload length varint, and 5 bytes for the data length varint and // block submessage length varint. The msgio varint reader rejects // messages strictly larger than MessageSizeMax, so the maximum block // that fits is 4194304 - 18 = 4194286 bytes. // // The hard limit varies slightly depending on the CID: a longer // multihash (e.g. SHA-512) increases the CID prefix and reduces the // maximum block payload by the same amount. libp2pMsgMax = 4 * 1024 * 1024 // 4194304 - libp2p network.MessageSizeMax bsBlockEnvelope = 18 // protobuf overhead for CIDv1 + raw + SHA2-256 maxTransferBlock = libp2pMsgMax - bsBlockEnvelope // 4194286 - largest block transferable via bitswap overMaxTransfer = maxTransferBlock + 1 // 4194287 ) // blockSize returns the block size in bytes for a given CID by parsing // the JSON output of `ipfs block stat --enc=json `. func blockSize(t *testing.T, node *harness.Node, cid string) int { t.Helper() res := node.IPFS("block", "stat", "--enc=json", cid) var stat struct { Key string Size int } require.NoError(t, json.Unmarshal(res.Stdout.Bytes(), &stat)) return stat.Size } // allBlockCIDs returns the root CID plus all recursive refs for a DAG. func allBlockCIDs(t *testing.T, node *harness.Node, root string) []string { t.Helper() cids := []string{root} res := node.IPFS("refs", "-r", "--unique", root) for line := range strings.SplitSeq(strings.TrimSpace(res.Stdout.String()), "\n") { if line != "" { cids = append(cids, line) } } return cids } // assertAllBlocksWithinLimit checks that every block in the DAG rooted at // root is at most twoMiB bytes. func assertAllBlocksWithinLimit(t *testing.T, node *harness.Node, root string) { t.Helper() for _, c := range allBlockCIDs(t, node, root) { size := blockSize(t, node, c) assert.LessOrEqual(t, size, twoMiB, fmt.Sprintf("block %s is %d bytes, exceeds 2MiB limit", c, size)) } } func TestBlockSizeBoundary(t *testing.T) { t.Parallel() t.Run("block put", func(t *testing.T) { t.Parallel() t.Run("exactly 2MiB succeeds", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() data := make([]byte, twoMiB) cid := strings.TrimSpace( node.PipeToIPFS(bytes.NewReader(data), "block", "put").Stdout.String(), ) got := node.IPFS("block", "get", cid) assert.Len(t, got.Stdout.Bytes(), twoMiB) }) t.Run("2MiB+1 fails without --allow-big-block", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() data := make([]byte, twoMiBPlus) res := node.RunPipeToIPFS(bytes.NewReader(data), "block", "put") assert.NotEqual(t, 0, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "produced block is over 2MiB: big blocks can't be exchanged with other peers. consider using UnixFS for automatic chunking of bigger files, or pass --allow-big-block to override") }) t.Run("2MiB+1 succeeds with --allow-big-block", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() data := make([]byte, twoMiBPlus) cid := strings.TrimSpace( node.PipeToIPFS(bytes.NewReader(data), "block", "put", "--allow-big-block").Stdout.String(), ) got := node.IPFS("block", "get", cid) assert.Len(t, got.Stdout.Bytes(), twoMiBPlus) }) }) t.Run("dag put", func(t *testing.T) { t.Parallel() t.Run("exactly 2MiB succeeds", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() data := make([]byte, twoMiB) cid := strings.TrimSpace( node.PipeToIPFS(bytes.NewReader(data), "dag", "put", "--input-codec=raw", "--store-codec=raw").Stdout.String(), ) got := node.IPFS("block", "get", cid) assert.Len(t, got.Stdout.Bytes(), twoMiB) }) t.Run("2MiB+1 fails without --allow-big-block", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() data := make([]byte, twoMiBPlus) res := node.RunPipeToIPFS(bytes.NewReader(data), "dag", "put", "--input-codec=raw", "--store-codec=raw") assert.NotEqual(t, 0, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "produced block is over 2MiB: big blocks can't be exchanged with other peers. consider using UnixFS for automatic chunking of bigger files, or pass --allow-big-block to override") }) t.Run("2MiB+1 succeeds with --allow-big-block", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() data := make([]byte, twoMiBPlus) cid := strings.TrimSpace( node.PipeToIPFS(bytes.NewReader(data), "dag", "put", "--input-codec=raw", "--store-codec=raw", "--allow-big-block").Stdout.String(), ) got := node.IPFS("block", "get", cid) assert.Len(t, got.Stdout.Bytes(), twoMiBPlus) }) }) t.Run("dag import and export", func(t *testing.T) { t.Parallel() t.Run("2MiB+1 block round-trips with --allow-big-block", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() // put an oversized raw block with override data := make([]byte, twoMiBPlus) cid := strings.TrimSpace( node.PipeToIPFS(bytes.NewReader(data), "dag", "put", "--input-codec=raw", "--store-codec=raw", "--allow-big-block").Stdout.String(), ) // export to CAR carPath := filepath.Join(node.Dir, "oversized.car") require.NoError(t, node.IPFSDagExport(cid, carPath)) // re-import without --allow-big-block should fail carFile, err := os.Open(carPath) require.NoError(t, err) res := node.RunPipeToIPFS(carFile, "dag", "import") carFile.Close() assert.NotEqual(t, 0, res.ExitCode()) assert.Contains(t, res.Stderr.String()+res.Stdout.String(), "produced block is over 2MiB: big blocks can't be exchanged with other peers. consider using UnixFS for automatic chunking of bigger files, or pass --allow-big-block to override") // re-import with --allow-big-block should succeed carFile, err = os.Open(carPath) require.NoError(t, err) res = node.RunPipeToIPFS(carFile, "dag", "import", "--allow-big-block") carFile.Close() assert.Equal(t, 0, res.ExitCode()) }) }) t.Run("ipfs add non-raw-leaves", func(t *testing.T) { t.Parallel() // The chunker enforces ChunkSizeLimit (maxChunkSize = 2MiB - 256 // as of boxo 2026Q1) regardless of leaf type. It does not know at parse time whether // raw or wrapped leaves will be used, so the 256-byte overhead // budget is applied uniformly. // // With --raw-leaves=false each chunk is wrapped in protobuf, // adding ~14 bytes overhead that pushes blocks past the chunk size. // The overhead budget ensures the wrapped block stays within 2MiB. // // With --raw-leaves=true there is no protobuf wrapper, so the // block is exactly the chunk size (maxChunkSize). The 256-byte // budget is unused in this case but the chunker still enforces it. // A full 2MiB chunk (--chunker=size-2097152) is rejected even // though the resulting raw block would fit within BlockSizeLimit. t.Run("1MiB chunk with protobuf wrapping succeeds under 2MiB limit", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() data := make([]byte, twoMiB) res := node.RunPipeToIPFS(bytes.NewReader(data), "add", "-q", "--chunker=size-1048576", "--raw-leaves=false") require.Equal(t, 0, res.ExitCode(), "stderr: %s", res.Stderr.String()) root := strings.TrimSpace(res.Stdout.String()) // the last line of `ipfs add -q` is the root CID lines := strings.Split(root, "\n") root = lines[len(lines)-1] assertAllBlocksWithinLimit(t, node, root) }) t.Run("max chunk with protobuf wrapping stays within block limit", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() // maxChunkSize leaves room for protobuf framing overhead data := make([]byte, maxChunkSize*2) res := node.RunPipeToIPFS(bytes.NewReader(data), "add", "-q", fmt.Sprintf("--chunker=size-%d", maxChunkSize), "--raw-leaves=false") require.Equal(t, 0, res.ExitCode(), "stderr: %s", res.Stderr.String()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") root := lines[len(lines)-1] assertAllBlocksWithinLimit(t, node, root) }) t.Run("chunk size over limit is rejected by chunker", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() data := make([]byte, twoMiB+twoMiB) res := node.RunPipeToIPFS(bytes.NewReader(data), "add", "-q", fmt.Sprintf("--chunker=size-%d", overMaxChunk), "--raw-leaves=false") assert.NotEqual(t, 0, res.ExitCode()) assert.Contains(t, res.Stderr.String(), fmt.Sprintf("chunker parameters may not exceed the maximum chunk size of %d", maxChunkSize)) }) t.Run("max chunk with raw leaves succeeds", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() // raw leaves have no protobuf wrapper, so max chunk size fits easily data := make([]byte, maxChunkSize*2) res := node.RunPipeToIPFS(bytes.NewReader(data), "add", "-q", fmt.Sprintf("--chunker=size-%d", maxChunkSize), "--raw-leaves=true") require.Equal(t, 0, res.ExitCode(), "stderr: %s", res.Stderr.String()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") root := lines[len(lines)-1] assertAllBlocksWithinLimit(t, node, root) }) }) t.Run("bitswap exchange", func(t *testing.T) { t.Parallel() t.Run("2MiB raw block transfers between peers", func(t *testing.T) { t.Parallel() h := harness.NewT(t) provider := h.NewNode().Init("--profile=unixfs-v1-2025").StartDaemon() defer provider.StopDaemon() requester := h.NewNode().Init("--profile=unixfs-v1-2025").StartDaemon() defer requester.StopDaemon() data := make([]byte, twoMiB) _, err := rand.Read(data) require.NoError(t, err) cid := strings.TrimSpace( provider.PipeToIPFS(bytes.NewReader(data), "block", "put").Stdout.String(), ) requester.Connect(provider) res := requester.IPFS("block", "get", cid) assert.Equal(t, data, res.Stdout.Bytes(), "retrieved block should match original") }) t.Run("unixfs-v1-2025: 2MiB file transfers between peers", func(t *testing.T) { t.Parallel() h := harness.NewT(t) provider := h.NewNode().Init("--profile=unixfs-v1-2025").StartDaemon() defer provider.StopDaemon() requester := h.NewNode().Init("--profile=unixfs-v1-2025").StartDaemon() defer requester.StopDaemon() // unixfs-v1-2025 profile uses CIDv1, raw leaves, SHA2-256, // and 1MiB chunks. A 2MiB file produces two 1MiB raw leaf // blocks plus a root node, all within the 2MiB spec limit. data := make([]byte, twoMiB) _, err := rand.Read(data) require.NoError(t, err) res := provider.RunPipeToIPFS(bytes.NewReader(data), "add", "-q") require.Equal(t, 0, res.ExitCode(), "stderr: %s", res.Stderr.String()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") root := lines[len(lines)-1] requester.Connect(provider) got := requester.IPFS("cat", root) assert.Equal(t, data, got.Stdout.Bytes(), "retrieved file should match original") }) // The following two tests guard the physical hard limit of the // libp2p transport layer (network.MessageSizeMax = 4MiB). This is // the actual ceiling for bitswap block transfer, independent of the // 2MiB soft limit from the bitswap spec. Knowing the exact hard // limit is important for backward-compatible protocol and standards // evolution: any future increase to the bitswap spec block size // must stay within the libp2p message framing budget, or the // transport layer must be updated first. t.Run("bitswap-over-libp2p: largest block that fits in message transfers", func(t *testing.T) { t.Parallel() h := harness.NewT(t) provider := h.NewNode().Init("--profile=unixfs-v1-2025").StartDaemon() defer provider.StopDaemon() requester := h.NewNode().Init("--profile=unixfs-v1-2025").StartDaemon() defer requester.StopDaemon() data := make([]byte, maxTransferBlock) _, err := rand.Read(data) require.NoError(t, err) cid := strings.TrimSpace( provider.PipeToIPFS(bytes.NewReader(data), "block", "put", "--allow-big-block").Stdout.String(), ) requester.Connect(provider) // successful transfers complete in ~1s timeout := time.After(5 * time.Second) dataChan := make(chan []byte, 1) go func() { res := requester.RunIPFS("block", "get", cid) dataChan <- res.Stdout.Bytes() }() select { case got := <-dataChan: assert.Equal(t, data, got, "retrieved block should match original") case <-timeout: t.Fatal("block get timed out: expected transfer to succeed at maxTransferBlock") } }) t.Run("bitswap-over-libp2p: one byte over message limit does not transfer", func(t *testing.T) { t.Parallel() h := harness.NewT(t) provider := h.NewNode().Init("--profile=unixfs-v1-2025").StartDaemon() defer provider.StopDaemon() requester := h.NewNode().Init("--profile=unixfs-v1-2025").StartDaemon() defer requester.StopDaemon() data := make([]byte, overMaxTransfer) _, err := rand.Read(data) require.NoError(t, err) cid := strings.TrimSpace( provider.PipeToIPFS(bytes.NewReader(data), "block", "put", "--allow-big-block").Stdout.String(), ) requester.Connect(provider) timeout := time.After(5 * time.Second) dataChan := make(chan []byte, 1) go func() { res := requester.RunIPFS("block", "get", cid) dataChan <- res.Stdout.Bytes() }() select { case got := <-dataChan: t.Fatalf("expected timeout, but block was retrieved (%d bytes)", len(got)) case <-timeout: t.Log("block get timed out as expected: block exceeds libp2p message size limit") } }) }) } ================================================ FILE: test/cli/bootstrap_auto_test.go ================================================ package cli import ( "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestBootstrapCommandsWithAutoPlaceholder(t *testing.T) { t.Parallel() t.Run("bootstrap add default", func(t *testing.T) { t.Parallel() // Test that 'ipfs bootstrap add default' works correctly node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{}) // Start with empty bootstrap // Add default bootstrap peers via "auto" placeholder result := node.RunIPFS("bootstrap", "add", "default") require.Equal(t, 0, result.ExitCode(), "bootstrap add default should succeed") output := result.Stdout.String() t.Logf("Bootstrap add default output: %s", output) assert.Contains(t, output, "added auto", "bootstrap add default should report adding 'auto'") // Verify bootstrap list shows "auto" listResult := node.RunIPFS("bootstrap", "list") require.Equal(t, 0, listResult.ExitCode(), "bootstrap list should succeed") listOutput := listResult.Stdout.String() t.Logf("Bootstrap list after add default: %s", listOutput) assert.Contains(t, listOutput, "auto", "bootstrap list should show 'auto' placeholder") }) t.Run("bootstrap add auto explicitly", func(t *testing.T) { t.Parallel() // Test that 'ipfs bootstrap add auto' works correctly node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{}) // Start with empty bootstrap // Add "auto" placeholder explicitly result := node.RunIPFS("bootstrap", "add", "auto") require.Equal(t, 0, result.ExitCode(), "bootstrap add auto should succeed") output := result.Stdout.String() t.Logf("Bootstrap add auto output: %s", output) assert.Contains(t, output, "added auto", "bootstrap add auto should report adding 'auto'") // Verify bootstrap list shows "auto" listResult := node.RunIPFS("bootstrap", "list") require.Equal(t, 0, listResult.ExitCode(), "bootstrap list should succeed") listOutput := listResult.Stdout.String() t.Logf("Bootstrap list after add auto: %s", listOutput) assert.Contains(t, listOutput, "auto", "bootstrap list should show 'auto' placeholder") }) t.Run("bootstrap add default converts to auto", func(t *testing.T) { t.Parallel() // Test that 'ipfs bootstrap add default' adds "auto" to the bootstrap list node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("Bootstrap", []string{}) // Start with empty bootstrap node.SetIPFSConfig("AutoConf.Enabled", true) // Enable AutoConf to allow adding "auto" // Add default bootstrap peers result := node.RunIPFS("bootstrap", "add", "default") require.Equal(t, 0, result.ExitCode(), "bootstrap add default should succeed") assert.Contains(t, result.Stdout.String(), "added auto", "should report adding 'auto'") // Verify bootstrap list shows "auto" var bootstrap []string node.GetIPFSConfig("Bootstrap", &bootstrap) require.Equal(t, []string{"auto"}, bootstrap, "Bootstrap should contain ['auto']") }) t.Run("bootstrap add default fails when AutoConf disabled", func(t *testing.T) { t.Parallel() // Test that adding default/auto fails when AutoConf is disabled node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("Bootstrap", []string{}) // Start with empty bootstrap node.SetIPFSConfig("AutoConf.Enabled", false) // Disable AutoConf // Try to add default - should fail result := node.RunIPFS("bootstrap", "add", "default") require.NotEqual(t, 0, result.ExitCode(), "bootstrap add default should fail when AutoConf disabled") assert.Contains(t, result.Stderr.String(), "AutoConf is disabled", "should mention AutoConf is disabled") // Try to add auto - should also fail result = node.RunIPFS("bootstrap", "add", "auto") require.NotEqual(t, 0, result.ExitCode(), "bootstrap add auto should fail when AutoConf disabled") assert.Contains(t, result.Stderr.String(), "AutoConf is disabled", "should mention AutoConf is disabled") }) t.Run("bootstrap rm with auto placeholder", func(t *testing.T) { t.Parallel() // Test that selective removal fails properly when "auto" is present node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Start with auto // Try to remove a specific peer - should fail with helpful error result := node.RunIPFS("bootstrap", "rm", "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN") require.NotEqual(t, 0, result.ExitCode(), "bootstrap rm of specific peer should fail when 'auto' is present") output := result.Stderr.String() t.Logf("Bootstrap rm error output: %s", output) assert.Contains(t, output, "cannot remove individual bootstrap peers when using 'auto' placeholder", "should provide helpful error message about auto placeholder") assert.Contains(t, output, "disable AutoConf", "should suggest disabling AutoConf as solution") assert.Contains(t, output, "ipfs bootstrap rm --all", "should suggest using rm --all as alternative") }) t.Run("bootstrap rm --all with auto placeholder", func(t *testing.T) { t.Parallel() // Test that 'ipfs bootstrap rm --all' works with "auto" placeholder node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Start with auto // Remove all bootstrap peers result := node.RunIPFS("bootstrap", "rm", "--all") require.Equal(t, 0, result.ExitCode(), "bootstrap rm --all should succeed with auto placeholder") output := result.Stdout.String() t.Logf("Bootstrap rm --all output: %s", output) assert.Contains(t, output, "removed auto", "bootstrap rm --all should report removing 'auto'") // Verify bootstrap list is now empty listResult := node.RunIPFS("bootstrap", "list") require.Equal(t, 0, listResult.ExitCode(), "bootstrap list should succeed") listOutput := listResult.Stdout.String() t.Logf("Bootstrap list after rm --all: %s", listOutput) assert.Empty(t, listOutput, "bootstrap list should be empty after rm --all") // Test the rm all subcommand too node.SetIPFSConfig("Bootstrap", []string{"auto"}) // Reset to auto result = node.RunIPFS("bootstrap", "rm", "all") require.Equal(t, 0, result.ExitCode(), "bootstrap rm all should succeed with auto placeholder") output = result.Stdout.String() t.Logf("Bootstrap rm all output: %s", output) assert.Contains(t, output, "removed auto", "bootstrap rm all should report removing 'auto'") }) t.Run("bootstrap mixed auto and specific peers", func(t *testing.T) { t.Parallel() // Test that bootstrap commands work when mixing "auto" with specific peers node := harness.NewT(t).NewNode().Init("--profile=test") node.SetIPFSConfig("AutoConf.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{}) // Start with empty bootstrap // Add a specific peer first specificPeer := "/ip4/127.0.0.1/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" result := node.RunIPFS("bootstrap", "add", specificPeer) require.Equal(t, 0, result.ExitCode(), "bootstrap add specific peer should succeed") // Add auto placeholder result = node.RunIPFS("bootstrap", "add", "auto") require.Equal(t, 0, result.ExitCode(), "bootstrap add auto should succeed") // Verify bootstrap list shows both listResult := node.RunIPFS("bootstrap", "list") require.Equal(t, 0, listResult.ExitCode(), "bootstrap list should succeed") listOutput := listResult.Stdout.String() t.Logf("Bootstrap list with mixed peers: %s", listOutput) assert.Contains(t, listOutput, "auto", "bootstrap list should contain 'auto' placeholder") assert.Contains(t, listOutput, specificPeer, "bootstrap list should contain specific peer") // Try to remove the specific peer - should fail because auto is present result = node.RunIPFS("bootstrap", "rm", specificPeer) require.NotEqual(t, 0, result.ExitCode(), "bootstrap rm of specific peer should fail when 'auto' is present") output := result.Stderr.String() assert.Contains(t, output, "cannot remove individual bootstrap peers when using 'auto' placeholder", "should provide helpful error message about auto placeholder") // Remove all should work and remove both auto and specific peer result = node.RunIPFS("bootstrap", "rm", "--all") require.Equal(t, 0, result.ExitCode(), "bootstrap rm --all should succeed") output = result.Stdout.String() t.Logf("Bootstrap rm --all output with mixed peers: %s", output) // Should report removing both the specific peer and auto assert.Contains(t, output, "removed", "should report removing peers") // Verify bootstrap list is now empty listResult = node.RunIPFS("bootstrap", "list") require.Equal(t, 0, listResult.ExitCode(), "bootstrap list should succeed") listOutput = listResult.Stdout.String() assert.Empty(t, listOutput, "bootstrap list should be empty after rm --all") }) } ================================================ FILE: test/cli/cid_profiles_test.go ================================================ package cli import ( "encoding/json" "os" "path/filepath" "strings" "testing" ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // cidProfileExpectations defines expected behaviors for a UnixFS import profile. // This allows DRY testing of multiple profiles with the same test logic. // // Each profile is tested against threshold boundaries to verify: // - CID format (version, hash function, raw leaves vs dag-pb wrapped) // - File chunking (UnixFSChunker size threshold) // - DAG structure (UnixFSFileMaxLinks rebalancing threshold) // - Directory sharding (HAMTThreshold for flat vs HAMT directories) type cidProfileExpectations struct { // Profile identification Name string // canonical profile name from IPIP-499 ProfileArgs []string // args to pass to ipfs init (empty for default behavior) // CID format expectations CIDVersion int // 0 or 1 HashFunc string // e.g., "sha2-256" RawLeaves bool // true = raw codec for small files, false = dag-pb wrapped // File chunking expectations (UnixFSChunker config) ChunkSize int // chunk size in bytes (e.g., 262144 for 256KiB, 1048576 for 1MiB) ChunkSizeHuman string // human-readable chunk size (e.g., "256KiB", "1MiB") FileMaxLinks int // max links before DAG rebalancing (UnixFSFileMaxLinks config) // HAMT directory sharding expectations (UnixFSHAMTDirectory* config). // Threshold behavior: boxo converts to HAMT when size > HAMTThreshold (not >=). // This means a directory exactly at the threshold stays as a basic (flat) directory. HAMTFanout int // max links per HAMT shard bucket (256) HAMTThreshold int // sharding threshold in bytes (262144 = 256 KiB) HAMTSizeEstimation string // "block" (protobuf size) or "links" (legacy name+cid) // Test vector parameters for threshold boundary tests. // - DirBasic: size == threshold (stays basic) // - DirHAMT: size > threshold (converts to HAMT) // For block estimation, last filename length is adjusted to hit exact thresholds. DirBasicNameLen int // filename length for basic directory (files 0 to N-2) DirBasicLastNameLen int // filename length for last file (0 = same as DirBasicNameLen) DirBasicFiles int // file count for basic directory (at exact threshold) DirHAMTNameLen int // filename length for HAMT directory (files 0 to N-2) DirHAMTLastNameLen int // filename length for last file (0 = same as DirHAMTNameLen) DirHAMTFiles int // total file count for HAMT directory (over threshold) // Expected deterministic CIDs for test vectors. // These serve as regression tests to detect unintended changes in CID generation. // SmallFileCID is the deterministic CID for "hello world" string. // Tests basic CID format (version, codec, hash). SmallFileCID string // FileAtChunkSizeCID is the deterministic CID for a file exactly at chunk size. // This file fits in a single block with no links: // - v0-2015: dag-pb wrapped TFile node (CIDv0) // - v1-2025: raw leaf block (CIDv1) FileAtChunkSizeCID string // FileOverChunkSizeCID is the deterministic CID for a file 1 byte over chunk size. // This file requires 2 chunks, producing a root dag-pb node with 2 links: // - v0-2015: links point to dag-pb wrapped TFile leaf nodes // - v1-2025: links point to raw leaf blocks FileOverChunkSizeCID string // FileAtMaxLinksCID is the deterministic CID for a file at UnixFSFileMaxLinks threshold. // File size = maxLinks * chunkSize, producing a single-layer DAG with exactly maxLinks children. FileAtMaxLinksCID string // FileOverMaxLinksCID is the deterministic CID for a file 1 byte over max links threshold. // The +1 byte requires an additional chunk, forcing DAG rebalancing to 2 layers. FileOverMaxLinksCID string // DirBasicCID is the deterministic CID for a directory exactly at HAMTThreshold. // With > comparison (not >=), directory at exact threshold stays as basic (flat) directory. DirBasicCID string // DirHAMTCID is the deterministic CID for a directory 1 byte over HAMTThreshold. // Crossing the threshold converts the directory to a HAMT sharded structure. DirHAMTCID string } // unixfsV02015 is the legacy profile for backward-compatible CID generation. // Alias: legacy-cid-v0 var unixfsV02015 = cidProfileExpectations{ Name: "unixfs-v0-2015", ProfileArgs: []string{"--profile=unixfs-v0-2015"}, CIDVersion: 0, HashFunc: "sha2-256", RawLeaves: false, ChunkSize: 262144, // 256 KiB ChunkSizeHuman: "256KiB", FileMaxLinks: 174, HAMTFanout: 256, HAMTThreshold: 262144, // 256 KiB HAMTSizeEstimation: "links", DirBasicNameLen: 30, // 4096 * (30 + 34) = 262144 exactly at threshold DirBasicFiles: 4096, // 4096 * 64 = 262144 (stays basic with >) DirHAMTNameLen: 31, // 4033 * (31 + 34) = 262145 exactly +1 over threshold DirHAMTLastNameLen: 0, // 0 = same as DirHAMTNameLen (uniform filenames) DirHAMTFiles: 4033, // 4033 * 65 = 262145 (becomes HAMT) SmallFileCID: "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD", // "hello world" dag-pb wrapped FileAtChunkSizeCID: "QmWmRj3dFDZdb6ABvbmKhEL6TmPbAfBZ1t5BxsEyJrcZhE", // 262144 bytes with seed "chunk-v0-seed" FileOverChunkSizeCID: "QmYyLxtzZyW22zpoVAtKANLRHpDjZtNeDjQdJrcQNWoRkJ", // 262145 bytes with seed "chunk-v0-seed" FileAtMaxLinksCID: "QmUbBALi174SnogsUzLpYbD4xPiBSFANF4iztWCsHbMKh2", // 174*256KiB bytes with seed "v0-seed" FileOverMaxLinksCID: "QmV81WL765sC8DXsRhE5fJv2rwhS4icHRaf3J9Zk5FdRnW", // 174*256KiB+1 bytes with seed "v0-seed" DirBasicCID: "QmX5GtRk3TSSEHtdrykgqm4eqMEn3n2XhfkFAis5fjyZmN", // 4096 files at threshold DirHAMTCID: "QmeMiJzmhpJAUgynAcxTQYek5PPKgdv3qEvFsdV3XpVnvP", // 4033 files +1 over threshold } // unixfsV12025 is the recommended profile for cross-implementation CID determinism. var unixfsV12025 = cidProfileExpectations{ Name: "unixfs-v1-2025", ProfileArgs: []string{"--profile=unixfs-v1-2025"}, CIDVersion: 1, HashFunc: "sha2-256", RawLeaves: true, ChunkSize: 1048576, // 1 MiB ChunkSizeHuman: "1MiB", FileMaxLinks: 1024, HAMTFanout: 256, HAMTThreshold: 262144, // 256 KiB HAMTSizeEstimation: "block", // Block size = numFiles * linkSize + 4 bytes overhead // LinkSerializedSize(11, 36, 1) = 55, LinkSerializedSize(21, 36, 1) = 65, LinkSerializedSize(22, 36, 1) = 66 DirBasicNameLen: 11, // 4765 files * 55 bytes DirBasicLastNameLen: 21, // last file: 65 bytes; total: 4765*55 + 65 + 4 = 262144 (at threshold) DirBasicFiles: 4766, // stays basic with > comparison DirHAMTNameLen: 11, // 4765 files * 55 bytes DirHAMTLastNameLen: 22, // last file: 66 bytes; total: 4765*55 + 66 + 4 = 262145 (+1 over threshold) DirHAMTFiles: 4766, // becomes HAMT SmallFileCID: "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e", // "hello world" raw leaf FileAtChunkSizeCID: "bafkreiacndfy443ter6qr2tmbbdhadvxxheowwf75s6zehscklu6ezxmta", // 1048576 bytes with seed "chunk-v1-seed" FileOverChunkSizeCID: "bafybeigmix7t42i6jacydtquhet7srwvgpizfg7gjbq7627d35mjomtu64", // 1048577 bytes with seed "chunk-v1-seed" FileAtMaxLinksCID: "bafybeihmf37wcuvtx4hpu7he5zl5qaf2ineo2lqlfrapokkm5zzw7zyhvm", // 1024*1MiB bytes with seed "v1-2025-seed" FileOverMaxLinksCID: "bafybeibdsi225ugbkmpbdohnxioyab6jsqrmkts3twhpvfnzp77xtzpyhe", // 1024*1MiB+1 bytes with seed "v1-2025-seed" DirBasicCID: "bafybeic3h7rwruealwxkacabdy45jivq2crwz6bufb5ljwupn36gicplx4", // 4766 files at 262144 bytes (threshold) DirHAMTCID: "bafybeiegvuterwurhdtkikfhbxcldohmxp566vpjdofhzmnhv6o4freidu", // 4766 files at 262145 bytes (+1 over) } // defaultProfile points to the profile that matches Kubo's implicit default behavior. // Today this is unixfs-v0-2015. When Kubo changes defaults, update this pointer. var defaultProfile = unixfsV02015 const ( cidV0Length = 34 // CIDv0 sha2-256 cidV1Length = 36 // CIDv1 sha2-256 ) // TestCIDProfiles generates deterministic test vectors for CID profile verification. // Set CID_PROFILES_CAR_OUTPUT environment variable to export CAR files. // Example: CID_PROFILES_CAR_OUTPUT=/tmp/cid-profiles go test -run TestCIDProfiles -v func TestCIDProfiles(t *testing.T) { t.Parallel() carOutputDir := os.Getenv("CID_PROFILES_CAR_OUTPUT") exportCARs := carOutputDir != "" if exportCARs { if err := os.MkdirAll(carOutputDir, 0o755); err != nil { t.Fatalf("failed to create CAR output directory: %v", err) } t.Logf("CAR export enabled, writing to: %s", carOutputDir) } // Test both IPIP-499 profiles for _, profile := range []cidProfileExpectations{unixfsV02015, unixfsV12025} { t.Run(profile.Name, func(t *testing.T) { t.Parallel() runProfileTests(t, profile, carOutputDir, exportCARs) }) } // Test default behavior (no profile specified) t.Run("default", func(t *testing.T) { t.Parallel() // Default behavior should match defaultProfile (currently unixfs-v0-2015) defaultExp := defaultProfile defaultExp.Name = "default" defaultExp.ProfileArgs = nil // no profile args = default behavior runProfileTests(t, defaultExp, carOutputDir, exportCARs) }) } // runProfileTests runs all test vectors for a given profile. // Tests verify threshold behaviors for: // - Small files (CID format verification) // - UnixFSChunker threshold (single block vs multi-block) // - UnixFSFileMaxLinks threshold (single-layer vs rebalanced DAG) // - HAMTThreshold (basic flat directory vs HAMT sharded) func runProfileTests(t *testing.T, exp cidProfileExpectations, carOutputDir string, exportCARs bool) { cidLen := cidV0Length if exp.CIDVersion == 1 { cidLen = cidV1Length } // Test: small file produces correct CID format // Verifies the profile sets the expected CID version, hash function, and leaf encoding. t.Run("small file produces correct CID format", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) node.StartDaemon() defer node.StopDaemon() // Use "hello world" for determinism cidStr := node.IPFSAddStr("hello world") // Verify CID version (v0 starts with "Qm", v1 with "b") verifyCIDVersion(t, node, cidStr, exp.CIDVersion) // Verify hash function (sha2-256 for both profiles) verifyHashFunction(t, node, cidStr, exp.HashFunc) // Verify raw leaves vs dag-pb wrapped // - v0-2015: dag-pb codec (wrapped) // - v1-2025: raw codec (raw leaves) verifyRawLeaves(t, node, cidStr, exp.RawLeaves) // Verify deterministic CID matches expected value if exp.SmallFileCID != "" { require.Equal(t, exp.SmallFileCID, cidStr, "expected deterministic CID for small file") } if exportCARs { carPath := filepath.Join(carOutputDir, exp.Name+"_small-file.car") require.NoError(t, node.IPFSDagExport(cidStr, carPath)) t.Logf("exported: %s -> %s", cidStr, carPath) } }) // Test: file at UnixFSChunker threshold (single block) // A file exactly at chunk size fits in one block with no links. // - v0-2015 (256KiB): produces dag-pb wrapped TFile node // - v1-2025 (1MiB): produces raw leaf block t.Run("file at UnixFSChunker threshold (single block)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) node.StartDaemon() defer node.StopDaemon() // File exactly at chunk size = single block (no links) seed := chunkSeedForProfile(exp) cidStr := node.IPFSAddDeterministicBytes(int64(exp.ChunkSize), seed) // Verify block structure based on raw leaves setting if exp.RawLeaves { // v1-2025: single block is a raw leaf (no dag-pb structure) codec := node.IPFS("cid", "format", "-f", "%c", cidStr).Stdout.Trimmed() require.Equal(t, "raw", codec, "single block file is raw leaf") } else { // v0-2015: single block is a dag-pb node with no links (TFile type) root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) require.Equal(t, 0, len(root.Links), "single block file has no links") fsType, err := node.UnixFSDataType(cidStr) require.NoError(t, err) require.Equal(t, ft.TFile, fsType, "single block file is dag-pb wrapped (TFile)") } verifyHashFunction(t, node, cidStr, exp.HashFunc) if exp.FileAtChunkSizeCID != "" { require.Equal(t, exp.FileAtChunkSizeCID, cidStr, "expected deterministic CID for file at chunk size") } if exportCARs { carPath := filepath.Join(carOutputDir, exp.Name+"_file-at-chunk-size.car") require.NoError(t, node.IPFSDagExport(cidStr, carPath)) t.Logf("exported: %s -> %s", cidStr, carPath) } }) // Test: file 1 byte over UnixFSChunker threshold (2 blocks) // A file 1 byte over chunk size requires 2 chunks. // Root is a dag-pb node with 2 links. Leaf encoding depends on profile: // - v0-2015: leaf blocks are dag-pb wrapped TFile nodes // - v1-2025: leaf blocks are raw codec blocks t.Run("file 1 byte over UnixFSChunker threshold (2 blocks)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) node.StartDaemon() defer node.StopDaemon() // File +1 byte over chunk size = 2 blocks seed := chunkSeedForProfile(exp) cidStr := node.IPFSAddDeterministicBytes(int64(exp.ChunkSize)+1, seed) root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) require.Equal(t, 2, len(root.Links), "file over chunk size has 2 links") // Verify leaf block encoding for _, link := range root.Links { if exp.RawLeaves { // v1-2025: leaves are raw blocks leafCodec := node.IPFS("cid", "format", "-f", "%c", link.Hash.Slash).Stdout.Trimmed() require.Equal(t, "raw", leafCodec, "leaf blocks are raw, not dag-pb") } else { // v0-2015: leaves are dag-pb wrapped (TFile type) leafType, err := node.UnixFSDataType(link.Hash.Slash) require.NoError(t, err) require.Equal(t, ft.TFile, leafType, "leaf blocks are dag-pb wrapped (TFile)") } } verifyHashFunction(t, node, cidStr, exp.HashFunc) if exp.FileOverChunkSizeCID != "" { require.Equal(t, exp.FileOverChunkSizeCID, cidStr, "expected deterministic CID for file over chunk size") } if exportCARs { carPath := filepath.Join(carOutputDir, exp.Name+"_file-over-chunk-size.car") require.NoError(t, node.IPFSDagExport(cidStr, carPath)) t.Logf("exported: %s -> %s", cidStr, carPath) } }) // Test: file at UnixFSFileMaxLinks threshold (single layer) // A file of exactly maxLinks * chunkSize bytes fits in a single DAG layer. // - v0-2015: 174 links (174 * 256KiB = ~44.6MiB) // - v1-2025: 1024 links (1024 * 1MiB = 1GiB) t.Run("file at UnixFSFileMaxLinks threshold (single layer)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) node.StartDaemon() defer node.StopDaemon() // File size = maxLinks * chunkSize (exactly at threshold) fileSize := fileAtMaxLinksBytes(exp) seed := seedForProfile(exp) cidStr := node.IPFSAddDeterministicBytes(fileSize, seed) root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) require.Equal(t, exp.FileMaxLinks, len(root.Links), "expected exactly %d links at max", exp.FileMaxLinks) verifyHashFunction(t, node, cidStr, exp.HashFunc) if exp.FileAtMaxLinksCID != "" { require.Equal(t, exp.FileAtMaxLinksCID, cidStr, "expected deterministic CID for file at max links") } if exportCARs { carPath := filepath.Join(carOutputDir, exp.Name+"_file-at-max-links.car") require.NoError(t, node.IPFSDagExport(cidStr, carPath)) t.Logf("exported: %s -> %s", cidStr, carPath) } }) // Test: file 1 byte over UnixFSFileMaxLinks threshold (rebalanced DAG) // Adding 1 byte requires an additional chunk, exceeding maxLinks. // This triggers DAG rebalancing: chunks are grouped into intermediate nodes, // producing a 2-layer DAG with 2 links at the root. t.Run("file 1 byte over UnixFSFileMaxLinks threshold (rebalanced DAG)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) node.StartDaemon() defer node.StopDaemon() // +1 byte over max links threshold triggers DAG rebalancing fileSize := fileOverMaxLinksBytes(exp) seed := seedForProfile(exp) cidStr := node.IPFSAddDeterministicBytes(fileSize, seed) root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) require.Equal(t, 2, len(root.Links), "expected 2 links after DAG rebalancing") verifyHashFunction(t, node, cidStr, exp.HashFunc) if exp.FileOverMaxLinksCID != "" { require.Equal(t, exp.FileOverMaxLinksCID, cidStr, "expected deterministic CID for rebalanced file") } if exportCARs { carPath := filepath.Join(carOutputDir, exp.Name+"_file-over-max-links.car") require.NoError(t, node.IPFSDagExport(cidStr, carPath)) t.Logf("exported: %s -> %s", cidStr, carPath) } }) // Test: directory at HAMTThreshold (basic flat dir) // A directory exactly at HAMTThreshold stays as a basic (flat) UnixFS directory. // Threshold uses > comparison (not >=), so size == threshold stays basic. // Size estimation method depends on profile: // - v0-2015 "links": size = sum(nameLen + cidLen) // - v1-2025 "block": size = serialized protobuf block size t.Run("directory at HAMTThreshold (basic flat dir)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) node.StartDaemon() defer node.StopDaemon() // Use consistent seed for deterministic CIDs seed := hamtSeedForProfile(exp) randDir, err := os.MkdirTemp(node.Dir, seed) require.NoError(t, err) // Create basic (flat) directory exactly at threshold basicLastNameLen := exp.DirBasicLastNameLen if basicLastNameLen == 0 { basicLastNameLen = exp.DirBasicNameLen } if exp.HAMTSizeEstimation == "block" { err = createDirectoryForHAMTBlockEstimation(randDir, exp.DirBasicFiles, exp.DirBasicNameLen, basicLastNameLen, seed) } else { err = createDirectoryForHAMTLinksEstimation(randDir, exp.DirBasicFiles, exp.DirBasicNameLen, basicLastNameLen, seed) } require.NoError(t, err) cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() // Verify UnixFS type is TDirectory (1), not THAMTShard (5) fsType, err := node.UnixFSDataType(cidStr) require.NoError(t, err) require.Equal(t, ft.TDirectory, fsType, "expected basic directory (type=1) at exact threshold") root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) require.Equal(t, exp.DirBasicFiles, len(root.Links), "expected basic directory with %d links", exp.DirBasicFiles) verifyHashFunction(t, node, cidStr, exp.HashFunc) // Verify size is exactly at threshold if exp.HAMTSizeEstimation == "block" { blockSize := getBlockSize(t, node, cidStr) require.Equal(t, exp.HAMTThreshold, blockSize, "expected basic directory block size to be exactly at threshold (%d), got %d", exp.HAMTThreshold, blockSize) } if exp.HAMTSizeEstimation == "links" { linksSize := 0 for _, link := range root.Links { linksSize += len(link.Name) + cidLen } require.Equal(t, exp.HAMTThreshold, linksSize, "expected basic directory links size to be exactly at threshold (%d), got %d", exp.HAMTThreshold, linksSize) } if exp.DirBasicCID != "" { require.Equal(t, exp.DirBasicCID, cidStr, "expected deterministic CID for basic directory") } if exportCARs { carPath := filepath.Join(carOutputDir, exp.Name+"_dir-basic.car") require.NoError(t, node.IPFSDagExport(cidStr, carPath)) t.Logf("exported: %s (%d files) -> %s", cidStr, exp.DirBasicFiles, carPath) } }) // Test: directory 1 byte over HAMTThreshold (HAMT sharded) // A directory 1 byte over HAMTThreshold is converted to a HAMT sharded structure. // HAMT distributes entries across buckets using consistent hashing. // Root has at most HAMTFanout links (256), with entries distributed across buckets. t.Run("directory 1 byte over HAMTThreshold (HAMT sharded)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init(exp.ProfileArgs...) node.StartDaemon() defer node.StopDaemon() // Use consistent seed for deterministic CIDs seed := hamtSeedForProfile(exp) randDir, err := os.MkdirTemp(node.Dir, seed) require.NoError(t, err) // Create HAMT (sharded) directory exactly +1 byte over threshold lastNameLen := exp.DirHAMTLastNameLen if lastNameLen == 0 { lastNameLen = exp.DirHAMTNameLen } if exp.HAMTSizeEstimation == "block" { err = createDirectoryForHAMTBlockEstimation(randDir, exp.DirHAMTFiles, exp.DirHAMTNameLen, lastNameLen, seed) } else { err = createDirectoryForHAMTLinksEstimation(randDir, exp.DirHAMTFiles, exp.DirHAMTNameLen, lastNameLen, seed) } require.NoError(t, err) cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() // Verify UnixFS type is THAMTShard (5), not TDirectory (1) fsType, err := node.UnixFSDataType(cidStr) require.NoError(t, err) require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory (type=5) when over threshold") // HAMT root has at most fanout links (actual count depends on hash distribution) root, err := node.InspectPBNode(cidStr) assert.NoError(t, err) require.LessOrEqual(t, len(root.Links), exp.HAMTFanout, "expected HAMT directory root to have <= %d links", exp.HAMTFanout) verifyHashFunction(t, node, cidStr, exp.HashFunc) if exp.DirHAMTCID != "" { require.Equal(t, exp.DirHAMTCID, cidStr, "expected deterministic CID for HAMT directory") } if exportCARs { carPath := filepath.Join(carOutputDir, exp.Name+"_dir-hamt.car") require.NoError(t, node.IPFSDagExport(cidStr, carPath)) t.Logf("exported: %s (%d files, HAMT root links: %d) -> %s", cidStr, exp.DirHAMTFiles, len(root.Links), carPath) } }) } // verifyCIDVersion checks that the CID has the expected version. func verifyCIDVersion(t *testing.T, _ *harness.Node, cidStr string, expectedVersion int) { t.Helper() if expectedVersion == 0 { require.True(t, strings.HasPrefix(cidStr, "Qm"), "expected CIDv0 (starts with Qm), got: %s", cidStr) } else { require.True(t, strings.HasPrefix(cidStr, "b"), "expected CIDv1 (base32, starts with b), got: %s", cidStr) } } // verifyHashFunction checks that the CID uses the expected hash function. func verifyHashFunction(t *testing.T, node *harness.Node, cidStr, expectedHash string) { t.Helper() // Use ipfs cid format to get hash function info // Format string %h gives the hash function name res := node.IPFS("cid", "format", "-f", "%h", cidStr) hashFunc := strings.TrimSpace(res.Stdout.String()) require.Equal(t, expectedHash, hashFunc, "expected hash function %s, got %s for CID %s", expectedHash, hashFunc, cidStr) } // verifyRawLeaves checks whether the CID represents a raw leaf or dag-pb wrapped block. // For CIDv1: raw leaves have codec 0x55 (raw), wrapped have codec 0x70 (dag-pb). // For CIDv0: always dag-pb (no raw leaves possible). func verifyRawLeaves(t *testing.T, node *harness.Node, cidStr string, expectRaw bool) { t.Helper() // Use ipfs cid format to get codec info // Format string %c gives the codec name res := node.IPFS("cid", "format", "-f", "%c", cidStr) codec := strings.TrimSpace(res.Stdout.String()) if expectRaw { require.Equal(t, "raw", codec, "expected raw codec for raw leaves, got %s for CID %s", codec, cidStr) } else { require.Equal(t, "dag-pb", codec, "expected dag-pb codec for wrapped leaves, got %s for CID %s", codec, cidStr) } } // getBlockSize returns the size of a block in bytes using ipfs block stat. func getBlockSize(t *testing.T, node *harness.Node, cidStr string) int { t.Helper() res := node.IPFS("block", "stat", "--enc=json", cidStr) var stat struct { Size int `json:"Size"` } require.NoError(t, json.Unmarshal(res.Stdout.Bytes(), &stat)) return stat.Size } // fileAtMaxLinksBytes returns the file size in bytes that produces exactly FileMaxLinks chunks. func fileAtMaxLinksBytes(exp cidProfileExpectations) int64 { return int64(exp.FileMaxLinks) * int64(exp.ChunkSize) } // fileOverMaxLinksBytes returns the file size in bytes that triggers DAG rebalancing (+1 byte over max links threshold). func fileOverMaxLinksBytes(exp cidProfileExpectations) int64 { return int64(exp.FileMaxLinks)*int64(exp.ChunkSize) + 1 } // seedForProfile returns the deterministic seed used in add_test.go for file max links tests. func seedForProfile(exp cidProfileExpectations) string { switch exp.Name { case "unixfs-v0-2015", "default": return "v0-seed" case "unixfs-v1-2025": return "v1-2025-seed" default: return exp.Name + "-seed" } } // chunkSeedForProfile returns the deterministic seed for chunk threshold tests. func chunkSeedForProfile(exp cidProfileExpectations) string { switch exp.Name { case "unixfs-v0-2015", "default": return "chunk-v0-seed" case "unixfs-v1-2025": return "chunk-v1-seed" default: return "chunk-" + exp.Name + "-seed" } } // hamtSeedForProfile returns the deterministic seed for HAMT directory tests. // Uses the same seed for both under/at threshold tests to ensure consistency. func hamtSeedForProfile(exp cidProfileExpectations) string { switch exp.Name { case "unixfs-v0-2015", "default": return "hamt-unixfs-v0-2015" case "unixfs-v1-2025": return "hamt-unixfs-v1-2025" default: return "hamt-" + exp.Name } } // TestDefaultMatchesExpectedProfile verifies that default ipfs add behavior // matches the expected profile (currently unixfs-v0-2015). func TestDefaultMatchesExpectedProfile(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.StartDaemon() defer node.StopDaemon() // Small file test cidDefault := node.IPFSAddStr("x") // Same file with explicit profile nodeWithProfile := harness.NewT(t).NewNode().Init(defaultProfile.ProfileArgs...) nodeWithProfile.StartDaemon() defer nodeWithProfile.StopDaemon() cidWithProfile := nodeWithProfile.IPFSAddStr("x") require.Equal(t, cidWithProfile, cidDefault, "default behavior should match %s profile", defaultProfile.Name) } // TestProtobufHelpers verifies the protobuf size calculation helpers. func TestProtobufHelpers(t *testing.T) { t.Parallel() t.Run("VarintLen", func(t *testing.T) { // Varint encoding: 7 bits per byte, MSB indicates continuation cases := []struct { value uint64 expected int }{ {0, 1}, {127, 1}, // 0x7F - max 1-byte varint {128, 2}, // 0x80 - min 2-byte varint {16383, 2}, // 0x3FFF - max 2-byte varint {16384, 3}, // 0x4000 - min 3-byte varint {2097151, 3}, // 0x1FFFFF - max 3-byte varint {2097152, 4}, // 0x200000 - min 4-byte varint {268435455, 4}, // 0xFFFFFFF - max 4-byte varint {268435456, 5}, // 0x10000000 - min 5-byte varint {34359738367, 5}, // 0x7FFFFFFFF - max 5-byte varint } for _, tc := range cases { got := testutils.VarintLen(tc.value) require.Equal(t, tc.expected, got, "VarintLen(%d)", tc.value) } }) t.Run("LinkSerializedSize", func(t *testing.T) { // Test typical cases for directory links cases := []struct { nameLen int cidLen int tsize uint64 expected int }{ // 255-char name, CIDv0 (34 bytes), tsize=0 // Inner: 1+1+34 + 1+2+255 + 1+1 = 296 // Outer: 1 + 2 + 296 = 299 {255, 34, 0, 299}, // 255-char name, CIDv1 (36 bytes), tsize=0 // Inner: 1+1+36 + 1+2+255 + 1+1 = 298 // Outer: 1 + 2 + 298 = 301 {255, 36, 0, 301}, // Short name (10 chars), CIDv1, tsize=0 // Inner: 1+1+36 + 1+1+10 + 1+1 = 52 // Outer: 1 + 1 + 52 = 54 {10, 36, 0, 54}, // 255-char name, CIDv1, large tsize // Inner: 1+1+36 + 1+2+255 + 1+5 = 302 (tsize uses 5-byte varint) // Outer: 1 + 2 + 302 = 305 {255, 36, 34359738367, 305}, } for _, tc := range cases { got := testutils.LinkSerializedSize(tc.nameLen, tc.cidLen, tc.tsize) require.Equal(t, tc.expected, got, "LinkSerializedSize(%d, %d, %d)", tc.nameLen, tc.cidLen, tc.tsize) } }) t.Run("EstimateFilesForBlockThreshold", func(t *testing.T) { threshold := 262144 nameLen := 255 cidLen := 36 var tsize uint64 = 0 numFiles := testutils.EstimateFilesForBlockThreshold(threshold, nameLen, cidLen, tsize) require.Equal(t, 870, numFiles, "expected 870 files for threshold 262144") numFilesUnder := testutils.EstimateFilesForBlockThreshold(threshold-1, nameLen, cidLen, tsize) require.Equal(t, 870, numFilesUnder, "expected 870 files for threshold 262143") numFilesOver := testutils.EstimateFilesForBlockThreshold(262185, nameLen, cidLen, tsize) require.Equal(t, 871, numFilesOver, "expected 871 files for threshold 262185") }) } ================================================ FILE: test/cli/cid_test.go ================================================ package cli import ( "encoding/json" "fmt" "strings" "testing" cid "github.com/ipfs/go-cid" "github.com/ipfs/kubo/test/cli/harness" peer "github.com/libp2p/go-libp2p/core/peer" mhash "github.com/multiformats/go-multihash" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestCidCommands(t *testing.T) { t.Parallel() t.Run("inspect", testCidInspect) t.Run("base32", testCidBase32) t.Run("format", testCidFormat) t.Run("bases", testCidBases) t.Run("codecs", testCidCodecs) t.Run("hashes", testCidHashes) } // testCidInspect tests 'ipfs cid inspect' subcommand func testCidInspect(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() t.Run("CIDv0", func(t *testing.T) { res := node.RunIPFS("cid", "inspect", "QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR") assert.Equal(t, 0, res.ExitCode()) out := res.Stdout.String() assert.Contains(t, out, "CID: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR") assert.Contains(t, out, "Version: 0") assert.Contains(t, out, "Multibase: base58btc (implicit)") assert.Contains(t, out, "Multicodec: dag-pb (0x70, implicit)") assert.Contains(t, out, "Multihash: sha2-256 (0x12, implicit)") assert.Contains(t, out, " Length: 32 bytes") assert.Contains(t, out, " Digest: c3c4733ec8affd06cf9e9ff50ffc6bcd2ec85a6170004bb709669c31de94391a") assert.Contains(t, out, "CIDv0: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR") assert.Contains(t, out, "CIDv1: bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") }) t.Run("CIDv1 base32 dag-pb", func(t *testing.T) { res := node.RunIPFS("cid", "inspect", "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") assert.Equal(t, 0, res.ExitCode()) out := res.Stdout.String() assert.Contains(t, out, "CID: bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") assert.Contains(t, out, "Version: 1") assert.Contains(t, out, "Multibase: base32 (b)") assert.Contains(t, out, "Multicodec: dag-pb (0x70)") assert.Contains(t, out, "Multihash: sha2-256 (0x12)") assert.Contains(t, out, " Length: 32 bytes") assert.Contains(t, out, " Digest: c3c4733ec8affd06cf9e9ff50ffc6bcd2ec85a6170004bb709669c31de94391a") assert.NotContains(t, out, "implicit") assert.Contains(t, out, "CIDv0: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR") assert.Contains(t, out, "CIDv1: bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") }) t.Run("CIDv1 raw codec", func(t *testing.T) { res := node.RunIPFS("cid", "inspect", "bafkreigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") assert.Equal(t, 0, res.ExitCode()) out := res.Stdout.String() assert.Contains(t, out, "CID: bafkreigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") assert.Contains(t, out, "Multibase: base32 (b)") assert.Contains(t, out, "Multicodec: raw (0x55)") assert.Contains(t, out, "Multihash: sha2-256 (0x12)") assert.Contains(t, out, " Length: 32 bytes") assert.Contains(t, out, " Digest: c3c4733ec8affd06cf9e9ff50ffc6bcd2ec85a6170004bb709669c31de94391a") assert.Contains(t, out, "CIDv0: not possible, requires dag-pb (0x70), got raw (0x55)") assert.Contains(t, out, "CIDv1: bafkreigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") }) t.Run("CIDv1 base36", func(t *testing.T) { res := node.RunIPFS("cid", "inspect", "k2jmtxw8rjh1z69c6not3wtdxb0u3urbzhyll1t9jg6ox26dhi5sfi1m") assert.Equal(t, 0, res.ExitCode()) out := res.Stdout.String() assert.Contains(t, out, "CID: k2jmtxw8rjh1z69c6not3wtdxb0u3urbzhyll1t9jg6ox26dhi5sfi1m") assert.Contains(t, out, "Multibase: base36 (k)") assert.Contains(t, out, "Multicodec: dag-pb (0x70)") assert.Contains(t, out, " Digest: c3c4733ec8affd06cf9e9ff50ffc6bcd2ec85a6170004bb709669c31de94391a") assert.Contains(t, out, "CIDv0: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR") assert.Contains(t, out, "CIDv1: bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") }) t.Run("invalid CID", func(t *testing.T) { res := node.RunIPFS("cid", "inspect", "garbage") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "invalid CID") }) t.Run("PeerID as input", func(t *testing.T) { res := node.RunIPFS("cid", "inspect", "12D3KooWD3eckifWpRn9wQpMG9R9hX3sD158z7EqHWmweQAJU5SA") assert.Equal(t, 1, res.ExitCode()) stderr := res.Stderr.String() assert.Contains(t, stderr, "PeerID") assert.Contains(t, stderr, "inspect its CID representation instead") // suggested CID should use base36 (k prefix) assert.Contains(t, stderr, "\n k") }) t.Run("libp2p-key CID uses base36", func(t *testing.T) { // Construct a libp2p-key CIDv1 from a known PeerID pid, err := peer.Decode("12D3KooWD3eckifWpRn9wQpMG9R9hX3sD158z7EqHWmweQAJU5SA") require.NoError(t, err) pidCid := peer.ToCid(pid) cidStr := pidCid.String() res := node.RunIPFS("cid", "inspect", cidStr) assert.Equal(t, 0, res.ExitCode()) out := res.Stdout.String() assert.Contains(t, out, "Multicodec: libp2p-key (0x72)") // CIDv1 should use base36 (k prefix) assert.Contains(t, out, "CIDv1: k") }) t.Run("identity multihash CID", func(t *testing.T) { // raw codec + identity multihash: digest is the raw content ("test" = 74657374) res := node.RunIPFS("cid", "inspect", "bafkqabdumvzxi") assert.Equal(t, 0, res.ExitCode()) out := res.Stdout.String() assert.Contains(t, out, "CID: bafkqabdumvzxi") assert.Contains(t, out, "Multicodec: raw (0x55)") assert.Contains(t, out, "Multihash: identity (0x0)") assert.Contains(t, out, " Length: 4 bytes") assert.Contains(t, out, " Digest: 74657374") }) t.Run("unknown codec", func(t *testing.T) { // Construct a CID with unknown codec 0x9999 mh, err := mhash.Sum([]byte("test"), mhash.SHA2_256, -1) require.NoError(t, err) unknownCID := cid.NewCidV1(0x9999, mh) cidStr := unknownCID.String() res := node.RunIPFS("cid", "inspect", cidStr) assert.Equal(t, 0, res.ExitCode()) out := res.Stdout.String() assert.Contains(t, out, "Multicodec: unknown (0x9999)") assert.Contains(t, out, "not possible, requires dag-pb (0x70), got unknown (0x9999)") }) t.Run("JSON output", func(t *testing.T) { res := node.RunIPFS("cid", "inspect", "--enc=json", "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") assert.Equal(t, 0, res.ExitCode()) var result map[string]any err := json.Unmarshal(res.Stdout.Bytes(), &result) require.NoError(t, err) // multibase.prefix should be a string, not a number mb := result["multibase"].(map[string]any) assert.IsType(t, "", mb["prefix"]) assert.Equal(t, "b", mb["prefix"]) // multihash.length should be a number (bytes) mh := result["multihash"].(map[string]any) assert.Equal(t, float64(32), mh["length"]) // cidV0 should be a clean CID string, no explanatory text cidV0 := result["cidV0"].(string) assert.True(t, strings.HasPrefix(cidV0, "Qm"), "cidV0 should be a valid CIDv0") // cidV1 should be a clean CID string cidV1 := result["cidV1"].(string) assert.True(t, strings.HasPrefix(cidV1, "b"), "cidV1 should be base32 encoded") }) t.Run("JSON output with empty CIDv0", func(t *testing.T) { // raw codec can't be CIDv0 res := node.RunIPFS("cid", "inspect", "--enc=json", "bafkreigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") assert.Equal(t, 0, res.ExitCode()) var result map[string]any err := json.Unmarshal(res.Stdout.Bytes(), &result) require.NoError(t, err) // cidV0 should not be present (omitempty) _, hasCidV0 := result["cidV0"] assert.False(t, hasCidV0, "cidV0 should be omitted when not possible") }) } // testCidBase32 tests 'ipfs cid base32' subcommand // Includes regression tests for https://github.com/ipfs/kubo/issues/9007 func testCidBase32(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() t.Run("converts valid CIDs to base32", func(t *testing.T) { t.Run("CIDv0 to base32", func(t *testing.T) { res := node.RunIPFS("cid", "base32", "QmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo") assert.Equal(t, 0, res.ExitCode()) assert.Equal(t, "bafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa\n", res.Stdout.String()) }) t.Run("CIDv1 base58 to base32", func(t *testing.T) { res := node.RunIPFS("cid", "base32", "zdj7WgefqQm5HogBQ2bckZuTYYDarRTUZi51GYCnerHD2G86j") assert.Equal(t, 0, res.ExitCode()) assert.Equal(t, "bafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa\n", res.Stdout.String()) }) t.Run("already base32 CID remains unchanged", func(t *testing.T) { res := node.RunIPFS("cid", "base32", "bafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa") assert.Equal(t, 0, res.ExitCode()) assert.Equal(t, "bafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa\n", res.Stdout.String()) }) t.Run("multiple valid CIDs", func(t *testing.T) { res := node.RunIPFS("cid", "base32", "QmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo", "bafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa") assert.Equal(t, 0, res.ExitCode()) assert.Empty(t, res.Stderr.String()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") assert.Equal(t, 2, len(lines)) assert.Equal(t, "bafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa", lines[0]) assert.Equal(t, "bafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa", lines[1]) }) }) t.Run("error handling", func(t *testing.T) { // Regression tests for https://github.com/ipfs/kubo/issues/9007 t.Run("returns error code 1 for single invalid CID", func(t *testing.T) { res := node.RunIPFS("cid", "base32", "invalid-cid") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "invalid-cid: invalid cid") assert.Contains(t, res.Stderr.String(), "Error: errors while displaying some entries") }) t.Run("returns error code 1 for mixed valid and invalid CIDs", func(t *testing.T) { res := node.RunIPFS("cid", "base32", "QmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo", "invalid-cid") assert.Equal(t, 1, res.ExitCode()) // Valid CID should be converted and printed to stdout assert.Contains(t, res.Stdout.String(), "bafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa") // Invalid CID error should be printed to stderr assert.Contains(t, res.Stderr.String(), "invalid-cid: invalid cid") assert.Contains(t, res.Stderr.String(), "Error: errors while displaying some entries") }) t.Run("returns error code 1 for stdin with invalid CIDs", func(t *testing.T) { input := "QmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo\nbad-cid\nbafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa" res := node.RunPipeToIPFS(strings.NewReader(input), "cid", "base32") assert.Equal(t, 1, res.ExitCode()) // Valid CIDs should be converted assert.Contains(t, res.Stdout.String(), "bafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa") // Invalid CID error should be in stderr assert.Contains(t, res.Stderr.String(), "bad-cid: invalid cid") }) }) } // testCidFormat tests 'ipfs cid format' subcommand // Includes regression tests for https://github.com/ipfs/kubo/issues/9007 func testCidFormat(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() t.Run("formats CIDs with various options", func(t *testing.T) { t.Run("default format preserves CID", func(t *testing.T) { res := node.RunIPFS("cid", "format", "QmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo") assert.Equal(t, 0, res.ExitCode()) assert.Equal(t, "QmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo\n", res.Stdout.String()) }) t.Run("convert to CIDv1 with base58btc", func(t *testing.T) { res := node.RunIPFS("cid", "format", "-v", "1", "-b", "base58btc", "QmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo") assert.Equal(t, 0, res.ExitCode()) assert.Equal(t, "zdj7WgefqQm5HogBQ2bckZuTYYDarRTUZi51GYCnerHD2G86j\n", res.Stdout.String()) }) t.Run("convert to CIDv0", func(t *testing.T) { res := node.RunIPFS("cid", "format", "-v", "0", "bafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa") assert.Equal(t, 0, res.ExitCode()) assert.Equal(t, "QmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo\n", res.Stdout.String()) }) t.Run("change codec to raw", func(t *testing.T) { res := node.RunIPFS("cid", "format", "--mc", "raw", "-b", "base32", "bafybeievd6mwe6vcwnkwo3eizs3h7w3a34opszbyfxziqdxguhjw7imdve") assert.Equal(t, 0, res.ExitCode()) assert.Equal(t, "bafkreievd6mwe6vcwnkwo3eizs3h7w3a34opszbyfxziqdxguhjw7imdve\n", res.Stdout.String()) }) t.Run("multiple valid CIDs with format options", func(t *testing.T) { res := node.RunIPFS("cid", "format", "-v", "1", "-b", "base58btc", "QmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo", "bafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa") assert.Equal(t, 0, res.ExitCode()) assert.Empty(t, res.Stderr.String()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") assert.Equal(t, 2, len(lines)) assert.Equal(t, "zdj7WgefqQm5HogBQ2bckZuTYYDarRTUZi51GYCnerHD2G86j", lines[0]) assert.Equal(t, "zdj7WgefqQm5HogBQ2bckZuTYYDarRTUZi51GYCnerHD2G86j", lines[1]) }) }) t.Run("error handling", func(t *testing.T) { // Regression tests for https://github.com/ipfs/kubo/issues/9007 t.Run("returns error code 1 for single invalid CID", func(t *testing.T) { res := node.RunIPFS("cid", "format", "not-a-cid") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "not-a-cid: invalid cid") assert.Contains(t, res.Stderr.String(), "Error: errors while displaying some entries") }) t.Run("returns error code 1 for mixed valid and invalid CIDs", func(t *testing.T) { res := node.RunIPFS("cid", "format", "not-a-cid", "QmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo") assert.Equal(t, 1, res.ExitCode()) // Valid CID should be printed to stdout assert.Contains(t, res.Stdout.String(), "QmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo") // Invalid CID error should be printed to stderr assert.Contains(t, res.Stderr.String(), "not-a-cid: invalid cid") assert.Contains(t, res.Stderr.String(), "Error: errors while displaying some entries") }) t.Run("returns error code 1 for stdin with invalid CIDs", func(t *testing.T) { input := "invalid\nQmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo" res := node.RunPipeToIPFS(strings.NewReader(input), "cid", "format", "-v", "1", "-b", "base58btc") assert.Equal(t, 1, res.ExitCode()) // Valid CID should be converted assert.Contains(t, res.Stdout.String(), "zdj7WgefqQm5HogBQ2bckZuTYYDarRTUZi51GYCnerHD2G86j") // Invalid CID error should be in stderr assert.Contains(t, res.Stderr.String(), "invalid: invalid cid") }) }) } // testCidBases tests 'ipfs cid bases' subcommand func testCidBases(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() t.Run("lists available bases", func(t *testing.T) { // This is a regression test to ensure we don't accidentally add or remove support // for multibase encodings. If a new base is intentionally added or removed, // this test should be updated accordingly. expectedBases := []string{ "identity", "base2", "base16", "base16upper", "base32", "base32upper", "base32pad", "base32padupper", "base32hex", "base32hexupper", "base32hexpad", "base32hexpadupper", "base36", "base36upper", "base58btc", "base58flickr", "base64", "base64pad", "base64url", "base64urlpad", "base256emoji", } res := node.RunIPFS("cid", "bases") assert.Equal(t, 0, res.ExitCode()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") assertExactSet(t, "bases", expectedBases, lines) }) t.Run("with --prefix flag shows single letter prefixes", func(t *testing.T) { // Regression test to catch any changes to the output format or supported bases expectedLines := []string{ "identity", "0 base2", "b base32", "B base32upper", "c base32pad", "C base32padupper", "f base16", "F base16upper", "k base36", "K base36upper", "m base64", "M base64pad", "t base32hexpad", "T base32hexpadupper", "u base64url", "U base64urlpad", "v base32hex", "V base32hexupper", "z base58btc", "Z base58flickr", "🚀 base256emoji", } res := node.RunIPFS("cid", "bases", "--prefix") assert.Equal(t, 0, res.ExitCode()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") assertExactSet(t, "bases --prefix output", expectedLines, lines) }) t.Run("with --numeric flag shows numeric codes", func(t *testing.T) { // Regression test to catch any changes to the output format or supported bases expectedLines := []string{ "0 identity", "48 base2", "98 base32", "66 base32upper", "99 base32pad", "67 base32padupper", "102 base16", "70 base16upper", "107 base36", "75 base36upper", "109 base64", "77 base64pad", "116 base32hexpad", "84 base32hexpadupper", "117 base64url", "85 base64urlpad", "118 base32hex", "86 base32hexupper", "122 base58btc", "90 base58flickr", "128640 base256emoji", } res := node.RunIPFS("cid", "bases", "--numeric") assert.Equal(t, 0, res.ExitCode()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") assertExactSet(t, "bases --numeric output", expectedLines, lines) }) t.Run("with both --prefix and --numeric flags", func(t *testing.T) { // Regression test to catch any changes to the output format or supported bases expectedLines := []string{ "0 identity", "0 48 base2", "b 98 base32", "B 66 base32upper", "c 99 base32pad", "C 67 base32padupper", "f 102 base16", "F 70 base16upper", "k 107 base36", "K 75 base36upper", "m 109 base64", "M 77 base64pad", "t 116 base32hexpad", "T 84 base32hexpadupper", "u 117 base64url", "U 85 base64urlpad", "v 118 base32hex", "V 86 base32hexupper", "z 122 base58btc", "Z 90 base58flickr", "🚀 128640 base256emoji", } res := node.RunIPFS("cid", "bases", "--prefix", "--numeric") assert.Equal(t, 0, res.ExitCode()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") assertExactSet(t, "bases --prefix --numeric output", expectedLines, lines) }) } // testCidCodecs tests 'ipfs cid codecs' subcommand func testCidCodecs(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() t.Run("lists available codecs", func(t *testing.T) { // This is a regression test to ensure we don't accidentally add or remove // IPLD codecs. If a codec is intentionally added or removed, // this test should be updated accordingly. expectedCodecs := []string{ "cbor", "raw", "dag-pb", "dag-cbor", "libp2p-key", "git-raw", "torrent-info", "torrent-file", "blake3-hashseq", "leofcoin-block", "leofcoin-tx", "leofcoin-pr", "dag-jose", "dag-cose", "eth-block", "eth-block-list", "eth-tx-trie", "eth-tx", "eth-tx-receipt-trie", "eth-tx-receipt", "eth-state-trie", "eth-account-snapshot", "eth-storage-trie", "eth-receipt-log-trie", "eth-receipt-log", "bitcoin-block", "bitcoin-tx", "bitcoin-witness-commitment", "zcash-block", "zcash-tx", "stellar-block", "stellar-tx", "decred-block", "decred-tx", "dash-block", "dash-tx", "swarm-manifest", "swarm-feed", "beeson", "dag-json", "swhid-1-snp", "json", "rdfc-1", "json-jcs", } res := node.RunIPFS("cid", "codecs") assert.Equal(t, 0, res.ExitCode()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") assertExactSet(t, "codecs", expectedCodecs, lines) }) t.Run("with --numeric flag shows codec numbers", func(t *testing.T) { // This is a regression test to ensure we don't accidentally add or remove // IPLD codecs. If a codec is intentionally added or removed, // this test should be updated accordingly. expectedLines := []string{ "81 cbor", "85 raw", "112 dag-pb", "113 dag-cbor", "114 libp2p-key", "120 git-raw", "123 torrent-info", "124 torrent-file", "128 blake3-hashseq", "129 leofcoin-block", "130 leofcoin-tx", "131 leofcoin-pr", "133 dag-jose", "134 dag-cose", "144 eth-block", "145 eth-block-list", "146 eth-tx-trie", "147 eth-tx", "148 eth-tx-receipt-trie", "149 eth-tx-receipt", "150 eth-state-trie", "151 eth-account-snapshot", "152 eth-storage-trie", "153 eth-receipt-log-trie", "154 eth-receipt-log", "176 bitcoin-block", "177 bitcoin-tx", "178 bitcoin-witness-commitment", "192 zcash-block", "193 zcash-tx", "208 stellar-block", "209 stellar-tx", "224 decred-block", "225 decred-tx", "240 dash-block", "241 dash-tx", "250 swarm-manifest", "251 swarm-feed", "252 beeson", "297 dag-json", "496 swhid-1-snp", "512 json", "46083 rdfc-1", "46593 json-jcs", } res := node.RunIPFS("cid", "codecs", "--numeric") assert.Equal(t, 0, res.ExitCode()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") assertExactSet(t, "codecs --numeric output", expectedLines, lines) }) t.Run("with --supported flag lists only supported codecs", func(t *testing.T) { // This is a regression test to ensure we don't accidentally change the list // of supported codecs. If a codec is intentionally added or removed from // support, this test should be updated accordingly. expectedSupportedCodecs := []string{ "cbor", "dag-cbor", "dag-jose", "dag-json", "dag-pb", "git-raw", "json", "libp2p-key", "raw", } res := node.RunIPFS("cid", "codecs", "--supported") assert.Equal(t, 0, res.ExitCode()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") assertExactSet(t, "supported codecs", expectedSupportedCodecs, lines) }) t.Run("with both --supported and --numeric flags", func(t *testing.T) { // Regression test to catch any changes to supported codecs or output format expectedLines := []string{ "81 cbor", "85 raw", "112 dag-pb", "113 dag-cbor", "114 libp2p-key", "120 git-raw", "133 dag-jose", "297 dag-json", "512 json", } res := node.RunIPFS("cid", "codecs", "--supported", "--numeric") assert.Equal(t, 0, res.ExitCode()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") assertExactSet(t, "codecs --supported --numeric output", expectedLines, lines) }) } // testCidHashes tests 'ipfs cid hashes' subcommand func testCidHashes(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() t.Run("lists available hashes", func(t *testing.T) { // This is a regression test to ensure we don't accidentally add or remove // support for hash functions. If a hash function is intentionally added // or removed, this test should be updated accordingly. expectedHashes := []string{ "identity", "sha1", "sha2-256", "sha2-512", "sha3-512", "sha3-384", "sha3-256", "sha3-224", "shake-256", "keccak-224", "keccak-256", "keccak-384", "keccak-512", "blake3", "dbl-sha2-256", } // Also expect all blake2b variants (160-512 in steps of 8) for i := 160; i <= 512; i += 8 { expectedHashes = append(expectedHashes, fmt.Sprintf("blake2b-%d", i)) } // Also expect all blake2s variants (160-256 in steps of 8) for i := 160; i <= 256; i += 8 { expectedHashes = append(expectedHashes, fmt.Sprintf("blake2s-%d", i)) } res := node.RunIPFS("cid", "hashes") assert.Equal(t, 0, res.ExitCode()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") assertExactSet(t, "hash functions", expectedHashes, lines) }) t.Run("with --numeric flag shows hash function codes", func(t *testing.T) { // This is a regression test to ensure we don't accidentally add or remove // support for hash functions. If a hash function is intentionally added // or removed, this test should be updated accordingly. expectedLines := []string{ "0 identity", "17 sha1", "18 sha2-256", "19 sha2-512", "20 sha3-512", "21 sha3-384", "22 sha3-256", "23 sha3-224", "25 shake-256", "26 keccak-224", "27 keccak-256", "28 keccak-384", "29 keccak-512", "30 blake3", "86 dbl-sha2-256", } // Add all blake2b variants (160-512 in steps of 8) for i := 160; i <= 512; i += 8 { expectedLines = append(expectedLines, fmt.Sprintf("%d blake2b-%d", 45568+i/8, i)) } // Add all blake2s variants (160-256 in steps of 8) for i := 160; i <= 256; i += 8 { expectedLines = append(expectedLines, fmt.Sprintf("%d blake2s-%d", 45632+i/8, i)) } res := node.RunIPFS("cid", "hashes", "--numeric") assert.Equal(t, 0, res.ExitCode()) lines := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n") assertExactSet(t, "hashes --numeric output", expectedLines, lines) }) } // assertExactSet compares expected vs actual items and reports clear errors for any differences. // This is used as a regression test to ensure we don't accidentally add or remove support. // Both expected and actual strings are trimmed of whitespace before comparison for maintainability. func assertExactSet(t *testing.T, itemType string, expected []string, actual []string) { t.Helper() // Normalize by trimming whitespace normalizedExpected := make([]string, len(expected)) for i, item := range expected { normalizedExpected[i] = strings.TrimSpace(item) } normalizedActual := make([]string, len(actual)) for i, item := range actual { normalizedActual[i] = strings.TrimSpace(item) } expectedSet := make(map[string]bool) for _, item := range normalizedExpected { expectedSet[item] = true } actualSet := make(map[string]bool) for _, item := range normalizedActual { actualSet[item] = true } var missing []string for _, item := range normalizedExpected { if !actualSet[item] { missing = append(missing, item) } } var unexpected []string for _, item := range normalizedActual { if !expectedSet[item] { unexpected = append(unexpected, item) } } if len(missing) > 0 { t.Errorf("Missing expected %s: %q", itemType, missing) } if len(unexpected) > 0 { t.Errorf("Unexpected %s found: %q", itemType, unexpected) } assert.Equal(t, len(expected), len(actual), "Expected %d %s but got %d", len(expected), itemType, len(actual)) } ================================================ FILE: test/cli/cli_https_test.go ================================================ package cli import ( "fmt" "net" "net/http" "net/http/httptest" "net/url" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/require" ) func TestCLIWithRemoteHTTPS(t *testing.T) { tests := []struct{ addrSuffix string }{{"https"}, {"tls/http"}} for _, tt := range tests { t.Run("with "+tt.addrSuffix+" multiaddr", func(t *testing.T) { // Create HTTPS test server server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.TLS == nil { t.Error("Mocked Kubo RPC received plain HTTP request instead of HTTPS TLS Handshake") } _, _ = w.Write([]byte("OK")) })) defer server.Close() serverURL, _ := url.Parse(server.URL) _, port, _ := net.SplitHostPort(serverURL.Host) // Create Kubo repo node := harness.NewT(t).NewNode().Init() // Attempt to talk to remote Kubo RPC endpoint over HTTPS resp := node.RunIPFS("id", "--api", fmt.Sprintf("/ip4/127.0.0.1/tcp/%s/%s", port, tt.addrSuffix)) // Expect HTTPS error (confirming TLS and https:// were used, and not Cleartext HTTP) require.Error(t, resp.Err) require.Contains(t, resp.Stderr.String(), "Error: tls: failed to verify certificate: x509: certificate signed by unknown authority") node.StopDaemon() }) } } ================================================ FILE: test/cli/commands_without_repo_test.go ================================================ package cli import ( "os" "os/exec" "strings" "testing" ) func TestCommandsWithoutRepo(t *testing.T) { t.Run("cid", func(t *testing.T) { t.Run("base32", func(t *testing.T) { cmd := exec.Command("ipfs", "cid", "base32", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv") cmd.Env = append(os.Environ(), "IPFS_PATH="+t.TempDir()) stdout, err := cmd.Output() if err != nil { t.Fatal(err) } expected := "bafybeibxm2nsadl3fnxv2sxcxmxaco2jl53wpeorjdzidjwf5aqdg7wa6u\n" if string(stdout) != expected { t.Fatalf("expected %q, got: %q", expected, stdout) } }) t.Run("format", func(t *testing.T) { cmd := exec.Command("ipfs", "cid", "format", "-v", "1", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv") cmd.Env = append(os.Environ(), "IPFS_PATH="+t.TempDir()) stdout, err := cmd.Output() if err != nil { t.Fatal(err) } expected := "zdj7WZAAFKPvYPPzyJLso2hhxo8a7ZACFQ4DvvfrNXTHidofr\n" if string(stdout) != expected { t.Fatalf("expected %q, got: %q", expected, stdout) } }) t.Run("bases", func(t *testing.T) { cmd := exec.Command("ipfs", "cid", "bases") cmd.Env = append(os.Environ(), "IPFS_PATH="+t.TempDir()) stdout, err := cmd.Output() if err != nil { t.Fatal(err) } if !strings.Contains(string(stdout), "base32") { t.Fatalf("expected base32 in output, got: %s", stdout) } }) t.Run("codecs", func(t *testing.T) { cmd := exec.Command("ipfs", "cid", "codecs") cmd.Env = append(os.Environ(), "IPFS_PATH="+t.TempDir()) stdout, err := cmd.Output() if err != nil { t.Fatal(err) } if !strings.Contains(string(stdout), "dag-pb") { t.Fatalf("expected dag-pb in output, got: %s", stdout) } }) t.Run("hashes", func(t *testing.T) { cmd := exec.Command("ipfs", "cid", "hashes") cmd.Env = append(os.Environ(), "IPFS_PATH="+t.TempDir()) stdout, err := cmd.Output() if err != nil { t.Fatal(err) } if !strings.Contains(string(stdout), "sha2-256") { t.Fatalf("expected sha2-256 in output, got: %s", stdout) } }) }) t.Run("multibase", func(t *testing.T) { t.Run("list", func(t *testing.T) { cmd := exec.Command("ipfs", "multibase", "list") cmd.Env = append(os.Environ(), "IPFS_PATH="+t.TempDir()) stdout, err := cmd.Output() if err != nil { t.Fatal(err) } if !strings.Contains(string(stdout), "base32") { t.Fatalf("expected base32 in output, got: %s", stdout) } }) t.Run("encode", func(t *testing.T) { cmd := exec.Command("ipfs", "multibase", "encode", "-b", "base32") cmd.Env = append(os.Environ(), "IPFS_PATH="+t.TempDir()) cmd.Stdin = strings.NewReader("hello\n") stdout, err := cmd.Output() if err != nil { t.Fatal(err) } expected := "bnbswy3dpbi" if string(stdout) != expected { t.Fatalf("expected %q, got: %q", expected, stdout) } }) t.Run("decode", func(t *testing.T) { cmd := exec.Command("ipfs", "multibase", "decode") cmd.Env = append(os.Environ(), "IPFS_PATH="+t.TempDir()) cmd.Stdin = strings.NewReader("bnbswy3dpbi") stdout, err := cmd.Output() if err != nil { t.Fatal(err) } expected := "hello\n" if string(stdout) != expected { t.Fatalf("expected %q, got: %q", expected, stdout) } }) t.Run("transcode", func(t *testing.T) { cmd := exec.Command("ipfs", "multibase", "transcode", "-b", "base64") cmd.Env = append(os.Environ(), "IPFS_PATH="+t.TempDir()) cmd.Stdin = strings.NewReader("bnbswy3dpbi") stdout, err := cmd.Output() if err != nil { t.Fatal(err) } expected := "maGVsbG8K" if string(stdout) != expected { t.Fatalf("expected %q, got: %q", expected, stdout) } }) }) } ================================================ FILE: test/cli/completion_test.go ================================================ package cli import ( "fmt" "testing" "github.com/ipfs/kubo/test/cli/harness" . "github.com/ipfs/kubo/test/cli/testutils" "github.com/stretchr/testify/assert" ) func TestBashCompletion(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode() res := node.IPFS("commands", "completion", "bash") length := len(res.Stdout.String()) if length < 100 { t.Fatalf("expected a long Bash completion file, but got one of length %d", length) } t.Run("completion file can be loaded in bash", func(t *testing.T) { RequiresLinux(t) completionFile := h.WriteToTemp(res.Stdout.String()) res = h.Sh(fmt.Sprintf("source %s && type -t _ipfs", completionFile)) assert.NoError(t, res.Err) }) } func TestZshCompletion(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode() res := node.IPFS("commands", "completion", "zsh") length := len(res.Stdout.String()) if length < 100 { t.Fatalf("expected a long Bash completion file, but got one of length %d", length) } t.Run("completion file can be loaded in bash", func(t *testing.T) { RequiresLinux(t) completionFile := h.WriteToTemp(res.Stdout.String()) res = h.Runner.Run(harness.RunRequest{ Path: "zsh", Args: []string{"-c", fmt.Sprintf("autoload -Uz compinit && compinit && source %s && echo -E $_comps[ipfs]", completionFile)}, }) assert.NoError(t, res.Err) assert.NotEmpty(t, res.Stdout.String()) }) } ================================================ FILE: test/cli/config_secrets_test.go ================================================ package cli import ( "strings" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/tidwall/sjson" ) func TestConfigSecrets(t *testing.T) { t.Parallel() t.Run("Identity.PrivKey protection", func(t *testing.T) { t.Parallel() t.Run("Identity.PrivKey is concealed in config show", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Read the actual config file to get the real PrivKey configFile := node.ReadFile(node.ConfigFile()) assert.Contains(t, configFile, "PrivKey") // config show should NOT contain the PrivKey configShow := node.RunIPFS("config", "show").Stdout.String() assert.NotContains(t, configShow, "PrivKey") }) t.Run("Identity.PrivKey cannot be read via ipfs config", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Attempting to read Identity.PrivKey should fail res := node.RunIPFS("config", "Identity.PrivKey") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "cannot show or change private key") }) t.Run("Identity.PrivKey cannot be read via ipfs config Identity", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Attempting to read Identity section should fail (it contains PrivKey) res := node.RunIPFS("config", "Identity") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "cannot show or change private key") }) t.Run("Identity.PrivKey cannot be set via config replace", func(t *testing.T) { t.Parallel() // Key rotation must be done in offline mode via the dedicated `ipfs key rotate` command. // This test ensures PrivKey cannot be changed via config replace. node := harness.NewT(t).NewNode().Init() configShow := node.RunIPFS("config", "show").Stdout.String() // Try to inject a PrivKey via config replace configJSON := MustVal(sjson.Set(configShow, "Identity.PrivKey", "CAASqAkwggSkAgEAAo")) node.WriteBytes("new-config", []byte(configJSON)) res := node.RunIPFS("config", "replace", "new-config") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "setting private key") }) t.Run("Identity.PrivKey is preserved when re-injecting config", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Read the original config file originalConfig := node.ReadFile(node.ConfigFile()) assert.Contains(t, originalConfig, "PrivKey") // Extract the PrivKey value for comparison var origPrivKey string assert.Contains(t, originalConfig, "PrivKey") // Simple extraction - find the PrivKey line for line := range strings.SplitSeq(originalConfig, "\n") { if strings.Contains(line, "\"PrivKey\":") { origPrivKey = line break } } assert.NotEmpty(t, origPrivKey) // Get config show output (which should NOT contain PrivKey) configShow := node.RunIPFS("config", "show").Stdout.String() assert.NotContains(t, configShow, "PrivKey") // Re-inject the config via config replace node.WriteBytes("config-show", []byte(configShow)) node.IPFS("config", "replace", "config-show") // The PrivKey should still be in the config file newConfig := node.ReadFile(node.ConfigFile()) assert.Contains(t, newConfig, "PrivKey") // Verify the PrivKey line is the same var newPrivKey string for line := range strings.SplitSeq(newConfig, "\n") { if strings.Contains(line, "\"PrivKey\":") { newPrivKey = line break } } assert.Equal(t, origPrivKey, newPrivKey, "PrivKey should be preserved") }) }) t.Run("TLS security validation", func(t *testing.T) { t.Parallel() t.Run("AutoConf.TLSInsecureSkipVerify defaults to false", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Check the default value in a fresh init res := node.RunIPFS("config", "AutoConf.TLSInsecureSkipVerify") // Field may not exist (exit code 1) or be false/empty (exit code 0) // Both are acceptable as they mean "not true" output := res.Stdout.String() assert.NotContains(t, output, "true", "default should not be true") }) t.Run("AutoConf.TLSInsecureSkipVerify can be set to true", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Set to true node.IPFS("config", "AutoConf.TLSInsecureSkipVerify", "true", "--json") // Verify it was set res := node.RunIPFS("config", "AutoConf.TLSInsecureSkipVerify") assert.Equal(t, 0, res.ExitCode()) assert.Contains(t, res.Stdout.String(), "true") }) t.Run("HTTPRetrieval.TLSInsecureSkipVerify defaults to false", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Check the default value in a fresh init res := node.RunIPFS("config", "HTTPRetrieval.TLSInsecureSkipVerify") // Field may not exist (exit code 1) or be false/empty (exit code 0) // Both are acceptable as they mean "not true" output := res.Stdout.String() assert.NotContains(t, output, "true", "default should not be true") }) t.Run("HTTPRetrieval.TLSInsecureSkipVerify can be set to true", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Set to true node.IPFS("config", "HTTPRetrieval.TLSInsecureSkipVerify", "true", "--json") // Verify it was set res := node.RunIPFS("config", "HTTPRetrieval.TLSInsecureSkipVerify") assert.Equal(t, 0, res.ExitCode()) assert.Contains(t, res.Stdout.String(), "true") }) }) } ================================================ FILE: test/cli/content_blocking_test.go ================================================ package cli import ( "context" "fmt" "io" "log" "net/http" "net/url" "os" "path/filepath" "strings" "testing" "github.com/ipfs/go-cid" "github.com/ipfs/kubo/test/cli/harness" carstore "github.com/ipld/go-car/v2/blockstore" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/peer" libp2phttp "github.com/libp2p/go-libp2p/p2p/http" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestContentBlocking(t *testing.T) { // NOTE: we can't run this with t.Parallel() because we set IPFS_NS_MAP // and running in parallel could impact other tests const blockedMsg = "blocked and cannot be provided" const statusExpl = "specific HTTP error code is expected" const bodyExpl = "Error message informing about content block is expected" h := harness.NewT(t) // Init IPFS_PATH node := h.NewNode().Init("--empty-repo", "--profile=test") // Create CIDs we use in test h.WriteFile("parent-dir/blocked-subdir/indirectly-blocked-file.txt", "indirectly blocked file content") allowedParentDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", "--pin=false", filepath.Join(h.Dir, "parent-dir")).Stdout.Trimmed() blockedSubDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", "--pin=false", filepath.Join(h.Dir, "parent-dir", "blocked-subdir")).Stdout.Trimmed() node.IPFS("block", "rm", blockedSubDirCID) h.WriteFile("directly-blocked-file.txt", "directly blocked file content") blockedCID := node.IPFS("add", "--raw-leaves", "-Q", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed() h.WriteFile("not-blocked-file.txt", "not blocked file content") allowedCID := node.IPFS("add", "--raw-leaves", "-Q", filepath.Join(h.Dir, "not-blocked-file.txt")).Stdout.Trimmed() // Create denylist at $IPFS_PATH/denylists/test.deny denylistTmp := h.WriteToTemp("name: test list\n---\n" + "//QmX9dhRcQcKUw3Ws8485T5a9dtjrSCQaUAHnG4iK9i4ceM\n" + // Double hash (sha256) CID block: base58btc(sha256-multihash(QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR)) "//gW813G35CnLsy7gRYYHuf63hrz71U1xoLFDVeV7actx6oX\n" + // Double hash (blake3) Path block under blake3 root CID: base58btc(blake3-multihash(gW7Nhu4HrfDtphEivm3Z9NNE7gpdh5Tga8g6JNZc1S8E47/path)) "//8526ba05eec55e28f8db5974cc891d0d92c8af69d386fc6464f1e9f372caf549\n" + // Legacy CID double-hash block: sha256(bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq/) "//e5b7d2ce2594e2e09901596d8e1f29fa249b74c8c9e32ea01eda5111e4d33f07\n" + // Legacy Path double-hash block: sha256(bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath) "/ipfs/" + blockedCID + "\n" + // block specific CID "/ipfs/" + allowedParentDirCID + "/blocked-subdir*\n" + // block only specific subpath "/ipns/blocked-cid.example.com\n" + "/ipns/blocked-dnslink.example.com\n") if err := os.MkdirAll(filepath.Join(node.Dir, "denylists"), 0o777); err != nil { log.Panicf("failed to create denylists dir: %s", err.Error()) } if err := os.Rename(denylistTmp, filepath.Join(node.Dir, "denylists", "test.deny")); err != nil { log.Panicf("failed to create test denylist: %s", err.Error()) } // Add two entries to namesys resolution cache // /ipns/blocked-cid.example.com point at a blocked CID (to confirm blocking impacts /ipns resolution) // /ipns/blocked-dnslink.example.com with safe CID (to test blocking of /ipns/ paths) os.Setenv("IPFS_NS_MAP", "blocked-cid.example.com:/ipfs/"+blockedCID+",blocked-dnslink.example.com/ipns/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") defer os.Unsetenv("IPFS_NS_MAP") // Enable GatewayOverLibp2p as we want to test denylist there too node.IPFS("config", "--json", "Experimental.GatewayOverLibp2p", "true") // Start daemon, it should pick up denylist from $IPFS_PATH/denylists/test.deny node.StartDaemon() // we need online mode for GatewayOverLibp2p tests t.Cleanup(func() { node.StopDaemon() }) client := node.GatewayClient() // First, confirm gateway works t.Run("Gateway Allows CID that is not blocked", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/" + allowedCID) assert.Equal(t, http.StatusOK, resp.StatusCode) assert.Equal(t, "not blocked file content", resp.Body) }) // Then, does the most basic blocking case work? t.Run("Gateway Denies directly blocked CID", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/" + blockedCID) assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) assert.NotEqual(t, "directly blocked file content", resp.Body) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) // Confirm parent of blocked subpath is not blocked t.Run("Gateway Allows parent Path that is not blocked", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/" + allowedParentDirCID) assert.Equal(t, http.StatusOK, resp.StatusCode) }) // Confirm CAR responses skip blocked subpaths t.Run("Gateway returns CAR without blocked subpath", func(t *testing.T) { resp := client.Get("/ipfs/" + allowedParentDirCID + "/subdir?format=car") assert.Equal(t, http.StatusOK, resp.StatusCode) bs, err := carstore.NewReadOnly(strings.NewReader(resp.Body), nil) assert.NoError(t, err) has, err := bs.Has(context.Background(), cid.MustParse(blockedSubDirCID)) assert.NoError(t, err) assert.False(t, has) }) /* TODO: this was already broken in 0.26, but we should fix it t.Run("Gateway returns CAR without directly blocked CID", func(t *testing.T) { allowedDirWithDirectlyBlockedCID := node.IPFS("add", "--raw-leaves", "-Q", "-rw", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed() resp := client.Get("/ipfs/" + allowedDirWithDirectlyBlockedCID + "?format=car") assert.Equal(t, http.StatusOK, resp.StatusCode) bs, err := carstore.NewReadOnly(strings.NewReader(resp.Body), nil) assert.NoError(t, err) has, err := bs.Has(context.Background(), cid.MustParse(blockedCID)) assert.NoError(t, err) assert.False(t, has, "Returned CAR should not include blockedCID") }) */ // Confirm CAR responses skip blocked subpaths t.Run("Gateway returns CAR without blocked subpath", func(t *testing.T) { resp := client.Get("/ipfs/" + allowedParentDirCID + "/subdir?format=car") assert.Equal(t, http.StatusOK, resp.StatusCode) bs, err := carstore.NewReadOnly(strings.NewReader(resp.Body), nil) assert.NoError(t, err) has, err := bs.Has(context.Background(), cid.MustParse(blockedSubDirCID)) assert.NoError(t, err) assert.False(t, has, "Returned CAR should not include blockedSubDirCID") }) // Ok, now the full list of test cases we want to cover in both CLI and Gateway testCases := []struct { name string path string }{ { name: "directly blocked file CID", path: "/ipfs/" + blockedCID, }, { name: "indirectly blocked file (on a blocked subpath)", path: "/ipfs/" + allowedParentDirCID + "/blocked-subdir/indirectly-blocked-file.txt", }, { name: "/ipns path that resolves to a blocked CID", path: "/ipns/blocked-cid.example.com", }, { name: "/ipns Path that is blocked by DNSLink name", path: "/ipns/blocked-dnslink.example.com", }, { name: "double-hash CID block (sha256-multihash)", path: "/ipfs/QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR", }, { name: "double-hash Path block (blake3-multihash)", path: "/ipfs/bafyb4ieqht3b2rssdmc7sjv2cy2gfdilxkfh7623nvndziyqnawkmo266a/path", }, { name: "legacy CID double-hash block (sha256)", path: "/ipfs/bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq", }, { name: "legacy Path double-hash block (sha256)", path: "/ipfs/bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath", }, } // Which specific cliCmds we test against testCases cliCmds := [][]string{ {"block", "get"}, {"block", "stat"}, {"dag", "get"}, {"dag", "export"}, {"dag", "stat"}, {"cat"}, {"ls"}, {"get"}, {"refs"}, } expectedMsg := blockedMsg for _, testCase := range testCases { // Confirm that denylist is active for every command in 'cliCmds' x 'testCases' for _, cmd := range cliCmds { cliTestName := fmt.Sprintf("CLI '%s' denies %s", strings.Join(cmd, " "), testCase.name) t.Run(cliTestName, func(t *testing.T) { t.Parallel() args := append(cmd, testCase.path) cmd := node.RunIPFS(args...) stdout := cmd.Stdout.Trimmed() stderr := cmd.Stderr.Trimmed() if !strings.Contains(stderr, expectedMsg) { t.Errorf("Expected STDERR error message %q, but got: %q", expectedMsg, stderr) if stdout != "" { t.Errorf("Expected STDOUT to be empty, but got: %q", stdout) } } }) } // Confirm that denylist is active for every content path in 'testCases' gwTestName := fmt.Sprintf("Gateway denies %s", testCase.name) t.Run(gwTestName, func(t *testing.T) { resp := client.Get(testCase.path) assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) } // Extra edge cases on subdomain gateway t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain redirect)", func(t *testing.T) { t.Parallel() gwURL, _ := url.Parse(node.GatewayURL()) resp := client.Get("/ipns/blocked-dnslink.example.com", func(r *http.Request) { r.Host = "localhost:" + gwURL.Port() }) assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain, no TLS)", func(t *testing.T) { t.Parallel() gwURL, _ := url.Parse(node.GatewayURL()) resp := client.Get("/", func(r *http.Request) { r.Host = "blocked-dnslink.example.com.ipns.localhost:" + gwURL.Port() }) assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain, inlined for TLS)", func(t *testing.T) { t.Parallel() gwURL, _ := url.Parse(node.GatewayURL()) resp := client.Get("/", func(r *http.Request) { // Inlined DNSLink to fit in single DNS label for TLS interop: // https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header r.Host = "blocked--dnslink-example-com.ipns.localhost:" + gwURL.Port() }) assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) // We need to confirm denylist is active when gateway is run in NoFetch // mode (which usually swaps blockservice to a read-only one, and that swap // may cause denylists to not be applied, as it is a separate code path) t.Run("GatewayNoFetch", func(t *testing.T) { // NOTE: we don't run this in parallel, as it requires restart with different config // Switch gateway to NoFetch mode node.StopDaemon() node.IPFS("config", "--json", "Gateway.NoFetch", "true") node.StartDaemon() // update client, as the port of test node might've changed after restart client = node.GatewayClient() // First, confirm gateway works t.Run("Allows CID that is not blocked", func(t *testing.T) { resp := client.Get("/ipfs/" + allowedCID) assert.Equal(t, http.StatusOK, resp.StatusCode) assert.Equal(t, "not blocked file content", resp.Body) }) // Then, does the most basic blocking case work? t.Run("Denies directly blocked CID", func(t *testing.T) { resp := client.Get("/ipfs/" + blockedCID) assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) assert.NotEqual(t, "directly blocked file content", resp.Body) assert.Contains(t, resp.Body, blockedMsg, bodyExpl) }) // Restore default node.StopDaemon() node.IPFS("config", "--json", "Gateway.NoFetch", "false") node.StartDaemon() client = node.GatewayClient() }) // We need to confirm denylist is active on the // trustless gateway exposed over libp2p // when Experimental.GatewayOverLibp2p=true // (https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#http-gateway-over-libp2p) // NOTE: this type of gateway is hardcoded to be NoFetch: it does not fetch // data that is not in local store, so we only need to run it once: a // simple smoke-test for allowed CID and blockedCID. t.Run("GatewayOverLibp2p", func(t *testing.T) { t.Parallel() // Create libp2p client that connects to our node over // /http1.1 and then talks gateway semantics over the /ipfs/gateway sub-protocol clientHost, err := libp2p.New(libp2p.NoListenAddrs) require.NoError(t, err) err = clientHost.Connect(context.Background(), peer.AddrInfo{ ID: node.PeerID(), Addrs: node.SwarmAddrs(), }) require.NoError(t, err) libp2pClient, err := (&libp2phttp.Host{StreamHost: clientHost}).NamespacedClient("/ipfs/gateway", peer.AddrInfo{ID: node.PeerID()}) require.NoError(t, err) t.Run("Serves Allowed CID", func(t *testing.T) { t.Parallel() resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=raw", allowedCID)) require.NoError(t, err) defer resp.Body.Close() assert.Equal(t, http.StatusOK, resp.StatusCode) body, err := io.ReadAll(resp.Body) require.NoError(t, err) require.Equal(t, string(body), "not blocked file content", bodyExpl) }) t.Run("Denies Blocked CID", func(t *testing.T) { t.Parallel() resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=raw", blockedCID)) require.NoError(t, err) defer resp.Body.Close() assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) body, err := io.ReadAll(resp.Body) require.NoError(t, err) assert.NotEqual(t, string(body), "directly blocked file content") assert.Contains(t, string(body), blockedMsg, bodyExpl) }) t.Run("Denies Blocked CID as CAR", func(t *testing.T) { t.Parallel() resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=car", blockedCID)) require.NoError(t, err) defer resp.Body.Close() assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) body, err := io.ReadAll(resp.Body) require.NoError(t, err) assert.NotContains(t, string(body), "directly blocked file content") assert.Contains(t, string(body), blockedMsg, bodyExpl) }) }) } ================================================ FILE: test/cli/content_routing_http_test.go ================================================ package cli import ( "net/http" "net/http/httptest" "os/exec" "testing" "time" "github.com/ipfs/boxo/routing/http/server" "github.com/ipfs/go-test/random" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils/httprouting" "github.com/stretchr/testify/assert" ) // userAgentRecorder records the user agent of every HTTP request type userAgentRecorder struct { delegate http.Handler userAgents []string } func (r *userAgentRecorder) ServeHTTP(w http.ResponseWriter, req *http.Request) { r.userAgents = append(r.userAgents, req.UserAgent()) r.delegate.ServeHTTP(w, req) } func TestContentRoutingHTTP(t *testing.T) { mockRouter := &httprouting.MockHTTPContentRouter{} // run the content routing HTTP server userAgentRecorder := &userAgentRecorder{delegate: server.Handler(mockRouter)} server := httptest.NewServer(userAgentRecorder) t.Cleanup(func() { server.Close() }) // setup the node node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { // setup Kubo node to use mocked HTTP Router cfg.Routing.DelegatedRouters = []string{server.URL} }) node.StartDaemon() // compute a random CID randStr := string(random.Bytes(100)) res := node.PipeStrToIPFS(randStr, "add", "-qn") wantCIDStr := res.Stdout.Trimmed() t.Run("fetching an uncached block results in an HTTP lookup", func(t *testing.T) { statRes := node.Runner.Run(harness.RunRequest{ Path: node.IPFSBin, Args: []string{"block", "stat", wantCIDStr}, RunFunc: (*exec.Cmd).Start, }) defer func() { if err := statRes.Cmd.Process.Kill(); err != nil { t.Logf("error killing 'block stat' cmd: %s", err) } }() // verify the content router was called assert.Eventually(t, func() bool { return mockRouter.NumFindProvidersCalls() > 0 }, time.Minute, 10*time.Millisecond) assert.NotEmpty(t, userAgentRecorder.userAgents) version := node.IPFS("id", "-f", "").Stdout.Trimmed() for _, userAgent := range userAgentRecorder.userAgents { assert.Equal(t, version, userAgent) } }) } ================================================ FILE: test/cli/daemon_test.go ================================================ package cli import ( "bytes" "crypto/rand" "fmt" "io" "net/http" "os/exec" "testing" "time" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" "github.com/stretchr/testify/require" ) func TestDaemon(t *testing.T) { t.Parallel() t.Run("daemon starts if api is set to null", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.SetIPFSConfig("Addresses.API", nil) node.Runner.MustRun(harness.RunRequest{ Path: node.IPFSBin, Args: []string{"daemon"}, RunFunc: (*exec.Cmd).Start, // Start without waiting for completion. }) node.StopDaemon() }) t.Run("daemon shuts down gracefully with active operations", func(t *testing.T) { t.Parallel() // Start daemon with multiple components active via config node := harness.NewT(t).NewNode().Init() // Enable experimental features and pubsub via config node.UpdateConfig(func(cfg *config.Config) { cfg.Pubsub.Enabled = config.True // Instead of --enable-pubsub-experiment cfg.Experimental.P2pHttpProxy = true // Enable P2P HTTP proxy cfg.Experimental.GatewayOverLibp2p = true // Enable gateway over libp2p }) node.StartDaemon("--enable-gc") // Start background operations to simulate real daemon workload: // 1. "ipfs add" simulates content onboarding/ingestion work // 2. Gateway request simulates content retrieval and gateway processing work // Background operation 1: Continuous add of random data to simulate onboarding addDone := make(chan struct{}) go func() { defer close(addDone) // Start the add command asynchronously res := node.Runner.Run(harness.RunRequest{ Path: node.IPFSBin, Args: []string{"add", "--progress=false", "-"}, RunFunc: (*exec.Cmd).Start, CmdOpts: []harness.CmdOpt{ harness.RunWithStdin(&infiniteReader{}), }, }) // Wait for command to finish (when daemon stops) if res.Cmd != nil { _ = res.Cmd.Wait() // Ignore error, expect command to be killed during shutdown } }() // Background operation 2: Gateway CAR request to simulate retrieval work gatewayDone := make(chan struct{}) go func() { defer close(gatewayDone) // First add a file sized to ensure gateway request takes ~1 minute largeData := make([]byte, 512*1024) // 512KB of data _, _ = rand.Read(largeData) // Always succeeds for crypto/rand testCID := node.IPFSAdd(bytes.NewReader(largeData)) // Get gateway address from config cfg := node.ReadConfig() gatewayMaddr, err := multiaddr.NewMultiaddr(cfg.Addresses.Gateway[0]) if err != nil { return } gatewayAddr, err := manet.ToNetAddr(gatewayMaddr) if err != nil { return } // Request CAR but slow reading to simulate heavy gateway load gatewayURL := fmt.Sprintf("http://%s/ipfs/%s?format=car", gatewayAddr, testCID) client := &http.Client{Timeout: 90 * time.Second} resp, err := client.Get(gatewayURL) if err == nil { defer resp.Body.Close() // Read response slowly: 512KB ÷ 1KB × 125ms = ~64 seconds (1+ minute) total // This ensures operation is still active when we shutdown at 2 seconds buf := make([]byte, 1024) // 1KB buffer for { if _, err := io.ReadFull(resp.Body, buf); err != nil { return } time.Sleep(125 * time.Millisecond) // 125ms delay = ~64s total for 512KB } } }() // Let operations run for 2 seconds to ensure they're active time.Sleep(2 * time.Second) // Trigger graceful shutdown shutdownStart := time.Now() node.StopDaemon() shutdownDuration := time.Since(shutdownStart) // Verify clean shutdown: // - Daemon should stop within reasonable time (not hang) require.Less(t, shutdownDuration, 10*time.Second, "daemon should shut down within 10 seconds") // Wait for background operations to complete (with timeout) select { case <-addDone: // Good, add operation terminated case <-time.After(5 * time.Second): t.Error("add operation did not terminate within 5 seconds after daemon shutdown") } select { case <-gatewayDone: // Good, gateway operation terminated case <-time.After(5 * time.Second): t.Error("gateway operation did not terminate within 5 seconds after daemon shutdown") } // Verify we can restart with same repo (no lock issues) node.StartDaemon() node.StopDaemon() }) } // infiniteReader provides an infinite stream of random data type infiniteReader struct{} func (r *infiniteReader) Read(p []byte) (n int, err error) { _, _ = rand.Read(p) // Always succeeds for crypto/rand time.Sleep(50 * time.Millisecond) // Rate limit to simulate steady stream return len(p), nil } ================================================ FILE: test/cli/dag_layout_test.go ================================================ package cli import ( "os" "path/filepath" "strings" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/require" ) // TestBalancedDAGLayout verifies that kubo uses the "balanced" DAG layout // (all leaves at same depth) rather than "balanced-packed" (varying leaf depths). // // DAG layout differences across implementations: // // - balanced: kubo, helia (all leaves at same depth, uniform traversal distance) // - balanced-packed: singularity (trailing leaves may be at different depths) // - trickle: kubo --trickle (varying depths, optimized for append-only/streaming) // // kubo does not implement balanced-packed. The trickle layout also produces // non-uniform leaf depths but with different trade-offs: trickle is optimized // for append-only and streaming reads (no seeking), while balanced-packed // minimizes node count. // // IPIP-499 documents the balanced vs balanced-packed distinction. Files larger // than dag_width × chunk_size will have different CIDs between implementations // using different layouts. // // Set DAG_LAYOUT_CAR_OUTPUT environment variable to export CAR files. // Example: DAG_LAYOUT_CAR_OUTPUT=/tmp/dag-layout go test -run TestBalancedDAGLayout -v func TestBalancedDAGLayout(t *testing.T) { t.Parallel() carOutputDir := os.Getenv("DAG_LAYOUT_CAR_OUTPUT") exportCARs := carOutputDir != "" if exportCARs { if err := os.MkdirAll(carOutputDir, 0755); err != nil { t.Fatalf("failed to create CAR output directory: %v", err) } t.Logf("CAR export enabled, writing to: %s", carOutputDir) } t.Run("balanced layout has uniform leaf depth", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() // Create file that triggers multi-level DAG. // For default v0: 175 chunks × 256KiB = ~44.8 MiB (just over 174 max links) // This creates a 2-level DAG where balanced layout ensures uniform depth. fileSize := "45MiB" seed := "balanced-test" cidStr := node.IPFSAddDeterministic(fileSize, seed) // Collect leaf depths by walking DAG depths := collectLeafDepths(t, node, cidStr, 0) // All leaves must be at same depth for balanced layout require.NotEmpty(t, depths, "expected at least one leaf node") firstDepth := depths[0] for i, d := range depths { require.Equal(t, firstDepth, d, "leaf %d at depth %d, expected %d (balanced layout requires uniform leaf depth)", i, d, firstDepth) } t.Logf("verified %d leaves all at depth %d (CID: %s)", len(depths), firstDepth, cidStr) if exportCARs { carPath := filepath.Join(carOutputDir, "balanced_"+fileSize+".car") require.NoError(t, node.IPFSDagExport(cidStr, carPath)) t.Logf("exported: %s -> %s", cidStr, carPath) } }) t.Run("trickle layout has varying leaf depth", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() fileSize := "45MiB" seed := "trickle-test" // Add with trickle layout (--trickle flag). // Trickle produces non-uniform leaf depths, optimized for append-only // and streaming reads (no seeking). This subtest validates the test // logic by confirming we can detect varying depths. cidStr := node.IPFSAddDeterministic(fileSize, seed, "--trickle") depths := collectLeafDepths(t, node, cidStr, 0) // Trickle layout should have varying depths require.NotEmpty(t, depths, "expected at least one leaf node") minDepth, maxDepth := depths[0], depths[0] for _, d := range depths { if d < minDepth { minDepth = d } if d > maxDepth { maxDepth = d } } require.NotEqual(t, minDepth, maxDepth, "trickle layout should have varying leaf depths, got uniform depth %d", minDepth) t.Logf("verified %d leaves with depths ranging from %d to %d (CID: %s)", len(depths), minDepth, maxDepth, cidStr) if exportCARs { carPath := filepath.Join(carOutputDir, "trickle_"+fileSize+".car") require.NoError(t, node.IPFSDagExport(cidStr, carPath)) t.Logf("exported: %s -> %s", cidStr, carPath) } }) } // collectLeafDepths recursively walks DAG and returns depth of each leaf node. // A node is a leaf if it's a raw block or a dag-pb node with no links. func collectLeafDepths(t *testing.T, node *harness.Node, cid string, depth int) []int { t.Helper() // Check codec to see if this is a raw leaf res := node.IPFS("cid", "format", "-f", "%c", cid) codec := strings.TrimSpace(res.Stdout.String()) if codec == "raw" { // Raw blocks are always leaves return []int{depth} } // Try to inspect as dag-pb node pbNode, err := node.InspectPBNode(cid) if err != nil { // Can't parse as dag-pb, treat as leaf return []int{depth} } // No links = leaf node if len(pbNode.Links) == 0 { return []int{depth} } // Recurse into children var depths []int for _, link := range pbNode.Links { childDepths := collectLeafDepths(t, node, link.Hash.Slash, depth+1) depths = append(depths, childDepths...) } return depths } ================================================ FILE: test/cli/dag_test.go ================================================ package cli import ( "encoding/json" "io" "os" "testing" "time" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) const ( fixtureFile = "./fixtures/TestDagStat.car" textOutputPath = "./fixtures/TestDagStatExpectedOutput.txt" node1Cid = "bafyreibmdfd7c5db4kls4ty57zljfhqv36gi43l6txl44pi423wwmeskwy" node2Cid = "bafyreie3njilzdi4ixumru4nzgecsnjtu7fzfcwhg7e6s4s5i7cnbslvn4" fixtureCid = "bafyreifrm6uf5o4dsaacuszf35zhibyojlqclabzrms7iak67pf62jygaq" ) type DagStat struct { Cid string `json:"Cid"` Size int `json:"Size"` NumBlocks int `json:"NumBlocks"` } type Data struct { UniqueBlocks int `json:"UniqueBlocks"` TotalSize int `json:"TotalSize"` SharedSize int `json:"SharedSize"` Ratio float64 `json:"Ratio"` DagStats []DagStat `json:"DagStats"` } // The Fixture file represents a dag where 2 nodes of size = 46B each, have a common child of 7B // when traversing the DAG from the root's children (node1 and node2) we count (46 + 7)x2 bytes (counting redundant bytes) = 106 // since both nodes share a common child of 7 bytes we actually had to read (46)x2 + 7 = 99 bytes // we should get a dedup ratio of 106/99 that results in approximately 1.0707071 func TestDag(t *testing.T) { t.Parallel() t.Run("ipfs dag stat --enc=json", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Import fixture r, err := os.Open(fixtureFile) assert.Nil(t, err) defer r.Close() err = node.IPFSDagImport(r, fixtureCid) assert.NoError(t, err) stat := node.RunIPFS("dag", "stat", "--progress=false", "--enc=json", node1Cid, node2Cid) var data Data err = json.Unmarshal(stat.Stdout.Bytes(), &data) assert.NoError(t, err) expectedUniqueBlocks := 3 expectedSharedSize := 7 expectedTotalSize := 99 expectedRatio := float64(expectedSharedSize+expectedTotalSize) / float64(expectedTotalSize) expectedDagStatsLength := 2 // Validate UniqueBlocks assert.Equal(t, expectedUniqueBlocks, data.UniqueBlocks) assert.Equal(t, expectedSharedSize, data.SharedSize) assert.Equal(t, expectedTotalSize, data.TotalSize) assert.Equal(t, testutils.FloatTruncate(expectedRatio, 4), testutils.FloatTruncate(data.Ratio, 4)) // Validate DagStats assert.Equal(t, expectedDagStatsLength, len(data.DagStats)) node1Output := data.DagStats[0] node2Output := data.DagStats[1] assert.Equal(t, node1Output.Cid, node1Cid) assert.Equal(t, node2Output.Cid, node2Cid) expectedNode1Size := (expectedTotalSize + expectedSharedSize) / 2 expectedNode2Size := (expectedTotalSize + expectedSharedSize) / 2 assert.Equal(t, expectedNode1Size, node1Output.Size) assert.Equal(t, expectedNode2Size, node2Output.Size) expectedNode1Blocks := 2 expectedNode2Blocks := 2 assert.Equal(t, expectedNode1Blocks, node1Output.NumBlocks) assert.Equal(t, expectedNode2Blocks, node2Output.NumBlocks) }) t.Run("ipfs dag stat", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() r, err := os.Open(fixtureFile) assert.NoError(t, err) defer r.Close() f, err := os.Open(textOutputPath) assert.NoError(t, err) defer f.Close() content, err := io.ReadAll(f) assert.NoError(t, err) err = node.IPFSDagImport(r, fixtureCid) assert.NoError(t, err) stat := node.RunIPFS("dag", "stat", "--progress=false", node1Cid, node2Cid) assert.Equal(t, content, stat.Stdout.Bytes()) }) } func TestDagImportFastProvide(t *testing.T) { t.Parallel() t.Run("fast-provide-root disabled via config: verify skipped in logs", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.FastProvideRoot = config.False }) // Start daemon with debug logging node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithEnv(map[string]string{ "GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug", }), }, }, "") defer node.StopDaemon() // Import CAR file r, err := os.Open(fixtureFile) require.NoError(t, err) defer r.Close() err = node.IPFSDagImport(r, fixtureCid) require.NoError(t, err) // Verify fast-provide-root was disabled daemonLog := node.Daemon.Stderr.String() require.Contains(t, daemonLog, "fast-provide-root: skipped") }) t.Run("fast-provide-root enabled with wait=false: verify async provide", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Use default config (FastProvideRoot=true, FastProvideWait=false) node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithEnv(map[string]string{ "GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug", }), }, }, "") defer node.StopDaemon() // Import CAR file r, err := os.Open(fixtureFile) require.NoError(t, err) defer r.Close() err = node.IPFSDagImport(r, fixtureCid) require.NoError(t, err) daemonLog := node.Daemon.Stderr // Should see async mode started require.Contains(t, daemonLog.String(), "fast-provide-root: enabled") require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously") require.Contains(t, daemonLog.String(), fixtureCid) // Should log the specific CID being provided // Wait for async completion or failure (slightly more than DefaultFastProvideTimeout) // In test environment with no DHT peers, this will fail with "failed to find any peer in table" timeout := config.DefaultFastProvideTimeout + time.Second completedOrFailed := waitForLogMessage(daemonLog, "async provide completed", timeout) || waitForLogMessage(daemonLog, "async provide failed", timeout) require.True(t, completedOrFailed, "async provide should complete or fail within timeout") }) t.Run("fast-provide-root enabled with wait=true: verify sync provide", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.FastProvideWait = config.True }) node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithEnv(map[string]string{ "GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug", }), }, }, "") defer node.StopDaemon() // Import CAR file - use Run instead of IPFSDagImport to handle expected error r, err := os.Open(fixtureFile) require.NoError(t, err) defer r.Close() res := node.Runner.Run(harness.RunRequest{ Path: node.IPFSBin, Args: []string{"dag", "import", "--pin-roots=false"}, CmdOpts: []harness.CmdOpt{ harness.RunWithStdin(r), }, }) // In sync mode (wait=true), provide errors propagate and fail the command. // Test environment uses 'test' profile with no bootstrappers, and CI has // insufficient peers for proper DHT puts, so we expect this to fail with // "failed to find any peer in table" error from the DHT. require.Equal(t, 1, res.ExitCode()) require.Contains(t, res.Stderr.String(), "Error: fast-provide: failed to find any peer in table") daemonLog := node.Daemon.Stderr.String() // Should see sync mode started require.Contains(t, daemonLog, "fast-provide-root: enabled") require.Contains(t, daemonLog, "fast-provide-root: providing synchronously") require.Contains(t, daemonLog, fixtureCid) // Should log the specific CID being provided require.Contains(t, daemonLog, "sync provide failed") // Verify the failure was logged }) t.Run("fast-provide-wait ignored when root disabled", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.FastProvideRoot = config.False cfg.Import.FastProvideWait = config.True }) node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithEnv(map[string]string{ "GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug", }), }, }, "") defer node.StopDaemon() // Import CAR file r, err := os.Open(fixtureFile) require.NoError(t, err) defer r.Close() err = node.IPFSDagImport(r, fixtureCid) require.NoError(t, err) daemonLog := node.Daemon.Stderr.String() require.Contains(t, daemonLog, "fast-provide-root: skipped") // Note: dag import doesn't log wait-flag-ignored like add does }) t.Run("CLI flag overrides config: flag=true overrides config=false", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.FastProvideRoot = config.False }) node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithEnv(map[string]string{ "GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug", }), }, }, "") defer node.StopDaemon() // Import CAR file with flag override r, err := os.Open(fixtureFile) require.NoError(t, err) defer r.Close() err = node.IPFSDagImport(r, fixtureCid, "--fast-provide-root=true") require.NoError(t, err) daemonLog := node.Daemon.Stderr // Flag should enable it despite config saying false require.Contains(t, daemonLog.String(), "fast-provide-root: enabled") require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously") require.Contains(t, daemonLog.String(), fixtureCid) // Should log the specific CID being provided }) t.Run("CLI flag overrides config: flag=false overrides config=true", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.FastProvideRoot = config.True }) node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithEnv(map[string]string{ "GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug", }), }, }, "") defer node.StopDaemon() // Import CAR file with flag override r, err := os.Open(fixtureFile) require.NoError(t, err) defer r.Close() err = node.IPFSDagImport(r, fixtureCid, "--fast-provide-root=false") require.NoError(t, err) daemonLog := node.Daemon.Stderr.String() // Flag should disable it despite config saying true require.Contains(t, daemonLog, "fast-provide-root: skipped") }) } ================================================ FILE: test/cli/delegated_routing_v1_http_client_test.go ================================================ package cli import ( "encoding/json" "io" "net/http" "net/http/httptest" "strings" "sync" "testing" "time" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" . "github.com/ipfs/kubo/test/cli/testutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestHTTPDelegatedRouting(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() fakeServer := func(contentType string, resp ...string) *httptest.Server { return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", contentType) for _, r := range resp { _, err := w.Write([]byte(r)) if err != nil { panic(err) } } })) } findProvsCID := "baeabep4vu3ceru7nerjjbk37sxb7wmftteve4hcosmyolsbsiubw2vr6pqzj6mw7kv6tbn6nqkkldnklbjgm5tzbi4hkpkled4xlcr7xz4bq" provs := []string{"12D3KooWAobjw92XDcnQ1rRmRJDA3zAQpdPYUpZKrJxH6yccSpje", "12D3KooWARYacCc6eoCqvsS9RW9MA2vo51CV75deoiqssx3YgyYJ"} t.Run("default routing config has no routers defined", func(t *testing.T) { assert.Nil(t, node.ReadConfig().Routing.Routers) }) t.Run("no routers means findprovs returns no results", func(t *testing.T) { res := node.IPFS("routing", "findprovs", findProvsCID).Stdout.String() assert.Empty(t, res) }) t.Run("no routers means findprovs returns no results", func(t *testing.T) { res := node.IPFS("routing", "findprovs", findProvsCID).Stdout.String() assert.Empty(t, res) }) node.StopDaemon() t.Run("missing method params make the daemon fail", func(t *testing.T) { node.UpdateConfig(func(cfg *config.Config) { cfg.Routing.Type = config.NewOptionalString("custom") cfg.Routing.Methods = config.Methods{ "find-peers": {RouterName: "TestDelegatedRouter"}, "find-providers": {RouterName: "TestDelegatedRouter"}, "get-ipns": {RouterName: "TestDelegatedRouter"}, "provide": {RouterName: "TestDelegatedRouter"}, } }) res := node.RunIPFS("daemon") assert.Equal(t, 1, res.ExitErr.ProcessState.ExitCode()) assert.Contains( t, res.Stderr.String(), `method name "put-ipns" is missing from Routing.Methods config param`, ) }) t.Run("having wrong methods makes daemon fail", func(t *testing.T) { node.UpdateConfig(func(cfg *config.Config) { cfg.Routing.Type = config.NewOptionalString("custom") cfg.Routing.Methods = config.Methods{ "find-peers": {RouterName: "TestDelegatedRouter"}, "find-providers": {RouterName: "TestDelegatedRouter"}, "get-ipns": {RouterName: "TestDelegatedRouter"}, "provide": {RouterName: "TestDelegatedRouter"}, "put-ipns": {RouterName: "TestDelegatedRouter"}, "NOT_SUPPORTED": {RouterName: "TestDelegatedRouter"}, } }) res := node.RunIPFS("daemon") assert.Equal(t, 1, res.ExitErr.ProcessState.ExitCode()) assert.Contains( t, res.Stderr.String(), `method name "NOT_SUPPORTED" is not a supported method on Routing.Methods config param`, ) }) t.Run("adding HTTP delegated routing endpoint to Routing.Routers config works", func(t *testing.T) { server := fakeServer("application/json", ToJSONStr(JSONObj{ "Providers": []JSONObj{ { "Schema": "bitswap", // Legacy bitswap schema. "Protocol": "transport-bitswap", "ID": provs[1], "Addrs": []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/tcp/4002"}, }, { "Schema": "peer", "Protocols": []string{"transport-bitswap"}, "ID": provs[0], "Addrs": []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/tcp/4002"}, }, }, })) t.Cleanup(server.Close) node.IPFS("config", "Routing.Type", "custom") node.IPFS("config", "Routing.Routers.TestDelegatedRouter", "--json", ToJSONStr(JSONObj{ "Type": "http", "Parameters": JSONObj{ "Endpoint": server.URL, }, })) node.IPFS("config", "Routing.Methods", "--json", ToJSONStr(JSONObj{ "find-peers": JSONObj{"RouterName": "TestDelegatedRouter"}, "find-providers": JSONObj{"RouterName": "TestDelegatedRouter"}, "get-ipns": JSONObj{"RouterName": "TestDelegatedRouter"}, "provide": JSONObj{"RouterName": "TestDelegatedRouter"}, "put-ipns": JSONObj{"RouterName": "TestDelegatedRouter"}, })) res := node.IPFS("config", "Routing.Routers.TestDelegatedRouter.Parameters.Endpoint") assert.Equal(t, res.Stdout.Trimmed(), server.URL) node.StartDaemon() res = node.IPFS("routing", "findprovs", findProvsCID) assert.Equal(t, provs[1]+"\n"+provs[0], res.Stdout.Trimmed()) }) node.StopDaemon() t.Run("adding HTTP delegated routing endpoint to Routing.Routers config works (streaming)", func(t *testing.T) { server := fakeServer("application/x-ndjson", ToJSONStr(JSONObj{ "Schema": "peer", "Protocols": []string{"transport-bitswap"}, "ID": provs[0], "Addrs": []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/tcp/4002"}, }), ToJSONStr(JSONObj{ "Schema": "bitswap", // Legacy bitswap schema. "Protocol": "transport-bitswap", "ID": provs[1], "Addrs": []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/tcp/4002"}, })) t.Cleanup(server.Close) node.IPFS("config", "Routing.Routers.TestDelegatedRouter", "--json", ToJSONStr(JSONObj{ "Type": "http", "Parameters": JSONObj{ "Endpoint": server.URL, }, })) res := node.IPFS("config", "Routing.Routers.TestDelegatedRouter.Parameters.Endpoint") assert.Equal(t, res.Stdout.Trimmed(), server.URL) node.StartDaemon() res = node.IPFS("routing", "findprovs", findProvsCID) assert.Equal(t, provs[0]+"\n"+provs[1], res.Stdout.Trimmed()) }) t.Run("HTTP client should emit OpenCensus metrics", func(t *testing.T) { resp := node.APIClient().Get("/debug/metrics/prometheus") assert.Contains(t, resp.Body, "routing_http_client_length_count") }) } // TestHTTPDelegatedRoutingProviderAddrs verifies that provider records sent to // HTTP routers contain the expected addresses based on Addresses configuration. // See https://github.com/ipfs/kubo/issues/11213 func TestHTTPDelegatedRoutingProviderAddrs(t *testing.T) { t.Parallel() // captureProviderAddrs returns a mock server and a function to retrieve captured addresses. captureProviderAddrs := func(t *testing.T) (*httptest.Server, func() []string) { t.Helper() var mu sync.Mutex var capturedAddrs []string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if (r.Method == http.MethodPut || r.Method == http.MethodPost) && strings.HasPrefix(r.URL.Path, "/routing/v1/providers") { body, _ := io.ReadAll(r.Body) var envelope struct { Providers []struct { Payload json.RawMessage `json:"Payload"` } `json:"Providers"` } if json.Unmarshal(body, &envelope) == nil { for _, prov := range envelope.Providers { var payload struct { Addrs []string `json:"Addrs"` } if json.Unmarshal(prov.Payload, &payload) == nil && len(payload.Addrs) > 0 { mu.Lock() capturedAddrs = payload.Addrs mu.Unlock() } } } w.WriteHeader(http.StatusOK) return } if strings.HasPrefix(r.URL.Path, "/routing/v1/") { w.WriteHeader(http.StatusOK) return } w.WriteHeader(http.StatusNotFound) })) t.Cleanup(srv.Close) return srv, func() []string { mu.Lock() defer mu.Unlock() return capturedAddrs } } customRoutingConf := func(endpoint string) map[string]any { return map[string]any{ "Type": "custom", "Methods": map[string]any{ "provide": map[string]any{"RouterName": "TestRouter"}, "find-providers": map[string]any{"RouterName": "TestRouter"}, "find-peers": map[string]any{"RouterName": "TestRouter"}, "get-ipns": map[string]any{"RouterName": "TestRouter"}, "put-ipns": map[string]any{"RouterName": "TestRouter"}, }, "Routers": map[string]any{ "TestRouter": map[string]any{ "Type": "http", "Parameters": map[string]any{"Endpoint": endpoint}, }, }, } } t.Run("provider records respect user-provided Addresses.Announce override", func(t *testing.T) { t.Parallel() srv, getAddrs := captureProviderAddrs(t) node := harness.NewT(t).NewNode().Init() node.SetIPFSConfig("Addresses.Announce", []string{"/ip4/1.2.3.4/tcp/4001"}) node.SetIPFSConfig("Routing", customRoutingConf(srv.URL)) node.StartDaemon() defer node.StopDaemon() cidStr := node.IPFSAddStr(time.Now().String()) node.IPFS("routing", "provide", cidStr) addrs := getAddrs() require.NotEmpty(t, addrs, "provider record should contain addresses") assert.Equal(t, []string{"/ip4/1.2.3.4/tcp/4001"}, addrs) }) t.Run("provider records respect user-provided Addresses.AppendAnnounce", func(t *testing.T) { t.Parallel() srv, getAddrs := captureProviderAddrs(t) node := harness.NewT(t).NewNode().Init() node.SetIPFSConfig("Addresses.AppendAnnounce", []string{"/ip4/5.6.7.8/tcp/4001"}) node.SetIPFSConfig("Routing", customRoutingConf(srv.URL)) node.StartDaemon() defer node.StopDaemon() cidStr := node.IPFSAddStr(time.Now().String()) node.IPFS("routing", "provide", cidStr) addrs := getAddrs() require.NotEmpty(t, addrs, "provider record should contain addresses") assert.Contains(t, addrs, "/ip4/5.6.7.8/tcp/4001", "AppendAnnounce address should be present") }) } ================================================ FILE: test/cli/delegated_routing_v1_http_proxy_test.go ================================================ package cli import ( "testing" "github.com/ipfs/boxo/ipns" "github.com/ipfs/go-test/random" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestRoutingV1Proxy(t *testing.T) { t.Parallel() setupNodes := func(t *testing.T) harness.Nodes { nodes := harness.NewT(t).NewNodes(3).Init() // Node 0 uses DHT and exposes the Routing API. For the DHT // to actually work there will need to be another DHT-enabled // node. nodes[0].UpdateConfig(func(cfg *config.Config) { cfg.Gateway.ExposeRoutingAPI = config.True cfg.Discovery.MDNS.Enabled = false cfg.Routing.Type = config.NewOptionalString("dht") }) nodes[0].StartDaemon() // Node 1 uses Node 0 as Routing V1 source, no DHT. nodes[1].UpdateConfig(func(cfg *config.Config) { cfg.Discovery.MDNS.Enabled = false cfg.Routing.Type = config.NewOptionalString("custom") cfg.Routing.Methods = config.Methods{ config.MethodNameFindPeers: {RouterName: "KuboA"}, config.MethodNameFindProviders: {RouterName: "KuboA"}, config.MethodNameGetIPNS: {RouterName: "KuboA"}, config.MethodNamePutIPNS: {RouterName: "KuboA"}, config.MethodNameProvide: {RouterName: "KuboA"}, } cfg.Routing.Routers = config.Routers{ "KuboA": config.RouterParser{ Router: config.Router{ Type: config.RouterTypeHTTP, Parameters: &config.HTTPRouterParams{ Endpoint: nodes[0].GatewayURL(), }, }, }, } }) nodes[1].StartDaemon() // This is the second DHT node. Only used so that the DHT is // operative. nodes[2].UpdateConfig(func(cfg *config.Config) { cfg.Gateway.ExposeRoutingAPI = config.True cfg.Discovery.MDNS.Enabled = false cfg.Routing.Type = config.NewOptionalString("dht") }) nodes[2].StartDaemon() t.Cleanup(func() { nodes.StopDaemons() }) // Connect them. nodes.Connect() return nodes } t.Run("Kubo can find provider for CID via Routing V1", func(t *testing.T) { t.Parallel() nodes := setupNodes(t) cidStr := nodes[0].IPFSAddStr(string(random.Bytes(1000))) // Reprovide as initialProviderDelay still ongoing waitUntilProvidesComplete(t, nodes[0]) res := nodes[1].IPFS("routing", "findprovs", cidStr) assert.Equal(t, nodes[0].PeerID().String(), res.Stdout.Trimmed()) }) t.Run("Kubo can find peer via Routing V1", func(t *testing.T) { t.Parallel() nodes := setupNodes(t) // Start lonely node that is not connected to other nodes. node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Discovery.MDNS.Enabled = false cfg.Routing.Type = config.NewOptionalString("dht") }) node.StartDaemon() // Connect Node 0 to Lonely Node. nodes[0].Connect(node) // Node 1 must find Lonely Node through Node 0 Routing V1. res := nodes[1].IPFS("routing", "findpeer", node.PeerID().String()) assert.Equal(t, node.SwarmAddrs()[0].String(), res.Stdout.Trimmed()) }) t.Run("Kubo can retrieve IPNS record via Routing V1", func(t *testing.T) { t.Parallel() nodes := setupNodes(t) nodeName := "/ipns/" + ipns.NameFromPeer(nodes[0].PeerID()).String() // Can't resolve the name as isn't published yet. res := nodes[1].RunIPFS("routing", "get", nodeName) require.Error(t, res.ExitErr) // Publish record on Node 0. path := "/ipfs/" + nodes[0].IPFSAddStr(string(random.Bytes(1000))) nodes[0].IPFS("name", "publish", "--allow-offline", path) // Get record on Node 1 (no DHT). res = nodes[1].IPFS("routing", "get", nodeName) record, err := ipns.UnmarshalRecord(res.Stdout.Bytes()) require.NoError(t, err) value, err := record.Value() require.NoError(t, err) require.Equal(t, path, value.String()) }) t.Run("Kubo can resolve IPNS name via Routing V1", func(t *testing.T) { t.Parallel() nodes := setupNodes(t) nodeName := "/ipns/" + ipns.NameFromPeer(nodes[0].PeerID()).String() // Can't resolve the name as isn't published yet. res := nodes[1].RunIPFS("routing", "get", nodeName) require.Error(t, res.ExitErr) // Publish name. path := "/ipfs/" + nodes[0].IPFSAddStr(string(random.Bytes(1000))) nodes[0].IPFS("name", "publish", "--allow-offline", path) // Resolve IPNS name res = nodes[1].IPFS("name", "resolve", nodeName) require.Equal(t, path, res.Stdout.Trimmed()) }) t.Run("Kubo can provide IPNS record via Routing V1", func(t *testing.T) { t.Parallel() nodes := setupNodes(t) // Publish something on Node 1 (no DHT). nodeName := "/ipns/" + ipns.NameFromPeer(nodes[1].PeerID()).String() path := "/ipfs/" + nodes[1].IPFSAddStr(string(random.Bytes(1000))) nodes[1].IPFS("name", "publish", "--allow-offline", path) // Retrieve through Node 0. res := nodes[0].IPFS("routing", "get", nodeName) record, err := ipns.UnmarshalRecord(res.Stdout.Bytes()) require.NoError(t, err) value, err := record.Value() require.NoError(t, err) require.Equal(t, path, value.String()) }) } ================================================ FILE: test/cli/delegated_routing_v1_http_server_test.go ================================================ package cli import ( "context" "strings" "testing" "time" "github.com/google/uuid" "github.com/ipfs/boxo/autoconf" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/routing/http/client" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" "github.com/ipfs/go-cid" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestRoutingV1Server(t *testing.T) { t.Parallel() setupNodes := func(t *testing.T) harness.Nodes { nodes := harness.NewT(t).NewNodes(5).Init() nodes.ForEachPar(func(node *harness.Node) { node.UpdateConfig(func(cfg *config.Config) { cfg.Gateway.ExposeRoutingAPI = config.True cfg.Routing.Type = config.NewOptionalString("dht") }) }) nodes.StartDaemons().Connect() t.Cleanup(func() { nodes.StopDaemons() }) return nodes } t.Run("Get Providers Responds With Correct Peers", func(t *testing.T) { t.Parallel() nodes := setupNodes(t) text := "hello world " + uuid.New().String() cidStr := nodes[2].IPFSAddStr(text) _ = nodes[3].IPFSAddStr(text) waitUntilProvidesComplete(t, nodes[3]) cid, err := cid.Decode(cidStr) assert.NoError(t, err) c, err := client.New(nodes[1].GatewayURL()) assert.NoError(t, err) resultsIter, err := c.FindProviders(context.Background(), cid) assert.NoError(t, err) records, err := iter.ReadAllResults(resultsIter) assert.NoError(t, err) var peers []peer.ID for _, record := range records { assert.Equal(t, types.SchemaPeer, record.GetSchema()) peer, ok := record.(*types.PeerRecord) assert.True(t, ok) peers = append(peers, *peer.ID) } assert.Contains(t, peers, nodes[2].PeerID()) assert.Contains(t, peers, nodes[3].PeerID()) }) t.Run("Get Peers Responds With Correct Peers", func(t *testing.T) { t.Parallel() nodes := setupNodes(t) c, err := client.New(nodes[1].GatewayURL()) assert.NoError(t, err) resultsIter, err := c.FindPeers(context.Background(), nodes[2].PeerID()) assert.NoError(t, err) records, err := iter.ReadAllResults(resultsIter) assert.NoError(t, err) assert.Len(t, records, 1) assert.IsType(t, records[0].GetSchema(), records[0].GetSchema()) assert.IsType(t, records[0], &types.PeerRecord{}) peer := records[0] assert.Equal(t, nodes[2].PeerID().String(), peer.ID.String()) assert.NotEmpty(t, peer.Addrs) }) t.Run("Get IPNS Record Responds With Correct Record", func(t *testing.T) { t.Parallel() nodes := setupNodes(t) text := "hello ipns test " + uuid.New().String() cidStr := nodes[0].IPFSAddStr(text) nodes[0].IPFS("name", "publish", "--allow-offline", cidStr) // Ask for record from a different peer. c, err := client.New(nodes[1].GatewayURL()) assert.NoError(t, err) record, err := c.GetIPNS(context.Background(), ipns.NameFromPeer(nodes[0].PeerID())) assert.NoError(t, err) value, err := record.Value() assert.NoError(t, err) assert.Equal(t, "/ipfs/"+cidStr, value.String()) }) t.Run("Put IPNS Record Succeeds", func(t *testing.T) { t.Parallel() nodes := setupNodes(t) // Publish a record and confirm the /routing/v1/ipns API exposes the IPNS record text := "hello ipns test " + uuid.New().String() cidStr := nodes[0].IPFSAddStr(text) nodes[0].IPFS("name", "publish", "--allow-offline", cidStr) c, err := client.New(nodes[0].GatewayURL()) assert.NoError(t, err) record, err := c.GetIPNS(context.Background(), ipns.NameFromPeer(nodes[0].PeerID())) assert.NoError(t, err) value, err := record.Value() assert.NoError(t, err) assert.Equal(t, "/ipfs/"+cidStr, value.String()) // Start lonely node that is not connected to other nodes. node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Gateway.ExposeRoutingAPI = config.True cfg.Routing.Type = config.NewOptionalString("dht") }) node.StartDaemon() defer node.StopDaemon() // Put IPNS record in lonely node. It should be accepted as it is a valid record. c, err = client.New(node.GatewayURL()) assert.NoError(t, err) err = c.PutIPNS(context.Background(), ipns.NameFromPeer(nodes[0].PeerID()), record) assert.NoError(t, err) // Get the record from lonely node and double check. record, err = c.GetIPNS(context.Background(), ipns.NameFromPeer(nodes[0].PeerID())) assert.NoError(t, err) value, err = record.Value() assert.NoError(t, err) assert.Equal(t, "/ipfs/"+cidStr, value.String()) }) t.Run("GetClosestPeers returns error when DHT is disabled", func(t *testing.T) { t.Parallel() // Test various routing types that don't support DHT routingTypes := []string{"none", "delegated", "custom"} for _, routingType := range routingTypes { t.Run("routing_type="+routingType, func(t *testing.T) { t.Parallel() // Create node with specified routing type (DHT disabled) node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Gateway.ExposeRoutingAPI = config.True cfg.Routing.Type = config.NewOptionalString(routingType) // For custom routing type, we need to provide minimal valid config // otherwise daemon startup will fail if routingType == "custom" { // Configure a minimal HTTP router (no DHT) cfg.Routing.Routers = map[string]config.RouterParser{ "http-only": { Router: config.Router{ Type: config.RouterTypeHTTP, Parameters: config.HTTPRouterParams{ Endpoint: "https://delegated-ipfs.dev", }, }, }, } cfg.Routing.Methods = map[config.MethodName]config.Method{ config.MethodNameProvide: {RouterName: "http-only"}, config.MethodNameFindProviders: {RouterName: "http-only"}, config.MethodNameFindPeers: {RouterName: "http-only"}, config.MethodNameGetIPNS: {RouterName: "http-only"}, config.MethodNamePutIPNS: {RouterName: "http-only"}, } } // For delegated routing type, ensure we have at least one HTTP router // to avoid daemon startup failure if routingType == "delegated" { // Use a minimal delegated router configuration cfg.Routing.DelegatedRouters = []string{"https://delegated-ipfs.dev"} // Delegated routing doesn't support providing, must be disabled cfg.Provide.Enabled = config.False } }) node.StartDaemon() defer node.StopDaemon() c, err := client.New(node.GatewayURL()) require.NoError(t, err) // Try to get closest peers - should fail gracefully with an error. // Use 60-second timeout (server has 30s routing timeout). testCid, err := cid.Decode("QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() _, err = c.GetClosestPeers(ctx, testCid) require.Error(t, err) // All these routing types should indicate DHT is not available // The exact error message may vary based on implementation details errStr := err.Error() assert.True(t, strings.Contains(errStr, "not supported") || strings.Contains(errStr, "not available") || strings.Contains(errStr, "500"), "Expected error indicating DHT not available for routing type %s, got: %s", routingType, errStr) }) } }) t.Run("GetClosestPeers returns peers", func(t *testing.T) { t.Parallel() routingTypes := []string{"auto", "autoclient", "dht", "dhtclient"} for _, routingType := range routingTypes { t.Run("routing_type="+routingType, func(t *testing.T) { t.Parallel() // Single node with DHT and real bootstrap peers node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Gateway.ExposeRoutingAPI = config.True cfg.Routing.Type = config.NewOptionalString(routingType) // Set real bootstrap peers from boxo/autoconf cfg.Bootstrap = autoconf.FallbackBootstrapPeers }) node.StartDaemon() defer node.StopDaemon() c, err := client.New(node.GatewayURL()) require.NoError(t, err) // Query for closest peers to our own peer ID key := peer.ToCid(node.PeerID()) // Wait for WAN DHT routing table to be populated. // The server has a 30-second routing timeout, so we use 60 seconds // per request to allow for network latency while preventing hangs. // Total wait time is 5 minutes to accommodate slow CI DHT bootstrapping. // Passing runs finish in 8-48s; failures are total bootstrap failures, // not slow convergence, so extra headroom doesn't waste time on success. var records []*types.PeerRecord require.EventuallyWithT(t, func(ct *assert.CollectT) { ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second) defer cancel() resultsIter, err := c.GetClosestPeers(ctx, key) if !assert.NoError(ct, err) { return } records, err = iter.ReadAllResults(resultsIter) assert.NoError(ct, err) }, 5*time.Minute, 5*time.Second) // Verify we got some peers back from WAN DHT require.NotEmpty(t, records, "should return peers close to own peerid") // Per IPIP-0476, GetClosestPeers returns at most 20 peers assert.LessOrEqual(t, len(records), 20, "IPIP-0476 limits GetClosestPeers to 20 peers") // Verify structure of returned records for _, record := range records { assert.Equal(t, types.SchemaPeer, record.Schema) assert.NotNil(t, record.ID) assert.NotEmpty(t, record.Addrs, "peer record should have addresses") } }) } }) } ================================================ FILE: test/cli/dht_autoclient_test.go ================================================ package cli import ( "bytes" "testing" "github.com/ipfs/go-test/random" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" ) func TestDHTAutoclient(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(10).Init() harness.Nodes(nodes[8:]).ForEachPar(func(node *harness.Node) { node.IPFS("config", "Routing.Type", "autoclient") }) nodes.StartDaemons().Connect() t.Cleanup(func() { nodes.StopDaemons() }) t.Run("file added on node in client mode is retrievable from node in client mode", func(t *testing.T) { t.Parallel() randomBytes := random.Bytes(1000) randomBytes = append(randomBytes, '\r') hash := nodes[8].IPFSAdd(bytes.NewReader(randomBytes)) res := nodes[9].IPFS("cat", hash) assert.Equal(t, randomBytes, []byte(res.Stdout.Trimmed())) }) t.Run("file added on node in server mode is retrievable from all nodes", func(t *testing.T) { t.Parallel() randomBytes := random.Bytes(1000) hash := nodes[0].IPFSAdd(bytes.NewReader(randomBytes)) for i := range 10 { res := nodes[i].IPFS("cat", hash) assert.Equal(t, randomBytes, []byte(res.Stdout.Trimmed())) } }) } ================================================ FILE: test/cli/dht_opt_prov_test.go ================================================ package cli import ( "testing" "github.com/ipfs/go-test/random" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" ) func TestDHTOptimisticProvide(t *testing.T) { t.Parallel() t.Run("optimistic provide smoke test", func(t *testing.T) { nodes := harness.NewT(t).NewNodes(2).Init() nodes[0].UpdateConfig(func(cfg *config.Config) { cfg.Experimental.OptimisticProvide = true // Optimistic provide only works with the legacy provider. cfg.Provide.DHT.SweepEnabled = config.False }) nodes.StartDaemons().Connect() defer nodes.StopDaemons() hash := nodes[0].IPFSAddStr(string(random.Bytes(100))) nodes[0].IPFS("routing", "provide", hash) res := nodes[1].IPFS("routing", "findprovs", "--num-providers=1", hash) assert.Equal(t, nodes[0].PeerID().String(), res.Stdout.Trimmed()) }) } ================================================ FILE: test/cli/diag_datastore_test.go ================================================ package cli import ( "encoding/json" "os" "path/filepath" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestDiagDatastore(t *testing.T) { t.Parallel() t.Run("diag datastore get returns error for non-existent key", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Don't start daemon - these commands require daemon to be stopped res := node.RunIPFS("diag", "datastore", "get", "/nonexistent/key") assert.Error(t, res.Err) assert.Contains(t, res.Stderr.String(), "key not found") }) t.Run("diag datastore get returns raw bytes by default", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Add some data to create a known datastore key // We need daemon for add, then stop it node.StartDaemon() cid := node.IPFSAddStr("test data for diag datastore") node.IPFS("pin", "add", cid) node.StopDaemon() // Test count to verify we have entries count := node.DatastoreCount("/") t.Logf("total datastore entries: %d", count) assert.NotEqual(t, int64(0), count, "should have datastore entries after pinning") }) t.Run("diag datastore get --hex returns hex dump", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Add and pin some data node.StartDaemon() cid := node.IPFSAddStr("test data for hex dump") node.IPFS("pin", "add", cid) node.StopDaemon() // Test with existing keys in pins namespace count := node.DatastoreCount("/pins/") t.Logf("pins datastore entries: %d", count) if count != 0 { t.Log("pins datastore has entries, hex dump format tested implicitly") } }) t.Run("diag datastore count returns 0 for empty prefix", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() count := node.DatastoreCount("/definitely/nonexistent/prefix/") assert.Equal(t, int64(0), count) }) t.Run("diag datastore count returns JSON with --enc=json", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() res := node.IPFS("diag", "datastore", "count", "/pubsub/seqno/", "--enc=json") assert.NoError(t, res.Err) var result struct { Prefix string `json:"prefix"` Count int64 `json:"count"` } err := json.Unmarshal(res.Stdout.Bytes(), &result) require.NoError(t, err) assert.Equal(t, "/pubsub/seqno/", result.Prefix) assert.Equal(t, int64(0), result.Count) }) t.Run("diag datastore get returns JSON with --enc=json", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Test error case with JSON encoding res := node.RunIPFS("diag", "datastore", "get", "/nonexistent", "--enc=json") assert.Error(t, res.Err) }) t.Run("diag datastore count counts entries correctly", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Add multiple pins to create multiple entries node.StartDaemon() cid1 := node.IPFSAddStr("data 1") cid2 := node.IPFSAddStr("data 2") cid3 := node.IPFSAddStr("data 3") node.IPFS("pin", "add", cid1) node.IPFS("pin", "add", cid2) node.IPFS("pin", "add", cid3) node.StopDaemon() // Count should reflect the pins (plus any system entries) count := node.DatastoreCount("/") t.Logf("total entries after adding 3 pins: %d", count) // Should have more than 0 entries assert.NotEqual(t, int64(0), count) }) t.Run("diag datastore commands work offline", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Don't start daemon - these commands require daemon to be stopped // Count should work offline count := node.DatastoreCount("/pubsub/seqno/") assert.Equal(t, int64(0), count) // Get should return error for missing key (but command should work) res := node.RunIPFS("diag", "datastore", "get", "/nonexistent/key") assert.Error(t, res.Err) assert.Contains(t, res.Stderr.String(), "key not found") }) t.Run("diag datastore put and get roundtrip", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.DatastorePut("/test/roundtrip", "hello world") assert.True(t, node.DatastoreHasKey("/test/roundtrip")) assert.Equal(t, []byte("hello world"), node.DatastoreGet("/test/roundtrip")) count := node.DatastoreCount("/test/") assert.Equal(t, int64(1), count) }) t.Run("diag datastore commands require daemon to be stopped", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Both get and count require repo lock, which is held by the running daemon res := node.RunIPFS("diag", "datastore", "get", "/test") assert.Error(t, res.Err, "get should fail when daemon is running") assert.Contains(t, res.Stderr.String(), "ipfs daemon is running") res = node.RunIPFS("diag", "datastore", "count", "/pubsub/seqno/") assert.Error(t, res.Err, "count should fail when daemon is running") assert.Contains(t, res.Stderr.String(), "ipfs daemon is running") }) t.Run("provider keystore datastores are visible in unified view", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) // Start daemon to create the provider-keystore datastores, then add data node.StartDaemon() cid := node.IPFSAddStr("data for provider keystore test") node.IPFS("pin", "add", cid) node.StopDaemon() // Verify the provider-keystore directory was created keystorePath := filepath.Join(node.Dir, "provider-keystore") _, err := os.Stat(keystorePath) require.NoError(t, err, "provider-keystore directory should exist after sweep-enabled daemon ran") // Count entries in each keystore namespace via the unified view for _, prefix := range []string{"/provider/keystore/0/", "/provider/keystore/1/"} { res := node.IPFS("diag", "datastore", "count", prefix) assert.NoError(t, res.Err) t.Logf("count %s: %s", prefix, res.Stdout.String()) } // The total count under /provider/keystore/ should include entries // from both keystore instances (0 and 1) count := node.DatastoreCount("/provider/keystore/") t.Logf("total /provider/keystore/ entries: %d", count) assert.Greater(t, count, int64(0), "should have provider keystore entries") }) t.Run("provider keystore count JSON output", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) node.StartDaemon() node.StopDaemon() res := node.IPFS("diag", "datastore", "count", "/provider/keystore/0/", "--enc=json") assert.NoError(t, res.Err) var result struct { Prefix string `json:"prefix"` Count int64 `json:"count"` } err := json.Unmarshal(res.Stdout.Bytes(), &result) require.NoError(t, err) assert.Equal(t, "/provider/keystore/0/", result.Prefix) assert.GreaterOrEqual(t, result.Count, int64(0), "count should be non-negative") }) t.Run("works without provider keystore", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // No sweep enabled, no provider-keystore dirs — should still work fine count := node.DatastoreCount("/provider/keystore/0/") assert.Zero(t, count) count = node.DatastoreCount("/") assert.Greater(t, count, int64(0)) }) } ================================================ FILE: test/cli/dns_resolvers_multiaddr_test.go ================================================ package cli import ( "strings" "testing" "time" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // testDomainSuffix is the default p2p-forge domain used in tests const testDomainSuffix = config.DefaultDomainSuffix // libp2p.direct // TestDNSResolversApplyToMultiaddr is a regression test for: // https://github.com/ipfs/kubo/issues/9199 // // It verifies that DNS.Resolvers config is used when resolving /dnsaddr, // /dns, /dns4, /dns6 multiaddrs during peer connections, not just for // DNSLink resolution. func TestDNSResolversApplyToMultiaddr(t *testing.T) { t.Parallel() t.Run("invalid DoH resolver causes multiaddr resolution to fail", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init("--profile=test") // Set an invalid DoH resolver that will fail when used. // If DNS.Resolvers is properly wired to multiaddr resolution, // swarm connect to a /dnsaddr will fail with an error mentioning // the invalid resolver URL. invalidResolver := "https://invalid.broken.resolver.test/dns-query" node.SetIPFSConfig("DNS.Resolvers", map[string]string{ ".": invalidResolver, }) // Clear bootstrap peers to prevent background connection attempts node.SetIPFSConfig("Bootstrap", []string{}) node.StartDaemon() defer node.StopDaemon() // Give daemon time to fully start time.Sleep(2 * time.Second) // Verify daemon is responsive result := node.RunIPFS("id") require.Equal(t, 0, result.ExitCode(), "daemon should be responsive") // Try to connect to a /dnsaddr peer - this should fail because // the DNS.Resolvers config points to an invalid DoH server result = node.RunIPFS("swarm", "connect", "/dnsaddr/bootstrap.libp2p.io") // The connection should fail require.NotEqual(t, 0, result.ExitCode(), "swarm connect should fail when DNS.Resolvers points to invalid DoH server") // The error should mention the invalid resolver, proving DNS.Resolvers // is being used for multiaddr resolution stderr := result.Stderr.String() assert.True(t, strings.Contains(stderr, "invalid.broken.resolver.test") || strings.Contains(stderr, "no such host") || strings.Contains(stderr, "lookup") || strings.Contains(stderr, "dial"), "error should indicate DNS resolution failure using custom resolver. got: %s", stderr) }) t.Run("libp2p.direct resolves locally even with broken DNS.Resolvers", func(t *testing.T) { t.Parallel() h := harness.NewT(t) nodes := h.NewNodes(2).Init("--profile=test") // Configure node0 with a broken DNS resolver // This would break all DNS resolution if libp2p.direct wasn't resolved locally invalidResolver := "https://invalid.broken.resolver.test/dns-query" nodes[0].SetIPFSConfig("DNS.Resolvers", map[string]string{ ".": invalidResolver, }) // Clear bootstrap peers on both nodes for _, n := range nodes { n.SetIPFSConfig("Bootstrap", []string{}) } nodes.StartDaemons() defer nodes.StopDaemons() // Get node1's peer ID in base36 format (what p2p-forge uses in DNS hostnames) // DNS is case-insensitive, and base36 is lowercase-only, making it ideal for DNS idResult := nodes[1].RunIPFS("id", "--peerid-base", "base36", "-f", "") require.Equal(t, 0, idResult.ExitCode()) node1IDBase36 := strings.TrimSpace(idResult.Stdout.String()) node1ID := nodes[1].PeerID().String() node1Addrs := nodes[1].SwarmAddrs() // Find a TCP address we can use var tcpAddr string for _, addr := range node1Addrs { addrStr := addr.String() if strings.Contains(addrStr, "/tcp/") && strings.Contains(addrStr, "/ip4/127.0.0.1") { tcpAddr = addrStr break } } require.NotEmpty(t, tcpAddr, "node1 should have a local TCP address") // Extract port from address like /ip4/127.0.0.1/tcp/12345/... parts := strings.Split(tcpAddr, "/") var port string for i, p := range parts { if p == "tcp" && i+1 < len(parts) { port = parts[i+1] break } } require.NotEmpty(t, port, "should find TCP port in address") // Construct a libp2p.direct hostname that encodes 127.0.0.1 // Format: /dns4/..libp2p.direct/tcp//p2p/ // p2p-forge uses base36 peerIDs in DNS hostnames (lowercase, DNS-safe) libp2pDirectAddr := "/dns4/127-0-0-1." + node1IDBase36 + "." + testDomainSuffix + "/tcp/" + port + "/p2p/" + node1ID // This connection should succeed because libp2p.direct is resolved locally // even though DNS.Resolvers points to a broken server result := nodes[0].RunIPFS("swarm", "connect", libp2pDirectAddr) // The connection should succeed - local resolution bypasses broken DNS assert.Equal(t, 0, result.ExitCode(), "swarm connect to libp2p.direct should succeed with local resolution. stderr: %s", result.Stderr.String()) // Verify the connection was actually established result = nodes[0].RunIPFS("swarm", "peers") require.Equal(t, 0, result.ExitCode()) assert.Contains(t, result.Stdout.String(), node1ID, "node0 should be connected to node1") }) } ================================================ FILE: test/cli/files_test.go ================================================ package cli import ( "encoding/json" "fmt" "os" "path/filepath" "strings" "testing" ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestFilesCp(t *testing.T) { t.Parallel() t.Run("files cp with valid UnixFS succeeds", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Create simple text file data := "testing files cp command" cid := node.IPFSAddStr(data) // Copy form IPFS => MFS res := node.IPFS("files", "cp", fmt.Sprintf("/ipfs/%s", cid), "/valid-file") assert.NoError(t, res.Err) // verification catRes := node.IPFS("files", "read", "/valid-file") assert.Equal(t, data, catRes.Stdout.Trimmed()) }) t.Run("files cp with unsupported DAG node type fails", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // MFS UnixFS is limited to dag-pb or raw, so we create a dag-cbor node to test this jsonData := `{"data": "not a UnixFS node"}` tempFile := filepath.Join(node.Dir, "test.json") err := os.WriteFile(tempFile, []byte(jsonData), 0644) require.NoError(t, err) cid := node.IPFS("dag", "put", "--input-codec=json", "--store-codec=dag-cbor", tempFile).Stdout.Trimmed() // copy without --force res := node.RunIPFS("files", "cp", fmt.Sprintf("/ipfs/%s", cid), "/invalid-file") assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), "Error: cp: source must be a valid UnixFS (dag-pb or raw codec)") }) t.Run("files cp with invalid UnixFS data structure fails", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Create an invalid proto file data := []byte{0xDE, 0xAD, 0xBE, 0xEF} // Invalid protobuf data tempFile := filepath.Join(node.Dir, "invalid-proto.bin") err := os.WriteFile(tempFile, data, 0644) require.NoError(t, err) res := node.IPFS("block", "put", "--format=raw", tempFile) require.NoError(t, res.Err) // we manually changed codec from raw to dag-pb to test "bad dag-pb" scenario cid := "bafybeic7pdbte5heh6u54vszezob3el6exadoiw4wc4ne7ny2x7kvajzkm" // should fail because node cannot be read as a valid dag-pb cpResNoForce := node.RunIPFS("files", "cp", fmt.Sprintf("/ipfs/%s", cid), "/invalid-proto") assert.NotEqual(t, 0, cpResNoForce.ExitErr.ExitCode()) assert.Contains(t, cpResNoForce.Stderr.String(), "Error") }) t.Run("files cp with raw node succeeds", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Create a raw node data := "raw data" tempFile := filepath.Join(node.Dir, "raw.bin") err := os.WriteFile(tempFile, []byte(data), 0644) require.NoError(t, err) res := node.IPFS("block", "put", "--format=raw", tempFile) require.NoError(t, res.Err) cid := res.Stdout.Trimmed() // Copy from IPFS to MFS (raw nodes should work without --force) cpRes := node.IPFS("files", "cp", fmt.Sprintf("/ipfs/%s", cid), "/raw-file") assert.NoError(t, cpRes.Err) // Verify the file was copied correctly catRes := node.IPFS("files", "read", "/raw-file") assert.Equal(t, data, catRes.Stdout.Trimmed()) }) t.Run("files cp creates intermediate directories with -p", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Create a simple text file and add it to IPFS data := "hello parent directories" tempFile := filepath.Join(node.Dir, "parent-test.txt") err := os.WriteFile(tempFile, []byte(data), 0644) require.NoError(t, err) cid := node.IPFS("add", "-Q", tempFile).Stdout.Trimmed() // Copy from IPFS to MFS with parent flag res := node.IPFS("files", "cp", "-p", fmt.Sprintf("/ipfs/%s", cid), "/parent/dir/file") assert.NoError(t, res.Err) // Verify the file and directories were created lsRes := node.IPFS("files", "ls", "/parent/dir") assert.Contains(t, lsRes.Stdout.String(), "file") catRes := node.IPFS("files", "read", "/parent/dir/file") assert.Equal(t, data, catRes.Stdout.Trimmed()) }) } func TestFilesRm(t *testing.T) { t.Parallel() t.Run("files rm with --flush=false returns error", func(t *testing.T) { // Test that files rm rejects --flush=false so user does not assume disabling flush works // (rm ignored it before, better to explicitly error) // See https://github.com/ipfs/kubo/issues/10842 t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Create a file to remove node.IPFS("files", "mkdir", "/test-dir") // Try to remove with --flush=false, should error res := node.RunIPFS("files", "rm", "-r", "--flush=false", "/test-dir") assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), "files rm always flushes for safety") assert.Contains(t, res.Stderr.String(), "cannot be set to false") // Verify the directory still exists (wasn't removed due to error) lsRes := node.IPFS("files", "ls", "/") assert.Contains(t, lsRes.Stdout.String(), "test-dir") }) t.Run("files rm with --flush=true works", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Create a file to remove node.IPFS("files", "mkdir", "/test-dir") // Remove with explicit --flush=true, should work res := node.IPFS("files", "rm", "-r", "--flush=true", "/test-dir") assert.NoError(t, res.Err) // Verify the directory was removed lsRes := node.IPFS("files", "ls", "/") assert.NotContains(t, lsRes.Stdout.String(), "test-dir") }) t.Run("files rm without flush flag works (default behavior)", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Create a file to remove node.IPFS("files", "mkdir", "/test-dir") // Remove without flush flag (should use default which is true) res := node.IPFS("files", "rm", "-r", "/test-dir") assert.NoError(t, res.Err) // Verify the directory was removed lsRes := node.IPFS("files", "ls", "/") assert.NotContains(t, lsRes.Stdout.String(), "test-dir") }) } func TestFilesNoFlushLimit(t *testing.T) { t.Parallel() t.Run("reaches default limit of 256 operations", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Perform 256 operations with --flush=false (should succeed) for i := range 256 { res := node.IPFS("files", "mkdir", "--flush=false", fmt.Sprintf("/dir%d", i)) assert.NoError(t, res.Err, "operation %d should succeed", i+1) } // 257th operation should fail res := node.RunIPFS("files", "mkdir", "--flush=false", "/dir256") require.NotNil(t, res.ExitErr, "command should have failed") assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), "reached limit of 256 unflushed MFS operations") assert.Contains(t, res.Stderr.String(), "run 'ipfs files flush'") assert.Contains(t, res.Stderr.String(), "use --flush=true") assert.Contains(t, res.Stderr.String(), "increase Internal.MFSNoFlushLimit") }) t.Run("custom limit via config", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Set custom limit to 5 node.UpdateConfig(func(cfg *config.Config) { limit := config.NewOptionalInteger(5) cfg.Internal.MFSNoFlushLimit = limit }) node.StartDaemon() defer node.StopDaemon() // Perform 5 operations (should succeed) for i := range 5 { res := node.IPFS("files", "mkdir", "--flush=false", fmt.Sprintf("/dir%d", i)) assert.NoError(t, res.Err, "operation %d should succeed", i+1) } // 6th operation should fail res := node.RunIPFS("files", "mkdir", "--flush=false", "/dir5") require.NotNil(t, res.ExitErr, "command should have failed") assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), "reached limit of 5 unflushed MFS operations") }) t.Run("flush=true resets counter", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Set limit to 3 for faster testing node.UpdateConfig(func(cfg *config.Config) { limit := config.NewOptionalInteger(3) cfg.Internal.MFSNoFlushLimit = limit }) node.StartDaemon() defer node.StopDaemon() // Do 2 operations with --flush=false node.IPFS("files", "mkdir", "--flush=false", "/dir1") node.IPFS("files", "mkdir", "--flush=false", "/dir2") // Operation with --flush=true should reset counter node.IPFS("files", "mkdir", "--flush=true", "/dir3") // Now we should be able to do 3 more operations with --flush=false for i := 4; i <= 6; i++ { res := node.IPFS("files", "mkdir", "--flush=false", fmt.Sprintf("/dir%d", i)) assert.NoError(t, res.Err, "operation after flush should succeed") } // 4th operation after reset should fail res := node.RunIPFS("files", "mkdir", "--flush=false", "/dir7") require.NotNil(t, res.ExitErr, "command should have failed") assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), "reached limit of 3 unflushed MFS operations") }) t.Run("explicit flush command resets counter", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Set limit to 3 for faster testing node.UpdateConfig(func(cfg *config.Config) { limit := config.NewOptionalInteger(3) cfg.Internal.MFSNoFlushLimit = limit }) node.StartDaemon() defer node.StopDaemon() // Do 2 operations with --flush=false node.IPFS("files", "mkdir", "--flush=false", "/dir1") node.IPFS("files", "mkdir", "--flush=false", "/dir2") // Explicit flush should reset counter node.IPFS("files", "flush") // Now we should be able to do 3 more operations for i := 3; i <= 5; i++ { res := node.IPFS("files", "mkdir", "--flush=false", fmt.Sprintf("/dir%d", i)) assert.NoError(t, res.Err, "operation after flush should succeed") } // 4th operation should fail res := node.RunIPFS("files", "mkdir", "--flush=false", "/dir6") require.NotNil(t, res.ExitErr, "command should have failed") assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), "reached limit of 3 unflushed MFS operations") }) t.Run("limit=0 disables the feature", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Set limit to 0 (disabled) node.UpdateConfig(func(cfg *config.Config) { limit := config.NewOptionalInteger(0) cfg.Internal.MFSNoFlushLimit = limit }) node.StartDaemon() defer node.StopDaemon() // Should be able to do many operations without error for i := range 300 { res := node.IPFS("files", "mkdir", "--flush=false", fmt.Sprintf("/dir%d", i)) assert.NoError(t, res.Err, "operation %d should succeed with limit disabled", i+1) } }) t.Run("different MFS commands count towards limit", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Set limit to 5 for testing node.UpdateConfig(func(cfg *config.Config) { limit := config.NewOptionalInteger(5) cfg.Internal.MFSNoFlushLimit = limit }) node.StartDaemon() defer node.StopDaemon() // Mix of different MFS operations (5 operations to hit the limit) node.IPFS("files", "mkdir", "--flush=false", "/testdir") // Create a file first, then copy it testCid := node.IPFSAddStr("test content") node.IPFS("files", "cp", "--flush=false", fmt.Sprintf("/ipfs/%s", testCid), "/testfile") node.IPFS("files", "cp", "--flush=false", "/testfile", "/testfile2") node.IPFS("files", "mv", "--flush=false", "/testfile2", "/testfile3") node.IPFS("files", "mkdir", "--flush=false", "/anotherdir") // 6th operation should fail res := node.RunIPFS("files", "mkdir", "--flush=false", "/another") require.NotNil(t, res.ExitErr, "command should have failed") assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), "reached limit of 5 unflushed MFS operations") }) } func TestFilesChroot(t *testing.T) { t.Parallel() // Known CIDs for testing emptyDirCid := "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" t.Run("requires --confirm flag", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Don't start daemon - chroot runs offline res := node.RunIPFS("files", "chroot") require.NotNil(t, res.ExitErr) assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), "pass --confirm to proceed") }) t.Run("resets to empty directory", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Start daemon to create MFS state node.StartDaemon() node.IPFS("files", "mkdir", "/testdir") node.StopDaemon() // Reset MFS to empty - should exit 0 res := node.RunIPFS("files", "chroot", "--confirm") assert.Nil(t, res.ExitErr, "expected exit code 0") assert.Contains(t, res.Stdout.String(), emptyDirCid) // Verify daemon starts and MFS is empty node.StartDaemon() defer node.StopDaemon() lsRes := node.IPFS("files", "ls", "/") assert.Empty(t, lsRes.Stdout.Trimmed()) }) t.Run("replaces with valid directory CID", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Start daemon to add content node.StartDaemon() node.IPFS("files", "mkdir", "/mydir") // Create a temp file for content tempFile := filepath.Join(node.Dir, "testfile.txt") require.NoError(t, os.WriteFile(tempFile, []byte("hello"), 0644)) node.IPFS("files", "write", "--create", "/mydir/file.txt", tempFile) statRes := node.IPFS("files", "stat", "--hash", "/mydir") dirCid := statRes.Stdout.Trimmed() node.StopDaemon() // Reset to empty first node.IPFS("files", "chroot", "--confirm") // Set root to the saved directory - should exit 0 res := node.RunIPFS("files", "chroot", "--confirm", dirCid) assert.Nil(t, res.ExitErr, "expected exit code 0") assert.Contains(t, res.Stdout.String(), dirCid) // Verify content node.StartDaemon() defer node.StopDaemon() readRes := node.IPFS("files", "read", "/file.txt") assert.Equal(t, "hello", readRes.Stdout.Trimmed()) }) t.Run("fails with non-existent CID", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() res := node.RunIPFS("files", "chroot", "--confirm", "bafybeibdxtd5thfoitjmnfhxhywokebwdmwnuqgkzjjdjhwjz7qh77777a") require.NotNil(t, res.ExitErr) assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), "does not exist locally") }) t.Run("fails with file CID", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Add a file to get a file CID node.StartDaemon() fileCid := node.IPFSAddStr("hello world") node.StopDaemon() // Try to set file as root - should fail with non-zero exit res := node.RunIPFS("files", "chroot", "--confirm", fileCid) require.NotNil(t, res.ExitErr) assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), "must be a directory") }) t.Run("fails while daemon is running", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() res := node.RunIPFS("files", "chroot", "--confirm") require.NotNil(t, res.ExitErr) assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), "opening repo") }) } // TestFilesMFSImportConfig tests that MFS operations respect Import.* configuration settings. // These tests verify that `ipfs files` commands use the same import settings as `ipfs add`. func TestFilesMFSImportConfig(t *testing.T) { t.Parallel() t.Run("files write respects Import.CidVersion=1", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.CidVersion = *config.NewOptionalInteger(1) }) node.StartDaemon() defer node.StopDaemon() // Write file via MFS tempFile := filepath.Join(node.Dir, "test.txt") require.NoError(t, os.WriteFile(tempFile, []byte("hello"), 0644)) node.IPFS("files", "write", "--create", "/test.txt", tempFile) // Get CID of written file cidStr := node.IPFS("files", "stat", "--hash", "/test.txt").Stdout.Trimmed() // Verify CIDv1 format (base32, starts with "b") require.True(t, strings.HasPrefix(cidStr, "b"), "expected CIDv1 (starts with b), got: %s", cidStr) }) t.Run("files write respects Import.UnixFSRawLeaves=true", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.CidVersion = *config.NewOptionalInteger(1) cfg.Import.UnixFSRawLeaves = config.True }) node.StartDaemon() defer node.StopDaemon() tempFile := filepath.Join(node.Dir, "test.txt") require.NoError(t, os.WriteFile(tempFile, []byte("hello world"), 0644)) node.IPFS("files", "write", "--create", "/test.txt", tempFile) cidStr := node.IPFS("files", "stat", "--hash", "/test.txt").Stdout.Trimmed() codec := node.IPFS("cid", "format", "-f", "%c", cidStr).Stdout.Trimmed() require.Equal(t, "raw", codec, "expected raw codec for small file with raw leaves") }) // This test verifies CID parity for single-block files only. // Multi-block files will have different CIDs because MFS uses trickle DAG layout // while 'ipfs add' uses balanced DAG layout. See "files write vs add for multi-block" test. t.Run("single-block file: files write produces same CID as ipfs add", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.CidVersion = *config.NewOptionalInteger(1) cfg.Import.UnixFSRawLeaves = config.True }) node.StartDaemon() defer node.StopDaemon() tempFile := filepath.Join(node.Dir, "test.txt") require.NoError(t, os.WriteFile(tempFile, []byte("hello world"), 0644)) node.IPFS("files", "write", "--create", "/test.txt", tempFile) mfsCid := node.IPFS("files", "stat", "--hash", "/test.txt").Stdout.Trimmed() addCid := node.IPFSAddStr("hello world") require.Equal(t, addCid, mfsCid, "MFS write should produce same CID as ipfs add for single-block files") }) t.Run("files mkdir respects Import.CidVersion=1", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.CidVersion = *config.NewOptionalInteger(1) }) node.StartDaemon() defer node.StopDaemon() node.IPFS("files", "mkdir", "/testdir") cidStr := node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() // Verify CIDv1 format require.True(t, strings.HasPrefix(cidStr, "b"), "expected CIDv1 (starts with b), got: %s", cidStr) }) t.Run("MFS subdirectory becomes HAMT when exceeding threshold", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { // Use small threshold for faster testing cfg.Import.UnixFSHAMTDirectorySizeThreshold = *config.NewOptionalBytes("1KiB") cfg.Import.UnixFSHAMTDirectorySizeEstimation = *config.NewOptionalString("block") }) node.StartDaemon() defer node.StopDaemon() node.IPFS("files", "mkdir", "/bigdir") content := "x" tempFile := filepath.Join(node.Dir, "content.txt") require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) // Add enough files to exceed 1KiB threshold for i := range 25 { node.IPFS("files", "write", "--create", fmt.Sprintf("/bigdir/file%02d", i), tempFile) } cidStr := node.IPFS("files", "stat", "--hash", "/bigdir").Stdout.Trimmed() fsType, err := node.UnixFSDataType(cidStr) require.NoError(t, err) require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory") }) t.Run("MFS root directory becomes HAMT when exceeding threshold", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.UnixFSHAMTDirectorySizeThreshold = *config.NewOptionalBytes("1KiB") cfg.Import.UnixFSHAMTDirectorySizeEstimation = *config.NewOptionalString("block") }) node.StartDaemon() defer node.StopDaemon() content := "x" tempFile := filepath.Join(node.Dir, "content.txt") require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) // Add files directly to root / for i := range 25 { node.IPFS("files", "write", "--create", fmt.Sprintf("/file%02d", i), tempFile) } cidStr := node.IPFS("files", "stat", "--hash", "/").Stdout.Trimmed() fsType, err := node.UnixFSDataType(cidStr) require.NoError(t, err) require.Equal(t, ft.THAMTShard, fsType, "expected MFS root to become HAMT") }) t.Run("MFS directory reverts from HAMT to basic when items removed", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.UnixFSHAMTDirectorySizeThreshold = *config.NewOptionalBytes("1KiB") cfg.Import.UnixFSHAMTDirectorySizeEstimation = *config.NewOptionalString("block") }) node.StartDaemon() defer node.StopDaemon() node.IPFS("files", "mkdir", "/testdir") content := "x" tempFile := filepath.Join(node.Dir, "content.txt") require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) // Add files to exceed threshold for i := range 25 { node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%02d", i), tempFile) } // Verify it became HAMT cidStr := node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() fsType, err := node.UnixFSDataType(cidStr) require.NoError(t, err) require.Equal(t, ft.THAMTShard, fsType, "should be HAMT after adding many files") // Remove files to get back below threshold for i := range 20 { node.IPFS("files", "rm", fmt.Sprintf("/testdir/file%02d", i)) } // Verify it reverted to basic directory cidStr = node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() fsType, err = node.UnixFSDataType(cidStr) require.NoError(t, err) require.Equal(t, ft.TDirectory, fsType, "should revert to basic directory after removing files") }) // Note: 'files write' produces DIFFERENT CIDs than 'ipfs add' for multi-block files because // MFS uses trickle DAG layout while 'ipfs add' uses balanced DAG layout. // Single-block files produce the same CID (tested above in "single-block file: files write..."). // For multi-block CID compatibility with 'ipfs add', use 'ipfs add --to-files' instead. t.Run("files cp preserves original CID", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.CidVersion = *config.NewOptionalInteger(1) cfg.Import.UnixFSRawLeaves = config.True }) node.StartDaemon() defer node.StopDaemon() // Add file via ipfs add originalCid := node.IPFSAddStr("hello world") // Copy to MFS node.IPFS("files", "cp", fmt.Sprintf("/ipfs/%s", originalCid), "/copied.txt") // Verify CID is preserved mfsCid := node.IPFS("files", "stat", "--hash", "/copied.txt").Stdout.Trimmed() require.Equal(t, originalCid, mfsCid, "files cp should preserve original CID") }) t.Run("add --to-files respects Import config", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.CidVersion = *config.NewOptionalInteger(1) cfg.Import.UnixFSRawLeaves = config.True }) node.StartDaemon() defer node.StopDaemon() // Create temp file tempFile := filepath.Join(node.Dir, "test.txt") require.NoError(t, os.WriteFile(tempFile, []byte("hello world"), 0644)) // Add with --to-files addCid := node.IPFS("add", "-Q", "--to-files=/added.txt", tempFile).Stdout.Trimmed() // Verify MFS file has same CID mfsCid := node.IPFS("files", "stat", "--hash", "/added.txt").Stdout.Trimmed() require.Equal(t, addCid, mfsCid) // Should be CIDv1 raw leaf codec := node.IPFS("cid", "format", "-f", "%c", mfsCid).Stdout.Trimmed() require.Equal(t, "raw", codec) }) t.Run("files mkdir respects Import.UnixFSDirectoryMaxLinks", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.CidVersion = *config.NewOptionalInteger(1) // Set low link threshold to trigger HAMT sharding at 5 links cfg.Import.UnixFSDirectoryMaxLinks = *config.NewOptionalInteger(5) // Also need size estimation enabled for switching to work cfg.Import.UnixFSHAMTDirectorySizeEstimation = *config.NewOptionalString("block") }) node.StartDaemon() defer node.StopDaemon() // Create directory with 6 files (exceeds max 5 links) node.IPFS("files", "mkdir", "/testdir") content := "x" tempFile := filepath.Join(node.Dir, "content.txt") require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) for i := range 6 { node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%d.txt", i), tempFile) } // Verify directory became HAMT sharded cidStr := node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() fsType, err := node.UnixFSDataType(cidStr) require.NoError(t, err) require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory after exceeding UnixFSDirectoryMaxLinks") }) t.Run("files write respects Import.UnixFSChunker", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.CidVersion = *config.NewOptionalInteger(1) cfg.Import.UnixFSRawLeaves = config.True cfg.Import.UnixFSChunker = *config.NewOptionalString("size-1024") // 1KB chunks }) node.StartDaemon() defer node.StopDaemon() // Create file larger than chunk size (3KB) data := make([]byte, 3*1024) for i := range data { data[i] = byte(i % 256) } tempFile := filepath.Join(node.Dir, "large.bin") require.NoError(t, os.WriteFile(tempFile, data, 0644)) node.IPFS("files", "write", "--create", "/large.bin", tempFile) // Verify chunking: 3KB file with 1KB chunks should have multiple child blocks cidStr := node.IPFS("files", "stat", "--hash", "/large.bin").Stdout.Trimmed() dagStatJSON := node.IPFS("dag", "stat", "--enc=json", cidStr).Stdout.Trimmed() var dagStat struct { UniqueBlocks int `json:"UniqueBlocks"` } require.NoError(t, json.Unmarshal([]byte(dagStatJSON), &dagStat)) // With 1KB chunks on a 3KB file, we expect 4 blocks (3 leaf + 1 root) assert.Greater(t, dagStat.UniqueBlocks, 1, "expected more than 1 block with 1KB chunker on 3KB file") }) t.Run("files write with custom chunker produces same CID as ipfs add --trickle", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Import.CidVersion = *config.NewOptionalInteger(1) cfg.Import.UnixFSRawLeaves = config.True cfg.Import.UnixFSChunker = *config.NewOptionalString("size-512") }) node.StartDaemon() defer node.StopDaemon() // Create test data (2KB to get multiple chunks) data := make([]byte, 2048) for i := range data { data[i] = byte(i % 256) } tempFile := filepath.Join(node.Dir, "test.bin") require.NoError(t, os.WriteFile(tempFile, data, 0644)) // Add via MFS node.IPFS("files", "write", "--create", "/test.bin", tempFile) mfsCid := node.IPFS("files", "stat", "--hash", "/test.bin").Stdout.Trimmed() // Add via ipfs add with same chunker and trickle (MFS always uses trickle) addCid := node.IPFS("add", "-Q", "--chunker=size-512", "--trickle", tempFile).Stdout.Trimmed() // CIDs should match when using same chunker + trickle layout require.Equal(t, addCid, mfsCid, "MFS and add --trickle should produce same CID with matching chunker") }) t.Run("files mkdir respects Import.UnixFSHAMTDirectoryMaxFanout", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { // Use non-default fanout of 64 (default is 256) cfg.Import.UnixFSHAMTDirectoryMaxFanout = *config.NewOptionalInteger(64) // Set low link threshold to trigger HAMT at 5 links cfg.Import.UnixFSDirectoryMaxLinks = *config.NewOptionalInteger(5) cfg.Import.UnixFSHAMTDirectorySizeEstimation = *config.NewOptionalString("disabled") }) node.StartDaemon() defer node.StopDaemon() node.IPFS("files", "mkdir", "/testdir") content := "x" tempFile := filepath.Join(node.Dir, "content.txt") require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) // Add 6 files (exceeds MaxLinks=5) to trigger HAMT for i := range 6 { node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%d.txt", i), tempFile) } // Verify directory became HAMT cidStr := node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() fsType, err := node.UnixFSDataType(cidStr) require.NoError(t, err) require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory") // Verify the HAMT uses the custom fanout (64) by inspecting the UnixFS Data field. fanout, err := node.UnixFSHAMTFanout(cidStr) require.NoError(t, err) require.Equal(t, uint64(64), fanout, "expected HAMT fanout 64") }) t.Run("files mkdir respects Import.UnixFSHAMTDirectorySizeThreshold", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { // Use very small threshold (100 bytes) to trigger HAMT quickly cfg.Import.UnixFSHAMTDirectorySizeThreshold = *config.NewOptionalBytes("100B") cfg.Import.UnixFSHAMTDirectorySizeEstimation = *config.NewOptionalString("block") }) node.StartDaemon() defer node.StopDaemon() node.IPFS("files", "mkdir", "/testdir") content := "test content" tempFile := filepath.Join(node.Dir, "content.txt") require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) // Add 3 files - each link adds ~40-50 bytes, so 3 should exceed 100B threshold for i := range 3 { node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%d.txt", i), tempFile) } // Verify directory became HAMT due to size threshold cidStr := node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() fsType, err := node.UnixFSDataType(cidStr) require.NoError(t, err) require.Equal(t, ft.THAMTShard, fsType, "expected HAMT directory after exceeding size threshold") }) t.Run("config change takes effect after daemon restart", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Start with high threshold (won't trigger HAMT) node.UpdateConfig(func(cfg *config.Config) { cfg.Import.UnixFSHAMTDirectorySizeThreshold = *config.NewOptionalBytes("256KiB") cfg.Import.UnixFSHAMTDirectorySizeEstimation = *config.NewOptionalString("block") }) node.StartDaemon() // Create directory with some files node.IPFS("files", "mkdir", "/testdir") content := "test" tempFile := filepath.Join(node.Dir, "content.txt") require.NoError(t, os.WriteFile(tempFile, []byte(content), 0644)) for i := range 3 { node.IPFS("files", "write", "--create", fmt.Sprintf("/testdir/file%d.txt", i), tempFile) } // Verify it's still a basic directory (threshold not exceeded) cidStr := node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() fsType, err := node.UnixFSDataType(cidStr) require.NoError(t, err) require.Equal(t, ft.TDirectory, fsType, "should be basic directory with high threshold") // Stop daemon node.StopDaemon() // Change config to use very low threshold node.UpdateConfig(func(cfg *config.Config) { cfg.Import.UnixFSHAMTDirectorySizeThreshold = *config.NewOptionalBytes("100B") }) // Restart daemon node.StartDaemon() defer node.StopDaemon() // Add one more file - this should trigger HAMT conversion with new threshold node.IPFS("files", "write", "--create", "/testdir/file3.txt", tempFile) // Verify it became HAMT (new threshold applied) cidStr = node.IPFS("files", "stat", "--hash", "/testdir").Stdout.Trimmed() fsType, err = node.UnixFSDataType(cidStr) require.NoError(t, err) require.Equal(t, ft.THAMTShard, fsType, "should be HAMT after daemon restart with lower threshold") }) } ================================================ FILE: test/cli/fixtures/README.md ================================================ # Dataset Description / Sources TestGatewayHAMTDirectory.car generated with: ```bash ipfs version # ipfs version 0.19.0 export HAMT_DIR=bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm export IPFS_PATH=$(mktemp -d) # Init and start daemon, ensure we have an empty repository. ipfs init --empty-repo ipfs daemon &> /dev/null & export IPFS_PID=$! # Retrieve the directory listing, forcing the daemon to download all required DAGs. Kill daemon. curl -o dir.html http://127.0.0.1:8080/ipfs/$HAMT_DIR/ kill $IPFS_PID # Get the list with all the downloaded refs and sanity check. ipfs refs local > required_refs cat required_refs | wc -l # 962 # Get the list of all the files CIDs inside the directory and sanity check. cat dir.html| pup '#content tbody .ipfs-hash attr{href}' | sed 's/\/ipfs\///g;s/\?filename=.*//g' > files_refs cat files_refs | wc -l # 10100 # Make and export our fixture. ipfs files mkdir --cid-version 1 /fixtures cat required_refs | xargs -I {} ipfs files cp /ipfs/{} /fixtures/{} cat files_refs | ipfs files write --create /fixtures/files_refs export FIXTURE_CID=$(ipfs files stat --hash /fixtures/) echo $FIXTURE_CID # bafybeig3yoibxe56aolixqa4zk55gp5sug3qgaztkakpndzk2b2ynobd4i ipfs dag export $FIXTURE_CID > TestGatewayHAMTDirectory.car ``` TestGatewayMultiRange.car generated with: ```sh ipfs version # ipfs version 0.19.0 export FILE_CID=bafybeiae5abzv6j3ucqbzlpnx3pcqbr2otbnpot7d2k5pckmpymin4guau export IPFS_PATH=$(mktemp -d) # Init and start daemon, ensure we have an empty repository. ipfs init --empty-repo ipfs daemon &> /dev/null & export IPFS_PID=$! # Get a specific byte range from the file. curl http://127.0.0.1:8080/ipfs/$FILE_CID -i -H "Range: bytes=1276-1279, 29839070-29839080" kill $IPFS_PID # Get the list with all the downloaded refs and sanity check. ipfs refs local > required_refs cat required_refs | wc -l # 19 # Make and export our fixture. ipfs files mkdir --cid-version 1 /fixtures cat required_refs | xargs -I {} ipfs files cp /ipfs/{} /fixtures/{} export FIXTURE_CID=$(ipfs files stat --hash /fixtures/) echo $FIXTURE_CID # bafybeicgsg3lwyn3yl75lw7sn4zhyj5dxtb7wfxwscpq6yzippetmr2w3y ipfs dag export $FIXTURE_CID > TestGatewayMultiRange.car ``` ================================================ FILE: test/cli/fixtures/TestDagStatExpectedOutput.txt ================================================ CID Blocks Size bafyreibmdfd7c5db4kls4ty57zljfhqv36gi43l6txl44pi423wwmeskwy 2 53 bafyreie3njilzdi4ixumru4nzgecsnjtu7fzfcwhg7e6s4s5i7cnbslvn4 2 53 Summary Total Size: 99 (99 B) Unique Blocks: 3 Shared Size: 7 (7 B) Ratio: 1.070707 ================================================ FILE: test/cli/fuse_test.go ================================================ package cli import ( "os" "os/exec" "path/filepath" "runtime" "strings" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils" "github.com/stretchr/testify/require" ) func TestFUSE(t *testing.T) { testutils.RequiresFUSE(t) t.Parallel() t.Run("mount and unmount work correctly", func(t *testing.T) { t.Parallel() // Create a node and start daemon node := harness.NewT(t).NewNode().Init() node.StartDaemon() // Create mount directories in the node's working directory nodeDir := node.Dir ipfsMount := filepath.Join(nodeDir, "ipfs") ipnsMount := filepath.Join(nodeDir, "ipns") mfsMount := filepath.Join(nodeDir, "mfs") err := os.MkdirAll(ipfsMount, 0755) require.NoError(t, err) err = os.MkdirAll(ipnsMount, 0755) require.NoError(t, err) err = os.MkdirAll(mfsMount, 0755) require.NoError(t, err) // Ensure any existing mounts are cleaned up first failOnError := false // mount points might not exist from previous runs doUnmount(t, ipfsMount, failOnError) doUnmount(t, ipnsMount, failOnError) doUnmount(t, mfsMount, failOnError) // Test mount operation result := node.IPFS("mount", "-f", ipfsMount, "-n", ipnsMount, "-m", mfsMount) // Verify mount output expectedOutput := "IPFS mounted at: " + ipfsMount + "\n" + "IPNS mounted at: " + ipnsMount + "\n" + "MFS mounted at: " + mfsMount + "\n" require.Equal(t, expectedOutput, result.Stdout.String()) // Test basic MFS functionality via FUSE mount testFile := filepath.Join(mfsMount, "testfile") testContent := "hello fuse world" // Create file via FUSE mount err = os.WriteFile(testFile, []byte(testContent), 0644) require.NoError(t, err) // Verify file appears in MFS via IPFS commands result = node.IPFS("files", "ls", "/") require.Contains(t, result.Stdout.String(), "testfile") // Read content back via MFS FUSE mount readContent, err := os.ReadFile(testFile) require.NoError(t, err) require.Equal(t, testContent, string(readContent)) // Get the CID of the MFS file result = node.IPFS("files", "stat", "/testfile", "--format=") fileCID := strings.TrimSpace(result.Stdout.String()) require.NotEmpty(t, fileCID, "should have a CID for the MFS file") // Read the same content via IPFS FUSE mount using the CID ipfsFile := filepath.Join(ipfsMount, fileCID) ipfsContent, err := os.ReadFile(ipfsFile) require.NoError(t, err) require.Equal(t, testContent, string(ipfsContent), "content should match between MFS and IPFS mounts") // Verify both FUSE mounts return identical data require.Equal(t, readContent, ipfsContent, "MFS and IPFS FUSE mounts should return identical data") // Test that mount directories cannot be removed while mounted err = os.Remove(ipfsMount) require.Error(t, err, "should not be able to remove mounted directory") // Stop daemon - this should trigger automatic unmount via context cancellation node.StopDaemon() // Daemon shutdown should handle unmount synchronously via context.AfterFunc // Verify directories can now be removed (indicating successful unmount) err = os.Remove(ipfsMount) require.NoError(t, err, "should be able to remove directory after unmount") err = os.Remove(ipnsMount) require.NoError(t, err, "should be able to remove directory after unmount") err = os.Remove(mfsMount) require.NoError(t, err, "should be able to remove directory after unmount") }) t.Run("explicit unmount works", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.StartDaemon() // Create mount directories nodeDir := node.Dir ipfsMount := filepath.Join(nodeDir, "ipfs") ipnsMount := filepath.Join(nodeDir, "ipns") mfsMount := filepath.Join(nodeDir, "mfs") err := os.MkdirAll(ipfsMount, 0755) require.NoError(t, err) err = os.MkdirAll(ipnsMount, 0755) require.NoError(t, err) err = os.MkdirAll(mfsMount, 0755) require.NoError(t, err) // Clean up any existing mounts failOnError := false // mount points might not exist from previous runs doUnmount(t, ipfsMount, failOnError) doUnmount(t, ipnsMount, failOnError) doUnmount(t, mfsMount, failOnError) // Mount node.IPFS("mount", "-f", ipfsMount, "-n", ipnsMount, "-m", mfsMount) // Explicit unmount via platform-specific command failOnError = true // test that explicit unmount works correctly doUnmount(t, ipfsMount, failOnError) doUnmount(t, ipnsMount, failOnError) doUnmount(t, mfsMount, failOnError) // Verify directories can be removed after explicit unmount err = os.Remove(ipfsMount) require.NoError(t, err) err = os.Remove(ipnsMount) require.NoError(t, err) err = os.Remove(mfsMount) require.NoError(t, err) node.StopDaemon() }) } // doUnmount performs platform-specific unmount, similar to sharness do_umount // failOnError: if true, unmount errors cause test failure; if false, errors are ignored (useful for cleanup) func doUnmount(t *testing.T, mountPoint string, failOnError bool) { t.Helper() var cmd *exec.Cmd if runtime.GOOS == "linux" { // fusermount -u: unmount filesystem (strict - fails if busy) cmd = exec.Command("fusermount", "-u", mountPoint) } else { cmd = exec.Command("umount", mountPoint) } err := cmd.Run() if err != nil && failOnError { t.Fatalf("failed to unmount %s: %v", mountPoint, err) } } ================================================ FILE: test/cli/gateway_limits_test.go ================================================ package cli import ( "net/http" "testing" "time" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" ) // TestGatewayLimits tests the gateway request limiting and timeout features. // These are basic integration tests that verify the configuration works. // For comprehensive tests, see: // - github.com/ipfs/boxo/gateway/middleware_retrieval_timeout_test.go // - github.com/ipfs/boxo/gateway/middleware_ratelimit_test.go func TestGatewayLimits(t *testing.T) { t.Parallel() t.Run("RetrievalTimeout", func(t *testing.T) { t.Parallel() // Create a node with a short retrieval timeout node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { // Set a 1 second timeout for retrieval cfg.Gateway.RetrievalTimeout = config.NewOptionalDuration(1 * time.Second) }) node.StartDaemon() defer node.StopDaemon() // Add content that can be retrieved quickly cid := node.IPFSAddStr("test content") client := node.GatewayClient() // Normal request should succeed (content is local) resp := client.Get("/ipfs/" + cid) assert.Equal(t, http.StatusOK, resp.StatusCode) assert.Equal(t, "test content", resp.Body) // Request for non-existent content should timeout // Using a CID that has no providers (generated with ipfs add -n) nonExistentCID := "bafkreif6lrhgz3fpiwypdk65qrqiey7svgpggruhbylrgv32l3izkqpsc4" // Create a client with longer timeout than the gateway's retrieval timeout // to ensure we get the gateway's 504 response clientWithTimeout := &harness.HTTPClient{ Client: &http.Client{ Timeout: 5 * time.Second, }, BaseURL: client.BaseURL, } resp = clientWithTimeout.Get("/ipfs/" + nonExistentCID) assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode, "Expected 504 Gateway Timeout for stuck retrieval") assert.Contains(t, resp.Body, "Unable to retrieve content within timeout period") }) t.Run("MaxRequestDuration", func(t *testing.T) { t.Parallel() // Create a node with a short max request duration node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { // Set a short absolute deadline (500ms) for the entire request cfg.Gateway.MaxRequestDuration = config.NewOptionalDuration(500 * time.Millisecond) // Set retrieval timeout much longer so MaxRequestDuration fires first cfg.Gateway.RetrievalTimeout = config.NewOptionalDuration(30 * time.Second) }) node.StartDaemon() defer node.StopDaemon() // Add content that can be retrieved quickly cid := node.IPFSAddStr("test content for max request duration") client := node.GatewayClient() // Fast request for local content should succeed (well within 500ms) resp := client.Get("/ipfs/" + cid) assert.Equal(t, http.StatusOK, resp.StatusCode) assert.Equal(t, "test content for max request duration", resp.Body) // Request for non-existent content should timeout due to MaxRequestDuration // This CID has no providers and will block during content routing nonExistentCID := "bafkreif6lrhgz3fpiwypdk65qrqiey7svgpggruhbylrgv32l3izkqpsc4" // Create a client with a longer timeout than MaxRequestDuration // to ensure we receive the gateway's 504 response clientWithTimeout := &harness.HTTPClient{ Client: &http.Client{ Timeout: 5 * time.Second, }, BaseURL: client.BaseURL, } resp = clientWithTimeout.Get("/ipfs/" + nonExistentCID) assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode, "Expected 504 when request exceeds MaxRequestDuration") }) t.Run("MaxConcurrentRequests", func(t *testing.T) { t.Parallel() // Create a node with a low concurrent request limit node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { // Allow only 1 concurrent request to make test deterministic cfg.Gateway.MaxConcurrentRequests = config.NewOptionalInteger(1) // Set retrieval timeout so blocking requests don't hang forever cfg.Gateway.RetrievalTimeout = config.NewOptionalDuration(2 * time.Second) }) node.StartDaemon() defer node.StopDaemon() // Add some content - use a non-existent CID that will block during retrieval // to ensure we can control timing blockingCID := "bafkreif6lrhgz3fpiwypdk65qrqiey7svgpggruhbylrgv32l3izkqpsc4" normalCID := node.IPFSAddStr("test content for concurrent request limiting") client := node.GatewayClient() // First, verify single request succeeds resp := client.Get("/ipfs/" + normalCID) assert.Equal(t, http.StatusOK, resp.StatusCode) // Now test deterministic 429 response: // Start a blocking request that will occupy the single slot, // then make another request that MUST get 429 blockingStarted := make(chan bool) blockingDone := make(chan bool) // Start a request that will block (searching for non-existent content) go func() { blockingStarted <- true // This will block until timeout looking for providers client.Get("/ipfs/" + blockingCID) blockingDone <- true }() // Wait for blocking request to start and occupy the slot <-blockingStarted time.Sleep(1 * time.Second) // Ensure it has acquired the semaphore // This request MUST get 429 because the slot is occupied resp = client.Get("/ipfs/" + normalCID + "?must-get-429=true") assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode, "Second request must get 429 when slot is occupied") // Verify 429 response headers retryAfter := resp.Headers.Get("Retry-After") assert.NotEmpty(t, retryAfter, "Retry-After header must be set on 429 response") assert.Equal(t, "60", retryAfter, "Retry-After must be 60 seconds") cacheControl := resp.Headers.Get("Cache-Control") assert.Equal(t, "no-store", cacheControl, "Cache-Control must be no-store on 429 response") assert.Contains(t, resp.Body, "Too many requests", "429 response must contain error message") // Clean up: wait for blocking request to timeout (it will timeout due to gateway retrieval timeout) select { case <-blockingDone: // Good, it completed case <-time.After(10 * time.Second): // Give it more time if needed } // Wait a bit more to ensure slot is fully released time.Sleep(1 * time.Second) // After blocking request completes, new request should succeed resp = client.Get("/ipfs/" + normalCID + "?after-limit-cleared=true") assert.Equal(t, http.StatusOK, resp.StatusCode, "Request must succeed after slot is freed") }) } ================================================ FILE: test/cli/gateway_range_test.go ================================================ package cli import ( "fmt" "net/http" "os" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" ) func TestGatewayHAMTDirectory(t *testing.T) { t.Parallel() const ( // The CID of the HAMT-sharded directory that has 10k items hamtCid = "bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm" // fixtureCid is the CID of root of the DAG that is a subset of hamtCid DAG // representing the minimal set of blocks necessary for directory listing. // It also includes a "files_refs" file with the list of the references // we do NOT needs to fetch (files inside the directory) fixtureCid = "bafybeig3yoibxe56aolixqa4zk55gp5sug3qgaztkakpndzk2b2ynobd4i" ) // Start node h := harness.NewT(t) node := h.NewNode().Init("--empty-repo", "--profile=test").StartDaemon("--offline") defer node.StopDaemon() client := node.GatewayClient() // Import fixtures r, err := os.Open("./fixtures/TestGatewayHAMTDirectory.car") assert.NoError(t, err) defer r.Close() err = node.IPFSDagImport(r, fixtureCid) assert.NoError(t, err) // Fetch HAMT directory succeeds with minimal refs resp := client.Get(fmt.Sprintf("/ipfs/%s/", hamtCid)) assert.Equal(t, http.StatusOK, resp.StatusCode) } func TestGatewayHAMTRanges(t *testing.T) { t.Parallel() const ( // fileCid is the CID of the large HAMT-sharded file. fileCid = "bafybeiae5abzv6j3ucqbzlpnx3pcqbr2otbnpot7d2k5pckmpymin4guau" // fixtureCid is the CID of root of the DAG that is a subset of fileCid DAG // representing the minimal set of blocks necessary for a simple byte range request. fixtureCid = "bafybeicgsg3lwyn3yl75lw7sn4zhyj5dxtb7wfxwscpq6yzippetmr2w3y" ) // Start node h := harness.NewT(t) node := h.NewNode().Init("--empty-repo", "--profile=test").StartDaemon("--offline") t.Cleanup(func() { node.StopDaemon() }) client := node.GatewayClient() // Import fixtures r, err := os.Open("./fixtures/TestGatewayMultiRange.car") assert.NoError(t, err) defer r.Close() err = node.IPFSDagImport(r, fixtureCid) assert.NoError(t, err) t.Run("Succeeds Fetching Range", func(t *testing.T) { t.Parallel() resp := client.Get(fmt.Sprintf("/ipfs/%s", fileCid), func(r *http.Request) { r.Header.Set("Range", "bytes=1276-1279") }) assert.Equal(t, http.StatusPartialContent, resp.StatusCode) assert.Equal(t, "bytes 1276-1279/109266405", resp.Headers.Get("Content-Range")) assert.Equal(t, "iana", resp.Body) }) t.Run("Succeeds Fetching Second Range", func(t *testing.T) { t.Parallel() resp := client.Get(fmt.Sprintf("/ipfs/%s", fileCid), func(r *http.Request) { r.Header.Set("Range", "bytes=29839070-29839080") }) assert.Equal(t, http.StatusPartialContent, resp.StatusCode) assert.Equal(t, "bytes 29839070-29839080/109266405", resp.Headers.Get("Content-Range")) assert.Equal(t, "EXAMPLE.COM", resp.Body) }) t.Run("Succeeds Fetching First Range of Multi-range Request", func(t *testing.T) { t.Parallel() resp := client.Get(fmt.Sprintf("/ipfs/%s", fileCid), func(r *http.Request) { r.Header.Set("Range", "bytes=1276-1279, 29839070-29839080") }) assert.Equal(t, http.StatusPartialContent, resp.StatusCode) assert.Equal(t, "bytes 1276-1279/109266405", resp.Headers.Get("Content-Range")) assert.Equal(t, "iana", resp.Body) }) } ================================================ FILE: test/cli/gateway_test.go ================================================ package cli import ( "bufio" "context" "encoding/json" "fmt" "net/http" "os" "path/filepath" "regexp" "strconv" "strings" "testing" "time" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" "github.com/multiformats/go-multibase" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestGateway(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init().StartDaemon("--offline") t.Cleanup(func() { node.StopDaemon() }) cid := node.IPFSAddStr("Hello Worlds!") peerID, err := peer.ToCid(node.PeerID()).StringOfBase(multibase.Base36) assert.NoError(t, err) client := node.GatewayClient() client.TemplateData = map[string]string{ "CID": cid, "PeerID": peerID, } t.Run("GET IPFS path succeeds", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/{{.CID}}") assert.Equal(t, 200, resp.StatusCode) }) t.Run("GET IPFS path with explicit ?filename succeeds with proper header", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/{{.CID}}?filename=testтест.pdf") assert.Equal(t, 200, resp.StatusCode) assert.Equal(t, `inline; filename="test____.pdf"; filename*=UTF-8''test%D1%82%D0%B5%D1%81%D1%82.pdf`, resp.Headers.Get("Content-Disposition"), ) }) t.Run("GET IPFS path with explicit ?filename and &download=true succeeds with proper header", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/{{.CID}}?filename=testтест.mp4&download=true") assert.Equal(t, 200, resp.StatusCode) assert.Equal(t, `attachment; filename="test____.mp4"; filename*=UTF-8''test%D1%82%D0%B5%D1%81%D1%82.mp4`, resp.Headers.Get("Content-Disposition"), ) }) // https://github.com/ipfs/go-ipfs/issues/4025#issuecomment-342250616 t.Run("GET for Server Worker registration outside of an IPFS content root errors", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/{{.CID}}?filename=sw.js", client.WithHeader("Service-Worker", "script")) assert.Equal(t, 400, resp.StatusCode) assert.Contains(t, resp.Body, "navigator.serviceWorker: registration is not allowed for this scope") }) t.Run("GET IPFS directory path succeeds", func(t *testing.T) { t.Parallel() client := node.GatewayClient().DisableRedirects() pageContents := "hello i am a webpage" fileContents := "12345" h.WriteFile("dir/test", fileContents) h.WriteFile("dir/dirwithindex/index.html", pageContents) cids := node.IPFS("add", "-r", "-q", filepath.Join(h.Dir, "dir")).Stdout.Lines() rootCID := cids[len(cids)-1] client.TemplateData = map[string]string{ "IndexFileCID": cids[0], "TestFileCID": cids[1], "RootCID": rootCID, } t.Run("GET IPFS the index file CID", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/{{.IndexFileCID}}") assert.Equal(t, 200, resp.StatusCode) assert.Equal(t, pageContents, resp.Body) }) t.Run("GET IPFS the test file CID", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/{{.TestFileCID}}") assert.Equal(t, 200, resp.StatusCode) assert.Equal(t, fileContents, resp.Body) }) t.Run("GET IPFS directory with index.html returns redirect to add trailing slash", func(t *testing.T) { t.Parallel() resp := client.Head("/ipfs/{{.RootCID}}/dirwithindex?query=to-remember") assert.Equal(t, 301, resp.StatusCode) assert.Equal(t, fmt.Sprintf("/ipfs/%s/dirwithindex/?query=to-remember", rootCID), resp.Headers.Get("Location"), ) }) // This enables go get to parse go-import meta tags from index.html files stored in IPFS // https://github.com/ipfs/kubo/pull/3963 t.Run("GET IPFS directory with index.html and no trailing slash returns expected output when go-get is passed", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/{{.RootCID}}/dirwithindex?go-get=1") assert.Equal(t, pageContents, resp.Body) }) t.Run("GET IPFS directory with index.html and trailing slash returns expected output", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/{{.RootCID}}/dirwithindex/?query=to-remember") assert.Equal(t, pageContents, resp.Body) }) t.Run("GET IPFS nonexistent file returns 404 (Not Found)", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/{{.RootCID}}/pleaseDontAddMe") assert.Equal(t, 404, resp.StatusCode) }) t.Run("GET IPFS invalid CID returns 400 (Bad Request)", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/QmInvalid/pleaseDontAddMe") assert.Equal(t, 400, resp.StatusCode) }) t.Run("GET IPFS inlined zero-length data object returns ok code (200)", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/bafkqaaa") assert.Equal(t, 200, resp.StatusCode) assert.Equal(t, "0", resp.Resp.Header.Get("Content-Length")) assert.Equal(t, "", resp.Body) }) t.Run("GET IPFS inlined zero-length data object with byte range returns ok code (200)", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/bafkqaaa", client.WithHeader("Range", "bytes=0-1048575")) assert.Equal(t, 200, resp.StatusCode) assert.Equal(t, "0", resp.Resp.Header.Get("Content-Length")) assert.Equal(t, "text/plain", resp.Resp.Header.Get("Content-Type")) }) t.Run("GET /ipfs/ipfs/{cid} returns redirect to the valid path", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/ipfs/bafkqaaa?query=to-remember") assert.Equal(t, 301, resp.StatusCode) assert.Equal(t, "/ipfs/bafkqaaa?query=to-remember", resp.Resp.Header.Get("Location")) }) }) t.Run("IPNS", func(t *testing.T) { t.Parallel() node.IPFS("name", "publish", "--allow-offline", "--ttl", "42h", cid) t.Run("GET invalid IPNS root returns 500 (Internal Server Error)", func(t *testing.T) { t.Parallel() resp := client.Get("/ipns/QmInvalid/pleaseDontAddMe") assert.Equal(t, 500, resp.StatusCode) }) t.Run("GET IPNS path succeeds", func(t *testing.T) { t.Parallel() resp := client.Get("/ipns/{{.PeerID}}") assert.Equal(t, 200, resp.StatusCode) assert.Equal(t, "Hello Worlds!", resp.Body) }) t.Run("GET IPNS path has correct Cache-Control", func(t *testing.T) { t.Parallel() resp := client.Get("/ipns/{{.PeerID}}") assert.Equal(t, 200, resp.StatusCode) cacheControl := resp.Headers.Get("Cache-Control") assert.True(t, strings.HasPrefix(cacheControl, "public, max-age=")) maxAge, err := strconv.Atoi(strings.TrimPrefix(cacheControl, "public, max-age=")) assert.NoError(t, err) assert.True(t, maxAge-151200 < 60) // MaxAge within 42h and 42h-1m }) t.Run("GET /ipfs/ipns/{peerid} returns redirect to the valid path", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/ipns/{{.PeerID}}?query=to-remember") assert.Equal(t, 301, resp.StatusCode) assert.Equal(t, fmt.Sprintf("/ipns/%s?query=to-remember", peerID), resp.Resp.Header.Get("Location")) }) }) t.Run("GET invalid IPFS path errors", func(t *testing.T) { t.Parallel() assert.Equal(t, 400, client.Get("/ipfs/12345").StatusCode) }) t.Run("GET invalid path errors", func(t *testing.T) { t.Parallel() assert.Equal(t, 404, client.Get("/12345").StatusCode) }) // TODO: these tests that use the API URL shouldn't be part of gateway tests... t.Run("GET /webui returns 301 or 302", func(t *testing.T) { t.Parallel() resp := node.APIClient().DisableRedirects().Get("/webui") assert.Contains(t, []int{302, 301, 307, 308}, resp.StatusCode) }) t.Run("GET /webui/ returns 301 or 302", func(t *testing.T) { t.Parallel() resp := node.APIClient().DisableRedirects().Get("/webui/") assert.Contains(t, []int{302, 301, 307, 308}, resp.StatusCode) }) t.Run("GET /webui/ returns user-specified headers", func(t *testing.T) { t.Parallel() header := "Access-Control-Allow-Origin" values := []string{"http://localhost:3000", "https://webui.ipfs.io"} node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.API.HTTPHeaders = map[string][]string{header: values} }) node.StartDaemon() defer node.StopDaemon() resp := node.APIClient().DisableRedirects().Get("/webui/") assert.Equal(t, resp.Headers.Values(header), values) assert.Contains(t, []int{302, 301}, resp.StatusCode) }) t.Run("POST /api/v0/version succeeds", func(t *testing.T) { t.Parallel() resp := node.APIClient().Post("/api/v0/version", nil) assert.Equal(t, 200, resp.StatusCode) assert.Len(t, resp.Resp.TransferEncoding, 1) assert.Equal(t, "chunked", resp.Resp.TransferEncoding[0]) vers := struct{ Version string }{} err := json.Unmarshal([]byte(resp.Body), &vers) require.NoError(t, err) assert.NotEmpty(t, vers.Version) }) t.Run("pprof", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() t.Cleanup(func() { node.StopDaemon() }) apiClient := node.APIClient() t.Run("mutex", func(t *testing.T) { t.Parallel() t.Run("setting the mutex fraction works (negative so it doesn't enable)", func(t *testing.T) { t.Parallel() resp := apiClient.Post("/debug/pprof-mutex/?fraction=-1", nil) assert.Equal(t, 200, resp.StatusCode) }) t.Run("mutex endpoint doesn't accept a string as an argument", func(t *testing.T) { t.Parallel() resp := apiClient.Post("/debug/pprof-mutex/?fraction=that_is_a_string", nil) assert.Equal(t, 400, resp.StatusCode) }) t.Run("mutex endpoint returns 405 on GET", func(t *testing.T) { t.Parallel() resp := apiClient.Get("/debug/pprof-mutex/?fraction=-1") assert.Equal(t, 405, resp.StatusCode) }) }) t.Run("block", func(t *testing.T) { t.Parallel() t.Run("setting the block profiler rate works (0 so it doesn't enable)", func(t *testing.T) { t.Parallel() resp := apiClient.Post("/debug/pprof-block/?rate=0", nil) assert.Equal(t, 200, resp.StatusCode) }) t.Run("block profiler endpoint doesn't accept a string as an argument", func(t *testing.T) { t.Parallel() resp := apiClient.Post("/debug/pprof-block/?rate=that_is_a_string", nil) assert.Equal(t, 400, resp.StatusCode) }) t.Run("block profiler endpoint returns 405 on GET", func(t *testing.T) { t.Parallel() resp := apiClient.Get("/debug/pprof-block/?rate=0") assert.Equal(t, 405, resp.StatusCode) }) }) }) t.Run("index content types", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init().StartDaemon() t.Cleanup(func() { node.StopDaemon() }) h.WriteFile("index/index.html", "

") cid := node.IPFS("add", "-Q", "-r", filepath.Join(h.Dir, "index")).Stderr.Trimmed() apiClient := node.APIClient() apiClient.TemplateData = map[string]string{"CID": cid} t.Run("GET index.html has correct content type", func(t *testing.T) { t.Parallel() res := apiClient.Get("/ipfs/{{.CID}}/") assert.Equal(t, "text/html; charset=utf-8", res.Resp.Header.Get("Content-Type")) }) t.Run("HEAD index.html has no content", func(t *testing.T) { t.Parallel() res := apiClient.Head("/ipfs/{{.CID}}/") assert.Equal(t, "", res.Body) assert.Equal(t, "", res.Resp.Header.Get("Content-Length")) }) }) t.Run("raw leaves node", func(t *testing.T) { t.Parallel() contents := "This is RAW!" cid := node.IPFSAddStr(contents, "--raw-leaves") assert.Equal(t, contents, client.Get("/ipfs/"+cid).Body) }) t.Run("compact blocks", func(t *testing.T) { t.Parallel() block1 := "\x0a\x09\x08\x02\x12\x03\x66\x6f\x6f\x18\x03" block2 := "\x0a\x04\x08\x02\x18\x06\x12\x24\x0a\x22\x12\x20\xcf\x92\xfd\xef\xcd\xc3\x4c\xac\x00\x9c" + "\x8b\x05\xeb\x66\x2b\xe0\x61\x8d\xb9\xde\x55\xec\xd4\x27\x85\xe9\xec\x67\x12\xf8\xdf\x65" + "\x12\x24\x0a\x22\x12\x20\xcf\x92\xfd\xef\xcd\xc3\x4c\xac\x00\x9c\x8b\x05\xeb\x66\x2b\xe0" + "\x61\x8d\xb9\xde\x55\xec\xd4\x27\x85\xe9\xec\x67\x12\xf8\xdf\x65" node.PipeStrToIPFS(block1, "block", "put") block2CID := node.PipeStrToIPFS(block2, "block", "put", "--cid-codec=dag-pb").Stdout.Trimmed() resp := client.Get("/ipfs/" + block2CID) assert.Equal(t, 200, resp.StatusCode) assert.Equal(t, "foofoo", resp.Body) }) t.Run("verify gateway file", func(t *testing.T) { t.Parallel() r := regexp.MustCompile(`Gateway server listening on (?P.+)\s`) matches := r.FindStringSubmatch(node.Daemon.Stdout.String()) ma, err := multiaddr.NewMultiaddr(matches[1]) require.NoError(t, err) netAddr, err := manet.ToNetAddr(ma) require.NoError(t, err) expURL := "http://" + netAddr.String() b, err := os.ReadFile(filepath.Join(node.Dir, "gateway")) require.NoError(t, err) assert.Equal(t, expURL, string(b)) }) t.Run("verify gateway file diallable while on unspecified", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Addresses.Gateway = config.Strings{"/ip4/127.0.0.1/tcp/32563"} }) node.StartDaemon() defer node.StopDaemon() b, err := os.ReadFile(filepath.Join(node.Dir, "gateway")) require.NoError(t, err) assert.Equal(t, "http://127.0.0.1:32563", string(b)) }) t.Run("NoFetch", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init() node1 := nodes[0] node2 := nodes[1] node1.UpdateConfig(func(cfg *config.Config) { cfg.Gateway.NoFetch = true }) node2PeerID, err := peer.ToCid(node2.PeerID()).StringOfBase(multibase.Base36) assert.NoError(t, err) nodes.StartDaemons().Connect() t.Cleanup(func() { nodes.StopDaemons() }) t.Run("not present", func(t *testing.T) { cidFoo := node2.IPFSAddStr("foo") t.Run("not present CID from node 1", func(t *testing.T) { t.Parallel() assert.Equal(t, 404, node1.GatewayClient().Get("/ipfs/"+cidFoo).StatusCode) }) t.Run("not present IPNS Record from node 1", func(t *testing.T) { t.Parallel() assert.Equal(t, 500, node1.GatewayClient().Get("/ipns/"+node2PeerID).StatusCode) }) }) t.Run("present", func(t *testing.T) { cidBar := node1.IPFSAddStr("bar") t.Run("present CID from node 1", func(t *testing.T) { t.Parallel() assert.Equal(t, 200, node1.GatewayClient().Get("/ipfs/"+cidBar).StatusCode) }) t.Run("present IPNS Record from node 1", func(t *testing.T) { t.Parallel() node2.IPFS("name", "publish", "/ipfs/"+cidBar) assert.Equal(t, 200, node1.GatewayClient().Get("/ipns/"+node2PeerID).StatusCode) }) }) }) t.Run("DeserializedResponses", func(t *testing.T) { type testCase struct { globalValue config.Flag gatewayValue config.Flag deserializedGlobalStatusCode int deserializedGatewayStaticCode int message string } setHost := func(r *http.Request) { r.Host = "example.com" } withAccept := func(accept string) func(r *http.Request) { return func(r *http.Request) { r.Header.Set("Accept", accept) } } withHostAndAccept := func(accept string) func(r *http.Request) { return func(r *http.Request) { setHost(r) withAccept(accept)(r) } } makeTest := func(test *testCase) func(t *testing.T) { return func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Gateway.DeserializedResponses = test.globalValue cfg.Gateway.PublicGateways = map[string]*config.GatewaySpec{ "example.com": { Paths: []string{"/ipfs", "/ipns"}, DeserializedResponses: test.gatewayValue, }, } }) node.StartDaemon() defer node.StopDaemon() cidFoo := node.IPFSAddStr("foo") client := node.GatewayClient() deserializedPath := "/ipfs/" + cidFoo blockPath := deserializedPath + "?format=raw" carPath := deserializedPath + "?format=car" // Global Check (Gateway.DeserializedResponses) assert.Equal(t, http.StatusOK, client.Get(blockPath).StatusCode) assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withAccept("application/vnd.ipld.raw")).StatusCode) assert.Equal(t, http.StatusOK, client.Get(carPath).StatusCode) assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withAccept("application/vnd.ipld.car")).StatusCode) assert.Equal(t, test.deserializedGlobalStatusCode, client.Get(deserializedPath).StatusCode) assert.Equal(t, test.deserializedGlobalStatusCode, client.Get(deserializedPath, withAccept("application/json")).StatusCode) // Public Gateway (example.com) Check (Gateway.PublicGateways[example.com].DeserializedResponses) assert.Equal(t, http.StatusOK, client.Get(blockPath, setHost).StatusCode) assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withHostAndAccept("application/vnd.ipld.raw")).StatusCode) assert.Equal(t, http.StatusOK, client.Get(carPath, setHost).StatusCode) assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withHostAndAccept("application/vnd.ipld.car")).StatusCode) assert.Equal(t, test.deserializedGatewayStaticCode, client.Get(deserializedPath, setHost).StatusCode) assert.Equal(t, test.deserializedGatewayStaticCode, client.Get(deserializedPath, withHostAndAccept("application/json")).StatusCode) } } for _, test := range []*testCase{ {config.True, config.Default, http.StatusOK, http.StatusOK, "when Gateway.DeserializedResponses is globally enabled, leaving implicit default for Gateway.PublicGateways[example.com] should inherit the global setting (enabled)"}, {config.False, config.Default, http.StatusNotAcceptable, http.StatusNotAcceptable, "when Gateway.DeserializedResponses is globally disabled, leaving implicit default on Gateway.PublicGateways[example.com] should inherit the global setting (disabled)"}, {config.False, config.True, http.StatusNotAcceptable, http.StatusOK, "when Gateway.DeserializedResponses is globally disabled, explicitly enabling on Gateway.PublicGateways[example.com] should override global (enabled)"}, {config.True, config.False, http.StatusOK, http.StatusNotAcceptable, "when Gateway.DeserializedResponses is globally enabled, explicitly disabling on Gateway.PublicGateways[example.com] should override global (disabled)"}, } { t.Run(test.message, makeTest(test)) } }) t.Run("DisableHTMLErrors", func(t *testing.T) { t.Parallel() t.Run("Returns HTML error without DisableHTMLErrors, Accept contains text/html", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.StartDaemon() defer node.StopDaemon() client := node.GatewayClient() res := client.Get("/ipfs/invalid-thing", func(r *http.Request) { r.Header.Set("Accept", "text/html") }) assert.NotEqual(t, http.StatusOK, res.StatusCode) assert.Contains(t, res.Resp.Header.Get("Content-Type"), "text/html") }) t.Run("Does not return HTML error with DisableHTMLErrors enabled, and Accept contains text/html", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Gateway.DisableHTMLErrors = config.True }) node.StartDaemon() defer node.StopDaemon() client := node.GatewayClient() res := client.Get("/ipfs/invalid-thing", func(r *http.Request) { r.Header.Set("Accept", "text/html") }) assert.NotEqual(t, http.StatusOK, res.StatusCode) assert.NotContains(t, res.Resp.Header.Get("Content-Type"), "text/html") }) }) } // TestLogs tests that GET /logs returns log messages. This test is separate // because it requires setting the server's log level to "info" which may // change the output expected by other tests. func TestLogs(t *testing.T) { h := harness.NewT(t) t.Setenv("GOLOG_LOG_LEVEL", "info") node := h.NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() cid := node.IPFSAddStr("Hello Worlds!") peerID, err := peer.ToCid(node.PeerID()).StringOfBase(multibase.Base36) assert.NoError(t, err) client := node.GatewayClient() client.TemplateData = map[string]string{ "CID": cid, "PeerID": peerID, } apiClient := node.APIClient() reqURL := apiClient.BuildURL("/logs") ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) require.NoError(t, err) resp, err := apiClient.Client.Do(req) require.NoError(t, err) defer resp.Body.Close() var found bool scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { if strings.Contains(scanner.Text(), "log API client connected") { found = true break } } assert.True(t, found) } ================================================ FILE: test/cli/harness/buffer.go ================================================ package harness import ( "strings" "sync" "github.com/ipfs/kubo/test/cli/testutils" ) // Buffer is a thread-safe byte buffer. type Buffer struct { b strings.Builder m sync.Mutex } func (b *Buffer) Write(p []byte) (n int, err error) { b.m.Lock() defer b.m.Unlock() return b.b.Write(p) } func (b *Buffer) String() string { b.m.Lock() defer b.m.Unlock() return b.b.String() } // Trimmed returns the bytes as a string, but with the trailing newline removed. // This only removes a single trailing newline, not all whitespace. func (b *Buffer) Trimmed() string { b.m.Lock() defer b.m.Unlock() s := b.b.String() if len(s) == 0 { return s } if s[len(s)-1] == '\n' { return s[:len(s)-1] } return s } func (b *Buffer) Bytes() []byte { b.m.Lock() defer b.m.Unlock() return []byte(b.b.String()) } func (b *Buffer) Lines() []string { b.m.Lock() defer b.m.Unlock() return testutils.SplitLines(b.b.String()) } ================================================ FILE: test/cli/harness/harness.go ================================================ package harness import ( "errors" "fmt" "os" "path/filepath" "strings" "testing" "time" logging "github.com/ipfs/go-log/v2" . "github.com/ipfs/kubo/test/cli/testutils" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" ) // Harness tracks state for a test, such as temp dirs and IFPS nodes, and cleans them up after the test. type Harness struct { Dir string IPFSBin string Runner *Runner NodesRoot string Nodes Nodes } // TODO: use zaptest.NewLogger(t) instead func EnableDebugLogging() { err := logging.SetLogLevel("testharness", "DEBUG") if err != nil { panic(err) } } // NewT constructs a harness that cleans up after the given test is done. func NewT(t *testing.T, options ...func(h *Harness)) *Harness { h := New(options...) t.Cleanup(h.Cleanup) return h } func New(options ...func(h *Harness)) *Harness { h := &Harness{Runner: &Runner{Env: osEnviron()}} // walk up to find the root dir, from which we can locate the binary wd, err := os.Getwd() if err != nil { panic(err) } goMod := FindUp("go.mod", wd) if goMod == "" { panic("unable to find root dir") } rootDir := filepath.Dir(goMod) h.IPFSBin = filepath.Join(rootDir, "cmd", "ipfs", "ipfs") // setup working dir tmpDir, err := os.MkdirTemp("", "") if err != nil { log.Panicf("error creating temp dir: %s", err) } h.Dir = tmpDir h.Runner.Dir = h.Dir h.NodesRoot = filepath.Join(h.Dir, ".nodes") // apply any customizations // this should happen after all initialization for _, o := range options { o(h) } return h } func osEnviron() map[string]string { m := map[string]string{} for _, entry := range os.Environ() { split := strings.Split(entry, "=") m[split[0]] = split[1] } return m } func (h *Harness) NewNode() *Node { nodeID := len(h.Nodes) node := BuildNode(h.IPFSBin, h.NodesRoot, nodeID) h.Nodes = append(h.Nodes, node) return node } func (h *Harness) NewNodes(count int) Nodes { var newNodes []*Node for range count { newNodes = append(newNodes, h.NewNode()) } return newNodes } // WriteToTemp writes the given contents to a guaranteed-unique temp file, returning its path. func (h *Harness) WriteToTemp(contents string) string { f := h.TempFile() _, err := f.WriteString(contents) if err != nil { log.Panicf("writing to temp file: %s", err.Error()) } err = f.Close() if err != nil { log.Panicf("closing temp file: %s", err.Error()) } return f.Name() } // TempFile creates a new unique temp file. func (h *Harness) TempFile() *os.File { f, err := os.CreateTemp(h.Dir, "") if err != nil { log.Panicf("creating temp file: %s", err.Error()) } return f } // WriteFile writes a file given a filename and its contents. // The filename must be a relative path, or this panics. func (h *Harness) WriteFile(filename, contents string) { if filepath.IsAbs(filename) { log.Panicf("%s must be a relative path", filename) } absPath := filepath.Join(h.Runner.Dir, filename) err := os.MkdirAll(filepath.Dir(absPath), 0o777) if err != nil { log.Panicf("creating intermediate dirs for %q: %s", filename, err.Error()) } err = os.WriteFile(absPath, []byte(contents), 0o644) if err != nil { log.Panicf("writing %q (%q): %s", filename, absPath, err.Error()) } } func WaitForFile(path string, timeout time.Duration) error { start := time.Now() timer := time.NewTimer(timeout) ticker := time.NewTicker(1 * time.Millisecond) defer timer.Stop() defer ticker.Stop() for { select { case <-timer.C: return fmt.Errorf("timeout waiting for %s after %v", path, time.Since(start)) case <-ticker.C: _, err := os.Stat(path) if err == nil { return nil } if errors.Is(err, os.ErrNotExist) { continue } return fmt.Errorf("error waiting for %s: %w", path, err) } } } func (h *Harness) Mkdirs(paths ...string) { for _, path := range paths { if filepath.IsAbs(path) { log.Panicf("%s must be a relative path when making dirs", path) } absPath := filepath.Join(h.Runner.Dir, path) err := os.MkdirAll(absPath, 0o777) if err != nil { log.Panicf("recursively making dirs under %s: %s", absPath, err) } } } func (h *Harness) Sh(expr string) *RunResult { return h.Runner.Run(RunRequest{ Path: "bash", Args: []string{"-c", expr}, }) } func (h *Harness) Cleanup() { log.Debugf("cleaning up cluster") h.Nodes.StopDaemons() // TODO: don't do this if test fails, not sure how? log.Debugf("removing harness dir") err := os.RemoveAll(h.Dir) if err != nil { log.Panicf("removing temp dir %s: %s", h.Dir, err) } } // ExtractPeerID extracts a peer ID from the given multiaddr, and fatals if it does not contain a peer ID. func (h *Harness) ExtractPeerID(m multiaddr.Multiaddr) peer.ID { var peerIDStr string multiaddr.ForEach(m, func(c multiaddr.Component) bool { if c.Protocol().Code == multiaddr.P_P2P { peerIDStr = c.Value() } return true }) if peerIDStr == "" { panic(multiaddr.ErrProtocolNotFound) } peerID, err := peer.Decode(peerIDStr) if err != nil { panic(err) } return peerID } ================================================ FILE: test/cli/harness/http_client.go ================================================ package harness import ( "io" "net/http" "strings" "text/template" "time" ) // HTTPClient is an HTTP client with some conveniences for testing. // URLs are constructed from a base URL. // The response body is buffered into a string. // Internal errors cause panics so that tests don't need to check errors. // The paths are evaluated as Go templates for readable string interpolation. type HTTPClient struct { Client *http.Client BaseURL string Timeout time.Duration TemplateData any } type HTTPResponse struct { Body string StatusCode int Headers http.Header // The raw response. The body will be closed on this response. Resp *http.Response } func (c *HTTPClient) WithHeader(k, v string) func(h *http.Request) { return func(h *http.Request) { h.Header.Add(k, v) } } func (c *HTTPClient) DisableRedirects() *HTTPClient { c.Client.CheckRedirect = func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse } return c } // Do executes the request unchanged. func (c *HTTPClient) Do(req *http.Request) *HTTPResponse { log.Debugf("making HTTP req %s to %q with headers %+v", req.Method, req.URL.String(), req.Header) resp, err := c.Client.Do(req) if resp != nil && resp.Body != nil { defer resp.Body.Close() } if err != nil { panic(err) } bodyStr, err := io.ReadAll(resp.Body) if err != nil { panic(err) } return &HTTPResponse{ Body: string(bodyStr), StatusCode: resp.StatusCode, Headers: resp.Header, Resp: resp, } } // BuildURL constructs a request URL from the given path by interpolating the string and then appending it to the base URL. func (c *HTTPClient) BuildURL(urlPath string) string { sb := &strings.Builder{} err := template.Must(template.New("test").Parse(urlPath)).Execute(sb, c.TemplateData) if err != nil { panic(err) } renderedPath := sb.String() return c.BaseURL + renderedPath } func (c *HTTPClient) Get(urlPath string, opts ...func(*http.Request)) *HTTPResponse { req, err := http.NewRequest(http.MethodGet, c.BuildURL(urlPath), nil) if err != nil { panic(err) } for _, o := range opts { o(req) } return c.Do(req) } func (c *HTTPClient) Post(urlPath string, body io.Reader, opts ...func(*http.Request)) *HTTPResponse { req, err := http.NewRequest(http.MethodPost, c.BuildURL(urlPath), body) if err != nil { panic(err) } for _, o := range opts { o(req) } return c.Do(req) } func (c *HTTPClient) PostStr(urlpath, body string, opts ...func(*http.Request)) *HTTPResponse { r := strings.NewReader(body) return c.Post(urlpath, r, opts...) } func (c *HTTPClient) Head(urlPath string, opts ...func(*http.Request)) *HTTPResponse { req, err := http.NewRequest(http.MethodHead, c.BuildURL(urlPath), nil) if err != nil { panic(err) } for _, o := range opts { o(req) } return c.Do(req) } ================================================ FILE: test/cli/harness/ipfs.go ================================================ package harness import ( "encoding/json" "fmt" "io" "os" "reflect" "strings" . "github.com/ipfs/kubo/test/cli/testutils" ) func (n *Node) IPFSCommands() []string { res := n.IPFS("commands").Stdout.String() res = strings.TrimSpace(res) split := SplitLines(res) var cmds []string for _, line := range split { trimmed := strings.TrimSpace(line) if trimmed == "ipfs" { continue } cmds = append(cmds, trimmed) } return cmds } func (n *Node) SetIPFSConfig(key string, val any, flags ...string) { valBytes, err := json.Marshal(val) if err != nil { log.Panicf("marshling config for key '%s': %s", key, err) } valStr := string(valBytes) args := []string{"config", "--json"} args = append(args, flags...) args = append(args, key, valStr) n.IPFS(args...) // validate the config was set correctly // Create a new value which is a pointer to the same type as the source. var newVal any if val != nil { // If it is not nil grab the type with reflect. newVal = reflect.New(reflect.TypeOf(val)).Interface() } else { // else just set a pointer to an any. var anything any newVal = &anything } n.GetIPFSConfig(key, newVal) // dereference newVal using reflect to load the resulting value if !reflect.DeepEqual(val, reflect.ValueOf(newVal).Elem().Interface()) { log.Panicf("key '%s' did not retain value '%s' after it was set, got '%s'", key, val, newVal) } } func (n *Node) GetIPFSConfig(key string, val any) { res := n.IPFS("config", key) valStr := strings.TrimSpace(res.Stdout.String()) // only when the result is a string is the result not well-formed JSON, // so check the value type and add quotes if it's expected to be a string reflectVal := reflect.ValueOf(val) if reflectVal.Kind() == reflect.Pointer && reflectVal.Elem().Kind() == reflect.String { valStr = fmt.Sprintf(`"%s"`, valStr) } err := json.Unmarshal([]byte(valStr), val) if err != nil { log.Fatalf("unmarshaling config for key '%s', value '%s': %s", key, valStr, err) } } func (n *Node) IPFSAddStr(content string, args ...string) string { log.Debugf("node %d adding content '%s' with args: %v", n.ID, PreviewStr(content), args) return n.IPFSAdd(strings.NewReader(content), args...) } // IPFSAddDeterministic produces a CID of a file of a certain size, filled with deterministically generated bytes based on some seed. // Size is specified as a humanize string (e.g., "256KiB", "1MiB"). // This ensures deterministic CID on the other end, that can be used in tests. func (n *Node) IPFSAddDeterministic(size string, seed string, args ...string) string { log.Debugf("node %d adding %s of deterministic pseudo-random data with seed %q and args: %v", n.ID, size, seed, args) reader, err := DeterministicRandomReader(size, seed) if err != nil { panic(err) } return n.IPFSAdd(reader, args...) } // IPFSAddDeterministicBytes produces a CID of a file of exactly `size` bytes, filled with deterministically generated bytes based on some seed. // Use this when exact byte precision is needed (e.g., threshold tests at T and T+1 bytes). func (n *Node) IPFSAddDeterministicBytes(size int64, seed string, args ...string) string { log.Debugf("node %d adding %d bytes of deterministic pseudo-random data with seed %q and args: %v", n.ID, size, seed, args) reader, err := DeterministicRandomReaderBytes(size, seed) if err != nil { panic(err) } return n.IPFSAdd(reader, args...) } func (n *Node) IPFSAdd(content io.Reader, args ...string) string { log.Debugf("node %d adding with args: %v", n.ID, args) fullArgs := []string{"add", "-q"} fullArgs = append(fullArgs, args...) res := n.Runner.MustRun(RunRequest{ Path: n.IPFSBin, Args: fullArgs, CmdOpts: []CmdOpt{RunWithStdin(content)}, }) out := strings.TrimSpace(res.Stdout.String()) log.Debugf("add result: %q", out) return out } func (n *Node) IPFSBlockPut(content io.Reader, args ...string) string { log.Debugf("node %d block put with args: %v", n.ID, args) fullArgs := []string{"block", "put"} fullArgs = append(fullArgs, args...) res := n.Runner.MustRun(RunRequest{ Path: n.IPFSBin, Args: fullArgs, CmdOpts: []CmdOpt{RunWithStdin(content)}, }) out := strings.TrimSpace(res.Stdout.String()) log.Debugf("block put result: %q", out) return out } func (n *Node) IPFSDAGPut(content io.Reader, args ...string) string { log.Debugf("node %d dag put with args: %v", n.ID, args) fullArgs := []string{"dag", "put"} fullArgs = append(fullArgs, args...) res := n.Runner.MustRun(RunRequest{ Path: n.IPFSBin, Args: fullArgs, CmdOpts: []CmdOpt{RunWithStdin(content)}, }) out := strings.TrimSpace(res.Stdout.String()) log.Debugf("dag put result: %q", out) return out } func (n *Node) IPFSDagImport(content io.Reader, cid string, args ...string) error { log.Debugf("node %d dag import with args: %v", n.ID, args) fullArgs := []string{"dag", "import", "--pin-roots=false"} fullArgs = append(fullArgs, args...) res := n.Runner.MustRun(RunRequest{ Path: n.IPFSBin, Args: fullArgs, CmdOpts: []CmdOpt{RunWithStdin(content)}, }) if res.Err != nil { return res.Err } res = n.Runner.MustRun(RunRequest{ Path: n.IPFSBin, Args: []string{"block", "stat", "--offline", cid}, }) return res.Err } // IPFSDagExport exports a DAG rooted at cid to a CAR file at carPath. func (n *Node) IPFSDagExport(cid string, carPath string) error { log.Debugf("node %d dag export of %s to %q", n.ID, cid, carPath) car, err := os.Create(carPath) if err != nil { return err } defer car.Close() res := n.Runner.MustRun(RunRequest{ Path: n.IPFSBin, Args: []string{"dag", "export", cid}, CmdOpts: []CmdOpt{RunWithStdout(car)}, }) return res.Err } ================================================ FILE: test/cli/harness/log.go ================================================ package harness import ( "fmt" "path/filepath" "runtime" "sort" "strings" "sync" "testing" "time" ) type event struct { timestamp time.Time msg string } type events []*event func (e events) Len() int { return len(e) } func (e events) Less(i, j int) bool { return e[i].timestamp.Before(e[j].timestamp) } func (e events) Swap(i, j int) { e[i], e[j] = e[j], e[i] } // TestLogger is a logger for tests. // It buffers output and only writes the output if the test fails or output is explicitly turned on. // The purpose of this logger is to allow Go test to run with the verbose flag without printing logs. // The verbose flag is useful since it streams test progress, but also printing logs makes the output too verbose. // // You can also add prefixes that are prepended to each log message, for extra logging context. // // This is implemented as a hierarchy of loggers, with children flushing log entries back to parents. // This works because t.Cleanup() processes entries in LIFO order, so children always flush first. // // Obviously this logger should never be used in production systems. type TestLogger struct { parent *TestLogger children []*TestLogger prefixes []string prefixesIface []any t *testing.T buf events m sync.Mutex logsEnabled bool } func NewTestLogger(t *testing.T) *TestLogger { l := &TestLogger{t: t, buf: make(events, 0)} t.Cleanup(l.flush) return l } func (t *TestLogger) buildPrefix(timestamp time.Time) string { d := timestamp.Format("2006-01-02T15:04:05.999999") _, file, lineno, _ := runtime.Caller(2) file = filepath.Base(file) caller := fmt.Sprintf("%s:%d", file, lineno) if len(t.prefixes) == 0 { return fmt.Sprintf("%s\t%s\t", d, caller) } prefixes := strings.Join(t.prefixes, ":") return fmt.Sprintf("%s\t%s\t%s: ", d, caller, prefixes) } func (t *TestLogger) Log(args ...any) { timestamp := time.Now() e := t.buildPrefix(timestamp) + fmt.Sprint(args...) t.add(&event{timestamp: timestamp, msg: e}) } func (t *TestLogger) Logf(format string, args ...any) { timestamp := time.Now() e := t.buildPrefix(timestamp) + fmt.Sprintf(format, args...) t.add(&event{timestamp: timestamp, msg: e}) } func (t *TestLogger) Fatal(args ...any) { timestamp := time.Now() e := t.buildPrefix(timestamp) + fmt.Sprint(append([]any{"fatal: "}, args...)...) t.add(&event{timestamp: timestamp, msg: e}) t.t.FailNow() } func (t *TestLogger) Fatalf(format string, args ...any) { timestamp := time.Now() e := t.buildPrefix(timestamp) + fmt.Sprintf(fmt.Sprintf("fatal: %s", format), args...) t.add(&event{timestamp: timestamp, msg: e}) t.t.FailNow() } func (t *TestLogger) add(e *event) { t.m.Lock() defer t.m.Unlock() t.buf = append(t.buf, e) } func (t *TestLogger) AddPrefix(prefix string) *TestLogger { l := &TestLogger{ prefixes: append(t.prefixes, prefix), prefixesIface: append(t.prefixesIface, prefix), t: t.t, parent: t, logsEnabled: t.logsEnabled, } t.m.Lock() defer t.m.Unlock() t.children = append(t.children, l) t.t.Cleanup(l.flush) return l } func (t *TestLogger) EnableLogs() { t.m.Lock() defer t.m.Unlock() t.logsEnabled = true if t.parent != nil { if t.parent.logsEnabled { t.parent.EnableLogs() } } fmt.Printf("enabling %d children\n", len(t.children)) for _, c := range t.children { if !c.logsEnabled { c.EnableLogs() } } } func (t *TestLogger) flush() { if t.t.Failed() || t.logsEnabled { t.m.Lock() defer t.m.Unlock() // if this is a child, send the events to the parent // the root parent will print all the events in sorted order if t.parent != nil { for _, e := range t.buf { t.parent.add(e) } } else { // we're the root, sort all the events and then print them sort.Sort(t.buf) fmt.Println() fmt.Printf("Logs for test %q:\n\n", t.t.Name()) for _, e := range t.buf { fmt.Println(e.msg) } fmt.Println() } t.buf = nil } } ================================================ FILE: test/cli/harness/node.go ================================================ package harness import ( "bytes" "encoding/json" "errors" "fmt" "io" "io/fs" "net/http" "os" "os/exec" "path/filepath" "runtime" "strconv" "strings" "syscall" "time" logging "github.com/ipfs/go-log/v2" "github.com/ipfs/kubo/config" serial "github.com/ipfs/kubo/config/serialize" "github.com/libp2p/go-libp2p/core/peer" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) var log = logging.Logger("testharness") // Node is a single Kubo node. // Each node has its own config and can run its own Kubo daemon. type Node struct { ID int Dir string APIListenAddr multiaddr.Multiaddr GatewayListenAddr multiaddr.Multiaddr SwarmAddr multiaddr.Multiaddr EnableMDNS bool IPFSBin string Runner *Runner Daemon *RunResult } func BuildNode(ipfsBin, baseDir string, id int) *Node { dir := filepath.Join(baseDir, strconv.Itoa(id)) if err := os.MkdirAll(dir, 0o755); err != nil { panic(err) } env := environToMap(os.Environ()) env["IPFS_PATH"] = dir // If using "ipfs" binary name, provide helpful binary information if ipfsBin == "ipfs" { // Check if cmd/ipfs/ipfs exists (simple relative path check) localBinary := "cmd/ipfs/ipfs" localExists := false if _, err := os.Stat(localBinary); err == nil { localExists = true if abs, err := filepath.Abs(localBinary); err == nil { localBinary = abs } } // Check if ipfs is available in PATH pathBinary, pathErr := exec.LookPath("ipfs") // Handle different scenarios if pathErr != nil { // No ipfs in PATH if localExists { fmt.Printf("WARNING: No 'ipfs' found in PATH, but local binary exists at %s\n", localBinary) fmt.Printf("Consider adding it to PATH or run: export PATH=\"$(pwd)/cmd/ipfs:$PATH\"\n") } else { fmt.Printf("ERROR: No 'ipfs' binary found in PATH and no local build at cmd/ipfs/ipfs\n") fmt.Printf("Run 'make build' first or install ipfs and add it to PATH\n") panic("ipfs binary not available") } } else { // ipfs found in PATH if localExists && localBinary != pathBinary { fmt.Printf("NOTE: Local binary at %s differs from PATH binary at %s\n", localBinary, pathBinary) fmt.Printf("Consider adding the local binary to PATH if you want to use the version built by 'make build'\n") } // If they match or no local binary, no message needed } } return &Node{ ID: id, Dir: dir, IPFSBin: ipfsBin, Runner: &Runner{ Env: env, Dir: dir, }, } } func (n *Node) WriteBytes(filename string, b []byte) { f, err := os.Create(filepath.Join(n.Dir, filename)) if err != nil { panic(err) } defer f.Close() _, err = io.Copy(f, bytes.NewReader(b)) if err != nil { panic(err) } } // ReadFile reads the specific file. If it is relative, it is relative the node's root dir. func (n *Node) ReadFile(filename string) string { f := filename if !filepath.IsAbs(filename) { f = filepath.Join(n.Dir, filename) } b, err := os.ReadFile(f) if err != nil { panic(err) } return string(b) } func (n *Node) ConfigFile() string { return filepath.Join(n.Dir, "config") } func (n *Node) ReadConfig() *config.Config { cfg, err := serial.Load(filepath.Join(n.Dir, "config")) if err != nil { panic(err) } return cfg } func (n *Node) WriteConfig(c *config.Config) { err := serial.WriteConfigFile(filepath.Join(n.Dir, "config"), c) if err != nil { panic(err) } } func (n *Node) UpdateConfig(f func(cfg *config.Config)) { cfg := n.ReadConfig() f(cfg) n.WriteConfig(cfg) } func (n *Node) ReadUserResourceOverrides() *rcmgr.PartialLimitConfig { var r rcmgr.PartialLimitConfig err := serial.ReadConfigFile(filepath.Join(n.Dir, "libp2p-resource-limit-overrides.json"), &r) switch err { case nil, serial.ErrNotInitialized: return &r default: panic(err) } } func (n *Node) WriteUserSuppliedResourceOverrides(c *rcmgr.PartialLimitConfig) { err := serial.WriteConfigFile(filepath.Join(n.Dir, "libp2p-resource-limit-overrides.json"), c) if err != nil { panic(err) } } func (n *Node) UpdateUserSuppliedResourceManagerOverrides(f func(overrides *rcmgr.PartialLimitConfig)) { overrides := n.ReadUserResourceOverrides() f(overrides) n.WriteUserSuppliedResourceOverrides(overrides) } func (n *Node) IPFS(args ...string) *RunResult { res := n.RunIPFS(args...) n.Runner.AssertNoError(res) return res } func (n *Node) PipeStrToIPFS(s string, args ...string) *RunResult { return n.PipeToIPFS(strings.NewReader(s), args...) } func (n *Node) PipeToIPFS(reader io.Reader, args ...string) *RunResult { res := n.RunPipeToIPFS(reader, args...) n.Runner.AssertNoError(res) return res } func (n *Node) RunPipeToIPFS(reader io.Reader, args ...string) *RunResult { return n.Runner.Run(RunRequest{ Path: n.IPFSBin, Args: args, CmdOpts: []CmdOpt{RunWithStdin(reader)}, }) } func (n *Node) RunIPFS(args ...string) *RunResult { return n.Runner.Run(RunRequest{ Path: n.IPFSBin, Args: args, }) } // Init initializes and configures the IPFS node, after which it is ready to run. func (n *Node) Init(ipfsArgs ...string) *Node { n.Runner.MustRun(RunRequest{ Path: n.IPFSBin, Args: append([]string{"init"}, ipfsArgs...), }) if n.SwarmAddr == nil { swarmAddr, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/0") if err != nil { panic(err) } n.SwarmAddr = swarmAddr } if n.APIListenAddr == nil { apiAddr, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/0") if err != nil { panic(err) } n.APIListenAddr = apiAddr } if n.GatewayListenAddr == nil { gatewayAddr, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/0") if err != nil { panic(err) } n.GatewayListenAddr = gatewayAddr } n.UpdateConfig(func(cfg *config.Config) { cfg.Bootstrap = []string{} cfg.Addresses.Swarm = []string{n.SwarmAddr.String()} cfg.Addresses.API = []string{n.APIListenAddr.String()} cfg.Addresses.Gateway = []string{n.GatewayListenAddr.String()} cfg.Swarm.DisableNatPortMap = true cfg.Discovery.MDNS.Enabled = n.EnableMDNS cfg.Routing.LoopbackAddressesOnLanDHT = config.True // Telemetry disabled by default in tests. cfg.Plugins = config.Plugins{ Plugins: map[string]config.Plugin{ "telemetry": config.Plugin{ Disabled: true, }, }, } }) return n } // StartDaemonWithReq runs a Kubo daemon with the given request. // This overwrites the request Path with the Kubo bin path. // // For example, if you want to run the daemon and see stderr and stdout to debug: // // node.StartDaemonWithReq(harness.RunRequest{ // CmdOpts: []harness.CmdOpt{ // harness.RunWithStderr(os.Stdout), // harness.RunWithStdout(os.Stdout), // }, // }) func (n *Node) StartDaemonWithReq(req RunRequest, authorization string) *Node { alive := n.IsAlive() if alive { log.Panicf("node %d is already running", n.ID) } newReq := req newReq.Path = n.IPFSBin newReq.Args = append([]string{"daemon"}, req.Args...) newReq.RunFunc = (*exec.Cmd).Start log.Debugf("starting node %d", n.ID) res := n.Runner.MustRun(newReq) n.Daemon = res log.Debugf("node %d started, checking API", n.ID) n.WaitOnAPI(authorization) return n } func (n *Node) StartDaemon(ipfsArgs ...string) *Node { return n.StartDaemonWithReq(RunRequest{ Args: ipfsArgs, }, "") } func (n *Node) StartDaemonWithAuthorization(secret string, ipfsArgs ...string) *Node { return n.StartDaemonWithReq(RunRequest{ Args: ipfsArgs, }, secret) } func (n *Node) signalAndWait(watch <-chan struct{}, signal os.Signal, t time.Duration) bool { err := n.Daemon.Cmd.Process.Signal(signal) if err != nil { if errors.Is(err, os.ErrProcessDone) { log.Debugf("process for node %d has already finished", n.ID) return true } log.Panicf("error killing daemon for node %d with peer ID %s: %s", n.ID, n.PeerID(), err.Error()) } timer := time.NewTimer(t) defer timer.Stop() select { case <-watch: return true case <-timer.C: return false } } func (n *Node) StopDaemon() *Node { log.Debugf("stopping node %d", n.ID) if n.Daemon == nil { log.Debugf("didn't stop node %d since no daemon present", n.ID) return n } watch := make(chan struct{}, 1) go func() { _, _ = n.Daemon.Cmd.Process.Wait() watch <- struct{}{} }() // os.Interrupt does not support interrupts on Windows https://github.com/golang/go/issues/46345 if runtime.GOOS == "windows" { if n.signalAndWait(watch, syscall.SIGKILL, 5*time.Second) { return n } log.Panicf("timed out stopping node %d with peer ID %s", n.ID, n.PeerID()) } log.Debugf("signaling node %d with SIGTERM", n.ID) if n.signalAndWait(watch, syscall.SIGTERM, 1*time.Second) { return n } log.Debugf("signaling node %d with SIGTERM", n.ID) if n.signalAndWait(watch, syscall.SIGTERM, 2*time.Second) { return n } log.Debugf("signaling node %d with SIGQUIT", n.ID) if n.signalAndWait(watch, syscall.SIGQUIT, 5*time.Second) { return n } log.Debugf("signaling node %d with SIGKILL", n.ID) if n.signalAndWait(watch, syscall.SIGKILL, 5*time.Second) { return n } log.Panicf("timed out stopping node %d with peer ID %s", n.ID, n.PeerID()) return n } func (n *Node) APIAddr() multiaddr.Multiaddr { ma, err := n.TryAPIAddr() if err != nil { panic(err) } return ma } func (n *Node) APIURL() string { apiAddr := n.APIAddr() netAddr, err := manet.ToNetAddr(apiAddr) if err != nil { panic(err) } return "http://" + netAddr.String() } func (n *Node) TryAPIAddr() (multiaddr.Multiaddr, error) { b, err := os.ReadFile(filepath.Join(n.Dir, "api")) if err != nil { return nil, err } ma, err := multiaddr.NewMultiaddr(string(b)) if err != nil { return nil, err } return ma, nil } func (n *Node) checkAPI(authorization string) bool { apiAddr, err := n.TryAPIAddr() if err != nil { log.Debugf("node %d API addr not available yet: %s", n.ID, err.Error()) return false } if unixAddr, err := apiAddr.ValueForProtocol(multiaddr.P_UNIX); err == nil { parts := strings.SplitN(unixAddr, "/", 2) if len(parts) < 1 { panic("malformed unix socket address") } fileName := "/" + parts[1] _, err := os.Stat(fileName) return !errors.Is(err, fs.ErrNotExist) } ip, err := apiAddr.ValueForProtocol(multiaddr.P_IP4) if err != nil { panic(err) } port, err := apiAddr.ValueForProtocol(multiaddr.P_TCP) if err != nil { panic(err) } url := fmt.Sprintf("http://%s:%s/api/v0/id", ip, port) log.Debugf("checking API for node %d at %s", n.ID, url) req, err := http.NewRequest(http.MethodPost, url, nil) if err != nil { panic(err) } if authorization != "" { req.Header.Set("Authorization", authorization) } httpResp, err := http.DefaultClient.Do(req) if err != nil { log.Debugf("node %d API check error: %s", err.Error()) return false } defer httpResp.Body.Close() resp := struct { ID string }{} respBytes, err := io.ReadAll(httpResp.Body) if err != nil { log.Debugf("error reading API check response for node %d: %s", n.ID, err.Error()) return false } log.Debugf("got API check response for node %d: %s", n.ID, string(respBytes)) err = json.Unmarshal(respBytes, &resp) if err != nil { log.Debugf("error decoding API check response for node %d: %s", n.ID, err.Error()) return false } if resp.ID == "" { log.Debugf("API check response for node %d did not contain a Peer ID", n.ID) return false } respPeerID, err := peer.Decode(resp.ID) if err != nil { panic(err) } peerID := n.PeerID() if respPeerID != peerID { log.Panicf("expected peer ID %s but got %s", peerID, resp.ID) } log.Debugf("API check for node %d successful", n.ID) return true } func (n *Node) PeerID() peer.ID { cfg := n.ReadConfig() id, err := peer.Decode(cfg.Identity.PeerID) if err != nil { panic(err) } return id } func (n *Node) WaitOnAPI(authorization string) *Node { log.Debugf("waiting on API for node %d", n.ID) for range 50 { if n.checkAPI(authorization) { log.Debugf("daemon API found, daemon stdout: %s", n.Daemon.Stdout.String()) return n } time.Sleep(400 * time.Millisecond) } log.Panicf("node %d with peer ID %s failed to come online: \n%s\n\n%s", n.ID, n.PeerID(), n.Daemon.Stderr.String(), n.Daemon.Stdout.String()) return n } func (n *Node) IsAlive() bool { if n.Daemon == nil || n.Daemon.Cmd == nil || n.Daemon.Cmd.Process == nil { return false } log.Debugf("signaling node %d daemon process for liveness check", n.ID) err := n.Daemon.Cmd.Process.Signal(syscall.Signal(0)) if err == nil { log.Debugf("node %d daemon is alive", n.ID) return true } log.Debugf("node %d daemon not alive: %s", err.Error()) return false } func (n *Node) SwarmAddrs() []multiaddr.Multiaddr { res := n.Runner.Run(RunRequest{ Path: n.IPFSBin, Args: []string{"swarm", "addrs", "local"}, }) if res.ExitCode() != 0 { // If swarm command fails (e.g., daemon not online), return empty slice log.Debugf("Node %d: swarm addrs local failed (exit %d): %s", n.ID, res.ExitCode(), res.Stderr.String()) return []multiaddr.Multiaddr{} } out := strings.TrimSpace(res.Stdout.String()) if out == "" { log.Debugf("Node %d: swarm addrs local returned empty output", n.ID) return []multiaddr.Multiaddr{} } log.Debugf("Node %d: swarm addrs local output: %s", n.ID, out) outLines := strings.Split(out, "\n") var addrs []multiaddr.Multiaddr for _, addrStr := range outLines { addrStr = strings.TrimSpace(addrStr) if addrStr == "" { continue } ma, err := multiaddr.NewMultiaddr(addrStr) if err != nil { panic(err) } addrs = append(addrs, ma) } log.Debugf("Node %d: parsed %d swarm addresses", n.ID, len(addrs)) return addrs } // SwarmAddrsWithTimeout waits for swarm addresses to be available func (n *Node) SwarmAddrsWithTimeout(timeout time.Duration) []multiaddr.Multiaddr { start := time.Now() for time.Since(start) < timeout { addrs := n.SwarmAddrs() if len(addrs) > 0 { return addrs } time.Sleep(100 * time.Millisecond) } return []multiaddr.Multiaddr{} } func (n *Node) SwarmAddrsWithPeerIDs() []multiaddr.Multiaddr { return n.SwarmAddrsWithPeerIDsTimeout(5 * time.Second) } func (n *Node) SwarmAddrsWithPeerIDsTimeout(timeout time.Duration) []multiaddr.Multiaddr { ipfsProtocol := multiaddr.ProtocolWithCode(multiaddr.P_IPFS).Name peerID := n.PeerID() var addrs []multiaddr.Multiaddr for _, ma := range n.SwarmAddrsWithTimeout(timeout) { // add the peer ID to the multiaddr if it doesn't have it _, err := ma.ValueForProtocol(multiaddr.P_IPFS) if errors.Is(err, multiaddr.ErrProtocolNotFound) { comp, err := multiaddr.NewComponent(ipfsProtocol, peerID.String()) if err != nil { panic(err) } ma = ma.Encapsulate(comp) } addrs = append(addrs, ma) } return addrs } func (n *Node) SwarmAddrsWithoutPeerIDs() []multiaddr.Multiaddr { var addrs []multiaddr.Multiaddr for _, ma := range n.SwarmAddrs() { i := 0 for _, c := range ma { if c.Protocol().Code == multiaddr.P_IPFS { continue } ma[i] = c i++ } ma = ma[:i] if len(ma) > 0 { addrs = append(addrs, ma) } } return addrs } func (n *Node) Connect(other *Node) *Node { // Get the peer addresses to connect to addrs := other.SwarmAddrsWithPeerIDs() if len(addrs) == 0 { // If no addresses available, skip connection log.Debugf("No swarm addresses available for connection") return n } // Use Run instead of MustRun to avoid panics on connection failures res := n.Runner.Run(RunRequest{ Path: n.IPFSBin, Args: []string{"swarm", "connect", addrs[0].String()}, }) if res.ExitCode() != 0 { log.Debugf("swarm connect failed: %s", res.Stderr.String()) } return n } // ConnectAndWait connects to another node and waits for the connection to be established func (n *Node) ConnectAndWait(other *Node, timeout time.Duration) error { // Get the peer addresses to connect to - wait up to half the timeout for addresses addrs := other.SwarmAddrsWithPeerIDsTimeout(timeout / 2) if len(addrs) == 0 { return fmt.Errorf("no swarm addresses available for node %d after waiting %v", other.ID, timeout/2) } otherPeerID := other.PeerID() // Try to connect res := n.Runner.Run(RunRequest{ Path: n.IPFSBin, Args: []string{"swarm", "connect", addrs[0].String()}, }) if res.ExitCode() != 0 { return fmt.Errorf("swarm connect failed: %s", res.Stderr.String()) } // Wait for connection to be established start := time.Now() for time.Since(start) < timeout { peers := n.Peers() for _, peerAddr := range peers { if peerID, err := peerAddr.ValueForProtocol(multiaddr.P_P2P); err == nil { if peerID == otherPeerID.String() { return nil // Connection established } } } time.Sleep(100 * time.Millisecond) } return fmt.Errorf("timeout waiting for connection to node %d (peer %s)", other.ID, otherPeerID) } func (n *Node) Peers() []multiaddr.Multiaddr { // Wait for daemon to be ready if it's supposed to be running if n.Daemon != nil && n.Daemon.Cmd != nil && n.Daemon.Cmd.Process != nil { // Give daemon a short time to become ready for range 10 { if n.IsAlive() { break } time.Sleep(100 * time.Millisecond) } } res := n.Runner.Run(RunRequest{ Path: n.IPFSBin, Args: []string{"swarm", "peers"}, }) if res.ExitCode() != 0 { // If swarm peers fails (e.g., daemon not online), return empty slice log.Debugf("swarm peers failed: %s", res.Stderr.String()) return []multiaddr.Multiaddr{} } var addrs []multiaddr.Multiaddr for _, line := range res.Stdout.Lines() { ma, err := multiaddr.NewMultiaddr(line) if err != nil { panic(err) } addrs = append(addrs, ma) } return addrs } func (n *Node) PeerWith(other *Node) { n.UpdateConfig(func(cfg *config.Config) { var addrs []multiaddr.Multiaddr for _, addrStr := range other.ReadConfig().Addresses.Swarm { ma, err := multiaddr.NewMultiaddr(addrStr) if err != nil { panic(err) } addrs = append(addrs, ma) } cfg.Peering.Peers = append(cfg.Peering.Peers, peer.AddrInfo{ ID: other.PeerID(), Addrs: addrs, }) }) } func (n *Node) Disconnect(other *Node) { n.IPFS("swarm", "disconnect", "/p2p/"+other.PeerID().String()) } // GatewayURL waits for the gateway file and then returns its contents or times out. func (n *Node) GatewayURL() string { timer := time.NewTimer(1 * time.Second) defer timer.Stop() for { select { case <-timer.C: panic("timeout waiting for gateway file") default: b, err := os.ReadFile(filepath.Join(n.Dir, "gateway")) if err == nil { return strings.TrimSpace(string(b)) } if !errors.Is(err, fs.ErrNotExist) { panic(err) } time.Sleep(1 * time.Millisecond) } } } func (n *Node) GatewayClient() *HTTPClient { return &HTTPClient{ Client: http.DefaultClient, BaseURL: n.GatewayURL(), } } func (n *Node) APIClient() *HTTPClient { return &HTTPClient{ Client: http.DefaultClient, BaseURL: n.APIURL(), } } // DatastoreCount returns the count of entries matching the given prefix. // Requires the daemon to be stopped. func (n *Node) DatastoreCount(prefix string) int64 { res := n.IPFS("diag", "datastore", "count", prefix) count, _ := strconv.ParseInt(strings.TrimSpace(res.Stdout.String()), 10, 64) return count } // DatastorePut writes a key-value pair to the datastore. // Requires the daemon to be stopped. func (n *Node) DatastorePut(key, value string) { n.IPFS("diag", "datastore", "put", key, value) } // DatastoreGet retrieves the value at the given key. // Requires the daemon to be stopped. Returns nil if key not found. func (n *Node) DatastoreGet(key string) []byte { res := n.RunIPFS("diag", "datastore", "get", key) if res.Err != nil { return nil } return res.Stdout.Bytes() } // DatastoreHasKey checks if a key exists in the datastore. // Requires the daemon to be stopped. func (n *Node) DatastoreHasKey(key string) bool { res := n.RunIPFS("diag", "datastore", "get", key) return res.Err == nil } ================================================ FILE: test/cli/harness/nodes.go ================================================ package harness import ( "sync" . "github.com/ipfs/kubo/test/cli/testutils" "github.com/multiformats/go-multiaddr" ) // Nodes is a collection of Kubo nodes along with operations on groups of nodes. type Nodes []*Node func (n Nodes) Init(args ...string) Nodes { ForEachPar(n, func(node *Node) { node.Init(args...) }) return n } func (n Nodes) ForEachPar(f func(*Node)) { var wg sync.WaitGroup for _, node := range n { wg.Add(1) node := node go func() { defer wg.Done() f(node) }() } wg.Wait() } func (n Nodes) Connect() Nodes { for i, node := range n { for j, otherNode := range n { if i == j { continue } // Do not connect in parallel, because that can cause TLS handshake problems on some platforms. node.Connect(otherNode) } } for _, node := range n { firstPeer := node.Peers()[0] if _, err := firstPeer.ValueForProtocol(multiaddr.P_P2P); err != nil { log.Panicf("unexpected state for node %d with peer ID %s: %s", node.ID, node.PeerID(), err) } } return n } func (n Nodes) StartDaemons(args ...string) Nodes { ForEachPar(n, func(node *Node) { node.StartDaemon(args...) }) return n } func (n Nodes) StopDaemons() Nodes { ForEachPar(n, func(node *Node) { node.StopDaemon() }) return n } ================================================ FILE: test/cli/harness/pbinspect.go ================================================ package harness import ( "bytes" "encoding/json" mdag "github.com/ipfs/boxo/ipld/merkledag" ft "github.com/ipfs/boxo/ipld/unixfs" pb "github.com/ipfs/boxo/ipld/unixfs/pb" ) // UnixFSDataType returns the UnixFS DataType for the given CID by fetching the // raw block and parsing the protobuf. This directly checks the Type field in // the UnixFS Data message (https://specs.ipfs.tech/unixfs/#data). // // Common types: // - ft.TDirectory (1) = basic flat directory // - ft.THAMTShard (5) = HAMT sharded directory func (n *Node) UnixFSDataType(cid string) (pb.Data_DataType, error) { log.Debugf("node %d block get %s", n.ID, cid) var blockData bytes.Buffer res := n.Runner.MustRun(RunRequest{ Path: n.IPFSBin, Args: []string{"block", "get", cid}, CmdOpts: []CmdOpt{RunWithStdout(&blockData)}, }) if res.Err != nil { return 0, res.Err } // Parse dag-pb block protoNode, err := mdag.DecodeProtobuf(blockData.Bytes()) if err != nil { return 0, err } // Parse UnixFS data fsNode, err := ft.FSNodeFromBytes(protoNode.Data()) if err != nil { return 0, err } return fsNode.Type(), nil } // UnixFSHAMTFanout returns the fanout value for a HAMT shard directory. // This is only valid for HAMT shards (THAMTShard type). func (n *Node) UnixFSHAMTFanout(cid string) (uint64, error) { log.Debugf("node %d block get %s for fanout", n.ID, cid) var blockData bytes.Buffer res := n.Runner.MustRun(RunRequest{ Path: n.IPFSBin, Args: []string{"block", "get", cid}, CmdOpts: []CmdOpt{RunWithStdout(&blockData)}, }) if res.Err != nil { return 0, res.Err } // Parse dag-pb block protoNode, err := mdag.DecodeProtobuf(blockData.Bytes()) if err != nil { return 0, err } // Parse UnixFS data fsNode, err := ft.FSNodeFromBytes(protoNode.Data()) if err != nil { return 0, err } return fsNode.Fanout(), nil } // InspectPBNode uses dag-json output of 'ipfs dag get' to inspect // "Logical Format" of DAG-PB as defined in // https://web.archive.org/web/20250403194752/https://ipld.io/specs/codecs/dag-pb/spec/#logical-format // (mainly used for inspecting Links without depending on any libraries) func (n *Node) InspectPBNode(cid string) (PBNode, error) { log.Debugf("node %d dag get %s as dag-json", n.ID, cid) var root PBNode var dagJsonOutput bytes.Buffer res := n.Runner.MustRun(RunRequest{ Path: n.IPFSBin, Args: []string{"dag", "get", "--output-codec=dag-json", cid}, CmdOpts: []CmdOpt{RunWithStdout(&dagJsonOutput)}, }) if res.Err != nil { return root, res.Err } err := json.Unmarshal(dagJsonOutput.Bytes(), &root) if err != nil { return root, err } return root, nil } // Define structs to match the JSON for type PBHash struct { Slash string `json:"/"` } type PBLink struct { Hash PBHash `json:"Hash"` Name string `json:"Name"` Tsize int `json:"Tsize"` } type PBData struct { Slash struct { Bytes string `json:"bytes"` } `json:"/"` } type PBNode struct { Data PBData `json:"Data"` Links []PBLink `json:"Links"` } ================================================ FILE: test/cli/harness/peering.go ================================================ package harness import ( "fmt" "math/rand" "net" "sync" "testing" "github.com/ipfs/kubo/config" ) type Peering struct { From int To int } var ( allocatedPorts = make(map[int]struct{}) portMutex sync.Mutex ) func NewRandPort() int { portMutex.Lock() defer portMutex.Unlock() for range 100 { l, err := net.Listen("tcp", "localhost:0") if err != nil { continue } port := l.Addr().(*net.TCPAddr).Port l.Close() if _, used := allocatedPorts[port]; !used { allocatedPorts[port] = struct{}{} return port } } // Fallback to random port if we can't get a unique one from the OS for range 1000 { port := 30000 + rand.Intn(10000) if _, used := allocatedPorts[port]; !used { allocatedPorts[port] = struct{}{} return port } } panic("failed to allocate unique port after 1100 attempts") } func CreatePeerNodes(t *testing.T, n int, peerings []Peering) (*Harness, Nodes) { h := NewT(t) nodes := h.NewNodes(n).Init() nodes.ForEachPar(func(node *Node) { node.UpdateConfig(func(cfg *config.Config) { cfg.Routing.Type = config.NewOptionalString("none") cfg.Addresses.Swarm = []string{fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", NewRandPort())} }) }) for _, peering := range peerings { nodes[peering.From].PeerWith(nodes[peering.To]) } return h, nodes } ================================================ FILE: test/cli/harness/run.go ================================================ package harness import ( "fmt" "io" "os" "os/exec" "strings" ) // Runner is a process runner which can run subprocesses and aggregate output. type Runner struct { Env map[string]string Dir string Verbose bool } type ( CmdOpt func(*exec.Cmd) RunFunc func(*exec.Cmd) error ) var RunFuncStart = (*exec.Cmd).Start type RunRequest struct { Path string Args []string // Options that are applied to the exec.Cmd just before running it CmdOpts []CmdOpt // Function to use to run the command. // If not specified, defaults to cmd.Run RunFunc func(*exec.Cmd) error Verbose bool } type RunResult struct { Stdout *Buffer Stderr *Buffer Err error ExitErr *exec.ExitError Cmd *exec.Cmd } func (r *RunResult) ExitCode() int { return r.Cmd.ProcessState.ExitCode() } func environToMap(environ []string) map[string]string { m := map[string]string{} for _, e := range environ { kv := strings.Split(e, "=") // Skip environment variables that start with = // These can occur in Windows https://github.com/golang/go/issues/61956 if kv[0] == "" { continue } m[kv[0]] = kv[1] } return m } func (r *Runner) Run(req RunRequest) *RunResult { cmd := exec.Command(req.Path, req.Args...) var stdout io.Writer var stderr io.Writer outbuf := &Buffer{} errbuf := &Buffer{} if r.Verbose { or, ow := io.Pipe() errr, errw := io.Pipe() stdout = io.MultiWriter(outbuf, ow) stderr = io.MultiWriter(errbuf, errw) go func() { _, _ = io.Copy(os.Stdout, or) }() go func() { _, _ = io.Copy(os.Stderr, errr) }() } else { stdout = outbuf stderr = errbuf } cmd.Stdout = stdout cmd.Stderr = stderr cmd.Dir = r.Dir for k, v := range r.Env { cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) } for _, o := range req.CmdOpts { o(cmd) } if req.RunFunc == nil { req.RunFunc = (*exec.Cmd).Run } log.Debugf("running %v", cmd.Args) err := req.RunFunc(cmd) result := RunResult{ Stdout: outbuf, Stderr: errbuf, Cmd: cmd, Err: err, } if exitErr, ok := err.(*exec.ExitError); ok { result.ExitErr = exitErr } return &result } // MustRun runs the command and fails the test if the command fails. func (r *Runner) MustRun(req RunRequest) *RunResult { result := r.Run(req) r.AssertNoError(result) return result } func (r *Runner) AssertNoError(result *RunResult) { if result.ExitErr != nil { log.Panicf("'%s' returned error, code: %d, err: %s\nstdout:%s\nstderr:%s\n", result.Cmd.Args, result.ExitErr.ExitCode(), result.ExitErr.Error(), result.Stdout.String(), result.Stderr.String()) } if result.Err != nil { log.Panicf("unable to run %s: %s", result.Cmd.Path, result.Err) } } func RunWithEnv(env map[string]string) CmdOpt { return func(cmd *exec.Cmd) { for k, v := range env { cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) } } } func RunWithPath(path string) CmdOpt { return func(cmd *exec.Cmd) { var newEnv []string for _, env := range cmd.Env { e := strings.Split(env, "=") if e[0] == "PATH" { paths := strings.Split(e[1], ":") paths = append(paths, path) e[1] = strings.Join(paths, ":") fmt.Printf("path: %s\n", strings.Join(e, "=")) } newEnv = append(newEnv, strings.Join(e, "=")) } cmd.Env = newEnv } } func RunWithStdin(reader io.Reader) CmdOpt { return func(cmd *exec.Cmd) { cmd.Stdin = reader } } func RunWithStdinStr(s string) CmdOpt { return RunWithStdin(strings.NewReader(s)) } func RunWithStdout(writer io.Writer) CmdOpt { return func(cmd *exec.Cmd) { cmd.Stdout = io.MultiWriter(writer, cmd.Stdout) } } func RunWithStderr(writer io.Writer) CmdOpt { return func(cmd *exec.Cmd) { cmd.Stderr = io.MultiWriter(writer, cmd.Stdout) } } ================================================ FILE: test/cli/http_gateway_over_libp2p_test.go ================================================ package cli import ( "context" "encoding/json" "fmt" "io" "net/http" "testing" "github.com/ipfs/go-cid" "github.com/ipfs/kubo/core/commands" "github.com/ipfs/kubo/test/cli/harness" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/peer" libp2phttp "github.com/libp2p/go-libp2p/p2p/http" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" "github.com/stretchr/testify/require" ) func TestGatewayOverLibp2p(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init() // Setup streaming functionality nodes.ForEachPar(func(node *harness.Node) { node.IPFS("config", "--json", "Experimental.Libp2pStreamMounting", "true") }) gwNode := nodes[0] p2pProxyNode := nodes[1] nodes.StartDaemons().Connect() defer nodes.StopDaemons() // Add data to the gateway node cidDataOnGatewayNode := cid.MustParse(gwNode.IPFSAddStr("Hello Worlds2!")) r := gwNode.GatewayClient().Get(fmt.Sprintf("/ipfs/%s?format=raw", cidDataOnGatewayNode)) blockDataOnGatewayNode := []byte(r.Body) // Add data to the non-gateway node cidDataNotOnGatewayNode := cid.MustParse(p2pProxyNode.IPFSAddStr("Hello Worlds!")) r = p2pProxyNode.GatewayClient().Get(fmt.Sprintf("/ipfs/%s?format=raw", cidDataNotOnGatewayNode)) blockDataNotOnGatewayNode := []byte(r.Body) _ = blockDataNotOnGatewayNode // Setup one of the nodes as http to http-over-libp2p proxy p2pProxyNode.IPFS("p2p", "forward", "--allow-custom-protocol", "/http/1.1", "/ip4/127.0.0.1/tcp/0", fmt.Sprintf("/p2p/%s", gwNode.PeerID())) lsOutput := commands.P2PLsOutput{} if err := json.Unmarshal(p2pProxyNode.IPFS("p2p", "ls", "--enc=json").Stdout.Bytes(), &lsOutput); err != nil { t.Fatal(err) } require.Len(t, lsOutput.Listeners, 1) p2pProxyNodeHTTPListenMA, err := multiaddr.NewMultiaddr(lsOutput.Listeners[0].ListenAddress) require.NoError(t, err) p2pProxyNodeHTTPListenAddr, err := manet.ToNetAddr(p2pProxyNodeHTTPListenMA) require.NoError(t, err) t.Run("DoesNotWorkWithoutExperimentalConfig", func(t *testing.T) { _, err := http.Get(fmt.Sprintf("http://%s/ipfs/%s?format=raw", p2pProxyNodeHTTPListenAddr, cidDataOnGatewayNode)) require.Error(t, err) }) // Enable the experimental feature and reconnect the nodes gwNode.IPFS("config", "--json", "Experimental.GatewayOverLibp2p", "true") gwNode.StopDaemon().StartDaemon() t.Cleanup(func() { gwNode.StopDaemon() }) nodes.Connect() // Note: the bare HTTP requests here assume that the gateway is mounted at `/` t.Run("WillNotServeRemoteContent", func(t *testing.T) { resp, err := http.Get(fmt.Sprintf("http://%s/ipfs/%s?format=raw", p2pProxyNodeHTTPListenAddr, cidDataNotOnGatewayNode)) require.NoError(t, err) require.Equal(t, http.StatusNotFound, resp.StatusCode) }) t.Run("WillNotServeDeserializedResponses", func(t *testing.T) { resp, err := http.Get(fmt.Sprintf("http://%s/ipfs/%s", p2pProxyNodeHTTPListenAddr, cidDataOnGatewayNode)) require.NoError(t, err) require.Equal(t, http.StatusNotAcceptable, resp.StatusCode) }) t.Run("ServeBlock", func(t *testing.T) { t.Run("UsingKuboProxy", func(t *testing.T) { resp, err := http.Get(fmt.Sprintf("http://%s/ipfs/%s?format=raw", p2pProxyNodeHTTPListenAddr, cidDataOnGatewayNode)) require.NoError(t, err) defer resp.Body.Close() require.Equal(t, 200, resp.StatusCode) body, err := io.ReadAll(resp.Body) require.NoError(t, err) require.Equal(t, blockDataOnGatewayNode, body) }) t.Run("UsingLibp2pClientWithPathDiscovery", func(t *testing.T) { clientHost, err := libp2p.New(libp2p.NoListenAddrs) require.NoError(t, err) err = clientHost.Connect(context.Background(), peer.AddrInfo{ ID: gwNode.PeerID(), Addrs: gwNode.SwarmAddrs(), }) require.NoError(t, err) client, err := (&libp2phttp.Host{StreamHost: clientHost}).NamespacedClient("/ipfs/gateway", peer.AddrInfo{ID: gwNode.PeerID()}) require.NoError(t, err) resp, err := client.Get(fmt.Sprintf("/ipfs/%s?format=raw", cidDataOnGatewayNode)) require.NoError(t, err) defer resp.Body.Close() require.Equal(t, 200, resp.StatusCode) body, err := io.ReadAll(resp.Body) require.NoError(t, err) require.Equal(t, blockDataOnGatewayNode, body) }) }) } ================================================ FILE: test/cli/http_retrieval_client_test.go ================================================ package cli import ( "fmt" "net" "net/http" "net/http/httptest" "net/url" "os" "strings" "testing" "github.com/ipfs/boxo/routing/http/server" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/go-cid" "github.com/ipfs/go-test/random" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils/httprouting" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/assert" ) func TestHTTPRetrievalClient(t *testing.T) { t.Parallel() // many moving pieces here, show more when debug is needed debug := os.Getenv("DEBUG") == "true" // usee local /routing/v1/providers/{cid} and // /ipfs/{cid} HTTP servers to confirm HTTP-only retrieval works end-to-end. t.Run("works end-to-end with an HTTP-only provider", func(t *testing.T) { // setup mocked HTTP Router to handle /routing/v1/providers/cid mockRouter := &httprouting.MockHTTPContentRouter{Debug: debug} delegatedRoutingServer := httptest.NewServer(server.Handler(mockRouter)) t.Cleanup(func() { delegatedRoutingServer.Close() }) // init Kubo repo node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { // explicitly enable http client cfg.HTTPRetrieval.Enabled = config.True // allow NewMockHTTPProviderServer to use self-signed TLS cert cfg.HTTPRetrieval.TLSInsecureSkipVerify = config.True // setup client-only routing which asks both HTTP + DHT // cfg.Routing.Type = config.NewOptionalString("autoclient") // setup Kubo node to use mocked HTTP Router cfg.Routing.DelegatedRouters = []string{delegatedRoutingServer.URL} }) // compute a random CID randStr := string(random.Bytes(100)) res := node.PipeStrToIPFS(randStr, "add", "-qn", "--cid-version", "1") // -n means dont add to local repo, just produce CID wantCIDStr := res.Stdout.Trimmed() testCid := cid.MustParse(wantCIDStr) // setup mock HTTP provider httpProviderServer := NewMockHTTPProviderServer(testCid, randStr, debug) t.Cleanup(func() { httpProviderServer.Close() }) httpHost, httpPort, err := splitHostPort(httpProviderServer.URL) assert.NoError(t, err) // setup /routing/v1/providers/cid result that points at our mocked HTTP provider mockHTTPProviderPeerID := "12D3KooWCjfPiojcCUmv78Wd1NJzi4Mraj1moxigp7AfQVQvGLwH" // static, it does not matter, we only care about multiaddr mockHTTPMultiaddr, _ := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%s/tls/http", httpHost, httpPort)) mpid, _ := peer.Decode(mockHTTPProviderPeerID) mockRouter.AddProvider(testCid, &types.PeerRecord{ Schema: types.SchemaPeer, ID: &mpid, Addrs: []types.Multiaddr{{Multiaddr: mockHTTPMultiaddr}}, // no explicit Protocols, ensure multiaddr alone is enough }) // Start Kubo node.StartDaemon() defer node.StopDaemon() if debug { fmt.Printf("delegatedRoutingServer.URL: %s\n", delegatedRoutingServer.URL) fmt.Printf("httpProviderServer.URL: %s\n", httpProviderServer.URL) fmt.Printf("httpProviderServer.Multiaddr: %s\n", mockHTTPMultiaddr) fmt.Printf("testCid: %s\n", testCid) } // Now, make Kubo to read testCid. it was not added to local blockstore, so it has only one provider -- a HTTP server. // First, confirm delegatedRoutingServer returned HTTP provider findprovsRes := node.IPFS("routing", "findprovs", testCid.String()) assert.Equal(t, mockHTTPProviderPeerID, findprovsRes.Stdout.Trimmed()) // Ok, now attempt retrieval. // If there was no timeout and returned bytes match expected body, HTTP routing and retrieval worked end-to-end. catRes := node.IPFS("cat", testCid.String()) assert.Equal(t, randStr, catRes.Stdout.Trimmed()) }) } // NewMockHTTPProviderServer pretends to be http provider that supports // block response https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw func NewMockHTTPProviderServer(c cid.Cid, body string, debug bool) *httptest.Server { expectedPathPrefix := "/ipfs/" + c.String() handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if debug { fmt.Printf("NewMockHTTPProviderServer GET %s\n", req.URL.Path) } if strings.HasPrefix(req.URL.Path, expectedPathPrefix) { w.Header().Set("Content-Type", "application/vnd.ipld.raw") w.WriteHeader(http.StatusOK) if req.Method == "GET" { _, err := w.Write([]byte(body)) if err != nil { fmt.Fprintf(os.Stderr, "NewMockHTTPProviderServer GET %s error: %v\n", req.URL.Path, err) } } } else if strings.HasPrefix(req.URL.Path, "/ipfs/bafkqaaa") { // This is probe from https://specs.ipfs.tech/http-gateways/trustless-gateway/#dedicated-probe-paths w.Header().Set("Content-Type", "application/vnd.ipld.raw") w.WriteHeader(http.StatusOK) } else { http.Error(w, "Not Found", http.StatusNotFound) } }) // Make it HTTP/2 with self-signed TLS cert srv := httptest.NewUnstartedServer(handler) srv.EnableHTTP2 = true srv.StartTLS() return srv } func splitHostPort(httpUrl string) (ipAddr string, port string, err error) { u, err := url.Parse(httpUrl) if err != nil { return "", "", err } if u.Scheme == "" || u.Host == "" { return "", "", fmt.Errorf("invalid URL format: missing scheme or host") } ipAddr, port, err = net.SplitHostPort(u.Host) if err != nil { return "", "", fmt.Errorf("failed to split host and port from %q: %w", u.Host, err) } return ipAddr, port, nil } ================================================ FILE: test/cli/identity_cid_test.go ================================================ package cli import ( "fmt" "os" "path/filepath" "strings" "testing" "github.com/ipfs/boxo/verifcid" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestIdentityCIDOverflowProtection(t *testing.T) { t.Parallel() t.Run("ipfs add --hash=identity with small data succeeds", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // small data that fits in identity CID smallData := "small data" tempFile := filepath.Join(node.Dir, "small.txt") err := os.WriteFile(tempFile, []byte(smallData), 0644) require.NoError(t, err) res := node.IPFS("add", "--hash=identity", tempFile) assert.NoError(t, res.Err) cid := strings.Fields(res.Stdout.String())[1] // verify it's actually using identity hash res = node.IPFS("cid", "format", "-f", "%h", cid) assert.NoError(t, res.Err) assert.Equal(t, "identity", res.Stdout.Trimmed()) }) t.Run("ipfs add --hash=identity with large data fails", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // data larger than verifcid.DefaultMaxIdentityDigestSize largeData := strings.Repeat("x", verifcid.DefaultMaxIdentityDigestSize+50) tempFile := filepath.Join(node.Dir, "large.txt") err := os.WriteFile(tempFile, []byte(largeData), 0644) require.NoError(t, err) res := node.RunIPFS("add", "--hash=identity", tempFile) assert.NotEqual(t, 0, res.ExitErr.ExitCode()) // should error with digest too large message assert.Contains(t, res.Stderr.String(), "digest too large") }) t.Run("ipfs add --inline with valid --inline-limit succeeds", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() smallData := "small inline data" tempFile := filepath.Join(node.Dir, "inline.txt") err := os.WriteFile(tempFile, []byte(smallData), 0644) require.NoError(t, err) // use limit just under the maximum limit := verifcid.DefaultMaxIdentityDigestSize - 10 res := node.IPFS("add", "--inline", fmt.Sprintf("--inline-limit=%d", limit), tempFile) assert.NoError(t, res.Err) cid := strings.Fields(res.Stdout.String())[1] // verify the CID is using identity hash (inline) res = node.IPFS("cid", "format", "-f", "%h", cid) assert.NoError(t, res.Err) assert.Equal(t, "identity", res.Stdout.Trimmed()) // verify the codec (may be dag-pb or raw depending on kubo version) res = node.IPFS("cid", "format", "-f", "%c", cid) assert.NoError(t, res.Err) // Accept either raw or dag-pb as both are valid for inline data codec := res.Stdout.Trimmed() assert.True(t, codec == "raw" || codec == "dag-pb", "expected raw or dag-pb codec, got %s", codec) }) t.Run("ipfs add --inline with excessive --inline-limit fails", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() smallData := "data" tempFile := filepath.Join(node.Dir, "inline2.txt") err := os.WriteFile(tempFile, []byte(smallData), 0644) require.NoError(t, err) excessiveLimit := verifcid.DefaultMaxIdentityDigestSize + 50 res := node.RunIPFS("add", "--inline", fmt.Sprintf("--inline-limit=%d", excessiveLimit), tempFile) assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), fmt.Sprintf("inline-limit %d exceeds maximum allowed size of %d bytes", excessiveLimit, verifcid.DefaultMaxIdentityDigestSize)) }) t.Run("ipfs files write --hash=identity appending to identity CID switches to configured hash", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // create initial small file with identity CID initialData := "initial" tempFile := filepath.Join(node.Dir, "initial.txt") err := os.WriteFile(tempFile, []byte(initialData), 0644) require.NoError(t, err) res := node.IPFS("add", "--hash=identity", tempFile) assert.NoError(t, res.Err) cid1 := strings.Fields(res.Stdout.String())[1] // verify initial CID uses identity res = node.IPFS("cid", "format", "-f", "%h", cid1) assert.NoError(t, res.Err) assert.Equal(t, "identity", res.Stdout.Trimmed()) // copy to MFS res = node.IPFS("files", "cp", fmt.Sprintf("/ipfs/%s", cid1), "/identity-file") assert.NoError(t, res.Err) // append data that would exceed identity CID limit appendData := strings.Repeat("a", verifcid.DefaultMaxIdentityDigestSize) appendFile := filepath.Join(node.Dir, "append.txt") err = os.WriteFile(appendFile, []byte(appendData), 0644) require.NoError(t, err) // append to the end of the file // get the current data size res = node.IPFS("files", "stat", "--format", "", "/identity-file") assert.NoError(t, res.Err) size := res.Stdout.Trimmed() // this should succeed because DagModifier in boxo handles the overflow res = node.IPFS("files", "write", "--hash=identity", "--offset="+size, "/identity-file", appendFile) assert.NoError(t, res.Err) // check that the file now uses non-identity hash res = node.IPFS("files", "stat", "--hash", "/identity-file") assert.NoError(t, res.Err) newCid := res.Stdout.Trimmed() // verify new CID does NOT use identity res = node.IPFS("cid", "format", "-f", "%h", newCid) assert.NoError(t, res.Err) assert.NotEqual(t, "identity", res.Stdout.Trimmed()) // verify it switched to a cryptographic hash assert.Equal(t, config.DefaultHashFunction, res.Stdout.Trimmed()) }) t.Run("ipfs files write --hash=identity with small write creates identity CID", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // create a small file with identity hash directly in MFS smallData := "small" tempFile := filepath.Join(node.Dir, "small.txt") err := os.WriteFile(tempFile, []byte(smallData), 0644) require.NoError(t, err) // write to MFS with identity hash res := node.IPFS("files", "write", "--create", "--hash=identity", "/mfs-identity", tempFile) assert.NoError(t, res.Err) // verify using identity CID res = node.IPFS("files", "stat", "--hash", "/mfs-identity") assert.NoError(t, res.Err) cid := res.Stdout.Trimmed() // verify CID uses identity hash res = node.IPFS("cid", "format", "-f", "%h", cid) assert.NoError(t, res.Err) assert.Equal(t, "identity", res.Stdout.Trimmed()) // verify content res = node.IPFS("files", "read", "/mfs-identity") assert.NoError(t, res.Err) assert.Equal(t, smallData, res.Stdout.Trimmed()) }) t.Run("raw node with identity CID converts to UnixFS when appending", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // create raw block with identity CID rawData := "raw" tempFile := filepath.Join(node.Dir, "raw.txt") err := os.WriteFile(tempFile, []byte(rawData), 0644) require.NoError(t, err) res := node.IPFS("block", "put", "--format=raw", "--mhtype=identity", tempFile) assert.NoError(t, res.Err) rawCid := res.Stdout.Trimmed() // verify initial CID uses identity hash and raw codec res = node.IPFS("cid", "format", "-f", "%h", rawCid) assert.NoError(t, res.Err) assert.Equal(t, "identity", res.Stdout.Trimmed()) res = node.IPFS("cid", "format", "-f", "%c", rawCid) assert.NoError(t, res.Err) assert.Equal(t, "raw", res.Stdout.Trimmed()) // copy to MFS res = node.IPFS("files", "cp", fmt.Sprintf("/ipfs/%s", rawCid), "/raw-identity") assert.NoError(t, res.Err) // append data appendData := "appended" appendFile := filepath.Join(node.Dir, "append-raw.txt") err = os.WriteFile(appendFile, []byte(appendData), 0644) require.NoError(t, err) // get current data size for appending res = node.IPFS("files", "stat", "--format", "", "/raw-identity") assert.NoError(t, res.Err) size := res.Stdout.Trimmed() res = node.IPFS("files", "write", "--hash=identity", "--offset="+size, "/raw-identity", appendFile) assert.NoError(t, res.Err) // verify content res = node.IPFS("files", "read", "/raw-identity") assert.NoError(t, res.Err) assert.Equal(t, rawData+appendData, res.Stdout.Trimmed()) // check that it's now a UnixFS structure (dag-pb) res = node.IPFS("files", "stat", "--hash", "/raw-identity") assert.NoError(t, res.Err) newCid := res.Stdout.Trimmed() res = node.IPFS("cid", "format", "-f", "%c", newCid) assert.NoError(t, res.Err) assert.Equal(t, "dag-pb", res.Stdout.Trimmed()) res = node.IPFS("files", "stat", "/raw-identity") assert.NoError(t, res.Err) assert.Contains(t, res.Stdout.String(), "Type: file") }) t.Run("ipfs add --inline-limit at exactly max size succeeds", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // create small data that will be inlined smallData := "test data for inline" tempFile := filepath.Join(node.Dir, "exact.txt") err := os.WriteFile(tempFile, []byte(smallData), 0644) require.NoError(t, err) // exactly at the limit should succeed res := node.IPFS("add", "--inline", fmt.Sprintf("--inline-limit=%d", verifcid.DefaultMaxIdentityDigestSize), tempFile) assert.NoError(t, res.Err) cid := strings.Fields(res.Stdout.String())[1] // verify it uses identity hash (inline) since data is small enough res = node.IPFS("cid", "format", "-f", "%h", cid) assert.NoError(t, res.Err) assert.Equal(t, "identity", res.Stdout.Trimmed()) }) t.Run("ipfs add --inline-limit one byte over max fails", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() smallData := "test" tempFile := filepath.Join(node.Dir, "oneover.txt") err := os.WriteFile(tempFile, []byte(smallData), 0644) require.NoError(t, err) // one byte over should fail overLimit := verifcid.DefaultMaxIdentityDigestSize + 1 res := node.RunIPFS("add", "--inline", fmt.Sprintf("--inline-limit=%d", overLimit), tempFile) assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), fmt.Sprintf("inline-limit %d exceeds maximum allowed size of %d bytes", overLimit, verifcid.DefaultMaxIdentityDigestSize)) }) t.Run("ipfs add --inline with data larger than limit uses configured hash", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // data larger than inline limit largeData := strings.Repeat("y", 100) tempFile := filepath.Join(node.Dir, "toolarge.txt") err := os.WriteFile(tempFile, []byte(largeData), 0644) require.NoError(t, err) // set inline limit smaller than data res := node.IPFS("add", "--inline", "--inline-limit=50", tempFile) assert.NoError(t, res.Err) cid := strings.Fields(res.Stdout.String())[1] // verify it's NOT using identity hash (data too large for inline) res = node.IPFS("cid", "format", "-f", "%h", cid) assert.NoError(t, res.Err) assert.NotEqual(t, "identity", res.Stdout.Trimmed()) // should use configured hash assert.Equal(t, config.DefaultHashFunction, res.Stdout.Trimmed()) }) } ================================================ FILE: test/cli/init_test.go ================================================ package cli import ( "fmt" "os" fp "path/filepath" "strings" "testing" "github.com/ipfs/kubo/test/cli/harness" . "github.com/ipfs/kubo/test/cli/testutils" pb "github.com/libp2p/go-libp2p/core/crypto/pb" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func validatePeerID(t *testing.T, peerID peer.ID, expErr error, expAlgo pb.KeyType) { assert.NoError(t, peerID.Validate()) pub, err := peerID.ExtractPublicKey() assert.ErrorIs(t, expErr, err) if expAlgo != 0 { assert.Equal(t, expAlgo, pub.Type()) } } func testInitAlgo(t *testing.T, initFlags []string, expOutputName string, expPeerIDPubKeyErr error, expPeerIDPubKeyType pb.KeyType) { t.Run("init", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() initRes := node.IPFS(StrCat("init", initFlags)...) lines := []string{ fmt.Sprintf("generating %s keypair...done", expOutputName), fmt.Sprintf("peer identity: %s", node.PeerID().String()), fmt.Sprintf("initializing IPFS node at %s\n", node.Dir), } expectedInitOutput := strings.Join(lines, "\n") assert.Equal(t, expectedInitOutput, initRes.Stdout.String()) assert.DirExists(t, node.Dir) assert.FileExists(t, fp.Join(node.Dir, "config")) assert.DirExists(t, fp.Join(node.Dir, "datastore")) assert.DirExists(t, fp.Join(node.Dir, "blocks")) assert.NoFileExists(t, fp.Join(node.Dir, "._check_writeable")) _, err := os.ReadDir(node.Dir) assert.NoError(t, err, "ipfs dir should be listable") validatePeerID(t, node.PeerID(), expPeerIDPubKeyErr, expPeerIDPubKeyType) res := node.IPFS("config", "Mounts.IPFS") assert.Equal(t, "/ipfs", res.Stdout.Trimmed()) catRes := node.RunIPFS("cat", fmt.Sprintf("/ipfs/%s/readme", CIDWelcomeDocs)) assert.NotEqual(t, 0, catRes.ExitErr.ExitCode(), "welcome readme doesn't exist") }) t.Run("init without empty repo", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() initRes := node.IPFS(StrCat("init", "--empty-repo=false", initFlags)...) validatePeerID(t, node.PeerID(), expPeerIDPubKeyErr, expPeerIDPubKeyType) lines := []string{ fmt.Sprintf("generating %s keypair...done", expOutputName), fmt.Sprintf("peer identity: %s", node.PeerID().String()), fmt.Sprintf("initializing IPFS node at %s", node.Dir), "to get started, enter:", fmt.Sprintf("\n\tipfs cat /ipfs/%s/readme\n\n", CIDWelcomeDocs), } expectedEmptyInitOutput := strings.Join(lines, "\n") assert.Equal(t, expectedEmptyInitOutput, initRes.Stdout.String()) node.IPFS("cat", fmt.Sprintf("/ipfs/%s/readme", CIDWelcomeDocs)) idRes := node.IPFS("id", "-f", "") version := node.IPFS("version", "-n").Stdout.Trimmed() assert.Contains(t, idRes.Stdout.String(), version) }) } func TestInit(t *testing.T) { t.Parallel() t.Run("init fails if the repo dir has no perms", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() badDir := fp.Join(node.Dir, ".badipfs") err := os.Mkdir(badDir, 0o000) require.NoError(t, err) res := node.RunIPFS("init", "--repo-dir", badDir) assert.NotEqual(t, 0, res.Cmd.ProcessState.ExitCode()) assert.Contains(t, res.Stderr.String(), "permission denied") }) t.Run("init with ed25519", func(t *testing.T) { t.Parallel() testInitAlgo(t, []string{"--algorithm=ed25519"}, "ED25519", nil, pb.KeyType_Ed25519) }) t.Run("init with rsa", func(t *testing.T) { t.Parallel() testInitAlgo(t, []string{"--bits=2048", "--algorithm=rsa"}, "2048-bit RSA", peer.ErrNoPublicKey, 0) }) t.Run("init with default algorithm", func(t *testing.T) { t.Parallel() testInitAlgo(t, []string{}, "ED25519", nil, pb.KeyType_Ed25519) }) t.Run("ipfs init --profile with invalid profile fails", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() res := node.RunIPFS("init", "--profile=invalid_profile") assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Equal(t, "Error: invalid configuration profile: invalid_profile", res.Stderr.Trimmed()) }) t.Run("ipfs init --profile with valid profile succeeds", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode() node.IPFS("init", "--profile=server") }) t.Run("ipfs config looks good", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init("--profile=server") lines := node.IPFS("config", "Swarm.AddrFilters").Stdout.Lines() assert.Len(t, lines, 18) out := node.IPFS("config", "Bootstrap").Stdout.Trimmed() assert.Equal(t, "[]", out) out = node.IPFS("config", "Addresses.API").Stdout.Trimmed() assert.Equal(t, "/ip4/127.0.0.1/tcp/0", out) }) t.Run("ipfs init from existing config succeeds", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2) node1 := nodes[0] node2 := nodes[1] node1.Init("--profile=server") node2.IPFS("init", fp.Join(node1.Dir, "config")) out := node2.IPFS("config", "Addresses.API").Stdout.Trimmed() assert.Equal(t, "/ip4/127.0.0.1/tcp/0", out) }) t.Run("ipfs init should not run while daemon is running", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() res := node.RunIPFS("init") assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), "Error: ipfs daemon is running. please stop it to run this command") }) } ================================================ FILE: test/cli/ipfswatch_test.go ================================================ //go:build !plan9 package cli import ( "fmt" "os" "os/exec" "path/filepath" "regexp" "testing" "time" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/require" ) func TestIPFSWatch(t *testing.T) { t.Parallel() // Build ipfswatch binary once before running parallel subtests. // This avoids race conditions and duplicate builds. h := harness.NewT(t) repoRoot := filepath.Dir(filepath.Dir(filepath.Dir(h.IPFSBin))) ipfswatchBin := filepath.Join(repoRoot, "cmd", "ipfswatch", "ipfswatch") if _, err := os.Stat(ipfswatchBin); os.IsNotExist(err) { // -C changes to repo root so go.mod is found cmd := exec.Command("go", "build", "-C", repoRoot, "-o", ipfswatchBin, "./cmd/ipfswatch") out, err := cmd.CombinedOutput() require.NoError(t, err, "failed to build ipfswatch: %s", string(out)) } t.Run("ipfswatch adds watched files to IPFS", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() // Create a temp directory to watch watchDir := filepath.Join(h.Dir, "watch") err := os.MkdirAll(watchDir, 0o755) require.NoError(t, err) // Start ipfswatch in background result := node.Runner.Run(harness.RunRequest{ Path: ipfswatchBin, Args: []string{"--repo", node.Dir, "--path", watchDir}, RunFunc: harness.RunFuncStart, }) require.NoError(t, result.Err, "ipfswatch should start without error") defer func() { if result.Cmd.Process != nil { _ = result.Cmd.Process.Kill() _, _ = result.Cmd.Process.Wait() } }() // Wait for ipfswatch to initialize time.Sleep(2 * time.Second) // Check for startup errors stderrStr := result.Stderr.String() require.NotContains(t, stderrStr, "unknown datastore type", "ipfswatch should recognize datastore plugins") // Create a test file with unique content based on timestamp testContent := fmt.Sprintf("ipfswatch test content generated at %s", time.Now().Format(time.RFC3339Nano)) testFile := filepath.Join(watchDir, "test.txt") err = os.WriteFile(testFile, []byte(testContent), 0o644) require.NoError(t, err) // Wait for ipfswatch to process the file and extract CID from log // Log format: "added %s... key: %s" cidPattern := regexp.MustCompile(`added .*/test\.txt\.\.\. key: (\S+)`) var cid string deadline := time.Now().Add(10 * time.Second) for time.Now().Before(deadline) { stderrStr = result.Stderr.String() if matches := cidPattern.FindStringSubmatch(stderrStr); len(matches) > 1 { cid = matches[1] break } time.Sleep(100 * time.Millisecond) } require.NotEmpty(t, cid, "ipfswatch should have added test.txt and logged the CID, got stderr: %s", stderrStr) // Kill ipfswatch to release the repo lock if result.Cmd.Process != nil { if err = result.Cmd.Process.Signal(os.Interrupt); err != nil { _ = result.Cmd.Process.Kill() } _, _ = result.Cmd.Process.Wait() } // Verify the content matches by reading it back via ipfs cat catRes := node.RunIPFS("cat", "--offline", cid) require.Equal(t, 0, catRes.Cmd.ProcessState.ExitCode(), "ipfs cat should succeed, cid=%s, stderr: %s", cid, catRes.Stderr.String()) require.Equal(t, testContent, catRes.Stdout.String(), "content read from IPFS should match what was written") }) t.Run("ipfswatch loads datastore plugins for pebbleds", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() // Configure pebbleds as the datastore node.UpdateConfig(func(cfg *config.Config) { cfg.Datastore.Spec = map[string]any{ "type": "mount", "mounts": []any{ map[string]any{ "mountpoint": "/blocks", "path": "blocks", "prefix": "flatfs.datastore", "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2", "sync": true, "type": "flatfs", }, map[string]any{ "mountpoint": "/", "path": "datastore", "prefix": "pebble.datastore", "type": "pebbleds", }, }, } }) // Re-initialize datastore directory for pebbleds // (the repo was initialized with levelds, need to remove it) dsPath := filepath.Join(node.Dir, "datastore") err := os.RemoveAll(dsPath) require.NoError(t, err) err = os.MkdirAll(dsPath, 0o755) require.NoError(t, err) // Create a temp directory to watch watchDir := filepath.Join(h.Dir, "watch") err = os.MkdirAll(watchDir, 0o755) require.NoError(t, err) // Start ipfswatch in background result := node.Runner.Run(harness.RunRequest{ Path: ipfswatchBin, Args: []string{"--repo", node.Dir, "--path", watchDir}, RunFunc: harness.RunFuncStart, }) require.NoError(t, result.Err, "ipfswatch should start without error") defer func() { if result.Cmd.Process != nil { _ = result.Cmd.Process.Kill() _, _ = result.Cmd.Process.Wait() } }() // Wait for ipfswatch to initialize and check for errors time.Sleep(3 * time.Second) stderrStr := result.Stderr.String() require.NotContains(t, stderrStr, "unknown datastore type", "ipfswatch should recognize pebbleds datastore plugin") }) } ================================================ FILE: test/cli/log_level_test.go ================================================ package cli import ( "bufio" "context" "encoding/json" "fmt" "net/http" "os" "os/exec" "strings" "testing" "time" "github.com/ipfs/kubo/test/cli/harness" . "github.com/ipfs/kubo/test/cli/testutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestLogLevel(t *testing.T) { t.Run("CLI", func(t *testing.T) { t.Run("level '*' shows all subsystems", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() expectedSubsystems := getExpectedSubsystems(t, node) res := node.IPFS("log", "level", "*") assert.NoError(t, res.Err) assert.Empty(t, res.Stderr.Lines()) actualSubsystems := parseCLIOutput(t, res.Stdout.String()) // Should show all subsystems plus the (default) entry assert.GreaterOrEqual(t, len(actualSubsystems), len(expectedSubsystems)) validateAllSubsystemsPresentCLI(t, expectedSubsystems, actualSubsystems, "CLI output") // Should have the (default) entry _, hasDefault := actualSubsystems["(default)"] assert.True(t, hasDefault, "Should have '(default)' entry") }) t.Run("level 'all' shows all subsystems (alias for '*')", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() expectedSubsystems := getExpectedSubsystems(t, node) res := node.IPFS("log", "level", "all") assert.NoError(t, res.Err) assert.Empty(t, res.Stderr.Lines()) actualSubsystems := parseCLIOutput(t, res.Stdout.String()) // Should show all subsystems plus the (default) entry assert.GreaterOrEqual(t, len(actualSubsystems), len(expectedSubsystems)) validateAllSubsystemsPresentCLI(t, expectedSubsystems, actualSubsystems, "CLI output") // Should have the (default) entry _, hasDefault := actualSubsystems["(default)"] assert.True(t, hasDefault, "Should have '(default)' entry") }) t.Run("get level for specific subsystem", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() node.IPFS("log", "level", "core", "debug") res := node.IPFS("log", "level", "core") assert.NoError(t, res.Err) assert.Empty(t, res.Stderr.Lines()) output := res.Stdout.String() lines := SplitLines(output) assert.Equal(t, 1, len(lines)) line := strings.TrimSpace(lines[0]) assert.Equal(t, "debug", line) }) t.Run("get level with no args returns default level", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() res1 := node.IPFS("log", "level", "*", "fatal") assert.NoError(t, res1.Err) assert.Empty(t, res1.Stderr.Lines()) res := node.IPFS("log", "level") assert.NoError(t, res.Err) assert.Equal(t, 0, len(res.Stderr.Lines())) output := res.Stdout.String() lines := SplitLines(output) assert.Equal(t, 1, len(lines)) line := strings.TrimSpace(lines[0]) assert.Equal(t, "fatal", line) }) t.Run("get level reflects runtime log level changes", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() node.IPFS("log", "level", "core", "debug") res := node.IPFS("log", "level", "core") assert.NoError(t, res.Err) output := res.Stdout.String() lines := SplitLines(output) assert.Equal(t, 1, len(lines)) line := strings.TrimSpace(lines[0]) assert.Equal(t, "debug", line) }) t.Run("get level with non-existent subsystem returns error", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() res := node.RunIPFS("log", "level", "non-existent-subsystem") assert.Error(t, res.Err) assert.NotEqual(t, 0, len(res.Stderr.Lines())) }) t.Run("set level to 'default' keyword", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // First set a specific subsystem to a different level res1 := node.IPFS("log", "level", "core", "debug") assert.NoError(t, res1.Err) assert.Contains(t, res1.Stdout.String(), "Changed log level of 'core' to 'debug'") // Verify it was set to debug res2 := node.IPFS("log", "level", "core") assert.NoError(t, res2.Err) assert.Equal(t, "debug", strings.TrimSpace(res2.Stdout.String())) // Get the current default level (should be 'error' since unchanged) res3 := node.IPFS("log", "level") assert.NoError(t, res3.Err) defaultLevel := strings.TrimSpace(res3.Stdout.String()) assert.Equal(t, "error", defaultLevel, "Default level should be 'error' when unchanged") // Now set the subsystem back to default res4 := node.IPFS("log", "level", "core", "default") assert.NoError(t, res4.Err) assert.Contains(t, res4.Stdout.String(), "Changed log level of 'core' to") // Verify it's now at the default level (should be 'error') res5 := node.IPFS("log", "level", "core") assert.NoError(t, res5.Err) assert.Equal(t, "error", strings.TrimSpace(res5.Stdout.String())) }) t.Run("set all subsystems with 'all' changes default (alias for '*')", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Initial state - default should be 'error' res := node.IPFS("log", "level") assert.NoError(t, res.Err) assert.Equal(t, "error", strings.TrimSpace(res.Stdout.String())) // Set one subsystem to a different level res = node.IPFS("log", "level", "core", "debug") assert.NoError(t, res.Err) // Default should still be 'error' res = node.IPFS("log", "level") assert.NoError(t, res.Err) assert.Equal(t, "error", strings.TrimSpace(res.Stdout.String())) // Now use 'all' to set everything to 'info' res = node.IPFS("log", "level", "all", "info") assert.NoError(t, res.Err) assert.Contains(t, res.Stdout.String(), "Changed log level of '*' to 'info'") // Default should now be 'info' res = node.IPFS("log", "level") assert.NoError(t, res.Err) assert.Equal(t, "info", strings.TrimSpace(res.Stdout.String())) // Core should also be 'info' (overwritten by 'all') res = node.IPFS("log", "level", "core") assert.NoError(t, res.Err) assert.Equal(t, "info", strings.TrimSpace(res.Stdout.String())) // Any other subsystem should also be 'info' res = node.IPFS("log", "level", "dht") assert.NoError(t, res.Err) assert.Equal(t, "info", strings.TrimSpace(res.Stdout.String())) }) t.Run("set all subsystems with '*' changes default", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Initial state - default should be 'error' res := node.IPFS("log", "level") assert.NoError(t, res.Err) assert.Equal(t, "error", strings.TrimSpace(res.Stdout.String())) // Set one subsystem to a different level res = node.IPFS("log", "level", "core", "debug") assert.NoError(t, res.Err) // Default should still be 'error' res = node.IPFS("log", "level") assert.NoError(t, res.Err) assert.Equal(t, "error", strings.TrimSpace(res.Stdout.String())) // Now use '*' to set everything to 'info' res = node.IPFS("log", "level", "*", "info") assert.NoError(t, res.Err) assert.Contains(t, res.Stdout.String(), "Changed log level of '*' to 'info'") // Default should now be 'info' res = node.IPFS("log", "level") assert.NoError(t, res.Err) assert.Equal(t, "info", strings.TrimSpace(res.Stdout.String())) // Core should also be 'info' (overwritten by '*') res = node.IPFS("log", "level", "core") assert.NoError(t, res.Err) assert.Equal(t, "info", strings.TrimSpace(res.Stdout.String())) // Any other subsystem should also be 'info' res = node.IPFS("log", "level", "dht") assert.NoError(t, res.Err) assert.Equal(t, "info", strings.TrimSpace(res.Stdout.String())) }) t.Run("'all' in get mode shows (default) entry (alias for '*')", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Get all levels with 'all' res := node.IPFS("log", "level", "all") assert.NoError(t, res.Err) output := res.Stdout.String() // Should contain "(default): error" entry assert.Contains(t, output, "(default): error", "Should show default level with (default) key") // Should also contain various subsystems assert.Contains(t, output, "core: error") assert.Contains(t, output, "dht: error") }) t.Run("'*' in get mode shows (default) entry", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Get all levels with '*' res := node.IPFS("log", "level", "*") assert.NoError(t, res.Err) output := res.Stdout.String() // Should contain "(default): error" entry assert.Contains(t, output, "(default): error", "Should show default level with (default) key") // Should also contain various subsystems assert.Contains(t, output, "core: error") assert.Contains(t, output, "dht: error") }) t.Run("set all subsystems to 'default' using 'all' (alias for '*')", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Get the original default level (just for reference, it should be "error") res0 := node.IPFS("log", "level") assert.NoError(t, res0.Err) assert.Equal(t, "error", strings.TrimSpace(res0.Stdout.String())) // First set all subsystems to debug using 'all' res1 := node.IPFS("log", "level", "all", "debug") assert.NoError(t, res1.Err) assert.Contains(t, res1.Stdout.String(), "Changed log level of '*' to 'debug'") // Verify a specific subsystem is at debug res2 := node.IPFS("log", "level", "core") assert.NoError(t, res2.Err) assert.Equal(t, "debug", strings.TrimSpace(res2.Stdout.String())) // Verify the default level is now debug res3 := node.IPFS("log", "level") assert.NoError(t, res3.Err) assert.Equal(t, "debug", strings.TrimSpace(res3.Stdout.String())) // Now set all subsystems back to default (which is now "debug") using 'all' res4 := node.IPFS("log", "level", "all", "default") assert.NoError(t, res4.Err) assert.Contains(t, res4.Stdout.String(), "Changed log level of '*' to") // The subsystem should still be at debug (because that's what default is now) res5 := node.IPFS("log", "level", "core") assert.NoError(t, res5.Err) assert.Equal(t, "debug", strings.TrimSpace(res5.Stdout.String())) // The behavior is correct: "default" uses the current default level, // which was changed to "debug" when we set "all" to "debug" }) t.Run("set all subsystems to 'default' keyword", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Get the original default level (just for reference, it should be "error") res0 := node.IPFS("log", "level") assert.NoError(t, res0.Err) // originalDefault := strings.TrimSpace(res0.Stdout.String()) assert.Equal(t, "error", strings.TrimSpace(res0.Stdout.String())) // First set all subsystems to debug res1 := node.IPFS("log", "level", "*", "debug") assert.NoError(t, res1.Err) assert.Contains(t, res1.Stdout.String(), "Changed log level of '*' to 'debug'") // Verify a specific subsystem is at debug res2 := node.IPFS("log", "level", "core") assert.NoError(t, res2.Err) assert.Equal(t, "debug", strings.TrimSpace(res2.Stdout.String())) // Verify the default level is now debug res3 := node.IPFS("log", "level") assert.NoError(t, res3.Err) assert.Equal(t, "debug", strings.TrimSpace(res3.Stdout.String())) // Now set all subsystems back to default (which is now "debug") res4 := node.IPFS("log", "level", "*", "default") assert.NoError(t, res4.Err) assert.Contains(t, res4.Stdout.String(), "Changed log level of '*' to") // The subsystem should still be at debug (because that's what default is now) res5 := node.IPFS("log", "level", "core") assert.NoError(t, res5.Err) assert.Equal(t, "debug", strings.TrimSpace(res5.Stdout.String())) // The behavior is correct: "default" uses the current default level, // which was changed to "debug" when we set "*" to "debug" }) t.Run("shell escaping variants for '*' wildcard", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init().StartDaemon() defer node.StopDaemon() // Test different shell escaping methods work for '*' // This tests the behavior documented in help text: '*' or "*" or \* // Test 1: Single quotes '*' (should work) cmd1 := fmt.Sprintf("IPFS_PATH='%s' %s --api='%s' log level '*' info", node.Dir, node.IPFSBin, node.APIAddr()) res1 := h.Sh(cmd1) assert.NoError(t, res1.Err) assert.Contains(t, res1.Stdout.String(), "Changed log level of '*' to 'info'") // Test 2: Double quotes "*" (should work) cmd2 := fmt.Sprintf("IPFS_PATH='%s' %s --api='%s' log level \"*\" debug", node.Dir, node.IPFSBin, node.APIAddr()) res2 := h.Sh(cmd2) assert.NoError(t, res2.Err) assert.Contains(t, res2.Stdout.String(), "Changed log level of '*' to 'debug'") // Test 3: Backslash escape \* (should work) cmd3 := fmt.Sprintf("IPFS_PATH='%s' %s --api='%s' log level \\* warn", node.Dir, node.IPFSBin, node.APIAddr()) res3 := h.Sh(cmd3) assert.NoError(t, res3.Err) assert.Contains(t, res3.Stdout.String(), "Changed log level of '*' to 'warn'") // Test 4: Verify the final state - should show 'warn' as default res4 := node.IPFS("log", "level") assert.NoError(t, res4.Err) assert.Equal(t, "warn", strings.TrimSpace(res4.Stdout.String())) // Test 5: Get all levels using escaped '*' to verify it shows all subsystems cmd5 := fmt.Sprintf("IPFS_PATH='%s' %s --api='%s' log level \\*", node.Dir, node.IPFSBin, node.APIAddr()) res5 := h.Sh(cmd5) assert.NoError(t, res5.Err) output := res5.Stdout.String() assert.Contains(t, output, "(default): warn", "Should show updated default level") assert.Contains(t, output, "core: warn", "Should show core subsystem at warn level") }) }) t.Run("HTTP RPC", func(t *testing.T) { t.Run("get default level returns JSON", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Make HTTP request to get default log level resp, err := http.Post(node.APIURL()+"/api/v0/log/level", "", nil) require.NoError(t, err) defer resp.Body.Close() // Parse JSON response var result map[string]any err = json.NewDecoder(resp.Body).Decode(&result) require.NoError(t, err) // Check that we have the Levels field levels, ok := result["Levels"].(map[string]any) require.True(t, ok, "Response should have 'Levels' field") // Should have exactly one entry for the default level assert.Equal(t, 1, len(levels)) // The default level should be present defaultLevel, ok := levels[""] require.True(t, ok, "Should have empty string key for default level") assert.Equal(t, "error", defaultLevel, "Default level should be 'error'") }) t.Run("get all levels using 'all' returns JSON (alias for '*')", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() expectedSubsystems := getExpectedSubsystems(t, node) // Make HTTP request to get all log levels using 'all' resp, err := http.Post(node.APIURL()+"/api/v0/log/level?arg=all", "", nil) require.NoError(t, err) defer resp.Body.Close() levels := parseHTTPResponse(t, resp) validateAllSubsystemsPresent(t, expectedSubsystems, levels, "JSON response") // Should have the (default) entry defaultLevel, ok := levels["(default)"] require.True(t, ok, "Should have '(default)' key") assert.Equal(t, "error", defaultLevel, "Default level should be 'error'") }) t.Run("get all levels returns JSON", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() expectedSubsystems := getExpectedSubsystems(t, node) // Make HTTP request to get all log levels resp, err := http.Post(node.APIURL()+"/api/v0/log/level?arg=*", "", nil) require.NoError(t, err) defer resp.Body.Close() levels := parseHTTPResponse(t, resp) validateAllSubsystemsPresent(t, expectedSubsystems, levels, "JSON response") // Should have the (default) entry defaultLevel, ok := levels["(default)"] require.True(t, ok, "Should have '(default)' key") assert.Equal(t, "error", defaultLevel, "Default level should be 'error'") }) t.Run("get specific subsystem level returns JSON", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // First set a specific level for a subsystem resp, err := http.Post(node.APIURL()+"/api/v0/log/level?arg=core&arg=debug", "", nil) require.NoError(t, err) resp.Body.Close() // Now get the level for that subsystem resp, err = http.Post(node.APIURL()+"/api/v0/log/level?arg=core", "", nil) require.NoError(t, err) defer resp.Body.Close() // Parse JSON response var result map[string]any err = json.NewDecoder(resp.Body).Decode(&result) require.NoError(t, err) // Check that we have the Levels field levels, ok := result["Levels"].(map[string]any) require.True(t, ok, "Response should have 'Levels' field") // Should have exactly one entry assert.Equal(t, 1, len(levels)) // Check the level for 'core' subsystem coreLevel, ok := levels["core"] require.True(t, ok, "Should have 'core' key") assert.Equal(t, "debug", coreLevel, "Core level should be 'debug'") }) t.Run("set level using 'all' returns JSON message (alias for '*')", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Set a log level using 'all' resp, err := http.Post(node.APIURL()+"/api/v0/log/level?arg=all&arg=info", "", nil) require.NoError(t, err) defer resp.Body.Close() // Parse JSON response var result map[string]any err = json.NewDecoder(resp.Body).Decode(&result) require.NoError(t, err) // Check that we have the Message field message, ok := result["Message"].(string) require.True(t, ok, "Response should have 'Message' field") // Check the message content (should show '*' in message even when 'all' was used) assert.Contains(t, message, "Changed log level of '*' to 'info'") }) t.Run("set level returns JSON message", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Set a log level resp, err := http.Post(node.APIURL()+"/api/v0/log/level?arg=core&arg=info", "", nil) require.NoError(t, err) defer resp.Body.Close() // Parse JSON response var result map[string]any err = json.NewDecoder(resp.Body).Decode(&result) require.NoError(t, err) // Check that we have the Message field message, ok := result["Message"].(string) require.True(t, ok, "Response should have 'Message' field") // Check the message content assert.Contains(t, message, "Changed log level of 'core' to 'info'") }) t.Run("set level to 'default' keyword", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // First set a subsystem to debug resp, err := http.Post(node.APIURL()+"/api/v0/log/level?arg=core&arg=debug", "", nil) require.NoError(t, err) resp.Body.Close() // Now set it back to default resp, err = http.Post(node.APIURL()+"/api/v0/log/level?arg=core&arg=default", "", nil) require.NoError(t, err) defer resp.Body.Close() // Parse JSON response var result map[string]any err = json.NewDecoder(resp.Body).Decode(&result) require.NoError(t, err) // Check that we have the Message field message, ok := result["Message"].(string) require.True(t, ok, "Response should have 'Message' field") // The message should indicate the change assert.True(t, strings.Contains(message, "Changed log level of 'core' to"), "Message should indicate level change") // Verify the level is back to error (default) resp, err = http.Post(node.APIURL()+"/api/v0/log/level?arg=core", "", nil) require.NoError(t, err) defer resp.Body.Close() var getResult map[string]any err = json.NewDecoder(resp.Body).Decode(&getResult) require.NoError(t, err) levels, _ := getResult["Levels"].(map[string]any) coreLevel, _ := levels["core"].(string) assert.Equal(t, "error", coreLevel, "Core level should be back to 'error' (default)") }) }) // Constants for slog interop tests const ( slogTestLogTailTimeout = 10 * time.Second slogTestLogWaitTimeout = 5 * time.Second slogTestLogStartupDelay = 1 * time.Second // Wait for log tail to start slogTestSubsystemCmdsHTTP = "cmds/http" // Native go-log subsystem slogTestSubsystemNetIdentify = "net/identify" // go-libp2p slog subsystem ) // logMatch represents a matched log entry for slog interop tests type logMatch struct { subsystem string line string } // startLogMonitoring starts ipfs log tail and returns command and channel for matched logs. startLogMonitoring := func(t *testing.T, node *harness.Node) (*exec.Cmd, chan logMatch) { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), slogTestLogTailTimeout) t.Cleanup(cancel) cmd := exec.CommandContext(ctx, node.IPFSBin, "log", "tail") cmd.Env = append([]string(nil), os.Environ()...) for k, v := range node.Runner.Env { cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) } cmd.Dir = node.Runner.Dir stdout, err := cmd.StdoutPipe() require.NoError(t, err) require.NoError(t, cmd.Start()) matches := make(chan logMatch, 10) go func() { scanner := bufio.NewScanner(stdout) for scanner.Scan() { line := scanner.Text() // Check for actual logger field in JSON, not just substring match if strings.Contains(line, `"logger":"cmds/http"`) { matches <- logMatch{slogTestSubsystemCmdsHTTP, line} } if strings.Contains(line, `"logger":"net/identify"`) { matches <- logMatch{slogTestSubsystemNetIdentify, line} } } }() return cmd, matches } // waitForBothSubsystems waits for both native go-log and slog subsystems to appear in logs. waitForBothSubsystems := func(t *testing.T, matches chan logMatch, timeout time.Duration) { t.Helper() seen := make(map[string]struct{}) deadline := time.After(timeout) for len(seen) < 2 { select { case match := <-matches: if _, exists := seen[match.subsystem]; !exists { t.Logf("Found %s log", match.subsystem) seen[match.subsystem] = struct{}{} } case <-deadline: t.Fatalf("Timeout waiting for logs. Seen: %v", seen) } } assert.Contains(t, seen, slogTestSubsystemCmdsHTTP, "should see cmds/http (native go-log)") assert.Contains(t, seen, slogTestSubsystemNetIdentify, "should see net/identify (slog from go-libp2p)") } // triggerIdentifyProtocol connects node1 to node2, triggering net/identify logs. triggerIdentifyProtocol := func(t *testing.T, node1, node2 *harness.Node) { t.Helper() // Get node2's peer ID and address node2ID := node2.PeerID().String() addrsRes := node2.IPFS("id", "-f", "") require.NoError(t, addrsRes.Err) addrs := strings.Split(strings.TrimSpace(addrsRes.Stdout.String()), "\n") require.NotEmpty(t, addrs, "node2 should have at least one address") // Connect node1 to node2 multiaddr := fmt.Sprintf("%s/p2p/%s", addrs[0], node2ID) res := node1.IPFS("swarm", "connect", multiaddr) require.NoError(t, res.Err) } // verifySlogInterop verifies that both native go-log and slog from go-libp2p // appear in ipfs log tail with correct formatting and level control. verifySlogInterop := func(t *testing.T, node1, node2 *harness.Node) { t.Helper() cmd, matches := startLogMonitoring(t, node1) defer func() { _ = cmd.Process.Kill() }() time.Sleep(slogTestLogStartupDelay) // Trigger cmds/http (native go-log) node1.IPFS("version") // Trigger net/identify (slog from go-libp2p) triggerIdentifyProtocol(t, node1, node2) waitForBothSubsystems(t, matches, slogTestLogWaitTimeout) } // This test verifies that go-log's slog bridge works with go-libp2p's gologshim // when log levels are set via GOLOG_LOG_LEVEL environment variable. // It tests both native go-log loggers (cmds/http) and slog-based loggers from // go-libp2p (net/identify), ensuring both types appear in `ipfs log tail`. t.Run("slog interop via env var", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node1 := h.NewNode().Init() node1.Runner.Env["GOLOG_LOG_LEVEL"] = "error,cmds/http=debug,net/identify=debug" node1.StartDaemon() defer node1.StopDaemon() node2 := h.NewNode().Init().StartDaemon() defer node2.StopDaemon() verifySlogInterop(t, node1, node2) }) // This test verifies that go-log's slog bridge works with go-libp2p's gologshim // when log levels are set dynamically via `ipfs log level` CLI commands. // It tests the key feature that SetLogLevel auto-creates level entries for subsystems // that don't exist yet, enabling `ipfs log level net/identify debug` to work even // before the net/identify logger is created. This is critical for slog interop. t.Run("slog interop via CLI", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node1 := h.NewNode().Init().StartDaemon() defer node1.StopDaemon() node2 := h.NewNode().Init().StartDaemon() defer node2.StopDaemon() // Set levels via CLI for both subsystems BEFORE triggering events res := node1.IPFS("log", "level", slogTestSubsystemCmdsHTTP, "debug") require.NoError(t, res.Err) res = node1.IPFS("log", "level", slogTestSubsystemNetIdentify, "debug") require.NoError(t, res.Err) // Auto-creates level entry for slog subsystem verifySlogInterop(t, node1, node2) }) } func getExpectedSubsystems(t *testing.T, node *harness.Node) []string { t.Helper() lsRes := node.IPFS("log", "ls") require.NoError(t, lsRes.Err) expectedSubsystems := SplitLines(lsRes.Stdout.String()) assert.Greater(t, len(expectedSubsystems), 10, "Should have many subsystems") return expectedSubsystems } func parseCLIOutput(t *testing.T, output string) map[string]string { t.Helper() lines := SplitLines(output) actualSubsystems := make(map[string]string) for _, line := range lines { if strings.TrimSpace(line) == "" { continue } parts := strings.Split(line, ": ") assert.Equal(t, 2, len(parts), "Line should have format 'subsystem: level', got: %s", line) assert.NotEmpty(t, parts[0], "Subsystem should not be empty") assert.NotEmpty(t, parts[1], "Level should not be empty") actualSubsystems[parts[0]] = parts[1] } return actualSubsystems } func parseHTTPResponse(t *testing.T, resp *http.Response) map[string]any { t.Helper() var result map[string]any err := json.NewDecoder(resp.Body).Decode(&result) require.NoError(t, err) levels, ok := result["Levels"].(map[string]any) require.True(t, ok, "Response should have 'Levels' field") assert.Greater(t, len(levels), 10, "Should have many subsystems") return levels } func validateAllSubsystemsPresent(t *testing.T, expectedSubsystems []string, actualLevels map[string]any, context string) { t.Helper() for _, expectedSub := range expectedSubsystems { expectedSub = strings.TrimSpace(expectedSub) if expectedSub == "" { continue } _, found := actualLevels[expectedSub] assert.True(t, found, "Expected subsystem '%s' should be present in %s", expectedSub, context) } } func validateAllSubsystemsPresentCLI(t *testing.T, expectedSubsystems []string, actualLevels map[string]string, context string) { t.Helper() for _, expectedSub := range expectedSubsystems { expectedSub = strings.TrimSpace(expectedSub) if expectedSub == "" { continue } _, found := actualLevels[expectedSub] assert.True(t, found, "Expected subsystem '%s' should be present in %s", expectedSub, context) } } ================================================ FILE: test/cli/ls_test.go ================================================ package cli import ( "os" "path/filepath" "strings" "testing" "time" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestLsLongFormat(t *testing.T) { t.Parallel() t.Run("long format shows mode and mtime when preserved", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Create a test directory structure with known permissions testDir := filepath.Join(node.Dir, "testdata") require.NoError(t, os.MkdirAll(testDir, 0755)) // Create files with specific permissions file1 := filepath.Join(testDir, "readable.txt") require.NoError(t, os.WriteFile(file1, []byte("hello"), 0644)) file2 := filepath.Join(testDir, "executable.sh") require.NoError(t, os.WriteFile(file2, []byte("#!/bin/sh\necho hi"), 0755)) // Set a known mtime in the past (to get year format, avoiding flaky time-based tests) oldTime := time.Date(2020, time.June, 15, 10, 30, 0, 0, time.UTC) require.NoError(t, os.Chtimes(file1, oldTime, oldTime)) require.NoError(t, os.Chtimes(file2, oldTime, oldTime)) // Add with preserved mode and mtime addRes := node.IPFS("add", "-r", "--preserve-mode", "--preserve-mtime", "-Q", testDir) dirCid := addRes.Stdout.Trimmed() // Run ls with --long flag lsRes := node.IPFS("ls", "--long", dirCid) output := lsRes.Stdout.String() // Verify format: Mode Hash Size ModTime Name lines := strings.Split(strings.TrimSpace(output), "\n") require.Len(t, lines, 2, "expected 2 files in output") // Check executable.sh line (should be first alphabetically) assert.Contains(t, lines[0], "-rwxr-xr-x", "executable should have 755 permissions") assert.Contains(t, lines[0], "Jun 15 2020", "should show mtime with year format") assert.Contains(t, lines[0], "executable.sh", "should show filename") // Check readable.txt line assert.Contains(t, lines[1], "-rw-r--r--", "readable file should have 644 permissions") assert.Contains(t, lines[1], "Jun 15 2020", "should show mtime with year format") assert.Contains(t, lines[1], "readable.txt", "should show filename") }) t.Run("long format shows dash for files without preserved mode or mtime", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Create and add a file without --preserve-mode or --preserve-mtime testFile := filepath.Join(node.Dir, "nopreserve.txt") require.NoError(t, os.WriteFile(testFile, []byte("test content"), 0644)) addRes := node.IPFS("add", "-Q", testFile) fileCid := addRes.Stdout.Trimmed() // Create a wrapper directory to list node.IPFS("files", "mkdir", "/testdir") node.IPFS("files", "cp", "/ipfs/"+fileCid, "/testdir/file.txt") statRes := node.IPFS("files", "stat", "--hash", "/testdir") dirCid := statRes.Stdout.Trimmed() // Run ls with --long flag lsRes := node.IPFS("ls", "--long", dirCid) output := lsRes.Stdout.String() // Files without preserved mode or mtime should show "-" for both columns // Format: "-" (mode) "-" (mtime) assert.Regexp(t, `^-\s+\S+\s+\d+\s+-\s+`, output, "missing mode and mtime should both show dash") }) t.Run("long format with headers shows correct column order", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Create a simple test file testDir := filepath.Join(node.Dir, "headertest") require.NoError(t, os.MkdirAll(testDir, 0755)) testFile := filepath.Join(testDir, "file.txt") require.NoError(t, os.WriteFile(testFile, []byte("hello"), 0644)) oldTime := time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC) require.NoError(t, os.Chtimes(testFile, oldTime, oldTime)) addRes := node.IPFS("add", "-r", "--preserve-mode", "--preserve-mtime", "-Q", testDir) dirCid := addRes.Stdout.Trimmed() // Run ls with --long and --headers (--size defaults to true) lsRes := node.IPFS("ls", "--long", "--headers", dirCid) output := lsRes.Stdout.String() lines := strings.Split(strings.TrimSpace(output), "\n") // First line should be headers in correct order: Mode Hash Size ModTime Name require.GreaterOrEqual(t, len(lines), 2) headerFields := strings.Fields(lines[0]) require.Len(t, headerFields, 5, "header should have 5 columns") assert.Equal(t, "Mode", headerFields[0]) assert.Equal(t, "Hash", headerFields[1]) assert.Equal(t, "Size", headerFields[2]) assert.Equal(t, "ModTime", headerFields[3]) assert.Equal(t, "Name", headerFields[4]) // Data line should have matching columns dataFields := strings.Fields(lines[1]) require.GreaterOrEqual(t, len(dataFields), 5) assert.Regexp(t, `^-[rwx-]{9}$`, dataFields[0], "first field should be mode") assert.Regexp(t, `^Qm`, dataFields[1], "second field should be CID") assert.Regexp(t, `^\d+$`, dataFields[2], "third field should be size") }) t.Run("long format with headers and size=false", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := filepath.Join(node.Dir, "headertest2") require.NoError(t, os.MkdirAll(testDir, 0755)) testFile := filepath.Join(testDir, "file.txt") require.NoError(t, os.WriteFile(testFile, []byte("hello"), 0644)) oldTime := time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC) require.NoError(t, os.Chtimes(testFile, oldTime, oldTime)) addRes := node.IPFS("add", "-r", "--preserve-mode", "--preserve-mtime", "-Q", testDir) dirCid := addRes.Stdout.Trimmed() // Run ls with --long --headers --size=false lsRes := node.IPFS("ls", "--long", "--headers", "--size=false", dirCid) output := lsRes.Stdout.String() lines := strings.Split(strings.TrimSpace(output), "\n") // Header should be: Mode Hash ModTime Name (no Size) require.GreaterOrEqual(t, len(lines), 2) headerFields := strings.Fields(lines[0]) require.Len(t, headerFields, 4, "header should have 4 columns without size") assert.Equal(t, "Mode", headerFields[0]) assert.Equal(t, "Hash", headerFields[1]) assert.Equal(t, "ModTime", headerFields[2]) assert.Equal(t, "Name", headerFields[3]) }) t.Run("long format for directories shows trailing slash", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() // Create nested directory structure testDir := filepath.Join(node.Dir, "dirtest") subDir := filepath.Join(testDir, "subdir") require.NoError(t, os.MkdirAll(subDir, 0755)) require.NoError(t, os.WriteFile(filepath.Join(subDir, "file.txt"), []byte("hi"), 0644)) addRes := node.IPFS("add", "-r", "--preserve-mode", "-Q", testDir) dirCid := addRes.Stdout.Trimmed() // Run ls with --long flag lsRes := node.IPFS("ls", "--long", dirCid) output := lsRes.Stdout.String() // Directory should end with / assert.Contains(t, output, "subdir/", "directory should have trailing slash") // Directory should show 'd' in mode assert.Contains(t, output, "drwxr-xr-x", "directory should show directory mode") }) t.Run("long format without size flag", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := filepath.Join(node.Dir, "nosizetest") require.NoError(t, os.MkdirAll(testDir, 0755)) testFile := filepath.Join(testDir, "file.txt") require.NoError(t, os.WriteFile(testFile, []byte("hello world"), 0644)) oldTime := time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC) require.NoError(t, os.Chtimes(testFile, oldTime, oldTime)) addRes := node.IPFS("add", "-r", "--preserve-mode", "--preserve-mtime", "-Q", testDir) dirCid := addRes.Stdout.Trimmed() // Run ls with --long but --size=false lsRes := node.IPFS("ls", "--long", "--size=false", dirCid) output := lsRes.Stdout.String() // Should still have mode and mtime, but format differs (no size column) assert.Contains(t, output, "-rw-r--r--") assert.Contains(t, output, "Jan 01 2020") assert.Contains(t, output, "file.txt") }) t.Run("long format output is stable", func(t *testing.T) { // This test ensures the output format doesn't change due to refactors t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() testDir := filepath.Join(node.Dir, "stabletest") require.NoError(t, os.MkdirAll(testDir, 0755)) testFile := filepath.Join(testDir, "test.txt") require.NoError(t, os.WriteFile(testFile, []byte("stable"), 0644)) // Use a fixed time for reproducibility fixedTime := time.Date(2020, time.December, 25, 12, 0, 0, 0, time.UTC) require.NoError(t, os.Chtimes(testFile, fixedTime, fixedTime)) addRes := node.IPFS("add", "-r", "--preserve-mode", "--preserve-mtime", "-Q", testDir) dirCid := addRes.Stdout.Trimmed() // The CID should be deterministic given same content, mode, and mtime // This is the expected CID for this specific test data lsRes := node.IPFS("ls", "--long", dirCid) output := strings.TrimSpace(lsRes.Stdout.String()) // Verify the format: ModeHashSizeModTimeName fields := strings.Fields(output) require.GreaterOrEqual(t, len(fields), 5, "output should have at least 5 fields") // Field 0: mode (10 chars, starts with - for regular file) assert.Regexp(t, `^-[rwx-]{9}$`, fields[0], "mode should be Unix permission format") // Field 1: CID (starts with Qm or bafy) assert.Regexp(t, `^(Qm|bafy)`, fields[1], "second field should be CID") // Field 2: size (numeric) assert.Regexp(t, `^\d+$`, fields[2], "third field should be numeric size") // Fields 3-4: date (e.g., "Dec 25 2020" or "Dec 25 12:00") // The date format is "Mon DD YYYY" for old files assert.Equal(t, "Dec", fields[3]) assert.Equal(t, "25", fields[4]) // Last field: filename assert.Equal(t, "test.txt", fields[len(fields)-1]) }) } ================================================ FILE: test/cli/migrations/migration_16_to_latest_test.go ================================================ package migrations // NOTE: These migration tests require the local Kubo binary (built with 'make build') to be in PATH. // // To run these tests successfully: // export PATH="$(pwd)/cmd/ipfs:$PATH" // go test ./test/cli/migrations/ import ( "bufio" "context" "encoding/json" "fmt" "io" "os" "os/exec" "path/filepath" "strings" "testing" "time" ipfs "github.com/ipfs/kubo" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // TestMigration16ToLatest tests migration from repo version 16 to the latest version. // // This test uses a real IPFS repository snapshot from Kubo v0.36.0 (the last version that used repo v16). // The intention is to confirm that users can upgrade from Kubo v0.36.0 to the latest version by applying // all intermediate migrations successfully. // // NOTE: This test comprehensively tests all migration methods (daemon --migrate, repo migrate, // and reverse migration) because 16-to-17 was the first embedded migration that did not fetch // external files. It serves as a reference implementation for migration testing. // // Future migrations can have simplified tests (like 17-to-18 in migration_17_to_latest_test.go) // that focus on specific migration logic rather than testing all migration methods. // // If you need to test migration of configuration keys that appeared in later repo versions, // create a new test file migration_N_to_latest_test.go with a separate IPFS repository test vector // from the appropriate Kubo version. func TestMigration16ToLatest(t *testing.T) { t.Parallel() // Primary tests using 'ipfs daemon --migrate' command (default in Docker) t.Run("daemon migrate: forward migration with auto values", testDaemonMigrationWithAuto) t.Run("daemon migrate: forward migration without auto values", testDaemonMigrationWithoutAuto) t.Run("daemon migrate: corrupted config handling", testDaemonCorruptedConfigHandling) t.Run("daemon migrate: missing fields handling", testDaemonMissingFieldsHandling) // Comparison tests using 'ipfs repo migrate' command t.Run("repo migrate: forward migration with auto values", testRepoMigrationWithAuto) t.Run("repo migrate: backward migration", testRepoBackwardMigration) // Temp file and backup cleanup tests t.Run("daemon migrate: no temp files after successful migration", testNoTempFilesAfterSuccessfulMigration) t.Run("daemon migrate: no temp files after failed migration", testNoTempFilesAfterFailedMigration) t.Run("daemon migrate: backup files persist after successful migration", testBackupFilesPersistAfterSuccessfulMigration) t.Run("repo migrate: backup files can revert migration", testBackupFilesCanRevertMigration) t.Run("repo migrate: conversion failure cleans up temp files", testConversionFailureCleanup) } // ============================================================================= // PRIMARY TESTS: 'ipfs daemon --migrate' command (default in Docker) // // These tests exercise the primary migration path used in production Docker // containers where --migrate is enabled by default. This covers: // - Normal forward migration scenarios // - Error handling with corrupted configs // - Migration with minimal/missing config fields // ============================================================================= func testDaemonMigrationWithAuto(t *testing.T) { // TEST: Forward migration using 'ipfs daemon --migrate' command (PRIMARY) // Use static v16 repo fixture from real Kubo 0.36 `ipfs init` // NOTE: This test may need to be revised/updated once repo version 18 is released, // at that point only keep tests that use 'ipfs repo migrate' node := setupStaticV16Repo(t) configPath := filepath.Join(node.Dir, "config") versionPath := filepath.Join(node.Dir, "version") // Static fixture already uses port 0 for random port assignment - no config update needed // Run migration using daemon --migrate (automatic during daemon startup) // This is the primary method used in Docker containers // Monitor output until daemon is ready, then shut it down gracefully stdoutOutput, migrationSuccess := runDaemonMigrationWithMonitoring(t, node) // Debug: Print the actual output t.Logf("Daemon output:\n%s", stdoutOutput) // Verify migration was successful based on monitoring require.True(t, migrationSuccess, "Migration should have been successful") require.Contains(t, stdoutOutput, "applying 16-to-17 repo migration", "Migration should have been triggered") require.Contains(t, stdoutOutput, "Migration 16-to-17 succeeded", "Migration should have completed successfully") // Verify version was updated to latest versionData, err := os.ReadFile(versionPath) require.NoError(t, err) expectedVersion := fmt.Sprint(ipfs.RepoVersion) require.Equal(t, expectedVersion, strings.TrimSpace(string(versionData)), "Version should be updated to %s (latest)", expectedVersion) // Verify migration results using DRY helper helper := NewMigrationTestHelper(t, configPath) helper.RequireAutoConfDefaults(). RequireArrayContains("Bootstrap", "auto"). RequireArrayLength("Bootstrap", 1). // Should only contain "auto" when all peers were defaults RequireArrayContains("Routing.DelegatedRouters", "auto"). RequireArrayContains("Ipns.DelegatedPublishers", "auto") // DNS resolver in static fixture should be empty, so "." should be set to "auto" helper.RequireFieldEquals("DNS.Resolvers[.]", "auto") } func testDaemonMigrationWithoutAuto(t *testing.T) { // TEST: Forward migration using 'ipfs daemon --migrate' command (PRIMARY) // Test migration of a config that already has some custom values // NOTE: This test may need to be revised/updated once repo version 18 is released, // at that point only keep tests that use 'ipfs repo migrate' // Should preserve existing settings and only add missing ones node := setupStaticV16Repo(t) // Modify the static fixture to add some custom values for testing mixed scenarios configPath := filepath.Join(node.Dir, "config") // Read existing config from static fixture var v16Config map[string]any configData, err := os.ReadFile(configPath) require.NoError(t, err) require.NoError(t, json.Unmarshal(configData, &v16Config)) // Add custom DNS resolver that should be preserved if v16Config["DNS"] == nil { v16Config["DNS"] = map[string]any{} } dnsSection := v16Config["DNS"].(map[string]any) dnsSection["Resolvers"] = map[string]string{ ".": "https://custom-dns.example.com/dns-query", "eth.": "https://dns.eth.limo/dns-query", // This is a default that will be replaced with "auto" } // Write modified config back modifiedConfigData, err := json.MarshalIndent(v16Config, "", " ") require.NoError(t, err) require.NoError(t, os.WriteFile(configPath, modifiedConfigData, 0644)) // Static fixture already uses port 0 for random port assignment - no config update needed // Run migration using daemon --migrate command (this is a daemon test) // Monitor output until daemon is ready, then shut it down gracefully stdoutOutput, migrationSuccess := runDaemonMigrationWithMonitoring(t, node) // Verify migration was successful based on monitoring require.True(t, migrationSuccess, "Migration should have been successful") require.Contains(t, stdoutOutput, "applying 16-to-17 repo migration", "Migration should have been triggered") require.Contains(t, stdoutOutput, "Migration 16-to-17 succeeded", "Migration should have completed successfully") // Verify migration results: custom values preserved alongside "auto" helper := NewMigrationTestHelper(t, configPath) helper.RequireAutoConfDefaults(). RequireArrayContains("Bootstrap", "auto"). RequireFieldEquals("DNS.Resolvers[.]", "https://custom-dns.example.com/dns-query") // Check that eth. resolver was replaced with "auto" since it uses a default URL helper.RequireFieldEquals("DNS.Resolvers[eth.]", "auto"). RequireFieldEquals("DNS.Resolvers[.]", "https://custom-dns.example.com/dns-query") } // ============================================================================= // Tests using 'ipfs daemon --migrate' command // ============================================================================= // Test helper structs and functions for cleaner, more DRY tests type ConfigField struct { Path string Expected any Message string } type MigrationTestHelper struct { t *testing.T config map[string]any } func NewMigrationTestHelper(t *testing.T, configPath string) *MigrationTestHelper { var config map[string]any configData, err := os.ReadFile(configPath) require.NoError(t, err) require.NoError(t, json.Unmarshal(configData, &config)) return &MigrationTestHelper{t: t, config: config} } func (h *MigrationTestHelper) RequireFieldExists(path string) *MigrationTestHelper { value := h.getNestedValue(path) require.NotNil(h.t, value, "Field %s should exist", path) return h } func (h *MigrationTestHelper) RequireFieldEquals(path string, expected any) *MigrationTestHelper { value := h.getNestedValue(path) require.Equal(h.t, expected, value, "Field %s should equal %v", path, expected) return h } func (h *MigrationTestHelper) RequireArrayContains(path string, expected any) *MigrationTestHelper { value := h.getNestedValue(path) require.IsType(h.t, []any{}, value, "Field %s should be an array", path) array := value.([]any) require.Contains(h.t, array, expected, "Array %s should contain %v", path, expected) return h } func (h *MigrationTestHelper) RequireArrayLength(path string, expectedLen int) *MigrationTestHelper { value := h.getNestedValue(path) require.IsType(h.t, []any{}, value, "Field %s should be an array", path) array := value.([]any) require.Len(h.t, array, expectedLen, "Array %s should have length %d", path, expectedLen) return h } func (h *MigrationTestHelper) RequireArrayDoesNotContain(path string, notExpected any) *MigrationTestHelper { value := h.getNestedValue(path) require.IsType(h.t, []any{}, value, "Field %s should be an array", path) array := value.([]any) require.NotContains(h.t, array, notExpected, "Array %s should not contain %v", path, notExpected) return h } func (h *MigrationTestHelper) RequireFieldAbsent(path string) *MigrationTestHelper { value := h.getNestedValue(path) require.Nil(h.t, value, "Field %s should not exist", path) return h } func (h *MigrationTestHelper) RequireAutoConfDefaults() *MigrationTestHelper { // AutoConf section should exist but be empty (using implicit defaults) return h.RequireFieldExists("AutoConf"). RequireFieldAbsent("AutoConf.Enabled"). // Should use implicit default (true) RequireFieldAbsent("AutoConf.URL"). // Should use implicit default (mainnet URL) RequireFieldAbsent("AutoConf.RefreshInterval"). // Should use implicit default (24h) RequireFieldAbsent("AutoConf.TLSInsecureSkipVerify") // Should use implicit default (false) } func (h *MigrationTestHelper) RequireAutoFieldsSetToAuto() *MigrationTestHelper { return h.RequireArrayContains("Bootstrap", "auto"). RequireFieldEquals("DNS.Resolvers[.]", "auto"). RequireArrayContains("Routing.DelegatedRouters", "auto"). RequireArrayContains("Ipns.DelegatedPublishers", "auto") } func (h *MigrationTestHelper) RequireNoAutoValues() *MigrationTestHelper { // Check Bootstrap if it exists if h.getNestedValue("Bootstrap") != nil { h.RequireArrayDoesNotContain("Bootstrap", "auto") } // Check DNS.Resolvers if it exists if h.getNestedValue("DNS.Resolvers") != nil { h.RequireMapDoesNotContainValue("DNS.Resolvers", "auto") } // Check Routing.DelegatedRouters if it exists if h.getNestedValue("Routing.DelegatedRouters") != nil { h.RequireArrayDoesNotContain("Routing.DelegatedRouters", "auto") } // Check Ipns.DelegatedPublishers if it exists if h.getNestedValue("Ipns.DelegatedPublishers") != nil { h.RequireArrayDoesNotContain("Ipns.DelegatedPublishers", "auto") } return h } func (h *MigrationTestHelper) RequireMapDoesNotContainValue(path string, notExpected any) *MigrationTestHelper { value := h.getNestedValue(path) require.IsType(h.t, map[string]any{}, value, "Field %s should be a map", path) mapValue := value.(map[string]any) for k, v := range mapValue { require.NotEqual(h.t, notExpected, v, "Map %s[%s] should not equal %v", path, k, notExpected) } return h } func (h *MigrationTestHelper) getNestedValue(path string) any { segments := h.parseKuboConfigPath(path) current := any(h.config) for _, segment := range segments { switch segment.Type { case "field": switch v := current.(type) { case map[string]any: current = v[segment.Key] default: return nil } case "mapKey": switch v := current.(type) { case map[string]any: current = v[segment.Key] default: return nil } default: return nil } if current == nil { return nil } } return current } type PathSegment struct { Type string // "field" or "mapKey" Key string } func (h *MigrationTestHelper) parseKuboConfigPath(path string) []PathSegment { var segments []PathSegment // Split path into parts, respecting bracket boundaries parts := h.splitKuboConfigPath(path) for _, part := range parts { if strings.Contains(part, "[") && strings.HasSuffix(part, "]") { // Handle field[key] notation bracketStart := strings.Index(part, "[") fieldName := part[:bracketStart] mapKey := part[bracketStart+1 : len(part)-1] // Remove [ and ] // Add field segment if present if fieldName != "" { segments = append(segments, PathSegment{Type: "field", Key: fieldName}) } // Add map key segment segments = append(segments, PathSegment{Type: "mapKey", Key: mapKey}) } else { // Regular field access if part != "" { segments = append(segments, PathSegment{Type: "field", Key: part}) } } } return segments } // splitKuboConfigPath splits a path on dots, but preserves bracket sections intact func (h *MigrationTestHelper) splitKuboConfigPath(path string) []string { var parts []string var current strings.Builder inBrackets := false for _, r := range path { switch r { case '[': inBrackets = true current.WriteRune(r) case ']': inBrackets = false current.WriteRune(r) case '.': if inBrackets { // Inside brackets, preserve the dot current.WriteRune(r) } else { // Outside brackets, split here if current.Len() > 0 { parts = append(parts, current.String()) current.Reset() } } default: current.WriteRune(r) } } // Add final part if any if current.Len() > 0 { parts = append(parts, current.String()) } return parts } // setupStaticV16Repo creates a test node using static v16 repo fixture from real Kubo 0.36 `ipfs init` // This ensures tests remain stable regardless of future changes to the IPFS binary // Each test gets its own copy in a temporary directory to allow modifications func setupStaticV16Repo(t *testing.T) *harness.Node { // Get absolute path to static v16 repo fixture v16FixturePath := "testdata/v16-repo" // Create a temporary test directory - each test gets its own copy // Sanitize test name for Windows - replace invalid characters sanitizedName := strings.Map(func(r rune) rune { if strings.ContainsRune(`<>:"/\|?*`, r) { return '_' } return r }, t.Name()) tmpDir := filepath.Join(t.TempDir(), "migration-test-"+sanitizedName) require.NoError(t, os.MkdirAll(tmpDir, 0755)) // Convert to absolute path for harness absTmpDir, err := filepath.Abs(tmpDir) require.NoError(t, err) // Use the built binary (should be in PATH) node := harness.BuildNode("ipfs", absTmpDir, 0) // Replace IPFS_PATH with static fixture files to test directory (creates independent copy per test) cloneStaticRepoFixture(t, v16FixturePath, node.Dir) return node } // cloneStaticRepoFixture recursively copies the v16 repo fixture to the target directory // It completely removes the target directory contents before copying to ensure no extra files remain func cloneStaticRepoFixture(t *testing.T, srcPath, dstPath string) { srcInfo, err := os.Stat(srcPath) require.NoError(t, err) if srcInfo.IsDir() { // Completely remove destination directory and all contents require.NoError(t, os.RemoveAll(dstPath)) // Create fresh destination directory require.NoError(t, os.MkdirAll(dstPath, srcInfo.Mode())) // Read source directory entries, err := os.ReadDir(srcPath) require.NoError(t, err) // Copy each entry recursively for _, entry := range entries { srcEntryPath := filepath.Join(srcPath, entry.Name()) dstEntryPath := filepath.Join(dstPath, entry.Name()) cloneStaticRepoFixture(t, srcEntryPath, dstEntryPath) } } else { // Copy file (destination directory should already be clean from parent call) srcFile, err := os.Open(srcPath) require.NoError(t, err) defer srcFile.Close() dstFile, err := os.Create(dstPath) require.NoError(t, err) defer dstFile.Close() _, err = io.Copy(dstFile, srcFile) require.NoError(t, err) // Copy file permissions require.NoError(t, dstFile.Chmod(srcInfo.Mode())) } } // Placeholder stubs for new test functions - to be implemented func testDaemonCorruptedConfigHandling(t *testing.T) { // TEST: Error handling using 'ipfs daemon --migrate' command with corrupted config (PRIMARY) // Test what happens when config file is corrupted during migration // NOTE: This test may need to be revised/updated once repo version 18 is released, // at that point only keep tests that use 'ipfs repo migrate' node := setupStaticV16Repo(t) // Create corrupted config configPath := filepath.Join(node.Dir, "config") corruptedJson := `{"Bootstrap": [invalid json}` require.NoError(t, os.WriteFile(configPath, []byte(corruptedJson), 0644)) // Write version file indicating v16 versionPath := filepath.Join(node.Dir, "version") require.NoError(t, os.WriteFile(versionPath, []byte("16"), 0644)) // Run daemon with --migrate flag - this should fail gracefully result := node.RunIPFS("daemon", "--migrate") // Verify graceful failure handling // The daemon should fail but migration error should be clear errorOutput := result.Stderr.String() + result.Stdout.String() require.True(t, strings.Contains(errorOutput, "json") || strings.Contains(errorOutput, "invalid character"), "Error should mention JSON parsing issue") // Verify atomic failure: version and config should remain unchanged versionData, err := os.ReadFile(versionPath) require.NoError(t, err) require.Equal(t, "16", strings.TrimSpace(string(versionData)), "Version should remain unchanged after failed migration") originalContent, err := os.ReadFile(configPath) require.NoError(t, err) require.Equal(t, corruptedJson, string(originalContent), "Original config should be unchanged after failed migration") } func testDaemonMissingFieldsHandling(t *testing.T) { // TEST: Migration using 'ipfs daemon --migrate' command with minimal config (PRIMARY) // Test migration when config is missing expected fields // NOTE: This test may need to be revised/updated once repo version 18 is released, // at that point only keep tests that use 'ipfs repo migrate' node := setupStaticV16Repo(t) // The static fixture already has all required fields, use it as-is configPath := filepath.Join(node.Dir, "config") versionPath := filepath.Join(node.Dir, "version") // Static fixture already uses port 0 for random port assignment - no config update needed // Run daemon migration stdoutOutput, migrationSuccess := runDaemonMigrationWithMonitoring(t, node) // Verify migration was successful require.True(t, migrationSuccess, "Migration should have been successful") require.Contains(t, stdoutOutput, "applying 16-to-17 repo migration", "Migration should have been triggered") require.Contains(t, stdoutOutput, "Migration 16-to-17 succeeded", "Migration should have completed successfully") // Verify version was updated to latest versionData, err := os.ReadFile(versionPath) require.NoError(t, err) expectedVersion := fmt.Sprint(ipfs.RepoVersion) require.Equal(t, expectedVersion, strings.TrimSpace(string(versionData)), "Version should be updated to %s (latest)", expectedVersion) // Verify migration adds all required fields to minimal config NewMigrationTestHelper(t, configPath). RequireAutoConfDefaults(). RequireAutoFieldsSetToAuto(). RequireFieldExists("Identity.PeerID") // Original identity preserved from static fixture } // ============================================================================= // COMPARISON TESTS: 'ipfs repo migrate' command // // These tests verify that repo migrate produces equivalent results to // daemon migrate, and test scenarios specific to repo migrate like // backward migration (which daemon doesn't support). // ============================================================================= func testRepoMigrationWithAuto(t *testing.T) { // TEST: Forward migration using 'ipfs repo migrate' command (COMPARISON) // Simple comparison test to verify repo migrate produces same results as daemon migrate node := setupStaticV16Repo(t) // Use static fixture as-is configPath := filepath.Join(node.Dir, "config") // Run migration using 'ipfs repo migrate' command result := node.RunIPFS("repo", "migrate") require.Empty(t, result.Stderr.String(), "Migration should succeed without errors") // Verify same results as daemon migrate helper := NewMigrationTestHelper(t, configPath) helper.RequireAutoConfDefaults(). RequireArrayContains("Bootstrap", "auto"). RequireArrayContains("Routing.DelegatedRouters", "auto"). RequireArrayContains("Ipns.DelegatedPublishers", "auto"). RequireFieldEquals("DNS.Resolvers[.]", "auto") } func testRepoBackwardMigration(t *testing.T) { // TEST: Backward migration using 'ipfs repo migrate --to=16 --allow-downgrade' command // This is kept as repo migrate since daemon doesn't support backward migration node := setupStaticV16Repo(t) // Use static fixture as-is configPath := filepath.Join(node.Dir, "config") versionPath := filepath.Join(node.Dir, "version") // First run forward migration to get to v17 result := node.RunIPFS("repo", "migrate") t.Logf("Forward migration stdout:\n%s", result.Stdout.String()) t.Logf("Forward migration stderr:\n%s", result.Stderr.String()) require.Empty(t, result.Stderr.String(), "Forward migration should succeed") // Verify we're at the latest version versionData, err := os.ReadFile(versionPath) require.NoError(t, err) expectedVersion := fmt.Sprint(ipfs.RepoVersion) require.Equal(t, expectedVersion, strings.TrimSpace(string(versionData)), "Should be at version %s (latest) after forward migration", expectedVersion) // Now run reverse migration back to v16 result = node.RunIPFS("repo", "migrate", "--to=16", "--allow-downgrade") t.Logf("Backward migration stdout:\n%s", result.Stdout.String()) t.Logf("Backward migration stderr:\n%s", result.Stderr.String()) require.Empty(t, result.Stderr.String(), "Reverse migration should succeed") // Verify version was downgraded to 16 versionData, err = os.ReadFile(versionPath) require.NoError(t, err) require.Equal(t, "16", strings.TrimSpace(string(versionData)), "Version should be downgraded to 16") // Verify backward migration results: AutoConf removed and no "auto" values remain NewMigrationTestHelper(t, configPath). RequireFieldAbsent("AutoConf"). RequireNoAutoValues() } // runDaemonMigrationWithMonitoring starts daemon --migrate, monitors output until "Daemon is ready", // then gracefully shuts down the daemon and returns the captured output and success status. // This monitors for all expected migrations from version 16 to latest. func runDaemonMigrationWithMonitoring(t *testing.T, node *harness.Node) (string, bool) { // Monitor migrations from repo v16 to latest return runDaemonWithExpectedMigrations(t, node, 16, ipfs.RepoVersion) } // runDaemonWithExpectedMigrations monitors daemon startup for a sequence of migrations from startVersion to endVersion func runDaemonWithExpectedMigrations(t *testing.T, node *harness.Node, startVersion, endVersion int) (string, bool) { // Build list of expected migrations var expectedMigrations []struct { pattern string success string } for v := startVersion; v < endVersion; v++ { from := v to := v + 1 expectedMigrations = append(expectedMigrations, struct { pattern string success string }{ pattern: fmt.Sprintf("applying %d-to-%d repo migration", from, to), success: fmt.Sprintf("Migration %d-to-%d succeeded", from, to), }) } return runDaemonWithMultipleMigrationMonitoring(t, node, expectedMigrations) } // runDaemonWithMultipleMigrationMonitoring monitors daemon startup for multiple sequential migrations func runDaemonWithMultipleMigrationMonitoring(t *testing.T, node *harness.Node, expectedMigrations []struct { pattern string success string }) (string, bool) { // Create context with timeout as safety net ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() // Set up daemon command with output monitoring cmd := exec.CommandContext(ctx, node.IPFSBin, "daemon", "--migrate") cmd.Dir = node.Dir // Set environment (especially IPFS_PATH) for k, v := range node.Runner.Env { cmd.Env = append(cmd.Env, k+"="+v) } // Set up pipes for output monitoring stdout, err := cmd.StdoutPipe() require.NoError(t, err) stderr, err := cmd.StderrPipe() require.NoError(t, err) // Start the daemon err = cmd.Start() require.NoError(t, err) var allOutput strings.Builder var daemonReady bool // Track which migrations have been detected migrationsDetected := make([]bool, len(expectedMigrations)) migrationsSucceeded := make([]bool, len(expectedMigrations)) // Monitor stdout for completion signals scanner := bufio.NewScanner(stdout) go func() { for scanner.Scan() { line := scanner.Text() allOutput.WriteString(line + "\n") // Check for migration messages for i, migration := range expectedMigrations { if strings.Contains(line, migration.pattern) { migrationsDetected[i] = true } if strings.Contains(line, migration.success) { migrationsSucceeded[i] = true } } if strings.Contains(line, "Daemon is ready") { daemonReady = true break // Exit monitoring loop } } }() // Also monitor stderr (but don't use it for completion detection) go func() { stderrScanner := bufio.NewScanner(stderr) for stderrScanner.Scan() { line := stderrScanner.Text() allOutput.WriteString("STDERR: " + line + "\n") } }() // Wait for daemon ready signal or timeout ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { select { case <-ctx.Done(): // Timeout - kill the process if cmd.Process != nil { _ = cmd.Process.Kill() } t.Logf("Daemon migration timed out after 60 seconds") return allOutput.String(), false case <-ticker.C: if daemonReady { // Daemon is ready - shut it down gracefully shutdownCmd := exec.Command(node.IPFSBin, "shutdown") shutdownCmd.Dir = node.Dir for k, v := range node.Runner.Env { shutdownCmd.Env = append(shutdownCmd.Env, k+"="+v) } if err := shutdownCmd.Run(); err != nil { t.Logf("Warning: ipfs shutdown failed: %v", err) // Force kill if graceful shutdown fails if cmd.Process != nil { _ = cmd.Process.Kill() } } // Wait for process to exit _ = cmd.Wait() // Check all migrations were detected and succeeded allDetected := true allSucceeded := true for i := range expectedMigrations { if !migrationsDetected[i] { allDetected = false t.Logf("Migration %s was not detected", expectedMigrations[i].pattern) } if !migrationsSucceeded[i] { allSucceeded = false t.Logf("Migration %s did not succeed", expectedMigrations[i].success) } } return allOutput.String(), allDetected && allSucceeded } // Check if process has exited (e.g., due to startup failure after migration) if cmd.ProcessState != nil && cmd.ProcessState.Exited() { // Process exited - migration may have completed but daemon failed to start // This is expected for corrupted config tests // Check all migrations status allDetected := true allSucceeded := true for i := range expectedMigrations { if !migrationsDetected[i] { allDetected = false } if !migrationsSucceeded[i] { allSucceeded = false } } return allOutput.String(), allDetected && allSucceeded } } } } // ============================================================================= // TEMP FILE AND BACKUP CLEANUP TESTS // ============================================================================= // Helper functions for test cleanup assertions func assertNoTempFiles(t *testing.T, dir string, msgAndArgs ...any) { t.Helper() tmpFiles, err := filepath.Glob(filepath.Join(dir, ".tmp-*")) require.NoError(t, err) assert.Empty(t, tmpFiles, msgAndArgs...) } func backupPath(configPath string, fromVer, toVer int) string { return fmt.Sprintf("%s.%d-to-%d.bak", configPath, fromVer, toVer) } func setupDaemonCmd(ctx context.Context, node *harness.Node, args ...string) *exec.Cmd { cmd := exec.CommandContext(ctx, node.IPFSBin, args...) cmd.Dir = node.Dir for k, v := range node.Runner.Env { cmd.Env = append(cmd.Env, k+"="+v) } return cmd } func testNoTempFilesAfterSuccessfulMigration(t *testing.T) { node := setupStaticV16Repo(t) // Run successful migration _, migrationSuccess := runDaemonMigrationWithMonitoring(t, node) require.True(t, migrationSuccess, "migration should succeed") assertNoTempFiles(t, node.Dir, "no temp files should remain after successful migration") } func testNoTempFilesAfterFailedMigration(t *testing.T) { node := setupStaticV16Repo(t) // Corrupt config to force migration failure configPath := filepath.Join(node.Dir, "config") corruptedJson := `{"Bootstrap": ["auto",` // Invalid JSON require.NoError(t, os.WriteFile(configPath, []byte(corruptedJson), 0644)) // Attempt migration (should fail) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() cmd := setupDaemonCmd(ctx, node, "daemon", "--migrate") output, _ := cmd.CombinedOutput() t.Logf("Failed migration output: %s", output) assertNoTempFiles(t, node.Dir, "no temp files should remain after failed migration") } func testBackupFilesPersistAfterSuccessfulMigration(t *testing.T) { node := setupStaticV16Repo(t) // Run migration from v16 to latest (v18) _, migrationSuccess := runDaemonMigrationWithMonitoring(t, node) require.True(t, migrationSuccess, "migration should succeed") // Check for backup files from each migration step configPath := filepath.Join(node.Dir, "config") backup16to17 := backupPath(configPath, 16, 17) backup17to18 := backupPath(configPath, 17, 18) // Both backup files should exist assert.FileExists(t, backup16to17, "16-to-17 backup should exist") assert.FileExists(t, backup17to18, "17-to-18 backup should exist") // Verify backup files contain valid JSON data16to17, err := os.ReadFile(backup16to17) require.NoError(t, err) var config16to17 map[string]any require.NoError(t, json.Unmarshal(data16to17, &config16to17), "16-to-17 backup should be valid JSON") data17to18, err := os.ReadFile(backup17to18) require.NoError(t, err) var config17to18 map[string]any require.NoError(t, json.Unmarshal(data17to18, &config17to18), "17-to-18 backup should be valid JSON") } func testBackupFilesCanRevertMigration(t *testing.T) { node := setupStaticV16Repo(t) configPath := filepath.Join(node.Dir, "config") versionPath := filepath.Join(node.Dir, "version") // Read original v16 config originalConfig, err := os.ReadFile(configPath) require.NoError(t, err) // Migrate to v17 only result := node.RunIPFS("repo", "migrate", "--to=17") require.Empty(t, result.Stderr.String(), "migration to v17 should succeed") // Verify backup exists backup16to17 := backupPath(configPath, 16, 17) assert.FileExists(t, backup16to17, "16-to-17 backup should exist") // Manually revert using backup backupData, err := os.ReadFile(backup16to17) require.NoError(t, err) require.NoError(t, os.WriteFile(configPath, backupData, 0600)) require.NoError(t, os.WriteFile(versionPath, []byte("16"), 0644)) // Verify config matches original revertedConfig, err := os.ReadFile(configPath) require.NoError(t, err) assert.JSONEq(t, string(originalConfig), string(revertedConfig), "reverted config should match original") // Verify version is back to 16 versionData, err := os.ReadFile(versionPath) require.NoError(t, err) assert.Equal(t, "16", strings.TrimSpace(string(versionData)), "version should be reverted to 16") } func testConversionFailureCleanup(t *testing.T) { // This test verifies that when a migration's conversion function fails, // all temporary files are cleaned up properly node := setupStaticV16Repo(t) configPath := filepath.Join(node.Dir, "config") // Create a corrupted config that will cause conversion to fail during JSON parsing // The migration will read this, attempt to parse as JSON, and fail corruptedJson := `{"Bootstrap": ["auto",` // Invalid JSON - missing closing bracket require.NoError(t, os.WriteFile(configPath, []byte(corruptedJson), 0644)) // Attempt migration (should fail during conversion) result := node.RunIPFS("repo", "migrate") require.NotEmpty(t, result.Stderr.String(), "migration should fail with error") assertNoTempFiles(t, node.Dir, "no temp files should remain after conversion failure") // Verify no backup files were created (failure happened before backup) backupFiles, err := filepath.Glob(filepath.Join(node.Dir, "config.*.bak")) require.NoError(t, err) assert.Empty(t, backupFiles, "no backup files should be created on conversion failure") // Verify corrupted config is unchanged (atomic operations prevented overwrite) currentConfig, err := os.ReadFile(configPath) require.NoError(t, err) assert.Equal(t, corruptedJson, string(currentConfig), "corrupted config should remain unchanged") } ================================================ FILE: test/cli/migrations/migration_17_to_latest_test.go ================================================ package migrations // NOTE: These migration tests require the local Kubo binary (built with 'make build') to be in PATH. // // To run these tests successfully: // export PATH="$(pwd)/cmd/ipfs:$PATH" // go test ./test/cli/migrations/ import ( "context" "encoding/json" "fmt" "os" "os/exec" "path/filepath" "strings" "testing" "time" ipfs "github.com/ipfs/kubo" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/require" ) // TestMigration17ToLatest tests migration from repo version 17 to the latest version. // // Since we don't have a v17 repo fixture, we start with v16 and migrate it to v17 first, // then test the 17-to-18 migration specifically. // // This test focuses on the Provider/Reprovider to Provide consolidation that happens in 17-to-18. func TestMigration17ToLatest(t *testing.T) { t.Parallel() // Tests for Provider/Reprovider to Provide migration (17-to-18) t.Run("daemon migrate: Provider/Reprovider to Provide consolidation", testProviderReproviderMigration) t.Run("daemon migrate: flat strategy conversion", testFlatStrategyConversion) t.Run("daemon migrate: empty Provider/Reprovider sections", testEmptyProviderReproviderMigration) t.Run("daemon migrate: partial configuration (Provider only)", testProviderOnlyMigration) t.Run("daemon migrate: partial configuration (Reprovider only)", testReproviderOnlyMigration) t.Run("repo migrate: invalid strategy values preserved", testInvalidStrategyMigration) t.Run("repo migrate: Provider/Reprovider to Provide consolidation", testRepoProviderReproviderMigration) } // ============================================================================= // MIGRATION 17-to-18 SPECIFIC TESTS: Provider/Reprovider to Provide consolidation // ============================================================================= func testProviderReproviderMigration(t *testing.T) { // TEST: 17-to-18 migration with explicit Provider/Reprovider configuration node := setupV17RepoWithProviderConfig(t) configPath := filepath.Join(node.Dir, "config") versionPath := filepath.Join(node.Dir, "version") // Run migration using daemon --migrate command stdoutOutput, migrationSuccess := runDaemonMigrationFromV17(t, node) // Debug: Print the actual output t.Logf("Daemon output:\n%s", stdoutOutput) // Verify migration was successful require.True(t, migrationSuccess, "Migration should have been successful") require.Contains(t, stdoutOutput, "applying 17-to-18 repo migration", "Migration 17-to-18 should have been triggered") require.Contains(t, stdoutOutput, "Migration 17-to-18 succeeded", "Migration 17-to-18 should have completed successfully") // Verify version was updated to latest versionData, err := os.ReadFile(versionPath) require.NoError(t, err) expectedVersion := fmt.Sprint(ipfs.RepoVersion) require.Equal(t, expectedVersion, strings.TrimSpace(string(versionData)), "Version should be updated to %s (latest)", expectedVersion) // ============================================================================= // MIGRATION 17-to-18 ASSERTIONS: Provider/Reprovider to Provide consolidation // ============================================================================= helper := NewMigrationTestHelper(t, configPath) // Verify Provider/Reprovider migration to Provide helper.RequireProviderMigration(). RequireFieldEquals("Provide.Enabled", true). // Migrated from Provider.Enabled RequireFieldEquals("Provide.DHT.MaxWorkers", float64(8)). // Migrated from Provider.WorkerCount RequireFieldEquals("Provide.Strategy", "roots"). // Migrated from Reprovider.Strategy RequireFieldEquals("Provide.DHT.Interval", "24h") // Migrated from Reprovider.Interval // Verify old sections are removed helper.RequireFieldAbsent("Provider"). RequireFieldAbsent("Reprovider") } func testFlatStrategyConversion(t *testing.T) { // TEST: 17-to-18 migration with "flat" strategy that should convert to "all" node := setupV17RepoWithFlatStrategy(t) configPath := filepath.Join(node.Dir, "config") // Run migration using daemon --migrate command stdoutOutput, migrationSuccess := runDaemonMigrationFromV17(t, node) // Verify migration was successful require.True(t, migrationSuccess, "Migration should have been successful") require.Contains(t, stdoutOutput, "applying 17-to-18 repo migration", "Migration 17-to-18 should have been triggered") require.Contains(t, stdoutOutput, "Migration 17-to-18 succeeded", "Migration 17-to-18 should have completed successfully") // ============================================================================= // MIGRATION 17-to-18 ASSERTIONS: "flat" to "all" strategy conversion // ============================================================================= helper := NewMigrationTestHelper(t, configPath) // Verify "flat" was converted to "all" helper.RequireProviderMigration(). RequireFieldEquals("Provide.Strategy", "all"). // "flat" converted to "all" RequireFieldEquals("Provide.DHT.Interval", "12h") } func testEmptyProviderReproviderMigration(t *testing.T) { // TEST: 17-to-18 migration with empty Provider and Reprovider sections node := setupV17RepoWithEmptySections(t) configPath := filepath.Join(node.Dir, "config") // Run migration stdoutOutput, migrationSuccess := runDaemonMigrationFromV17(t, node) // Verify migration was successful require.True(t, migrationSuccess, "Migration should have been successful") require.Contains(t, stdoutOutput, "Migration 17-to-18 succeeded") // Verify empty sections are removed and no Provide section is created helper := NewMigrationTestHelper(t, configPath) helper.RequireFieldAbsent("Provider"). RequireFieldAbsent("Reprovider"). RequireFieldAbsent("Provide") // No Provide section should be created for empty configs } func testProviderOnlyMigration(t *testing.T) { // TEST: 17-to-18 migration with only Provider configuration node := setupV17RepoWithProviderOnly(t) configPath := filepath.Join(node.Dir, "config") // Run migration stdoutOutput, migrationSuccess := runDaemonMigrationFromV17(t, node) // Verify migration was successful require.True(t, migrationSuccess, "Migration should have been successful") require.Contains(t, stdoutOutput, "Migration 17-to-18 succeeded") // Verify only Provider fields are migrated helper := NewMigrationTestHelper(t, configPath) helper.RequireProviderMigration(). RequireFieldEquals("Provide.Enabled", false). RequireFieldEquals("Provide.DHT.MaxWorkers", float64(32)). RequireFieldAbsent("Provide.Strategy"). // No Reprovider.Strategy to migrate RequireFieldAbsent("Provide.DHT.Interval") // No Reprovider.Interval to migrate } func testReproviderOnlyMigration(t *testing.T) { // TEST: 17-to-18 migration with only Reprovider configuration node := setupV17RepoWithReproviderOnly(t) configPath := filepath.Join(node.Dir, "config") // Run migration stdoutOutput, migrationSuccess := runDaemonMigrationFromV17(t, node) // Verify migration was successful require.True(t, migrationSuccess, "Migration should have been successful") require.Contains(t, stdoutOutput, "Migration 17-to-18 succeeded") // Verify only Reprovider fields are migrated helper := NewMigrationTestHelper(t, configPath) helper.RequireProviderMigration(). RequireFieldEquals("Provide.Strategy", "pinned"). RequireFieldEquals("Provide.DHT.Interval", "48h"). RequireFieldAbsent("Provide.Enabled"). // No Provider.Enabled to migrate RequireFieldAbsent("Provide.DHT.MaxWorkers") // No Provider.WorkerCount to migrate } func testInvalidStrategyMigration(t *testing.T) { // TEST: 17-to-18 migration with invalid strategy values (should be preserved as-is) // The migration itself should succeed, but daemon start will fail due to invalid strategy node := setupV17RepoWithInvalidStrategy(t) configPath := filepath.Join(node.Dir, "config") // Run the migration using 'ipfs repo migrate' (not daemon --migrate) // because daemon would fail to start with invalid strategy after migration result := node.RunIPFS("repo", "migrate") require.Empty(t, result.Stderr.String(), "Migration should succeed without errors") // Verify invalid strategy is preserved as-is (not validated during migration) helper := NewMigrationTestHelper(t, configPath) helper.RequireProviderMigration(). RequireFieldEquals("Provide.Strategy", "invalid-strategy") // Should be preserved // Now verify that daemon fails to start with invalid strategy // Note: We cannot use --offline as it skips provider validation // Use a context with timeout to avoid hanging ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() cmd := exec.CommandContext(ctx, node.IPFSBin, "daemon") cmd.Dir = node.Dir for k, v := range node.Runner.Env { cmd.Env = append(cmd.Env, k+"="+v) } output, err := cmd.CombinedOutput() // The daemon should fail (either with error or timeout if it's hanging) require.Error(t, err, "Daemon should fail to start with invalid strategy") // Check if we got the expected error message outputStr := string(output) t.Logf("Daemon output with invalid strategy: %s", outputStr) // The error should mention unknown strategy require.Contains(t, outputStr, "unknown strategy", "Should report unknown strategy error") } func testRepoProviderReproviderMigration(t *testing.T) { // TEST: 17-to-18 migration using 'ipfs repo migrate' command node := setupV17RepoWithProviderConfig(t) configPath := filepath.Join(node.Dir, "config") // Run migration using 'ipfs repo migrate' command result := node.RunIPFS("repo", "migrate") require.Empty(t, result.Stderr.String(), "Migration should succeed without errors") // Verify same results as daemon migrate helper := NewMigrationTestHelper(t, configPath) helper.RequireProviderMigration(). RequireFieldEquals("Provide.Enabled", true). RequireFieldEquals("Provide.DHT.MaxWorkers", float64(8)). RequireFieldEquals("Provide.Strategy", "roots"). RequireFieldEquals("Provide.DHT.Interval", "24h") } // ============================================================================= // HELPER FUNCTIONS // ============================================================================= // setupV17RepoWithProviderConfig creates a v17 repo with Provider/Reprovider configuration func setupV17RepoWithProviderConfig(t *testing.T) *harness.Node { return setupV17RepoWithConfig(t, map[string]any{ "Enabled": true, "WorkerCount": 8, }, map[string]any{ "Strategy": "roots", "Interval": "24h", }) } // setupV17RepoWithFlatStrategy creates a v17 repo with "flat" strategy for testing conversion func setupV17RepoWithFlatStrategy(t *testing.T) *harness.Node { return setupV17RepoWithConfig(t, map[string]any{ "Enabled": false, }, map[string]any{ "Strategy": "flat", // This should be converted to "all" "Interval": "12h", }) } // setupV17RepoWithConfig is a helper that creates a v17 repo with specified Provider/Reprovider config func setupV17RepoWithConfig(t *testing.T, providerConfig, reproviderConfig map[string]any) *harness.Node { node := setupStaticV16Repo(t) // First migrate to v17 result := node.RunIPFS("repo", "migrate", "--to=17") require.Empty(t, result.Stderr.String(), "Migration to v17 should succeed") // Update config with specified Provider and Reprovider settings configPath := filepath.Join(node.Dir, "config") var config map[string]any configData, err := os.ReadFile(configPath) require.NoError(t, err) require.NoError(t, json.Unmarshal(configData, &config)) if providerConfig != nil { config["Provider"] = providerConfig } else { config["Provider"] = map[string]any{} } if reproviderConfig != nil { config["Reprovider"] = reproviderConfig } else { config["Reprovider"] = map[string]any{} } modifiedConfigData, err := json.MarshalIndent(config, "", " ") require.NoError(t, err) require.NoError(t, os.WriteFile(configPath, modifiedConfigData, 0644)) return node } // setupV17RepoWithEmptySections creates a v17 repo with empty Provider/Reprovider sections func setupV17RepoWithEmptySections(t *testing.T) *harness.Node { return setupV17RepoWithConfig(t, map[string]any{}, map[string]any{}) } // setupV17RepoWithProviderOnly creates a v17 repo with only Provider configuration func setupV17RepoWithProviderOnly(t *testing.T) *harness.Node { return setupV17RepoWithConfig(t, map[string]any{ "Enabled": false, "WorkerCount": 32, }, map[string]any{}) } // setupV17RepoWithReproviderOnly creates a v17 repo with only Reprovider configuration func setupV17RepoWithReproviderOnly(t *testing.T) *harness.Node { return setupV17RepoWithConfig(t, map[string]any{}, map[string]any{ "Strategy": "pinned", "Interval": "48h", }) } // setupV17RepoWithInvalidStrategy creates a v17 repo with an invalid strategy value func setupV17RepoWithInvalidStrategy(t *testing.T) *harness.Node { return setupV17RepoWithConfig(t, map[string]any{}, map[string]any{ "Strategy": "invalid-strategy", // This is not a valid strategy "Interval": "24h", }) } // runDaemonMigrationFromV17 monitors daemon startup for 17-to-18 migration only func runDaemonMigrationFromV17(t *testing.T, node *harness.Node) (string, bool) { // Monitor only the 17-to-18 migration expectedMigrations := []struct { pattern string success string }{ { pattern: "applying 17-to-18 repo migration", success: "Migration 17-to-18 succeeded", }, } return runDaemonWithMultipleMigrationMonitoring(t, node, expectedMigrations) } // RequireProviderMigration verifies that Provider/Reprovider have been migrated to Provide section func (h *MigrationTestHelper) RequireProviderMigration() *MigrationTestHelper { return h.RequireFieldExists("Provide"). RequireFieldAbsent("Provider"). RequireFieldAbsent("Reprovider") } ================================================ FILE: test/cli/migrations/migration_concurrent_test.go ================================================ package migrations // NOTE: These concurrent migration tests require the local Kubo binary (built with 'make build') to be in PATH. // // To run these tests successfully: // export PATH="$(pwd)/cmd/ipfs:$PATH" // go test ./test/cli/migrations/ import ( "context" "testing" "time" "github.com/stretchr/testify/require" ) const daemonStartupWait = 2 * time.Second // TestConcurrentMigrations tests concurrent daemon --migrate attempts func TestConcurrentMigrations(t *testing.T) { t.Parallel() t.Run("concurrent daemon migrations prevented by lock", testConcurrentDaemonMigrations) } func testConcurrentDaemonMigrations(t *testing.T) { node := setupStaticV16Repo(t) // Start first daemon --migrate in background (holds repo.lock) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() firstDaemon := setupDaemonCmd(ctx, node, "daemon", "--migrate") require.NoError(t, firstDaemon.Start()) defer func() { // Shutdown first daemon shutdownCmd := setupDaemonCmd(context.Background(), node, "shutdown") _ = shutdownCmd.Run() _ = firstDaemon.Wait() }() // Wait for first daemon to start and acquire lock time.Sleep(daemonStartupWait) // Attempt second daemon --migrate (should fail due to lock) secondDaemon := setupDaemonCmd(context.Background(), node, "daemon", "--migrate") output, err := secondDaemon.CombinedOutput() t.Logf("Second daemon output: %s", output) // Should fail with lock error require.Error(t, err, "second daemon should fail when first daemon holds lock") require.Contains(t, string(output), "lock", "error should mention lock") assertNoTempFiles(t, node.Dir, "no temp files should be created when lock fails") } ================================================ FILE: test/cli/migrations/migration_mixed_15_to_latest_test.go ================================================ package migrations // NOTE: These mixed migration tests validate the transition from old Kubo versions that used external // migration binaries to the latest version with embedded migrations. This ensures users can upgrade // from very old installations (v15) to the latest version seamlessly. // // The tests verify hybrid migration paths: // - Forward: external binary (15→16) + embedded migrations (16→latest) // - Backward: embedded migrations (latest→16) + external binary (16→15) // // This confirms compatibility between the old external migration system and the new embedded system. // // To run these tests successfully: // export PATH="$(pwd)/cmd/ipfs:$PATH" // go test ./test/cli/migrations/ import ( "bufio" "context" "encoding/json" "fmt" "io" "os" "os/exec" "path/filepath" "runtime" "slices" "strings" "testing" "time" ipfs "github.com/ipfs/kubo" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/require" ) // TestMixedMigration15ToLatest tests migration from old Kubo (v15 with external migrations) // to the latest version using a hybrid approach: external binary for 15→16, then embedded // migrations for 16→latest. This ensures backward compatibility for users upgrading from // very old Kubo installations. func TestMixedMigration15ToLatest(t *testing.T) { t.Parallel() // Test mixed migration from v15 to latest (combines external 15→16 + embedded 16→latest) t.Run("daemon migrate: mixed 15 to latest", testDaemonMigration15ToLatest) t.Run("repo migrate: mixed 15 to latest", testRepoMigration15ToLatest) } // TestMixedMigrationLatestTo15Downgrade tests downgrading from the latest version back to v15 // using a hybrid approach: embedded migrations for latest→16, then external binary for 16→15. // This ensures the migration system works bidirectionally for recovery scenarios. func TestMixedMigrationLatestTo15Downgrade(t *testing.T) { t.Parallel() // Test reverse hybrid migration from latest to v15 (embedded latest→16 + external 16→15) t.Run("repo migrate: reverse hybrid latest to 15", testRepoReverseHybridMigrationLatestTo15) } func testDaemonMigration15ToLatest(t *testing.T) { // TEST: Migration from v15 to latest using 'ipfs daemon --migrate' // This tests the mixed migration path: external binary (15→16) + embedded (16→latest) node := setupStaticV15Repo(t) // Create mock migration binary for 15→16 (16→17 will use embedded migration) mockBinDir := createMockMigrationBinary(t, "15", "16") customPath := buildCustomPath(mockBinDir) configPath := filepath.Join(node.Dir, "config") versionPath := filepath.Join(node.Dir, "version") // Verify starting conditions versionData, err := os.ReadFile(versionPath) require.NoError(t, err) require.Equal(t, "15", strings.TrimSpace(string(versionData)), "Should start at version 15") // Read original config to verify preservation of key fields var originalConfig map[string]any configData, err := os.ReadFile(configPath) require.NoError(t, err) require.NoError(t, json.Unmarshal(configData, &originalConfig)) originalPeerID := getNestedValue(originalConfig, "Identity.PeerID") // Run dual migration using daemon --migrate stdoutOutput, migrationSuccess := runDaemonWithLegacyMigrationMonitoring(t, node, customPath) // Debug output t.Logf("Daemon output:\n%s", stdoutOutput) // Verify hybrid migration was successful require.True(t, migrationSuccess, "Hybrid migration should have been successful") require.Contains(t, stdoutOutput, "Phase 1: External migration from v15 to v16", "Should detect external migration phase") // Verify each embedded migration step from 16 to latest verifyMigrationSteps(t, stdoutOutput, 16, ipfs.RepoVersion, true) require.Contains(t, stdoutOutput, fmt.Sprintf("Phase 2: Embedded migration from v16 to v%d", ipfs.RepoVersion), "Should detect embedded migration phase") require.Contains(t, stdoutOutput, "Hybrid migration completed successfully", "Should confirm hybrid migration completion") // Verify final version is latest versionData, err = os.ReadFile(versionPath) require.NoError(t, err) latestVersion := fmt.Sprintf("%d", ipfs.RepoVersion) require.Equal(t, latestVersion, strings.TrimSpace(string(versionData)), "Version should be updated to latest") // Verify config is still valid JSON and key fields preserved var finalConfig map[string]any configData, err = os.ReadFile(configPath) require.NoError(t, err) require.NoError(t, json.Unmarshal(configData, &finalConfig), "Config should remain valid JSON") // Verify essential fields preserved finalPeerID := getNestedValue(finalConfig, "Identity.PeerID") require.Equal(t, originalPeerID, finalPeerID, "Identity.PeerID should be preserved") // Verify bootstrap exists (may be modified by 16→17 migration) finalBootstrap := getNestedValue(finalConfig, "Bootstrap") require.NotNil(t, finalBootstrap, "Bootstrap should exist after migration") // Verify AutoConf was added by 16→17 migration autoConf := getNestedValue(finalConfig, "AutoConf") require.NotNil(t, autoConf, "AutoConf should be added by 16→17 migration") } func testRepoMigration15ToLatest(t *testing.T) { // TEST: Migration from v15 to latest using 'ipfs repo migrate' // Comparison test to verify repo migrate produces same results as daemon migrate node := setupStaticV15Repo(t) // Create mock migration binary for 15→16 (16→17 will use embedded migration) mockBinDir := createMockMigrationBinary(t, "15", "16") customPath := buildCustomPath(mockBinDir) configPath := filepath.Join(node.Dir, "config") versionPath := filepath.Join(node.Dir, "version") // Verify starting version versionData, err := os.ReadFile(versionPath) require.NoError(t, err) require.Equal(t, "15", strings.TrimSpace(string(versionData)), "Should start at version 15") // Run migration using 'ipfs repo migrate' with custom PATH result := runMigrationWithCustomPath(node, customPath, "repo", "migrate") require.Empty(t, result.Stderr.String(), "Migration should succeed without errors") // Verify final version is latest versionData, err = os.ReadFile(versionPath) require.NoError(t, err) latestVersion := fmt.Sprintf("%d", ipfs.RepoVersion) require.Equal(t, latestVersion, strings.TrimSpace(string(versionData)), "Version should be updated to latest") // Verify config is valid JSON var finalConfig map[string]any configData, err := os.ReadFile(configPath) require.NoError(t, err) require.NoError(t, json.Unmarshal(configData, &finalConfig), "Config should remain valid JSON") // Verify essential fields exist require.NotNil(t, getNestedValue(finalConfig, "Identity.PeerID"), "Identity.PeerID should exist") require.NotNil(t, getNestedValue(finalConfig, "Bootstrap"), "Bootstrap should exist") require.NotNil(t, getNestedValue(finalConfig, "AutoConf"), "AutoConf should be added") } // setupStaticV15Repo creates a test node using static v15 repo fixture // This ensures tests remain stable and validates migration from very old repos func setupStaticV15Repo(t *testing.T) *harness.Node { // Get path to static v15 repo fixture v15FixturePath := "testdata/v15-repo" // Create temporary test directory using Go's testing temp dir tmpDir := t.TempDir() // Use the built binary (should be in PATH) node := harness.BuildNode("ipfs", tmpDir, 0) // Copy static fixture to test directory cloneStaticRepoFixture(t, v15FixturePath, node.Dir) return node } // runDaemonWithLegacyMigrationMonitoring monitors for hybrid migration patterns func runDaemonWithLegacyMigrationMonitoring(t *testing.T, node *harness.Node, customPath string) (string, bool) { // Monitor for hybrid migration completion - use "Hybrid migration completed successfully" as success pattern stdoutOutput, daemonStarted := runDaemonWithMigrationMonitoringCustomEnv(t, node, "Using hybrid migration strategy", "Hybrid migration completed successfully", map[string]string{ "PATH": customPath, // Pass custom PATH with our mock binaries }) // Check for hybrid migration patterns in output hasHybridStart := strings.Contains(stdoutOutput, "Using hybrid migration strategy") hasPhase1 := strings.Contains(stdoutOutput, "Phase 1: External migration from v15 to v16") hasPhase2 := strings.Contains(stdoutOutput, fmt.Sprintf("Phase 2: Embedded migration from v16 to v%d", ipfs.RepoVersion)) hasHybridSuccess := strings.Contains(stdoutOutput, "Hybrid migration completed successfully") // Success requires daemon to start and hybrid migration patterns to be detected hybridMigrationSuccess := daemonStarted && hasHybridStart && hasPhase1 && hasPhase2 && hasHybridSuccess return stdoutOutput, hybridMigrationSuccess } // runDaemonWithMigrationMonitoringCustomEnv is like runDaemonWithMigrationMonitoring but allows custom environment func runDaemonWithMigrationMonitoringCustomEnv(t *testing.T, node *harness.Node, migrationPattern, successPattern string, extraEnv map[string]string) (string, bool) { // Create context with timeout as safety net ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() // Set up daemon command with output monitoring cmd := exec.CommandContext(ctx, node.IPFSBin, "daemon", "--migrate") cmd.Dir = node.Dir // Set environment (especially IPFS_PATH) for k, v := range node.Runner.Env { cmd.Env = append(cmd.Env, k+"="+v) } // Add extra environment variables (like PATH with mock binaries) for k, v := range extraEnv { cmd.Env = append(cmd.Env, k+"="+v) } // Set up pipes for output monitoring stdout, err := cmd.StdoutPipe() require.NoError(t, err) stderr, err := cmd.StderrPipe() require.NoError(t, err) // Start the daemon require.NoError(t, cmd.Start()) // Monitor output from both streams var outputBuffer strings.Builder done := make(chan bool) migrationStarted := false migrationCompleted := false go func() { scanner := bufio.NewScanner(io.MultiReader(stdout, stderr)) for scanner.Scan() { line := scanner.Text() outputBuffer.WriteString(line + "\n") // Check for migration start if strings.Contains(line, migrationPattern) { migrationStarted = true } // Check for migration completion if strings.Contains(line, successPattern) { migrationCompleted = true } // Check for daemon ready if strings.Contains(line, "Daemon is ready") { done <- true return } } done <- false }() // Wait for daemon to be ready or timeout daemonReady := false select { case ready := <-done: daemonReady = ready case <-ctx.Done(): t.Log("Daemon startup timed out") } // Stop the daemon using ipfs shutdown command for graceful shutdown if cmd.Process != nil { shutdownCmd := exec.Command(node.IPFSBin, "shutdown") shutdownCmd.Dir = node.Dir for k, v := range node.Runner.Env { shutdownCmd.Env = append(shutdownCmd.Env, k+"="+v) } if err := shutdownCmd.Run(); err != nil { // If graceful shutdown fails, force kill _ = cmd.Process.Kill() } // Wait for process to exit _ = cmd.Wait() } return outputBuffer.String(), daemonReady && migrationStarted && migrationCompleted } // buildCustomPath creates a custom PATH with mock migration binaries prepended. // This is necessary for test isolation when running tests in parallel with t.Parallel(). // Without isolated PATH handling, parallel tests can interfere with each other through // global PATH modifications, causing tests to download real migration binaries instead // of using the test mocks. func buildCustomPath(mockBinDirs ...string) string { // Prepend mock directories to ensure they're found first pathElements := append(mockBinDirs, os.Getenv("PATH")) return strings.Join(pathElements, string(filepath.ListSeparator)) } // runMigrationWithCustomPath runs a migration command with a custom PATH environment. // This ensures the migration uses our mock binaries instead of downloading real ones. func runMigrationWithCustomPath(node *harness.Node, customPath string, args ...string) *harness.RunResult { return node.Runner.Run(harness.RunRequest{ Path: node.IPFSBin, Args: args, CmdOpts: []harness.CmdOpt{ func(cmd *exec.Cmd) { // Remove existing PATH entries using slices.DeleteFunc cmd.Env = slices.DeleteFunc(cmd.Env, func(s string) bool { return strings.HasPrefix(s, "PATH=") }) // Add custom PATH cmd.Env = append(cmd.Env, "PATH="+customPath) }, }, }) } // createMockMigrationBinary creates a platform-agnostic Go binary for migration testing. // Returns the directory containing the binary to be added to PATH. func createMockMigrationBinary(t *testing.T, fromVer, toVer string) string { // Create bin directory for migration binaries binDir := t.TempDir() // Create Go source for mock migration binary scriptName := fmt.Sprintf("fs-repo-%s-to-%s", fromVer, toVer) sourceFile := filepath.Join(binDir, scriptName+".go") binaryPath := filepath.Join(binDir, scriptName) if runtime.GOOS == "windows" { binaryPath += ".exe" } // Generate minimal mock migration binary code goSource := fmt.Sprintf(`package main import ("fmt"; "os"; "path/filepath"; "strings"; "time") func main() { var path string var revert bool for _, a := range os.Args[1:] { if strings.HasPrefix(a, "-path=") { path = a[6:] } if a == "-revert" { revert = true } } if path == "" { fmt.Fprintln(os.Stderr, "missing -path="); os.Exit(1) } from, to := "%s", "%s" if revert { from, to = to, from } fmt.Printf("fake applying %%s-to-%%s repo migration\n", from, to) // Create and immediately remove lock file to simulate proper locking behavior lockPath := filepath.Join(path, "repo.lock") lockFile, err := os.Create(lockPath) if err != nil && !os.IsExist(err) { fmt.Fprintf(os.Stderr, "Error creating lock: %%v\n", err) os.Exit(1) } if lockFile != nil { lockFile.Close() defer os.Remove(lockPath) } // Small delay to simulate migration work time.Sleep(10 * time.Millisecond) if err := os.WriteFile(filepath.Join(path, "version"), []byte(to), 0644); err != nil { fmt.Fprintf(os.Stderr, "Error: %%v\n", err) os.Exit(1) } }`, fromVer, toVer) require.NoError(t, os.WriteFile(sourceFile, []byte(goSource), 0644)) // Compile the Go binary cmd := exec.Command("go", "build", "-o", binaryPath, sourceFile) cmd.Env = append(os.Environ(), "CGO_ENABLED=0") // Ensure static binary require.NoError(t, cmd.Run()) // Verify the binary exists and is executable _, err := os.Stat(binaryPath) require.NoError(t, err, "Mock binary should exist") // Return the bin directory to be added to PATH return binDir } // expectedMigrationSteps generates the expected migration step strings for a version range. // For forward migrations (from < to), it returns strings like "Running embedded migration fs-repo-16-to-17" // For reverse migrations (from > to), it returns strings for the reverse path. func expectedMigrationSteps(from, to int, forward bool) []string { var steps []string if forward { // Forward migration: increment by 1 each step for v := from; v < to; v++ { migrationName := fmt.Sprintf("fs-repo-%d-to-%d", v, v+1) steps = append(steps, fmt.Sprintf("Running embedded migration %s", migrationName)) } } else { // Reverse migration: decrement by 1 each step for v := from; v > to; v-- { migrationName := fmt.Sprintf("fs-repo-%d-to-%d", v, v-1) steps = append(steps, fmt.Sprintf("Running reverse migration %s", migrationName)) } } return steps } // verifyMigrationSteps checks that all expected migration steps appear in the output func verifyMigrationSteps(t *testing.T, output string, from, to int, forward bool) { steps := expectedMigrationSteps(from, to, forward) for _, step := range steps { require.Contains(t, output, step, "Migration output should contain: %s", step) } } // getNestedValue retrieves a nested value from a config map using dot notation func getNestedValue(config map[string]any, path string) any { parts := strings.Split(path, ".") current := any(config) for _, part := range parts { switch v := current.(type) { case map[string]any: current = v[part] default: return nil } if current == nil { return nil } } return current } func testRepoReverseHybridMigrationLatestTo15(t *testing.T) { // TEST: Reverse hybrid migration from latest to v15 using 'ipfs repo migrate --to=15 --allow-downgrade' // This tests reverse hybrid migration: embedded (17→16) + external (16→15) // Start with v15 fixture and migrate forward to latest to create proper backup files node := setupStaticV15Repo(t) // Create mock migration binaries for both forward and reverse migrations mockBinDirs := []string{ createMockMigrationBinary(t, "15", "16"), // for forward migration createMockMigrationBinary(t, "16", "15"), // for downgrade } customPath := buildCustomPath(mockBinDirs...) configPath := filepath.Join(node.Dir, "config") versionPath := filepath.Join(node.Dir, "version") // Step 1: Forward migration from v15 to latest to create backup files t.Logf("Step 1: Forward migration v15 → v%d", ipfs.RepoVersion) result := runMigrationWithCustomPath(node, customPath, "repo", "migrate") // Debug: print the output to see what happened t.Logf("Forward migration stdout:\n%s", result.Stdout.String()) t.Logf("Forward migration stderr:\n%s", result.Stderr.String()) require.Empty(t, result.Stderr.String(), "Forward migration should succeed without errors") // Verify we're at latest version after forward migration versionData, err := os.ReadFile(versionPath) require.NoError(t, err) latestVersion := fmt.Sprintf("%d", ipfs.RepoVersion) require.Equal(t, latestVersion, strings.TrimSpace(string(versionData)), "Should be at latest version after forward migration") // Read config after forward migration to use as baseline for downgrade var latestConfig map[string]any configData, err := os.ReadFile(configPath) require.NoError(t, err) require.NoError(t, json.Unmarshal(configData, &latestConfig)) originalPeerID := getNestedValue(latestConfig, "Identity.PeerID") // Step 2: Reverse hybrid migration from latest to v15 t.Logf("Step 2: Reverse hybrid migration v%d → v15", ipfs.RepoVersion) result = runMigrationWithCustomPath(node, customPath, "repo", "migrate", "--to=15", "--allow-downgrade") require.Empty(t, result.Stderr.String(), "Reverse hybrid migration should succeed without errors") // Debug output t.Logf("Downgrade migration output:\n%s", result.Stdout.String()) // Verify final version is 15 versionData, err = os.ReadFile(versionPath) require.NoError(t, err) require.Equal(t, "15", strings.TrimSpace(string(versionData)), "Version should be updated to 15") // Verify config is still valid JSON and key fields preserved var finalConfig map[string]any configData, err = os.ReadFile(configPath) require.NoError(t, err) require.NoError(t, json.Unmarshal(configData, &finalConfig), "Config should remain valid JSON") // Verify essential fields preserved finalPeerID := getNestedValue(finalConfig, "Identity.PeerID") require.Equal(t, originalPeerID, finalPeerID, "Identity.PeerID should be preserved") // Verify bootstrap exists (may be modified by migrations) finalBootstrap := getNestedValue(finalConfig, "Bootstrap") require.NotNil(t, finalBootstrap, "Bootstrap should exist after migration") // AutoConf should be removed by the downgrade (was added in 16→17) autoConf := getNestedValue(finalConfig, "AutoConf") require.Nil(t, autoConf, "AutoConf should be removed by downgrade to v15") } ================================================ FILE: test/cli/migrations/testdata/v15-repo/blocks/SHARDING ================================================ /repo/flatfs/shard/v1/next-to-last/2 ================================================ FILE: test/cli/migrations/testdata/v15-repo/blocks/X3/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data ================================================  ================================================ FILE: test/cli/migrations/testdata/v15-repo/blocks/_README ================================================ This is a repository of IPLD objects. Each IPLD object is in a single file, named .data. Where is the "base32" encoding of the CID (as specified in https://github.com/multiformats/multibase) without the 'B' prefix. All the object files are placed in a tree of directories, based on a function of the CID. This is a form of sharding similar to the objects directory in git repositories. Previously, we used prefixes, we now use the next-to-last two characters. func NextToLast(base32cid string) { nextToLastLen := 2 offset := len(base32cid) - nextToLastLen - 1 return str[offset : offset+nextToLastLen] } For example, an object with a base58 CIDv1 of zb2rhYSxw4ZjuzgCnWSt19Q94ERaeFhu9uSqRgjSdx9bsgM6f has a base32 CIDv1 of BAFKREIA22FLID5AJ2KU7URG47MDLROZIH6YF2KALU2PWEFPVI37YLKRSCA and will be placed at SC/AFKREIA22FLID5AJ2KU7URG47MDLROZIH6YF2KALU2PWEFPVI37YLKRSCA.data with 'SC' being the last-to-next two characters and the 'B' at the beginning of the CIDv1 string is the multibase prefix that is not stored in the filename. ================================================ FILE: test/cli/migrations/testdata/v15-repo/blocks/diskUsage.cache ================================================ {"diskUsage":13452,"accuracy":"initial-exact"} ================================================ FILE: test/cli/migrations/testdata/v15-repo/config ================================================ { "Identity": { "PeerID": "12D3KooWPeo9gaDV6URwwwyWWjEJsCaMeZ7PBE5vpqvR1KFnPv3B", "PrivKey": "CAESQGPAQlzI5P/KnsbQ3e7dPNbv5Ztw8YwLv9k1dtS3pkd1zZAOR2796fXBZSKyo8Lw/wOqFb9plijC0iW0vTDuxXI=" }, "Datastore": { "StorageMax": "10GB", "StorageGCWatermark": 90, "GCPeriod": "1h", "Spec": { "mounts": [ { "child": { "path": "blocks", "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2", "sync": true, "type": "flatfs" }, "mountpoint": "/blocks", "prefix": "flatfs.datastore", "type": "measure" }, { "child": { "compression": "none", "path": "datastore", "type": "levelds" }, "mountpoint": "/", "prefix": "leveldb.datastore", "type": "measure" } ], "type": "mount" }, "HashOnRead": false, "BloomFilterSize": 0 }, "Addresses": { "Swarm": [ "/ip4/0.0.0.0/tcp/4001", "/ip6/::/tcp/4001", "/ip4/0.0.0.0/udp/4001/quic-v1", "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", "/ip6/::/udp/4001/quic-v1", "/ip6/::/udp/4001/quic-v1/webtransport" ], "Announce": [], "AppendAnnounce": [], "NoAnnounce": [], "API": "/ip4/127.0.0.1/tcp/5001", "Gateway": "/ip4/127.0.0.1/tcp/8080" }, "Mounts": { "IPFS": "/ipfs", "IPNS": "/ipns", "FuseAllowOther": false }, "Discovery": { "MDNS": { "Enabled": true } }, "Routing": { "Routers": null, "Methods": null }, "Ipns": { "RepublishPeriod": "", "RecordLifetime": "", "ResolveCacheSize": 128 }, "Bootstrap": [ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" ], "Gateway": { "HTTPHeaders": {}, "RootRedirect": "", "NoFetch": false, "NoDNSLink": false, "DeserializedResponses": null, "DisableHTMLErrors": null, "PublicGateways": null, "ExposeRoutingAPI": null }, "API": { "HTTPHeaders": {} }, "Swarm": { "AddrFilters": null, "DisableBandwidthMetrics": false, "DisableNatPortMap": false, "RelayClient": {}, "RelayService": {}, "Transports": { "Network": {}, "Security": {}, "Multiplexers": {} }, "ConnMgr": {}, "ResourceMgr": {} }, "AutoNAT": {}, "Pubsub": { "Router": "", "DisableSigning": false }, "Peering": { "Peers": null }, "DNS": { "Resolvers": {} }, "Migration": { "DownloadSources": [], "Keep": "" }, "Provider": { "Strategy": "" }, "Reprovider": {}, "Experimental": { "FilestoreEnabled": false, "UrlstoreEnabled": false, "Libp2pStreamMounting": false, "P2pHttpProxy": false, "StrategicProviding": false, "OptimisticProvide": false, "OptimisticProvideJobsPoolSize": 0 }, "Plugins": { "Plugins": null }, "Pinning": { "RemoteServices": {} }, "Import": { "CidVersion": null, "UnixFSRawLeaves": null, "UnixFSChunker": null, "HashFunction": null }, "Internal": {} } ================================================ FILE: test/cli/migrations/testdata/v15-repo/datastore/CURRENT ================================================ MANIFEST-000000 ================================================ FILE: test/cli/migrations/testdata/v15-repo/datastore/LOCK ================================================ ================================================ FILE: test/cli/migrations/testdata/v15-repo/datastore/LOG ================================================ =============== Aug 4, 2025 (CEST) =============== 01:47:33.360920 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed 01:47:33.384586 db@open opening 01:47:33.385359 version@stat F·[] S·0B[] Sc·[] 01:47:33.397679 db@janitor F·2 G·0 01:47:33.397725 db@open done T·13.097186ms 01:47:33.460539 db@close closing 01:47:33.460679 db@close done T·135.605µs ================================================ FILE: test/cli/migrations/testdata/v15-repo/datastore_spec ================================================ {"mounts":[{"mountpoint":"/blocks","path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"},{"mountpoint":"/","path":"datastore","type":"levelds"}],"type":"mount"} ================================================ FILE: test/cli/migrations/testdata/v15-repo/version ================================================ 15 ================================================ FILE: test/cli/migrations/testdata/v16-repo/blocks/SHARDING ================================================ /repo/flatfs/shard/v1/next-to-last/2 ================================================ FILE: test/cli/migrations/testdata/v16-repo/blocks/X3/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data ================================================  ================================================ FILE: test/cli/migrations/testdata/v16-repo/blocks/_README ================================================ This is a repository of IPLD objects. Each IPLD object is in a single file, named .data. Where is the "base32" encoding of the CID (as specified in https://github.com/multiformats/multibase) without the 'B' prefix. All the object files are placed in a tree of directories, based on a function of the CID. This is a form of sharding similar to the objects directory in git repositories. Previously, we used prefixes, we now use the next-to-last two characters. func NextToLast(base32cid string) { nextToLastLen := 2 offset := len(base32cid) - nextToLastLen - 1 return str[offset : offset+nextToLastLen] } For example, an object with a base58 CIDv1 of zb2rhYSxw4ZjuzgCnWSt19Q94ERaeFhu9uSqRgjSdx9bsgM6f has a base32 CIDv1 of BAFKREIA22FLID5AJ2KU7URG47MDLROZIH6YF2KALU2PWEFPVI37YLKRSCA and will be placed at SC/AFKREIA22FLID5AJ2KU7URG47MDLROZIH6YF2KALU2PWEFPVI37YLKRSCA.data with 'SC' being the last-to-next two characters and the 'B' at the beginning of the CIDv1 string is the multibase prefix that is not stored in the filename. ================================================ FILE: test/cli/migrations/testdata/v16-repo/blocks/diskUsage.cache ================================================ {"diskUsage":13452,"accuracy":"initial-exact"} ================================================ FILE: test/cli/migrations/testdata/v16-repo/config ================================================ { "Identity": { "PeerID": "12D3KooWGU72UzYkzVAiTyNLugX72zoDPTGkRegoKcTfB8oWxSuu", "PrivKey": "CAESQNfpGWI4zS+x+HSggBd7qqBai+Je5fopjmBylaTo7uZZYtESGX1PLDr5HmS3NJmrK7glW5kGRuYDvpqwJ2hnC2g=" }, "Datastore": { "StorageMax": "10GB", "StorageGCWatermark": 90, "GCPeriod": "1h", "Spec": { "mounts": [ { "mountpoint": "/blocks", "path": "blocks", "prefix": "flatfs.datastore", "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2", "sync": false, "type": "flatfs" }, { "compression": "none", "mountpoint": "/", "path": "datastore", "prefix": "leveldb.datastore", "type": "levelds" } ], "type": "mount" }, "HashOnRead": false, "BloomFilterSize": 0, "BlockKeyCacheSize": null }, "Addresses": { "Swarm": [ "/ip4/0.0.0.0/tcp/0" ], "Announce": [], "AppendAnnounce": [], "NoAnnounce": [], "API": "/ip4/127.0.0.1/tcp/0", "Gateway": "/ip4/127.0.0.1/tcp/0" }, "Mounts": { "IPFS": "/ipfs", "IPNS": "/ipns", "MFS": "/mfs", "FuseAllowOther": false }, "Discovery": { "MDNS": { "Enabled": true } }, "Routing": {}, "Ipns": { "RepublishPeriod": "", "RecordLifetime": "", "ResolveCacheSize": 128 }, "Bootstrap": [ "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", "/dnsaddr/va1.bootstrap.libp2p.io/p2p/12D3KooWKnDdG3iXw9eTFijk3EWSunZcFi54Zka4wmtqtt6rPxc8", "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN" ], "Gateway": { "HTTPHeaders": {}, "RootRedirect": "", "NoFetch": false, "NoDNSLink": false, "DeserializedResponses": null, "DisableHTMLErrors": null, "PublicGateways": null, "ExposeRoutingAPI": null }, "API": { "HTTPHeaders": {} }, "Swarm": { "AddrFilters": null, "DisableBandwidthMetrics": false, "DisableNatPortMap": false, "RelayClient": {}, "RelayService": {}, "Transports": { "Network": {}, "Security": {}, "Multiplexers": {} }, "ConnMgr": {}, "ResourceMgr": {} }, "AutoNAT": {}, "AutoTLS": {}, "Pubsub": { "Router": "", "DisableSigning": false }, "Peering": { "Peers": null }, "DNS": { "Resolvers": {} }, "Migration": { "DownloadSources": [], "Keep": "" }, "Provider": {}, "Reprovider": {}, "HTTPRetrieval": {}, "Experimental": { "FilestoreEnabled": false, "UrlstoreEnabled": false, "Libp2pStreamMounting": false, "P2pHttpProxy": false, "OptimisticProvide": false, "OptimisticProvideJobsPoolSize": 0 }, "Plugins": { "Plugins": null }, "Pinning": { "RemoteServices": {} }, "Import": { "CidVersion": null, "UnixFSRawLeaves": null, "UnixFSChunker": null, "HashFunction": null, "UnixFSFileMaxLinks": null, "UnixFSDirectoryMaxLinks": null, "UnixFSHAMTDirectoryMaxFanout": null, "UnixFSHAMTDirectorySizeThreshold": null, "BatchMaxNodes": null, "BatchMaxSize": null }, "Version": {}, "Internal": {}, "Bitswap": {} } ================================================ FILE: test/cli/migrations/testdata/v16-repo/datastore/CURRENT ================================================ MANIFEST-000000 ================================================ FILE: test/cli/migrations/testdata/v16-repo/datastore/LOCK ================================================ ================================================ FILE: test/cli/migrations/testdata/v16-repo/datastore/LOG ================================================ =============== Jul 23, 2025 (CEST) =============== 19:18:16.721510 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed 19:18:16.746720 db@open opening 19:18:16.747562 version@stat F·[] S·0B[] Sc·[] 19:18:16.763409 db@janitor F·2 G·0 19:18:16.763468 db@open done T·16.722352ms 19:18:16.831746 db@close closing 19:18:16.831861 db@close done T·110.694µs ================================================ FILE: test/cli/migrations/testdata/v16-repo/datastore_spec ================================================ {"mounts":[{"mountpoint":"/blocks","path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"},{"mountpoint":"/","path":"datastore","type":"levelds"}],"type":"mount"} ================================================ FILE: test/cli/migrations/testdata/v16-repo/version ================================================ 16 ================================================ FILE: test/cli/must.go ================================================ package cli func MustVal[V any](val V, err error) V { if err != nil { panic(err) } return val } ================================================ FILE: test/cli/name_test.go ================================================ // Tests for `ipfs name` CLI commands. // - TestName: tests name publish, resolve, and inspect // - TestNameGetPut: tests name get and put for raw IPNS record handling package cli import ( "bytes" "encoding/json" "fmt" "os" "path/filepath" "strings" "testing" "github.com/ipfs/boxo/ipns" "github.com/ipfs/kubo/core/commands/name" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/require" ) func TestName(t *testing.T) { const ( fixturePath = "fixtures/TestName.car" fixtureCid = "bafybeidg3uxibfrt7uqh7zd5yaodetik7wjwi4u7rwv2ndbgj6ec7lsv2a" dagCid = "bafyreidgts62p4rtg3rtmptmbv2dt46zjzin275fr763oku3wfod3quzay" ) makeDaemon := func(t *testing.T, initArgs []string) *harness.Node { node := harness.NewT(t).NewNode().Init(append([]string{"--profile=test"}, initArgs...)...) r, err := os.Open(fixturePath) require.Nil(t, err) defer r.Close() err = node.IPFSDagImport(r, fixtureCid) require.NoError(t, err) return node } testPublishingWithSelf := func(keyType string) { t.Run("Publishing with self (keyType = "+keyType+")", func(t *testing.T) { t.Parallel() args := []string{} if keyType != "default" { args = append(args, "-a="+keyType) } node := makeDaemon(t, args) name := ipns.NameFromPeer(node.PeerID()) t.Run("Publishing a CID", func(t *testing.T) { publishPath := "/ipfs/" + fixtureCid res := node.IPFS("name", "publish", "--allow-offline", publishPath) require.Equal(t, fmt.Sprintf("Published to %s: %s\n", name.String(), publishPath), res.Stdout.String()) res = node.IPFS("name", "resolve", "/ipns/"+name.String()) require.Equal(t, publishPath+"\n", res.Stdout.String()) }) t.Run("Publishing a CID with -Q option", func(t *testing.T) { publishPath := "/ipfs/" + fixtureCid res := node.IPFS("name", "publish", "--allow-offline", "-Q", publishPath) require.Equal(t, name.String()+"\n", res.Stdout.String()) res = node.IPFS("name", "resolve", "/ipns/"+name.String()) require.Equal(t, publishPath+"\n", res.Stdout.String()) }) t.Run("Publishing a CID+subpath", func(t *testing.T) { publishPath := "/ipfs/" + fixtureCid + "/hello" res := node.IPFS("name", "publish", "--allow-offline", publishPath) require.Equal(t, fmt.Sprintf("Published to %s: %s\n", name.String(), publishPath), res.Stdout.String()) res = node.IPFS("name", "resolve", "/ipns/"+name.String()) require.Equal(t, publishPath+"\n", res.Stdout.String()) }) t.Run("Publishing nothing fails", func(t *testing.T) { res := node.RunIPFS("name", "publish") require.Error(t, res.Err) require.Equal(t, 1, res.ExitCode()) require.Contains(t, res.Stderr.String(), `argument "ipfs-path" is required`) }) t.Run("Publishing with IPLD works", func(t *testing.T) { publishPath := "/ipld/" + dagCid + "/thing" res := node.IPFS("name", "publish", "--allow-offline", publishPath) require.Equal(t, fmt.Sprintf("Published to %s: %s\n", name.String(), publishPath), res.Stdout.String()) res = node.IPFS("name", "resolve", "/ipns/"+name.String()) require.Equal(t, publishPath+"\n", res.Stdout.String()) }) publishPath := "/ipfs/" + fixtureCid res := node.IPFS("name", "publish", "--allow-offline", publishPath) require.Equal(t, fmt.Sprintf("Published to %s: %s\n", name.String(), publishPath), res.Stdout.String()) t.Run("Resolving self offline succeeds (daemon off)", func(t *testing.T) { res = node.IPFS("name", "resolve", "--offline", "/ipns/"+name.String()) require.Equal(t, publishPath+"\n", res.Stdout.String()) // Test without cache. res = node.IPFS("name", "resolve", "--offline", "-n", "/ipns/"+name.String()) require.Equal(t, publishPath+"\n", res.Stdout.String()) }) node.StartDaemon() defer node.StopDaemon() t.Run("Resolving self offline succeeds (daemon on)", func(t *testing.T) { res = node.IPFS("name", "resolve", "--offline", "/ipns/"+name.String()) require.Equal(t, publishPath+"\n", res.Stdout.String()) // Test without cache. res = node.IPFS("name", "resolve", "--offline", "-n", "/ipns/"+name.String()) require.Equal(t, publishPath+"\n", res.Stdout.String()) }) }) } testPublishingWithSelf("default") testPublishingWithSelf("rsa") testPublishingWithSelf("ed25519") testPublishWithKey := func(name string, keyArgs ...string) { t.Run(name, func(t *testing.T) { t.Parallel() node := makeDaemon(t, nil) keyGenArgs := []string{"key", "gen"} keyGenArgs = append(keyGenArgs, keyArgs...) keyGenArgs = append(keyGenArgs, "key") res := node.IPFS(keyGenArgs...) key := strings.TrimSpace(res.Stdout.String()) publishPath := "/ipfs/" + fixtureCid name, err := ipns.NameFromString(key) require.NoError(t, err) res = node.IPFS("name", "publish", "--allow-offline", "--key="+key, publishPath) require.Equal(t, fmt.Sprintf("Published to %s: %s\n", name.String(), publishPath), res.Stdout.String()) }) } testPublishWithKey("Publishing with RSA (with b58mh) Key", "--ipns-base=b58mh", "--type=rsa", "--size=2048") testPublishWithKey("Publishing with ED25519 (with b58mh) Key", "--ipns-base=b58mh", "--type=ed25519") testPublishWithKey("Publishing with ED25519 (with base36) Key", "--ipns-base=base36", "--type=ed25519") t.Run("Fails to publish in offline mode", func(t *testing.T) { t.Parallel() node := makeDaemon(t, nil).StartDaemon("--offline") defer node.StopDaemon() res := node.RunIPFS("name", "publish", "/ipfs/"+fixtureCid) require.Error(t, res.Err) require.Equal(t, 1, res.ExitCode()) require.Contains(t, res.Stderr.String(), "can't publish while offline: pass `--allow-offline` to override or `--allow-delegated` if Ipns.DelegatedPublishers are set up") }) t.Run("Publish V2-only record", func(t *testing.T) { t.Parallel() node := makeDaemon(t, nil).StartDaemon() defer node.StopDaemon() ipnsName := ipns.NameFromPeer(node.PeerID()).String() ipnsPath := ipns.NamespacePrefix + ipnsName publishPath := "/ipfs/" + fixtureCid res := node.IPFS("name", "publish", "--ttl=30m", "--v1compat=false", publishPath) require.Equal(t, fmt.Sprintf("Published to %s: %s\n", ipnsName, publishPath), res.Stdout.String()) res = node.IPFS("name", "resolve", ipnsPath) require.Equal(t, publishPath+"\n", res.Stdout.String()) res = node.IPFS("routing", "get", ipnsPath) record := res.Stdout.Bytes() res = node.PipeToIPFS(bytes.NewReader(record), "name", "inspect") out := res.Stdout.String() require.Contains(t, out, "This record was not verified.") require.Contains(t, out, publishPath) require.Contains(t, out, "30m") res = node.PipeToIPFS(bytes.NewReader(record), "name", "inspect", "--verify="+ipnsPath) out = res.Stdout.String() require.Contains(t, out, "Valid: true") require.Contains(t, out, "Signature Type: V2") require.Contains(t, out, fmt.Sprintf("Protobuf Size: %d", len(record))) }) t.Run("Publish with TTL and inspect record", func(t *testing.T) { t.Parallel() node := makeDaemon(t, nil).StartDaemon() t.Cleanup(func() { node.StopDaemon() }) ipnsPath := ipns.NamespacePrefix + ipns.NameFromPeer(node.PeerID()).String() publishPath := "/ipfs/" + fixtureCid _ = node.IPFS("name", "publish", "--ttl=30m", publishPath) res := node.IPFS("routing", "get", ipnsPath) record := res.Stdout.Bytes() t.Run("Inspect record shows correct TTL and that it is not validated", func(t *testing.T) { t.Parallel() res = node.PipeToIPFS(bytes.NewReader(record), "name", "inspect") out := res.Stdout.String() require.Contains(t, out, "This record was not verified.") require.Contains(t, out, publishPath) require.Contains(t, out, "30m") require.Contains(t, out, "Signature Type: V1+V2") require.Contains(t, out, fmt.Sprintf("Protobuf Size: %d", len(record))) }) t.Run("Inspect record shows valid with correct name", func(t *testing.T) { t.Parallel() res := node.PipeToIPFS(bytes.NewReader(record), "name", "inspect", "--enc=json", "--verify="+ipnsPath) val := name.IpnsInspectResult{} err := json.Unmarshal(res.Stdout.Bytes(), &val) require.NoError(t, err) require.True(t, val.Validation.Valid) }) t.Run("Inspect record shows invalid with wrong name", func(t *testing.T) { t.Parallel() res := node.PipeToIPFS(bytes.NewReader(record), "name", "inspect", "--enc=json", "--verify=12D3KooWRirYjmmQATx2kgHBfky6DADsLP7ex1t7BRxJ6nqLs9WH") val := name.IpnsInspectResult{} err := json.Unmarshal(res.Stdout.Bytes(), &val) require.NoError(t, err) require.False(t, val.Validation.Valid) }) }) t.Run("Inspect with verification using wrong RSA key errors", func(t *testing.T) { t.Parallel() node := makeDaemon(t, nil).StartDaemon() defer node.StopDaemon() // Prepare RSA Key 1 res := node.IPFS("key", "gen", "--type=rsa", "--size=4096", "key1") key1 := strings.TrimSpace(res.Stdout.String()) name1, err := ipns.NameFromString(key1) require.NoError(t, err) // Prepare RSA Key 2 res = node.IPFS("key", "gen", "--type=rsa", "--size=4096", "key2") key2 := strings.TrimSpace(res.Stdout.String()) name2, err := ipns.NameFromString(key2) require.NoError(t, err) // Publish using Key 1 publishPath := "/ipfs/" + fixtureCid res = node.IPFS("name", "publish", "--allow-offline", "--key="+key1, publishPath) require.Equal(t, fmt.Sprintf("Published to %s: %s\n", name1.String(), publishPath), res.Stdout.String()) // Get IPNS Record res = node.IPFS("routing", "get", ipns.NamespacePrefix+name1.String()) record := res.Stdout.Bytes() // Validate with correct key succeeds res = node.PipeToIPFS(bytes.NewReader(record), "name", "inspect", "--verify="+name1.String(), "--enc=json") val := name.IpnsInspectResult{} err = json.Unmarshal(res.Stdout.Bytes(), &val) require.NoError(t, err) require.True(t, val.Validation.Valid) // Validate with wrong key fails res = node.PipeToIPFS(bytes.NewReader(record), "name", "inspect", "--verify="+name2.String(), "--enc=json") val = name.IpnsInspectResult{} err = json.Unmarshal(res.Stdout.Bytes(), &val) require.NoError(t, err) require.False(t, val.Validation.Valid) }) t.Run("Publishing with custom sequence number", func(t *testing.T) { t.Parallel() node := makeDaemon(t, nil) publishPath := "/ipfs/" + fixtureCid name := ipns.NameFromPeer(node.PeerID()) t.Run("Publish with sequence=0 is not allowed", func(t *testing.T) { // Sequence=0 is never valid, even on a fresh node res := node.RunIPFS("name", "publish", "--allow-offline", "--ttl=0", "--sequence=0", publishPath) require.NotEqual(t, 0, res.ExitCode(), "Expected publish with sequence=0 to fail") require.Contains(t, res.Stderr.String(), "sequence number must be greater than the current record sequence") }) t.Run("Publish with sequence=1 on fresh node", func(t *testing.T) { // Sequence=1 is the minimum valid sequence number for first publish res := node.IPFS("name", "publish", "--allow-offline", "--ttl=0", "--sequence=1", publishPath) require.Equal(t, fmt.Sprintf("Published to %s: %s\n", name.String(), publishPath), res.Stdout.String()) }) t.Run("Publish with sequence=42", func(t *testing.T) { res := node.IPFS("name", "publish", "--allow-offline", "--ttl=0", "--sequence=42", publishPath) require.Equal(t, fmt.Sprintf("Published to %s: %s\n", name.String(), publishPath), res.Stdout.String()) }) t.Run("Publish with large sequence number", func(t *testing.T) { res := node.IPFS("name", "publish", "--allow-offline", "--ttl=0", "--sequence=18446744073709551615", publishPath) // Max uint64 require.Equal(t, fmt.Sprintf("Published to %s: %s\n", name.String(), publishPath), res.Stdout.String()) }) }) t.Run("Sequence number monotonic check", func(t *testing.T) { t.Parallel() node := makeDaemon(t, nil).StartDaemon() defer node.StopDaemon() publishPath1 := "/ipfs/" + fixtureCid publishPath2 := "/ipfs/" + dagCid // Different content name := ipns.NameFromPeer(node.PeerID()) // First, publish with a high sequence number (1000) res := node.IPFS("name", "publish", "--ttl=0", "--sequence=1000", publishPath1) require.Equal(t, fmt.Sprintf("Published to %s: %s\n", name.String(), publishPath1), res.Stdout.String()) // Verify the record was published successfully res = node.IPFS("name", "resolve", name.String()) require.Contains(t, res.Stdout.String(), publishPath1) // Now try to publish different content with a LOWER sequence number (500) // This should fail due to monotonic sequence check res = node.RunIPFS("name", "publish", "--ttl=0", "--sequence=500", publishPath2) require.NotEqual(t, 0, res.ExitCode(), "Expected publish with lower sequence to fail") require.Contains(t, res.Stderr.String(), "sequence number", "Expected error about sequence number") // Verify the original content is still published (not overwritten) res = node.IPFS("name", "resolve", name.String()) require.Contains(t, res.Stdout.String(), publishPath1, "Original content should still be published") require.NotContains(t, res.Stdout.String(), publishPath2, "New content should not have been published") // Publishing with a HIGHER sequence number should succeed res = node.IPFS("name", "publish", "--ttl=0", "--sequence=2000", publishPath2) require.Equal(t, fmt.Sprintf("Published to %s: %s\n", name.String(), publishPath2), res.Stdout.String()) // Verify the new content is now published res = node.IPFS("name", "resolve", name.String()) require.Contains(t, res.Stdout.String(), publishPath2, "New content should now be published") }) } func TestNameGetPut(t *testing.T) { t.Parallel() const ( fixturePath = "fixtures/TestName.car" fixtureCid = "bafybeidg3uxibfrt7uqh7zd5yaodetik7wjwi4u7rwv2ndbgj6ec7lsv2a" ) makeDaemon := func(t *testing.T, daemonArgs ...string) *harness.Node { node := harness.NewT(t).NewNode().Init("--profile=test") r, err := os.Open(fixturePath) require.NoError(t, err) defer r.Close() err = node.IPFSDagImport(r, fixtureCid) require.NoError(t, err) return node.StartDaemon(daemonArgs...) } // makeKey creates a unique IPNS key for a test and returns the IPNS name makeKey := func(t *testing.T, node *harness.Node, keyName string) ipns.Name { res := node.IPFS("key", "gen", "--type=ed25519", keyName) keyID := strings.TrimSpace(res.Stdout.String()) name, err := ipns.NameFromString(keyID) require.NoError(t, err) return name } // makeExternalRecord creates an IPNS record on an ephemeral node that is // shut down before returning. This ensures the test node has no local // knowledge of the record, properly testing put/get functionality. // We use short --lifetime so if IPNS records from tests get published on // the public DHT, they won't waste storage for long. makeExternalRecord := func(t *testing.T, h *harness.Harness, publishPath string, publishArgs ...string) (ipns.Name, []byte) { node := h.NewNode().Init("--profile=test") r, err := os.Open(fixturePath) require.NoError(t, err) defer r.Close() err = node.IPFSDagImport(r, fixtureCid) require.NoError(t, err) node.StartDaemon() res := node.IPFS("key", "gen", "--type=ed25519", "ephemeral-key") keyID := strings.TrimSpace(res.Stdout.String()) ipnsName, err := ipns.NameFromString(keyID) require.NoError(t, err) args := []string{"name", "publish", "--key=ephemeral-key", "--lifetime=5m"} args = append(args, publishArgs...) args = append(args, publishPath) node.IPFS(args...) res = node.IPFS("name", "get", ipnsName.String()) record := res.Stdout.Bytes() require.NotEmpty(t, record) node.StopDaemon() return ipnsName, record } t.Run("name get retrieves IPNS record", func(t *testing.T) { t.Parallel() node := makeDaemon(t) defer node.StopDaemon() publishPath := "/ipfs/" + fixtureCid ipnsName := makeKey(t, node, "testkey") // publish a record first node.IPFS("name", "publish", "--key=testkey", "--lifetime=5m", publishPath) // retrieve the record using name get res := node.IPFS("name", "get", ipnsName.String()) record := res.Stdout.Bytes() require.NotEmpty(t, record, "expected non-empty IPNS record") // verify the record is valid by inspecting it res = node.PipeToIPFS(bytes.NewReader(record), "name", "inspect", "--verify="+ipnsName.String()) require.Contains(t, res.Stdout.String(), "Valid: true") require.Contains(t, res.Stdout.String(), publishPath) }) t.Run("name get accepts /ipns/ prefix", func(t *testing.T) { t.Parallel() node := makeDaemon(t) defer node.StopDaemon() publishPath := "/ipfs/" + fixtureCid ipnsName := makeKey(t, node, "testkey") node.IPFS("name", "publish", "--key=testkey", "--lifetime=5m", publishPath) // retrieve with /ipns/ prefix res := node.IPFS("name", "get", "/ipns/"+ipnsName.String()) record := res.Stdout.Bytes() require.NotEmpty(t, record) // verify the record res = node.PipeToIPFS(bytes.NewReader(record), "name", "inspect", "--verify="+ipnsName.String()) require.Contains(t, res.Stdout.String(), "Valid: true") }) t.Run("name get fails for non-existent name", func(t *testing.T) { t.Parallel() node := makeDaemon(t) defer node.StopDaemon() // try to get a record for a random peer ID that doesn't exist res := node.RunIPFS("name", "get", "12D3KooWRirYjmmQATx2kgHBfky6DADsLP7ex1t7BRxJ6nqLs9WH") require.Error(t, res.Err) require.NotEqual(t, 0, res.ExitCode()) }) t.Run("name get fails for invalid name format", func(t *testing.T) { t.Parallel() node := makeDaemon(t) defer node.StopDaemon() res := node.RunIPFS("name", "get", "not-a-valid-ipns-name") require.Error(t, res.Err) require.NotEqual(t, 0, res.ExitCode()) }) t.Run("name put accepts /ipns/ prefix", func(t *testing.T) { t.Parallel() node := makeDaemon(t) defer node.StopDaemon() publishPath := "/ipfs/" + fixtureCid ipnsName := makeKey(t, node, "testkey") node.IPFS("name", "publish", "--key=testkey", "--lifetime=5m", publishPath) res := node.IPFS("name", "get", ipnsName.String()) record := res.Stdout.Bytes() // put with /ipns/ prefix res = node.PipeToIPFS(bytes.NewReader(record), "name", "put", "--force", "/ipns/"+ipnsName.String()) require.NoError(t, res.Err) }) t.Run("name put fails for invalid name format", func(t *testing.T) { t.Parallel() node := makeDaemon(t) defer node.StopDaemon() // create a dummy file recordFile := filepath.Join(node.Dir, "dummy.bin") err := os.WriteFile(recordFile, []byte("dummy"), 0644) require.NoError(t, err) res := node.RunIPFS("name", "put", "not-a-valid-ipns-name", recordFile) require.Error(t, res.Err) require.Contains(t, res.Stderr.String(), "invalid IPNS name") }) t.Run("name put rejects oversized record", func(t *testing.T) { t.Parallel() node := makeDaemon(t) defer node.StopDaemon() ipnsName := makeKey(t, node, "testkey") // create a file larger than 10 KiB oversizedRecord := make([]byte, 11*1024) recordFile := filepath.Join(node.Dir, "oversized.bin") err := os.WriteFile(recordFile, oversizedRecord, 0644) require.NoError(t, err) res := node.RunIPFS("name", "put", ipnsName.String(), recordFile) require.Error(t, res.Err) require.Contains(t, res.Stderr.String(), "exceeds maximum size") }) t.Run("name put --force skips size check", func(t *testing.T) { t.Parallel() node := makeDaemon(t) defer node.StopDaemon() ipnsName := makeKey(t, node, "testkey") // create a file larger than 10 KiB oversizedRecord := make([]byte, 11*1024) recordFile := filepath.Join(node.Dir, "oversized.bin") err := os.WriteFile(recordFile, oversizedRecord, 0644) require.NoError(t, err) // with --force, size check is skipped (but routing will likely reject it) res := node.RunIPFS("name", "put", "--force", ipnsName.String(), recordFile) // the command itself should not fail on size, but routing may reject // we just verify it doesn't fail with "exceeds maximum size" if res.Err != nil { require.NotContains(t, res.Stderr.String(), "exceeds maximum size") } }) t.Run("name put stores IPNS record", func(t *testing.T) { t.Parallel() h := harness.NewT(t) publishPath := "/ipfs/" + fixtureCid // create a record on an ephemeral node (shut down before test node starts) ipnsName, record := makeExternalRecord(t, h, publishPath) // start test node (has no local knowledge of the record) node := makeDaemon(t) defer node.StopDaemon() // put the record (should succeed since no existing record) recordFile := filepath.Join(node.Dir, "record.bin") err := os.WriteFile(recordFile, record, 0644) require.NoError(t, err) res := node.RunIPFS("name", "put", ipnsName.String(), recordFile) require.NoError(t, res.Err) // verify the record was stored by getting it back res = node.IPFS("name", "get", ipnsName.String()) retrievedRecord := res.Stdout.Bytes() require.Equal(t, record, retrievedRecord, "stored record should match original") }) t.Run("name put with --force overwrites existing record", func(t *testing.T) { t.Parallel() h := harness.NewT(t) publishPath := "/ipfs/" + fixtureCid // create a record on an ephemeral node ipnsName, record := makeExternalRecord(t, h, publishPath) // start test node node := makeDaemon(t) defer node.StopDaemon() // first put the record normally recordFile := filepath.Join(node.Dir, "record.bin") err := os.WriteFile(recordFile, record, 0644) require.NoError(t, err) res := node.RunIPFS("name", "put", ipnsName.String(), recordFile) require.NoError(t, res.Err) // put the same record again (identical record republishing is allowed) res = node.RunIPFS("name", "put", ipnsName.String(), recordFile) require.NoError(t, res.Err) // put the record with --force (should succeed) res = node.RunIPFS("name", "put", "--force", ipnsName.String(), recordFile) require.NoError(t, res.Err) }) t.Run("name put validates signature against name", func(t *testing.T) { t.Parallel() h := harness.NewT(t) publishPath := "/ipfs/" + fixtureCid // create a record on an ephemeral node _, record := makeExternalRecord(t, h, publishPath) // start test node node := makeDaemon(t) defer node.StopDaemon() // write the record to a file recordFile := filepath.Join(node.Dir, "record.bin") err := os.WriteFile(recordFile, record, 0644) require.NoError(t, err) // try to put with a wrong name (should fail validation) wrongName := "12D3KooWRirYjmmQATx2kgHBfky6DADsLP7ex1t7BRxJ6nqLs9WH" res := node.RunIPFS("name", "put", wrongName, recordFile) require.Error(t, res.Err) require.Contains(t, res.Stderr.String(), "record validation failed") }) t.Run("name put with --force skips command validation", func(t *testing.T) { t.Parallel() h := harness.NewT(t) publishPath := "/ipfs/" + fixtureCid // create a record on an ephemeral node ipnsName, record := makeExternalRecord(t, h, publishPath) // start test node node := makeDaemon(t) defer node.StopDaemon() // with --force the command skips its own validation (signature, sequence check) // and passes the record directly to the routing layer res := node.PipeToIPFS(bytes.NewReader(record), "name", "put", "--force", ipnsName.String()) require.NoError(t, res.Err) }) t.Run("name put rejects empty record", func(t *testing.T) { t.Parallel() node := makeDaemon(t) defer node.StopDaemon() ipnsName := makeKey(t, node, "testkey") // create an empty file recordFile := filepath.Join(node.Dir, "empty.bin") err := os.WriteFile(recordFile, []byte{}, 0644) require.NoError(t, err) res := node.RunIPFS("name", "put", ipnsName.String(), recordFile) require.Error(t, res.Err) require.Contains(t, res.Stderr.String(), "record is empty") }) t.Run("name put rejects invalid record", func(t *testing.T) { t.Parallel() node := makeDaemon(t) defer node.StopDaemon() ipnsName := makeKey(t, node, "testkey") // create a file with garbage data recordFile := filepath.Join(node.Dir, "garbage.bin") err := os.WriteFile(recordFile, []byte("not a valid ipns record"), 0644) require.NoError(t, err) res := node.RunIPFS("name", "put", ipnsName.String(), recordFile) require.Error(t, res.Err) require.Contains(t, res.Stderr.String(), "invalid IPNS record") }) t.Run("name put accepts stdin", func(t *testing.T) { t.Parallel() h := harness.NewT(t) publishPath := "/ipfs/" + fixtureCid // create a record on an ephemeral node ipnsName, record := makeExternalRecord(t, h, publishPath) // start test node (has no local knowledge of the record) node := makeDaemon(t) defer node.StopDaemon() // put via stdin (no --force needed since no existing record) res := node.PipeToIPFS(bytes.NewReader(record), "name", "put", ipnsName.String()) require.NoError(t, res.Err) }) t.Run("name put fails when offline without --allow-offline", func(t *testing.T) { t.Parallel() h := harness.NewT(t) publishPath := "/ipfs/" + fixtureCid // create a record on an ephemeral node ipnsName, record := makeExternalRecord(t, h, publishPath) // write the record to a file recordFile := filepath.Join(h.Dir, "record.bin") err := os.WriteFile(recordFile, record, 0644) require.NoError(t, err) // start test node in offline mode node := h.NewNode().Init("--profile=test") node.StartDaemon("--offline") defer node.StopDaemon() // try to put without --allow-offline (should fail) res := node.RunIPFS("name", "put", ipnsName.String(), recordFile) require.Error(t, res.Err) // error can come from our command or from the routing layer stderr := res.Stderr.String() require.True(t, strings.Contains(stderr, "offline") || strings.Contains(stderr, "online mode"), "expected offline-related error, got: %s", stderr) }) t.Run("name put succeeds with --allow-offline", func(t *testing.T) { t.Parallel() h := harness.NewT(t) publishPath := "/ipfs/" + fixtureCid // create a record on an ephemeral node ipnsName, record := makeExternalRecord(t, h, publishPath) // write the record to a file recordFile := filepath.Join(h.Dir, "record.bin") err := os.WriteFile(recordFile, record, 0644) require.NoError(t, err) // start test node in offline mode node := h.NewNode().Init("--profile=test") node.StartDaemon("--offline") defer node.StopDaemon() // put with --allow-offline (should succeed, no --force needed since no existing record) res := node.RunIPFS("name", "put", "--allow-offline", ipnsName.String(), recordFile) require.NoError(t, res.Err) }) t.Run("name get/put round trip preserves record bytes", func(t *testing.T) { t.Parallel() h := harness.NewT(t) publishPath := "/ipfs/" + fixtureCid // create a record on an ephemeral node ipnsName, originalRecord := makeExternalRecord(t, h, publishPath) // start test node (has no local knowledge of the record) node := makeDaemon(t) defer node.StopDaemon() // put the record res := node.PipeToIPFS(bytes.NewReader(originalRecord), "name", "put", ipnsName.String()) require.NoError(t, res.Err) // get the record back res = node.IPFS("name", "get", ipnsName.String()) retrievedRecord := res.Stdout.Bytes() // the records should be byte-for-byte identical require.Equal(t, originalRecord, retrievedRecord, "record bytes should be preserved after get/put round trip") }) t.Run("name put --force allows storing lower sequence record", func(t *testing.T) { t.Parallel() h := harness.NewT(t) publishPath := "/ipfs/" + fixtureCid // create an ephemeral node to generate two records with different sequences ephNode := h.NewNode().Init("--profile=test") r, err := os.Open(fixturePath) require.NoError(t, err) err = ephNode.IPFSDagImport(r, fixtureCid) r.Close() require.NoError(t, err) ephNode.StartDaemon() res := ephNode.IPFS("key", "gen", "--type=ed25519", "ephemeral-key") keyID := strings.TrimSpace(res.Stdout.String()) ipnsName, err := ipns.NameFromString(keyID) require.NoError(t, err) // publish record with sequence 100 ephNode.IPFS("name", "publish", "--key=ephemeral-key", "--lifetime=5m", "--sequence=100", publishPath) res = ephNode.IPFS("name", "get", ipnsName.String()) record100 := res.Stdout.Bytes() // publish record with sequence 200 ephNode.IPFS("name", "publish", "--key=ephemeral-key", "--lifetime=5m", "--sequence=200", publishPath) res = ephNode.IPFS("name", "get", ipnsName.String()) record200 := res.Stdout.Bytes() ephNode.StopDaemon() // start test node (has no local knowledge of the records) node := makeDaemon(t) defer node.StopDaemon() // helper to get sequence from record getSequence := func(record []byte) uint64 { res := node.PipeToIPFS(bytes.NewReader(record), "name", "inspect", "--enc=json") var result name.IpnsInspectResult err := json.Unmarshal(res.Stdout.Bytes(), &result) require.NoError(t, err) require.NotNil(t, result.Entry.Sequence) return *result.Entry.Sequence } // verify we have the right records require.Equal(t, uint64(100), getSequence(record100)) require.Equal(t, uint64(200), getSequence(record200)) // put record with sequence 200 first res = node.PipeToIPFS(bytes.NewReader(record200), "name", "put", ipnsName.String()) require.NoError(t, res.Err) // verify current record has sequence 200 res = node.IPFS("name", "get", ipnsName.String()) require.Equal(t, uint64(200), getSequence(res.Stdout.Bytes())) // now put the lower sequence record (100) with --force // this should succeed (--force bypasses our sequence check) res = node.PipeToIPFS(bytes.NewReader(record100), "name", "put", "--force", ipnsName.String()) require.NoError(t, res.Err, "putting lower sequence record with --force should succeed") // note: when we get the record, IPNS resolution returns the "best" record // (highest sequence), so we'll get the sequence 200 record back // this is expected IPNS behavior - the put succeeded, but get returns the best record res = node.IPFS("name", "get", ipnsName.String()) retrievedSeq := getSequence(res.Stdout.Bytes()) require.Equal(t, uint64(200), retrievedSeq, "IPNS get returns the best (highest sequence) record") }) t.Run("name put sequence conflict detection", func(t *testing.T) { t.Parallel() h := harness.NewT(t) publishPath := "/ipfs/" + fixtureCid // create an ephemeral node to generate two records with different sequences ephNode := h.NewNode().Init("--profile=test") r, err := os.Open(fixturePath) require.NoError(t, err) err = ephNode.IPFSDagImport(r, fixtureCid) r.Close() require.NoError(t, err) ephNode.StartDaemon() res := ephNode.IPFS("key", "gen", "--type=ed25519", "ephemeral-key") keyID := strings.TrimSpace(res.Stdout.String()) ipnsName, err := ipns.NameFromString(keyID) require.NoError(t, err) // publish record with sequence 100 ephNode.IPFS("name", "publish", "--key=ephemeral-key", "--lifetime=5m", "--sequence=100", publishPath) res = ephNode.IPFS("name", "get", ipnsName.String()) record100 := res.Stdout.Bytes() // publish record with sequence 200 ephNode.IPFS("name", "publish", "--key=ephemeral-key", "--lifetime=5m", "--sequence=200", publishPath) res = ephNode.IPFS("name", "get", ipnsName.String()) record200 := res.Stdout.Bytes() ephNode.StopDaemon() // start test node (has no local knowledge of the records) node := makeDaemon(t) defer node.StopDaemon() // put record with sequence 200 first res = node.PipeToIPFS(bytes.NewReader(record200), "name", "put", ipnsName.String()) require.NoError(t, res.Err) // try to put record with sequence 100 (lower than current 200) recordFile := filepath.Join(node.Dir, "record100.bin") err = os.WriteFile(recordFile, record100, 0644) require.NoError(t, err) res = node.RunIPFS("name", "put", ipnsName.String(), recordFile) require.Error(t, res.Err) require.Contains(t, res.Stderr.String(), "existing IPNS record has sequence 200 >= new record sequence 100") }) t.Run("name put allows identical record republishing", func(t *testing.T) { t.Parallel() h := harness.NewT(t) publishPath := "/ipfs/" + fixtureCid ipnsName, record := makeExternalRecord(t, h, publishPath, "--sequence=100") node := makeDaemon(t) defer node.StopDaemon() // put the record res := node.PipeToIPFS(bytes.NewReader(record), "name", "put", ipnsName.String()) require.NoError(t, res.Err) // put the exact same record again (same bytes, same sequence) // this should succeed: republishing an identical record is a valid use case res = node.PipeToIPFS(bytes.NewReader(record), "name", "put", ipnsName.String()) require.NoError(t, res.Err) require.Contains(t, res.Stdout.String(), ipnsName.String()) }) t.Run("name put rejects different record with same sequence", func(t *testing.T) { t.Parallel() h := harness.NewT(t) // create two different records signed by the same key with the same // sequence number by using two ephemeral nodes that share a key ephNode1 := h.NewNode().Init("--profile=test") r, err := os.Open(fixturePath) require.NoError(t, err) err = ephNode1.IPFSDagImport(r, fixtureCid) r.Close() require.NoError(t, err) ephNode1.StartDaemon() res := ephNode1.IPFS("key", "gen", "--type=ed25519", "shared-key") keyID := strings.TrimSpace(res.Stdout.String()) ipnsName, err := ipns.NameFromString(keyID) require.NoError(t, err) // publish record A (sequence=100, value=fixtureCid) ephNode1.IPFS("name", "publish", "--key=shared-key", "--lifetime=5m", "--sequence=100", "/ipfs/"+fixtureCid) res = ephNode1.IPFS("name", "get", ipnsName.String()) recordA := res.Stdout.Bytes() // export key and import into second ephemeral node keyFile := filepath.Join(ephNode1.Dir, "shared-key.key") ephNode1.IPFS("key", "export", "--output="+keyFile, "shared-key") ephNode1.StopDaemon() ephNode2 := h.NewNode().Init("--profile=test") ephNode2.StartDaemon() ephNode2.IPFS("key", "import", "shared-key", keyFile) // publish record B (sequence=100, different value) ephNode2.IPFS("name", "publish", "--key=shared-key", "--lifetime=5m", "--sequence=100", "/ipfs/bafkqaaa") res = ephNode2.IPFS("name", "get", ipnsName.String()) recordB := res.Stdout.Bytes() ephNode2.StopDaemon() // verify records have same sequence but different bytes require.NotEqual(t, recordA, recordB, "records should have different bytes") // start test node and try the put scenario node := makeDaemon(t) defer node.StopDaemon() // put record A res = node.PipeToIPFS(bytes.NewReader(recordA), "name", "put", ipnsName.String()) require.NoError(t, res.Err) // try to put record B (different bytes, same sequence=100) recordFile := filepath.Join(node.Dir, "recordB.bin") err = os.WriteFile(recordFile, recordB, 0644) require.NoError(t, err) res = node.RunIPFS("name", "put", ipnsName.String(), recordFile) require.Error(t, res.Err) require.Contains(t, res.Stderr.String(), "existing IPNS record has sequence 100 >= new record sequence 100") }) } ================================================ FILE: test/cli/p2p_test.go ================================================ package cli import ( "encoding/json" "fmt" "io" "net" "net/http" "os/exec" "slices" "syscall" "testing" "time" "github.com/ipfs/kubo/core/commands" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/require" ) // waitForListenerCount waits until the node has exactly the expected number of listeners. func waitForListenerCount(t *testing.T, node *harness.Node, expectedCount int) { t.Helper() require.Eventually(t, func() bool { lsOut := node.IPFS("p2p", "ls", "--enc=json") var lsResult commands.P2PLsOutput if err := json.Unmarshal(lsOut.Stdout.Bytes(), &lsResult); err != nil { return false } return len(lsResult.Listeners) == expectedCount }, 5*time.Second, 100*time.Millisecond, "expected %d listeners", expectedCount) } // waitForListenerProtocol waits until the node has a listener with the given protocol. func waitForListenerProtocol(t *testing.T, node *harness.Node, protocol string) { t.Helper() require.Eventually(t, func() bool { lsOut := node.IPFS("p2p", "ls", "--enc=json") var lsResult commands.P2PLsOutput if err := json.Unmarshal(lsOut.Stdout.Bytes(), &lsResult); err != nil { return false } return slices.ContainsFunc(lsResult.Listeners, func(l commands.P2PListenerInfoOutput) bool { return l.Protocol == protocol }) }, 5*time.Second, 100*time.Millisecond, "expected listener with protocol %s", protocol) } func TestP2PForeground(t *testing.T) { t.Parallel() t.Run("listen foreground creates listener and removes on interrupt", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.IPFS("config", "--json", "Experimental.Libp2pStreamMounting", "true") node.StartDaemon() listenPort := harness.NewRandPort() // Start foreground listener asynchronously res := node.Runner.Run(harness.RunRequest{ Path: node.IPFSBin, Args: []string{"p2p", "listen", "--foreground", "/x/fgtest", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", listenPort)}, RunFunc: (*exec.Cmd).Start, }) require.NoError(t, res.Err) // Wait for listener to be created waitForListenerProtocol(t, node, "/x/fgtest") // Send SIGTERM _ = res.Cmd.Process.Signal(syscall.SIGTERM) _ = res.Cmd.Wait() // Wait for listener to be removed waitForListenerCount(t, node, 0) }) t.Run("listen foreground text output on SIGTERM", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.IPFS("config", "--json", "Experimental.Libp2pStreamMounting", "true") node.StartDaemon() listenPort := harness.NewRandPort() // Run without --enc=json to test actual text output users see res := node.Runner.Run(harness.RunRequest{ Path: node.IPFSBin, Args: []string{"p2p", "listen", "--foreground", "/x/sigterm", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", listenPort)}, RunFunc: (*exec.Cmd).Start, }) require.NoError(t, res.Err) waitForListenerProtocol(t, node, "/x/sigterm") _ = res.Cmd.Process.Signal(syscall.SIGTERM) _ = res.Cmd.Wait() // Verify stdout shows "waiting for interrupt" message stdout := res.Stdout.String() require.Contains(t, stdout, "waiting for interrupt") // Note: "Received interrupt, removing listener" message is NOT visible to CLI on SIGTERM // because the command runs in the daemon via RPC and the response stream closes before // the message can be emitted. The important behavior is verified in the first test: // the listener IS removed when SIGTERM is sent. }) t.Run("forward foreground creates forwarder and removes on interrupt", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init() nodes.ForEachPar(func(n *harness.Node) { n.IPFS("config", "--json", "Experimental.Libp2pStreamMounting", "true") }) nodes.StartDaemons().Connect() forwardPort := harness.NewRandPort() // Start foreground forwarder asynchronously on node 0 res := nodes[0].Runner.Run(harness.RunRequest{ Path: nodes[0].IPFSBin, Args: []string{"p2p", "forward", "--foreground", "/x/fgfwd", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", forwardPort), "/p2p/" + nodes[1].PeerID().String()}, RunFunc: (*exec.Cmd).Start, }) require.NoError(t, res.Err) // Wait for forwarder to be created waitForListenerCount(t, nodes[0], 1) // Send SIGTERM _ = res.Cmd.Process.Signal(syscall.SIGTERM) _ = res.Cmd.Wait() // Wait for forwarder to be removed waitForListenerCount(t, nodes[0], 0) }) t.Run("forward foreground text output on SIGTERM", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init() nodes.ForEachPar(func(n *harness.Node) { n.IPFS("config", "--json", "Experimental.Libp2pStreamMounting", "true") }) nodes.StartDaemons().Connect() forwardPort := harness.NewRandPort() // Run without --enc=json to test actual text output users see res := nodes[0].Runner.Run(harness.RunRequest{ Path: nodes[0].IPFSBin, Args: []string{"p2p", "forward", "--foreground", "/x/fwdsigterm", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", forwardPort), "/p2p/" + nodes[1].PeerID().String()}, RunFunc: (*exec.Cmd).Start, }) require.NoError(t, res.Err) waitForListenerCount(t, nodes[0], 1) _ = res.Cmd.Process.Signal(syscall.SIGTERM) _ = res.Cmd.Wait() // Verify stdout shows "waiting for interrupt" message stdout := res.Stdout.String() require.Contains(t, stdout, "waiting for interrupt") // Note: "Received interrupt, removing forwarder" message is NOT visible to CLI on SIGTERM // because the response stream closes before the message can be emitted. }) t.Run("listen without foreground returns immediately and persists", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.IPFS("config", "--json", "Experimental.Libp2pStreamMounting", "true") node.StartDaemon() listenPort := harness.NewRandPort() // This should return immediately (not block) node.IPFS("p2p", "listen", "/x/nofg", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", listenPort)) // Listener should still exist waitForListenerProtocol(t, node, "/x/nofg") // Clean up node.IPFS("p2p", "close", "-p", "/x/nofg") }) t.Run("listen foreground text output on p2p close", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.IPFS("config", "--json", "Experimental.Libp2pStreamMounting", "true") node.StartDaemon() listenPort := harness.NewRandPort() // Run without --enc=json to test actual text output users see res := node.Runner.Run(harness.RunRequest{ Path: node.IPFSBin, Args: []string{"p2p", "listen", "--foreground", "/x/closetest", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", listenPort)}, RunFunc: (*exec.Cmd).Start, }) require.NoError(t, res.Err) // Wait for listener to be created waitForListenerProtocol(t, node, "/x/closetest") // Close the listener via ipfs p2p close command node.IPFS("p2p", "close", "-p", "/x/closetest") // Wait for foreground command to exit (it should exit quickly after close) done := make(chan error, 1) go func() { done <- res.Cmd.Wait() }() select { case <-done: // Good - command exited case <-time.After(5 * time.Second): _ = res.Cmd.Process.Kill() t.Fatal("foreground command did not exit after listener was closed via ipfs p2p close") } // Wait for listener to be removed waitForListenerCount(t, node, 0) // Verify text output shows BOTH messages when closed via p2p close // (unlike SIGTERM, the stream is still open so "Received interrupt" is emitted) out := res.Stdout.String() require.Contains(t, out, "waiting for interrupt") require.Contains(t, out, "Received interrupt, removing listener") }) t.Run("forward foreground text output on p2p close", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init() nodes.ForEachPar(func(n *harness.Node) { n.IPFS("config", "--json", "Experimental.Libp2pStreamMounting", "true") }) nodes.StartDaemons().Connect() forwardPort := harness.NewRandPort() // Run without --enc=json to test actual text output users see res := nodes[0].Runner.Run(harness.RunRequest{ Path: nodes[0].IPFSBin, Args: []string{"p2p", "forward", "--foreground", "/x/fwdclose", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", forwardPort), "/p2p/" + nodes[1].PeerID().String()}, RunFunc: (*exec.Cmd).Start, }) require.NoError(t, res.Err) // Wait for forwarder to be created waitForListenerCount(t, nodes[0], 1) // Close the forwarder via ipfs p2p close command nodes[0].IPFS("p2p", "close", "-a") // Wait for foreground command to exit done := make(chan error, 1) go func() { done <- res.Cmd.Wait() }() select { case <-done: // Good - command exited case <-time.After(5 * time.Second): _ = res.Cmd.Process.Kill() t.Fatal("foreground command did not exit after forwarder was closed via ipfs p2p close") } // Wait for forwarder to be removed waitForListenerCount(t, nodes[0], 0) // Verify text output shows BOTH messages when closed via p2p close out := res.Stdout.String() require.Contains(t, out, "waiting for interrupt") require.Contains(t, out, "Received interrupt, removing forwarder") }) t.Run("listen foreground tunnel transfers data and cleans up on SIGTERM", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init() nodes.ForEachPar(func(n *harness.Node) { n.IPFS("config", "--json", "Experimental.Libp2pStreamMounting", "true") }) nodes.StartDaemons().Connect() httpServerPort := harness.NewRandPort() forwardPort := harness.NewRandPort() // Start HTTP server expectedBody := "Hello from p2p tunnel!" httpServer := &http.Server{ Addr: fmt.Sprintf("127.0.0.1:%d", httpServerPort), Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(expectedBody)) }), } listener, err := net.Listen("tcp", httpServer.Addr) require.NoError(t, err) go func() { _ = httpServer.Serve(listener) }() defer httpServer.Close() // Node 0: listen --foreground listenRes := nodes[0].Runner.Run(harness.RunRequest{ Path: nodes[0].IPFSBin, Args: []string{"p2p", "listen", "--foreground", "/x/httptest", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", httpServerPort)}, RunFunc: (*exec.Cmd).Start, }) require.NoError(t, listenRes.Err) // Wait for listener to be created waitForListenerProtocol(t, nodes[0], "/x/httptest") // Node 1: forward (non-foreground) nodes[1].IPFS("p2p", "forward", "/x/httptest", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", forwardPort), "/p2p/"+nodes[0].PeerID().String()) // Verify data flows through tunnel resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/", forwardPort)) require.NoError(t, err) body, err := io.ReadAll(resp.Body) resp.Body.Close() require.NoError(t, err) require.Equal(t, expectedBody, string(body)) // Clean up forwarder on node 1 nodes[1].IPFS("p2p", "close", "-a") // SIGTERM the listen --foreground command _ = listenRes.Cmd.Process.Signal(syscall.SIGTERM) _ = listenRes.Cmd.Wait() // Wait for listener to be removed on node 0 waitForListenerCount(t, nodes[0], 0) }) t.Run("forward foreground tunnel transfers data and cleans up on SIGTERM", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init() nodes.ForEachPar(func(n *harness.Node) { n.IPFS("config", "--json", "Experimental.Libp2pStreamMounting", "true") }) nodes.StartDaemons().Connect() httpServerPort := harness.NewRandPort() forwardPort := harness.NewRandPort() // Start HTTP server expectedBody := "Hello from forward foreground tunnel!" httpServer := &http.Server{ Addr: fmt.Sprintf("127.0.0.1:%d", httpServerPort), Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(expectedBody)) }), } listener, err := net.Listen("tcp", httpServer.Addr) require.NoError(t, err) go func() { _ = httpServer.Serve(listener) }() defer httpServer.Close() // Node 0: listen (non-foreground) nodes[0].IPFS("p2p", "listen", "/x/httptest", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", httpServerPort)) // Node 1: forward --foreground forwardRes := nodes[1].Runner.Run(harness.RunRequest{ Path: nodes[1].IPFSBin, Args: []string{"p2p", "forward", "--foreground", "/x/httptest", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", forwardPort), "/p2p/" + nodes[0].PeerID().String()}, RunFunc: (*exec.Cmd).Start, }) require.NoError(t, forwardRes.Err) // Wait for forwarder to be created waitForListenerCount(t, nodes[1], 1) // Verify data flows through tunnel resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/", forwardPort)) require.NoError(t, err) body, err := io.ReadAll(resp.Body) resp.Body.Close() require.NoError(t, err) require.Equal(t, expectedBody, string(body)) // SIGTERM the forward --foreground command _ = forwardRes.Cmd.Process.Signal(syscall.SIGTERM) _ = forwardRes.Cmd.Wait() // Wait for forwarder to be removed on node 1 waitForListenerCount(t, nodes[1], 0) // Clean up listener on node 0 nodes[0].IPFS("p2p", "close", "-a") }) t.Run("foreground command exits when daemon shuts down", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.IPFS("config", "--json", "Experimental.Libp2pStreamMounting", "true") node.StartDaemon() listenPort := harness.NewRandPort() // Start foreground listener res := node.Runner.Run(harness.RunRequest{ Path: node.IPFSBin, Args: []string{"p2p", "listen", "--foreground", "/x/daemontest", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", listenPort)}, RunFunc: (*exec.Cmd).Start, }) require.NoError(t, res.Err) // Wait for listener to be created waitForListenerProtocol(t, node, "/x/daemontest") // Stop the daemon node.StopDaemon() // Wait for foreground command to exit done := make(chan error, 1) go func() { done <- res.Cmd.Wait() }() select { case <-done: // Good - foreground command exited when daemon stopped case <-time.After(5 * time.Second): _ = res.Cmd.Process.Kill() t.Fatal("foreground command did not exit when daemon was stopped") } }) } ================================================ FILE: test/cli/peering_test.go ================================================ package cli import ( "slices" "testing" "time" "github.com/ipfs/kubo/test/cli/harness" . "github.com/ipfs/kubo/test/cli/testutils" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/assert" ) func TestPeering(t *testing.T) { t.Parallel() containsPeerID := func(p peer.ID, peers []peer.ID) bool { return slices.Contains(peers, p) } assertPeered := func(h *harness.Harness, from *harness.Node, to *harness.Node) { assert.Eventuallyf(t, func() bool { fromPeers := from.Peers() if len(fromPeers) == 0 { return false } var fromPeerIDs []peer.ID for _, p := range fromPeers { fromPeerIDs = append(fromPeerIDs, h.ExtractPeerID(p)) } return containsPeerID(to.PeerID(), fromPeerIDs) }, time.Minute, 10*time.Millisecond, "%d -> %d not peered", from.ID, to.ID) } assertNotPeered := func(h *harness.Harness, from *harness.Node, to *harness.Node) { assert.Eventuallyf(t, func() bool { fromPeers := from.Peers() if len(fromPeers) == 0 { return false } var fromPeerIDs []peer.ID for _, p := range fromPeers { fromPeerIDs = append(fromPeerIDs, h.ExtractPeerID(p)) } return !containsPeerID(to.PeerID(), fromPeerIDs) }, 20*time.Second, 10*time.Millisecond, "%d -> %d peered", from.ID, to.ID) } assertPeerings := func(h *harness.Harness, nodes []*harness.Node, peerings []harness.Peering) { ForEachPar(peerings, func(peering harness.Peering) { assertPeered(h, nodes[peering.From], nodes[peering.To]) }) } t.Run("bidirectional peering should work (simultaneous connect)", func(t *testing.T) { t.Parallel() peerings := []harness.Peering{{From: 0, To: 1}, {From: 1, To: 0}, {From: 1, To: 2}} h, nodes := harness.CreatePeerNodes(t, 3, peerings) nodes.StartDaemons() defer nodes.StopDaemons() assertPeerings(h, nodes, peerings) nodes[0].Disconnect(nodes[1]) assertPeerings(h, nodes, peerings) }) t.Run("1 should reconnect to 2 when 2 disconnects from 1", func(t *testing.T) { t.Parallel() peerings := []harness.Peering{{From: 0, To: 1}, {From: 1, To: 0}, {From: 1, To: 2}} h, nodes := harness.CreatePeerNodes(t, 3, peerings) nodes.StartDaemons() defer nodes.StopDaemons() assertPeerings(h, nodes, peerings) nodes[2].Disconnect(nodes[1]) assertPeerings(h, nodes, peerings) }) t.Run("1 will peer with 2 when it comes online", func(t *testing.T) { t.Parallel() peerings := []harness.Peering{{From: 0, To: 1}, {From: 1, To: 0}, {From: 1, To: 2}} h, nodes := harness.CreatePeerNodes(t, 3, peerings) defer nodes.StopDaemons() nodes[0].StartDaemon() nodes[1].StartDaemon() assertPeerings(h, nodes, []harness.Peering{{From: 0, To: 1}, {From: 1, To: 0}}) nodes[2].StartDaemon() assertPeerings(h, nodes, peerings) }) t.Run("1 will re-peer with 2 when it disconnects and then comes back online", func(t *testing.T) { t.Parallel() peerings := []harness.Peering{{From: 0, To: 1}, {From: 1, To: 0}, {From: 1, To: 2}} h, nodes := harness.CreatePeerNodes(t, 3, peerings) nodes.StartDaemons() defer nodes.StopDaemons() assertPeerings(h, nodes, peerings) nodes[2].StopDaemon() assertNotPeered(h, nodes[1], nodes[2]) nodes[2].StartDaemon() assertPeerings(h, nodes, peerings) }) } ================================================ FILE: test/cli/pin_ls_names_test.go ================================================ package cli import ( "encoding/json" "fmt" "os" "path/filepath" "strings" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/require" ) // pinInfo represents the JSON structure for pin ls output type pinInfo struct { Type string `json:"Type"` Name string `json:"Name"` } // pinLsJSON represents the JSON output structure for pin ls command type pinLsJSON struct { Keys map[string]pinInfo `json:"Keys"` } // Helper function to initialize a test node with daemon func setupTestNode(t *testing.T) *harness.Node { t.Helper() node := harness.NewT(t).NewNode().Init() node.StartDaemon("--offline") t.Cleanup(func() { node.StopDaemon() }) return node } // Helper function to assert pin name and CID are present in output func assertPinOutput(t *testing.T, output, cid, pinName string) { t.Helper() require.Contains(t, output, pinName, "pin name '%s' not found in output: %s", pinName, output) require.Contains(t, output, cid, "CID %s not found in output: %s", cid, output) } // Helper function to assert CID is present but name is not func assertCIDOnly(t *testing.T, output, cid string) { t.Helper() require.Contains(t, output, cid, "CID %s not found in output: %s", cid, output) } // Helper function to assert neither CID nor name are present func assertNotPresent(t *testing.T, output, cid, pinName string) { t.Helper() require.NotContains(t, output, cid, "CID %s should not be present in output: %s", cid, output) require.NotContains(t, output, pinName, "pin name '%s' should not be present in output: %s", pinName, output) } // Test that pin ls returns names when querying specific CIDs with --names flag func TestPinLsWithNamesForSpecificCIDs(t *testing.T) { t.Parallel() t.Run("pin ls with specific CID returns name", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Add content without pinning cidA := node.IPFSAddStr("content A", "--pin=false") cidB := node.IPFSAddStr("content B", "--pin=false") cidC := node.IPFSAddStr("content C", "--pin=false") // Pin with names node.IPFS("pin", "add", "--name=pin-a", cidA) node.IPFS("pin", "add", "--name=pin-b", cidB) node.IPFS("pin", "add", cidC) // No name // Test: pin ls --names should return the name res := node.IPFS("pin", "ls", cidA, "--names") assertPinOutput(t, res.Stdout.String(), cidA, "pin-a") res = node.IPFS("pin", "ls", cidB, "--names") assertPinOutput(t, res.Stdout.String(), cidB, "pin-b") // Test: pin without name should work res = node.IPFS("pin", "ls", cidC, "--names") output := res.Stdout.String() assertCIDOnly(t, output, cidC) require.Contains(t, output, "recursive", "pin type 'recursive' not found for CID %s in output: %s", cidC, output) // Test: without --names flag, no names returned res = node.IPFS("pin", "ls", cidA) output = res.Stdout.String() require.NotContains(t, output, "pin-a", "pin name 'pin-a' should not be present without --names flag, but found in: %s", output) assertCIDOnly(t, output, cidA) }) t.Run("pin ls with multiple CIDs returns names", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Create test content cidA := node.IPFSAddStr("multi A", "--pin=false") cidB := node.IPFSAddStr("multi B", "--pin=false") // Pin with names node.IPFS("pin", "add", "--name=multi-pin-a", cidA) node.IPFS("pin", "add", "--name=multi-pin-b", cidB) // Test multiple CIDs at once res := node.IPFS("pin", "ls", cidA, cidB, "--names") output := res.Stdout.String() assertPinOutput(t, output, cidA, "multi-pin-a") assertPinOutput(t, output, cidB, "multi-pin-b") }) t.Run("pin ls without CID lists all pins with names", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Create and pin content with names cidA := node.IPFSAddStr("list all A", "--pin=false") cidB := node.IPFSAddStr("list all B", "--pin=false") cidC := node.IPFSAddStr("list all C", "--pin=false") node.IPFS("pin", "add", "--name=all-pin-a", cidA) node.IPFS("pin", "add", "--name=all-pin-b", "--recursive=false", cidB) node.IPFS("pin", "add", cidC) // No name // Test: pin ls --names (without CID) should list all pins with their names res := node.IPFS("pin", "ls", "--names") output := res.Stdout.String() // Should contain all pins with their names assertPinOutput(t, output, cidA, "all-pin-a") assertPinOutput(t, output, cidB, "all-pin-b") assertCIDOnly(t, output, cidC) // Pin C should appear but without a name (just type) lines := strings.SplitSeq(output, "\n") for line := range lines { if strings.Contains(line, cidC) { // Should have CID and type but no name require.Contains(t, line, "recursive", "pin type 'recursive' not found for unnamed pin %s in line: %s", cidC, line) require.NotContains(t, line, "all-pin", "pin name should not be present for unnamed pin %s, but found in line: %s", cidC, line) } } }) t.Run("pin ls --type with --names", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Create test content cidDirect := node.IPFSAddStr("direct content", "--pin=false") cidRecursive := node.IPFSAddStr("recursive content", "--pin=false") // Create a DAG for indirect testing childCid := node.IPFSAddStr("child for indirect", "--pin=false") parentContent := fmt.Sprintf(`{"link": "/ipfs/%s"}`, childCid) parentCid := node.PipeStrToIPFS(parentContent, "dag", "put", "--input-codec=json", "--store-codec=dag-cbor").Stdout.Trimmed() // Pin with different types and names node.IPFS("pin", "add", "--name=direct-pin", "--recursive=false", cidDirect) node.IPFS("pin", "add", "--name=recursive-pin", cidRecursive) node.IPFS("pin", "add", "--name=parent-pin", parentCid) // Test: --type=direct --names res := node.IPFS("pin", "ls", "--type=direct", "--names") output := res.Stdout.String() assertPinOutput(t, output, cidDirect, "direct-pin") assertNotPresent(t, output, cidRecursive, "recursive-pin") // Test: --type=recursive --names res = node.IPFS("pin", "ls", "--type=recursive", "--names") output = res.Stdout.String() assertPinOutput(t, output, cidRecursive, "recursive-pin") assertPinOutput(t, output, parentCid, "parent-pin") assertNotPresent(t, output, cidDirect, "direct-pin") // Test: --type=indirect with proper directory structure // Create a directory with a file for indirect pin testing dirPath := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dirPath, "file.txt"), []byte("test content"), 0644)) // Add directory recursively dirAddRes := node.IPFS("add", "-r", "-q", dirPath) dirCidStr := strings.TrimSpace(dirAddRes.Stdout.Lines()[len(dirAddRes.Stdout.Lines())-1]) // Add file separately without pinning to get its CID fileAddRes := node.IPFS("add", "-q", "--pin=false", filepath.Join(dirPath, "file.txt")) fileCidStr := strings.TrimSpace(fileAddRes.Stdout.String()) // Check if file shows as indirect res = node.IPFS("pin", "ls", "--type=indirect", fileCidStr) output = res.Stdout.String() require.Contains(t, output, fileCidStr, "indirect pin CID %s not found in output: %s", fileCidStr, output) require.Contains(t, output, "indirect through "+dirCidStr, "indirect relationship not found for CID %s through %s in output: %s", fileCidStr, dirCidStr, output) // Test: --type=all --names res = node.IPFS("pin", "ls", "--type=all", "--names") output = res.Stdout.String() assertPinOutput(t, output, cidDirect, "direct-pin") assertPinOutput(t, output, cidRecursive, "recursive-pin") assertPinOutput(t, output, parentCid, "parent-pin") // Indirect pins are included in --type=all output }) t.Run("pin ls JSON output with names", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Add and pin content with name cidA := node.IPFSAddStr("json content", "--pin=false") node.IPFS("pin", "add", "--name=json-pin", cidA) // Test JSON output with specific CID res := node.IPFS("pin", "ls", cidA, "--names", "--enc=json") var pinOutput pinLsJSON err := json.Unmarshal([]byte(res.Stdout.String()), &pinOutput) require.NoError(t, err, "failed to unmarshal JSON output: %s", res.Stdout.String()) pinData, ok := pinOutput.Keys[cidA] require.True(t, ok, "CID %s should be in Keys map, got: %+v", cidA, pinOutput.Keys) require.Equal(t, "recursive", pinData.Type, "expected pin type 'recursive', got '%s'", pinData.Type) require.Equal(t, "json-pin", pinData.Name, "expected pin name 'json-pin', got '%s'", pinData.Name) // Without names flag res = node.IPFS("pin", "ls", cidA, "--enc=json") err = json.Unmarshal([]byte(res.Stdout.String()), &pinOutput) require.NoError(t, err, "failed to unmarshal JSON output: %s", res.Stdout.String()) pinData, ok = pinOutput.Keys[cidA] require.True(t, ok, "CID %s should be in Keys map, got: %+v", cidA, pinOutput.Keys) // Name should be empty without --names flag require.Equal(t, "", pinData.Name, "pin name should be empty without --names flag, got '%s'", pinData.Name) // Test JSON output without CID (list all) res = node.IPFS("pin", "ls", "--names", "--enc=json") var listOutput pinLsJSON err = json.Unmarshal([]byte(res.Stdout.String()), &listOutput) require.NoError(t, err, "failed to unmarshal JSON list output: %s", res.Stdout.String()) // Should have at least one pin (the one we just added) require.NotEmpty(t, listOutput.Keys, "pin list should not be empty") // Check that our pin is in the list pinData, ok = listOutput.Keys[cidA] require.True(t, ok, "our pin with CID %s should be in the list, got: %+v", cidA, listOutput.Keys) require.Equal(t, "json-pin", pinData.Name, "expected pin name 'json-pin' in list, got '%s'", pinData.Name) }) t.Run("direct and indirect pins with names", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Create a small DAG: parent -> child childCid := node.IPFSAddStr("child content", "--pin=false") // Create parent that references child parentContent := fmt.Sprintf(`{"link": "/ipfs/%s"}`, childCid) parentCid := node.PipeStrToIPFS(parentContent, "dag", "put", "--input-codec=json", "--store-codec=dag-cbor").Stdout.Trimmed() // Pin child directly with a name node.IPFS("pin", "add", "--name=direct-child", "--recursive=false", childCid) // Pin parent recursively with a name node.IPFS("pin", "add", "--name=recursive-parent", parentCid) // Check direct pin with specific CID res := node.IPFS("pin", "ls", "--type=direct", childCid, "--names") output := res.Stdout.String() require.Contains(t, output, "direct-child", "pin name 'direct-child' not found in output: %s", output) require.Contains(t, output, "direct", "pin type 'direct' not found in output: %s", output) // Check recursive pin with specific CID res = node.IPFS("pin", "ls", "--type=recursive", parentCid, "--names") output = res.Stdout.String() require.Contains(t, output, "recursive-parent", "pin name 'recursive-parent' not found in output: %s", output) require.Contains(t, output, "recursive", "pin type 'recursive' not found in output: %s", output) // Child is both directly pinned and indirectly pinned through parent // Both relationships are valid and can be checked }) t.Run("pin update preserves name", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Create two pieces of content cidOld := node.IPFSAddStr("old content", "--pin=false") cidNew := node.IPFSAddStr("new content", "--pin=false") // Pin with name node.IPFS("pin", "add", "--name=my-pin", cidOld) // Update pin node.IPFS("pin", "update", cidOld, cidNew) // Check that new pin has the same name res := node.IPFS("pin", "ls", cidNew, "--names") require.Contains(t, res.Stdout.String(), "my-pin", "pin name 'my-pin' not preserved after update, output: %s", res.Stdout.String()) // Old pin should not exist res = node.RunIPFS("pin", "ls", cidOld) require.Equal(t, 1, res.ExitCode(), "expected exit code 1 for unpinned CID, got %d", res.ExitCode()) require.Contains(t, res.Stderr.String(), "is not pinned", "expected 'is not pinned' error for old CID %s, got: %s", cidOld, res.Stderr.String()) }) t.Run("pin ls with invalid CID returns error", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() res := node.RunIPFS("pin", "ls", "invalid-cid") require.Equal(t, 1, res.ExitCode(), "expected exit code 1 for invalid CID, got %d", res.ExitCode()) require.Contains(t, res.Stderr.String(), "invalid", "expected 'invalid' in error message, got: %s", res.Stderr.String()) }) t.Run("pin ls with unpinned CID returns error", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Add content without pinning cid := node.IPFSAddStr("unpinned content", "--pin=false") res := node.RunIPFS("pin", "ls", cid) require.Equal(t, 1, res.ExitCode(), "expected exit code 1 for unpinned CID, got %d", res.ExitCode()) require.Contains(t, res.Stderr.String(), "is not pinned", "expected 'is not pinned' error for CID %s, got: %s", cid, res.Stderr.String()) }) t.Run("pin with special characters in name", func(t *testing.T) { t.Parallel() node := setupTestNode(t) testCases := []struct { name string pinName string }{ {"unicode", "test-📌-pin"}, {"spaces", "test pin name"}, {"special chars", "test!@#$%"}, {"path-like", "test/pin/name"}, {"dots", "test.pin.name"}, {"long name", strings.Repeat("a", 255)}, {"empty name", ""}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { cid := node.IPFSAddStr("content for "+tc.name, "--pin=false") node.IPFS("pin", "add", "--name="+tc.pinName, cid) res := node.IPFS("pin", "ls", cid, "--names") if tc.pinName != "" { require.Contains(t, res.Stdout.String(), tc.pinName, "pin name '%s' not found in output for test case '%s'", tc.pinName, tc.name) } }) } }) t.Run("concurrent pin operations with names", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Create multiple goroutines adding pins with names numPins := 10 done := make(chan struct{}, numPins) for i := range numPins { go func(idx int) { defer func() { done <- struct{}{} }() content := fmt.Sprintf("concurrent content %d", idx) cid := node.IPFSAddStr(content, "--pin=false") pinName := fmt.Sprintf("concurrent-pin-%d", idx) node.IPFS("pin", "add", "--name="+pinName, cid) }(i) } // Wait for all goroutines for range numPins { <-done } // Verify all pins have correct names res := node.IPFS("pin", "ls", "--names") output := res.Stdout.String() for i := range numPins { pinName := fmt.Sprintf("concurrent-pin-%d", i) require.Contains(t, output, pinName, "concurrent pin name '%s' not found in output", pinName) } }) t.Run("pin rm removes name association", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Add and pin with name cid := node.IPFSAddStr("content to remove", "--pin=false") node.IPFS("pin", "add", "--name=to-be-removed", cid) // Verify pin exists with name res := node.IPFS("pin", "ls", cid, "--names") require.Contains(t, res.Stdout.String(), "to-be-removed") // Remove pin node.IPFS("pin", "rm", cid) // Verify pin and name are gone res = node.RunIPFS("pin", "ls", cid) require.Equal(t, 1, res.ExitCode()) require.Contains(t, res.Stderr.String(), "is not pinned") }) t.Run("garbage collection preserves named pins", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Add content with and without pin names cidNamed := node.IPFSAddStr("named content", "--pin=false") cidUnnamed := node.IPFSAddStr("unnamed content", "--pin=false") cidUnpinned := node.IPFSAddStr("unpinned content", "--pin=false") node.IPFS("pin", "add", "--name=important-data", cidNamed) node.IPFS("pin", "add", cidUnnamed) // Run garbage collection node.IPFS("repo", "gc") // Named and unnamed pins should still exist res := node.IPFS("pin", "ls", cidNamed, "--names") require.Contains(t, res.Stdout.String(), "important-data") res = node.IPFS("pin", "ls", cidUnnamed) require.Contains(t, res.Stdout.String(), cidUnnamed) // Unpinned content should be gone (cat should fail) res = node.RunIPFS("cat", cidUnpinned) require.NotEqual(t, 0, res.ExitCode(), "unpinned content should be garbage collected") }) t.Run("pin add with same name can be used for multiple pins", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Add two different pieces of content cid1 := node.IPFSAddStr("first content", "--pin=false") cid2 := node.IPFSAddStr("second content", "--pin=false") // Pin both with the same name - this is allowed node.IPFS("pin", "add", "--name=shared-name", cid1) node.IPFS("pin", "add", "--name=shared-name", cid2) // List all pins with names res := node.IPFS("pin", "ls", "--names") output := res.Stdout.String() // Both CIDs should be pinned require.Contains(t, output, cid1) require.Contains(t, output, cid2) // Both pins can have the same name lines := strings.Split(output, "\n") foundCid1WithName := false foundCid2WithName := false for _, line := range lines { if strings.Contains(line, cid1) && strings.Contains(line, "shared-name") { foundCid1WithName = true } if strings.Contains(line, cid2) && strings.Contains(line, "shared-name") { foundCid2WithName = true } } require.True(t, foundCid1WithName, "first pin should have the name") require.True(t, foundCid2WithName, "second pin should have the name") }) t.Run("pin names persist across daemon restarts", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.StartDaemon("--offline") // Add content with pin name cid := node.IPFSAddStr("persistent content") node.IPFS("pin", "add", "--name=persistent-pin", cid) // Restart daemon node.StopDaemon() node.StartDaemon("--offline") // Check pin name persisted res := node.IPFS("pin", "ls", cid, "--names") require.Contains(t, res.Stdout.String(), "persistent-pin", "pin name should persist across daemon restarts") node.StopDaemon() }) } // TestPinLsEdgeCases tests edge cases for pin ls command func TestPinLsEdgeCases(t *testing.T) { t.Parallel() t.Run("invalid pin type returns error", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Try to list pins with invalid type res := node.RunIPFS("pin", "ls", "--type=invalid") require.NotEqual(t, 0, res.ExitCode()) require.Contains(t, res.Stderr.String(), "invalid type 'invalid'") require.Contains(t, res.Stderr.String(), "must be one of {direct, indirect, recursive, all}") }) t.Run("known but non-listable pin type returns error", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // "internal" is a valid pin.Mode in boxo but not a valid --type for pin ls. // Before the fix, this caused a panic instead of returning an error. res := node.RunIPFS("pin", "ls", "--type=internal") require.NotEqual(t, 0, res.ExitCode()) require.Contains(t, res.Stderr.String(), "invalid type 'internal'") }) t.Run("non-existent path returns proper error", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Try to list a non-existent CID fakeCID := "QmNonExistent123456789" res := node.RunIPFS("pin", "ls", fakeCID) require.NotEqual(t, 0, res.ExitCode()) }) t.Run("unpinned CID returns not pinned error", func(t *testing.T) { t.Parallel() node := setupTestNode(t) // Add content but don't pin it explicitly (it's just in blockstore) unpinnedCID := node.IPFSAddStr("unpinned content", "--pin=false") // Try to list specific unpinned CID res := node.RunIPFS("pin", "ls", unpinnedCID) require.NotEqual(t, 0, res.ExitCode()) require.Contains(t, res.Stderr.String(), "is not pinned") }) } ================================================ FILE: test/cli/pin_name_validation_test.go ================================================ package cli import ( "fmt" "strings" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/require" ) func TestPinNameValidation(t *testing.T) { t.Parallel() // Create a test node and add a test file node := harness.NewT(t).NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() // Add a test file to get a CID testContent := "test content for pin name validation" testCID := node.IPFSAddStr(testContent, "--pin=false") t.Run("pin add accepts valid names", func(t *testing.T) { testCases := []struct { name string pinName string description string }{ { name: "empty_name", pinName: "", description: "Empty name should be allowed", }, { name: "short_name", pinName: "test", description: "Short ASCII name should be allowed", }, { name: "max_255_bytes", pinName: strings.Repeat("a", 255), description: "Exactly 255 bytes should be allowed", }, { name: "unicode_within_limit", pinName: "测试名称🔥", // Chinese characters and emoji description: "Unicode characters within 255 bytes should be allowed", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var args []string if tc.pinName != "" { args = []string{"pin", "add", "--name", tc.pinName, testCID} } else { args = []string{"pin", "add", testCID} } res := node.RunIPFS(args...) require.Equal(t, 0, res.ExitCode(), tc.description) // Clean up - unpin node.RunIPFS("pin", "rm", testCID) }) } }) t.Run("pin add rejects names exceeding 255 bytes", func(t *testing.T) { testCases := []struct { name string pinName string description string }{ { name: "256_bytes", pinName: strings.Repeat("a", 256), description: "256 bytes should be rejected", }, { name: "300_bytes", pinName: strings.Repeat("b", 300), description: "300 bytes should be rejected", }, { name: "unicode_exceeding_limit", pinName: strings.Repeat("测", 100), // Each Chinese character is 3 bytes, total 300 bytes description: "Unicode string exceeding 255 bytes should be rejected", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { res := node.RunIPFS("pin", "add", "--name", tc.pinName, testCID) require.NotEqual(t, 0, res.ExitCode(), tc.description) require.Contains(t, res.Stderr.String(), "max 255 bytes", "Error should mention the 255 byte limit") }) } }) t.Run("pin ls with name filter validates length", func(t *testing.T) { // Test valid filter res := node.RunIPFS("pin", "ls", "--name", strings.Repeat("a", 255)) require.Equal(t, 0, res.ExitCode(), "255-byte name filter should be accepted") // Test invalid filter res = node.RunIPFS("pin", "ls", "--name", strings.Repeat("a", 256)) require.NotEqual(t, 0, res.ExitCode(), "256-byte name filter should be rejected") require.Contains(t, res.Stderr.String(), "max 255 bytes", "Error should mention the 255 byte limit") }) } func TestAddPinNameValidation(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon("--offline") defer node.StopDaemon() // Create a test file testFile := "test.txt" node.WriteBytes(testFile, []byte("test content for add command")) t.Run("ipfs add with --pin-name accepts valid names", func(t *testing.T) { testCases := []struct { name string pinName string description string }{ { name: "short_name", pinName: "test-add", description: "Short ASCII name should be allowed", }, { name: "max_255_bytes", pinName: strings.Repeat("x", 255), description: "Exactly 255 bytes should be allowed", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { res := node.RunIPFS("add", fmt.Sprintf("--pin-name=%s", tc.pinName), "-q", testFile) require.Equal(t, 0, res.ExitCode(), tc.description) cid := strings.TrimSpace(res.Stdout.String()) // Verify pin exists with name lsRes := node.RunIPFS("pin", "ls", "--names", "--type=recursive", cid) require.Equal(t, 0, lsRes.ExitCode()) require.Contains(t, lsRes.Stdout.String(), tc.pinName, "Pin should have the specified name") // Clean up node.RunIPFS("pin", "rm", cid) }) } }) t.Run("ipfs add with --pin-name rejects names exceeding 255 bytes", func(t *testing.T) { testCases := []struct { name string pinName string description string }{ { name: "256_bytes", pinName: strings.Repeat("y", 256), description: "256 bytes should be rejected", }, { name: "500_bytes", pinName: strings.Repeat("z", 500), description: "500 bytes should be rejected", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { res := node.RunIPFS("add", fmt.Sprintf("--pin-name=%s", tc.pinName), testFile) require.NotEqual(t, 0, res.ExitCode(), tc.description) require.Contains(t, res.Stderr.String(), "max 255 bytes", "Error should mention the 255 byte limit") }) } }) } ================================================ FILE: test/cli/ping_test.go ================================================ package cli import ( "fmt" "strings" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" ) func TestPing(t *testing.T) { t.Parallel() t.Run("other", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init().StartDaemons().Connect() defer nodes.StopDaemons() node1 := nodes[0] node2 := nodes[1] node1.IPFS("ping", "-n", "2", "--", node2.PeerID().String()) node2.IPFS("ping", "-n", "2", "--", node1.PeerID().String()) }) t.Run("ping unreachable peer", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init().StartDaemons().Connect() defer nodes.StopDaemons() node1 := nodes[0] badPeer := "QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJx" res := node1.RunIPFS("ping", "-n", "2", "--", badPeer) assert.Contains(t, res.Stdout.String(), fmt.Sprintf("Looking up peer %s", badPeer)) msg := res.Stderr.String() assert.Truef(t, strings.HasPrefix(msg, "Error:"), "should fail got this instead: %q", msg) }) t.Run("self", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init().StartDaemons() defer nodes.StopDaemons() node1 := nodes[0] node2 := nodes[1] res := node1.RunIPFS("ping", "-n", "2", "--", node1.PeerID().String()) assert.Equal(t, 1, res.Cmd.ProcessState.ExitCode()) assert.Contains(t, res.Stderr.String(), "can't ping self") res = node2.RunIPFS("ping", "-n", "2", "--", node2.PeerID().String()) assert.Equal(t, 1, res.Cmd.ProcessState.ExitCode()) assert.Contains(t, res.Stderr.String(), "can't ping self") }) t.Run("0", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init().StartDaemons().Connect() defer nodes.StopDaemons() node1 := nodes[0] node2 := nodes[1] res := node1.RunIPFS("ping", "-n", "0", "--", node2.PeerID().String()) assert.Equal(t, 1, res.Cmd.ProcessState.ExitCode()) assert.Contains(t, res.Stderr.String(), "ping count must be greater than 0") }) t.Run("offline", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init().StartDaemons().Connect() defer nodes.StopDaemons() node1 := nodes[0] node2 := nodes[1] node2.StopDaemon() res := node1.RunIPFS("ping", "-n", "2", "--", node2.PeerID().String()) assert.Equal(t, 1, res.Cmd.ProcessState.ExitCode()) assert.Contains(t, res.Stderr.String(), "ping failed") }) } ================================================ FILE: test/cli/pinning_remote_test.go ================================================ package cli import ( "errors" "fmt" "net" "net/http" "testing" "time" "github.com/google/uuid" "github.com/ipfs/go-test/random" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils/pinningservice" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) func runPinningService(t *testing.T, authToken string) (*pinningservice.PinningService, string) { svc := pinningservice.New() router := pinningservice.NewRouter(authToken, svc) server := &http.Server{Handler: router} listener, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) go func() { err := server.Serve(listener) if err != nil && !errors.Is(err, net.ErrClosed) && !errors.Is(err, http.ErrServerClosed) { t.Logf("Serve error: %s", err) } }() t.Cleanup(func() { listener.Close() }) return svc, fmt.Sprintf("http://%s/api/v1", listener.Addr().String()) } func TestRemotePinning(t *testing.T) { t.Parallel() authToken := "testauthtoken" t.Run("MFS pinning", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.Runner.Env["MFS_PIN_POLL_INTERVAL"] = "10ms" _, svcURL := runPinningService(t, authToken) node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.RepinInterval", `"1s"`) node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.PinName", `"test_pin"`) node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.Enable", "true") node.StartDaemon() t.Cleanup(func() { node.StopDaemon() }) node.IPFS("files", "cp", "/ipfs/bafkqaaa", "/mfs-pinning-test-"+uuid.NewString()) node.IPFS("files", "flush") res := node.IPFS("files", "stat", "/", "--enc=json") hash := gjson.Get(res.Stdout.String(), "Hash").Str assert.Eventually(t, func() bool { res = node.IPFS("pin", "remote", "ls", "--service=svc", "--name=test_pin", "--status=queued,pinning,pinned,failed", "--enc=json", ) pinnedHash := gjson.Get(res.Stdout.String(), "Cid").Str return hash == pinnedHash }, 10*time.Second, 10*time.Millisecond, ) t.Run("MFS root is repinned on CID change", func(t *testing.T) { node.IPFS("files", "cp", "/ipfs/bafkqaaa", "/mfs-pinning-repin-test-"+uuid.NewString()) node.IPFS("files", "flush") res = node.IPFS("files", "stat", "/", "--enc=json") hash := gjson.Get(res.Stdout.String(), "Hash").Str assert.Eventually(t, func() bool { res := node.IPFS("pin", "remote", "ls", "--service=svc", "--name=test_pin", "--status=queued,pinning,pinned,failed", "--enc=json", ) pinnedHash := gjson.Get(res.Stdout.String(), "Cid").Str return hash == pinnedHash }, 10*time.Second, 10*time.Millisecond, ) }) }) // Pinning.RemoteServices includes API.Key, so we give it the same treatment // as Identity,PrivKey to prevent exposing it on the network t.Run("access token security", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.IPFS("pin", "remote", "service", "add", "1", "http://example1.com", "testkey") res := node.RunIPFS("config", "Pinning") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "cannot show or change pinning services credentials") assert.NotContains(t, res.Stdout.String(), "testkey") res = node.RunIPFS("config", "Pinning.RemoteServices.1.API.Key") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "cannot show or change pinning services credentials") assert.NotContains(t, res.Stdout.String(), "testkey") configShow := node.RunIPFS("config", "show").Stdout.String() assert.NotContains(t, configShow, "testkey") t.Run("re-injecting config with 'ipfs config replace' preserves the API keys", func(t *testing.T) { node.WriteBytes("config-show", []byte(configShow)) node.IPFS("config", "replace", "config-show") assert.Contains(t, node.ReadFile(node.ConfigFile()), "testkey") }) t.Run("injecting config with 'ipfs config replace' with API keys returns an error", func(t *testing.T) { // remove Identity.PrivKey to ensure error is triggered by Pinning.RemoteServices configJSON := MustVal(sjson.Delete(configShow, "Identity.PrivKey")) configJSON = MustVal(sjson.Set(configJSON, "Pinning.RemoteServices.1.API.Key", "testkey")) node.WriteBytes("new-config", []byte(configJSON)) res := node.RunIPFS("config", "replace", "new-config") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "cannot change remote pinning services api info with `config replace`") }) }) t.Run("pin remote service ls --stat", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() _, svcURL := runPinningService(t, authToken) node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) node.IPFS("pin", "remote", "service", "add", "invalid-svc", svcURL+"/invalidpath", authToken) res := node.IPFS("pin", "remote", "service", "ls", "--stat") assert.Contains(t, res.Stdout.String(), " 0/0/0/0") stats := node.IPFS("pin", "remote", "service", "ls", "--stat", "--enc=json").Stdout.String() assert.Equal(t, "valid", gjson.Get(stats, `RemoteServices.#(Service == "svc").Stat.Status`).Str) assert.Equal(t, "invalid", gjson.Get(stats, `RemoteServices.#(Service == "invalid-svc").Stat.Status`).Str) // no --stat returns no stat obj t.Run("no --stat returns no stat obj", func(t *testing.T) { res := node.IPFS("pin", "remote", "service", "ls", "--enc=json") assert.False(t, gjson.Get(res.Stdout.String(), `RemoteServices.#(Service == "svc").Stat`).Exists()) }) }) t.Run("adding service with invalid URL fails", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() res := node.RunIPFS("pin", "remote", "service", "add", "svc", "invalid-service.example.com", "key") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "service endpoint must be a valid HTTP URL") res = node.RunIPFS("pin", "remote", "service", "add", "svc", "xyz://invalid-service.example.com", "key") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "service endpoint must be a valid HTTP URL") }) t.Run("unauthorized pinning service calls fail", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() _, svcURL := runPinningService(t, authToken) node.IPFS("pin", "remote", "service", "add", "svc", svcURL, "othertoken") res := node.RunIPFS("pin", "remote", "ls", "--service=svc") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "access denied") }) t.Run("pinning service calls fail when there is a wrong path", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() _, svcURL := runPinningService(t, authToken) node.IPFS("pin", "remote", "service", "add", "svc", svcURL+"/invalid-path", authToken) res := node.RunIPFS("pin", "remote", "ls", "--service=svc") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "404") }) t.Run("pinning service calls fail when DNS resolution fails", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() node.IPFS("pin", "remote", "service", "add", "svc", "https://invalid-service.example.com", authToken) res := node.RunIPFS("pin", "remote", "ls", "--service=svc") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "no such host") }) t.Run("pin remote service rm", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() node.IPFS("pin", "remote", "service", "add", "svc", "https://example.com", authToken) node.IPFS("pin", "remote", "service", "rm", "svc") res := node.IPFS("pin", "remote", "service", "ls") assert.NotContains(t, res.Stdout.String(), "svc") }) t.Run("remote pinning", func(t *testing.T) { t.Parallel() verifyStatus := func(node *harness.Node, name, hash, status string) { resJSON := node.IPFS("pin", "remote", "ls", "--service=svc", "--enc=json", "--name="+name, "--status="+status, ).Stdout.String() assert.Equal(t, status, gjson.Get(resJSON, "Status").Str) assert.Equal(t, hash, gjson.Get(resJSON, "Cid").Str) assert.Equal(t, name, gjson.Get(resJSON, "Name").Str) } t.Run("'ipfs pin remote add --background=true'", func(t *testing.T) { node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() svc, svcURL := runPinningService(t, authToken) node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) // retain a ptr to the pin that's in the DB so we can directly mutate its status // to simulate async work pinCh := make(chan *pinningservice.PinStatus, 1) svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) { pinCh <- pin } hash := node.IPFSAddStr("foo") node.IPFS("pin", "remote", "add", "--background=true", "--service=svc", "--name=pin1", hash, ) pin := <-pinCh transitionStatus := func(status string) { pin.M.Lock() pin.Status = status pin.M.Unlock() } verifyStatus(node, "pin1", hash, "queued") transitionStatus("pinning") verifyStatus(node, "pin1", hash, "pinning") transitionStatus("pinned") verifyStatus(node, "pin1", hash, "pinned") transitionStatus("failed") verifyStatus(node, "pin1", hash, "failed") }) t.Run("'ipfs pin remote add --background=false'", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() svc, svcURL := runPinningService(t, authToken) node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) { pin.M.Lock() defer pin.M.Unlock() pin.Status = "pinned" } hash := node.IPFSAddStr("foo") node.IPFS("pin", "remote", "add", "--background=false", "--service=svc", "--name=pin2", hash, ) verifyStatus(node, "pin2", hash, "pinned") }) t.Run("'ipfs pin remote ls' with multiple statuses", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() svc, svcURL := runPinningService(t, authToken) node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) hash := node.IPFSAddStr("foo") desiredStatuses := map[string]string{ "pin-queued": "queued", "pin-pinning": "pinning", "pin-pinned": "pinned", "pin-failed": "failed", } var pins []*pinningservice.PinStatus svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) { pin.M.Lock() defer pin.M.Unlock() pins = append(pins, pin) // this must be "pinned" for the 'pin remote add' command to return // after 'pin remote add', we change the status to its real status pin.Status = "pinned" } for pinName := range desiredStatuses { node.IPFS("pin", "remote", "add", "--service=svc", "--name="+pinName, hash, ) } for _, pin := range pins { pin.M.Lock() pin.Status = desiredStatuses[pin.Pin.Name] pin.M.Unlock() } res := node.IPFS("pin", "remote", "ls", "--service=svc", "--status=queued,pinning,pinned,failed", "--enc=json", ) actualStatuses := map[string]string{} for _, line := range res.Stdout.Lines() { name := gjson.Get(line, "Name").Str status := gjson.Get(line, "Status").Str // drop statuses of other pins we didn't add if _, ok := desiredStatuses[name]; ok { actualStatuses[name] = status } } assert.Equal(t, desiredStatuses, actualStatuses) }) t.Run("'ipfs pin remote ls' by CID", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() svc, svcURL := runPinningService(t, authToken) node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) transitionedCh := make(chan struct{}, 1) svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) { pin.M.Lock() defer pin.M.Unlock() pin.Status = "pinned" transitionedCh <- struct{}{} } hash := node.IPFSAddStr(string(random.Bytes(1000))) node.IPFS("pin", "remote", "add", "--background=false", "--service=svc", hash) <-transitionedCh res := node.IPFS("pin", "remote", "ls", "--service=svc", "--cid="+hash, "--enc=json").Stdout.String() assert.Contains(t, res, hash) }) t.Run("'ipfs pin remote rm --name' without --force when multiple pins match", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() svc, svcURL := runPinningService(t, authToken) node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) { pin.M.Lock() defer pin.M.Unlock() pin.Status = "pinned" } hash := node.IPFSAddStr(string(random.Bytes(1000))) node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash) node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash) t.Run("fails", func(t *testing.T) { res := node.RunIPFS("pin", "remote", "rm", "--service=svc", "--name=force-test-name") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "Error: multiple remote pins are matching this query, add --force to confirm the bulk removal") }) t.Run("matching pins are not removed", func(t *testing.T) { lines := node.IPFS("pin", "remote", "ls", "--service=svc", "--name=force-test-name").Stdout.Lines() assert.Contains(t, lines[0], "force-test-name") assert.Contains(t, lines[1], "force-test-name") }) }) t.Run("'ipfs pin remote rm --name --force' remove multiple pins", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() svc, svcURL := runPinningService(t, authToken) node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) { pin.M.Lock() defer pin.M.Unlock() pin.Status = "pinned" } hash := node.IPFSAddStr(string(random.Bytes(1000))) node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash) node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash) node.IPFS("pin", "remote", "rm", "--service=svc", "--name=force-test-name", "--force") out := node.IPFS("pin", "remote", "ls", "--service=svc", "--name=force-test-name").Stdout.Trimmed() assert.Empty(t, out) }) t.Run("'ipfs pin remote rm --force' removes all pins", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() svc, svcURL := runPinningService(t, authToken) node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) { pin.M.Lock() defer pin.M.Unlock() pin.Status = "pinned" } for i := range 4 { hash := node.IPFSAddStr(string(random.Bytes(1000))) name := fmt.Sprintf("--name=%d", i) node.IPFS("pin", "remote", "add", "--service=svc", "--name="+name, hash) } lines := node.IPFS("pin", "remote", "ls", "--service=svc").Stdout.Lines() assert.Len(t, lines, 4) node.IPFS("pin", "remote", "rm", "--service=svc", "--force") lines = node.IPFS("pin", "remote", "ls", "--service=svc").Stdout.Lines() assert.Len(t, lines, 0) }) }) t.Run("'ipfs pin remote add' shows a warning message when offline", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() _, svcURL := runPinningService(t, authToken) node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken) hash := node.IPFSAddStr(string(random.Bytes(1000))) res := node.IPFS("pin", "remote", "add", "--service=svc", "--background", hash) warningMsg := "WARNING: the local node is offline and remote pinning may fail if there is no other provider for this CID" assert.Contains(t, res.Stdout.String(), warningMsg) }) } ================================================ FILE: test/cli/pins_test.go ================================================ package cli import ( "fmt" "strings" "testing" "github.com/ipfs/go-cid" "github.com/ipfs/go-test/random" "github.com/ipfs/kubo/test/cli/harness" . "github.com/ipfs/kubo/test/cli/testutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) type testPinsArgs struct { runDaemon bool pinArg string lsArg string baseArg string } func testPins(t *testing.T, args testPinsArgs) { t.Run(fmt.Sprintf("test pins with args=%+v", args), func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() if args.runDaemon { node.StartDaemon("--offline") defer node.StopDaemon() } strs := []string{"a", "b", "c", "d", "e", "f", "g"} dataToCid := map[string]string{} cids := []string{} ipfsAdd := func(t *testing.T, content string) string { cidStr := node.IPFSAddStr(content, StrCat(args.baseArg, "--pin=false")...) _, err := cid.Decode(cidStr) require.NoError(t, err) dataToCid[content] = cidStr cids = append(cids, cidStr) return cidStr } ipfsPinAdd := func(cids []string) []string { input := strings.Join(cids, "\n") return node.PipeStrToIPFS(input, StrCat("pin", "add", args.pinArg, args.baseArg)...).Stdout.Lines() } ipfsPinLS := func() string { return node.IPFS(StrCat("pin", "ls", args.lsArg, args.baseArg)...).Stdout.Trimmed() } for _, s := range strs { ipfsAdd(t, s) } // these subtests run sequentially since they depend on state t.Run("check output of pin command", func(t *testing.T) { resLines := ipfsPinAdd(cids) for i, s := range resLines { assert.Equal(t, fmt.Sprintf("pinned %s recursively", cids[i]), s, ) } }) t.Run("pin verify should succeed", func(t *testing.T) { node.IPFS("pin", "verify") }) t.Run("'pin verify --verbose' should include all the cids", func(t *testing.T) { verboseVerifyOut := node.IPFS(StrCat("pin", "verify", "--verbose", args.baseArg)...).Stdout.String() for _, cid := range cids { assert.Contains(t, verboseVerifyOut, fmt.Sprintf("%s ok", cid)) } }) t.Run("ls output should contain the cids", func(t *testing.T) { lsOut := ipfsPinLS() for _, cid := range cids { assert.Contains(t, lsOut, cid) } }) t.Run("check 'pin ls hash' output", func(t *testing.T) { lsHashOut := node.IPFS(StrCat("pin", "ls", args.lsArg, args.baseArg, dataToCid["b"])...) lsHashOutStr := lsHashOut.Stdout.String() assert.Equal(t, fmt.Sprintf("%s recursive\n", dataToCid["b"]), lsHashOutStr) }) t.Run("unpinning works", func(t *testing.T) { node.PipeStrToIPFS(strings.Join(cids, "\n"), "pin", "rm") }) t.Run("test pin update", func(t *testing.T) { cidA := dataToCid["a"] cidB := dataToCid["b"] ipfsPinAdd([]string{cidA}) beforeUpdate := ipfsPinLS() assert.Contains(t, beforeUpdate, cidA) assert.NotContains(t, beforeUpdate, cidB) node.IPFS("pin", "update", "--unpin=true", cidA, cidB) afterUpdate := ipfsPinLS() assert.NotContains(t, afterUpdate, cidA) assert.Contains(t, afterUpdate, cidB) node.IPFS("pin", "update", "--unpin=true", cidB, cidB) afterIdempotentUpdate := ipfsPinLS() assert.Contains(t, afterIdempotentUpdate, cidB) node.IPFS("pin", "rm", cidB) }) }) } func testPinsErrorReporting(t *testing.T, args testPinsArgs) { t.Run(fmt.Sprintf("test pins error reporting with args=%+v", args), func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() if args.runDaemon { node.StartDaemon("--offline") defer node.StopDaemon() } randomCID := "Qme8uX5n9hn15pw9p6WcVKoziyyC9LXv4LEgvsmKMULjnV" res := node.RunIPFS(StrCat("pin", "add", args.pinArg, randomCID)...) assert.NotEqual(t, 0, res.ExitErr.ExitCode()) assert.Contains(t, res.Stderr.String(), "ipld: could not find") }) } func testPinDAG(t *testing.T, args testPinsArgs) { t.Run(fmt.Sprintf("test pin DAG with args=%+v", args), func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() if args.runDaemon { node.StartDaemon("--offline") defer node.StopDaemon() } bytes := random.Bytes(1 << 20) // 1 MiB tmpFile := h.WriteToTemp(string(bytes)) cid := node.IPFS(StrCat("add", args.pinArg, "--pin=false", "-q", tmpFile)...).Stdout.Trimmed() node.IPFS("pin", "add", "--recursive=true", cid) node.IPFS("pin", "rm", cid) // remove part of the DAG part := node.IPFS("refs", cid).Stdout.Lines()[0] node.IPFS("block", "rm", part) res := node.RunIPFS("pin", "add", "--recursive=true", cid) assert.NotEqual(t, 0, res) assert.Contains(t, res.Stderr.String(), "ipld: could not find") }) } func testPinProgress(t *testing.T, args testPinsArgs) { t.Run(fmt.Sprintf("test pin progress with args=%+v", args), func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() if args.runDaemon { node.StartDaemon("--offline") defer node.StopDaemon() } bytes := random.Bytes(1 << 20) // 1 MiB tmpFile := h.WriteToTemp(string(bytes)) cid := node.IPFS(StrCat("add", args.pinArg, "--pin=false", "-q", tmpFile)...).Stdout.Trimmed() res := node.RunIPFS("pin", "add", "--progress", cid) node.Runner.AssertNoError(res) assert.Contains(t, res.Stderr.String(), " 5 nodes (1.0 MB)") }) } func TestPins(t *testing.T) { t.Parallel() t.Run("test pinning without daemon running", func(t *testing.T) { t.Parallel() testPinsErrorReporting(t, testPinsArgs{}) testPinsErrorReporting(t, testPinsArgs{pinArg: "--progress"}) testPinDAG(t, testPinsArgs{}) testPinDAG(t, testPinsArgs{pinArg: "--raw-leaves"}) testPinProgress(t, testPinsArgs{}) testPins(t, testPinsArgs{}) testPins(t, testPinsArgs{pinArg: "--progress"}) testPins(t, testPinsArgs{pinArg: "--progress", lsArg: "--stream"}) testPins(t, testPinsArgs{baseArg: "--cid-base=base32"}) testPins(t, testPinsArgs{lsArg: "--stream", baseArg: "--cid-base=base32"}) }) t.Run("test pinning with daemon running without network", func(t *testing.T) { t.Parallel() testPinsErrorReporting(t, testPinsArgs{runDaemon: true}) testPinsErrorReporting(t, testPinsArgs{runDaemon: true, pinArg: "--progress"}) testPinDAG(t, testPinsArgs{runDaemon: true}) testPinDAG(t, testPinsArgs{runDaemon: true, pinArg: "--raw-leaves"}) testPinProgress(t, testPinsArgs{runDaemon: true}) testPins(t, testPinsArgs{runDaemon: true}) testPins(t, testPinsArgs{runDaemon: true, pinArg: "--progress"}) testPins(t, testPinsArgs{runDaemon: true, pinArg: "--progress", lsArg: "--stream"}) testPins(t, testPinsArgs{runDaemon: true, baseArg: "--cid-base=base32"}) testPins(t, testPinsArgs{runDaemon: true, lsArg: "--stream", baseArg: "--cid-base=base32"}) }) pinLs := func(node *harness.Node, args ...string) []string { return strings.Split(node.IPFS(StrCat("pin", "ls", args)...).Stdout.Trimmed(), "\n") } t.Run("test pinning with names cli text output", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() cidAStr := node.IPFSAddStr(string(random.Bytes(1000)), "--pin=false") cidBStr := node.IPFSAddStr(string(random.Bytes(1000)), "--pin=false") _ = node.IPFS("pin", "add", "--name", "testPin", cidAStr) outARegular := cidAStr + " recursive" outADetailed := outARegular + " testPin" outBRegular := cidBStr + " recursive" outBDetailed := outBRegular + " testPin" lsOut := pinLs(node, "-t=recursive") require.Contains(t, lsOut, outARegular) require.NotContains(t, lsOut, outADetailed) lsOut = pinLs(node, "-t=recursive", "--names") require.Contains(t, lsOut, outADetailed) require.NotContains(t, lsOut, outARegular) _ = node.IPFS("pin", "update", cidAStr, cidBStr) lsOut = pinLs(node, "-t=recursive", "--names") require.Contains(t, lsOut, outBDetailed) require.NotContains(t, lsOut, outADetailed) }) t.Run("test listing pins with names that contain specific string", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() cidAStr := node.IPFSAddStr(string(random.Bytes(1000)), "--pin=false") cidBStr := node.IPFSAddStr(string(random.Bytes(1000)), "--pin=false") cidCStr := node.IPFSAddStr(string(random.Bytes(1000)), "--pin=false") outA := cidAStr + " recursive testPin" outB := cidBStr + " recursive testPin" outC := cidCStr + " recursive randPin" // make sure both -n and --name work for _, nameParam := range []string{"--name", "-n"} { _ = node.IPFS("pin", "add", "--name", "testPin", cidAStr) lsOut := pinLs(node, "-t=recursive", nameParam+"=test") require.Contains(t, lsOut, outA) lsOut = pinLs(node, "-t=recursive", nameParam+"=randomLabel") require.NotContains(t, lsOut, outA) _ = node.IPFS("pin", "add", "--name", "testPin", cidBStr) lsOut = pinLs(node, "-t=recursive", nameParam+"=test") require.Contains(t, lsOut, outA) require.Contains(t, lsOut, outB) _ = node.IPFS("pin", "add", "--name", "randPin", cidCStr) lsOut = pinLs(node, "-t=recursive", nameParam+"=rand") require.NotContains(t, lsOut, outA) require.NotContains(t, lsOut, outB) require.Contains(t, lsOut, outC) lsOut = pinLs(node, "-t=recursive", nameParam+"=testPin") require.Contains(t, lsOut, outA) require.Contains(t, lsOut, outB) require.NotContains(t, lsOut, outC) } }) t.Run("test overwriting pin with name", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() cidStr := node.IPFSAddStr(string(random.Bytes(1000)), "--pin=false") outBefore := cidStr + " recursive A" outAfter := cidStr + " recursive B" _ = node.IPFS("pin", "add", "--name", "A", cidStr) lsOut := pinLs(node, "-t=recursive", "--names") require.Contains(t, lsOut, outBefore) require.NotContains(t, lsOut, outAfter) _ = node.IPFS("pin", "add", "--name", "B", cidStr) lsOut = pinLs(node, "-t=recursive", "--names") require.Contains(t, lsOut, outAfter) require.NotContains(t, lsOut, outBefore) }) // JSON that is also the wire format of /api/v0 t.Run("test pinning with names json output", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() cidAStr := node.IPFSAddStr(string(random.Bytes(1000)), "--pin=false") cidBStr := node.IPFSAddStr(string(random.Bytes(1000)), "--pin=false") _ = node.IPFS("pin", "add", "--name", "testPinJson", cidAStr) outARegular := `"` + cidAStr + `":{"Type":"recursive"` outADetailed := outARegular + `,"Name":"testPinJson"` outBRegular := `"` + cidBStr + `":{"Type":"recursive"` outBDetailed := outBRegular + `,"Name":"testPinJson"` pinLs := func(args ...string) string { return node.IPFS(StrCat("pin", "ls", "--enc=json", args)...).Stdout.Trimmed() } lsOut := pinLs("-t=recursive") require.Contains(t, lsOut, outARegular) require.NotContains(t, lsOut, outADetailed) lsOut = pinLs("-t=recursive", "--names") require.Contains(t, lsOut, outADetailed) _ = node.IPFS("pin", "update", cidAStr, cidBStr) lsOut = pinLs("-t=recursive", "--names") require.Contains(t, lsOut, outBDetailed) }) } ================================================ FILE: test/cli/provide_stats_test.go ================================================ package cli import ( "bufio" "encoding/json" "os" "path/filepath" "strings" "testing" "time" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) const ( provideStatEventuallyTimeout = 15 * time.Second provideStatEventuallyTick = 100 * time.Millisecond ) // sweepStats mirrors the subset of JSON fields actually used by tests. // This type is intentionally independent from upstream types to detect breaking changes. // Only includes fields that tests actually access to keep it simple and maintainable. type sweepStats struct { Sweep struct { Closed bool `json:"closed"` Connectivity struct { Status string `json:"status"` } `json:"connectivity"` Queues struct { PendingKeyProvides int `json:"pending_key_provides"` } `json:"queues"` Schedule struct { Keys int `json:"keys"` } `json:"schedule"` } `json:"Sweep"` } // parseSweepStats parses JSON output from ipfs provide stat command. // Tests will naturally fail if upstream removes/renames fields we depend on. func parseSweepStats(t *testing.T, jsonOutput string) sweepStats { t.Helper() var stats sweepStats err := json.Unmarshal([]byte(jsonOutput), &stats) require.NoError(t, err, "failed to parse provide stat JSON output") return stats } // TestProvideStatAllMetricsDocumented verifies that all metrics output by // `ipfs provide stat --all` are documented in docs/provide-stats.md. // // The test works as follows: // 1. Starts an IPFS node with Provide.DHT.SweepEnabled=true // 2. Runs `ipfs provide stat --all` to get all metrics // 3. Parses the output and extracts all lines with exactly 2 spaces indent // (these are the actual metric lines) // 4. Reads docs/provide-stats.md and extracts all ### section headers // 5. Ensures every metric in the output has a corresponding ### section in the docs func TestProvideStatAllMetricsDocumented(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() // Enable sweep provider node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) node.StartDaemon() defer node.StopDaemon() // Run `ipfs provide stat --all` to get all metrics res := node.IPFS("provide", "stat", "--all") require.NoError(t, res.Err) // Parse metrics from the command output // Only consider lines with exactly two spaces of padding (" ") // These are the actual metric lines as shown in provide.go outputMetrics := make(map[string]bool) scanner := bufio.NewScanner(strings.NewReader(res.Stdout.String())) // Only consider lines that start with exactly two spaces indent := " " for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, indent) || strings.HasPrefix(line, indent) { continue } // Remove the indent line = strings.TrimPrefix(line, indent) // Extract metric name - everything before the first ':' parts := strings.SplitN(line, ":", 2) if len(parts) >= 1 { metricName := strings.TrimSpace(parts[0]) if metricName != "" { outputMetrics[metricName] = true } } } require.NoError(t, scanner.Err()) // Read docs/provide-stats.md // Find the repo root by looking for go.mod repoRoot := ".." for range 6 { if _, err := os.Stat(filepath.Join(repoRoot, "go.mod")); err == nil { break } repoRoot = filepath.Join("..", repoRoot) } docsPath := filepath.Join(repoRoot, "docs", "provide-stats.md") docsFile, err := os.Open(docsPath) require.NoError(t, err, "Failed to open provide-stats.md") defer docsFile.Close() // Parse all ### metric headers from the docs documentedMetrics := make(map[string]bool) docsScanner := bufio.NewScanner(docsFile) for docsScanner.Scan() { line := docsScanner.Text() if metricName, found := strings.CutPrefix(line, "### "); found { metricName = strings.TrimSpace(metricName) documentedMetrics[metricName] = true } } require.NoError(t, docsScanner.Err()) // Check that all output metrics are documented var undocumentedMetrics []string for metric := range outputMetrics { if !documentedMetrics[metric] { undocumentedMetrics = append(undocumentedMetrics, metric) } } require.Empty(t, undocumentedMetrics, "The following metrics from 'ipfs provide stat --all' are not documented in docs/provide-stats.md: %v\n"+ "All output metrics: %v\n"+ "Documented metrics: %v", undocumentedMetrics, outputMetrics, documentedMetrics) } // TestProvideStatBasic tests basic functionality of ipfs provide stat func TestProvideStatBasic(t *testing.T) { t.Parallel() t.Run("works with Sweep provider and shows brief output", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) node.StartDaemon() defer node.StopDaemon() res := node.IPFS("provide", "stat") require.NoError(t, res.Err) assert.Empty(t, res.Stderr.String()) output := res.Stdout.String() // Brief output should contain specific full labels assert.Contains(t, output, "Provide queue:") assert.Contains(t, output, "Reprovide queue:") assert.Contains(t, output, "CIDs scheduled:") assert.Contains(t, output, "Regions scheduled:") assert.Contains(t, output, "Avg record holders:") assert.Contains(t, output, "Ongoing provides:") assert.Contains(t, output, "Ongoing reprovides:") assert.Contains(t, output, "Total CIDs provided:") }) t.Run("requires daemon to be online", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() res := node.RunIPFS("provide", "stat") assert.Error(t, res.Err) assert.Contains(t, res.Stderr.String(), "this command must be run in online mode") }) } // TestProvideStatFlags tests various command flags func TestProvideStatFlags(t *testing.T) { t.Parallel() t.Run("--all flag shows all sections with headings", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) node.StartDaemon() defer node.StopDaemon() res := node.IPFS("provide", "stat", "--all") require.NoError(t, res.Err) output := res.Stdout.String() // Should contain section headings with colons assert.Contains(t, output, "Connectivity:") assert.Contains(t, output, "Queues:") assert.Contains(t, output, "Schedule:") assert.Contains(t, output, "Timings:") assert.Contains(t, output, "Network:") assert.Contains(t, output, "Operations:") assert.Contains(t, output, "Workers:") // Should contain detailed metrics not in brief mode assert.Contains(t, output, "Uptime:") assert.Contains(t, output, "Cycle started:") assert.Contains(t, output, "Reprovide interval:") assert.Contains(t, output, "Peers swept:") assert.Contains(t, output, "Full keyspace coverage:") }) t.Run("--compact requires --all", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) node.StartDaemon() defer node.StopDaemon() res := node.RunIPFS("provide", "stat", "--compact") assert.Error(t, res.Err) assert.Contains(t, res.Stderr.String(), "--compact requires --all flag") }) t.Run("--compact with --all shows 2-column layout", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) node.StartDaemon() defer node.StopDaemon() res := node.IPFS("provide", "stat", "--all", "--compact") require.NoError(t, res.Err) output := res.Stdout.String() lines := strings.Split(strings.TrimSpace(output), "\n") require.NotEmpty(t, lines) // In compact mode, find a line that has both Schedule and Connectivity metrics // This confirms 2-column layout is working foundTwoColumns := false for _, line := range lines { if strings.Contains(line, "CIDs scheduled:") && strings.Contains(line, "Status:") { foundTwoColumns = true break } } assert.True(t, foundTwoColumns, "Should have at least one line with both 'CIDs scheduled:' and 'Status:' confirming 2-column layout") }) t.Run("individual section flags work with full labels", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) node.StartDaemon() defer node.StopDaemon() testCases := []struct { flag string contains []string }{ { flag: "--connectivity", contains: []string{"Status:"}, }, { flag: "--queues", contains: []string{"Provide queue:", "Reprovide queue:"}, }, { flag: "--schedule", contains: []string{"CIDs scheduled:", "Regions scheduled:", "Avg prefix length:", "Next region prefix:", "Next region reprovide:"}, }, { flag: "--timings", contains: []string{"Uptime:", "Current time offset:", "Cycle started:", "Reprovide interval:"}, }, { flag: "--network", contains: []string{"Avg record holders:", "Peers swept:", "Full keyspace coverage:", "Reachable peers:", "Avg region size:", "Replication factor:"}, }, { flag: "--operations", contains: []string{"Ongoing provides:", "Ongoing reprovides:", "Total CIDs provided:", "Total records provided:", "Total provide errors:"}, }, { flag: "--workers", contains: []string{"Active workers:", "Free workers:", "Workers stats:", "Periodic", "Burst"}, }, } for _, tc := range testCases { res := node.IPFS("provide", "stat", tc.flag) require.NoError(t, res.Err, "flag %s should work", tc.flag) output := res.Stdout.String() for _, expected := range tc.contains { assert.Contains(t, output, expected, "flag %s should contain '%s'", tc.flag, expected) } } }) t.Run("multiple section flags can be combined", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) node.StartDaemon() defer node.StopDaemon() res := node.IPFS("provide", "stat", "--network", "--operations") require.NoError(t, res.Err) output := res.Stdout.String() // Should have section headings when multiple flags combined assert.Contains(t, output, "Network:") assert.Contains(t, output, "Operations:") assert.Contains(t, output, "Avg record holders:") assert.Contains(t, output, "Ongoing provides:") }) } // TestProvideStatLegacyProvider tests Legacy provider specific behavior func TestProvideStatLegacyProvider(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", false) node.SetIPFSConfig("Provide.Enabled", true) node.StartDaemon() defer node.StopDaemon() t.Run("shows legacy stats from old provider system", func(t *testing.T) { res := node.IPFS("provide", "stat") require.NoError(t, res.Err) // Legacy provider shows stats from the old reprovider system output := res.Stdout.String() assert.Contains(t, output, "TotalReprovides:") assert.Contains(t, output, "AvgReprovideDuration:") assert.Contains(t, output, "LastReprovideDuration:") }) t.Run("rejects flags with legacy provider", func(t *testing.T) { flags := []string{"--all", "--connectivity", "--queues", "--network", "--workers"} for _, flag := range flags { res := node.RunIPFS("provide", "stat", flag) assert.Error(t, res.Err, "flag %s should be rejected for legacy provider", flag) assert.Contains(t, res.Stderr.String(), "cannot use flags with legacy provide stats") } }) t.Run("rejects --lan flag with legacy provider", func(t *testing.T) { res := node.RunIPFS("provide", "stat", "--lan") assert.Error(t, res.Err) assert.Contains(t, res.Stderr.String(), "LAN stats only available for Sweep provider with Dual DHT") }) } // TestProvideStatOutputFormats tests different output formats func TestProvideStatOutputFormats(t *testing.T) { t.Parallel() t.Run("JSON output with Sweep provider", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) node.StartDaemon() defer node.StopDaemon() res := node.IPFS("provide", "stat", "--enc=json") require.NoError(t, res.Err) // Parse JSON to verify structure var result struct { Sweep map[string]any `json:"Sweep"` Legacy map[string]any `json:"Legacy"` } err := json.Unmarshal([]byte(res.Stdout.String()), &result) require.NoError(t, err, "Output should be valid JSON") assert.NotNil(t, result.Sweep, "Sweep stats should be present") assert.Nil(t, result.Legacy, "Legacy stats should not be present") }) t.Run("JSON output with Legacy provider", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", false) node.SetIPFSConfig("Provide.Enabled", true) node.StartDaemon() defer node.StopDaemon() res := node.IPFS("provide", "stat", "--enc=json") require.NoError(t, res.Err) // Parse JSON to verify structure var result struct { Sweep map[string]any `json:"Sweep"` Legacy map[string]any `json:"Legacy"` } err := json.Unmarshal([]byte(res.Stdout.String()), &result) require.NoError(t, err, "Output should be valid JSON") assert.Nil(t, result.Sweep, "Sweep stats should not be present") assert.NotNil(t, result.Legacy, "Legacy stats should be present") }) } // TestProvideStatIntegration tests integration with provide operations func TestProvideStatIntegration(t *testing.T) { t.Parallel() t.Run("stats reflect content being added to schedule", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) node.SetIPFSConfig("Provide.DHT.Interval", "1h") node.StartDaemon() defer node.StopDaemon() // Get initial scheduled CID count res1 := node.IPFS("provide", "stat", "--enc=json") require.NoError(t, res1.Err) initialKeys := parseSweepStats(t, res1.Stdout.String()).Sweep.Schedule.Keys // Add content - this should increase CIDs scheduled node.IPFSAddStr("test content for stats") // Wait for content to appear in schedule (with timeout) // The buffered provider may take a moment to schedule items require.Eventually(t, func() bool { res := node.IPFS("provide", "stat", "--enc=json") require.NoError(t, res.Err) stats := parseSweepStats(t, res.Stdout.String()) return stats.Sweep.Schedule.Keys > initialKeys }, provideStatEventuallyTimeout, provideStatEventuallyTick, "Content should appear in schedule after adding") }) t.Run("stats work with all documented strategies", func(t *testing.T) { t.Parallel() // Test all strategies documented in docs/config.md#providestrategy strategies := []string{"all", "pinned", "roots", "mfs", "pinned+mfs"} for _, strategy := range strategies { h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) node.SetIPFSConfig("Provide.Strategy", strategy) node.StartDaemon() res := node.IPFS("provide", "stat") require.NoError(t, res.Err, "stats should work with strategy %s", strategy) output := res.Stdout.String() assert.NotEmpty(t, output) assert.Contains(t, output, "CIDs scheduled:") node.StopDaemon() } }) } // TestProvideStatDisabledConfig tests behavior when provide system is disabled func TestProvideStatDisabledConfig(t *testing.T) { t.Parallel() t.Run("Provide.Enabled=false returns error stats not available", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", false) node.StartDaemon() defer node.StopDaemon() res := node.RunIPFS("provide", "stat") assert.Error(t, res.Err) assert.Contains(t, res.Stderr.String(), "stats not available") }) t.Run("Provide.Enabled=true with Provide.DHT.Interval=0 returns error stats not available", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) node.SetIPFSConfig("Provide.DHT.Interval", "0") node.StartDaemon() defer node.StopDaemon() res := node.RunIPFS("provide", "stat") assert.Error(t, res.Err) assert.Contains(t, res.Stderr.String(), "stats not available") }) } ================================================ FILE: test/cli/provider_test.go ================================================ package cli import ( "bytes" "encoding/json" "fmt" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync/atomic" "testing" "time" "github.com/ipfs/go-test/random" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) const ( timeStep = 20 * time.Millisecond timeout = time.Second ) type cfgApplier func(*harness.Node) func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) { t.Helper() initNodes := func(t *testing.T, n int, fn func(n *harness.Node)) harness.Nodes { nodes := harness.NewT(t).NewNodes(n).Init() nodes.ForEachPar(apply) nodes.ForEachPar(fn) nodes = nodes.StartDaemons().Connect() time.Sleep(500 * time.Millisecond) // wait for DHT clients to be bootstrapped return nodes } initNodesWithoutStart := func(t *testing.T, n int, fn func(n *harness.Node)) harness.Nodes { nodes := harness.NewT(t).NewNodes(n).Init() nodes.ForEachPar(apply) nodes.ForEachPar(fn) return nodes } expectNoProviders := func(t *testing.T, cid string, nodes ...*harness.Node) { for _, node := range nodes { res := node.IPFS("routing", "findprovs", "-n=1", cid) require.Empty(t, res.Stdout.String()) } } expectProviders := func(t *testing.T, cid, expectedProvider string, nodes ...*harness.Node) { outerLoop: for _, node := range nodes { for i := time.Duration(0); i*timeStep < timeout; i++ { res := node.IPFS("routing", "findprovs", "-n=1", cid) if res.Stdout.Trimmed() == expectedProvider { continue outerLoop } } require.FailNowf(t, "found no providers", "expected a provider for %s", cid) } } t.Run("Provide.Enabled=true announces new CIDs created by ipfs add", func(t *testing.T) { t.Parallel() nodes := initNodes(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Enabled", true) }) defer nodes.StopDaemons() cid := nodes[0].IPFSAddStr(time.Now().String()) expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) t.Run("Provide.Enabled=true announces new CIDs created by ipfs add --pin=false with default strategy", func(t *testing.T) { t.Parallel() nodes := initNodes(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Enabled", true) // Default strategy is "all" which should provide even unpinned content }) defer nodes.StopDaemons() cid := nodes[0].IPFSAddStr(time.Now().String(), "--pin=false") expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) t.Run("Provide.Enabled=true announces new CIDs created by ipfs block put --pin=false with default strategy", func(t *testing.T) { t.Parallel() nodes := initNodes(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Enabled", true) // Default strategy is "all" which should provide unpinned content from block put }) defer nodes.StopDaemons() data := random.Bytes(256) cid := nodes[0].IPFSBlockPut(bytes.NewReader(data), "--pin=false") expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) t.Run("Provide.Enabled=true announces new CIDs created by ipfs dag put --pin=false with default strategy", func(t *testing.T) { t.Parallel() nodes := initNodes(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Enabled", true) // Default strategy is "all" which should provide unpinned content from dag put }) defer nodes.StopDaemons() dagData := `{"hello": "world", "timestamp": "` + time.Now().String() + `"}` cid := nodes[0].IPFSDAGPut(bytes.NewReader([]byte(dagData)), "--pin=false") expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) t.Run("Provide.Enabled=false disables announcement of new CID from ipfs add", func(t *testing.T) { t.Parallel() nodes := initNodes(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Enabled", false) }) defer nodes.StopDaemons() cid := nodes[0].IPFSAddStr(time.Now().String()) expectNoProviders(t, cid, nodes[1:]...) }) t.Run("Provide.Enabled=false disables manual announcement via RPC command", func(t *testing.T) { t.Parallel() nodes := initNodes(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Enabled", false) }) defer nodes.StopDaemons() cid := nodes[0].IPFSAddStr(time.Now().String()) res := nodes[0].RunIPFS("routing", "provide", cid) assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Provide.Enabled is set to 'false'") assert.Equal(t, 1, res.ExitCode()) expectNoProviders(t, cid, nodes[1:]...) }) t.Run("manual provide fails when no libp2p peers and no custom HTTP router", func(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() apply(node) node.SetIPFSConfig("Provide.Enabled", true) node.StartDaemon() defer node.StopDaemon() cid := node.IPFSAddStr(time.Now().String()) res := node.RunIPFS("routing", "provide", cid) assert.Contains(t, res.Stderr.Trimmed(), "cannot provide, no connected peers") assert.Equal(t, 1, res.ExitCode()) }) t.Run("manual provide succeeds via custom HTTP router when no libp2p peers", func(t *testing.T) { t.Parallel() // Create a mock HTTP server that accepts provide requests. // This simulates the undocumented API behavior described in // https://discuss.ipfs.tech/t/only-peers-found-from-dht-seem-to-be-getting-used-as-relays-so-cant-use-http-routers/19545/9 // Note: This is NOT IPIP-378, which was not implemented. mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Accept both PUT and POST requests to /routing/v1/providers and /routing/v1/ipns if (r.Method == http.MethodPut || r.Method == http.MethodPost) && (strings.HasPrefix(r.URL.Path, "/routing/v1/providers") || strings.HasPrefix(r.URL.Path, "/routing/v1/ipns")) { // Return HTTP 200 to indicate successful publishing w.WriteHeader(http.StatusOK) } else { w.WriteHeader(http.StatusNotFound) } })) defer mockServer.Close() h := harness.NewT(t) node := h.NewNode().Init() apply(node) node.SetIPFSConfig("Provide.Enabled", true) // Configure a custom HTTP router for providing. // Using our mock server that will accept the provide requests. routingConf := map[string]any{ "Type": "custom", // https://github.com/ipfs/kubo/blob/master/docs/delegated-routing.md#configuration-file-example "Methods": map[string]any{ "provide": map[string]any{"RouterName": "MyCustomRouter"}, "get-ipns": map[string]any{"RouterName": "MyCustomRouter"}, "put-ipns": map[string]any{"RouterName": "MyCustomRouter"}, "find-peers": map[string]any{"RouterName": "MyCustomRouter"}, "find-providers": map[string]any{"RouterName": "MyCustomRouter"}, }, "Routers": map[string]any{ "MyCustomRouter": map[string]any{ "Type": "http", "Parameters": map[string]any{ // Use the mock server URL "Endpoint": mockServer.URL, }, }, }, } node.SetIPFSConfig("Routing", routingConf) node.StartDaemon() defer node.StopDaemon() cid := node.IPFSAddStr(time.Now().String()) // The command should successfully provide via HTTP even without libp2p peers res := node.RunIPFS("routing", "provide", cid) assert.Empty(t, res.Stderr.String(), "Should have no errors when providing via HTTP router") assert.Equal(t, 0, res.ExitCode(), "Should succeed with exit code 0") }) // Right now Provide and Reprovide are tied together t.Run("Reprovide.Interval=0 disables announcement of new CID too", func(t *testing.T) { t.Parallel() nodes := initNodes(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.DHT.Interval", "0") }) defer nodes.StopDaemons() cid := nodes[0].IPFSAddStr(time.Now().String()) expectNoProviders(t, cid, nodes[1:]...) }) // It is a lesser evil - forces users to fix their config and have some sort of interval t.Run("Manual Reprovide trigger does not work when periodic reprovide is disabled", func(t *testing.T) { t.Parallel() nodes := initNodes(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.DHT.Interval", "0") }) defer nodes.StopDaemons() cid := nodes[0].IPFSAddStr(time.Now().String()) expectNoProviders(t, cid, nodes[1:]...) res := nodes[0].RunIPFS("routing", "reprovide") assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Provide.DHT.Interval is set to '0'") assert.Equal(t, 1, res.ExitCode()) expectNoProviders(t, cid, nodes[1:]...) }) // It is a lesser evil - forces users to fix their config and have some sort of interval t.Run("Manual Reprovide trigger does not work when Provide system is disabled", func(t *testing.T) { t.Parallel() nodes := initNodes(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Enabled", false) }) defer nodes.StopDaemons() cid := nodes[0].IPFSAddStr(time.Now().String()) expectNoProviders(t, cid, nodes[1:]...) res := nodes[0].RunIPFS("routing", "reprovide") assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Provide.Enabled is set to 'false'") assert.Equal(t, 1, res.ExitCode()) expectNoProviders(t, cid, nodes[1:]...) }) t.Run("Provide with 'all' strategy", func(t *testing.T) { t.Parallel() nodes := initNodes(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Strategy", "all") }) defer nodes.StopDaemons() cid := nodes[0].IPFSAddStr("all strategy") expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) t.Run("Provide with 'pinned' strategy", func(t *testing.T) { t.Parallel() nodes := initNodes(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Strategy", "pinned") }) defer nodes.StopDaemons() // Add a non-pinned CID (should not be provided) cid := nodes[0].IPFSAddStr("pinned strategy", "--pin=false") expectNoProviders(t, cid, nodes[1:]...) // Pin the CID (should now be provided) nodes[0].IPFS("pin", "add", cid) expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) t.Run("Provide with 'pinned+mfs' strategy", func(t *testing.T) { t.Parallel() nodes := initNodes(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Strategy", "pinned+mfs") }) defer nodes.StopDaemons() // Add a pinned CID (should be provided) cidPinned := nodes[0].IPFSAddStr("pinned content") cidUnpinned := nodes[0].IPFSAddStr("unpinned content", "--pin=false") cidMFS := nodes[0].IPFSAddStr("mfs content", "--pin=false") nodes[0].IPFS("files", "cp", "/ipfs/"+cidMFS, "/myfile") n0pid := nodes[0].PeerID().String() expectProviders(t, cidPinned, n0pid, nodes[1:]...) expectNoProviders(t, cidUnpinned, nodes[1:]...) expectProviders(t, cidMFS, n0pid, nodes[1:]...) }) t.Run("Provide with 'roots' strategy", func(t *testing.T) { t.Parallel() nodes := initNodes(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Strategy", "roots") }) defer nodes.StopDaemons() // Add a root CID (should be provided) cidRoot := nodes[0].IPFSAddStr("roots strategy", "-w", "-Q") // the same without wrapping should give us a child node. cidChild := nodes[0].IPFSAddStr("root strategy", "--pin=false") expectProviders(t, cidRoot, nodes[0].PeerID().String(), nodes[1:]...) expectNoProviders(t, cidChild, nodes[1:]...) }) t.Run("Provide with 'mfs' strategy", func(t *testing.T) { t.Parallel() nodes := initNodes(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Strategy", "mfs") }) defer nodes.StopDaemons() // Add a file to MFS (should be provided) data := random.Bytes(1000) cid := nodes[0].IPFSAdd(bytes.NewReader(data), "-Q") // not yet in MFS expectNoProviders(t, cid, nodes[1:]...) nodes[0].IPFS("files", "cp", "/ipfs/"+cid, "/myfile") expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) if reprovide { t.Run("Reprovides with 'all' strategy when strategy is '' (empty)", func(t *testing.T) { t.Parallel() nodes := initNodesWithoutStart(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Strategy", "") }) cid := nodes[0].IPFSAddStr(time.Now().String()) nodes = nodes.StartDaemons().Connect() defer nodes.StopDaemons() expectNoProviders(t, cid, nodes[1:]...) nodes[0].IPFS("routing", "reprovide") expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) t.Run("Reprovides with 'all' strategy", func(t *testing.T) { t.Parallel() nodes := initNodesWithoutStart(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Strategy", "all") }) cid := nodes[0].IPFSAddStr(time.Now().String()) nodes = nodes.StartDaemons().Connect() defer nodes.StopDaemons() expectNoProviders(t, cid, nodes[1:]...) nodes[0].IPFS("routing", "reprovide") expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) t.Run("Reprovides with 'pinned' strategy", func(t *testing.T) { t.Parallel() foo := random.Bytes(1000) bar := random.Bytes(1000) nodes := initNodesWithoutStart(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Strategy", "pinned") }) // Add a pin while offline so it cannot be provided cidBarDir := nodes[0].IPFSAdd(bytes.NewReader(bar), "-Q", "-w") nodes = nodes.StartDaemons().Connect() defer nodes.StopDaemons() // Add content without pinning while daemon line cidFoo := nodes[0].IPFSAdd(bytes.NewReader(foo), "--pin=false") cidBar := nodes[0].IPFSAdd(bytes.NewReader(bar), "--pin=false") // Nothing should have been provided. The pin was offline, and // the others should not be provided per the strategy. expectNoProviders(t, cidFoo, nodes[1:]...) expectNoProviders(t, cidBar, nodes[1:]...) expectNoProviders(t, cidBarDir, nodes[1:]...) nodes[0].IPFS("routing", "reprovide") // cidFoo is not pinned so should not be provided. expectNoProviders(t, cidFoo, nodes[1:]...) // cidBar gets provided by being a child from cidBarDir even though we added with pin=false. expectProviders(t, cidBar, nodes[0].PeerID().String(), nodes[1:]...) expectProviders(t, cidBarDir, nodes[0].PeerID().String(), nodes[1:]...) }) t.Run("Reprovides with 'roots' strategy", func(t *testing.T) { t.Parallel() foo := random.Bytes(1000) bar := random.Bytes(1000) nodes := initNodesWithoutStart(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Strategy", "roots") }) n0pid := nodes[0].PeerID().String() // Add a pin. Only root should get pinned but not provided // because node not started cidBarDir := nodes[0].IPFSAdd(bytes.NewReader(bar), "-Q", "-w") nodes = nodes.StartDaemons().Connect() defer nodes.StopDaemons() cidFoo := nodes[0].IPFSAdd(bytes.NewReader(foo)) cidBar := nodes[0].IPFSAdd(bytes.NewReader(bar), "--pin=false") // cidFoo will get provided per the strategy but cidBar will not. expectProviders(t, cidFoo, n0pid, nodes[1:]...) expectNoProviders(t, cidBar, nodes[1:]...) nodes[0].IPFS("routing", "reprovide") expectProviders(t, cidFoo, n0pid, nodes[1:]...) expectNoProviders(t, cidBar, nodes[1:]...) expectProviders(t, cidBarDir, n0pid, nodes[1:]...) }) t.Run("Reprovides with 'mfs' strategy", func(t *testing.T) { t.Parallel() bar := random.Bytes(1000) nodes := initNodesWithoutStart(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Strategy", "mfs") }) n0pid := nodes[0].PeerID().String() // add something and lets put it in MFS cidBar := nodes[0].IPFSAdd(bytes.NewReader(bar), "--pin=false", "-Q") nodes[0].IPFS("files", "cp", "/ipfs/"+cidBar, "/myfile") nodes = nodes.StartDaemons().Connect() defer nodes.StopDaemons() // cidBar is in MFS but not provided expectNoProviders(t, cidBar, nodes[1:]...) nodes[0].IPFS("routing", "reprovide") // And now is provided expectProviders(t, cidBar, n0pid, nodes[1:]...) }) t.Run("Reprovides with 'pinned+mfs' strategy", func(t *testing.T) { t.Parallel() nodes := initNodesWithoutStart(t, 2, func(n *harness.Node) { n.SetIPFSConfig("Provide.Strategy", "pinned+mfs") }) n0pid := nodes[0].PeerID().String() // Add a pinned CID (should be provided) cidPinned := nodes[0].IPFSAddStr("pinned content", "--pin=true") // Add a CID to MFS (should be provided) cidMFS := nodes[0].IPFSAddStr("mfs content") nodes[0].IPFS("files", "cp", "/ipfs/"+cidMFS, "/myfile") // Add a CID that is neither pinned nor in MFS (should not be provided) cidNeither := nodes[0].IPFSAddStr("neither content", "--pin=false") nodes = nodes.StartDaemons().Connect() defer nodes.StopDaemons() // Trigger reprovide nodes[0].IPFS("routing", "reprovide") // Check that pinned CID is provided expectProviders(t, cidPinned, n0pid, nodes[1:]...) // Check that MFS CID is provided expectProviders(t, cidMFS, n0pid, nodes[1:]...) // Check that neither CID is not provided expectNoProviders(t, cidNeither, nodes[1:]...) }) } t.Run("provide clear command removes items from provide queue", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(1).Init() nodes.ForEachPar(func(n *harness.Node) { n.SetIPFSConfig("Provide.Enabled", true) n.SetIPFSConfig("Provide.DHT.Interval", "22h") n.SetIPFSConfig("Provide.Strategy", "all") }) nodes.StartDaemons() defer nodes.StopDaemons() // Clear the provide queue first time - works regardless of queue state res1 := nodes[0].IPFS("provide", "clear") require.NoError(t, res1.Err) // Should report cleared items and proper message format assert.Contains(t, res1.Stdout.String(), "removed") assert.Contains(t, res1.Stdout.String(), "items from provide queue") // Clear the provide queue second time - should definitely report 0 items res2 := nodes[0].IPFS("provide", "clear") require.NoError(t, res2.Err) // Should report 0 items cleared since queue was already cleared assert.Contains(t, res2.Stdout.String(), "removed 0 items from provide queue") }) t.Run("provide clear command with quiet option", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(1).Init() nodes.ForEachPar(func(n *harness.Node) { n.SetIPFSConfig("Provide.Enabled", true) n.SetIPFSConfig("Provide.DHT.Interval", "22h") n.SetIPFSConfig("Provide.Strategy", "all") }) nodes.StartDaemons() defer nodes.StopDaemons() // Clear the provide queue with quiet option res := nodes[0].IPFS("provide", "clear", "-q") require.NoError(t, res.Err) // Should have no output when quiet assert.Empty(t, res.Stdout.String()) }) t.Run("provide clear command works when provider is disabled", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(1).Init() nodes.ForEachPar(func(n *harness.Node) { n.SetIPFSConfig("Provide.Enabled", false) n.SetIPFSConfig("Provide.DHT.Interval", "22h") n.SetIPFSConfig("Provide.Strategy", "all") }) nodes.StartDaemons() defer nodes.StopDaemons() // Clear should succeed even when provider is disabled res := nodes[0].IPFS("provide", "clear") require.NoError(t, res.Err) }) t.Run("provide clear command returns JSON with removed item count", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(1).Init() nodes.ForEachPar(func(n *harness.Node) { n.SetIPFSConfig("Provide.Enabled", true) n.SetIPFSConfig("Provide.DHT.Interval", "22h") n.SetIPFSConfig("Provide.Strategy", "all") }) nodes.StartDaemons() defer nodes.StopDaemons() // Clear the provide queue with JSON encoding res := nodes[0].IPFS("provide", "clear", "--enc=json") require.NoError(t, res.Err) // Should return valid JSON with the number of removed items output := res.Stdout.String() assert.NotEmpty(t, output) // Parse JSON to verify structure var result int err := json.Unmarshal([]byte(output), &result) require.NoError(t, err, "Output should be valid JSON") // Should be a non-negative integer (0 or positive) assert.GreaterOrEqual(t, result, 0) }) } // runResumeTests validates Provide.DHT.ResumeEnabled behavior for SweepingProvider. // // Background: The provider tracks current_time_offset = (now - cycleStart) % interval // where cycleStart is the timestamp marking the beginning of the reprovide cycle. // With ResumeEnabled=true, cycleStart persists in the datastore across restarts. // With ResumeEnabled=false, cycleStart resets to 'now' on each startup. func runResumeTests(t *testing.T, apply cfgApplier) { t.Helper() const ( reprovideInterval = 30 * time.Second initialRuntime = 10 * time.Second // Let cycle progress downtime = 5 * time.Second // Simulated offline period restartTime = 2 * time.Second // Daemon restart stabilization // Thresholds account for timing jitter (~2-3s margin) minOffsetBeforeRestart = 8 * time.Second // Expect ~10s minOffsetAfterResume = 12 * time.Second // Expect ~17s (10s + 5s + 2s) maxOffsetAfterReset = 5 * time.Second // Expect ~2s (fresh start) ) setupNode := func(t *testing.T, resumeEnabled bool) *harness.Node { node := harness.NewT(t).NewNode().Init() apply(node) // Sets Provide.DHT.SweepEnabled=true node.SetIPFSConfig("Provide.DHT.ResumeEnabled", resumeEnabled) node.SetIPFSConfig("Provide.DHT.Interval", reprovideInterval.String()) node.SetIPFSConfig("Bootstrap", []string{}) node.StartDaemon() return node } t.Run("preserves cycle state across restart", func(t *testing.T) { t.Parallel() node := setupNode(t, true) defer node.StopDaemon() for i := range 10 { node.IPFSAddStr(fmt.Sprintf("resume-test-%d-%d", i, time.Now().UnixNano())) } time.Sleep(initialRuntime) beforeRestart := node.IPFS("provide", "stat", "--enc=json") offsetBeforeRestart, _, err := parseProvideStatJSON(beforeRestart.Stdout.String()) require.NoError(t, err) require.Greater(t, offsetBeforeRestart, minOffsetBeforeRestart, "cycle should have progressed") node.StopDaemon() time.Sleep(downtime) node.StartDaemon() time.Sleep(restartTime) afterRestart := node.IPFS("provide", "stat", "--enc=json") offsetAfterRestart, _, err := parseProvideStatJSON(afterRestart.Stdout.String()) require.NoError(t, err) assert.GreaterOrEqual(t, offsetAfterRestart, minOffsetAfterResume, "offset should account for downtime") }) t.Run("resets cycle when disabled", func(t *testing.T) { t.Parallel() node := setupNode(t, false) defer node.StopDaemon() for i := range 10 { node.IPFSAddStr(fmt.Sprintf("no-resume-%d-%d", i, time.Now().UnixNano())) } time.Sleep(initialRuntime) beforeRestart := node.IPFS("provide", "stat", "--enc=json") offsetBeforeRestart, _, err := parseProvideStatJSON(beforeRestart.Stdout.String()) require.NoError(t, err) require.Greater(t, offsetBeforeRestart, minOffsetBeforeRestart, "cycle should have progressed") node.StopDaemon() time.Sleep(downtime) node.StartDaemon() time.Sleep(restartTime) afterRestart := node.IPFS("provide", "stat", "--enc=json") offsetAfterRestart, _, err := parseProvideStatJSON(afterRestart.Stdout.String()) require.NoError(t, err) assert.Less(t, offsetAfterRestart, maxOffsetAfterReset, "offset should reset to near zero") }) } type provideStatJSON struct { Sweep struct { Timing struct { CurrentTimeOffset int64 `json:"current_time_offset"` // nanoseconds } `json:"timing"` Schedule struct { NextReprovidePrefix string `json:"next_reprovide_prefix"` } `json:"schedule"` } `json:"Sweep"` } // parseProvideStatJSON extracts timing and schedule information from // the JSON output of 'ipfs provide stat --enc=json'. // Note: prefix is unused in current tests but kept for potential future use. func parseProvideStatJSON(output string) (offset time.Duration, prefix string, err error) { var stat provideStatJSON if err := json.Unmarshal([]byte(output), &stat); err != nil { return 0, "", err } offset = time.Duration(stat.Sweep.Timing.CurrentTimeOffset) prefix = stat.Sweep.Schedule.NextReprovidePrefix return offset, prefix, nil } func TestProvider(t *testing.T) { t.Parallel() variants := []struct { name string reprovide bool apply cfgApplier }{ { name: "LegacyProvider", reprovide: true, apply: func(n *harness.Node) { n.SetIPFSConfig("Provide.DHT.SweepEnabled", false) }, }, { name: "SweepingProvider", reprovide: false, apply: func(n *harness.Node) { n.SetIPFSConfig("Provide.DHT.SweepEnabled", true) }, }, } for _, v := range variants { t.Run(v.name, func(t *testing.T) { // t.Parallel() runProviderSuite(t, v.reprovide, v.apply) // Resume tests only apply to SweepingProvider if v.name == "SweepingProvider" { runResumeTests(t, v.apply) } }) } } // TestHTTPOnlyProviderWithSweepEnabled tests that provider records are correctly // sent to HTTP routers when Routing.Type="custom" with only HTTP routers configured, // even when Provide.DHT.SweepEnabled=true (the default since v0.39). // // This is a regression test for https://github.com/ipfs/kubo/issues/11089 func TestHTTPOnlyProviderWithSweepEnabled(t *testing.T) { t.Parallel() // Track provide requests received by the mock HTTP router var provideRequests atomic.Int32 mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if (r.Method == http.MethodPut || r.Method == http.MethodPost) && strings.HasPrefix(r.URL.Path, "/routing/v1/providers") { provideRequests.Add(1) w.WriteHeader(http.StatusOK) } else if strings.HasPrefix(r.URL.Path, "/routing/v1/providers") && r.Method == http.MethodGet { // Return empty providers for findprovs w.Header().Set("Content-Type", "application/x-ndjson") w.WriteHeader(http.StatusOK) } else { w.WriteHeader(http.StatusNotFound) } })) defer mockServer.Close() h := harness.NewT(t) node := h.NewNode().Init() // Explicitly set SweepEnabled=true (the default since v0.39, but be explicit for test clarity) node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) // Configure HTTP-only custom routing (no DHT) with explicit Routing.Type=custom routingConf := map[string]any{ "Type": "custom", // Explicitly set Routing.Type=custom "Methods": map[string]any{ "provide": map[string]any{"RouterName": "HTTPRouter"}, "get-ipns": map[string]any{"RouterName": "HTTPRouter"}, "put-ipns": map[string]any{"RouterName": "HTTPRouter"}, "find-peers": map[string]any{"RouterName": "HTTPRouter"}, "find-providers": map[string]any{"RouterName": "HTTPRouter"}, }, "Routers": map[string]any{ "HTTPRouter": map[string]any{ "Type": "http", "Parameters": map[string]any{ "Endpoint": mockServer.URL, }, }, }, } node.SetIPFSConfig("Routing", routingConf) node.StartDaemon() defer node.StopDaemon() // Add content and manually provide it cid := node.IPFSAddStr(time.Now().String()) // Manual provide should succeed even without libp2p peers res := node.RunIPFS("routing", "provide", cid) // Check that the command succeeded (exit code 0) and no provide-related errors assert.Equal(t, 0, res.ExitCode(), "routing provide should succeed with HTTP-only routing and SweepEnabled=true") assert.NotContains(t, res.Stderr.String(), "cannot provide", "should not have provide errors") // Verify HTTP router received at least one provide request assert.Greater(t, provideRequests.Load(), int32(0), "HTTP router should have received provide requests") // Verify 'provide stat' works with HTTP-only routing (regression test for stats) statRes := node.RunIPFS("provide", "stat") assert.Equal(t, 0, statRes.ExitCode(), "provide stat should succeed with HTTP-only routing") assert.NotContains(t, statRes.Stderr.String(), "stats not available", "should not report stats unavailable") // LegacyProvider outputs "TotalReprovides:" in its stats assert.Contains(t, statRes.Stdout.String(), "TotalReprovides:", "should show legacy provider stats") } // TestProviderKeystoreDatastoreCompaction verifies that the SweepingProvider's // keystore uses a datastore factory that creates separate physical datastores // and reclaims disk space by deleting old datastores after each reset cycle. // // The keystore uses two alternating namespaces ("0" and "1") plus a "meta" // namespace. The lifecycle is: // 1. First start: namespace "0" is created as the initial active datastore // 2. First reset (keystore sync at startup): "1" is created, data is written, // namespaces swap, "0" is destroyed from disk via os.RemoveAll // 3. Restart: "1" and "meta" survive on disk // 4. Second reset: "0" is recreated, namespaces swap, "1" is destroyed func TestProviderKeystoreDatastorePurge(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{}) // Add content offline so the keystore has something to sync on startup. for i := range 5 { node.IPFSAddStr(fmt.Sprintf("keystore-compaction-test-%d", i)) } keystoreBase := filepath.Join(node.Dir, "provider-keystore") ns0 := filepath.Join(keystoreBase, "0") ns1 := filepath.Join(keystoreBase, "1") // Directory should not exist before starting the daemon. _, err := os.Stat(keystoreBase) require.True(t, os.IsNotExist(err), "provider-keystore should not exist before daemon start") // --- First start: triggers keystore sync (ResetCids) --- // Init creates "0", then reset swaps to "1" and destroys "0". node.StartDaemon() require.Eventually(t, func() bool { return dirExists(ns1) && !dirExists(ns0) }, 30*time.Second, 200*time.Millisecond, "after first reset: ns1 should exist, ns0 should be destroyed") // --- Restart: triggers a second keystore sync (ResetCids) --- // Reset swaps back to "0" and destroys "1". node.StopDaemon() // Between restarts: ns1 survives on disk, ns0 does not. assert.True(t, dirExists(ns1), "ns1 should survive shutdown") assert.False(t, dirExists(ns0), "ns0 should not reappear between restarts") node.StartDaemon() require.Eventually(t, func() bool { return dirExists(ns0) && !dirExists(ns1) }, 30*time.Second, 200*time.Millisecond, "after second reset: ns0 should exist, ns1 should be destroyed") node.StopDaemon() } // TestProviderKeystoreMigrationPurge verifies that orphaned keystore data // left in the shared repo datastore by older Kubo versions is purged on // the first sweep-enabled daemon start. The migration is triggered by the // absence of the /provider-keystore/ directory. func TestProviderKeystoreMigrationPurge(t *testing.T) { t.Parallel() h := harness.NewT(t) node := h.NewNode().Init() node.SetIPFSConfig("Provide.DHT.SweepEnabled", true) node.SetIPFSConfig("Provide.Enabled", true) node.SetIPFSConfig("Bootstrap", []string{}) keystoreBase := filepath.Join(node.Dir, "provider-keystore") // Pre-seed orphaned keystore data into the shared datastore, simulating // the layout produced by older Kubo that stored keystore entries inline. const numOrphans = 10 for i := range numOrphans { node.DatastorePut( fmt.Sprintf("/provider/keystore/%d/fake-key-%d", i%2, i), fmt.Sprintf("orphan-%d", i), ) } // The orphaned keys should be visible via diag datastore. count := node.DatastoreCount("/provider/keystore/") require.Equal(t, int64(numOrphans), count, "orphaned keys should be present before migration") // The provider-keystore directory must not exist yet (its absence // triggers the migration). require.False(t, dirExists(keystoreBase), "provider-keystore/ should not exist before first sweep-enabled start") // Start the daemon: this triggers the one-time migration purge. node.StartDaemon() node.StopDaemon() // After migration the seeded orphaned keys should be gone from the // shared datastore. The diag datastore count command mounts the // separate provider-keystore datastores, so we check for the specific // fake keys we seeded to confirm they were purged. for i := range numOrphans { key := fmt.Sprintf("/provider/keystore/%d/fake-key-%d", i%2, i) assert.False(t, node.DatastoreHasKey(key), "orphaned key %s should be purged after migration", key) } // The provider-keystore directory should now exist. assert.True(t, dirExists(keystoreBase), "provider-keystore/ should exist after sweep-enabled daemon ran") } func dirExists(path string) bool { info, err := os.Stat(path) return err == nil && info.IsDir() } ================================================ FILE: test/cli/pubsub_test.go ================================================ package cli import ( "context" "encoding/json" "slices" "testing" "time" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // waitForSubscription waits until the node has a subscription to the given topic. func waitForSubscription(t *testing.T, node *harness.Node, topic string) { t.Helper() require.Eventually(t, func() bool { res := node.RunIPFS("pubsub", "ls") if res.Err != nil { return false } return slices.Contains(res.Stdout.Lines(), topic) }, 5*time.Second, 100*time.Millisecond, "expected subscription to topic %s", topic) } // waitForMessagePropagation waits for pubsub messages to propagate through the network // and for seqno state to be persisted to the datastore. func waitForMessagePropagation(t *testing.T) { t.Helper() time.Sleep(1 * time.Second) } // publishMessages publishes n messages from publisher to the given topic with // a small delay between each to allow for ordered delivery. func publishMessages(t *testing.T, publisher *harness.Node, topic string, n int) { t.Helper() for range n { publisher.PipeStrToIPFS("msg", "pubsub", "pub", topic) time.Sleep(50 * time.Millisecond) } } // TestPubsub tests pubsub functionality and the persistent seqno validator. // // Pubsub has two deduplication layers: // // Layer 1: MessageID-based TimeCache (in-memory) // - Controlled by Pubsub.SeenMessagesTTL config (default 120s) // - Tested in go-libp2p-pubsub (see timecache in github.com/libp2p/go-libp2p-pubsub) // - Only tested implicitly here via message delivery (timing-sensitive, not practical for CLI tests) // // Layer 2: Per-peer seqno validator (persistent in datastore) // - Stores max seen seqno per peer at /pubsub/seqno/ // - Tested directly below: persistence, updates, reset, survives restart // - Validator: go-libp2p-pubsub BasicSeqnoValidator func TestPubsub(t *testing.T) { t.Parallel() // enablePubsub configures a node with pubsub enabled enablePubsub := func(n *harness.Node) { n.SetIPFSConfig("Pubsub.Enabled", true) n.SetIPFSConfig("Routing.Type", "none") // simplify test setup } t.Run("basic pub/sub message delivery", func(t *testing.T) { t.Parallel() h := harness.NewT(t) // Create two connected nodes with pubsub enabled nodes := h.NewNodes(2).Init() nodes.ForEachPar(enablePubsub) nodes = nodes.StartDaemons().Connect() defer nodes.StopDaemons() subscriber := nodes[0] publisher := nodes[1] const topic = "test-topic" const message = "hello pubsub" // Start subscriber in background ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() // Use a channel to receive the message msgChan := make(chan string, 1) go func() { // Subscribe and wait for one message res := subscriber.RunIPFS("pubsub", "sub", "--enc=json", topic) if res.Err == nil { // Parse JSON output to get message data lines := res.Stdout.Lines() if len(lines) > 0 { var msg struct { Data []byte `json:"data"` } if json.Unmarshal([]byte(lines[0]), &msg) == nil { msgChan <- string(msg.Data) } } } }() // Wait for subscriber to be ready waitForSubscription(t, subscriber, topic) // Publish message publisher.PipeStrToIPFS(message, "pubsub", "pub", topic) // Wait for message or timeout select { case received := <-msgChan: assert.Equal(t, message, received) case <-ctx.Done(): // Subscriber may not receive in time due to test timing - that's OK // The main goal is to test the seqno validator state persistence t.Log("subscriber did not receive message in time (this is acceptable)") } }) t.Run("seqno validator state is persisted", func(t *testing.T) { t.Parallel() h := harness.NewT(t) // Create two connected nodes with pubsub nodes := h.NewNodes(2).Init() nodes.ForEachPar(enablePubsub) nodes = nodes.StartDaemons().Connect() node1 := nodes[0] node2 := nodes[1] node2PeerID := node2.PeerID().String() const topic = "seqno-test" // Start subscriber on node1 go func() { node1.RunIPFS("pubsub", "sub", topic) }() waitForSubscription(t, node1, topic) // Publish multiple messages from node2 to trigger seqno validation publishMessages(t, node2, topic, 3) // Wait for messages to propagate and seqno to be stored waitForMessagePropagation(t) // Stop daemons to check datastore (diag datastore requires daemon to be stopped) nodes.StopDaemons() // Check that seqno state exists count := node1.DatastoreCount("/pubsub/seqno/") t.Logf("seqno entries count: %d", count) // There should be at least one seqno entry (from node2) assert.NotEqual(t, int64(0), count, "expected seqno state to be persisted") // Verify the specific peer's key exists and test --hex output format key := "/pubsub/seqno/" + node2PeerID res := node1.RunIPFS("diag", "datastore", "get", "--hex", key) if res.Err == nil { t.Logf("seqno for peer %s:\n%s", node2PeerID, res.Stdout.String()) assert.Contains(t, res.Stdout.String(), "Hex Dump:") } else { // Key might not exist if messages didn't propagate - log but don't fail t.Logf("seqno key not found for peer %s (messages may not have propagated)", node2PeerID) } }) t.Run("seqno updates when receiving multiple messages", func(t *testing.T) { t.Parallel() h := harness.NewT(t) // Create two connected nodes with pubsub nodes := h.NewNodes(2).Init() nodes.ForEachPar(enablePubsub) nodes = nodes.StartDaemons().Connect() node1 := nodes[0] node2 := nodes[1] node2PeerID := node2.PeerID().String() const topic = "seqno-update-test" seqnoKey := "/pubsub/seqno/" + node2PeerID // Start subscriber on node1 go func() { node1.RunIPFS("pubsub", "sub", topic) }() waitForSubscription(t, node1, topic) // Send first message node2.PipeStrToIPFS("msg1", "pubsub", "pub", topic) time.Sleep(500 * time.Millisecond) // Stop daemons to check seqno (diag datastore requires daemon to be stopped) nodes.StopDaemons() // Get seqno after first message res1 := node1.RunIPFS("diag", "datastore", "get", seqnoKey) var seqno1 []byte if res1.Err == nil { seqno1 = res1.Stdout.Bytes() t.Logf("seqno after first message: %d bytes", len(seqno1)) } else { t.Logf("seqno not found after first message (message may not have propagated)") } // Restart daemons for second message nodes = nodes.StartDaemons().Connect() // Resubscribe go func() { node1.RunIPFS("pubsub", "sub", topic) }() waitForSubscription(t, node1, topic) // Send second message node2.PipeStrToIPFS("msg2", "pubsub", "pub", topic) time.Sleep(500 * time.Millisecond) // Stop daemons to check seqno nodes.StopDaemons() // Get seqno after second message res2 := node1.RunIPFS("diag", "datastore", "get", seqnoKey) var seqno2 []byte if res2.Err == nil { seqno2 = res2.Stdout.Bytes() t.Logf("seqno after second message: %d bytes", len(seqno2)) } else { t.Logf("seqno not found after second message") } // If both messages were received, seqno should have been updated // The seqno is a uint64 that should increase with each message if len(seqno1) > 0 && len(seqno2) > 0 { // seqno2 should be >= seqno1 (it's the max seen seqno) // We just verify they're both non-empty and potentially different t.Logf("seqno1: %x", seqno1) t.Logf("seqno2: %x", seqno2) // The seqno validator stores the max seqno seen, so seqno2 >= seqno1 // We can't do a simple byte comparison due to potential endianness // but both should be valid uint64 values (8 bytes) assert.Equal(t, 8, len(seqno2), "seqno should be 8 bytes (uint64)") } }) t.Run("pubsub reset clears seqno state", func(t *testing.T) { t.Parallel() h := harness.NewT(t) // Create two connected nodes nodes := h.NewNodes(2).Init() nodes.ForEachPar(enablePubsub) nodes = nodes.StartDaemons().Connect() node1 := nodes[0] node2 := nodes[1] const topic = "reset-test" // Start subscriber and exchange messages go func() { node1.RunIPFS("pubsub", "sub", topic) }() waitForSubscription(t, node1, topic) publishMessages(t, node2, topic, 3) waitForMessagePropagation(t) // Stop daemons to check initial count nodes.StopDaemons() // Verify there is state before resetting initialCount := node1.DatastoreCount("/pubsub/seqno/") t.Logf("initial seqno count: %d", initialCount) // Restart node1 to run pubsub reset node1.StartDaemon() // Reset all seqno state (while daemon is running) res := node1.IPFS("pubsub", "reset") assert.NoError(t, res.Err) t.Logf("reset output: %s", res.Stdout.String()) // Stop daemon to verify state was cleared node1.StopDaemon() // Verify state was cleared finalCount := node1.DatastoreCount("/pubsub/seqno/") t.Logf("final seqno count: %d", finalCount) assert.Equal(t, int64(0), finalCount, "seqno state should be cleared after reset") }) t.Run("pubsub reset with peer flag", func(t *testing.T) { t.Parallel() h := harness.NewT(t) // Create three connected nodes nodes := h.NewNodes(3).Init() nodes.ForEachPar(enablePubsub) nodes = nodes.StartDaemons().Connect() node1 := nodes[0] node2 := nodes[1] node3 := nodes[2] node2PeerID := node2.PeerID().String() node3PeerID := node3.PeerID().String() const topic = "peer-reset-test" // Start subscriber on node1 go func() { node1.RunIPFS("pubsub", "sub", topic) }() waitForSubscription(t, node1, topic) // Publish from both node2 and node3 for range 3 { node2.PipeStrToIPFS("msg2", "pubsub", "pub", topic) node3.PipeStrToIPFS("msg3", "pubsub", "pub", topic) time.Sleep(50 * time.Millisecond) } waitForMessagePropagation(t) // Stop node2 and node3 node2.StopDaemon() node3.StopDaemon() // Reset only node2's state (while node1 daemon is running) res := node1.IPFS("pubsub", "reset", "--peer", node2PeerID) require.NoError(t, res.Err) t.Logf("reset output: %s", res.Stdout.String()) // Stop node1 daemon to check datastore node1.StopDaemon() // Check that node2's key is gone res = node1.RunIPFS("diag", "datastore", "get", "/pubsub/seqno/"+node2PeerID) assert.Error(t, res.Err, "node2's seqno key should be deleted") // Check that node3's key still exists (if it was created) res = node1.RunIPFS("diag", "datastore", "get", "/pubsub/seqno/"+node3PeerID) // Note: node3's key might not exist if messages didn't propagate // So we just log the result without asserting if res.Err == nil { t.Logf("node3's seqno key still exists (as expected)") } else { t.Logf("node3's seqno key not found (messages may not have propagated)") } }) t.Run("seqno state survives daemon restart", func(t *testing.T) { t.Parallel() h := harness.NewT(t) // Create and start single node node := h.NewNode().Init() enablePubsub(node) node.StartDaemon() // We need another node to publish messages node2 := h.NewNode().Init() enablePubsub(node2) node2.StartDaemon() node.Connect(node2) const topic = "restart-test" // Start subscriber and exchange messages go func() { node.RunIPFS("pubsub", "sub", topic) }() waitForSubscription(t, node, topic) publishMessages(t, node2, topic, 3) waitForMessagePropagation(t) // Stop daemons to check datastore node.StopDaemon() node2.StopDaemon() // Get count before restart beforeCount := node.DatastoreCount("/pubsub/seqno/") t.Logf("seqno count before restart: %d", beforeCount) // Restart node (simulate restart scenario) node.StartDaemon() time.Sleep(500 * time.Millisecond) // Stop daemon to check datastore again node.StopDaemon() // Get count after restart afterCount := node.DatastoreCount("/pubsub/seqno/") t.Logf("seqno count after restart: %d", afterCount) // Count should be the same (state persisted) assert.Equal(t, beforeCount, afterCount, "seqno state should survive daemon restart") }) } ================================================ FILE: test/cli/rcmgr_test.go ================================================ package cli import ( "encoding/json" "testing" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/node/libp2p" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestRcmgr(t *testing.T) { t.Parallel() t.Run("Resource manager disabled", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Swarm.ResourceMgr.Enabled = config.False }) node.StartDaemon() defer node.StopDaemon() t.Run("swarm resources should fail", func(t *testing.T) { res := node.RunIPFS("swarm", "resources") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "missing ResourceMgr") }) }) t.Run("Node with resource manager disabled", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Swarm.ResourceMgr.Enabled = config.False }) node.StartDaemon() defer node.StopDaemon() t.Run("swarm resources should fail", func(t *testing.T) { res := node.RunIPFS("swarm", "resources") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "missing ResourceMgr") }) }) t.Run("Very high connmgr highwater", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Swarm.ConnMgr.HighWater = config.NewOptionalInteger(1000) }) node.StartDaemon() defer node.StopDaemon() res := node.RunIPFS("swarm", "resources", "--enc=json") require.Equal(t, 0, res.ExitCode()) limits := unmarshalLimits(t, res.Stdout.Bytes()) rl := limits.System.ToResourceLimits() s := rl.Build(rcmgr.BaseLimit{}) assert.GreaterOrEqual(t, s.ConnsInbound, 2000) assert.GreaterOrEqual(t, s.StreamsInbound, 2000) }) t.Run("default configuration", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Swarm.ConnMgr.HighWater = config.NewOptionalInteger(1000) }) node.StartDaemon() t.Cleanup(func() { node.StopDaemon() }) t.Run("conns and streams are above 800 for default connmgr settings", func(t *testing.T) { t.Parallel() res := node.RunIPFS("swarm", "resources", "--enc=json") require.Equal(t, 0, res.ExitCode()) limits := unmarshalLimits(t, res.Stdout.Bytes()) if limits.System.ConnsInbound > rcmgr.DefaultLimit { assert.GreaterOrEqual(t, limits.System.ConnsInbound, 800) } if limits.System.StreamsInbound > rcmgr.DefaultLimit { assert.GreaterOrEqual(t, limits.System.StreamsInbound, 800) } }) t.Run("limits should succeed", func(t *testing.T) { t.Parallel() res := node.RunIPFS("swarm", "resources", "--enc=json") assert.Equal(t, 0, res.ExitCode()) limits := rcmgr.PartialLimitConfig{} err := json.Unmarshal(res.Stdout.Bytes(), &limits) require.NoError(t, err) assert.NotEqual(t, limits.Transient.Memory, rcmgr.BlockAllLimit64) assert.NotEqual(t, limits.System.Memory, rcmgr.BlockAllLimit64) assert.NotEqual(t, limits.System.FD, rcmgr.BlockAllLimit) assert.NotEqual(t, limits.System.Conns, rcmgr.BlockAllLimit) assert.NotEqual(t, limits.System.ConnsInbound, rcmgr.BlockAllLimit) assert.NotEqual(t, limits.System.ConnsOutbound, rcmgr.BlockAllLimit) assert.NotEqual(t, limits.System.Streams, rcmgr.BlockAllLimit) assert.NotEqual(t, limits.System.StreamsInbound, rcmgr.BlockAllLimit) assert.NotEqual(t, limits.System.StreamsOutbound, rcmgr.BlockAllLimit) }) t.Run("swarm stats works", func(t *testing.T) { t.Parallel() res := node.RunIPFS("swarm", "resources", "--enc=json") require.Equal(t, 0, res.ExitCode()) limits := unmarshalLimits(t, res.Stdout.Bytes()) // every scope has the same fields, so we only inspect system assert.Zero(t, limits.System.MemoryUsage) assert.Zero(t, limits.System.FDUsage) assert.Zero(t, limits.System.ConnsInboundUsage) assert.Zero(t, limits.System.ConnsOutboundUsage) assert.Zero(t, limits.System.StreamsInboundUsage) assert.Zero(t, limits.System.StreamsOutboundUsage) assert.Zero(t, limits.Transient.MemoryUsage) }) }) t.Run("smoke test unlimited System inbounds", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateUserSuppliedResourceManagerOverrides(func(overrides *rcmgr.PartialLimitConfig) { overrides.System.StreamsInbound = rcmgr.Unlimited overrides.System.ConnsInbound = rcmgr.Unlimited }) node.StartDaemon() defer node.StopDaemon() res := node.RunIPFS("swarm", "resources", "--enc=json") limits := unmarshalLimits(t, res.Stdout.Bytes()) assert.Equal(t, rcmgr.Unlimited, limits.System.ConnsInbound) assert.Equal(t, rcmgr.Unlimited, limits.System.StreamsInbound) }) t.Run("smoke test transient scope", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateUserSuppliedResourceManagerOverrides(func(overrides *rcmgr.PartialLimitConfig) { overrides.Transient.Memory = 88888 }) node.StartDaemon() defer node.StopDaemon() res := node.RunIPFS("swarm", "resources", "--enc=json") limits := unmarshalLimits(t, res.Stdout.Bytes()) assert.Equal(t, rcmgr.LimitVal64(88888), limits.Transient.Memory) }) t.Run("smoke test service scope", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateUserSuppliedResourceManagerOverrides(func(overrides *rcmgr.PartialLimitConfig) { overrides.Service = map[string]rcmgr.ResourceLimits{"foo": {Memory: 77777}} }) node.StartDaemon() defer node.StopDaemon() res := node.RunIPFS("swarm", "resources", "--enc=json") limits := unmarshalLimits(t, res.Stdout.Bytes()) assert.Equal(t, rcmgr.LimitVal64(77777), limits.Services["foo"].Memory) }) t.Run("smoke test protocol scope", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateUserSuppliedResourceManagerOverrides(func(overrides *rcmgr.PartialLimitConfig) { overrides.Protocol = map[protocol.ID]rcmgr.ResourceLimits{"foo": {Memory: 66666}} }) node.StartDaemon() defer node.StopDaemon() res := node.RunIPFS("swarm", "resources", "--enc=json") limits := unmarshalLimits(t, res.Stdout.Bytes()) assert.Equal(t, rcmgr.LimitVal64(66666), limits.Protocols["foo"].Memory) }) t.Run("smoke test peer scope", func(t *testing.T) { t.Parallel() validPeerID, err := peer.Decode("QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN") assert.NoError(t, err) node := harness.NewT(t).NewNode().Init() node.UpdateUserSuppliedResourceManagerOverrides(func(overrides *rcmgr.PartialLimitConfig) { overrides.Peer = map[peer.ID]rcmgr.ResourceLimits{validPeerID: {Memory: 55555}} }) node.StartDaemon() defer node.StopDaemon() res := node.RunIPFS("swarm", "resources", "--enc=json") limits := unmarshalLimits(t, res.Stdout.Bytes()) assert.Equal(t, rcmgr.LimitVal64(55555), limits.Peers[validPeerID].Memory) }) t.Run("blocking and allowlists", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(3).Init() node0, node1, node2 := nodes[0], nodes[1], nodes[2] peerID1, peerID2 := node1.PeerID().String(), node2.PeerID().String() node0.UpdateConfig(func(cfg *config.Config) { cfg.Swarm.ResourceMgr.Enabled = config.True cfg.Swarm.ResourceMgr.Allowlist = []string{"/ip4/0.0.0.0/ipcidr/0/p2p/" + peerID2} }) node0.UpdateUserSuppliedResourceManagerOverrides(func(overrides *rcmgr.PartialLimitConfig) { *overrides = rcmgr.PartialLimitConfig{ System: rcmgr.ResourceLimits{ Conns: rcmgr.BlockAllLimit, ConnsInbound: rcmgr.BlockAllLimit, ConnsOutbound: rcmgr.BlockAllLimit, }, } }) nodes.StartDaemons() t.Cleanup(func() { nodes.StopDaemons() }) t.Run("node 0 should fail to connect to and ping node 1", func(t *testing.T) { t.Parallel() res := node0.Runner.Run(harness.RunRequest{ Path: node0.IPFSBin, Args: []string{"swarm", "connect", node1.SwarmAddrsWithPeerIDs()[0].String()}, }) assert.Equal(t, 1, res.ExitCode()) testutils.AssertStringContainsOneOf(t, res.Stderr.String(), "failed to find any peer in table", "resource limit exceeded", ) res = node0.RunIPFS("ping", "-n2", peerID1) assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "Error: ping failed") }) t.Run("node 0 should connect to and ping node 2 since it is allowlisted", func(t *testing.T) { t.Parallel() res := node0.Runner.Run(harness.RunRequest{ Path: node0.IPFSBin, Args: []string{"swarm", "connect", node2.SwarmAddrsWithPeerIDs()[0].String()}, }) assert.Equal(t, 0, res.ExitCode()) res = node0.RunIPFS("ping", "-n2", peerID2) assert.Equal(t, 0, res.ExitCode()) }) }) t.Run("daemon should refuse to start if connmgr.highwater < resources inbound", func(t *testing.T) { t.Run("system conns", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Swarm.ConnMgr.HighWater = config.NewOptionalInteger(128) cfg.Swarm.ConnMgr.LowWater = config.NewOptionalInteger(64) }) node.UpdateUserSuppliedResourceManagerOverrides(func(overrides *rcmgr.PartialLimitConfig) { *overrides = rcmgr.PartialLimitConfig{ System: rcmgr.ResourceLimits{Conns: 128}, } }) res := node.RunIPFS("daemon") assert.Equal(t, 1, res.ExitCode()) }) t.Run("system conns inbound", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Swarm.ConnMgr.HighWater = config.NewOptionalInteger(128) cfg.Swarm.ConnMgr.LowWater = config.NewOptionalInteger(64) }) node.UpdateUserSuppliedResourceManagerOverrides(func(overrides *rcmgr.PartialLimitConfig) { *overrides = rcmgr.PartialLimitConfig{ System: rcmgr.ResourceLimits{ConnsInbound: 128}, } }) res := node.RunIPFS("daemon") assert.Equal(t, 1, res.ExitCode()) }) t.Run("system streams", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Swarm.ConnMgr.HighWater = config.NewOptionalInteger(128) cfg.Swarm.ConnMgr.LowWater = config.NewOptionalInteger(64) }) node.UpdateUserSuppliedResourceManagerOverrides(func(overrides *rcmgr.PartialLimitConfig) { *overrides = rcmgr.PartialLimitConfig{ System: rcmgr.ResourceLimits{Streams: 128}, } }) res := node.RunIPFS("daemon") assert.Equal(t, 1, res.ExitCode()) }) t.Run("system streams inbound", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Swarm.ConnMgr.HighWater = config.NewOptionalInteger(128) cfg.Swarm.ConnMgr.LowWater = config.NewOptionalInteger(64) }) node.UpdateUserSuppliedResourceManagerOverrides(func(overrides *rcmgr.PartialLimitConfig) { *overrides = rcmgr.PartialLimitConfig{ System: rcmgr.ResourceLimits{StreamsInbound: 128}, } }) res := node.RunIPFS("daemon") assert.Equal(t, 1, res.ExitCode()) }) }) } func unmarshalLimits(t *testing.T, b []byte) *libp2p.LimitsConfigAndUsage { limits := &libp2p.LimitsConfigAndUsage{} err := json.Unmarshal(b, limits) require.NoError(t, err) return limits } ================================================ FILE: test/cli/repo_verify_test.go ================================================ package cli import ( "fmt" "os" "path/filepath" "strings" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // Well-known block file names in flatfs blockstore that should not be corrupted during testing. // Flatfs stores each block as a separate .data file on disk. const ( // emptyFileFlatfsFilename is the flatfs filename for an empty UnixFS file block emptyFileFlatfsFilename = "CIQL7TG2PB52XIZLLHDYIUFMHUQLMMZWBNBZSLDXFCPZ5VDNQQ2WDZQ" // emptyDirFlatfsFilename is the flatfs filename for an empty UnixFS directory block. // This block has special handling and may be served from memory even when corrupted on disk. emptyDirFlatfsFilename = "CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y" ) // getEligibleFlatfsBlockFiles returns flatfs block files (*.data) that are safe to corrupt in tests. // Filters out well-known blocks (empty file/dir) that cause test flakiness. // // Note: This helper is specific to the flatfs blockstore implementation where each block // is stored as a separate file on disk under blocks/*/*.data. func getEligibleFlatfsBlockFiles(t *testing.T, node *harness.Node) []string { blockFiles, err := filepath.Glob(filepath.Join(node.Dir, "blocks", "*", "*.data")) require.NoError(t, err) require.NotEmpty(t, blockFiles, "no flatfs block files found") var eligible []string for _, f := range blockFiles { name := filepath.Base(f) if !strings.Contains(name, emptyFileFlatfsFilename) && !strings.Contains(name, emptyDirFlatfsFilename) { eligible = append(eligible, f) } } return eligible } // corruptRandomBlock corrupts a random block file in the flatfs blockstore. // Returns the path to the corrupted file. func corruptRandomBlock(t *testing.T, node *harness.Node) string { eligible := getEligibleFlatfsBlockFiles(t, node) require.NotEmpty(t, eligible, "no eligible blocks to corrupt") toCorrupt := eligible[0] err := os.WriteFile(toCorrupt, []byte("corrupted data"), 0644) require.NoError(t, err) return toCorrupt } // corruptMultipleBlocks corrupts multiple block files in the flatfs blockstore. // Returns the paths to the corrupted files. func corruptMultipleBlocks(t *testing.T, node *harness.Node, count int) []string { eligible := getEligibleFlatfsBlockFiles(t, node) require.GreaterOrEqual(t, len(eligible), count, "not enough eligible blocks to corrupt") var corrupted []string for i := 0; i < count && i < len(eligible); i++ { err := os.WriteFile(eligible[i], fmt.Appendf(nil, "corrupted data %d", i), 0644) require.NoError(t, err) corrupted = append(corrupted, eligible[i]) } return corrupted } func TestRepoVerify(t *testing.T) { t.Run("healthy repo passes", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.IPFS("add", "-q", "--raw-leaves=false", "-r", node.IPFSBin) res := node.IPFS("repo", "verify") assert.Contains(t, res.Stdout.String(), "all blocks validated") }) t.Run("detects corruption", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.IPFSAddStr("test content") corruptRandomBlock(t, node) res := node.RunIPFS("repo", "verify") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stdout.String(), "was corrupt") assert.Contains(t, res.Stderr.String(), "1 blocks corrupt") }) t.Run("drop removes corrupt blocks", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() cid := node.IPFSAddStr("test content") corruptRandomBlock(t, node) res := node.RunIPFS("repo", "verify", "--drop") assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully") output := res.Stdout.String() assert.Contains(t, output, "1 blocks corrupt") assert.Contains(t, output, "1 removed") // Verify block is gone res = node.RunIPFS("block", "stat", cid) assert.NotEqual(t, 0, res.ExitCode()) }) t.Run("heal requires online mode", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.IPFSAddStr("test content") corruptRandomBlock(t, node) res := node.RunIPFS("repo", "verify", "--heal") assert.NotEqual(t, 0, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "online mode") }) t.Run("heal repairs from network", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init() nodes.StartDaemons().Connect() defer nodes.StopDaemons() // Add content to node 0 cid := nodes[0].IPFSAddStr("test content for healing") // Wait for it to appear on node 1 nodes[1].IPFS("block", "get", cid) // Corrupt on node 1 corruptRandomBlock(t, nodes[1]) // Heal should restore from node 0 res := nodes[1].RunIPFS("repo", "verify", "--heal") assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks healed successfully") output := res.Stdout.String() // Should report corruption and healing with specific counts assert.Contains(t, output, "1 blocks corrupt") assert.Contains(t, output, "1 removed") assert.Contains(t, output, "1 healed") // Verify block is restored nodes[1].IPFS("block", "stat", cid) }) t.Run("healed blocks contain correct data", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init() nodes.StartDaemons().Connect() defer nodes.StopDaemons() // Add specific content to node 0 testContent := "this is the exact content that should be healed correctly" cid := nodes[0].IPFSAddStr(testContent) // Fetch to node 1 and verify the content is correct initially nodes[1].IPFS("block", "get", cid) res := nodes[1].IPFS("cat", cid) assert.Equal(t, testContent, res.Stdout.String()) // Corrupt on node 1 corruptRandomBlock(t, nodes[1]) // Heal the corruption res = nodes[1].RunIPFS("repo", "verify", "--heal") assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks healed successfully") output := res.Stdout.String() assert.Contains(t, output, "1 blocks corrupt") assert.Contains(t, output, "1 removed") assert.Contains(t, output, "1 healed") // Verify the healed content matches the original exactly res = nodes[1].IPFS("cat", cid) assert.Equal(t, testContent, res.Stdout.String(), "healed content should match original") // Also verify via block get that the raw block data is correct block0 := nodes[0].IPFS("block", "get", cid) block1 := nodes[1].IPFS("block", "get", cid) assert.Equal(t, block0.Stdout.String(), block1.Stdout.String(), "raw block data should match") }) t.Run("multiple corrupt blocks", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Create 20 blocks for i := range 20 { node.IPFSAddStr(strings.Repeat("test content ", i+1)) } // Corrupt 5 blocks corruptMultipleBlocks(t, node, 5) // Verify detects all corruptions res := node.RunIPFS("repo", "verify") assert.Equal(t, 1, res.ExitCode()) // Error summary is in stderr assert.Contains(t, res.Stderr.String(), "5 blocks corrupt") // Test with --drop res = node.RunIPFS("repo", "verify", "--drop") assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully") assert.Contains(t, res.Stdout.String(), "5 blocks corrupt") assert.Contains(t, res.Stdout.String(), "5 removed") }) t.Run("empty repository", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Verify empty repo passes res := node.IPFS("repo", "verify") assert.Equal(t, 0, res.ExitCode()) assert.Contains(t, res.Stdout.String(), "all blocks validated") // Should work with --drop and --heal too res = node.IPFS("repo", "verify", "--drop") assert.Equal(t, 0, res.ExitCode()) assert.Contains(t, res.Stdout.String(), "all blocks validated") }) t.Run("partial heal success", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init() // Start both nodes and connect them nodes.StartDaemons().Connect() defer nodes.StopDaemons() // Add 5 blocks to node 0, pin them to keep available cid1 := nodes[0].IPFSAddStr("content available for healing 1") cid2 := nodes[0].IPFSAddStr("content available for healing 2") cid3 := nodes[0].IPFSAddStr("content available for healing 3") cid4 := nodes[0].IPFSAddStr("content available for healing 4") cid5 := nodes[0].IPFSAddStr("content available for healing 5") // Pin these on node 0 to ensure they stay available nodes[0].IPFS("pin", "add", cid1) nodes[0].IPFS("pin", "add", cid2) nodes[0].IPFS("pin", "add", cid3) nodes[0].IPFS("pin", "add", cid4) nodes[0].IPFS("pin", "add", cid5) // Node 1 fetches these blocks nodes[1].IPFS("block", "get", cid1) nodes[1].IPFS("block", "get", cid2) nodes[1].IPFS("block", "get", cid3) nodes[1].IPFS("block", "get", cid4) nodes[1].IPFS("block", "get", cid5) // Now remove some blocks from node 0 to simulate partial availability nodes[0].IPFS("pin", "rm", cid3) nodes[0].IPFS("pin", "rm", cid4) nodes[0].IPFS("pin", "rm", cid5) nodes[0].IPFS("repo", "gc") // Verify node 1 is still connected peers := nodes[1].IPFS("swarm", "peers") require.Contains(t, peers.Stdout.String(), nodes[0].PeerID().String()) // Corrupt 5 blocks on node 1 corruptMultipleBlocks(t, nodes[1], 5) // Heal should partially succeed (only cid1 and cid2 available from node 0) res := nodes[1].RunIPFS("repo", "verify", "--heal") assert.Equal(t, 1, res.ExitCode()) // Should show mixed results with specific counts in stderr errOutput := res.Stderr.String() assert.Contains(t, errOutput, "5 blocks corrupt") assert.Contains(t, errOutput, "5 removed") // Only cid1 and cid2 are available for healing, cid3-5 were GC'd assert.Contains(t, errOutput, "2 healed") assert.Contains(t, errOutput, "3 failed to heal") }) t.Run("heal with block not available on network", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init() // Start both nodes and connect nodes.StartDaemons().Connect() defer nodes.StopDaemons() // Add unique content only to node 1 nodes[1].IPFSAddStr("unique content that exists nowhere else") // Ensure nodes are connected peers := nodes[1].IPFS("swarm", "peers") require.Contains(t, peers.Stdout.String(), nodes[0].PeerID().String()) // Corrupt the block on node 1 corruptRandomBlock(t, nodes[1]) // Heal should fail - node 0 doesn't have this content res := nodes[1].RunIPFS("repo", "verify", "--heal") assert.Equal(t, 1, res.ExitCode()) // Should report heal failure with specific counts in stderr errOutput := res.Stderr.String() assert.Contains(t, errOutput, "1 blocks corrupt") assert.Contains(t, errOutput, "1 removed") assert.Contains(t, errOutput, "1 failed to heal") }) t.Run("large repository scale test", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Create 1000 small blocks for i := range 1000 { node.IPFSAddStr(fmt.Sprintf("content-%d", i)) } // Corrupt 10 blocks corruptMultipleBlocks(t, node, 10) // Verify handles large repos efficiently res := node.RunIPFS("repo", "verify") assert.Equal(t, 1, res.ExitCode()) // Should report exactly 10 corrupt blocks in stderr assert.Contains(t, res.Stderr.String(), "10 blocks corrupt") // Test --drop at scale res = node.RunIPFS("repo", "verify", "--drop") assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully") output := res.Stdout.String() assert.Contains(t, output, "10 blocks corrupt") assert.Contains(t, output, "10 removed") }) t.Run("drop with partial removal failures", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // Create several blocks for i := range 5 { node.IPFSAddStr(fmt.Sprintf("content for removal test %d", i)) } // Corrupt 3 blocks corruptedFiles := corruptMultipleBlocks(t, node, 3) require.Len(t, corruptedFiles, 3) // Make one of the corrupted files read-only to simulate removal failure err := os.Chmod(corruptedFiles[0], 0400) // read-only require.NoError(t, err) defer func() { _ = os.Chmod(corruptedFiles[0], 0644) }() // cleanup // Also make the directory read-only to prevent deletion blockDir := filepath.Dir(corruptedFiles[0]) originalPerm, err := os.Stat(blockDir) require.NoError(t, err) err = os.Chmod(blockDir, 0500) // read+execute only, no write require.NoError(t, err) defer func() { _ = os.Chmod(blockDir, originalPerm.Mode()) }() // cleanup // Try to drop - should fail because at least one block can't be removed res := node.RunIPFS("repo", "verify", "--drop") assert.Equal(t, 1, res.ExitCode(), "should exit 1 when some blocks fail to remove") // Restore permissions for verification _ = os.Chmod(blockDir, originalPerm.Mode()) _ = os.Chmod(corruptedFiles[0], 0644) // Should report both successes and failures with specific counts errOutput := res.Stderr.String() assert.Contains(t, errOutput, "3 blocks corrupt") assert.Contains(t, errOutput, "2 removed") assert.Contains(t, errOutput, "1 failed to remove") }) } ================================================ FILE: test/cli/routing_dht_test.go ================================================ package cli import ( "fmt" "strconv" "strings" "testing" "time" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func waitUntilProvidesComplete(t *testing.T, n *harness.Node) { getCidsCount := func(line string) int { trimmed := strings.TrimSpace(line) countStr := strings.SplitN(trimmed, " ", 2)[0] count, err := strconv.Atoi(countStr) require.NoError(t, err) return count } queuedProvides, ongoingProvides := true, true for queuedProvides || ongoingProvides { res := n.IPFS("provide", "stat", "-a") require.NoError(t, res.Err) for _, line := range res.Stdout.Lines() { if trimmed, ok := strings.CutPrefix(line, " Provide queue:"); ok { provideQueueSize := getCidsCount(trimmed) queuedProvides = provideQueueSize > 0 } if trimmed, ok := strings.CutPrefix(line, " Ongoing provides:"); ok { ongoingProvideCount := getCidsCount(trimmed) ongoingProvides = ongoingProvideCount > 0 } } time.Sleep(10 * time.Millisecond) } } func testRoutingDHT(t *testing.T, enablePubsub bool) { t.Run(fmt.Sprintf("enablePubSub=%v", enablePubsub), func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(5).Init() nodes.ForEachPar(func(node *harness.Node) { node.IPFS("config", "Routing.Type", "dht") }) var daemonArgs []string if enablePubsub { daemonArgs = []string{ "--enable-pubsub-experiment", "--enable-namesys-pubsub", } } nodes.StartDaemons(daemonArgs...).Connect() t.Cleanup(func() { nodes.StopDaemons() }) t.Run("ipfs routing findpeer", func(t *testing.T) { t.Parallel() res := nodes[1].RunIPFS("routing", "findpeer", nodes[0].PeerID().String()) assert.Equal(t, 0, res.ExitCode()) swarmAddr := nodes[0].SwarmAddrsWithoutPeerIDs()[0] require.Equal(t, swarmAddr.String(), res.Stdout.Trimmed()) }) t.Run("ipfs routing get ", func(t *testing.T) { t.Parallel() hash := nodes[2].IPFSAddStr("hello world") nodes[2].IPFS("name", "publish", "/ipfs/"+hash) res := nodes[1].IPFS("routing", "get", "/ipns/"+nodes[2].PeerID().String()) assert.Contains(t, res.Stdout.String(), "/ipfs/"+hash) t.Run("put round trips (#3124)", func(t *testing.T) { t.Parallel() nodes[0].WriteBytes("get_result", res.Stdout.Bytes()) res := nodes[0].IPFS("routing", "put", "/ipns/"+nodes[2].PeerID().String(), "get_result") assert.Greater(t, len(res.Stdout.Lines()), 0, "should put to at least one node") }) t.Run("put with bad keys fails (issue #5113, #4611)", func(t *testing.T) { t.Parallel() keys := []string{"foo", "/pk/foo", "/ipns/foo"} for _, key := range keys { t.Run(key, func(t *testing.T) { t.Parallel() res := nodes[0].RunIPFS("routing", "put", key) assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "invalid") assert.Empty(t, res.Stdout.String()) }) } }) t.Run("get with bad keys (issue #4611)", func(t *testing.T) { for _, key := range []string{"foo", "/pk/foo"} { t.Run(key, func(t *testing.T) { t.Parallel() res := nodes[0].RunIPFS("routing", "get", key) assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "invalid") assert.Empty(t, res.Stdout.String()) }) } }) }) t.Run("ipfs routing findprovs", func(t *testing.T) { t.Parallel() hash := nodes[3].IPFSAddStr("some stuff") waitUntilProvidesComplete(t, nodes[3]) res := nodes[4].IPFS("routing", "findprovs", hash) assert.Equal(t, nodes[3].PeerID().String(), res.Stdout.Trimmed()) }) t.Run("routing commands fail when offline", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() // these cannot be run in parallel due to repo locking // this seems like a bug, we should be able to run these without locking the repo t.Run("routing findprovs", func(t *testing.T) { res := node.RunIPFS("routing", "findprovs", testutils.CIDEmptyDir) assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "this command must be run in online mode") }) t.Run("routing findpeer", func(t *testing.T) { res := node.RunIPFS("routing", "findpeer", testutils.CIDEmptyDir) assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "this command must be run in online mode") }) t.Run("routing put", func(t *testing.T) { node.WriteBytes("foo", []byte("foo")) res := node.RunIPFS("routing", "put", "/ipns/"+node.PeerID().String(), "foo") assert.Equal(t, 1, res.ExitCode()) assert.Contains(t, res.Stderr.String(), "can't put while offline: pass `--allow-offline` to override") }) }) }) } func testSelfFindDHT(t *testing.T) { t.Run("ipfs routing findpeer fails for self", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(1).Init() nodes.ForEachPar(func(node *harness.Node) { node.IPFS("config", "Routing.Type", "dht") }) nodes.StartDaemons() defer nodes.StopDaemons() res := nodes[0].RunIPFS("dht", "findpeer", nodes[0].PeerID().String()) assert.Equal(t, 1, res.ExitCode()) }) } func TestRoutingDHT(t *testing.T) { testRoutingDHT(t, false) testRoutingDHT(t, true) testSelfFindDHT(t) } ================================================ FILE: test/cli/rpc_auth_test.go ================================================ package cli import ( "net/http" "testing" "github.com/ipfs/kubo/client/rpc/auth" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) const rpcDeniedMsg = "Kubo RPC Access Denied: Please provide a valid authorization token as defined in the API.Authorizations configuration." func TestRPCAuth(t *testing.T) { t.Parallel() makeAndStartProtectedNode := func(t *testing.T, authorizations map[string]*config.RPCAuthScope) *harness.Node { authorizations["test-node-starter"] = &config.RPCAuthScope{ AuthSecret: "bearer:test-node-starter", AllowedPaths: []string{"/api/v0"}, } node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.API.Authorizations = authorizations }) node.StartDaemonWithAuthorization("Bearer test-node-starter") return node } makeHTTPTest := func(authSecret, header string) func(t *testing.T) { return func(t *testing.T) { t.Parallel() t.Log(authSecret, header) node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{ "userA": { AuthSecret: authSecret, AllowedPaths: []string{"/api/v0/id"}, }, }) apiClient := node.APIClient() apiClient.Client = &http.Client{ Transport: auth.NewAuthorizedRoundTripper(header, http.DefaultTransport), } // Can access /id with valid token resp := apiClient.Post("/api/v0/id", nil) assert.Equal(t, 200, resp.StatusCode) // But not /config/show resp = apiClient.Post("/api/v0/config/show", nil) assert.Equal(t, 403, resp.StatusCode) // create client which sends invalid access token invalidApiClient := node.APIClient() invalidApiClient.Client = &http.Client{ Transport: auth.NewAuthorizedRoundTripper("Bearer invalid", http.DefaultTransport), } // Can't access /id with invalid token errResp := invalidApiClient.Post("/api/v0/id", nil) assert.Equal(t, 403, errResp.StatusCode) node.StopDaemon() } } makeCLITest := func(authSecret string) func(t *testing.T) { return func(t *testing.T) { t.Parallel() node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{ "userA": { AuthSecret: authSecret, AllowedPaths: []string{"/api/v0/id"}, }, }) // Can access 'ipfs id' resp := node.RunIPFS("id", "--api-auth", authSecret) require.NoError(t, resp.Err) // But not 'ipfs config show' resp = node.RunIPFS("config", "show", "--api-auth", authSecret) require.Error(t, resp.Err) require.Contains(t, resp.Stderr.String(), rpcDeniedMsg) node.StopDaemon() } } for _, testCase := range []struct { name string authSecret string header string }{ {"Bearer (no type)", "myToken", "Bearer myToken"}, {"Bearer", "bearer:myToken", "Bearer myToken"}, {"Basic (user:pass)", "basic:user:pass", "Basic dXNlcjpwYXNz"}, {"Basic (encoded)", "basic:dXNlcjpwYXNz", "Basic dXNlcjpwYXNz"}, } { t.Run("AllowedPaths on CLI "+testCase.name, makeCLITest(testCase.authSecret)) t.Run("AllowedPaths on HTTP "+testCase.name, makeHTTPTest(testCase.authSecret, testCase.header)) } t.Run("AllowedPaths set to /api/v0 Gives Full Access", func(t *testing.T) { t.Parallel() node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{ "userA": { AuthSecret: "bearer:userAToken", AllowedPaths: []string{"/api/v0"}, }, }) apiClient := node.APIClient() apiClient.Client = &http.Client{ Transport: auth.NewAuthorizedRoundTripper("Bearer userAToken", http.DefaultTransport), } resp := apiClient.Post("/api/v0/id", nil) assert.Equal(t, 200, resp.StatusCode) node.StopDaemon() }) t.Run("API.Authorizations set to nil disables Authorization header check", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.API.Authorizations = nil }) node.StartDaemon() apiClient := node.APIClient() resp := apiClient.Post("/api/v0/id", nil) assert.Equal(t, 200, resp.StatusCode) node.StopDaemon() }) t.Run("API.Authorizations set to empty map disables Authorization header check", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.API.Authorizations = map[string]*config.RPCAuthScope{} }) node.StartDaemon() apiClient := node.APIClient() resp := apiClient.Post("/api/v0/id", nil) assert.Equal(t, 200, resp.StatusCode) node.StopDaemon() }) t.Run("Requests without Authorization header are rejected when auth is enabled", func(t *testing.T) { t.Parallel() node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{ "userA": { AuthSecret: "bearer:mytoken", AllowedPaths: []string{"/api/v0"}, }, }) // Create client with NO auth apiClient := node.APIClient() // Uses http.DefaultClient with no auth headers // Should be denied without auth header resp := apiClient.Post("/api/v0/id", nil) assert.Equal(t, 403, resp.StatusCode) // Should contain denial message assert.Contains(t, resp.Body, rpcDeniedMsg) node.StopDaemon() }) t.Run("Version endpoint is always accessible even with limited AllowedPaths", func(t *testing.T) { t.Parallel() node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{ "userA": { AuthSecret: "bearer:mytoken", AllowedPaths: []string{"/api/v0/id"}, // Only /id allowed }, }) apiClient := node.APIClient() apiClient.Client = &http.Client{ Transport: auth.NewAuthorizedRoundTripper("Bearer mytoken", http.DefaultTransport), } // Can access /version even though not in AllowedPaths resp := apiClient.Post("/api/v0/version", nil) assert.Equal(t, 200, resp.StatusCode) node.StopDaemon() }) t.Run("User cannot access API with another user's secret", func(t *testing.T) { t.Parallel() node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{ "alice": { AuthSecret: "bearer:alice-secret", AllowedPaths: []string{"/api/v0/id"}, }, "bob": { AuthSecret: "bearer:bob-secret", AllowedPaths: []string{"/api/v0/config"}, }, }) // Alice tries to use Bob's secret apiClient := node.APIClient() apiClient.Client = &http.Client{ Transport: auth.NewAuthorizedRoundTripper("Bearer bob-secret", http.DefaultTransport), } // Bob's secret should work for Bob's paths resp := apiClient.Post("/api/v0/config/show", nil) assert.Equal(t, 200, resp.StatusCode) // But not for Alice's paths (Bob doesn't have access to /id) resp = apiClient.Post("/api/v0/id", nil) assert.Equal(t, 403, resp.StatusCode) node.StopDaemon() }) t.Run("Empty AllowedPaths denies all access except version", func(t *testing.T) { t.Parallel() node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{ "userA": { AuthSecret: "bearer:mytoken", AllowedPaths: []string{}, // Empty! }, }) apiClient := node.APIClient() apiClient.Client = &http.Client{ Transport: auth.NewAuthorizedRoundTripper("Bearer mytoken", http.DefaultTransport), } // Should deny everything resp := apiClient.Post("/api/v0/id", nil) assert.Equal(t, 403, resp.StatusCode) resp = apiClient.Post("/api/v0/config/show", nil) assert.Equal(t, 403, resp.StatusCode) // Except version resp = apiClient.Post("/api/v0/version", nil) assert.Equal(t, 200, resp.StatusCode) node.StopDaemon() }) t.Run("CLI commands fail without --api-auth when auth is enabled", func(t *testing.T) { t.Parallel() node := makeAndStartProtectedNode(t, map[string]*config.RPCAuthScope{ "userA": { AuthSecret: "bearer:mytoken", AllowedPaths: []string{"/api/v0"}, }, }) // Try to run command without --api-auth flag resp := node.RunIPFS("id") // No --api-auth flag require.Error(t, resp.Err) require.Contains(t, resp.Stderr.String(), rpcDeniedMsg) node.StopDaemon() }) } ================================================ FILE: test/cli/rpc_content_type_test.go ================================================ // Tests HTTP RPC Content-Type headers. // These tests verify that RPC endpoints return correct Content-Type headers // for binary responses (CAR, tar, gzip, raw blocks, IPNS records). package cli import ( "bytes" "encoding/base64" "encoding/json" "io" "net/http" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // TestRPCDagExportContentType verifies that the RPC endpoint for `ipfs dag export` // returns the correct Content-Type header for CAR output. func TestRPCDagExportContentType(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.StartDaemon("--offline") // add test content cid := node.IPFSAddStr("test content for dag export") url := node.APIURL() + "/api/v0/dag/export?arg=" + cid req, err := http.NewRequest(http.MethodPost, url, nil) require.NoError(t, err) resp, err := http.DefaultClient.Do(req) require.NoError(t, err) defer resp.Body.Close() assert.Equal(t, http.StatusOK, resp.StatusCode) assert.Equal(t, "application/vnd.ipld.car", resp.Header.Get("Content-Type"), "dag export should return application/vnd.ipld.car") } // TestRPCBlockGetContentType verifies that the RPC endpoint for `ipfs block get` // returns the correct Content-Type header for raw block data. func TestRPCBlockGetContentType(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.StartDaemon("--offline") // add test content cid := node.IPFSAddStr("test content for block get") url := node.APIURL() + "/api/v0/block/get?arg=" + cid req, err := http.NewRequest(http.MethodPost, url, nil) require.NoError(t, err) resp, err := http.DefaultClient.Do(req) require.NoError(t, err) defer resp.Body.Close() assert.Equal(t, http.StatusOK, resp.StatusCode) assert.Equal(t, "application/vnd.ipld.raw", resp.Header.Get("Content-Type"), "block get should return application/vnd.ipld.raw") } // TestRPCProfileContentType verifies that the RPC endpoint for `ipfs diag profile` // returns the correct Content-Type header for ZIP output. func TestRPCProfileContentType(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.StartDaemon("--offline") // use profile-time=0 to skip sampling profiles and return quickly url := node.APIURL() + "/api/v0/diag/profile?profile-time=0" req, err := http.NewRequest(http.MethodPost, url, nil) require.NoError(t, err) resp, err := http.DefaultClient.Do(req) require.NoError(t, err) defer resp.Body.Close() assert.Equal(t, http.StatusOK, resp.StatusCode) assert.Equal(t, "application/zip", resp.Header.Get("Content-Type"), "diag profile should return application/zip") } // TestHTTPRPCNameGet verifies the behavior of `ipfs name get` vs `ipfs routing get`: // // `ipfs name get `: // - Purpose: dedicated command for retrieving IPNS records // - Returns: raw IPNS record bytes (protobuf) // - Content-Type: application/vnd.ipfs.ipns-record // // `ipfs routing get /ipns/`: // - Purpose: generic routing get for any key type // - Returns: JSON with base64-encoded record in "Extra" field // - Content-Type: application/json // // Both commands retrieve the same underlying IPNS record data. func TestHTTPRPCNameGet(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.StartDaemon() // must be online to use routing // add test content and publish IPNS record cid := node.IPFSAddStr("test content for name get") node.IPFS("name", "publish", cid) // get the node's peer ID (which is also the IPNS name) peerID := node.PeerID().String() // Test ipfs name get - returns raw IPNS record bytes with specific Content-Type nameGetURL := node.APIURL() + "/api/v0/name/get?arg=" + peerID nameGetReq, err := http.NewRequest(http.MethodPost, nameGetURL, nil) require.NoError(t, err) nameGetResp, err := http.DefaultClient.Do(nameGetReq) require.NoError(t, err) defer nameGetResp.Body.Close() assert.Equal(t, http.StatusOK, nameGetResp.StatusCode) assert.Equal(t, "application/vnd.ipfs.ipns-record", nameGetResp.Header.Get("Content-Type"), "name get should return application/vnd.ipfs.ipns-record") nameGetBytes, err := io.ReadAll(nameGetResp.Body) require.NoError(t, err) // Test ipfs routing get /ipns/... - returns JSON with base64-encoded record routingGetURL := node.APIURL() + "/api/v0/routing/get?arg=/ipns/" + peerID routingGetReq, err := http.NewRequest(http.MethodPost, routingGetURL, nil) require.NoError(t, err) routingGetResp, err := http.DefaultClient.Do(routingGetReq) require.NoError(t, err) defer routingGetResp.Body.Close() assert.Equal(t, http.StatusOK, routingGetResp.StatusCode) assert.Equal(t, "application/json", routingGetResp.Header.Get("Content-Type"), "routing get should return application/json") // Parse JSON response and decode base64 record from "Extra" field var routingResp struct { Extra string `json:"Extra"` Type int `json:"Type"` } err = json.NewDecoder(routingGetResp.Body).Decode(&routingResp) require.NoError(t, err) routingGetBytes, err := base64.StdEncoding.DecodeString(routingResp.Extra) require.NoError(t, err) // Verify both commands return identical IPNS record bytes assert.Equal(t, nameGetBytes, routingGetBytes, "name get and routing get should return identical IPNS record bytes") // Verify the record can be inspected and contains the published CID inspectOutput := node.PipeToIPFS(bytes.NewReader(nameGetBytes), "name", "inspect") assert.Contains(t, inspectOutput.Stdout.String(), cid, "ipfs name inspect should show the published CID") } ================================================ FILE: test/cli/rpc_get_output_test.go ================================================ package cli import ( "net/http" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // TestRPCGetContentType verifies that the RPC endpoint for `ipfs get` returns // the correct Content-Type header based on output format options. // // Output formats and expected Content-Type: // - default (no flags): tar (transport format) -> application/x-tar // - --archive: tar archive -> application/x-tar // - --compress: gzip -> application/gzip // - --archive --compress: tar.gz -> application/gzip // // Fixes: https://github.com/ipfs/kubo/issues/2376 func TestRPCGetContentType(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.StartDaemon("--offline") // add test content cid := node.IPFSAddStr("test content for Content-Type header verification") tests := []struct { name string query string expectedContentType string }{ { name: "default returns application/x-tar", query: "?arg=" + cid, expectedContentType: "application/x-tar", }, { name: "archive=true returns application/x-tar", query: "?arg=" + cid + "&archive=true", expectedContentType: "application/x-tar", }, { name: "compress=true returns application/gzip", query: "?arg=" + cid + "&compress=true", expectedContentType: "application/gzip", }, { name: "archive=true&compress=true returns application/gzip", query: "?arg=" + cid + "&archive=true&compress=true", expectedContentType: "application/gzip", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { url := node.APIURL() + "/api/v0/get" + tt.query req, err := http.NewRequest(http.MethodPost, url, nil) require.NoError(t, err) resp, err := http.DefaultClient.Do(req) require.NoError(t, err) defer resp.Body.Close() assert.Equal(t, http.StatusOK, resp.StatusCode) assert.Equal(t, tt.expectedContentType, resp.Header.Get("Content-Type"), "Content-Type header mismatch for %s", tt.name) }) } } ================================================ FILE: test/cli/rpc_unixsocket_test.go ================================================ package cli import ( "context" "path" "testing" rpcapi "github.com/ipfs/kubo/client/rpc" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/require" ) func TestRPCUnixSocket(t *testing.T) { node := harness.NewT(t).NewNode().Init() sockDir := node.Dir sockAddr := path.Join("/unix", sockDir, "sock") node.UpdateConfig(func(cfg *config.Config) { //cfg.Addresses.API = append(cfg.Addresses.API, sockPath) cfg.Addresses.API = []string{sockAddr} }) t.Log("Starting daemon with unix socket:", sockAddr) node.StartDaemon() unixMaddr, err := multiaddr.NewMultiaddr(sockAddr) require.NoError(t, err) apiClient, err := rpcapi.NewApi(unixMaddr) require.NoError(t, err) var ver struct { Version string } err = apiClient.Request("version").Exec(context.Background(), &ver) require.NoError(t, err) require.NotEmpty(t, ver) t.Log("Got version:", ver.Version) var res struct { ID string } err = apiClient.Request("id").Exec(context.Background(), &res) require.NoError(t, err) require.NotEmpty(t, res) t.Log("Got ID:", res.ID) node.StopDaemon() } ================================================ FILE: test/cli/stats_test.go ================================================ package cli import ( "testing" "github.com/stretchr/testify/assert" "github.com/ipfs/kubo/test/cli/harness" ) func TestStats(t *testing.T) { t.Parallel() t.Run("stats dht", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(2).Init().StartDaemons().Connect() defer nodes.StopDaemons() node1 := nodes[0] res := node1.IPFS("stats", "dht") assert.NoError(t, res.Err) assert.Equal(t, 0, len(res.Stderr.Lines())) assert.NotEqual(t, 0, len(res.Stdout.Lines())) }) } ================================================ FILE: test/cli/swarm_test.go ================================================ package cli import ( "encoding/json" "fmt" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" ) // TODO: Migrate the rest of the sharness swarm test. func TestSwarm(t *testing.T) { type identifyType struct { ID string PublicKey string Addresses []string AgentVersion string Protocols []string } type peer struct { Identify identifyType } type expectedOutputType struct { Peers []peer } t.Parallel() t.Run("ipfs swarm peers returns empty peers when a node is not connected to any peers", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() res := node.RunIPFS("swarm", "peers", "--enc=json", "--identify") var output expectedOutputType err := json.Unmarshal(res.Stdout.Bytes(), &output) assert.NoError(t, err) assert.Equal(t, 0, len(output.Peers)) }) t.Run("ipfs swarm peers with flag identify outputs expected identify information about connected peers", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() otherNode := harness.NewT(t).NewNode().Init().StartDaemon() defer otherNode.StopDaemon() node.Connect(otherNode) res := node.RunIPFS("swarm", "peers", "--enc=json", "--identify") var output expectedOutputType err := json.Unmarshal(res.Stdout.Bytes(), &output) assert.NoError(t, err) actualID := output.Peers[0].Identify.ID actualPublicKey := output.Peers[0].Identify.PublicKey actualAgentVersion := output.Peers[0].Identify.AgentVersion actualAddresses := output.Peers[0].Identify.Addresses actualProtocols := output.Peers[0].Identify.Protocols expectedID := otherNode.PeerID().String() expectedAddresses := []string{fmt.Sprintf("%s/p2p/%s", otherNode.SwarmAddrs()[0], actualID)} assert.Equal(t, actualID, expectedID) assert.NotNil(t, actualPublicKey) assert.NotNil(t, actualAgentVersion) assert.Len(t, actualAddresses, 1) assert.Equal(t, expectedAddresses[0], actualAddresses[0]) assert.Greater(t, len(actualProtocols), 0) }) t.Run("ipfs swarm peers with flag identify outputs Identify field with data that matches calling ipfs id on a peer", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() otherNode := harness.NewT(t).NewNode().Init().StartDaemon() defer otherNode.StopDaemon() node.Connect(otherNode) otherNodeIDResponse := otherNode.RunIPFS("id", "--enc=json") var otherNodeIDOutput identifyType err := json.Unmarshal(otherNodeIDResponse.Stdout.Bytes(), &otherNodeIDOutput) assert.NoError(t, err) res := node.RunIPFS("swarm", "peers", "--enc=json", "--identify") var output expectedOutputType err = json.Unmarshal(res.Stdout.Bytes(), &output) assert.NoError(t, err) outputIdentify := output.Peers[0].Identify assert.Equal(t, outputIdentify.ID, otherNodeIDOutput.ID) assert.Equal(t, outputIdentify.PublicKey, otherNodeIDOutput.PublicKey) assert.Equal(t, outputIdentify.AgentVersion, otherNodeIDOutput.AgentVersion) assert.ElementsMatch(t, outputIdentify.Addresses, otherNodeIDOutput.Addresses) assert.ElementsMatch(t, outputIdentify.Protocols, otherNodeIDOutput.Protocols) }) t.Run("ipfs swarm addrs autonat returns valid reachability status", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() res := node.RunIPFS("swarm", "addrs", "autonat", "--enc=json") assert.NoError(t, res.Err) var output struct { Reachability string `json:"reachability"` Reachable []string `json:"reachable"` Unreachable []string `json:"unreachable"` Unknown []string `json:"unknown"` } err := json.Unmarshal(res.Stdout.Bytes(), &output) assert.NoError(t, err) // Reachability must be one of the valid states // Note: network.Reachability constants use capital first letter validStates := []string{"Public", "Private", "Unknown"} assert.Contains(t, validStates, output.Reachability, "Reachability should be one of: Public, Private, Unknown") // For a newly started node, reachability is typically Unknown initially // as AutoNAT hasn't completed probing yet. This is expected behavior. // The important thing is that the command runs and returns valid data. totalAddrs := len(output.Reachable) + len(output.Unreachable) + len(output.Unknown) t.Logf("Reachability: %s, Total addresses: %d (reachable: %d, unreachable: %d, unknown: %d)", output.Reachability, totalAddrs, len(output.Reachable), len(output.Unreachable), len(output.Unknown)) }) } ================================================ FILE: test/cli/telemetry_test.go ================================================ package cli import ( "encoding/json" "io" "maps" "net/http" "net/http/httptest" "os" "path/filepath" "slices" "testing" "time" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestTelemetry(t *testing.T) { t.Parallel() t.Run("opt-out via environment variable", func(t *testing.T) { t.Parallel() // Create a new node node := harness.NewT(t).NewNode().Init() node.SetIPFSConfig("Plugins.Plugins.telemetry.Disabled", false) // Set the opt-out environment variable node.Runner.Env["IPFS_TELEMETRY"] = "off" node.Runner.Env["GOLOG_LOG_LEVEL"] = "telemetry=debug" // Capture daemon output stdout := &harness.Buffer{} stderr := &harness.Buffer{} // Start daemon with output capture node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithStdout(stdout), harness.RunWithStderr(stderr), }, }, "") time.Sleep(500 * time.Millisecond) // Get daemon output output := stdout.String() + stderr.String() // Check that telemetry is disabled assert.Contains(t, output, "telemetry disabled via opt-out", "Expected telemetry disabled message") // Stop daemon node.StopDaemon() // Verify UUID file was not created or was removed uuidPath := filepath.Join(node.Dir, "telemetry_uuid") _, err := os.Stat(uuidPath) assert.True(t, os.IsNotExist(err), "UUID file should not exist when opted out") }) t.Run("opt-out via config", func(t *testing.T) { t.Parallel() // Create a new node node := harness.NewT(t).NewNode().Init() node.SetIPFSConfig("Plugins.Plugins.telemetry.Disabled", false) // Set opt-out via config node.IPFS("config", "Plugins.Plugins.telemetry.Config.Mode", "off") // Enable debug logging node.Runner.Env["GOLOG_LOG_LEVEL"] = "telemetry=debug" // Capture daemon output stdout := &harness.Buffer{} stderr := &harness.Buffer{} // Start daemon with output capture node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithStdout(stdout), harness.RunWithStderr(stderr), }, }, "") time.Sleep(500 * time.Millisecond) // Get daemon output output := stdout.String() + stderr.String() // Check that telemetry is disabled assert.Contains(t, output, "telemetry disabled via opt-out", "Expected telemetry disabled message") assert.Contains(t, output, "telemetry collection skipped: opted out", "Expected telemetry skipped message") // Stop daemon node.StopDaemon() // Verify UUID file was not created or was removed uuidPath := filepath.Join(node.Dir, "telemetry_uuid") _, err := os.Stat(uuidPath) assert.True(t, os.IsNotExist(err), "UUID file should not exist when opted out") }) t.Run("opt-out removes existing UUID file", func(t *testing.T) { t.Parallel() // Create a new node node := harness.NewT(t).NewNode().Init() node.SetIPFSConfig("Plugins.Plugins.telemetry.Disabled", false) // Create a UUID file manually to simulate previous telemetry run uuidPath := filepath.Join(node.Dir, "telemetry_uuid") testUUID := "test-uuid-12345" err := os.WriteFile(uuidPath, []byte(testUUID), 0600) require.NoError(t, err, "Failed to create test UUID file") // Verify file exists _, err = os.Stat(uuidPath) require.NoError(t, err, "UUID file should exist before opt-out") // Set the opt-out environment variable node.Runner.Env["IPFS_TELEMETRY"] = "off" node.Runner.Env["GOLOG_LOG_LEVEL"] = "telemetry=debug" // Capture daemon output stdout := &harness.Buffer{} stderr := &harness.Buffer{} // Start daemon with output capture node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithStdout(stdout), harness.RunWithStderr(stderr), }, }, "") time.Sleep(500 * time.Millisecond) // Get daemon output output := stdout.String() + stderr.String() // Check that UUID file was removed assert.Contains(t, output, "removed existing telemetry UUID file due to opt-out", "Expected UUID removal message") // Stop daemon node.StopDaemon() // Verify UUID file was removed _, err = os.Stat(uuidPath) assert.True(t, os.IsNotExist(err), "UUID file should be removed after opt-out") }) t.Run("telemetry enabled shows info message", func(t *testing.T) { t.Parallel() // Create a new node node := harness.NewT(t).NewNode().Init() node.SetIPFSConfig("Plugins.Plugins.telemetry.Disabled", false) // Capture daemon output stdout := &harness.Buffer{} stderr := &harness.Buffer{} // Don't set opt-out, so telemetry will be enabled // This should trigger the info message on first run node.StartDaemonWithReq(harness.RunRequest{ CmdOpts: []harness.CmdOpt{ harness.RunWithStdout(stdout), harness.RunWithStderr(stderr), }, }, "") time.Sleep(500 * time.Millisecond) // Get daemon output output := stdout.String() + stderr.String() // First run - should show info message assert.Contains(t, output, "Anonymous telemetry") assert.Contains(t, output, "No data sent yet", "Expected no data sent message") assert.Contains(t, output, "To opt-out before collection starts", "Expected opt-out instructions") assert.Contains(t, output, "Learn more:", "Expected learn more link") // Stop daemon node.StopDaemon() // Verify UUID file was created uuidPath := filepath.Join(node.Dir, "telemetry_uuid") _, err := os.Stat(uuidPath) assert.NoError(t, err, "UUID file should exist when daemon started without telemetry opt-out") }) t.Run("telemetry schema regression guard", func(t *testing.T) { t.Parallel() // Define the exact set of expected telemetry fields // This list must be updated whenever telemetry fields change expectedFields := []string{ "uuid", "agent_version", "private_network", "bootstrappers_custom", "repo_size_bucket", "uptime_bucket", "reprovider_strategy", "provide_dht_sweep_enabled", "provide_dht_interval_custom", "provide_dht_max_workers_custom", "routing_type", "routing_accelerated_dht_client", "routing_delegated_count", "autonat_service_mode", "autonat_reachability", "swarm_enable_hole_punching", "swarm_circuit_addresses", "swarm_ipv4_public_addresses", "swarm_ipv6_public_addresses", "auto_tls_auto_wss", "auto_tls_domain_suffix_custom", "autoconf", "autoconf_custom", "discovery_mdns_enabled", "platform_os", "platform_arch", "platform_containerized", "platform_vm", } // Channel to receive captured telemetry data telemetryChan := make(chan map[string]any, 1) // Create a mock HTTP server to capture telemetry mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != "POST" { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } body, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "Failed to read body", http.StatusBadRequest) return } var telemetryData map[string]any if err := json.Unmarshal(body, &telemetryData); err != nil { http.Error(w, "Invalid JSON", http.StatusBadRequest) return } // Send captured data through channel select { case telemetryChan <- telemetryData: default: } w.WriteHeader(http.StatusOK) })) defer mockServer.Close() // Create a new node node := harness.NewT(t).NewNode().Init() node.SetIPFSConfig("Plugins.Plugins.telemetry.Disabled", false) // Configure telemetry with a very short delay for testing node.IPFS("config", "Plugins.Plugins.telemetry.Config.Delay", "100ms") node.IPFS("config", "Plugins.Plugins.telemetry.Config.Endpoint", mockServer.URL) // Enable debug logging to see what's being sent node.Runner.Env["GOLOG_LOG_LEVEL"] = "telemetry=debug" // Start daemon node.StartDaemon() defer node.StopDaemon() // Wait for telemetry to be sent (configured delay + buffer) select { case telemetryData := <-telemetryChan: receivedFields := slices.Collect(maps.Keys(telemetryData)) slices.Sort(expectedFields) slices.Sort(receivedFields) // Fast path: check if fields match exactly if !slices.Equal(expectedFields, receivedFields) { var missingFields, unexpectedFields []string for _, field := range expectedFields { if _, ok := telemetryData[field]; !ok { missingFields = append(missingFields, field) } } expectedSet := make(map[string]struct{}, len(expectedFields)) for _, f := range expectedFields { expectedSet[f] = struct{}{} } for field := range telemetryData { if _, ok := expectedSet[field]; !ok { unexpectedFields = append(unexpectedFields, field) } } t.Fatalf("Telemetry field mismatch:\n"+ " Missing fields: %v\n"+ " Unexpected fields: %v\n"+ " Note: Update expectedFields list in this test when adding/removing telemetry fields", missingFields, unexpectedFields) } t.Logf("Telemetry field validation passed: %d fields verified", len(expectedFields)) case <-time.After(5 * time.Second): t.Fatal("Timeout waiting for telemetry data to be sent") } }) } ================================================ FILE: test/cli/testutils/asserts.go ================================================ package testutils import ( "strings" "testing" ) func AssertStringContainsOneOf(t *testing.T, str string, ss ...string) { for _, s := range ss { if strings.Contains(str, s) { return } } t.Errorf("%q does not contain one of %v", str, ss) } ================================================ FILE: test/cli/testutils/cids.go ================================================ package testutils const ( CIDWelcomeDocs = "QmQPeNsJPyVWPFDVHb77w8G42Fvo15z4bG2X8D2GhfbSXc" CIDEmptyDir = "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" ) ================================================ FILE: test/cli/testutils/files.go ================================================ package testutils import ( "log" "os" "path/filepath" ) func MustOpen(name string) *os.File { f, err := os.Open(name) if err != nil { log.Panicf("opening %s: %s", name, err) } return f } // Searches for a file in a dir, then the parent dir, etc. // If the file is not found, an empty string is returned. func FindUp(name, dir string) string { curDir := dir for { entries, err := os.ReadDir(curDir) if err != nil { panic(err) } for _, e := range entries { if name == e.Name() { return filepath.Join(curDir, name) } } newDir := filepath.Dir(curDir) if newDir == curDir { return "" } curDir = newDir } } ================================================ FILE: test/cli/testutils/floats.go ================================================ package testutils func FloatTruncate(value float64, decimalPlaces int) float64 { pow := 1.0 for range decimalPlaces { pow *= 10.0 } return float64(int(value*pow)) / pow } ================================================ FILE: test/cli/testutils/httprouting/mock_http_content_router.go ================================================ package httprouting import ( "context" "fmt" "sync" "time" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/routing/http/server" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" ) // MockHTTPContentRouter provides /routing/v1 // (https://specs.ipfs.tech/routing/http-routing-v1/) server implementation // based on github.com/ipfs/boxo/routing/http/server type MockHTTPContentRouter struct { m sync.Mutex provideBitswapCalls int findProvidersCalls int findPeersCalls int getClosestPeersCalls int providers map[cid.Cid][]types.Record peers map[peer.ID][]*types.PeerRecord Debug bool } func (r *MockHTTPContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) { if r.Debug { fmt.Printf("MockHTTPContentRouter.FindProviders(%s)\n", key.String()) } r.m.Lock() defer r.m.Unlock() r.findProvidersCalls++ if r.providers == nil { r.providers = make(map[cid.Cid][]types.Record) } records, found := r.providers[key] if !found { return iter.FromSlice([]iter.Result[types.Record]{}), nil } results := make([]iter.Result[types.Record], len(records)) for i, rec := range records { results[i] = iter.Result[types.Record]{Val: rec} if r.Debug { fmt.Printf("MockHTTPContentRouter.FindProviders(%s) result: %+v\n", key.String(), rec) } } return iter.FromSlice(results), nil } // nolint deprecated func (r *MockHTTPContentRouter) ProvideBitswap(ctx context.Context, req *server.BitswapWriteProvideRequest) (time.Duration, error) { r.m.Lock() defer r.m.Unlock() r.provideBitswapCalls++ return 0, nil } func (r *MockHTTPContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[*types.PeerRecord], error) { r.m.Lock() defer r.m.Unlock() r.findPeersCalls++ if r.peers == nil { r.peers = make(map[peer.ID][]*types.PeerRecord) } records, found := r.peers[pid] if !found { return iter.FromSlice([]iter.Result[*types.PeerRecord]{}), nil } results := make([]iter.Result[*types.PeerRecord], len(records)) for i, rec := range records { results[i] = iter.Result[*types.PeerRecord]{Val: rec} if r.Debug { fmt.Printf("MockHTTPContentRouter.FindPeers(%s) result: %+v\n", pid.String(), rec) } } return iter.FromSlice(results), nil } func (r *MockHTTPContentRouter) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { return nil, routing.ErrNotSupported } func (r *MockHTTPContentRouter) PutIPNS(ctx context.Context, name ipns.Name, rec *ipns.Record) error { return routing.ErrNotSupported } func (r *MockHTTPContentRouter) NumFindProvidersCalls() int { r.m.Lock() defer r.m.Unlock() return r.findProvidersCalls } // AddProvider adds a record for a given CID func (r *MockHTTPContentRouter) AddProvider(key cid.Cid, record types.Record) { r.m.Lock() defer r.m.Unlock() if r.providers == nil { r.providers = make(map[cid.Cid][]types.Record) } r.providers[key] = append(r.providers[key], record) peerRecord, ok := record.(*types.PeerRecord) if ok { if r.peers == nil { r.peers = make(map[peer.ID][]*types.PeerRecord) } pid := peerRecord.ID r.peers[*pid] = append(r.peers[*pid], peerRecord) } } func (r *MockHTTPContentRouter) GetClosestPeers(ctx context.Context, key cid.Cid) (iter.ResultIter[*types.PeerRecord], error) { r.m.Lock() defer r.m.Unlock() r.getClosestPeersCalls++ if r.peers == nil { r.peers = make(map[peer.ID][]*types.PeerRecord) } pid, err := peer.FromCid(key) if err != nil { return iter.FromSlice([]iter.Result[*types.PeerRecord]{}), nil } records, found := r.peers[pid] if !found { return iter.FromSlice([]iter.Result[*types.PeerRecord]{}), nil } results := make([]iter.Result[*types.PeerRecord], len(records)) for i, rec := range records { results[i] = iter.Result[*types.PeerRecord]{Val: rec} if r.Debug { fmt.Printf("MockHTTPContentRouter.GetPeers(%s) result: %+v\n", pid.String(), rec) } } return iter.FromSlice(results), nil } ================================================ FILE: test/cli/testutils/json.go ================================================ package testutils import "encoding/json" type JSONObj map[string]any func ToJSONStr(m JSONObj) string { b, err := json.Marshal(m) if err != nil { panic(err) } return string(b) } ================================================ FILE: test/cli/testutils/pinningservice/pinning.go ================================================ package pinningservice import ( "encoding/json" "fmt" "net/http" "reflect" "strconv" "strings" "sync" "time" "github.com/google/uuid" "github.com/julienschmidt/httprouter" ) func NewRouter(authToken string, svc *PinningService) http.Handler { router := httprouter.New() router.GET("/api/v1/pins", svc.listPins) router.POST("/api/v1/pins", svc.addPin) router.GET("/api/v1/pins/:requestID", svc.getPin) router.POST("/api/v1/pins/:requestID", svc.replacePin) router.DELETE("/api/v1/pins/:requestID", svc.removePin) handler := authHandler(authToken, router) return handler } func authHandler(authToken string, delegate http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { authz := r.Header.Get("Authorization") if !strings.HasPrefix(authz, "Bearer ") { errResp(w, "invalid authorization token, must start with 'Bearer '", "", http.StatusBadRequest) return } token := strings.TrimPrefix(authz, "Bearer ") if token != authToken { errResp(w, "access denied", "", http.StatusUnauthorized) return } delegate.ServeHTTP(w, r) }) } func New() *PinningService { return &PinningService{ PinAdded: func(*AddPinRequest, *PinStatus) {}, } } // PinningService is a basic pinning service that implements the Remote Pinning API, for testing Kubo's integration with remote pinning services. // Pins are not persisted, they are just kept in-memory, and this provides callbacks for controlling the behavior of the pinning service. type PinningService struct { m sync.Mutex // PinAdded is a callback that is invoked after a new pin is added via the API. PinAdded func(*AddPinRequest, *PinStatus) pins []*PinStatus } type Pin struct { CID string `json:"cid"` Name string `json:"name"` Origins []string `json:"origins"` Meta map[string]any `json:"meta"` } type PinStatus struct { M sync.Mutex RequestID string Status string Created time.Time Pin Pin Delegates []string Info map[string]any } func (p *PinStatus) MarshalJSON() ([]byte, error) { type pinStatusJSON struct { RequestID string `json:"requestid"` Status string `json:"status"` Created time.Time `json:"created"` Pin Pin `json:"pin"` Delegates []string `json:"delegates"` Info map[string]any `json:"info"` } // lock the pin before marshaling it to protect against data races while marshaling p.M.Lock() pinJSON := pinStatusJSON{ RequestID: p.RequestID, Status: p.Status, Created: p.Created, Pin: p.Pin, Delegates: p.Delegates, Info: p.Info, } p.M.Unlock() return json.Marshal(pinJSON) } func (p *PinStatus) Clone() PinStatus { return PinStatus{ RequestID: p.RequestID, Status: p.Status, Created: p.Created, Pin: p.Pin, Delegates: p.Delegates, Info: p.Info, } } const ( matchExact = "exact" matchIExact = "iexact" matchPartial = "partial" matchIPartial = "ipartial" statusQueued = "queued" statusPinning = "pinning" statusPinned = "pinned" statusFailed = "failed" timeLayout = "2006-01-02T15:04:05.999Z" ) func errResp(w http.ResponseWriter, reason, details string, statusCode int) { type errorObj struct { Reason string `json:"reason"` Details string `json:"details"` } type errorResp struct { Error errorObj `json:"error"` } resp := errorResp{ Error: errorObj{ Reason: reason, Details: details, }, } writeJSON(w, resp, statusCode) } func writeJSON(w http.ResponseWriter, val any, statusCode int) { b, err := json.Marshal(val) if err != nil { w.Header().Set("Content-Type", "text/plain") errResp(w, fmt.Sprintf("marshaling response: %s", err), "", http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(statusCode) _, _ = w.Write(b) } type AddPinRequest struct { CID string `json:"cid"` Name string `json:"name"` Origins []string `json:"origins"` Meta map[string]any `json:"meta"` } func (p *PinningService) addPin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) { var addReq AddPinRequest err := json.NewDecoder(req.Body).Decode(&addReq) if err != nil { errResp(writer, fmt.Sprintf("unmarshaling req: %s", err), "", http.StatusBadRequest) return } pin := &PinStatus{ RequestID: uuid.NewString(), Status: statusQueued, Created: time.Now(), Pin: Pin(addReq), } p.m.Lock() p.pins = append(p.pins, pin) p.m.Unlock() writeJSON(writer, &pin, http.StatusAccepted) p.PinAdded(&addReq, pin) } type ListPinsResponse struct { Count int `json:"count"` Results []*PinStatus `json:"results"` } func (p *PinningService) listPins(writer http.ResponseWriter, req *http.Request, params httprouter.Params) { q := req.URL.Query() cidStr := q.Get("cid") name := q.Get("name") match := q.Get("match") status := q.Get("status") beforeStr := q.Get("before") afterStr := q.Get("after") limitStr := q.Get("limit") metaStr := q.Get("meta") if limitStr == "" { limitStr = "10" } limit, err := strconv.Atoi(limitStr) if err != nil { errResp(writer, fmt.Sprintf("parsing limit: %s", err), "", http.StatusBadRequest) return } var cids []string if cidStr != "" { cids = strings.Split(cidStr, ",") } var statuses []string if status != "" { statuses = strings.Split(status, ",") } p.m.Lock() defer p.m.Unlock() var pins []*PinStatus for _, pinStatus := range p.pins { // clone it so we can immediately release the lock pinStatus.M.Lock() clonedPS := pinStatus.Clone() pinStatus.M.Unlock() // cid var matchesCID bool if len(cids) == 0 { matchesCID = true } else { for _, cid := range cids { if cid == clonedPS.Pin.CID { matchesCID = true } } } if !matchesCID { continue } // name if match == "" { match = matchExact } if name != "" { switch match { case matchExact: if name != clonedPS.Pin.Name { continue } case matchIExact: if !strings.EqualFold(name, clonedPS.Pin.Name) { continue } case matchPartial: if !strings.Contains(clonedPS.Pin.Name, name) { continue } case matchIPartial: if !strings.Contains(strings.ToLower(clonedPS.Pin.Name), strings.ToLower(name)) { continue } default: errResp(writer, fmt.Sprintf("unknown match %q", match), "", http.StatusBadRequest) return } } // status var matchesStatus bool if len(statuses) == 0 { statuses = []string{statusPinned} } for _, status := range statuses { if status == clonedPS.Status { matchesStatus = true } } if !matchesStatus { continue } // before if beforeStr != "" { before, err := time.Parse(timeLayout, beforeStr) if err != nil { errResp(writer, fmt.Sprintf("parsing before: %s", err), "", http.StatusBadRequest) return } if !clonedPS.Created.Before(before) { continue } } // after if afterStr != "" { after, err := time.Parse(timeLayout, afterStr) if err != nil { errResp(writer, fmt.Sprintf("parsing before: %s", err), "", http.StatusBadRequest) return } if !clonedPS.Created.After(after) { continue } } // meta if metaStr != "" { meta := map[string]any{} err := json.Unmarshal([]byte(metaStr), &meta) if err != nil { errResp(writer, fmt.Sprintf("parsing meta: %s", err), "", http.StatusBadRequest) return } var matchesMeta bool for k, v := range meta { pinV, contains := clonedPS.Pin.Meta[k] if !contains || !reflect.DeepEqual(pinV, v) { matchesMeta = false break } } if !matchesMeta { continue } } // add the original pin status, not the cloned one pins = append(pins, pinStatus) if len(pins) == limit { break } } out := ListPinsResponse{ Count: len(pins), Results: pins, } writeJSON(writer, out, http.StatusOK) } func (p *PinningService) getPin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) { requestID := params.ByName("requestID") p.m.Lock() defer p.m.Unlock() for _, pin := range p.pins { if pin.RequestID == requestID { writeJSON(writer, pin, http.StatusOK) return } } errResp(writer, "", "", http.StatusNotFound) } func (p *PinningService) replacePin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) { requestID := params.ByName("requestID") var replaceReq Pin err := json.NewDecoder(req.Body).Decode(&replaceReq) if err != nil { errResp(writer, fmt.Sprintf("decoding request: %s", err), "", http.StatusBadRequest) return } p.m.Lock() defer p.m.Unlock() for _, pin := range p.pins { if pin.RequestID == requestID { pin.M.Lock() pin.Pin = replaceReq pin.M.Unlock() writer.WriteHeader(http.StatusAccepted) return } } errResp(writer, "", "", http.StatusNotFound) } func (p *PinningService) removePin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) { requestID := params.ByName("requestID") p.m.Lock() defer p.m.Unlock() for i, pin := range p.pins { if pin.RequestID == requestID { p.pins = append(p.pins[0:i], p.pins[i+1:]...) writer.WriteHeader(http.StatusAccepted) return } } errResp(writer, "", "", http.StatusNotFound) } ================================================ FILE: test/cli/testutils/protobuf.go ================================================ package testutils import "math/bits" // VarintLen returns the number of bytes needed to encode v as a protobuf varint. func VarintLen(v uint64) int { return int(9*uint32(bits.Len64(v))+64) / 64 } // LinkSerializedSize calculates the serialized size of a single PBLink in a dag-pb block. // This matches the calculation in boxo/ipld/unixfs/io/directory.go estimatedBlockSize(). // // The protobuf wire format for a PBLink is: // // PBNode.Links wrapper tag (1 byte) // + varint length of inner message // + Hash field: tag (1) + varint(cidLen) + cidLen // + Name field: tag (1) + varint(nameLen) + nameLen // + Tsize field: tag (1) + varint(tsize) func LinkSerializedSize(nameLen, cidLen int, tsize uint64) int { // Inner link message size linkLen := 1 + VarintLen(uint64(cidLen)) + cidLen + // Hash field 1 + VarintLen(uint64(nameLen)) + nameLen + // Name field 1 + VarintLen(tsize) // Tsize field // Outer wrapper: tag (1 byte) + varint(linkLen) + linkLen return 1 + VarintLen(uint64(linkLen)) + linkLen } // EstimateFilesForBlockThreshold estimates how many files with given name/cid lengths // will fit under the block size threshold. // Returns the number of files that keeps the block size just under the threshold. func EstimateFilesForBlockThreshold(threshold, nameLen, cidLen int, tsize uint64) int { linkSize := LinkSerializedSize(nameLen, cidLen, tsize) // Base overhead for empty directory node (Data field + minimal structure) // Empirically determined to be 4 bytes for dag-pb directories baseOverhead := 4 return (threshold - baseOverhead) / linkSize } ================================================ FILE: test/cli/testutils/random_deterministic.go ================================================ package testutils import ( "crypto/sha256" "io" "github.com/dustin/go-humanize" "golang.org/x/crypto/chacha20" ) type randomReader struct { cipher *chacha20.Cipher remaining int64 } func (r *randomReader) Read(p []byte) (int, error) { if r.remaining <= 0 { return 0, io.EOF } n := min(int64(len(p)), r.remaining) // Generate random bytes directly into the provided buffer r.cipher.XORKeyStream(p[:n], make([]byte, n)) r.remaining -= n return int(n), nil } // DeterministicRandomReader produces specified number of pseudo-random bytes // from a seed. Size can be specified as a humanize string (e.g., "256KiB", "1MiB"). func DeterministicRandomReader(sizeStr string, seed string) (io.Reader, error) { size, err := humanize.ParseBytes(sizeStr) if err != nil { return nil, err } return DeterministicRandomReaderBytes(int64(size), seed) } // DeterministicRandomReaderBytes produces exactly `size` pseudo-random bytes // from a seed. Use this when exact byte precision is needed. func DeterministicRandomReaderBytes(size int64, seed string) (io.Reader, error) { // Hash the seed string to a 32-byte key for ChaCha20 key := sha256.Sum256([]byte(seed)) // Use ChaCha20 for deterministic random bytes var nonce [chacha20.NonceSize]byte // Zero nonce for simplicity cipher, err := chacha20.NewUnauthenticatedCipher(key[:chacha20.KeySize], nonce[:]) if err != nil { return nil, err } return &randomReader{cipher: cipher, remaining: size}, nil } ================================================ FILE: test/cli/testutils/requires.go ================================================ package testutils import ( "os" "os/exec" "runtime" "testing" ) func RequiresDocker(t *testing.T) { if os.Getenv("TEST_DOCKER") != "1" { t.SkipNow() } } func RequiresFUSE(t *testing.T) { // Skip if FUSE tests are explicitly disabled if os.Getenv("TEST_FUSE") == "0" { t.Skip("FUSE tests disabled via TEST_FUSE=0") } // If TEST_FUSE=1 is set, always run (for backwards compatibility) if os.Getenv("TEST_FUSE") == "1" { return } // Auto-detect FUSE availability based on platform and tools if !isFUSEAvailable(t) { t.Skip("FUSE not available (no fusermount/umount found or unsupported platform)") } } // isFUSEAvailable checks if FUSE is available on the current system func isFUSEAvailable(t *testing.T) bool { t.Helper() // Check platform support switch runtime.GOOS { case "linux", "darwin", "freebsd", "openbsd", "netbsd": // These platforms potentially support FUSE case "windows": // Windows has limited FUSE support via WinFsp, but skip for now return false default: // Unknown platform, assume no FUSE support return false } // Check for required unmount tools var unmountCmd string if runtime.GOOS == "linux" { unmountCmd = "fusermount" } else { unmountCmd = "umount" } _, err := exec.LookPath(unmountCmd) return err == nil } func RequiresExpensive(t *testing.T) { if os.Getenv("TEST_EXPENSIVE") == "1" || testing.Short() { t.SkipNow() } } func RequiresPlugins(t *testing.T) { if os.Getenv("TEST_PLUGIN") != "1" { t.SkipNow() } } func RequiresLinux(t *testing.T) { if runtime.GOOS != "linux" { t.SkipNow() } } ================================================ FILE: test/cli/testutils/strings.go ================================================ package testutils import ( "bufio" "fmt" "net" "net/netip" "net/url" "strings" "sync" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) var ( AlphabetEasy = []rune("abcdefghijklmnopqrstuvwxyz01234567890-_") AlphabetHard = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890!@#$%^&*()-_+= ;.,<>'\"[]{}() ") ) // StrCat takes a bunch of strings or string slices // and concats them all together into one string slice. // If an arg is not one of those types, this panics. // If an arg is an empty string, it is dropped. func StrCat(args ...any) []string { res := make([]string, 0) for _, a := range args { if s, ok := a.(string); ok { if s != "" { res = append(res, s) } continue } if ss, ok := a.([]string); ok { for _, s := range ss { if s != "" { res = append(res, s) } } continue } panic(fmt.Sprintf("arg '%v' must be a string or string slice, but is '%T'", a, a)) } return res } // PreviewStr returns a preview of s, which is a prefix for logging that avoids dumping a huge string to logs. func PreviewStr(s string) string { suffix := "..." previewLength := 10 if len(s) < previewLength { previewLength = len(s) suffix = "" } return s[0:previewLength] + suffix } func SplitLines(s string) []string { var lines []string scanner := bufio.NewScanner(strings.NewReader(s)) for scanner.Scan() { lines = append(lines, scanner.Text()) } return lines } // URLStrToMultiaddr converts a URL string like http://localhost:80 to a multiaddr. func URLStrToMultiaddr(u string) multiaddr.Multiaddr { parsedURL, err := url.Parse(u) if err != nil { panic(err) } addrPort, err := netip.ParseAddrPort(parsedURL.Host) if err != nil { panic(err) } tcpAddr := net.TCPAddrFromAddrPort(addrPort) ma, err := manet.FromNetAddr(tcpAddr) if err != nil { panic(err) } return ma } // ForEachPar invokes f in a new goroutine for each element of s and waits for all to complete. func ForEachPar[T any](s []T, f func(T)) { wg := sync.WaitGroup{} wg.Add(len(s)) for _, x := range s { go func(x T) { defer wg.Done() f(x) }(x) } wg.Wait() } ================================================ FILE: test/cli/tracing_test.go ================================================ package cli import ( "fmt" "os" "os/exec" "path/filepath" "strings" "testing" "time" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) var otelCollectorConfigYAML = ` receivers: otlp: protocols: grpc: processors: batch: exporters: file: path: /traces/traces.json service: pipelines: traces: receivers: [otlp] processors: [batch] exporters: [file] ` func TestTracing(t *testing.T) { testutils.RequiresDocker(t) t.Parallel() node := harness.NewT(t).NewNode().Init() node.WriteBytes("collector-config.yaml", []byte(otelCollectorConfigYAML)) // touch traces.json and give it 777 perms in case Docker runs as a different user node.WriteBytes("traces.json", nil) err := os.Chmod(filepath.Join(node.Dir, "traces.json"), 0o777) require.NoError(t, err) dockerBin, err := exec.LookPath("docker") require.NoError(t, err) node.Runner.MustRun(harness.RunRequest{ Path: dockerBin, Args: []string{ "run", "--rm", "--detach", "--volume", fmt.Sprintf("%s:/config.yaml", filepath.Join(node.Dir, "collector-config.yaml")), "--volume", fmt.Sprintf("%s:/traces", node.Dir), "--net", "host", "--name", "ipfs-test-otel-collector", "otel/opentelemetry-collector-contrib:0.52.0", "--config", "/config.yaml", }, }) t.Cleanup(func() { node.Runner.MustRun(harness.RunRequest{ Path: dockerBin, Args: []string{"stop", "ipfs-test-otel-collector"}, }) }) node.Runner.Env["OTEL_TRACES_EXPORTER"] = "otlp" node.Runner.Env["OTEL_EXPORTER_OTLP_PROTOCOL"] = "grpc" node.Runner.Env["OTEL_EXPORTER_OTLP_ENDPOINT"] = "http://localhost:4317" node.StartDaemon() defer node.StopDaemon() assert.Eventually(t, func() bool { b, err := os.ReadFile(filepath.Join(node.Dir, "traces.json")) require.NoError(t, err) return strings.Contains(string(b), "go-ipfs") }, 5*time.Minute, 10*time.Millisecond, ) } ================================================ FILE: test/cli/transports_test.go ================================================ package cli import ( "fmt" "os" "path/filepath" "testing" "github.com/ipfs/go-test/random" "github.com/ipfs/go-test/random/files" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestTransports(t *testing.T) { disableRouting := func(nodes harness.Nodes) { nodes.ForEachPar(func(n *harness.Node) { n.UpdateConfig(func(cfg *config.Config) { cfg.Routing.Type = config.NewOptionalString("none") cfg.Bootstrap = nil }) }) } checkSingleFile := func(nodes harness.Nodes) { s := string(random.Bytes(100)) hash := nodes[0].IPFSAddStr(s) nodes.ForEachPar(func(n *harness.Node) { val := n.IPFS("cat", hash).Stdout.String() assert.Equal(t, s, val) }) } checkRandomDir := func(nodes harness.Nodes) { randDir := filepath.Join(nodes[0].Dir, "foobar") require.NoError(t, os.Mkdir(randDir, 0o777)) rfCfg := files.DefaultConfig() rfCfg.Dirs = 3 rfCfg.Files = 6 rfCfg.Depth = 4 require.NoError(t, files.Create(rfCfg, randDir)) hash := nodes[1].IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() nodes.ForEachPar(func(n *harness.Node) { res := n.RunIPFS("refs", "-r", hash) assert.Equal(t, 0, res.ExitCode()) }) } runTests := func(nodes harness.Nodes) { checkSingleFile(nodes) checkRandomDir(nodes) } tcpNodes := func(t *testing.T) harness.Nodes { nodes := harness.NewT(t).NewNodes(2).Init() nodes.ForEachPar(func(n *harness.Node) { n.UpdateConfig(func(cfg *config.Config) { cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/tcp/0"} cfg.Swarm.Transports.Network.QUIC = config.False cfg.Swarm.Transports.Network.Relay = config.False cfg.Swarm.Transports.Network.WebTransport = config.False cfg.Swarm.Transports.Network.WebRTCDirect = config.False cfg.Swarm.Transports.Network.Websocket = config.False // Disable AutoTLS since we're disabling WebSocket transport cfg.AutoTLS.Enabled = config.False }) }) disableRouting(nodes) return nodes } t.Run("tcp", func(t *testing.T) { t.Parallel() nodes := tcpNodes(t).StartDaemons().Connect() runTests(nodes) nodes.StopDaemons() }) t.Run("tcp with NOISE", func(t *testing.T) { t.Parallel() nodes := tcpNodes(t) nodes.ForEachPar(func(n *harness.Node) { n.UpdateConfig(func(cfg *config.Config) { cfg.Swarm.Transports.Security.TLS = config.Disabled }) }) nodes.StartDaemons().Connect() runTests(nodes) nodes.StopDaemons() }) t.Run("QUIC", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(5).Init() nodes.ForEachPar(func(n *harness.Node) { n.UpdateConfig(func(cfg *config.Config) { cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/udp/0/quic-v1"} cfg.Swarm.Transports.Network.TCP = config.False cfg.Swarm.Transports.Network.QUIC = config.True cfg.Swarm.Transports.Network.WebTransport = config.False cfg.Swarm.Transports.Network.WebRTCDirect = config.False cfg.Swarm.Transports.Network.Websocket = config.False }) }) disableRouting(nodes) nodes.StartDaemons().Connect() runTests(nodes) nodes.StopDaemons() }) t.Run("QUIC+Webtransport", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(5).Init() nodes.ForEachPar(func(n *harness.Node) { n.UpdateConfig(func(cfg *config.Config) { cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/udp/0/quic-v1/webtransport"} cfg.Swarm.Transports.Network.TCP = config.False cfg.Swarm.Transports.Network.QUIC = config.True cfg.Swarm.Transports.Network.WebTransport = config.True cfg.Swarm.Transports.Network.WebRTCDirect = config.False cfg.Swarm.Transports.Network.Websocket = config.False }) }) disableRouting(nodes) nodes.StartDaemons().Connect() runTests(nodes) nodes.StopDaemons() }) t.Run("QUIC connects with non-dialable transports", func(t *testing.T) { // This test targets specific Kubo internals which may change later. This checks // if we can announce an address we do not listen on, and then are able to connect // via a different address that is available. t.Parallel() nodes := harness.NewT(t).NewNodes(5).Init() nodes.ForEachPar(func(n *harness.Node) { n.UpdateConfig(func(cfg *config.Config) { // We need a specific port to announce so we first generate a random port. // We can't use 0 here to automatically assign an available port because // that would only work with Swarm, but not for the announcing. port := harness.NewRandPort() quicAddr := fmt.Sprintf("/ip4/127.0.0.1/udp/%d/quic-v1", port) cfg.Addresses.Swarm = []string{quicAddr} cfg.Addresses.Announce = []string{quicAddr, quicAddr + "/webtransport"} }) }) disableRouting(nodes) nodes.StartDaemons().Connect() runTests(nodes) nodes.StopDaemons() }) t.Run("WebRTC Direct", func(t *testing.T) { t.Parallel() nodes := harness.NewT(t).NewNodes(5).Init() nodes.ForEachPar(func(n *harness.Node) { n.UpdateConfig(func(cfg *config.Config) { cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/udp/0/webrtc-direct"} cfg.Swarm.Transports.Network.TCP = config.False cfg.Swarm.Transports.Network.QUIC = config.False cfg.Swarm.Transports.Network.WebTransport = config.False cfg.Swarm.Transports.Network.WebRTCDirect = config.True cfg.Swarm.Transports.Network.Websocket = config.False }) }) disableRouting(nodes) nodes.StartDaemons().Connect() runTests(nodes) nodes.StopDaemons() }) } ================================================ FILE: test/cli/webui_test.go ================================================ package cli import ( "net/http" "testing" "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" ) func TestWebUI(t *testing.T) { t.Parallel() t.Run("NoFetch=true shows not available error", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Gateway.NoFetch = true }) node.StartDaemon("--offline") apiClient := node.APIClient() resp := apiClient.Get("/webui/") // Should return 503 Service Unavailable when WebUI is not in local store assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode) // Check response contains helpful information body := resp.Body assert.Contains(t, body, "IPFS WebUI Not Available") assert.Contains(t, body, "Gateway.NoFetch=true") assert.Contains(t, body, "ipfs pin add") assert.Contains(t, body, "ipfs dag import") assert.Contains(t, body, "https://github.com/ipfs/ipfs-webui/releases") }) t.Run("DeserializedResponses=false shows incompatible error", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Gateway.DeserializedResponses = config.False }) node.StartDaemon() apiClient := node.APIClient() resp := apiClient.Get("/webui/") // Should return 503 Service Unavailable assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode) // Check response contains incompatibility message body := resp.Body assert.Contains(t, body, "IPFS WebUI Incompatible") assert.Contains(t, body, "Gateway.DeserializedResponses=false") assert.Contains(t, body, "WebUI requires deserializing IPFS responses") assert.Contains(t, body, "Gateway.DeserializedResponses=true") }) t.Run("Both NoFetch=true and DeserializedResponses=false shows incompatible error", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init() node.UpdateConfig(func(cfg *config.Config) { cfg.Gateway.NoFetch = true cfg.Gateway.DeserializedResponses = config.False }) node.StartDaemon("--offline") apiClient := node.APIClient() resp := apiClient.Get("/webui/") // Should return 503 Service Unavailable assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode) // DeserializedResponses=false takes priority body := resp.Body assert.Contains(t, body, "IPFS WebUI Incompatible") assert.Contains(t, body, "Gateway.DeserializedResponses=false") // Should NOT mention NoFetch since DeserializedResponses check comes first assert.NotContains(t, body, "NoFetch") }) } ================================================ FILE: test/dependencies/GNUmakefile ================================================ all: restore restore: @echo "*** $@ ***" which godep mkdir -p tmp_gopath OLD_GOPATH="$$GOPATH" export GOPATH=$$(pwd)/tmp_gopath cd ../.. godep restore cd - rm -rf tmp_gopath export GOPATH="$$OLD_GOPATH" .PHONY: all restore ================================================ FILE: test/dependencies/dependencies.go ================================================ //go:build tools package tools import ( _ "github.com/Kubuxu/gocovmerge" _ "github.com/golangci/golangci-lint/cmd/golangci-lint" _ "github.com/ipfs/go-cidutil/cid-fmt" _ "github.com/ipfs/go-test/cli/random-data" _ "github.com/ipfs/go-test/cli/random-files" _ "github.com/ipfs/hang-fds" _ "github.com/multiformats/go-multihash/multihash" _ "gotest.tools/gotestsum" ) ================================================ FILE: test/dependencies/go.mod ================================================ module github.com/ipfs/kubo/test/dependencies go 1.25.7 replace github.com/ipfs/kubo => ../../ require ( github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd github.com/golangci/golangci-lint v1.64.8 github.com/ipfs/go-cidutil v0.1.1 github.com/ipfs/go-log/v2 v2.9.1 github.com/ipfs/go-test v0.2.3 github.com/ipfs/hang-fds v0.1.0 github.com/ipfs/iptb v1.4.1 github.com/ipfs/iptb-plugins v0.5.1 github.com/multiformats/go-multiaddr v0.16.1 github.com/multiformats/go-multihash v0.2.3 gotest.tools/gotestsum v1.13.0 ) require ( 4d63.com/gocheckcompilerdirectives v1.3.0 // indirect 4d63.com/gochecknoglobals v0.2.2 // indirect github.com/4meepo/tagalign v1.4.2 // indirect github.com/Abirdcfly/dupword v0.1.3 // indirect github.com/Antonboom/errname v1.0.0 // indirect github.com/Antonboom/nilnil v1.0.1 // indirect github.com/Antonboom/testifylint v1.5.2 // indirect github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect github.com/Crocmagnon/fatcontext v0.7.1 // indirect github.com/DataDog/zstd v1.5.7 // indirect github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.1 // indirect github.com/Jorropo/jsync v1.0.1 // indirect github.com/Masterminds/semver/v3 v3.3.0 // indirect github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect github.com/RaduBerinde/axisds v0.1.0 // indirect github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect github.com/alecthomas/go-check-sumtype v0.3.1 // indirect github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/alexkohler/nakedret/v2 v2.0.5 // indirect github.com/alexkohler/prealloc v1.0.0 // indirect github.com/alingse/asasalint v0.0.11 // indirect github.com/alingse/nilnesserr v0.1.2 // indirect github.com/ashanbrown/forbidigo v1.6.0 // indirect github.com/ashanbrown/makezero v1.2.0 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bitfield/gotestdox v0.2.2 // indirect github.com/bkielbasa/cyclop v1.2.3 // indirect github.com/blizzy78/varnamelen v0.8.0 // indirect github.com/bombsimon/wsl/v4 v4.5.0 // indirect github.com/breml/bidichk v0.3.2 // indirect github.com/breml/errchkjson v0.4.0 // indirect github.com/butuzov/ireturn v0.3.1 // indirect github.com/butuzov/mirror v1.3.0 // indirect github.com/caddyserver/certmagic v0.23.0 // indirect github.com/caddyserver/zerossl v0.1.3 // indirect github.com/catenacyber/perfsprint v0.8.2 // indirect github.com/ccojocar/zxcvbn-go v1.0.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charithe/durationcheck v0.0.10 // indirect github.com/chavacava/garif v0.1.0 // indirect github.com/ckaznocha/intrange v0.3.0 // indirect github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect github.com/cockroachdb/errors v1.11.3 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/pebble/v2 v2.1.4 // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf // indirect github.com/curioswitch/go-reassign v0.3.0 // indirect github.com/daixiang0/gci v0.13.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/dnephin/pflag v1.0.7 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ettle/strcase v0.2.0 // indirect github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect github.com/fatih/color v1.18.0 // indirect github.com/fatih/structtag v1.2.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/filecoin-project/go-clock v0.1.0 // indirect github.com/firefart/nonamedreturns v1.0.5 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fzipp/gocyclo v0.6.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gammazero/chanqueue v1.1.2 // indirect github.com/gammazero/deque v1.2.1 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect github.com/ghostiam/protogetter v0.3.9 // indirect github.com/go-critic/go-critic v0.12.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-toolsmith/astcast v1.1.0 // indirect github.com/go-toolsmith/astcopy v1.1.0 // indirect github.com/go-toolsmith/astequal v1.2.0 // indirect github.com/go-toolsmith/astfmt v1.1.0 // indirect github.com/go-toolsmith/astp v1.1.0 // indirect github.com/go-toolsmith/strparse v1.1.0 // indirect github.com/go-toolsmith/typep v1.1.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/go-xmlfmt/xmlfmt v1.1.3 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gofrs/flock v0.12.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 // indirect github.com/golangci/go-printf-func-name v0.1.0 // indirect github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d // indirect github.com/golangci/misspell v0.6.0 // indirect github.com/golangci/plugin-module-register v0.1.1 // indirect github.com/golangci/revgrep v0.8.0 // indirect github.com/golangci/unconvert v0.0.0-20240309020433-c5143eacb3ed // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gordonklaus/ineffassign v0.1.0 // indirect github.com/gostaticanalysis/analysisutil v0.7.1 // indirect github.com/gostaticanalysis/comment v1.5.0 // indirect github.com/gostaticanalysis/forcetypeassert v0.2.0 // indirect github.com/gostaticanalysis/nilerr v0.1.1 // indirect github.com/hashicorp/go-immutable-radix/v2 v2.1.0 // indirect github.com/hashicorp/go-version v1.8.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/boxo v0.37.1-0.20260317235537-851246983422 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect github.com/ipfs/go-datastore v0.9.1 // indirect github.com/ipfs/go-dsqueue v0.2.0 // indirect github.com/ipfs/go-ipfs-cmds v0.16.0 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect github.com/ipfs/go-ipld-cbor v0.2.1 // indirect github.com/ipfs/go-ipld-format v0.6.3 // indirect github.com/ipfs/go-ipld-legacy v0.3.0 // indirect github.com/ipfs/go-metrics-interface v0.3.0 // indirect github.com/ipfs/go-unixfsnode v1.10.3 // indirect github.com/ipfs/kubo v0.31.0 // indirect github.com/ipld/go-car/v2 v2.16.0 // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect github.com/ipld/go-ipld-prime v0.22.0 // indirect github.com/ipshipyard/p2p-forge v0.7.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jgautheron/goconst v1.7.1 // indirect github.com/jingyugao/rowserrcheck v1.1.1 // indirect github.com/jjti/go-spancheck v0.6.4 // indirect github.com/julz/importas v0.2.0 // indirect github.com/karamaru-alpha/copyloopvar v1.2.1 // indirect github.com/kisielk/errcheck v1.9.0 // indirect github.com/kkHAIKE/contextcheck v1.1.6 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/koron/go-ssdp v0.0.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kulti/thelper v0.6.3 // indirect github.com/kunwardeep/paralleltest v1.0.10 // indirect github.com/lasiar/canonicalheader v1.1.2 // indirect github.com/ldez/exptostd v0.4.2 // indirect github.com/ldez/gomoddirectives v0.6.1 // indirect github.com/ldez/grignotin v0.9.0 // indirect github.com/ldez/tagliatelle v0.7.1 // indirect github.com/ldez/usetesting v0.4.2 // indirect github.com/leonklingele/grouper v1.1.2 // indirect github.com/libdns/libdns v1.0.0-beta.1 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-doh-resolver v0.5.0 // indirect github.com/libp2p/go-flow-metrics v0.3.0 // indirect github.com/libp2p/go-libp2p v0.48.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect github.com/libp2p/go-libp2p-kad-dht v0.39.0 // indirect github.com/libp2p/go-libp2p-kbucket v0.8.0 // indirect github.com/libp2p/go-libp2p-record v0.3.1 // indirect github.com/libp2p/go-libp2p-routing-helpers v0.7.5 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-netroute v0.4.0 // indirect github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/macabu/inamedparam v0.1.3 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/maratori/testableexamples v1.0.0 // indirect github.com/maratori/testpackage v1.1.1 // indirect github.com/matoous/godox v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mattn/go-shellwords v1.0.12 // indirect github.com/mgechev/revive v1.7.0 // indirect github.com/mholt/acmez/v3 v3.1.2 // indirect github.com/miekg/dns v1.1.72 // indirect github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moricho/tparallel v0.3.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr-dns v0.5.0 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multicodec v0.10.0 // indirect github.com/multiformats/go-multistream v0.6.1 // indirect github.com/multiformats/go-varint v0.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nakabonne/nestif v0.3.1 // indirect github.com/nishanths/exhaustive v0.12.0 // indirect github.com/nishanths/predeclared v0.2.2 // indirect github.com/nunnatsa/ginkgolinter v0.19.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pion/datachannel v1.5.10 // indirect github.com/pion/dtls/v3 v3.1.2 // indirect github.com/pion/ice/v4 v4.0.10 // indirect github.com/pion/interceptor v0.1.40 // indirect github.com/pion/logging v0.2.4 // indirect github.com/pion/mdns/v2 v2.0.7 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/rtcp v1.2.16 // indirect github.com/pion/rtp v1.8.19 // indirect github.com/pion/sctp v1.8.39 // indirect github.com/pion/sdp/v3 v3.0.18 // indirect github.com/pion/srtp/v3 v3.0.6 // indirect github.com/pion/stun/v3 v3.1.1 // indirect github.com/pion/transport/v3 v3.0.7 // indirect github.com/pion/transport/v4 v4.0.1 // indirect github.com/pion/turn/v4 v4.0.2 // indirect github.com/pion/webrtc/v4 v4.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect github.com/polyfloyd/go-errorlint v1.7.1 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.20.1 // indirect github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1 // indirect github.com/quasilyte/go-ruleguard/dsl v0.3.22 // indirect github.com/quasilyte/gogrep v0.5.0 // indirect github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect github.com/quic-go/quic-go v0.59.0 // indirect github.com/raeperd/recvcheck v0.2.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryancurrah/gomodguard v1.3.5 // indirect github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sanposhiho/wastedassign/v2 v2.1.0 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect github.com/sashamelentyev/interfacebloat v1.1.0 // indirect github.com/sashamelentyev/usestdlibvars v1.28.0 // indirect github.com/securego/gosec/v2 v2.22.2 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sivchari/containedctx v1.0.3 // indirect github.com/sivchari/tenv v1.12.1 // indirect github.com/sonatard/noctx v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/sourcegraph/go-diff v0.7.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.12.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cobra v1.9.1 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/spf13/viper v1.19.0 // indirect github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect github.com/stbenjam/no-sprintf-host-port v0.2.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/stretchr/testify v1.11.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tdakkota/asciicheck v0.4.1 // indirect github.com/tetafro/godot v1.5.0 // indirect github.com/timakin/bodyclose v0.0.0-20241017074812-ed6a65f985e3 // indirect github.com/timonwong/loggercheck v0.10.1 // indirect github.com/tomarrell/wrapcheck/v2 v2.10.0 // indirect github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/ultraware/funlen v0.2.0 // indirect github.com/ultraware/whitespace v0.2.0 // indirect github.com/urfave/cli v1.22.16 // indirect github.com/uudashr/gocognit v1.2.0 // indirect github.com/uudashr/iface v1.3.1 // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.3.1 // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/wlynxg/anet v0.0.5 // indirect github.com/xen0n/gosmopolitan v1.2.2 // indirect github.com/yagipy/maintidx v1.0.0 // indirect github.com/yeya24/promlinter v0.3.0 // indirect github.com/ykadowak/zerologlint v0.1.5 // indirect github.com/zeebo/blake3 v0.2.4 // indirect gitlab.com/bosi/decorder v0.4.2 // indirect go-simpler.org/musttag v0.13.0 // indirect go-simpler.org/sloglint v0.9.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect go.opentelemetry.io/otel v1.42.0 // indirect go.opentelemetry.io/otel/metric v1.42.0 // indirect go.opentelemetry.io/otel/trace v1.42.0 // indirect go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/dig v1.19.0 // indirect go.uber.org/fx v1.24.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.uber.org/zap/exp v0.3.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 // indirect golang.org/x/exp/typeparams v0.0.0-20250210185358-939b2ce775ac // indirect golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/term v0.41.0 // indirect golang.org/x/text v0.35.0 // indirect golang.org/x/time v0.12.0 // indirect golang.org/x/tools v0.43.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gonum.org/v1/gonum v0.17.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/tools v0.6.1 // indirect lukechampine.com/blake3 v1.4.1 // indirect mvdan.cc/gofumpt v0.7.0 // indirect mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f // indirect ) ================================================ FILE: test/dependencies/go.sum ================================================ 4d63.com/gocheckcompilerdirectives v1.3.0 h1:Ew5y5CtcAAQeTVKUVFrE7EwHMrTO6BggtEj8BZSjZ3A= 4d63.com/gocheckcompilerdirectives v1.3.0/go.mod h1:ofsJ4zx2QAuIP/NO/NAh1ig6R1Fb18/GI7RVMwz7kAY= 4d63.com/gochecknoglobals v0.2.2 h1:H1vdnwnMaZdQW/N+NrkT1SZMTBmcwHe9Vq8lJcYYTtU= 4d63.com/gochecknoglobals v0.2.2/go.mod h1:lLxwTQjL5eIesRbvnzIP3jZtG140FnTdz+AlMa+ogt0= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5 h1:JA0fFr+kxpqTdxR9LOBiTWpGNchqmkcsgmdeJZRclZ0= filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI= filippo.io/keygen v0.0.0-20260114151900-8e2790ea4c5b h1:REI1FbdW71yO56Are4XAxD+OS/e+BQsB3gE4mZRQEXY= filippo.io/keygen v0.0.0-20260114151900-8e2790ea4c5b/go.mod h1:9nnw1SlYHYuPSo/3wjQzNjSbeHlq2NsKo5iEtfJPWP0= github.com/4meepo/tagalign v1.4.2 h1:0hcLHPGMjDyM1gHG58cS73aQF8J4TdVR96TZViorO9E= github.com/4meepo/tagalign v1.4.2/go.mod h1:+p4aMyFM+ra7nb41CnFG6aSDXqRxU/w1VQqScKqDARI= github.com/Abirdcfly/dupword v0.1.3 h1:9Pa1NuAsZvpFPi9Pqkd93I7LIYRURj+A//dFd5tgBeE= github.com/Abirdcfly/dupword v0.1.3/go.mod h1:8VbB2t7e10KRNdwTVoxdBaxla6avbhGzb8sCTygUMhw= github.com/Antonboom/errname v1.0.0 h1:oJOOWR07vS1kRusl6YRSlat7HFnb3mSfMl6sDMRoTBA= github.com/Antonboom/errname v1.0.0/go.mod h1:gMOBFzK/vrTiXN9Oh+HFs+e6Ndl0eTFbtsRTSRdXyGI= github.com/Antonboom/nilnil v1.0.1 h1:C3Tkm0KUxgfO4Duk3PM+ztPncTFlOf0b2qadmS0s4xs= github.com/Antonboom/nilnil v1.0.1/go.mod h1:CH7pW2JsRNFgEh8B2UaPZTEPhCMuFowP/e8Udp9Nnb0= github.com/Antonboom/testifylint v1.5.2 h1:4s3Xhuv5AvdIgbd8wOOEeo0uZG7PbDKQyKY5lGoQazk= github.com/Antonboom/testifylint v1.5.2/go.mod h1:vxy8VJ0bc6NavlYqjZfmp6EfqXMtBgQ4+mhCojwC1P8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs= github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Crocmagnon/fatcontext v0.7.1 h1:SC/VIbRRZQeQWj/TcQBS6JmrXcfA+BU4OGSVUt54PjM= github.com/Crocmagnon/fatcontext v0.7.1/go.mod h1:1wMvv3NXEBJucFGfwOJBxSVWcoIO6emV215SMkW9MFU= github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 h1:sHglBQTwgx+rWPdisA5ynNEsoARbiCBOyGcJM4/OzsM= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.1 h1:Sz1JIXEcSfhz7fUi7xHnhpIE0thVASYjvosApmHuD2k= github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.1/go.mod h1:n/LSCXNuIYqVfBlVXyHfMQkZDdp1/mmxfSjADd3z1Zg= github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd h1:HNhzThEtZW714v8Eda8sWWRcu9WSzJC+oCyjRjvZgRA= github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd/go.mod h1:bqoB8kInrTeEtYAwaIXoSRqdwnjQmFhsfusnzyui6yY= github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= github.com/RaduBerinde/axisds v0.1.0 h1:YItk/RmU5nvlsv/awo2Fjx97Mfpt4JfgtEVAGPrLdz8= github.com/RaduBerinde/axisds v0.1.0/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y= github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk= github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/go-check-sumtype v0.3.1 h1:u9aUvbGINJxLVXiFvHUlPEaD7VDULsrxJb4Aq31NLkU= github.com/alecthomas/go-check-sumtype v0.3.1/go.mod h1:A8TSiN3UPRw3laIgWEUOHHLPa6/r9MtoigdlP5h3K/E= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/alexkohler/nakedret/v2 v2.0.5 h1:fP5qLgtwbx9EJE8dGEERT02YwS8En4r9nnZ71RK+EVU= github.com/alexkohler/nakedret/v2 v2.0.5/go.mod h1:bF5i0zF2Wo2o4X4USt9ntUWve6JbFv02Ff4vlkmS/VU= github.com/alexkohler/prealloc v1.0.0 h1:Hbq0/3fJPQhNkN0dR95AVrr6R7tou91y0uHG5pOcUuw= github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE= github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQRnw= github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= github.com/alingse/nilnesserr v0.1.2 h1:Yf8Iwm3z2hUUrP4muWfW83DF4nE3r1xZ26fGWUKCZlo= github.com/alingse/nilnesserr v0.1.2/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/ashanbrown/forbidigo v1.6.0 h1:D3aewfM37Yb3pxHujIPSpTf6oQk9sc9WZi8gerOIVIY= github.com/ashanbrown/forbidigo v1.6.0/go.mod h1:Y8j9jy9ZYAEHXdu723cUlraTqbzjKF1MUyfOKL+AjcU= github.com/ashanbrown/makezero v1.2.0 h1:/2Lp1bypdmK9wDIq7uWBlDF1iMUpIIS4A+pF6C9IEUU= github.com/ashanbrown/makezero v1.2.0/go.mod h1:dxlPhHbDMC6N6xICzFBSK+4njQDdK8euNO0qjQMtGY4= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE= github.com/bitfield/gotestdox v0.2.2/go.mod h1:D+gwtS0urjBrzguAkTM2wodsTQYFHdpx8eqRJ3N+9pY= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bkielbasa/cyclop v1.2.3 h1:faIVMIGDIANuGPWH031CZJTi2ymOQBULs9H21HSMa5w= github.com/bkielbasa/cyclop v1.2.3/go.mod h1:kHTwA9Q0uZqOADdupvcFJQtp/ksSnytRMe8ztxG8Fuo= github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M= github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k= github.com/bombsimon/wsl/v4 v4.5.0 h1:iZRsEvDdyhd2La0FVi5k6tYehpOR/R7qIUjmKk7N74A= github.com/bombsimon/wsl/v4 v4.5.0/go.mod h1:NOQ3aLF4nD7N5YPXMruR6ZXDOAqLoM0GEpLwTdvmOSc= github.com/breml/bidichk v0.3.2 h1:xV4flJ9V5xWTqxL+/PMFF6dtJPvZLPsyixAoPe8BGJs= github.com/breml/bidichk v0.3.2/go.mod h1:VzFLBxuYtT23z5+iVkamXO386OB+/sVwZOpIj6zXGos= github.com/breml/errchkjson v0.4.0 h1:gftf6uWZMtIa/Is3XJgibewBm2ksAQSY/kABDNFTAdk= github.com/breml/errchkjson v0.4.0/go.mod h1:AuBOSTHyLSaaAFlWsRSuRBIroCh3eh7ZHh5YeelDIk8= github.com/butuzov/ireturn v0.3.1 h1:mFgbEI6m+9W8oP/oDdfA34dLisRFCj2G6o/yiI1yZrY= github.com/butuzov/ireturn v0.3.1/go.mod h1:ZfRp+E7eJLC0NQmk1Nrm1LOrn/gQlOykv+cVPdiXH5M= github.com/butuzov/mirror v1.3.0 h1:HdWCXzmwlQHdVhwvsfBb2Au0r3HyINry3bDWLYXiKoc= github.com/butuzov/mirror v1.3.0/go.mod h1:AEij0Z8YMALaq4yQj9CPPVYOyJQyiexpQEQgihajRfI= github.com/caddyserver/certmagic v0.23.0 h1:CfpZ/50jMfG4+1J/u2LV6piJq4HOfO6ppOnOf7DkFEU= github.com/caddyserver/certmagic v0.23.0/go.mod h1:9mEZIWqqWoI+Gf+4Trh04MOVPD0tGSxtqsxg87hAIH4= github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA= github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4= github.com/catenacyber/perfsprint v0.8.2 h1:+o9zVmCSVa7M4MvabsWvESEhpsMkhfE7k0sHNGL95yw= github.com/catenacyber/perfsprint v0.8.2/go.mod h1:q//VWC2fWbcdSLEY1R3l8n0zQCDPdE4IjZwyY1HMunM= github.com/ccojocar/zxcvbn-go v1.0.2 h1:na/czXU8RrhXO4EZme6eQJLR4PzcGsahsBOAwU6I3Vg= github.com/ccojocar/zxcvbn-go v1.0.2/go.mod h1:g1qkXtUSvHP8lhHp5GrSmTz6uWALGRMQdw6Qnz/hi60= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charithe/durationcheck v0.0.10 h1:wgw73BiocdBDQPik+zcEoBG/ob8uyBHf2iyoHGPf5w4= github.com/charithe/durationcheck v0.0.10/go.mod h1:bCWXb7gYRysD1CU3C+u4ceO49LoGOY1C1L6uouGNreQ= github.com/chavacava/garif v0.1.0 h1:2JHa3hbYf5D9dsgseMKAmc/MZ109otzgNFk5s87H9Pc= github.com/chavacava/garif v0.1.0/go.mod h1:XMyYCkEL58DF0oyW4qDjjnPWONs2HBqYKI+UIPD+Gww= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/ckaznocha/intrange v0.3.0 h1:VqnxtK32pxgkhJgYQEeOArVidIPg+ahLP7WBOXZd5ZY= github.com/ckaznocha/intrange v0.3.0/go.mod h1:+I/o2d2A1FBHgGELbGxzIcyd3/9l9DuwjM8FsbSS3Lo= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= github.com/cockroachdb/pebble/v2 v2.1.4 h1:j9wPgMDbkErFdAKYFGhsoCcvzcjR+6zrJ4jhKtJ6bOk= github.com/cockroachdb/pebble/v2 v2.1.4/go.mod h1:Reo1RTniv1UjVTAu/Fv74y5i3kJ5gmVrPhO9UtFiKn8= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b h1:VXvSNzmr8hMj8XTuY0PT9Ane9qZGul/p67vGYwl9BFI= github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf h1:dwGgBWn84wUS1pVikGiruW+x5XM4amhjaZO20vCjay4= github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/curioswitch/go-reassign v0.3.0 h1:dh3kpQHuADL3cobV/sSGETA8DOv457dwl+fbBAhrQPs= github.com/curioswitch/go-reassign v0.3.0/go.mod h1:nApPCCTtqLJN/s8HfItCcKV0jIPwluBOvZP+dsJGA88= github.com/daixiang0/gci v0.13.5 h1:kThgmH1yBmZSBCh1EJVxQ7JsHpm5Oms0AMed/0LaH4c= github.com/daixiang0/gci v0.13.5/go.mod h1:12etP2OniiIdP4q+kjUGrC/rUagga7ODbqsom5Eo5Yk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8= github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= github.com/dunglas/httpsfv v1.1.0 h1:Jw76nAyKWKZKFrpMMcL76y35tOpYHqQPzHQiwDvpe54= github.com/dunglas/httpsfv v1.1.0/go.mod h1:zID2mqw9mFsnt7YC3vYQ9/cjq30q41W+1AnDwH8TiMg= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ettle/strcase v0.2.0 h1:fGNiVF21fHXpX1niBgk0aROov1LagYsOwV/xqKDKR/Q= github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs= github.com/firefart/nonamedreturns v1.0.5 h1:tM+Me2ZaXs8tfdDw3X6DOX++wMCOqzYUho6tUTYIdRA= github.com/firefart/nonamedreturns v1.0.5/go.mod h1:gHJjDqhGM4WyPt639SOZs+G89Ko7QKH5R5BhnO6xJhw= github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gammazero/chanqueue v1.1.2 h1:dZEsxlyANZMyeTRemABqZF8QM9BnE4NBI43Oh3y5fIU= github.com/gammazero/chanqueue v1.1.2/go.mod h1:XDN1X/jjAbmSceNFOQbtKToeSkxtdVdpKu90LiEdBEE= github.com/gammazero/deque v1.2.1 h1:9fnQVFCCZ9/NOc7ccTNqzoKd1tCWOqeI05/lPqFPMGQ= github.com/gammazero/deque v1.2.1/go.mod h1:5nSFkzVm+afG9+gy0VIowlqVAW4N8zNcMne+CMQVD2g= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 h1:r5GgOLGbza2wVHRzK7aAj6lWZjfbAwiu/RDCVOKjRyM= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghostiam/protogetter v0.3.9 h1:j+zlLLWzqLay22Cz/aYwTHKQ88GE2DQ6GkWSYFOI4lQ= github.com/ghostiam/protogetter v0.3.9/go.mod h1:WZ0nw9pfzsgxuRsPOFQomgDVSWtDLJRfQJEhsGbmQMA= github.com/go-critic/go-critic v0.12.0 h1:iLosHZuye812wnkEz1Xu3aBwn5ocCPfc9yqmFG9pa6w= github.com/go-critic/go-critic v0.12.0/go.mod h1:DpE0P6OVc6JzVYzmM5gq5jMU31zLr4am5mB/VfFK64w= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-toolsmith/astcast v1.1.0 h1:+JN9xZV1A+Re+95pgnMgDboWNVnIMMQXwfBwLRPgSC8= github.com/go-toolsmith/astcast v1.1.0/go.mod h1:qdcuFWeGGS2xX5bLM/c3U9lewg7+Zu4mr+xPwZIB4ZU= github.com/go-toolsmith/astcopy v1.1.0 h1:YGwBN0WM+ekI/6SS6+52zLDEf8Yvp3n2seZITCUBt5s= github.com/go-toolsmith/astcopy v1.1.0/go.mod h1:hXM6gan18VA1T/daUEHCFcYiW8Ai1tIwIzHY6srfEAw= github.com/go-toolsmith/astequal v1.0.3/go.mod h1:9Ai4UglvtR+4up+bAD4+hCj7iTo4m/OXVTSLnCyTAx4= github.com/go-toolsmith/astequal v1.1.0/go.mod h1:sedf7VIdCL22LD8qIvv7Nn9MuWJruQA/ysswh64lffQ= github.com/go-toolsmith/astequal v1.2.0 h1:3Fs3CYZ1k9Vo4FzFhwwewC3CHISHDnVUPC4x0bI2+Cw= github.com/go-toolsmith/astequal v1.2.0/go.mod h1:c8NZ3+kSFtFY/8lPso4v8LuJjdJiUFVnSuU3s0qrrDY= github.com/go-toolsmith/astfmt v1.1.0 h1:iJVPDPp6/7AaeLJEruMsBUlOYCmvg0MoCfJprsOmcco= github.com/go-toolsmith/astfmt v1.1.0/go.mod h1:OrcLlRwu0CuiIBp/8b5PYF9ktGVZUjlNMV634mhwuQ4= github.com/go-toolsmith/astp v1.1.0 h1:dXPuCl6u2llURjdPLLDxJeZInAeZ0/eZwFJmqZMnpQA= github.com/go-toolsmith/astp v1.1.0/go.mod h1:0T1xFGz9hicKs8Z5MfAqSUitoUYS30pDMsRVIDHs8CA= github.com/go-toolsmith/pkgload v1.2.2 h1:0CtmHq/02QhxcF7E9N5LIFcYFsMR5rdovfqTtRKkgIk= github.com/go-toolsmith/pkgload v1.2.2/go.mod h1:R2hxLNRKuAsiXCo2i5J6ZQPhnPMOVtU+f0arbFPWCus= github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQiyP2Bvw= github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ= github.com/go-toolsmith/typep v1.1.0 h1:fIRYDyF+JywLfqzyhdiHzRop/GQDxxNhLGQ6gFUNHus= github.com/go-toolsmith/typep v1.1.0/go.mod h1:fVIw+7zjdsMxDA3ITWnH1yOiw1rnTQKCsF/sk2H/qig= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/go-xmlfmt/xmlfmt v1.1.3 h1:t8Ey3Uy7jDSEisW2K3somuMKIpzktkWptA0iFCnRUWY= github.com/go-xmlfmt/xmlfmt v1.1.3/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e h1:4bw4WeyTYPp0smaXiJZCNnLrvVBqirQVreixayXezGc= github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 h1:WUvBfQL6EW/40l6OmeSBYQJNSif4O11+bmWEz+C7FYw= github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32/go.mod h1:NUw9Zr2Sy7+HxzdjIULge71wI6yEg1lWQr7Evcu8K0E= github.com/golangci/go-printf-func-name v0.1.0 h1:dVokQP+NMTO7jwO4bwsRwLWeudOVUPPyAKJuzv8pEJU= github.com/golangci/go-printf-func-name v0.1.0/go.mod h1:wqhWFH5mUdJQhweRnldEywnR5021wTdZSNgwYceV14s= github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d h1:viFft9sS/dxoYY0aiOTsLKO2aZQAPT4nlQCsimGcSGE= github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d/go.mod h1:ivJ9QDg0XucIkmwhzCDsqcnxxlDStoTl89jDMIoNxKY= github.com/golangci/golangci-lint v1.64.8 h1:y5TdeVidMtBGG32zgSC7ZXTFNHrsJkDnpO4ItB3Am+I= github.com/golangci/golangci-lint v1.64.8/go.mod h1:5cEsUQBSr6zi8XI8OjmcY2Xmliqc4iYL7YoPrL+zLJ4= github.com/golangci/misspell v0.6.0 h1:JCle2HUTNWirNlDIAUO44hUsKhOFqGPoC4LZxlaSXDs= github.com/golangci/misspell v0.6.0/go.mod h1:keMNyY6R9isGaSAu+4Q8NMBwMPkh15Gtc8UCVoDtAWo= github.com/golangci/plugin-module-register v0.1.1 h1:TCmesur25LnyJkpsVrupv1Cdzo+2f7zX0H6Jkw1Ol6c= github.com/golangci/plugin-module-register v0.1.1/go.mod h1:TTpqoB6KkwOJMV8u7+NyXMrkwwESJLOkfl9TxR1DGFc= github.com/golangci/revgrep v0.8.0 h1:EZBctwbVd0aMeRnNUsFogoyayvKHyxlV3CdUA46FX2s= github.com/golangci/revgrep v0.8.0/go.mod h1:U4R/s9dlXZsg8uJmaR1GrloUr14D7qDl8gi2iPXJH8k= github.com/golangci/unconvert v0.0.0-20240309020433-c5143eacb3ed h1:IURFTjxeTfNFP0hTEi1YKjB/ub8zkpaOqFFMApi2EAs= github.com/golangci/unconvert v0.0.0-20240309020433-c5143eacb3ed/go.mod h1:XLXN8bNw4CGRPaqgl3bv/lhz7bsGPh4/xSaMTbo2vkQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a h1://KbezygeMJZCSHH+HgUZiTeSoiuFspbMg1ge+eFj18= github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gordonklaus/ineffassign v0.1.0 h1:y2Gd/9I7MdY1oEIt+n+rowjBNDcLQq3RsH5hwJd0f9s= github.com/gordonklaus/ineffassign v0.1.0/go.mod h1:Qcp2HIAYhR7mNUVSIxZww3Guk4it82ghYcEXIAk+QT0= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gostaticanalysis/analysisutil v0.7.1 h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk= github.com/gostaticanalysis/analysisutil v0.7.1/go.mod h1:v21E3hY37WKMGSnbsw2S/ojApNWb6C1//mXO48CXbVc= github.com/gostaticanalysis/comment v1.4.1/go.mod h1:ih6ZxzTHLdadaiSnF5WY3dxUoXfXAlTaRzuaNDlSado= github.com/gostaticanalysis/comment v1.4.2/go.mod h1:KLUTGDv6HOCotCH8h2erHKmpci2ZoR8VPu34YA2uzdM= github.com/gostaticanalysis/comment v1.5.0 h1:X82FLl+TswsUMpMh17srGRuKaaXprTaytmEpgnKIDu8= github.com/gostaticanalysis/comment v1.5.0/go.mod h1:V6eb3gpCv9GNVqb6amXzEUX3jXLVK/AdA+IrAMSqvEc= github.com/gostaticanalysis/forcetypeassert v0.2.0 h1:uSnWrrUEYDr86OCxWa4/Tp2jeYDlogZiZHzGkWFefTk= github.com/gostaticanalysis/forcetypeassert v0.2.0/go.mod h1:M5iPavzE9pPqWyeiVXSFghQjljW1+l/Uke3PXHS6ILY= github.com/gostaticanalysis/nilerr v0.1.1 h1:ThE+hJP0fEp4zWLkWHWcRyI2Od0p7DlgYG3Uqrmrcpk= github.com/gostaticanalysis/nilerr v0.1.1/go.mod h1:wZYb6YI5YAxxq0i1+VJbY0s2YONW0HU0GPE3+5PWN4A= github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod h1:D+FIZ+7OahH3ePw/izIEeH5I06eKs1IKI4Xr64/Am3M= github.com/gostaticanalysis/testutil v0.5.0 h1:Dq4wT1DdTwTGCQQv3rl3IvD5Ld0E6HiY+3Zh0sUGqw8= github.com/gostaticanalysis/testutil v0.5.0/go.mod h1:OLQSbuM6zw2EvCcXTz1lVq5unyoNft372msDY0nY5Hs= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix/v2 v2.1.0 h1:CUW5RYIcysz+D3B+l1mDeXrQ7fUvGGCwJfdASSzbrfo= github.com/hashicorp/go-immutable-radix/v2 v2.1.0/go.mod h1:hgdqLXA4f6NIjRVisM1TJ9aOJVNRqKZj+xDGF6m7PBw= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.37.1-0.20260317235537-851246983422 h1:yY3ot/DU1bqTzHDBARACM76Tbx9s4xzcRbzifG1e/es= github.com/ipfs/boxo v0.37.1-0.20260317235537-851246983422/go.mod h1:8yyiRn54F2CsW13n0zwXEPrVsZix/gFj9SYIRYMZ6KE= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk= github.com/ipfs/go-block-format v0.2.3/go.mod h1:WJaQmPAKhD3LspLixqlqNFxiZ3BZ3xgqxxoSR/76pnA= github.com/ipfs/go-cid v0.6.0 h1:DlOReBV1xhHBhhfy/gBNNTSyfOM6rLiIx9J7A4DGf30= github.com/ipfs/go-cid v0.6.0/go.mod h1:NC4kS1LZjzfhK40UGmpXv5/qD2kcMzACYJNntCUiDhQ= github.com/ipfs/go-cidutil v0.1.1 h1:COuby6H8C2ml0alvHYX3WdbFM4F07YtbY0UlT5j+sgI= github.com/ipfs/go-cidutil v0.1.1/go.mod h1:SCoUftGEUgoXe5Hjeyw5CiLZF8cwYn/TbtpFQXJCP6k= github.com/ipfs/go-datastore v0.9.1 h1:67Po2epre/o0UxrmkzdS9ZTe2GFGODgTd2odx8Wh6Yo= github.com/ipfs/go-datastore v0.9.1/go.mod h1:zi07Nvrpq1bQwSkEnx3bfjz+SQZbdbWyCNvyxMh9pN0= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-dsqueue v0.2.0 h1:MBi9w3oSiX98Xc+Y7NuJ9G8MI6mAT4IGdO9dHEMCZzU= github.com/ipfs/go-dsqueue v0.2.0/go.mod h1:8FfNQC4DMF/KkzBXRNB9Rb3MKDW0Sh98HMtXYl1mLQE= github.com/ipfs/go-ipfs-cmds v0.16.0 h1:Oq39Gzz3pWrPwP25SjbfBQugFjKyjFdsHlAMIXdzYzM= github.com/ipfs/go-ipfs-cmds v0.16.0/go.mod h1:iRNtY9ipM/vH0Yr+/0FY+JfT+trZDQIztDoesmmoTo4= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-pq v0.0.4 h1:U7jjENWJd1jhcrR8X/xHTaph14PTAK9O+yaLJbjqgOw= github.com/ipfs/go-ipfs-pq v0.0.4/go.mod h1:9UdLOIIb99IFrgT0Fc53pvbvlJBhpUb4GJuAQf3+O2A= github.com/ipfs/go-ipfs-redirects-file v0.1.2 h1:QCK7VtL91FH17KROVVy5KrzDx2hu68QvB2FTWk08ZQk= github.com/ipfs/go-ipfs-redirects-file v0.1.2/go.mod h1:yIiTlLcDEM/8lS6T3FlCEXZktPPqSOyuY6dEzVqw7Fw= github.com/ipfs/go-ipld-cbor v0.2.1 h1:H05yEJbK/hxg0uf2AJhyerBDbjOuHX4yi+1U/ogRa7E= github.com/ipfs/go-ipld-cbor v0.2.1/go.mod h1:x9Zbeq8CoE5R2WicYgBMcr/9mnkQ0lHddYWJP2sMV3A= github.com/ipfs/go-ipld-format v0.6.3 h1:9/lurLDTotJpZSuL++gh3sTdmcFhVkCwsgx2+rAh4j8= github.com/ipfs/go-ipld-format v0.6.3/go.mod h1:74ilVN12NXVMIV+SrBAyC05UJRk0jVvGqdmrcYZvCBk= github.com/ipfs/go-ipld-legacy v0.3.0 h1:7XhFKkRyCvP5upOlQfKUFIqL3S5DEZnbUE4bQmQ/tNE= github.com/ipfs/go-ipld-legacy v0.3.0/go.mod h1:Ukef9ARQiX+RVetwH2XiReLgJvQDEXcUPszrZ1KRjKI= github.com/ipfs/go-log/v2 v2.9.1 h1:3JXwHWU31dsCpvQ+7asz6/QsFJHqFr4gLgQ0FWteujk= github.com/ipfs/go-log/v2 v2.9.1/go.mod h1:evFx7sBiohUN3AG12mXlZBw5hacBQld3ZPHrowlJYoo= github.com/ipfs/go-metrics-interface v0.3.0 h1:YwG7/Cy4R94mYDUuwsBfeziJCVm9pBMJ6q/JR9V40TU= github.com/ipfs/go-metrics-interface v0.3.0/go.mod h1:OxxQjZDGocXVdyTPocns6cOLwHieqej/jos7H4POwoY= github.com/ipfs/go-peertaskqueue v0.8.3 h1:tBPpGJy+A92RqtRFq5amJn0Uuj8Pw8tXi0X3eHfHM8w= github.com/ipfs/go-peertaskqueue v0.8.3/go.mod h1:OqVync4kPOcXEGdj/LKvox9DCB5mkSBeXsPczCxLtYA= github.com/ipfs/go-test v0.2.3 h1:Z/jXNAReQFtCYyn7bsv/ZqUwS6E7iIcSpJ2CuzCvnrc= github.com/ipfs/go-test v0.2.3/go.mod h1:QW8vSKkwYvWFwIZQLGQXdkt9Ud76eQXRQ9Ao2H+cA1o= github.com/ipfs/go-unixfsnode v1.10.3 h1:c8sJjuGNkxXAQH75P+f5ngPda/9T+DrboVA0TcDGvGI= github.com/ipfs/go-unixfsnode v1.10.3/go.mod h1:2Jlc7DoEwr12W+7l8Hr6C7XF4NHST3gIkqSArLhGSxU= github.com/ipfs/hang-fds v0.1.0 h1:deBiFlWHsVGzJ0ZMaqscEqRM1r2O1rFZ59UiQXb1Xko= github.com/ipfs/hang-fds v0.1.0/go.mod h1:29VLWOn3ftAgNNgXg/al7b11UzuQ+w7AwtCGcTaWkbM= github.com/ipfs/iptb v1.4.1 h1:faXd3TKGPswbHyZecqqg6UfbES7RDjTKQb+6VFPKDUo= github.com/ipfs/iptb v1.4.1/go.mod h1:nTsBMtVYFEu0FjC5DgrErnABm3OG9ruXkFXGJoTV5OA= github.com/ipfs/iptb-plugins v0.5.1 h1:11PNTNEt2+SFxjUcO5qpyCTXqDj6T8Tx9pU/G4ytCIQ= github.com/ipfs/iptb-plugins v0.5.1/go.mod h1:mscJAjRnu4g16QK6oUBn9RGpcp8ueJmLfmPxIG/At78= github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco= github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.22.0 h1:YJhDhjEOvOYaqshd3b4atIWUoRg/rKrgmwCyUHwlbuY= github.com/ipld/go-ipld-prime v0.22.0/go.mod h1:ol7vKxOOVgEh0iAPuiDalM+0gScXVMA5ZZa4DVrTnEA= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20250821084354-a425e60cd714 h1:cqNk8PEwHnK0vqWln+U/YZhQc9h2NB3KjUjDPZo5Q2s= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20250821084354-a425e60cd714/go.mod h1:ZEUdra3CoqRVRYgAX/jAJO9aZGz6SKtKEG628fHHktY= github.com/ipshipyard/p2p-forge v0.7.0 h1:PQayexxZC1FR2Vx0XOSbmZ6wDPliidS48I+xXWuF+YU= github.com/ipshipyard/p2p-forge v0.7.0/go.mod h1:i2wg0p7WmHGyo5vYaK9COZBp8BN5Drncfu3WoQNZlQY= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= github.com/jgautheron/goconst v1.7.1 h1:VpdAG7Ca7yvvJk5n8dMwQhfEZJh95kl/Hl9S1OI5Jkk= github.com/jgautheron/goconst v1.7.1/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4= github.com/jingyugao/rowserrcheck v1.1.1 h1:zibz55j/MJtLsjP1OF4bSdgXxwL1b+Vn7Tjzq7gFzUs= github.com/jingyugao/rowserrcheck v1.1.1/go.mod h1:4yvlZSDb3IyDTUZJUmpZfm2Hwok+Dtp+nu2qOq+er9c= github.com/jjti/go-spancheck v0.6.4 h1:Tl7gQpYf4/TMU7AT84MN83/6PutY21Nb9fuQjFTpRRc= github.com/jjti/go-spancheck v0.6.4/go.mod h1:yAEYdKJ2lRkDA8g7X+oKUHXOWVAXSBJRv04OhF+QUjk= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julz/importas v0.2.0 h1:y+MJN/UdL63QbFJHws9BVC5RpA2iq0kpjrFajTGivjQ= github.com/julz/importas v0.2.0/go.mod h1:pThlt589EnCYtMnmhmRYY/qn9lCf/frPOK+WMx3xiJY= github.com/karamaru-alpha/copyloopvar v1.2.1 h1:wmZaZYIjnJ0b5UoKDjUHrikcV0zuPyyxI4SVplLd2CI= github.com/karamaru-alpha/copyloopvar v1.2.1/go.mod h1:nFmMlFNlClC2BPvNaHMdkirmTJxVCY0lhxBtlfOypMM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/errcheck v1.9.0 h1:9xt1zI9EBfcYBvdU1nVrzMzzUPUtPKs9bVSIM3TAb3M= github.com/kisielk/errcheck v1.9.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= github.com/koron/go-ssdp v0.0.6/go.mod h1:0R9LfRJGek1zWTjN3JUNlm5INCDYGpRDfAptnct63fI= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kulti/thelper v0.6.3 h1:ElhKf+AlItIu+xGnI990no4cE2+XaSu1ULymV2Yulxs= github.com/kulti/thelper v0.6.3/go.mod h1:DsqKShOvP40epevkFrvIwkCMNYxMeTNjdWL4dqWHZ6I= github.com/kunwardeep/paralleltest v1.0.10 h1:wrodoaKYzS2mdNVnc4/w31YaXFtsc21PCTdvWJ/lDDs= github.com/kunwardeep/paralleltest v1.0.10/go.mod h1:2C7s65hONVqY7Q5Efj5aLzRCNLjw2h4eMc9EcypGjcY= github.com/lasiar/canonicalheader v1.1.2 h1:vZ5uqwvDbyJCnMhmFYimgMZnJMjwljN5VGY0VKbMXb4= github.com/lasiar/canonicalheader v1.1.2/go.mod h1:qJCeLFS0G/QlLQ506T+Fk/fWMa2VmBUiEI2cuMK4djI= github.com/ldez/exptostd v0.4.2 h1:l5pOzHBz8mFOlbcifTxzfyYbgEmoUqjxLFHZkjlbHXs= github.com/ldez/exptostd v0.4.2/go.mod h1:iZBRYaUmcW5jwCR3KROEZ1KivQQp6PHXbDPk9hqJKCQ= github.com/ldez/gomoddirectives v0.6.1 h1:Z+PxGAY+217f/bSGjNZr/b2KTXcyYLgiWI6geMBN2Qc= github.com/ldez/gomoddirectives v0.6.1/go.mod h1:cVBiu3AHR9V31em9u2kwfMKD43ayN5/XDgr+cdaFaKs= github.com/ldez/grignotin v0.9.0 h1:MgOEmjZIVNn6p5wPaGp/0OKWyvq42KnzAt/DAb8O4Ow= github.com/ldez/grignotin v0.9.0/go.mod h1:uaVTr0SoZ1KBii33c47O1M8Jp3OP3YDwhZCmzT9GHEk= github.com/ldez/tagliatelle v0.7.1 h1:bTgKjjc2sQcsgPiT902+aadvMjCeMHrY7ly2XKFORIk= github.com/ldez/tagliatelle v0.7.1/go.mod h1:3zjxUpsNB2aEZScWiZTHrAXOl1x25t3cRmzfK1mlo2I= github.com/ldez/usetesting v0.4.2 h1:J2WwbrFGk3wx4cZwSMiCQQ00kjGR0+tuuyW0Lqm4lwA= github.com/ldez/usetesting v0.4.2/go.mod h1:eEs46T3PpQ+9RgN9VjpY6qWdiw2/QmfiDeWmdZdrjIQ= github.com/leonklingele/grouper v1.1.2 h1:o1ARBDLOmmasUaNDesWqWCIFH3u7hoFlM84YrjT3mIY= github.com/leonklingele/grouper v1.1.2/go.mod h1:6D0M/HVkhs2yRKRFZUoGjeDy7EZTfFBE9gl4kjmIGkA= github.com/libdns/libdns v1.0.0-beta.1 h1:KIf4wLfsrEpXpZ3vmc/poM8zCATXT2klbdPe6hyOBjQ= github.com/libdns/libdns v1.0.0-beta.1/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-doh-resolver v0.5.0 h1:4h7plVVW+XTS+oUBw2+8KfoM1jF6w8XmO7+skhePFdE= github.com/libp2p/go-doh-resolver v0.5.0/go.mod h1:aPDxfiD2hNURgd13+hfo29z9IC22fv30ee5iM31RzxU= github.com/libp2p/go-flow-metrics v0.3.0 h1:q31zcHUvHnwDO0SHaukewPYgwOBSxtt830uJtUx6784= github.com/libp2p/go-flow-metrics v0.3.0/go.mod h1:nuhlreIwEguM1IvHAew3ij7A8BMlyHQJ279ao24eZZo= github.com/libp2p/go-libp2p v0.48.0 h1:h2BrLAgrj7X8bEN05K7qmrjpNHYA+6tnsGRdprjTnvo= github.com/libp2p/go-libp2p v0.48.0/go.mod h1:Q1fBZNdmC2Hf82husCTfkKJVfHm2we5zk+NWmOGEmWk= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-kad-dht v0.39.0 h1:mww38eBYiUvdsu+Xl/GLlBC0Aa8M+5HAwvafkFOygAM= github.com/libp2p/go-libp2p-kad-dht v0.39.0/go.mod h1:Po2JugFEkDq9Vig/JXtc153ntOi0q58o4j7IuITCOVs= github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s= github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4= github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= github.com/libp2p/go-libp2p-routing-helpers v0.7.5 h1:HdwZj9NKovMx0vqq6YNPTh6aaNzey5zHD7HeLJtq6fI= github.com/libp2p/go-libp2p-routing-helpers v0.7.5/go.mod h1:3YaxrwP0OBPDD7my3D0KxfR89FlcX/IEbxDEDfAmj98= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-netroute v0.4.0 h1:sZZx9hyANYUx9PZyqcgE/E1GUG3iEtTZHUEvdtXT7/Q= github.com/libp2p/go-netroute v0.4.0/go.mod h1:Nkd5ShYgSMS5MUKy/MU2T57xFoOKvvLR92Lic48LEyA= github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= github.com/libp2p/go-yamux/v5 v5.0.1 h1:f0WoX/bEF2E8SbE4c/k1Mo+/9z0O4oC/hWEA+nfYRSg= github.com/libp2p/go-yamux/v5 v5.0.1/go.mod h1:en+3cdX51U0ZslwRdRLrvQsdayFt3TSUKvBGErzpWbU= github.com/macabu/inamedparam v0.1.3 h1:2tk/phHkMlEL/1GNe/Yf6kkR/hkcUdAEY3L0hjYV1Mk= github.com/macabu/inamedparam v0.1.3/go.mod h1:93FLICAIk/quk7eaPPQvbzihUdn/QkGDwIZEoLtpH6I= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/maratori/testableexamples v1.0.0 h1:dU5alXRrD8WKSjOUnmJZuzdxWOEQ57+7s93SLMxb2vI= github.com/maratori/testableexamples v1.0.0/go.mod h1:4rhjL1n20TUTT4vdh3RDqSizKLyXp7K2u6HgraZCGzE= github.com/maratori/testpackage v1.1.1 h1:S58XVV5AD7HADMmD0fNnziNHqKvSdDuEKdPD1rNTU04= github.com/maratori/testpackage v1.1.1/go.mod h1:s4gRK/ym6AMrqpOa/kEbQTV4Q4jb7WeLZzVhVVVOQMc= github.com/marcopolo/simnet v0.0.4 h1:50Kx4hS9kFGSRIbrt9xUS3NJX33EyPqHVmpXvaKLqrY= github.com/marcopolo/simnet v0.0.4/go.mod h1:tfQF1u2DmaB6WHODMtQaLtClEf3a296CKQLq5gAsIS0= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= github.com/matoous/godox v1.1.0 h1:W5mqwbyWrwZv6OQ5Z1a/DHGMOvXYCBP3+Ht7KMoJhq4= github.com/matoous/godox v1.1.0/go.mod h1:jgE/3fUXiTurkdHOLT5WEkThTSuE7yxHv5iWPa80afs= github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mgechev/revive v1.7.0 h1:JyeQ4yO5K8aZhIKf5rec56u0376h8AlKNQEmjfkjKlY= github.com/mgechev/revive v1.7.0/go.mod h1:qZnwcNhoguE58dfi96IJeSTPeZQejNeoMQLUZGi4SW4= github.com/mholt/acmez/v3 v3.1.2 h1:auob8J/0FhmdClQicvJvuDavgd5ezwLBfKuYmynhYzc= github.com/mholt/acmez/v3 v3.1.2/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/moricho/tparallel v0.3.2 h1:odr8aZVFA3NZrNybggMkYO3rgPRcqjeQUlBBFVxKHTI= github.com/moricho/tparallel v0.3.2/go.mod h1:OQ+K3b4Ln3l2TZveGCywybl68glfLEwFGqvnjok8b+U= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.16.1 h1:fgJ0Pitow+wWXzN9do+1b8Pyjmo8m5WhGfzpL82MpCw= github.com/multiformats/go-multiaddr v0.16.1/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= github.com/multiformats/go-multiaddr-dns v0.5.0 h1:p/FTyHKX0nl59f+S+dEUe8HRK+i5Ow/QHMw8Nh3gPCo= github.com/multiformats/go-multiaddr-dns v0.5.0/go.mod h1:yJ349b8TPIAANUyuOzn1oz9o22tV9f+06L+cCeMxC14= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multicodec v0.10.0 h1:UpP223cig/Cx8J76jWt91njpK3GTAO1w02sdcjZDSuc= github.com/multiformats/go-multicodec v0.10.0/go.mod h1:wg88pM+s2kZJEQfRCKBNU+g32F5aWBEjyFHXvZLTcLI= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.6.1 h1:4aoX5v6T+yWmc2raBHsTvzmFhOI8WVOer28DeBBEYdQ= github.com/multiformats/go-multistream v0.6.1/go.mod h1:ksQf6kqHAb6zIsyw7Zm+gAuVo57Qbq84E27YlYqavqw= github.com/multiformats/go-varint v0.1.0 h1:i2wqFp4sdl3IcIxfAonHQV9qU5OsZ4Ts9IOoETFs5dI= github.com/multiformats/go-varint v0.1.0/go.mod h1:5KVAVXegtfmNQQm/lCY+ATvDzvJJhSkUlGQV9wgObdI= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U= github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nishanths/exhaustive v0.12.0 h1:vIY9sALmw6T/yxiASewa4TQcFsVYZQQRUQJhKRf3Swg= github.com/nishanths/exhaustive v0.12.0/go.mod h1:mEZ95wPIZW+x8kC4TgC+9YCUgiST7ecevsVDTgc2obs= github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk= github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= github.com/nunnatsa/ginkgolinter v0.19.1 h1:mjwbOlDQxZi9Cal+KfbEJTCz327OLNfwNvoZ70NJ+c4= github.com/nunnatsa/ginkgolinter v0.19.1/go.mod h1:jkQ3naZDmxaZMXPWaS9rblH+i+GWXQCaS/JFIWcOH2s= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= github.com/onsi/gomega v1.36.3 h1:hID7cr8t3Wp26+cYnfcjR6HpJ00fdogN6dqZ1t6IylU= github.com/onsi/gomega v1.36.3/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o= github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M= github.com/pion/dtls/v3 v3.1.2 h1:gqEdOUXLtCGW+afsBLO0LtDD8GnuBBjEy6HRtyofZTc= github.com/pion/dtls/v3 v3.1.2/go.mod h1:Hw/igcX4pdY69z1Hgv5x7wJFrUkdgHwAn/Q/uo7YHRo= github.com/pion/ice/v4 v4.0.10 h1:P59w1iauC/wPk9PdY8Vjl4fOFL5B+USq1+xbDcN6gT4= github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw= github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4= github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic= github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM= github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.16 h1:fk1B1dNW4hsI78XUCljZJlC4kZOPk67mNRuQ0fcEkSo= github.com/pion/rtcp v1.2.16/go.mod h1:/as7VKfYbs5NIb4h6muQ35kQF/J0ZVNz2Z3xKoCBYOo= github.com/pion/rtp v1.8.19 h1:jhdO/3XhL/aKm/wARFVmvTfq0lC/CvN1xwYKmduly3c= github.com/pion/rtp v1.8.19/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk= github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE= github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE= github.com/pion/sdp/v3 v3.0.18 h1:l0bAXazKHpepazVdp+tPYnrsy9dfh7ZbT8DxesH5ZnI= github.com/pion/sdp/v3 v3.0.18/go.mod h1:ZREGo6A9ZygQ9XkqAj5xYCQtQpif0i6Pa81HOiAdqQ8= github.com/pion/srtp/v3 v3.0.6 h1:E2gyj1f5X10sB/qILUGIkL4C2CqK269Xq167PbGCc/4= github.com/pion/srtp/v3 v3.0.6/go.mod h1:BxvziG3v/armJHAaJ87euvkhHqWe9I7iiOy50K2QkhY= github.com/pion/stun/v3 v3.1.1 h1:CkQxveJ4xGQjulGSROXbXq94TAWu8gIX2dT+ePhUkqw= github.com/pion/stun/v3 v3.1.1/go.mod h1:qC1DfmcCTQjl9PBaMa5wSn3x9IPmKxSdcCsxBcDBndM= github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0= github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= github.com/pion/transport/v4 v4.0.1 h1:sdROELU6BZ63Ab7FrOLn13M6YdJLY20wldXW2Cu2k8o= github.com/pion/transport/v4 v4.0.1/go.mod h1:nEuEA4AD5lPdcIegQDpVLgNoDGreqM/YqmEx3ovP4jM= github.com/pion/turn/v4 v4.0.2 h1:ZqgQ3+MjP32ug30xAbD6Mn+/K4Sxi3SdNOTFf+7mpps= github.com/pion/turn/v4 v4.0.2/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs= github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54= github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a h1:cgqrm0F3zwf9IPzca7xN4w+Zy6MC9ZkPvAC8QEWa/iQ= github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a/go.mod h1:ocZfO/tLSHqfScRDNTJbAJR1by4D1lewauX9OwTaPuY= github.com/polyfloyd/go-errorlint v1.7.1 h1:RyLVXIbosq1gBdk/pChWA8zWYLsq9UEw7a1L5TVMCnA= github.com/polyfloyd/go-errorlint v1.7.1/go.mod h1:aXjNb1x2TNhoLsk26iv1yl7a+zTnXPhwEMtEXukiLR8= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1 h1:+Wl/0aFp0hpuHM3H//KMft64WQ1yX9LdJY64Qm/gFCo= github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1/go.mod h1:GJLgqsLeo4qgavUoL8JeGFNS7qcisx3awV/w9eWTmNI= github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE= github.com/quasilyte/go-ruleguard/dsl v0.3.22/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo= github.com/quasilyte/gogrep v0.5.0/go.mod h1:Cm9lpz9NZjEoL1tgZ2OgeUKPIxL1meE7eo60Z6Sk+Ng= github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl980XxGFEZSS6KlBGIV0diGdySzxATTWoqaU= github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/quic-go/webtransport-go v0.10.0 h1:LqXXPOXuETY5Xe8ITdGisBzTYmUOy5eSj+9n4hLTjHI= github.com/quic-go/webtransport-go v0.10.0/go.mod h1:LeGIXr5BQKE3UsynwVBeQrU1TPrbh73MGoC6jd+V7ow= github.com/raeperd/recvcheck v0.2.0 h1:GnU+NsbiCqdC2XX5+vMZzP+jAJC5fht7rcVTAhX74UI= github.com/raeperd/recvcheck v0.2.0/go.mod h1:n04eYkwIR0JbgD73wT8wL4JjPC3wm0nFtzBnWNocnYU= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryancurrah/gomodguard v1.3.5 h1:cShyguSwUEeC0jS7ylOiG/idnd1TpJ1LfHGpV3oJmPU= github.com/ryancurrah/gomodguard v1.3.5/go.mod h1:MXlEPQRxgfPQa62O8wzK3Ozbkv9Rkqr+wKjSxTdsNJE= github.com/ryanrolds/sqlclosecheck v0.5.1 h1:dibWW826u0P8jNLsLN+En7+RqWWTYrjCB9fJfSfdyCU= github.com/ryanrolds/sqlclosecheck v0.5.1/go.mod h1:2g3dUjoS6AL4huFdv6wn55WpLIDjY7ZgUR4J8HOO/XQ= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sanposhiho/wastedassign/v2 v2.1.0 h1:crurBF7fJKIORrV85u9UUpePDYGWnwvv3+A96WvwXT0= github.com/sanposhiho/wastedassign/v2 v2.1.0/go.mod h1:+oSmSC+9bQ+VUAxA66nBb0Z7N8CK7mscKTDYC6aIek4= github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 h1:PKK9DyHxif4LZo+uQSgXNqs0jj5+xZwwfKHgph2lxBw= github.com/santhosh-tekuri/jsonschema/v6 v6.0.1/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/sashamelentyev/interfacebloat v1.1.0 h1:xdRdJp0irL086OyW1H/RTZTr1h/tMEOsumirXcOJqAw= github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ= github.com/sashamelentyev/usestdlibvars v1.28.0 h1:jZnudE2zKCtYlGzLVreNp5pmCdOxXUzwsMDBkR21cyQ= github.com/sashamelentyev/usestdlibvars v1.28.0/go.mod h1:9nl0jgOfHKWNFS43Ojw0i7aRoS4j6EBye3YBhmAIRF8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/securego/gosec/v2 v2.22.2 h1:IXbuI7cJninj0nRpZSLCUlotsj8jGusohfONMrHoF6g= github.com/securego/gosec/v2 v2.22.2/go.mod h1:UEBGA+dSKb+VqM6TdehR7lnQtIIMorYJ4/9CW1KVQBE= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sivchari/containedctx v1.0.3 h1:x+etemjbsh2fB5ewm5FeLNi5bUjK0V8n0RB+Wwfd0XE= github.com/sivchari/containedctx v1.0.3/go.mod h1:c1RDvCbnJLtH4lLcYD/GqwiBSSf4F5Qk0xld2rBqzJ4= github.com/sivchari/tenv v1.12.1 h1:+E0QzjktdnExv/wwsnnyk4oqZBUfuh89YMQT1cyuvSY= github.com/sivchari/tenv v1.12.1/go.mod h1:1LjSOUCc25snIr5n3DtGGrENhX3LuWefcplwVGC24mw= github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/sonatard/noctx v0.1.0 h1:JjqOc2WN16ISWAjAk8M5ej0RfExEXtkEyExl2hLW+OM= github.com/sonatard/noctx v0.1.0/go.mod h1:0RvBxqY8D4j9cTTTWE8ylt2vqj2EPI8fHmrxHdsaZ2c= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/go-diff v0.7.0 h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0= github.com/sourcegraph/go-diff v0.7.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YEwQ0= github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= github.com/stbenjam/no-sprintf-host-port v0.2.0 h1:i8pxvGrt1+4G0czLr/WnmyH7zbZ8Bg8etvARQ1rpyl4= github.com/stbenjam/no-sprintf-host-port v0.2.0/go.mod h1:eL0bQ9PasS0hsyTyfTjjG+E80QIyPnBVQbYZyv20Jfk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/tdakkota/asciicheck v0.4.1 h1:bm0tbcmi0jezRA2b5kg4ozmMuGAFotKI3RZfrhfovg8= github.com/tdakkota/asciicheck v0.4.1/go.mod h1:0k7M3rCfRXb0Z6bwgvkEIMleKH3kXNz9UqJ9Xuqopr8= github.com/tenntenn/modver v1.0.1 h1:2klLppGhDgzJrScMpkj9Ujy3rXPUspSjAcev9tSEBgA= github.com/tenntenn/modver v1.0.1/go.mod h1:bePIyQPb7UeioSRkw3Q0XeMhYZSMx9B8ePqg6SAMGH0= github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpRQGxTSkNYKJ51yaw6ChIqO+Je8UqsTKN/cDag= github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= github.com/tetafro/godot v1.5.0 h1:aNwfVI4I3+gdxjMgYPus9eHmoBeJIbnajOyqZYStzuw= github.com/tetafro/godot v1.5.0/go.mod h1:2oVxTBSftRTh4+MVfUaUXR6bn2GDXCaMcOG4Dk3rfio= github.com/timakin/bodyclose v0.0.0-20241017074812-ed6a65f985e3 h1:y4mJRFlM6fUyPhoXuFg/Yu02fg/nIPFMOY8tOqppoFg= github.com/timakin/bodyclose v0.0.0-20241017074812-ed6a65f985e3/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460= github.com/timonwong/loggercheck v0.10.1 h1:uVZYClxQFpw55eh+PIoqM7uAOHMrhVcDoWDery9R8Lg= github.com/timonwong/loggercheck v0.10.1/go.mod h1:HEAWU8djynujaAVX7QI65Myb8qgfcZ1uKbdpg3ZzKl8= github.com/tomarrell/wrapcheck/v2 v2.10.0 h1:SzRCryzy4IrAH7bVGG4cK40tNUhmVmMDuJujy4XwYDg= github.com/tomarrell/wrapcheck/v2 v2.10.0/go.mod h1:g9vNIyhb5/9TQgumxQyOEqDHsmGYcGsVMOx/xGkqdMo= github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw= github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM= github.com/ultraware/funlen v0.2.0 h1:gCHmCn+d2/1SemTdYMiKLAHFYxTYz7z9VIDRaTGyLkI= github.com/ultraware/funlen v0.2.0/go.mod h1:ZE0q4TsJ8T1SQcjmkhN/w+MceuatI6pBFSxxyteHIJA= github.com/ultraware/whitespace v0.2.0 h1:TYowo2m9Nfj1baEQBjuHzvMRbp19i+RCcRYrSWoFa+g= github.com/ultraware/whitespace v0.2.0/go.mod h1:XcP1RLD81eV4BW8UhQlpaR+SDc2givTvyI8a586WjW8= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.16 h1:MH0k6uJxdwdeWQTwhSO42Pwr4YLrNLwBtg1MRgTqPdQ= github.com/urfave/cli v1.22.16/go.mod h1:EeJR6BKodywf4zciqrdw6hpCPk68JO9z5LazXZMn5Po= github.com/uudashr/gocognit v1.2.0 h1:3BU9aMr1xbhPlvJLSydKwdLN3tEUUrzPSSM8S4hDYRA= github.com/uudashr/gocognit v1.2.0/go.mod h1:k/DdKPI6XBZO1q7HgoV2juESI2/Ofj9AcHPZhBBdrTU= github.com/uudashr/iface v1.3.1 h1:bA51vmVx1UIhiIsQFSNq6GZ6VPTk3WNMZgRiCe9R29U= github.com/uudashr/iface v1.3.1/go.mod h1:4QvspiRd3JLPAEXBQ9AiZpLbJlrWWgRChOKDJEuQTdg= github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.3.1 h1:82ioxmhEYut7LBVGhGq8xoRkXPLElVuh5mV67AFfdv0= github.com/whyrusleeping/cbor-gen v0.3.1/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= github.com/xen0n/gosmopolitan v1.2.2 h1:/p2KTnMzwRexIW8GlKawsTWOxn7UHA+jCMF/V8HHtvU= github.com/xen0n/gosmopolitan v1.2.2/go.mod h1:7XX7Mj61uLYrj0qmeN0zi7XDon9JRAEhYQqAPLVNTeg= github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM= github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk= github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5JsjqtoFs= github.com/yeya24/promlinter v0.3.0/go.mod h1:cDfJQQYv9uYciW60QT0eeHlFodotkYZlL+YcPQN+mW4= github.com/ykadowak/zerologlint v0.1.5 h1:Gy/fMz1dFQN9JZTPjv1hxEk+sRWm05row04Yoolgdiw= github.com/ykadowak/zerologlint v0.1.5/go.mod h1:KaUskqF3e/v59oPmdq1U1DnKcuHokl2/K1U4pmIELKg= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI= github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= gitlab.com/bosi/decorder v0.4.2 h1:qbQaV3zgwnBZ4zPMhGLW4KZe7A7NwxEhJx39R3shffo= gitlab.com/bosi/decorder v0.4.2/go.mod h1:muuhHoaJkA9QLcYHq4Mj8FJUwDZ+EirSHRiaTcTf6T8= go-simpler.org/assert v0.9.0 h1:PfpmcSvL7yAnWyChSjOz6Sp6m9j5lyK8Ok9pEL31YkQ= go-simpler.org/assert v0.9.0/go.mod h1:74Eqh5eI6vCK6Y5l3PI8ZYFXG4Sa+tkr70OIPJAUr28= go-simpler.org/musttag v0.13.0 h1:Q/YAW0AHvaoaIbsPj3bvEI5/QFP7w696IMUpnKXQfCE= go-simpler.org/musttag v0.13.0/go.mod h1:FTzIGeK6OkKlUDVpj0iQUXZLUO1Js9+mvykDQy9C5yM= go-simpler.org/sloglint v0.9.0 h1:/40NQtjRx9txvsB/RN022KsUJU+zaaSb/9q9BSefSrE= go-simpler.org/sloglint v0.9.0/go.mod h1:G/OrAF6uxj48sHahCzrbarVMptL2kjWTaUeC8+fOGww= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 h1:jiDhWWeC7jfWqR9c/uplMOqJ0sbNlNWv0UkzE0vX1MA= golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90/go.mod h1:xE1HEv6b+1SCZ5/uscMRjUBKtIxworgEcEi+/n9NQDQ= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20250210185358-939b2ce775ac h1:TSSpLIG4v+p0rPv1pNOQtl1I8knsO4S9trOxNMOLVP4= golang.org/x/exp/typeparams v0.0.0-20250210185358-939b2ce775ac/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c h1:6a8FdnNk6bTXBjR4AGKFgUKuo+7GnR3FX5L7CbveeZc= golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200820010801-b793a1359eac/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201023174141-c8cfbd0f21e6/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1-0.20210205202024-ef80cdb6ec6d/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= golang.org/x/tools v0.1.1-0.20210302220138-2ac05c832e1a/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/gotestsum v1.13.0 h1:+Lh454O9mu9AMG1APV4o0y7oDYKyik/3kBOiCqiEpRo= gotest.tools/gotestsum v1.13.0/go.mod h1:7f0NS5hFb0dWr4NtcsAsF0y1kzjEFfAil0HiBQJE03Q= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.6.1 h1:R094WgE8K4JirYjBaOpz/AvTyUu/3wbmAoskKN/pxTI= honnef.co/go/tools v0.6.1/go.mod h1:3puzxxljPCe8RGJX7BIy1plGbxEOZni5mR2aXe3/uk4= lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= mvdan.cc/gofumpt v0.7.0 h1:bg91ttqXmi9y2xawvkuMXyvAA/1ZGJqYAEGjXuP0JXU= mvdan.cc/gofumpt v0.7.0/go.mod h1:txVFJy/Sc/mvaycET54pV8SW8gWxTlUuGHVEcncmNUo= mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f h1:lMpcwN6GxNbWtbpI1+xzFLSW8XzX0u72NttUGVFjO3U= mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f/go.mod h1:RSLa7mKKCNeTTMHBw5Hsy2rfJmd6O2ivt9Dw9ZqCQpQ= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= ================================================ FILE: test/dependencies/iptb/iptb.go ================================================ package main import ( "fmt" "os" cli "github.com/ipfs/iptb/cli" testbed "github.com/ipfs/iptb/testbed" plugin "github.com/ipfs/iptb-plugins/local" ) func init() { _, err := testbed.RegisterPlugin(testbed.IptbPlugin{ From: "", NewNode: plugin.NewNode, GetAttrList: plugin.GetAttrList, GetAttrDesc: plugin.GetAttrDesc, PluginName: plugin.PluginName, BuiltIn: true, }, false) if err != nil { panic(err) } } func main() { cli := cli.NewCli() if err := cli.Run(os.Args); err != nil { fmt.Fprintf(cli.ErrWriter, "%s\n", err) os.Exit(1) } } ================================================ FILE: test/dependencies/ma-pipe-unidir/LICENSE ================================================ The MIT License (MIT) Copyright (c) 2017 Łukasz Magiera Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: test/dependencies/ma-pipe-unidir/main.go ================================================ package main import ( "flag" "fmt" "io" "os" "strconv" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) const USAGE = "ma-pipe-unidir [-l|--listen] [--pidFile=path] [-h|--help] \n" type Opts struct { Listen bool PidFile string } func app() int { opts := Opts{} flag.BoolVar(&opts.Listen, "l", false, "") flag.BoolVar(&opts.Listen, "listen", false, "") flag.StringVar(&opts.PidFile, "pidFile", "", "") flag.Usage = func() { fmt.Print(USAGE) } flag.Parse() args := flag.Args() if len(args) < 2 { // fmt.Print(USAGE) return 1 } mode := args[0] addr := args[1] if mode != "send" && mode != "recv" { fmt.Print(USAGE) return 1 } maddr, err := ma.NewMultiaddr(addr) if err != nil { return 1 } var conn manet.Conn if opts.Listen { listener, err := manet.Listen(maddr) if err != nil { return 1 } if len(opts.PidFile) > 0 { data := []byte(strconv.Itoa(os.Getpid())) err := os.WriteFile(opts.PidFile, data, 0o644) if err != nil { return 1 } defer os.Remove(opts.PidFile) } conn, err = listener.Accept() if err != nil { return 1 } } else { var err error conn, err = manet.Dial(maddr) if err != nil { return 1 } if len(opts.PidFile) > 0 { data := []byte(strconv.Itoa(os.Getpid())) err := os.WriteFile(opts.PidFile, data, 0o644) if err != nil { return 1 } defer os.Remove(opts.PidFile) } } defer conn.Close() switch mode { case "recv": _, err = io.Copy(os.Stdout, conn) case "send": _, err = io.Copy(conn, os.Stdin) default: return 1 } if err != nil { return 1 } return 0 } func main() { os.Exit(app()) } ================================================ FILE: test/dependencies/pollEndpoint/main.go ================================================ // pollEndpoint is a helper utility that waits for a http endpoint to be reachable and return with http.StatusOK package main import ( "context" "flag" "io" "net" "net/http" "os" "time" logging "github.com/ipfs/go-log/v2" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) var ( host = flag.String("host", "/ip4/127.0.0.1/tcp/5001", "the multiaddr host to dial on") tries = flag.Int("tries", 10, "how many tries to make before failing") timeout = flag.Duration("tout", time.Second, "how long to wait between attempts") httpURL = flag.String("http-url", "", "HTTP URL to fetch") httpOut = flag.Bool("http-out", false, "Print the HTTP response body to stdout") verbose = flag.Bool("v", false, "verbose logging") ) var log = logging.Logger("pollEndpoint") func main() { flag.Parse() // extract address from host flag addr, err := ma.NewMultiaddr(*host) if err != nil { log.Fatal("NewMultiaddr() failed: ", err) } if *verbose { // lower log level logging.SetDebugLogging() } // show what we got start := time.Now() log.Debugf("starting at %s, tries: %d, timeout: %s, addr: %s", start, *tries, *timeout, addr) connTries := *tries for connTries > 0 { c, err := manet.Dial(addr) if err == nil { log.Debugf("ok - endpoint reachable with %d tries remaining, took %s", *tries, time.Since(start)) c.Close() break } log.Debug("connect failed: ", err) time.Sleep(*timeout) connTries-- } if err != nil { goto Fail } if *httpURL != "" { dialer := &connDialer{addr: addr} httpClient := http.Client{Transport: &http.Transport{ DialContext: dialer.DialContext, }} reqTries := *tries for reqTries > 0 { try := (*tries - reqTries) + 1 log.Debugf("trying HTTP req %d: '%s'", try, *httpURL) if tryHTTPGet(&httpClient, *httpURL) { log.Debugf("HTTP req %d to '%s' succeeded", try, *httpURL) goto Success } log.Debugf("HTTP req %d to '%s' failed", try, *httpURL) time.Sleep(*timeout) reqTries-- } goto Fail } Success: os.Exit(0) Fail: log.Error("failed") os.Exit(1) } func tryHTTPGet(client *http.Client, url string) bool { resp, err := client.Get(*httpURL) if resp != nil && resp.Body != nil { defer resp.Body.Close() } if err != nil { return false } if resp.StatusCode != http.StatusOK { return false } if *httpOut { _, err := io.Copy(os.Stdout, resp.Body) if err != nil { panic(err) } } return true } type connDialer struct { addr ma.Multiaddr } func (d connDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { return (&manet.Dialer{}).DialContext(ctx, d.addr) } ================================================ FILE: test/integration/GNUmakefile ================================================ # This Makefile provides a way to really simple way to run benchmarks in a # docker environment. IPFS_ROOT = ../.. CONTAINER = go-ipfs-bench PACKAGE = integrationtest PACKAGE_DIR = test/integration BUILD_DIR = ./build/bench CONTAINER_WORKING_DIR = /go CPU_PROF_NAME = cpu.out EXTRA_TEST_ARGS = all: collect collect: clean build_image run_profiler cp_pprof_from_container cp_pprof_from_container: docker cp $(CONTAINER):$(CONTAINER_WORKING_DIR)/$(CPU_PROF_NAME) $(BUILD_DIR) docker cp $(CONTAINER):$(CONTAINER_WORKING_DIR)/$(PACKAGE).test $(BUILD_DIR) build_image: cd $(IPFS_ROOT) && docker build -t $(IMAGE) . run_profiler: docker run --name $(CONTAINER) -it --entrypoint go $(IMAGE) test ./src/github.com/ipfs/go-ipfs/$(PACKAGE_DIR) --cpuprofile=$(CPU_PROF_NAME) $(EXTRA_TEST_ARGS) clean: docker rm $(CONTAINER) || true rm -rf $(BUILD_DIR) analyze: go tool pprof $(BUILD_DIR)/$(PACKAGE).test $(BUILD_DIR)/$(CPU_PROF_NAME) ================================================ FILE: test/integration/addcat_test.go ================================================ package integrationtest import ( "bytes" "context" "errors" "fmt" "io" "math" "os" "testing" "time" "github.com/ipfs/boxo/bootstrap" "github.com/ipfs/boxo/files" logging "github.com/ipfs/go-log/v2" "github.com/ipfs/go-test/random" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/coreapi" mock "github.com/ipfs/kubo/core/mock" "github.com/ipfs/kubo/thirdparty/unit" testutil "github.com/libp2p/go-libp2p-testing/net" "github.com/libp2p/go-libp2p/core/peer" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" ) var log = logging.Logger("epictest") const kSeed = 1 func Test1KBInstantaneous(t *testing.T) { conf := testutil.LatencyConfig{ NetworkLatency: 0, RoutingLatency: 0, BlockstoreLatency: 0, } if err := DirectAddCat(RandomBytes(1*unit.KB), conf); err != nil { t.Fatal(err) } } func TestDegenerateSlowBlockstore(t *testing.T) { SkipUnlessEpic(t) conf := testutil.LatencyConfig{BlockstoreLatency: 50 * time.Millisecond} if err := AddCatPowers(conf, 128); err != nil { t.Fatal(err) } } func TestDegenerateSlowNetwork(t *testing.T) { SkipUnlessEpic(t) conf := testutil.LatencyConfig{NetworkLatency: 400 * time.Millisecond} if err := AddCatPowers(conf, 128); err != nil { t.Fatal(err) } } func TestDegenerateSlowRouting(t *testing.T) { SkipUnlessEpic(t) conf := testutil.LatencyConfig{RoutingLatency: 400 * time.Millisecond} if err := AddCatPowers(conf, 128); err != nil { t.Fatal(err) } } func Test100MBMacbookCoastToCoast(t *testing.T) { SkipUnlessEpic(t) conf := testutil.LatencyConfig{}.NetworkNYtoSF().BlockstoreSlowSSD2014().RoutingSlow() if err := DirectAddCat(RandomBytes(100*1024*1024), conf); err != nil { t.Fatal(err) } } func AddCatPowers(conf testutil.LatencyConfig, megabytesMax int64) error { var i int64 for i = 1; i < megabytesMax; i = i * 2 { fmt.Printf("%d MB\n", i) if err := DirectAddCat(RandomBytes(i*1024*1024), conf); err != nil { return err } } return nil } func RandomBytes(n int64) []byte { random.SetSeed(kSeed) return random.Bytes(int(n)) } func DirectAddCat(data []byte, conf testutil.LatencyConfig) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() // create network mn := mocknet.New() mn.SetLinkDefaults(mocknet.LinkOptions{ Latency: conf.NetworkLatency, // TODO add to conf. This is tricky because we want 0 values to be functional. Bandwidth: math.MaxInt32, }) adder, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer adder.Close() catter, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer catter.Close() adderAPI, err := coreapi.NewCoreAPI(adder) if err != nil { return err } catterAPI, err := coreapi.NewCoreAPI(catter) if err != nil { return err } err = mn.LinkAll() if err != nil { return err } bs1 := []peer.AddrInfo{adder.Peerstore.PeerInfo(adder.Identity)} bs2 := []peer.AddrInfo{catter.Peerstore.PeerInfo(catter.Identity)} if err := catter.Bootstrap(bootstrap.BootstrapConfigWithPeers(bs1)); err != nil { return err } if err := adder.Bootstrap(bootstrap.BootstrapConfigWithPeers(bs2)); err != nil { return err } added, err := adderAPI.Unixfs().Add(ctx, files.NewBytesFile(data)) if err != nil { return err } readerCatted, err := catterAPI.Unixfs().Get(ctx, added) if err != nil { return err } // verify var bufout bytes.Buffer _, err = io.Copy(&bufout, readerCatted.(io.Reader)) if err != nil { return err } if !bytes.Equal(bufout.Bytes(), data) { return errors.New("catted data does not match added data") } return nil } func SkipUnlessEpic(t *testing.T) { if os.Getenv("IPFS_EPIC_TEST") == "" { t.SkipNow() } } ================================================ FILE: test/integration/bench_cat_test.go ================================================ package integrationtest import ( "bytes" "context" "errors" "io" "math" "testing" "github.com/ipfs/boxo/bootstrap" "github.com/ipfs/boxo/files" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/coreapi" mock "github.com/ipfs/kubo/core/mock" "github.com/ipfs/kubo/thirdparty/unit" testutil "github.com/libp2p/go-libp2p-testing/net" "github.com/libp2p/go-libp2p/core/peer" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" ) func BenchmarkCat1MB(b *testing.B) { benchmarkVarCat(b, unit.MB*1) } func BenchmarkCat2MB(b *testing.B) { benchmarkVarCat(b, unit.MB*2) } func BenchmarkCat4MB(b *testing.B) { benchmarkVarCat(b, unit.MB*4) } func benchmarkVarCat(b *testing.B, size int64) { data := RandomBytes(size) b.SetBytes(size) for n := 0; n < b.N; n++ { err := benchCat(b, data, instant) if err != nil { b.Fatal(err) } } } func benchCat(b *testing.B, data []byte, conf testutil.LatencyConfig) error { b.StopTimer() ctx, cancel := context.WithCancel(context.Background()) defer cancel() // create network mn := mocknet.New() mn.SetLinkDefaults(mocknet.LinkOptions{ Latency: conf.NetworkLatency, // TODO add to conf. This is tricky because we want 0 values to be functional. Bandwidth: math.MaxInt32, }) adder, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer adder.Close() catter, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer catter.Close() adderAPI, err := coreapi.NewCoreAPI(adder) if err != nil { return err } catterAPI, err := coreapi.NewCoreAPI(catter) if err != nil { return err } err = mn.LinkAll() if err != nil { return err } bs1 := []peer.AddrInfo{adder.Peerstore.PeerInfo(adder.Identity)} bs2 := []peer.AddrInfo{catter.Peerstore.PeerInfo(catter.Identity)} if err := catter.Bootstrap(bootstrap.BootstrapConfigWithPeers(bs1)); err != nil { return err } if err := adder.Bootstrap(bootstrap.BootstrapConfigWithPeers(bs2)); err != nil { return err } added, err := adderAPI.Unixfs().Add(ctx, files.NewBytesFile(data)) if err != nil { return err } b.StartTimer() readerCatted, err := catterAPI.Unixfs().Get(ctx, added) if err != nil { return err } // verify var bufout bytes.Buffer _, err = io.Copy(&bufout, readerCatted.(io.Reader)) if err != nil { return err } if !bytes.Equal(bufout.Bytes(), data) { return errors.New("catted data does not match added data") } return nil } ================================================ FILE: test/integration/bench_test.go ================================================ package integrationtest import ( "testing" "github.com/ipfs/kubo/thirdparty/unit" testutil "github.com/libp2p/go-libp2p-testing/net" ) func benchmarkAddCat(numBytes int64, conf testutil.LatencyConfig, b *testing.B) { b.StopTimer() b.SetBytes(numBytes) data := RandomBytes(numBytes) // we don't want to measure the time it takes to generate this data b.StartTimer() for n := 0; n < b.N; n++ { if err := DirectAddCat(data, conf); err != nil { b.Fatal(err) } } } var instant = testutil.LatencyConfig{}.AllInstantaneous() func BenchmarkInstantaneousAddCat1KB(b *testing.B) { benchmarkAddCat(1*unit.KB, instant, b) } func BenchmarkInstantaneousAddCat1MB(b *testing.B) { benchmarkAddCat(1*unit.MB, instant, b) } func BenchmarkInstantaneousAddCat2MB(b *testing.B) { benchmarkAddCat(2*unit.MB, instant, b) } func BenchmarkInstantaneousAddCat4MB(b *testing.B) { benchmarkAddCat(4*unit.MB, instant, b) } func BenchmarkInstantaneousAddCat8MB(b *testing.B) { benchmarkAddCat(8*unit.MB, instant, b) } func BenchmarkInstantaneousAddCat16MB(b *testing.B) { benchmarkAddCat(16*unit.MB, instant, b) } func BenchmarkInstantaneousAddCat32MB(b *testing.B) { benchmarkAddCat(32*unit.MB, instant, b) } func BenchmarkInstantaneousAddCat64MB(b *testing.B) { benchmarkAddCat(64*unit.MB, instant, b) } func BenchmarkInstantaneousAddCat128MB(b *testing.B) { benchmarkAddCat(128*unit.MB, instant, b) } func BenchmarkInstantaneousAddCat256MB(b *testing.B) { benchmarkAddCat(256*unit.MB, instant, b) } var routing = testutil.LatencyConfig{}.RoutingSlow() func BenchmarkRoutingSlowAddCat1MB(b *testing.B) { benchmarkAddCat(1*unit.MB, routing, b) } func BenchmarkRoutingSlowAddCat2MB(b *testing.B) { benchmarkAddCat(2*unit.MB, routing, b) } func BenchmarkRoutingSlowAddCat4MB(b *testing.B) { benchmarkAddCat(4*unit.MB, routing, b) } func BenchmarkRoutingSlowAddCat8MB(b *testing.B) { benchmarkAddCat(8*unit.MB, routing, b) } func BenchmarkRoutingSlowAddCat16MB(b *testing.B) { benchmarkAddCat(16*unit.MB, routing, b) } func BenchmarkRoutingSlowAddCat32MB(b *testing.B) { benchmarkAddCat(32*unit.MB, routing, b) } func BenchmarkRoutingSlowAddCat64MB(b *testing.B) { benchmarkAddCat(64*unit.MB, routing, b) } func BenchmarkRoutingSlowAddCat128MB(b *testing.B) { benchmarkAddCat(128*unit.MB, routing, b) } func BenchmarkRoutingSlowAddCat256MB(b *testing.B) { benchmarkAddCat(256*unit.MB, routing, b) } func BenchmarkRoutingSlowAddCat512MB(b *testing.B) { benchmarkAddCat(512*unit.MB, routing, b) } var network = testutil.LatencyConfig{}.NetworkNYtoSF() func BenchmarkNetworkSlowAddCat1MB(b *testing.B) { benchmarkAddCat(1*unit.MB, network, b) } func BenchmarkNetworkSlowAddCat2MB(b *testing.B) { benchmarkAddCat(2*unit.MB, network, b) } func BenchmarkNetworkSlowAddCat4MB(b *testing.B) { benchmarkAddCat(4*unit.MB, network, b) } func BenchmarkNetworkSlowAddCat8MB(b *testing.B) { benchmarkAddCat(8*unit.MB, network, b) } func BenchmarkNetworkSlowAddCat16MB(b *testing.B) { benchmarkAddCat(16*unit.MB, network, b) } func BenchmarkNetworkSlowAddCat32MB(b *testing.B) { benchmarkAddCat(32*unit.MB, network, b) } func BenchmarkNetworkSlowAddCat64MB(b *testing.B) { benchmarkAddCat(64*unit.MB, network, b) } func BenchmarkNetworkSlowAddCat128MB(b *testing.B) { benchmarkAddCat(128*unit.MB, network, b) } func BenchmarkNetworkSlowAddCat256MB(b *testing.B) { benchmarkAddCat(256*unit.MB, network, b) } var hdd = testutil.LatencyConfig{}.Blockstore7200RPM() func BenchmarkBlockstoreSlowAddCat1MB(b *testing.B) { benchmarkAddCat(1*unit.MB, hdd, b) } func BenchmarkBlockstoreSlowAddCat2MB(b *testing.B) { benchmarkAddCat(2*unit.MB, hdd, b) } func BenchmarkBlockstoreSlowAddCat4MB(b *testing.B) { benchmarkAddCat(4*unit.MB, hdd, b) } func BenchmarkBlockstoreSlowAddCat8MB(b *testing.B) { benchmarkAddCat(8*unit.MB, hdd, b) } func BenchmarkBlockstoreSlowAddCat16MB(b *testing.B) { benchmarkAddCat(16*unit.MB, hdd, b) } func BenchmarkBlockstoreSlowAddCat32MB(b *testing.B) { benchmarkAddCat(32*unit.MB, hdd, b) } func BenchmarkBlockstoreSlowAddCat64MB(b *testing.B) { benchmarkAddCat(64*unit.MB, hdd, b) } func BenchmarkBlockstoreSlowAddCat128MB(b *testing.B) { benchmarkAddCat(128*unit.MB, hdd, b) } func BenchmarkBlockstoreSlowAddCat256MB(b *testing.B) { benchmarkAddCat(256*unit.MB, hdd, b) } var mixed = testutil.LatencyConfig{}.NetworkNYtoSF().BlockstoreSlowSSD2014().RoutingSlow() func BenchmarkMixedAddCat1MBXX(b *testing.B) { benchmarkAddCat(1*unit.MB, mixed, b) } func BenchmarkMixedAddCat2MBXX(b *testing.B) { benchmarkAddCat(2*unit.MB, mixed, b) } func BenchmarkMixedAddCat4MBXX(b *testing.B) { benchmarkAddCat(4*unit.MB, mixed, b) } func BenchmarkMixedAddCat8MBXX(b *testing.B) { benchmarkAddCat(8*unit.MB, mixed, b) } func BenchmarkMixedAddCat16MBX(b *testing.B) { benchmarkAddCat(16*unit.MB, mixed, b) } func BenchmarkMixedAddCat32MBX(b *testing.B) { benchmarkAddCat(32*unit.MB, mixed, b) } func BenchmarkMixedAddCat64MBX(b *testing.B) { benchmarkAddCat(64*unit.MB, mixed, b) } func BenchmarkMixedAddCat128MB(b *testing.B) { benchmarkAddCat(128*unit.MB, mixed, b) } func BenchmarkMixedAddCat256MB(b *testing.B) { benchmarkAddCat(256*unit.MB, mixed, b) } ================================================ FILE: test/integration/bitswap_wo_routing_test.go ================================================ package integrationtest import ( "bytes" "testing" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/kubo/core" coremock "github.com/ipfs/kubo/core/mock" "github.com/ipfs/kubo/core/node/libp2p" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" ) func TestBitswapWithoutRouting(t *testing.T) { ctx := t.Context() const numPeers = 4 // create network mn := mocknet.New() var nodes []*core.IpfsNode for range numPeers { n, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: coremock.MockHostOption(mn), Routing: libp2p.NilRouterOption, // no routing }) if err != nil { t.Fatal(err) } defer n.Close() nodes = append(nodes, n) } err := mn.LinkAll() if err != nil { t.Fatal(err) } // connect them for _, n1 := range nodes { for _, n2 := range nodes { if n1 == n2 { continue } log.Debug("connecting to other hosts") p2 := n2.PeerHost.Peerstore().PeerInfo(n2.PeerHost.ID()) if err := n1.PeerHost.Connect(ctx, p2); err != nil { t.Fatal(err) } } } // add blocks to each before log.Debug("adding block.") block0 := blocks.NewBlock([]byte("block0")) block1 := blocks.NewBlock([]byte("block1")) // put 1 before if err := nodes[0].Blockstore.Put(ctx, block0); err != nil { t.Fatal(err) } // get it out. for i, n := range nodes { // skip first because block not in its exchange. will hang. if i == 0 { continue } log.Debugf("%d %s get block.", i, n.Identity) b, err := n.Blocks.GetBlock(ctx, cid.NewCidV0(block0.Multihash())) if err != nil { t.Error(err) } else if !bytes.Equal(b.RawData(), block0.RawData()) { t.Error("byte comparison fail") } else { log.Debug("got block: %s", b.Cid()) } } // put 1 after if err := nodes[1].Blockstore.Put(ctx, block1); err != nil { t.Fatal(err) } // get it out. for _, n := range nodes { b, err := n.Blocks.GetBlock(ctx, cid.NewCidV0(block1.Multihash())) if err != nil { t.Error(err) } else if !bytes.Equal(b.RawData(), block1.RawData()) { t.Error("byte comparison fail") } else { log.Debug("got block: %s", b.Cid()) } } } ================================================ FILE: test/integration/three_legged_cat_test.go ================================================ package integrationtest import ( "bytes" "context" "errors" "io" "math" "testing" "time" bootstrap2 "github.com/ipfs/boxo/bootstrap" "github.com/ipfs/kubo/core/coreapi" mock "github.com/ipfs/kubo/core/mock" "github.com/ipfs/kubo/thirdparty/unit" "github.com/ipfs/boxo/files" testutil "github.com/libp2p/go-libp2p-testing/net" "github.com/libp2p/go-libp2p/core/peer" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" ) func TestThreeLeggedCatTransfer(t *testing.T) { conf := testutil.LatencyConfig{ NetworkLatency: 0, RoutingLatency: 0, BlockstoreLatency: 0, } if err := RunThreeLeggedCat(RandomBytes(1*unit.MB), conf); err != nil { t.Fatal(err) } } func TestThreeLeggedCatDegenerateSlowBlockstore(t *testing.T) { SkipUnlessEpic(t) conf := testutil.LatencyConfig{BlockstoreLatency: 50 * time.Millisecond} if err := RunThreeLeggedCat(RandomBytes(1*unit.KB), conf); err != nil { t.Fatal(err) } } func TestThreeLeggedCatDegenerateSlowNetwork(t *testing.T) { SkipUnlessEpic(t) conf := testutil.LatencyConfig{NetworkLatency: 400 * time.Millisecond} if err := RunThreeLeggedCat(RandomBytes(1*unit.KB), conf); err != nil { t.Fatal(err) } } func TestThreeLeggedCatDegenerateSlowRouting(t *testing.T) { SkipUnlessEpic(t) conf := testutil.LatencyConfig{RoutingLatency: 400 * time.Millisecond} if err := RunThreeLeggedCat(RandomBytes(1*unit.KB), conf); err != nil { t.Fatal(err) } } func TestThreeLeggedCat100MBMacbookCoastToCoast(t *testing.T) { SkipUnlessEpic(t) conf := testutil.LatencyConfig{}.NetworkNYtoSF().BlockstoreSlowSSD2014().RoutingSlow() if err := RunThreeLeggedCat(RandomBytes(100*unit.MB), conf); err != nil { t.Fatal(err) } } func RunThreeLeggedCat(data []byte, conf testutil.LatencyConfig) error { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() // create network mn := mocknet.New() mn.SetLinkDefaults(mocknet.LinkOptions{ Latency: conf.NetworkLatency, // TODO add to conf. This is tricky because we want 0 values to be functional. Bandwidth: math.MaxInt32, }) bootstrap, err := mock.MockPublicNode(ctx, mn) if err != nil { return err } defer bootstrap.Close() adder, err := mock.MockPublicNode(ctx, mn) if err != nil { return err } defer adder.Close() catter, err := mock.MockPublicNode(ctx, mn) if err != nil { return err } defer catter.Close() adderAPI, err := coreapi.NewCoreAPI(adder) if err != nil { return err } catterAPI, err := coreapi.NewCoreAPI(catter) if err != nil { return err } err = mn.LinkAll() if err != nil { return err } bis := bootstrap.Peerstore.PeerInfo(bootstrap.PeerHost.ID()) bcfg := bootstrap2.BootstrapConfigWithPeers([]peer.AddrInfo{bis}) if err := adder.Bootstrap(bcfg); err != nil { return err } if err := catter.Bootstrap(bcfg); err != nil { return err } added, err := adderAPI.Unixfs().Add(ctx, files.NewBytesFile(data)) if err != nil { return err } // Explicitly provide the root CID to the DHT so the catter can discover // the adder. Without this, the async reprovider may not have propagated // the record before the catter queries. if err := adder.Routing.Provide(ctx, added.RootCid(), true); err != nil { return err } readerCatted, err := catterAPI.Unixfs().Get(ctx, added) if err != nil { return err } // verify var bufout bytes.Buffer _, err = io.Copy(&bufout, readerCatted.(io.Reader)) if err != nil { return err } if !bytes.Equal(bufout.Bytes(), data) { return errors.New("catted data does not match added data") } return nil } ================================================ FILE: test/integration/wan_lan_dht_test.go ================================================ package integrationtest import ( "context" "encoding/binary" "fmt" "math" "math/rand" "net" "testing" "time" "github.com/ipfs/go-cid" "github.com/ipfs/kubo/core" mock "github.com/ipfs/kubo/core/mock" libp2p2 "github.com/ipfs/kubo/core/node/libp2p" testutil "github.com/libp2p/go-libp2p-testing/net" corenet "github.com/libp2p/go-libp2p/core/network" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" ma "github.com/multiformats/go-multiaddr" ) func TestDHTConnectivityFast(t *testing.T) { conf := testutil.LatencyConfig{ NetworkLatency: 0, RoutingLatency: 0, BlockstoreLatency: 0, } if err := RunDHTConnectivity(conf, 5); err != nil { t.Fatal(err) } } func TestDHTConnectivitySlowNetwork(t *testing.T) { SkipUnlessEpic(t) conf := testutil.LatencyConfig{NetworkLatency: 400 * time.Millisecond} if err := RunDHTConnectivity(conf, 5); err != nil { t.Fatal(err) } } func TestDHTConnectivitySlowRouting(t *testing.T) { SkipUnlessEpic(t) conf := testutil.LatencyConfig{RoutingLatency: 400 * time.Millisecond} if err := RunDHTConnectivity(conf, 5); err != nil { t.Fatal(err) } } // wan prefix must have a real corresponding ASN for the peer diversity filter to work. var ( wanPrefix = net.ParseIP("2001:218:3004::") lanPrefix = net.ParseIP("fe80::") ) func makeAddr(n uint32, wan bool) ma.Multiaddr { var ip net.IP if wan { ip = append(net.IP{}, wanPrefix...) } else { ip = append(net.IP{}, lanPrefix...) } binary.LittleEndian.PutUint32(ip[12:], n) addr, _ := ma.NewMultiaddr(fmt.Sprintf("/ip6/%s/tcp/4242", ip)) return addr } func RunDHTConnectivity(conf testutil.LatencyConfig, numPeers int) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() // create network mn := mocknet.New() mn.SetLinkDefaults(mocknet.LinkOptions{ Latency: conf.NetworkLatency, Bandwidth: math.MaxInt32, }) testPeer, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer testPeer.Close() wanPeers := []*core.IpfsNode{} lanPeers := []*core.IpfsNode{} connectionContext, connCtxCancel := context.WithTimeout(ctx, 15*time.Second) defer connCtxCancel() for i := range numPeers { wanPeer, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Routing: libp2p2.DHTServerOption, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer wanPeer.Close() wanAddr := makeAddr(uint32(i), true) _ = wanPeer.PeerHost.Network().Listen(wanAddr) for _, p := range wanPeers { _, _ = mn.LinkPeers(p.Identity, wanPeer.Identity) _ = wanPeer.PeerHost.Connect(connectionContext, p.Peerstore.PeerInfo(p.Identity)) } wanPeers = append(wanPeers, wanPeer) lanPeer, err := core.NewNode(ctx, &core.BuildCfg{ Online: true, Host: mock.MockHostOption(mn), }) if err != nil { return err } defer lanPeer.Close() lanAddr := makeAddr(uint32(i), false) _ = lanPeer.PeerHost.Network().Listen(lanAddr) for _, p := range lanPeers { _, _ = mn.LinkPeers(p.Identity, lanPeer.Identity) _ = lanPeer.PeerHost.Connect(connectionContext, p.Peerstore.PeerInfo(p.Identity)) } lanPeers = append(lanPeers, lanPeer) } connCtxCancel() // Add interfaces / addresses to test peer. wanAddr := makeAddr(0, true) _ = testPeer.PeerHost.Network().Listen(wanAddr) lanAddr := makeAddr(0, false) _ = testPeer.PeerHost.Network().Listen(lanAddr) // The test peer is connected to one lan peer. for _, p := range lanPeers { if _, err := mn.LinkPeers(testPeer.Identity, p.Identity); err != nil { return err } } err = testPeer.PeerHost.Connect(ctx, lanPeers[0].Peerstore.PeerInfo(lanPeers[0].Identity)) if err != nil { return err } startupCtx, startupCancel := context.WithTimeout(ctx, time.Second*60) StartupWait: for { select { case err := <-testPeer.DHT.LAN.RefreshRoutingTable(): if err != nil { fmt.Printf("Error refreshing routing table: %v\n", err) } if testPeer.DHT.LAN.RoutingTable() == nil || testPeer.DHT.LAN.RoutingTable().Size() == 0 || err != nil { time.Sleep(100 * time.Millisecond) continue } break StartupWait case <-startupCtx.Done(): startupCancel() return fmt.Errorf("expected faster dht bootstrap") } } startupCancel() // choose a lan peer and validate lan DHT is functioning. i := rand.Intn(len(lanPeers)) if testPeer.PeerHost.Network().Connectedness(lanPeers[i].Identity) == corenet.Connected { i = (i + 1) % len(lanPeers) if testPeer.PeerHost.Network().Connectedness(lanPeers[i].Identity) == corenet.Connected { _ = testPeer.PeerHost.Network().ClosePeer(lanPeers[i].Identity) testPeer.PeerHost.Peerstore().ClearAddrs(lanPeers[i].Identity) } } // That peer will provide a new CID, and we'll validate the test node can find it. provideCid := cid.NewCidV1(cid.Raw, []byte("Lan Provide Record")) provideCtx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() if err := lanPeers[i].DHT.Provide(provideCtx, provideCid, true); err != nil { return err } provChan := testPeer.DHT.FindProvidersAsync(provideCtx, provideCid, 0) prov, ok := <-provChan if !ok || prov.ID == "" { return fmt.Errorf("Expected provider. stream closed early") } if prov.ID != lanPeers[i].Identity { return fmt.Errorf("Unexpected lan peer provided record") } // Now, connect with a wan peer. for _, p := range wanPeers { if _, err := mn.LinkPeers(testPeer.Identity, p.Identity); err != nil { return err } } err = testPeer.PeerHost.Connect(ctx, wanPeers[0].Peerstore.PeerInfo(wanPeers[0].Identity)) if err != nil { return err } startupCtx, startupCancel = context.WithTimeout(ctx, time.Second*60) WanStartupWait: for { select { case err := <-testPeer.DHT.WAN.RefreshRoutingTable(): // if err != nil { // fmt.Printf("Error refreshing routing table: %v\n", err) // } if testPeer.DHT.WAN.RoutingTable() == nil || testPeer.DHT.WAN.RoutingTable().Size() == 0 || err != nil { time.Sleep(100 * time.Millisecond) continue } break WanStartupWait case <-startupCtx.Done(): startupCancel() return fmt.Errorf("expected faster wan dht bootstrap") } } startupCancel() // choose a wan peer and validate wan DHT is functioning. i = rand.Intn(len(wanPeers)) if testPeer.PeerHost.Network().Connectedness(wanPeers[i].Identity) == corenet.Connected { i = (i + 1) % len(wanPeers) if testPeer.PeerHost.Network().Connectedness(wanPeers[i].Identity) == corenet.Connected { _ = testPeer.PeerHost.Network().ClosePeer(wanPeers[i].Identity) testPeer.PeerHost.Peerstore().ClearAddrs(wanPeers[i].Identity) } } // That peer will provide a new CID, and we'll validate the test node can find it. wanCid := cid.NewCidV1(cid.Raw, []byte("Wan Provide Record")) wanProvideCtx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() if err := wanPeers[i].DHT.Provide(wanProvideCtx, wanCid, true); err != nil { return err } provChan = testPeer.DHT.FindProvidersAsync(wanProvideCtx, wanCid, 0) prov, ok = <-provChan if !ok || prov.ID == "" { return fmt.Errorf("Expected one provider, closed early") } if prov.ID != wanPeers[i].Identity { return fmt.Errorf("Unexpected lan peer provided record") } // Finally, re-share the lan provided cid from a wan peer and expect a merged result. i = rand.Intn(len(wanPeers)) if testPeer.PeerHost.Network().Connectedness(wanPeers[i].Identity) == corenet.Connected { _ = testPeer.PeerHost.Network().ClosePeer(wanPeers[i].Identity) testPeer.PeerHost.Peerstore().ClearAddrs(wanPeers[i].Identity) } provideCtx, cancel = context.WithTimeout(ctx, time.Second) defer cancel() if err := wanPeers[i].DHT.Provide(provideCtx, provideCid, true); err != nil { return err } provChan = testPeer.DHT.FindProvidersAsync(provideCtx, provideCid, 0) prov, ok = <-provChan if !ok { return fmt.Errorf("Expected two providers, got 0") } prov, ok = <-provChan if !ok { return fmt.Errorf("Expected two providers, got 1") } return nil } ================================================ FILE: test/ipfs-test-lib.sh ================================================ # Generic test functions for go-ipfs ansi_strip() { sed 's/\x1b\[[0-9;]*m//g' } # Quote arguments for sh eval shellquote() { _space='' for _arg do # On macOS, sed adds a newline character. # With a printf wrapper the extra newline is removed. printf "$_space'%s'" "$(printf "%s" "$_arg" | sed -e "s/'/'\\\\''/g;")" _space=' ' done printf '\n' } # Echo the args, run the cmd, and then also fail, # making sure a test case fails. test_fsh() { echo "> $@" eval $(shellquote "$@") echo "" false } # Same as sharness' test_cmp but using test_fsh (to see the output). # We have to do it twice, so the first diff output doesn't show unless it's # broken. test_cmp() { diff -q "$@" >/dev/null || test_fsh diff -u "$@" } # Same as test_cmp above, but we sort files before comparing them. test_sort_cmp() { sort "$1" >"$1_sorted" && sort "$2" >"$2_sorted" && test_cmp "$1_sorted" "$2_sorted" } # Same as test_cmp above, but we standardize directory # separators before comparing the files. test_path_cmp() { sed -e "s/\\\\/\//g" "$1" >"$1_std" && sed -e "s/\\\\/\//g" "$2" >"$2_std" && test_cmp "$1_std" "$2_std" } # Docker # This takes a Dockerfile, a tag name, and a build context directory docker_build() { docker build --rm --tag "$1" --file "$2" "$3" | ansi_strip } # This takes an image as argument and writes a docker ID on stdout docker_run() { docker run --detach "$1" } # This takes a docker ID and a command as arguments docker_exec() { docker exec --tty "$1" /bin/sh -c "$2" } # This takes a docker ID as argument docker_stop() { docker stop "$1" } # This takes a docker ID as argument docker_rm() { docker rm --force --volumes "$1" > /dev/null } # This takes a docker image name as argument docker_rmi() { docker rmi --force "$1" > /dev/null } # Test whether all the expected lines are included in a file. The file # can have extra lines. # # $1 - Path to file with expected lines. # $2 - Path to file with actual output. # # Examples # # test_expect_success 'foo says hello' ' # echo hello >expected && # foo >actual && # test_cmp expected actual # ' # # Returns the exit code of the command set by TEST_CMP. test_includes_lines() { sort "$1" >"$1_sorted" && sort "$2" >"$2_sorted" && comm -2 -3 "$1_sorted" "$2_sorted" >"$2_missing" && [ ! -s "$2_missing" ] || test_fsh comm -2 -3 "$1_sorted" "$2_sorted" } # Depending on GNU seq availability is not nice. # Git also has test_seq but it uses Perl. test_seq() { test "$1" -le "$2" || return i="$1" j="$2" while test "$i" -le "$j" do echo "$i" i=$(expr "$i" + 1) done } b64decode() { case `uname` in Linux|FreeBSD) base64 -d ;; Darwin) base64 -D ;; *) echo "no compatible base64 command found" >&2 return 1 esac } ================================================ FILE: test/sharness/.gitignore ================================================ # symlinks to lib/sharness /sharness.sh /lib-sharness # clone of sharness lib/sharness/ # deps downloaded by lib/*.sh scripts lib/dependencies/ # sharness files test-results/ trash directory.*.sh/ # makefile files plugins # macos files *.DS_Store ================================================ FILE: test/sharness/GNUmakefile ================================================ # default target is to run all tests all: aggregate SH := $(wildcard t[0-9][0-9][0-9][0-9]-*.sh) .DEFAULT $(SH): ALWAYS $(MAKE) -C ../.. test/sharness/$@ ALWAYS: .PHONY: ALWAYS ================================================ FILE: test/sharness/README.md ================================================ # ipfs whole tests using the [sharness framework](https://github.com/pl-strflt/sharness/tree/feat/junit) ## Running all the tests Just use `make` in this directory to run all the tests. Run with `TEST_VERBOSE=1` to get helpful verbose output. ``` TEST_VERBOSE=1 make ``` The usual ipfs env flags also apply: ```sh # the output will make your eyes bleed GOLOG_LOG_LEVEL=debug TEST_VERBOSE=1 make ``` To make the tests abort as soon as an error occurs, use the TEST_IMMEDIATE env variable: ```sh # this will abort as soon the first error occurs TEST_IMMEDIATE=1 make ``` ## Running just one test You can run only one test script by launching it like a regular shell script: ``` $ ./t0010-basic-commands.sh ``` ## Debugging one test You can use the `-v` option to make it verbose and the `-i` option to make it stop as soon as one test fails. For example: ``` $ ./t0010-basic-commands.sh -v -i ``` ## Sharness When running sharness tests from main Makefile or when `test_sharness_deps` target is run dependencies for sharness will be downloaded from its GitHub repo and installed in a "lib/sharness" directory. Please do not change anything in the "lib/sharness" directory. If you really need some changes in sharness, please fork it from [its canonical repo](https://github.com/mlafeldt/sharness/) and send pull requests there. ## Writing Tests Please have a look at existing tests and try to follow their example. When possible and not too inefficient, that means most of the time, an ipfs command should not be on the left side of a pipe, because if the ipfs command fails (exit non zero), the pipe will mask this failure. For example after `false | true`, `echo $?` prints 0 (despite `false` failing). It should be possible to put most of the code inside `test_expect_success`, or sometimes `test_expect_failure`, blocks, and to chain all the commands inside those blocks with `&&`, or `||` for diagnostic commands. ### Diagnostics Make your test case output helpful for when running sharness verbosely. This means cating certain files, or running diagnostic commands. For example: ``` test_expect_success ".ipfs/ has been created" ' test -d ".ipfs" && test -f ".ipfs/config" && test -d ".ipfs/datastore" && test -d ".ipfs/blocks" || test_fsh ls -al .ipfs ' ``` The `|| ...` is a diagnostic run when the preceding command fails. test_fsh is a shell function that echoes the args, runs the cmd, and then also fails, making sure the test case fails. (wouldn't want the diagnostic accidentally returning true and making it _seem_ like the test case succeeded!). ### Testing commands on daemon or mounted Use the provided functions in `lib/test-lib.sh` to run the daemon or mount: To init, run daemon, and mount in one go: ```sh test_launch_ipfs_daemon_and_mount test_expect_success "'ipfs add --help' succeeds" ' ipfs add --help >actual ' # other tests here... # don't forget to kill the daemon!! test_kill_ipfs_daemon ``` To init, run daemon, and then mount separately: ```sh test_init_ipfs # tests inited but not running here test_launch_ipfs_daemon # tests running but not mounted here test_mount_ipfs # tests mounted here # don't forget to kill the daemon!! test_kill_ipfs_daemon ``` ================================================ FILE: test/sharness/Rules.mk ================================================ include mk/header.mk SHARNESS_$(d) = $(d)/lib/sharness/sharness.sh T_$(d) = $(sort $(wildcard $(d)/t[0-9][0-9][0-9][0-9]-*.sh)) DEPS_$(d) := test/bin/multihash test/bin/pollEndpoint test/bin/iptb \ test/bin/go-sleep test/bin/random-data test/bin/random-files \ test/bin/go-timeout test/bin/hang-fds test/bin/ma-pipe-unidir \ test/bin/cid-fmt DEPS_$(d) += cmd/ipfs/ipfs DEPS_$(d) += $(d)/clean-test-results DEPS_$(d) += $(SHARNESS_$(d)) ifeq ($(OS),Linux) PLUGINS_DIR_$(d) := $(d)/plugins/ ORIGIN_PLUGINS_$(d) := $(plugin/plugins_plugins_so) PLUGINS_$(d) := $(addprefix $(PLUGINS_DIR_$(d)),$(notdir $(ORIGIN_PLUGINS_$(d)))) $(PLUGINS_$(d)): $(ORIGIN_PLUGINS_$(d)) @mkdir -p $(@D) cp -f plugin/plugins/$(@F) $@ ifneq ($(TEST_PLUGIN),0) DEPS_$(d) += $(PLUGINS_$(d)) endif endif export MAKE_SKIP_PATH=1 $(T_$(d)): $$(DEPS_$(d)) # use second expansion so coverage can inject dependency @echo "*** $@ ***" ifeq ($(CONTINUE_ON_S_FAILURE),1) -@(cd $(@D) && ./$(@F)) 2>&1 else @(cd $(@D) && ./$(@F)) 2>&1 endif .PHONY: $(T_$(d)) $(d)/aggregate: $(T_$(d)) @echo "*** $@ ***" @(cd $(@D) && ./lib/test-aggregate-results.sh) .PHONY: $(d)/aggregate $(d)/test-results/sharness.xml: $(T_$(d)) @echo "*** $@ ***" @(cd $(@D)/.. && ./lib/test-aggregate-junit-reports.sh) .PHONY: $(d)/test-results/sharness.xml $(d)/clean-test-results: rm -rf $(@D)/test-results .PHONY: $(d)/clean-test-results CLEAN += $(wildcard $(d)/test-results/*) $(SHARNESS_$(d)): $(d) ALWAYS @clonedir=$(dir $(@D)) $&2 "$@" exit 1 } if test -d "$clonedir/$sharnessdir"; then giturl="git@github.com:${gitrepo}.git" echo "Checking if $giturl is already cloned (and if its origin is correct)" if ! test -d "$gitdir" || test "$(git --git-dir "$gitdir" remote get-url origin)" != "$giturl"; then echo "Removing $clonedir/$sharnessdir" rm -rf "$clonedir/$sharnessdir" || die "Could not remove $clonedir/$sharnessdir" fi fi if ! test -d "$clonedir/$sharnessdir"; then giturl="https://github.com/${gitrepo}.git" echo "Cloning $giturl into $clonedir/$sharnessdir" git clone "$giturl" "$clonedir/$sharnessdir" || die "Could not clone $giturl into $clonedir/$sharnessdir" fi echo "Changing directory to $clonedir/$sharnessdir" cd "$clonedir/$sharnessdir" || die "Could not cd into '$clonedir/$sharnessdir' directory" echo "Checking if $githash is already fetched" if ! git show "$githash" >/dev/null 2>&1; then echo "Fetching $githash" git fetch origin "$githash" || die "Could not fetch $githash" fi echo "Resetting to $githash" git reset --hard "$githash" || die "Could not reset to $githash" exit 0 ================================================ FILE: test/sharness/lib/iptb-lib.sh ================================================ # iptb test framework # # Copyright (c) 2014, 2016 Jeromy Johnson, Christian Couder # MIT Licensed; see the LICENSE file in this repository. export IPTB_ROOT="$(pwd)/.iptb" ipfsi() { dir="$1" shift IPFS_PATH="$IPTB_ROOT/testbeds/default/$dir" ipfs "$@" } check_has_connection() { node="$1" ipfsi "$node" swarm peers >"swarm_peers_$node" && grep "p2p" "swarm_peers_$node" >/dev/null } iptb() { if ! command iptb "$@"; then case "$1" in start|stop|connect) test_fsh command iptb logs ;; esac return 1 fi } startup_cluster() { num_nodes="$1" shift other_args="$@" bound=$(expr "$num_nodes" - 1) test_expect_success "set Routing.LoopbackAddressesOnLanDHT to true" ' iptb run [0-$bound] -- ipfs config --json "Routing.LoopbackAddressesOnLanDHT" true ' if test -n "$other_args"; then test_expect_success "start up nodes with additional args" " iptb start -wait [0-$bound] -- ${other_args[@]} " else test_expect_success "start up nodes" ' iptb start -wait [0-$bound] ' fi test_expect_success "connect nodes to each other" ' iptb connect [1-$bound] 0 ' for i in $(test_seq 0 "$bound") do test_expect_success "node $i is connected" ' check_has_connection "$i" || test_fsh cat "swarm_peers_$i" ' done } iptb_wait_stop() { while ! iptb run -- sh -c '! { test -e "$IPFS_PATH/repo.lock" && fuser -f "$IPFS_PATH/repo.lock" >/dev/null; }'; do go-sleep 10ms done } ================================================ FILE: test/sharness/lib/test-aggregate-junit-reports.sh ================================================ #!/bin/sh # # Script to aggregate results using Sharness # # Copyright (c) 2014, 2022 Christian Couder, Piotr Galar # MIT Licensed; see the LICENSE file in this repository. # SHARNESS_AGGREGATE_JUNIT="lib/sharness/aggregate-junit-reports.sh" test -f "$SHARNESS_AGGREGATE_JUNIT" || { echo >&2 "Cannot find: $SHARNESS_AGGREGATE_JUNIT" echo >&2 "Please check Sharness installation." exit 1 } ls test-results/t*-*.sh.*.xml.part | "$SHARNESS_AGGREGATE_JUNIT" > test-results/sharness.xml ================================================ FILE: test/sharness/lib/test-aggregate-results.sh ================================================ #!/bin/sh # # Script to aggregate results using Sharness # # Copyright (c) 2014 Christian Couder # MIT Licensed; see the LICENSE file in this repository. # SHARNESS_AGGREGATE="lib/sharness/aggregate-results.sh" test -f "$SHARNESS_AGGREGATE" || { echo >&2 "Cannot find: $SHARNESS_AGGREGATE" echo >&2 "Please check Sharness installation." exit 1 } ls test-results/t*-*.sh.*.counts | "$SHARNESS_AGGREGATE" ================================================ FILE: test/sharness/lib/test-lib-hashes.sh ================================================ # this file defines several useful hashes used across the test codebase. # thus they can be defined + changed in one place HASH_WELCOME_DOCS="QmQPeNsJPyVWPFDVHb77w8G42Fvo15z4bG2X8D2GhfbSXc" HASH_EMPTY_DIR="QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" ================================================ FILE: test/sharness/lib/test-lib.sh ================================================ # Test framework for go-ipfs # # Copyright (c) 2014 Christian Couder # MIT Licensed; see the LICENSE file in this repository. # # We are using sharness (https://github.com/pl-strflt/sharness/tree/feat/junit) # which was extracted from the Git test framework. # use the ipfs tool to test against # add current directory to path, for ipfs tool. if test "$MAKE_SKIP_PATH" != "1"; then BIN=$(cd .. && echo `pwd`/bin) BIN2=$(cd ../.. && echo `pwd`/cmd/ipfs) PATH=${BIN2}:${BIN}:${PATH} # assert the `ipfs` we're using is the right one. if test `which ipfs` != ${BIN2}/ipfs; then echo >&2 "Cannot find the tests' local ipfs tool." echo >&2 "Please check test and ipfs tool installation." exit 1 fi fi # set sharness verbosity. we set the env var directly as # it's too late to pass in --verbose, and --verbose is harder # to pass through in some cases. test "$TEST_VERBOSE" = 1 && verbose=t test "$TEST_IMMEDIATE" = 1 && immediate=t test "$TEST_JUNIT" = 1 && junit=t test "$TEST_NO_COLOR" = 1 && no_color=t # source the common hashes first. . lib/test-lib-hashes.sh ln -sf lib/sharness/sharness.sh . ln -sf lib/sharness/lib-sharness . . "sharness.sh" || { echo >&2 "Cannot source: sharness.sh" echo >&2 "Please check Sharness installation." exit 1 } # Please put go-ipfs specific shell functions below ### # BEGIN Check for pre-existing daemon being stuck ### wait_prev_cleanup_tick_secs=1 wait_prev_cleanup_max_secs=5 cur_test_pwd="$(pwd)" while true ; do echo -n > stuck_cwd_list timeout 5 lsof -c ipfs -Ffn 2>/dev/null | grep -A1 '^fcwd$' | grep '^n' | cut -b 2- | while read -r pwd_of_stuck ; do case "$pwd_of_stuck" in "$cur_test_pwd"*) echo "$pwd_of_stuck" >> stuck_cwd_list ;; *) ;; esac done test -s stuck_cwd_list || break test "$wait_prev_cleanup_max_secs" -le 0 && break echo "Daemons still running, waiting for ${wait_prev_cleanup_max_secs}s" sleep $wait_prev_cleanup_tick_secs wait_prev_cleanup_max_secs="$(( $wait_prev_cleanup_max_secs - $wait_prev_cleanup_tick_secs ))" done if test -s stuck_cwd_list ; then test_expect_success "ipfs daemon (s)seems to be running with CWDs of $(cat stuck_cwd_list) Almost certainly a leftover from a prior test, ABORTING" 'false' test_done fi ### # END Check for pre-existing daemon being stuck ### # Make sure the ipfs path is set, also set in test_init_ipfs but that # is not always used. export IPFS_PATH="$(pwd)/.ipfs" # Ask programs to please not print ANSI codes export TERM=dumb TEST_OS="$(uname -s | tr '[a-z]' '[A-Z]')" # grab + output options test "$TEST_FUSE" = 1 && test_set_prereq FUSE test "$TEST_EXPENSIVE" = 1 && test_set_prereq EXPENSIVE test "$TEST_DOCKER" = 1 && type docker >/dev/null 2>&1 && groups | egrep "\bdocker\b" && test_set_prereq DOCKER test "$TEST_PLUGIN" = 1 && test "$TEST_OS" = "LINUX" && test_set_prereq PLUGIN # this may not be available, skip a few dependent tests type socat >/dev/null 2>&1 && test_set_prereq SOCAT type unzip >/dev/null 2>&1 && test_set_prereq UNZIP # Set a prereq as error messages are often different on Windows/Cygwin expr "$TEST_OS" : "CYGWIN_NT" >/dev/null || test_set_prereq STD_ERR_MSG if test "$TEST_VERBOSE" = 1; then echo '# TEST_VERBOSE='"$TEST_VERBOSE" echo '# TEST_IMMEDIATE='"$TEST_IMMEDIATE" echo '# TEST_FUSE='"$TEST_FUSE" echo '# TEST_DOCKER='"$TEST_DOCKER" echo '# TEST_PLUGIN='"$TEST_PLUGIN" echo '# TEST_EXPENSIVE='"$TEST_EXPENSIVE" echo '# TEST_OS='"$TEST_OS" echo '# TEST_JUNIT='"$TEST_JUNIT" echo '# TEST_NO_COLOR='"$TEST_NO_COLOR" echo '# TEST_ULIMIT_PRESET='"$TEST_ULIMIT_PRESET" fi # source our generic test lib . ../../ipfs-test-lib.sh # source iptb lib . ../lib/iptb-lib.sh test_cmp_repeat_10_sec() { for i in $(test_seq 1 100) do test_cmp "$1" "$2" >/dev/null && return go-sleep 100ms done test_cmp "$1" "$2" } test_run_repeat_60_sec() { for i in $(test_seq 1 600) do (test_eval_ "$1") && return go-sleep 100ms done return 1 # failed } test_wait_output_n_lines() { for i in $(test_seq 1 3600) do test $(cat "$1" | wc -l | tr -d " ") -ge $2 && return go-sleep 100ms done actual=$(cat "$1" | wc -l | tr -d " ") test_fsh "expected $2 lines of output. got $actual" } test_wait_open_tcp_port_10_sec() { for i in $(test_seq 1 100) do # this is not a perfect check, but it's portable. # can't count on ss. not installed everywhere. # can't count on netstat using : or . as port delim. differ across platforms. echo $(netstat -aln | egrep "^tcp.*LISTEN" | egrep "[.:]$1" | wc -l) -gt 0 if [ $(netstat -aln | egrep "^tcp.*LISTEN" | egrep "[.:]$1" | wc -l) -gt 0 ]; then return 0 fi go-sleep 100ms done return 1 } # test_config_set helps us make sure _we really did set_ a config value. # it sets it and then tests it. This became elaborate because ipfs config # was setting really weird things and am not sure why. test_config_set() { # grab flags (like --bool in "ipfs config --bool") test_cfg_flags="" # unset in case. test "$#" = 3 && { test_cfg_flags=$1; shift; } test_cfg_key=$1 test_cfg_val=$2 # when verbose, tell the user what config values are being set test_cfg_cmd="ipfs config $test_cfg_flags \"$test_cfg_key\" \"$test_cfg_val\"" test "$TEST_VERBOSE" = 1 && echo "$test_cfg_cmd" # ok try setting the config key/val pair. ipfs config $test_cfg_flags "$test_cfg_key" "$test_cfg_val" echo "$test_cfg_val" >cfg_set_expected ipfs config "$test_cfg_key" >cfg_set_actual test_cmp cfg_set_expected cfg_set_actual } test_init_ipfs() { args=("$@") # we set the Addresses.API config variable. # the cli client knows to use it, so only need to set. # todo: in the future, use env? test_expect_success "ipfs init succeeds" ' export IPFS_PATH="$(pwd)/.ipfs" && ipfs init "${args[@]}" --profile=test > /dev/null ' test_expect_success "disable telemetry" ' test_config_set --bool Plugins.Plugins.telemetry.Disabled "true" ' test_expect_success "prepare config -- mounting" ' mkdir mountdir ipfs ipns mfs && test_config_set Mounts.IPFS "$(pwd)/ipfs" && test_config_set Mounts.IPNS "$(pwd)/ipns" && test_config_set Mounts.MFS "$(pwd)/mfs" || test_fsh cat "\"$IPFS_PATH/config\"" ' } test_init_ipfs_measure() { args=("$@") # we set the Addresses.API config variable. # the cli client knows to use it, so only need to set. # todo: in the future, use env? test_expect_success "ipfs init succeeds" ' export IPFS_PATH="$(pwd)/.ipfs" && ipfs init "${args[@]}" --profile=test,flatfs-measure > /dev/null ' test_expect_success "disable telemetry" ' test_config_set --bool Plugins.Plugins.telemetry.Disabled "true" ' test_expect_success "prepare config -- mounting" ' mkdir mountdir ipfs ipns && test_config_set Mounts.IPFS "$(pwd)/ipfs" && test_config_set Mounts.IPNS "$(pwd)/ipns" || test_fsh cat "\"$IPFS_PATH/config\"" ' } test_wait_for_file() { loops=$1 delay=$2 file=$3 fwaitc=0 while ! test -f "$file" do if test $fwaitc -ge $loops then echo "Error: timed out waiting for file: $file" return 1 fi go-sleep $delay fwaitc=`expr $fwaitc + 1` done } test_set_address_vars() { daemon_output="$1" test_expect_success "set up address variables" ' API_MADDR=$(cat "$IPFS_PATH/api") && API_ADDR=$(convert_tcp_maddr $API_MADDR) && API_PORT=$(port_from_maddr $API_MADDR) && GWAY_MADDR=$(sed -n "s/^Gateway server listening on //p" "$daemon_output") && GWAY_ADDR=$(convert_tcp_maddr $GWAY_MADDR) && GWAY_PORT=$(port_from_maddr $GWAY_MADDR) ' if ipfs swarm addrs local >/dev/null 2>&1; then test_expect_success "get swarm addresses" ' ipfs swarm addrs local > addrs_out ' test_expect_success "set swarm address vars" ' SWARM_MADDR=$(grep "127.0.0.1" addrs_out) && SWARM_PORT=$(port_from_maddr $SWARM_MADDR) ' fi } test_launch_ipfs_daemon() { args=("$@") test "$TEST_ULIMIT_PRESET" != 1 && ulimit -n 2048 test_expect_success "'ipfs daemon' succeeds" ' ipfs daemon "${args[@]}" >actual_daemon 2>daemon_err & IPFS_PID=$! ' # wait for api file to show up test_expect_success "api file shows up" ' test_wait_for_file 50 200ms "$IPFS_PATH/api" ' test_set_address_vars actual_daemon # we say the daemon is ready when the API server is ready. test_expect_success "'ipfs daemon' is ready" ' pollEndpoint -host=$API_MADDR -v -tout=1s -tries=60 2>poll_apierr > poll_apiout || test_fsh cat actual_daemon || test_fsh cat daemon_err || test_fsh cat poll_apierr || test_fsh cat poll_apiout ' } test_launch_ipfs_daemon_without_network() { test_launch_ipfs_daemon --offline "$@" } do_umount() { local mount_point="$1" local max_retries=3 local retry_delay=0.5 # Try normal unmount first (without lazy flag) for i in $(seq 1 $max_retries); do if [ "$(uname -s)" = "Linux" ]; then # First attempt: standard unmount if fusermount -u "$mount_point" 2>/dev/null; then return 0 fi else if umount "$mount_point" 2>/dev/null; then return 0 fi fi # If not last attempt, wait before retry if [ $i -lt $max_retries ]; then go-sleep "${retry_delay}s" fi done # If normal unmount failed, try lazy unmount as last resort (Linux only) if [ "$(uname -s)" = "Linux" ]; then # Log that we're falling back to lazy unmount test "$TEST_VERBOSE" = 1 && echo "# Warning: falling back to lazy unmount for $mount_point" fusermount -z -u "$mount_point" 2>/dev/null else # On non-Linux, try force unmount umount -f "$mount_point" 2>/dev/null || true fi } test_mount_ipfs() { # make sure stuff is unmounted first. test_expect_success FUSE "'ipfs mount' succeeds" ' do_umount "$(pwd)/ipfs" || true && do_umount "$(pwd)/ipns" || true && do_umount "$(pwd)/mfs" || true && ipfs mount >actual ' test_expect_success FUSE "'ipfs mount' output looks good" ' echo "IPFS mounted at: $(pwd)/ipfs" >expected && echo "IPNS mounted at: $(pwd)/ipns" >>expected && echo "MFS mounted at: $(pwd)/mfs" >>expected && test_cmp expected actual ' } test_launch_ipfs_daemon_and_mount() { test_init_ipfs test_launch_ipfs_daemon test_mount_ipfs } test_kill_repeat_10_sec() { # try to shut down once + wait for graceful exit kill $1 for i in $(test_seq 1 100) do go-sleep 100ms ! kill -0 $1 2>/dev/null && return done # if not, try once more, which will skip graceful exit kill $1 go-sleep 1s ! kill -0 $1 2>/dev/null && return # ok, no hope. kill it to prevent it messing with other tests kill -9 $1 2>/dev/null return 1 } test_kill_ipfs_daemon() { test_expect_success "'ipfs daemon' is still running" ' kill -0 $IPFS_PID ' test_expect_success "'ipfs daemon' can be killed" ' test_kill_repeat_10_sec $IPFS_PID ' } test_curl_resp_http_code() { curl -I "$1" >curl_output || { echo "curl error with url: '$1'" echo "curl output was:" cat curl_output return 1 } shift && RESP=$(head -1 curl_output) && while test "$#" -gt 0 do expr "$RESP" : "$1" >/dev/null && return shift done echo "curl response didn't match!" echo "curl response was: '$RESP'" echo "curl output was:" cat curl_output return 1 } test_must_be_empty() { if test -s "$1" then echo "'$1' is not empty, it contains:" cat "$1" return 1 fi } test_should_contain() { test "$#" = 2 || error "bug in the test script: not 2 parameters to test_should_contain" if ! grep -q "$1" "$2" then echo "'$2' does not contain '$1', it contains:" cat "$2" return 1 fi } test_should_not_contain() { test "$#" = 2 || error "bug in the test script: not 2 parameters to test_should_not_contain" if grep -q "$1" "$2" then echo "'$2' contains undesired value '$1'" return 1 fi } test_str_contains() { find=$1 shift echo "$@" | egrep "\b$find\b" >/dev/null } disk_usage() { # normalize du across systems case $(uname -s) in Linux) DU="du -sb" M=1 ;; FreeBSD) DU="du -s -A -B 1" M=512 ;; Darwin | DragonFly | *) DU="du -s" M=512 ;; esac expr $($DU "$1" | awk "{print \$1}") "*" "$M" } # output a file's permission in human readable format generic_stat() { # normalize stat across systems case $(uname -s) in Linux) _STAT="stat -c %A" ;; FreeBSD | Darwin | DragonFly) _STAT="stat -f %Sp" ;; *) echo "unsupported OS" >&2 exit 1 ;; esac $_STAT "$1" || echo "failed" # Avoid returning nothing. } # output a file's permission in human readable format file_size() { case $(uname -s) in Linux) _STAT="stat --format=%s" ;; FreeBSD | Darwin | DragonFly) _STAT="stat -f%z" ;; *) echo "unsupported OS" >&2 exit 1 ;; esac $_STAT "$1" } # len 46: 2048-bit RSA keys, b58mh-encoded # len 52: ED25519 keys, b58mh-encoded # len 56: 2048-bit RSA keys, base36-encoded # len 62: ED25519 keys, base36-encoded test_check_peerid() { peeridlen=$(echo "$1" | tr -dC "[:alnum:]" | wc -c | tr -d " ") && test "$peeridlen" = "46" -o "$peeridlen" = "52" -o "$peeridlen" = "56" -o "$peeridlen" = "62" || { echo "Bad peerid '$1' with len '$peeridlen'" return 1 } } test_check_rsa2048_b58mh_peerid() { peeridlen=$(echo "$1" | tr -dC "[:alnum:]" | wc -c | tr -d " ") && test "$peeridlen" = "46" || { echo "Bad RSA2048 B58MH peerid '$1' with len '$peeridlen'" return 1 } } test_check_ed25519_b58mh_peerid() { peeridlen=$(echo "$1" | tr -dC "[:alnum:]" | wc -c | tr -d " ") && test "$peeridlen" = "52" || { echo "Bad ED25519 B58MH peerid '$1' with len '$peeridlen'" return 1 } } test_check_rsa2048_base36_peerid() { peeridlen=$(echo "$1" | tr -dC "[:alnum:]" | wc -c | tr -d " ") && test "$peeridlen" = "56" || { echo "Bad RSA2048 B36CID peerid '$1' with len '$peeridlen'" return 1 } } test_check_ed25519_base36_peerid() { peeridlen=$(echo "$1" | tr -dC "[:alnum:]" | wc -c | tr -d " ") && test "$peeridlen" = "62" || { echo "Bad ED25519 B36CID peerid '$1' with len '$peeridlen'" return 1 } } convert_tcp_maddr() { echo $1 | awk -F'/' '{ printf "%s:%s", $3, $5 }' } port_from_maddr() { echo $1 | awk -F'/' '{ print $NF }' } findprovs_empty() { test_expect_success 'findprovs '$1' succeeds' ' ipfsi 1 routing findprovs -n 1 '$1' > findprovsOut ' test_expect_success "findprovs $1 output is empty" ' test_must_be_empty findprovsOut ' } findprovs_expect() { test_expect_success 'findprovs '$1' succeeds' ' ipfsi 1 routing findprovs -n 1 '$1' > findprovsOut && echo '$2' > expected ' test_expect_success "findprovs $1 output looks good" ' test_cmp findprovsOut expected ' } purge_blockstore() { ipfs pin ls --quiet --type=recursive | ipfs pin rm &>/dev/null ipfs repo gc --silent &>/dev/null test_expect_success "pinlist empty" ' [[ -z "$( ipfs pin ls )" ]] ' test_expect_success "nothing left to gc" ' [[ -z "$( ipfs repo gc )" ]] ' } ================================================ FILE: test/sharness/t0001-tests-work.sh ================================================ #!/usr/bin/env bash test_description="Test sharness tests are correctly written" . lib/test-lib.sh for file in $(find .. -maxdepth 1 -name 't*.sh' -type f); do test_expect_success "test in $file finishes" ' grep -q "^test_done\b" "$file" ' test_expect_success "test in $file has a description" ' grep -q "^test_description=" "$file" ' # We have some tests that manually kill. case "$(basename "$file")" in t0060-daemon.sh|t0023-shutdown.sh) continue ;; esac test_expect_success "test in $file has matching ipfs start/stop" ' awk "/^ *[^#]*test_launch_ipfs_daemon/ { if (count != 0) { exit(1) }; count++ } /^ *[^#]*test_kill_ipfs_daemon/ { if (count != 1) { exit(1) }; count-- } END { exit(count) }" "$file" ' done test_done ================================================ FILE: test/sharness/t0002-docker-image.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Christian Couder # MIT Licensed; see the LICENSE file in this repository. # test_description="Test docker image" . lib/test-lib.sh # if in travis CI on OSX, docker is not available if ! test_have_prereq DOCKER; then skip_all='skipping docker tests, docker not available' test_done fi test_expect_success "'docker --version' works" ' docker --version >actual ' test_expect_success "'docker --version' output looks good" ' egrep "^Docker version" actual ' TEST_TRASH_DIR=$(pwd) TEST_SCRIPTS_DIR=$(dirname "$TEST_TRASH_DIR") TEST_TESTS_DIR=$(dirname "$TEST_SCRIPTS_DIR") APP_ROOT_DIR=$(dirname "$TEST_TESTS_DIR") IMAGE_TAG=kubo_test test_expect_success "docker image build succeeds" ' docker_build "$IMAGE_TAG" "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" || test_fsh echo "TEST_TESTS_DIR: $TEST_TESTS_DIR" || test_fsh echo "APP_ROOT_DIR : $APP_ROOT_DIR" ' test_expect_success "write init scripts" ' echo "ipfs config Mounts.IPFS Bar" > 001.sh && echo "ipfs config Pubsub.Router Qux" > 002.sh && chmod +x 002.sh ' test_expect_success "docker image runs" ' DOC_ID=$(docker run -d \ -p 127.0.0.1:5001:5001 -p 127.0.0.1:8080:8080 \ -v "$PWD/001.sh":/container-init.d/001.sh \ -v "$PWD/002.sh":/container-init.d/002.sh \ "$IMAGE_TAG") ' test_expect_success "docker container gateway is up" ' pollEndpoint -host=/ip4/127.0.0.1/tcp/8080 -http-url http://localhost:8080/ipfs/bafkqaddimvwgy3zao5xxe3debi -v -tries 30 -tout 1s ' test_expect_success "docker container API is up" ' pollEndpoint -host=/ip4/127.0.0.1/tcp/5001 -http-url http://localhost:5001/version -v -tries 30 -tout 1s ' test_expect_success "check that init scripts were run correctly and in the correct order" " echo -e \"Sourcing '/container-init.d/001.sh'...\nExecuting '/container-init.d/002.sh'...\" > expected && docker logs $DOC_ID 2>/dev/null | grep -e 001.sh -e 002.sh > actual && test_cmp actual expected " test_expect_success "check that init script configs were applied" ' echo Bar > expected && docker exec "$DOC_ID" ipfs config Mounts.IPFS > actual && test_cmp actual expected && echo Qux > expected && docker exec "$DOC_ID" ipfs config Pubsub.Router > actual && test_cmp actual expected ' test_expect_success "simple ipfs add/cat can be run in docker container" ' echo "Hello Worlds" | tr -d "[:cntrl:]" > expected && HASH=$(docker_exec "$DOC_ID" "echo $(cat expected) | ipfs add -q" | tr -d "[:cntrl:]") && docker_exec "$DOC_ID" "ipfs cat $HASH" | tr -d "[:cntrl:]" > actual && test_cmp expected actual ' read testcode <actual ; \ test -s actual ; \ docker exec -i "$DOC_ID" ipfs version --enc json \ | sed 's/^.*"Commit":"\\\([^"]*\\\)".*$/\\\1/g' >expected ; \ test -s expected ; \ test_cmp expected actual EOF test_expect_success "version CurrentCommit is set" "$testcode" test_expect_success "stop docker container" ' docker_stop "$DOC_ID" ' docker_rm "$DOC_ID" docker_rmi "$IMAGE_TAG" test_done ================================================ FILE: test/sharness/t0003-docker-migrate.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2017 Whyrusleeping # MIT Licensed; see the LICENSE file in this repository. # test_description="Test docker image migration" . lib/test-lib.sh # if in travis CI on OSX, docker is not available if ! test_have_prereq DOCKER; then skip_all='skipping '$test_description', docker not available' test_done fi if ! test_have_prereq SOCAT; then skip_all="skipping '$test_description': socat is not available" test_done fi TEST_TRASH_DIR=$(pwd) TEST_SCRIPTS_DIR=$(dirname "$TEST_TRASH_DIR") TEST_TESTS_DIR=$(dirname "$TEST_SCRIPTS_DIR") APP_ROOT_DIR=$(dirname "$TEST_TESTS_DIR") IMAGE_TAG=kubo_migrate test_expect_success "docker image build succeeds" ' docker_build "$IMAGE_TAG" "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" ' test_init_ipfs test_expect_success "configure migration sources" ' ipfs config --json Migration.DownloadSources "[\"http://127.0.0.1:17233\"]" ' test_expect_success "setup http response" ' mkdir migration && echo "v1.1.1" > migration/versions && mkdir -p migration/fs-repo-6-to-7 && echo "v1.1.1" > migration/fs-repo-6-to-7/versions && CID=$(ipfs add -r -Q migration) && echo "HTTP/1.1 200 OK" > vers_resp && echo "Content-Type: application/vnd.ipld.car" >> vers_resp && echo "" >> vers_resp && ipfs dag export $CID >> vers_resp ' test_expect_success "make repo be version 4" ' echo 4 > "$IPFS_PATH/version" ' test_expect_success "startup fake dists server" ' ( socat tcp-listen:17233,fork,bind=127.0.0.1,reuseaddr "SYSTEM:cat vers_resp"!!STDERR 2> dist_serv_out ) & echo $! > netcat_pid ' test_expect_success "docker image runs" ' DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs -e IPFS_DIST_PATH=/ipfs/$CID --net=host "$IMAGE_TAG") ' test_expect_success "docker container tries to pull migrations from netcat" ' sleep 4 && cat dist_serv_out ' test_expect_success "see logs" ' docker logs $DOC_ID ' test_expect_success "stop docker container" ' docker_stop "$DOC_ID" ' test_expect_success "kill the net cat" ' kill $(cat netcat_pid) || true ' test_expect_success "correct version was requested" ' grep "/fs-repo-6-to-7/v1.1.1/fs-repo-6-to-7_v1.1.1_linux-amd64.tar.gz" dist_serv_out > /dev/null ' docker_rm "$DOC_ID" docker_rmi "$IMAGE_TAG" test_done ================================================ FILE: test/sharness/t0012-completion-fish.sh ================================================ #!/usr/bin/env bash test_description="Test generated fish completions" . lib/test-lib.sh test_expect_success "'ipfs commands completion fish' succeeds" ' ipfs commands completion fish > completions.fish ' test_expect_success "generated completions completes 'ipfs version'" ' fish -c "source completions.fish && complete -C \"ipfs ver\" | grep -q \"version.Show IPFS version information.\" " ' test_done ================================================ FILE: test/sharness/t0015-basic-sh-functions.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Christian Couder # MIT Licensed; see the LICENSE file in this repository. # test_description="Test some basic shell functions" . lib/test-lib.sh test_expect_success "shellquote works with simple stuff" ' var=$(shellquote one two) ' test_expect_success "shellquote output looks good" ' test "$var" = "'\''one'\'' '\''two'\''" || test_fsh echo "var is \"$var\" instead of \"'\''one'\'' '\''two'\''\"" ' # The following two printf statements are equivalent: # printf "%s\n" \''"foo\ # bar' # printf "\047\042\146\157\157\134\012\142\141\162\012" # We use the second one to simplify quoting. test_expect_success "shellquote works with complex printf" ' eval "$(shellquote printf "\047\042\146\157\157\134\012\142\141\162\012")" >actual ' test_expect_success "shellquote output looks good" ' printf "\047\042\146\157\157\134\012\142\141\162\012" >expected && test_cmp expected actual ' test_expect_success "shellquote works with many different bytes" ' bytes_sans_NUL=$( printf "\001\002\003\004\005\006\007\010\011\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037\040\041\042\043\044%%\046\047\050\051\052\053\054\055\056\057\060\061\062\063\064\065\066\067\070\071\072\073\074\075\076\077\100\101\102\103\104\105\106\107\110\111\112\113\114\115\116\117\120\121\122\123\124\125\126\127\130\131\132\133\134\135\136\137\140\141\142\143\144\145\146\147\150\151\152\153\154\155\156\157\160\161\162\163\164\165\166\167\170\171\172\173\174\175\176\177\200\201\202\203\204\205\206\207\210\211\212\213\214\215\216\217\220\221\222\223\224\225\226\227\230\231\232\233\234\235\236\237\240\241\242\243\244\245\246\247\250\251\252\253\254\255\256\257\260\261\262\263\264\265\266\267\270\271\272\273\274\275\276\277\300\301\302\303\304\305\306\307\310\311\312\313\314\315\316\317\320\321\322\323\324\325\326\327\330\331\332\333\334\335\336\337\340\341\342\343\344\345\346\347\350\351\352\353\354\355\356\357\360\361\362\363\364\365\366\367\370\371\372\373\374\375\376\377" ) && eval "$(shellquote printf "%s" "$bytes_sans_NUL")" >actual ' test_expect_success "shellquote output looks good" ' printf "%s" "$bytes_sans_NUL" >expected && test_cmp expected actual ' test_done ================================================ FILE: test/sharness/t0018-indent.sh ================================================ #!/usr/bin/env bash test_description="Test sharness test indent" . lib/test-lib.sh for file in $(find .. -name 't*.sh' -type f); do if [ "$(basename "$file")" = "t0290-cid.sh" ]; then continue fi test_expect_success "indent in $file is not using tabs" ' test_must_fail grep -P "^ *\t" $file ' done test_done ================================================ FILE: test/sharness/t0021-config.sh ================================================ #!/usr/bin/env bash test_description="Test config command" . lib/test-lib.sh # we use a function so that we can run it both offline + online test_config_cmd_set() { # flags (like --bool in "ipfs config --bool") cfg_flags="" # unset in case. test "$#" = 3 && { cfg_flags=$1; shift; } cfg_key=$1 cfg_val=$2 test_expect_success "ipfs config succeeds" " ipfs config $cfg_flags \"$cfg_key\" \"$cfg_val\" " test_expect_success "ipfs config output looks good" " echo \"$cfg_val\" >expected && if [$cfg_flags != \"--json\"]; then ipfs config \"$cfg_key\" >actual && test_cmp expected actual else ipfs config \"$cfg_key\" | tr -d \"\\n\\t \" >actual && echo >>actual && test_cmp expected actual fi " } test_profile_apply_revert() { profile=$1 inverse_profile=$2 test_expect_success "save expected config" ' ipfs config show >expected ' test_expect_success "'ipfs config profile apply ${profile}' works" ' ipfs config profile apply '${profile}' ' test_expect_success "profile ${profile} changed something" ' ipfs config show >actual && test_must_fail test_cmp expected actual ' test_expect_success "'ipfs config profile apply ${inverse_profile}' works" ' ipfs config profile apply '${inverse_profile}' ' test_expect_success "config is back to previous state after ${inverse_profile} was applied" ' ipfs config show >actual && test_cmp expected actual ' } test_profile_apply_dry_run_not_alter() { profile=$1 test_expect_success "'ipfs config profile apply ${profile} --dry-run' doesn't alter config" ' cat "$IPFS_PATH/config" >expected && ipfs config profile apply '${profile}' --dry-run && cat "$IPFS_PATH/config" >actual && test_cmp expected actual ' } test_config_cmd() { test_config_cmd_set "Addresses.API" "foo" test_config_cmd_set "Addresses.Gateway" "bar" test_config_cmd_set "Datastore.GCPeriod" "baz" test_config_cmd_set "AutoNAT.ServiceMode" "enabled" test_config_cmd_set "--bool" "Discovery.MDNS.Enabled" "true" test_config_cmd_set "--bool" "Discovery.MDNS.Enabled" "false" test_config_cmd_set "--json" "Datastore.HashOnRead" "true" test_config_cmd_set "--json" "Datastore.HashOnRead" "false" test_config_cmd_set "--json" "Experimental.FilestoreEnabled" "true" test_config_cmd_set "--json" "Import.BatchMaxSize" "null" test_config_cmd_set "--json" "Import.UnixFSRawLeaves" "true" test_config_cmd_set "--json" "Routing.Routers.Test" "{\\\"Parameters\\\":\\\"Test\\\",\\\"Type\\\":\\\"Test\\\"}" test_config_cmd_set "--json" "Experimental.OptimisticProvideJobsPoolSize" "1337" test_config_cmd_set "--json" "Addresses.Swarm" "[\\\"test\\\",\\\"test\\\",\\\"test\\\"]" test_config_cmd_set "--json" "Gateway.PublicGateways.Foo" "{\\\"DeserializedResponses\\\":true,\\\"InlineDNSLink\\\":false,\\\"NoDNSLink\\\":false,\\\"Paths\\\":[\\\"Bar\\\",\\\"Baz\\\"],\\\"UseSubdomains\\\":true}" test_config_cmd_set "--bool" "Gateway.PublicGateways.Foo.UseSubdomains" "false" test_expect_success "'ipfs config show' works" ' ipfs config show >actual ' test_expect_success "'ipfs config show' output looks good" ' grep "\"API\": \"foo\"," actual && grep "\"Gateway\": \"bar\"" actual && grep "\"Enabled\": false" actual && grep "\"HashOnRead\": false" actual ' test_expect_success "'ipfs config show --config-file' works" ' mv "$IPFS_PATH/config" "$IPFS_PATH/config-moved" && ipfs config --config-file "$IPFS_PATH/config-moved" show >moved && test_cmp moved actual && mv "$IPFS_PATH/config-moved" "$IPFS_PATH/config" ' test_expect_success "setup for config replace test" ' cp "$IPFS_PATH/config" newconfig.json && sed -i"~" -e /PrivKey/d -e s/10GB/11GB/ newconfig.json && sed -i"~" -e '"'"'/PeerID/ {'"'"' -e '"'"' s/,$// '"'"' -e '"'"' } '"'"' newconfig.json ' test_expect_success "run 'ipfs config replace'" ' ipfs config replace - < newconfig.json ' test_expect_success "check resulting config after 'ipfs config replace'" ' sed -e /PrivKey/d "$IPFS_PATH/config" > replconfig.json && sed -i"~" -e '"'"'/PeerID/ {'"'"' -e '"'"' s/,$// '"'"' -e '"'"' } '"'"' replconfig.json && test_cmp replconfig.json newconfig.json ' # SECURITY # Those tests are here to prevent exposing the PrivKey on the network test_expect_success "'ipfs config Identity' fails" ' test_expect_code 1 ipfs config Identity 2> ident_out ' test_expect_success "output looks good" ' echo "Error: cannot show or change private key through API" > ident_exp && test_cmp ident_exp ident_out ' test_expect_success "'ipfs config Identity.PrivKey' fails" ' test_expect_code 1 ipfs config Identity.PrivKey 2> ident_out ' test_expect_success "output looks good" ' test_cmp ident_exp ident_out ' test_expect_success "lower cased PrivKey" ' sed -i"~" -e '\''s/PrivKey/privkey/'\'' "$IPFS_PATH/config" && test_expect_code 1 ipfs config Identity.privkey 2> ident_out ' test_expect_success "output looks good" ' test_cmp ident_exp ident_out ' test_expect_success "fix it back" ' sed -i"~" -e '\''s/privkey/PrivKey/'\'' "$IPFS_PATH/config" ' test_expect_success "'ipfs config show' doesn't include privkey" ' ipfs config show > show_config && test_expect_code 1 grep PrivKey show_config ' test_expect_success "'ipfs config replace' injects privkey back" ' ipfs config replace show_config && grep "\"PrivKey\":" "$IPFS_PATH/config" | grep -e ": \".\+\"" >/dev/null ' test_expect_success "'ipfs config replace' with privkey errors out" ' cp "$IPFS_PATH/config" real_config && test_expect_code 1 ipfs config replace - < real_config 2> replace_out ' test_expect_success "output looks good" ' echo "Error: setting private key with API is not supported" > replace_expected test_cmp replace_out replace_expected ' test_expect_success "'ipfs config replace' with lower case privkey errors out" ' cp "$IPFS_PATH/config" real_config && sed -i -e '\''s/PrivKey/privkey/'\'' real_config && test_expect_code 1 ipfs config replace - < real_config 2> replace_out ' test_expect_success "output looks good" ' echo "Error: setting private key with API is not supported" > replace_expected test_cmp replace_out replace_expected ' test_expect_success "'ipfs config Swarm.AddrFilters' looks good" ' ipfs config Swarm.AddrFilters > actual_config && test $(cat actual_config | wc -l) = 1 ' test_expect_success "copy ipfs config" ' cp "$IPFS_PATH/config" before_patch ' test_expect_success "'ipfs config profile apply server' works" ' ipfs config profile apply server ' test_expect_success "backup was created and looks good" ' test_cmp "$(find "$IPFS_PATH" -name "config-*")" before_patch ' test_expect_success "'ipfs config Swarm.AddrFilters' looks good with server profile" ' ipfs config Swarm.AddrFilters > actual_config && test $(cat actual_config | wc -l) = 18 ' test_expect_success "'ipfs config profile apply local-discovery' works" ' ipfs config profile apply local-discovery ' test_expect_success "'ipfs config Swarm.AddrFilters' looks good with applied local-discovery profile" ' ipfs config Swarm.AddrFilters > actual_config && test $(cat actual_config | wc -l) = 1 ' test_profile_apply_revert server local-discovery # tests above mess with values this profile changes, need to do that before testing test profile test_expect_success "ensure test profile is applied fully" ' ipfs config profile apply test ' # need to do this in reverse as the test profile is already applied in sharness test_profile_apply_revert default-networking test test_profile_apply_dry_run_not_alter server test_profile_apply_dry_run_not_alter local-discovery test_profile_apply_dry_run_not_alter test test_expect_success "'ipfs config profile apply local-discovery --dry-run' looks good with different profile info" ' ipfs config profile apply local-discovery --dry-run > diff_info && test `grep "DisableNatPortMap" diff_info | wc -l` = 2 ' test_expect_success "'ipfs config profile apply server --dry-run' looks good with same profile info" ' ipfs config profile apply server --dry-run > diff_info && test `grep "DisableNatPortMap" diff_info | wc -l` = 1 ' test_expect_success "'ipfs config profile apply server' looks good with same profile info" ' ipfs config profile apply server > diff_info && test `grep "DisableNatPortMap" diff_info | wc -l` = 1 ' test_expect_success "'ipfs config profile apply local-discovery' looks good with different profile info" ' ipfs config profile apply local-discovery > diff_info && test `grep "DisableNatPortMap" diff_info | wc -l` = 2 ' test_expect_success "'ipfs config profile apply test' looks good with different profile info" ' ipfs config profile apply test > diff_info && test `grep "DisableNatPortMap" diff_info | wc -l` = 2 ' test_expect_success "'ipfs config profile apply test --dry-run' doesn't include privkey" ' ipfs config profile apply test --dry-run > show_config && test_expect_code 1 grep PrivKey show_config ' test_expect_success "'ipfs config profile apply test' doesn't include privkey" ' ipfs config profile apply test > show_config && test_expect_code 1 grep PrivKey show_config ' # won't work as it changes datastore definition, which makes ipfs not launch # without converting first # test_profile_apply_revert pebbleds test_expect_success "cleanup config backups" ' find "$IPFS_PATH" -name "config-*" -exec rm {} \; ' } test_init_ipfs # should work offline test_config_cmd # should work online test_launch_ipfs_daemon test_config_cmd test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0022-init-default.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Christian Couder # MIT Licensed; see the LICENSE file in this repository. # test_description="Test init command with default config" . lib/test-lib.sh cfg_key="Addresses.API" cfg_val="/ip4/0.0.0.0/tcp/5001" # test that init succeeds test_expect_success "ipfs init succeeds" ' export IPFS_PATH="$(pwd)/.ipfs" && echo "IPFS_PATH: \"$IPFS_PATH\"" && BITS="2048" && ipfs init >actual_init || test_fsh cat actual_init ' test_expect_success ".ipfs/config has been created" ' test -f "$IPFS_PATH"/config || test_fsh ls -al .ipfs ' test_expect_success "ipfs config succeeds" ' ipfs config $cfg_flags "$cfg_key" "$cfg_val" ' test_expect_success "ipfs read config succeeds" ' IPFS_DEFAULT_CONFIG=$(cat "$IPFS_PATH"/config) ' test_expect_success "clean up ipfs dir" ' rm -rf "$IPFS_PATH" ' test_expect_success "ipfs init default config succeeds" ' echo $IPFS_DEFAULT_CONFIG | ipfs init - >actual_init || test_fsh cat actual_init ' test_expect_success "ipfs config output looks good" ' echo "$cfg_val" >expected && ipfs config "$cfg_key" >actual && test_cmp expected actual ' test_done ================================================ FILE: test/sharness/t0023-shutdown.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2017 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test shutdown command" . lib/test-lib.sh test_init_ipfs test_launch_ipfs_daemon test_expect_success "shutdown succeeds" ' ipfs shutdown ' test_expect_success "daemon no longer running" ' for i in $(test_seq 1 100) do go-sleep 100ms ! kill -0 $IPFS_PID 2>/dev/null && return done ' test_launch_ipfs_daemon_without_network test_expect_success "shutdown succeeds" ' ipfs shutdown ' test_expect_success "daemon no longer running" ' for i in $(test_seq 1 100) do go-sleep 100ms ! kill -0 $IPFS_PID 2>/dev/null && return done ' test_done ================================================ FILE: test/sharness/t0024-datastore-config.sh ================================================ #!/usr/bin/env bash test_description="Test datastore config" . lib/test-lib.sh test_init_ipfs test_launch_ipfs_daemon test_kill_ipfs_daemon SPEC_NOSYNC=$(cat ../t0024-files/spec-nosync) SPEC_NEWSHARDFUN=$(cat ../t0024-files/spec-newshardfun) test_expect_success "change runtime value in spec config" ' ipfs config --json Datastore.Spec "$SPEC_NOSYNC" ' test_launch_ipfs_daemon test_kill_ipfs_daemon test_expect_success "change on-disk value in spec config" ' ipfs config --json Datastore.Spec "$SPEC_NEWSHARDFUN" ' test_expect_success "can not launch daemon after on-disk value change" ' test_must_fail ipfs daemon ' test_done ================================================ FILE: test/sharness/t0024-files/spec-newshardfun ================================================ { "mounts": [ { "child": { "path": "blocks", "shardFunc": "/repo/flatfs/shard/v1/next-to-last/3", "sync": true, "type": "flatfs" }, "mountpoint": "/blocks", "prefix": "flatfs.datastore", "type": "measure" }, { "child": { "compression": "none", "path": "datastore", "type": "levelds" }, "mountpoint": "/", "prefix": "leveldb.datastore", "type": "measure" } ], "type": "mount" } ================================================ FILE: test/sharness/t0024-files/spec-nosync ================================================ { "mounts": [ { "child": { "path": "blocks", "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2", "sync": false, "type": "flatfs" }, "mountpoint": "/blocks", "prefix": "flatfs.datastore", "type": "measure" }, { "child": { "compression": "none", "path": "datastore", "type": "levelds" }, "mountpoint": "/", "prefix": "leveldb.datastore", "type": "measure" } ], "type": "mount" } ================================================ FILE: test/sharness/t0025-datastores.sh ================================================ #!/usr/bin/env bash test_description="Test non-standard datastores" . lib/test-lib.sh profiles=("flatfs" "pebbleds" "badgerds") proot="$(mktemp -d "${TMPDIR:-/tmp}/t0025.XXXXXX")" for profile in "${profiles[@]}"; do test_expect_success "'ipfs init --empty-repo=false --profile=$profile' succeeds" ' BITS="2048" && IPFS_PATH="$proot/$profile" && ipfs init --empty-repo=false --profile=$profile ' test_expect_success "'ipfs pin add' and 'pin ls' works with $profile" ' export IPFS_PATH="$proot/$profile" && echo -n "hello_$profile" | ipfs block put --pin=true > hello_cid && ipfs pin ls -t recursive "$(cat hello_cid)" ' done test_done ================================================ FILE: test/sharness/t0026-id.sh ================================================ #!/usr/bin/env bash test_description="Test to make sure our identity information looks sane" . lib/test-lib.sh test_init_ipfs test_id_compute_agent() { local AGENT_SUFFIX AGENT_SUFFIX=$1 AGENT_VERSION="$(ipfs version --number)" || return 1 AGENT_COMMIT="$(ipfs version --number --commit)" || return 1 if test "$AGENT_COMMIT" = "$AGENT_VERSION"; then AGENT_COMMIT="" else AGENT_COMMIT="${AGENT_COMMIT##$AGENT_VERSION-}" fi AGENT_VERSION="kubo/$AGENT_VERSION" if test -n "$AGENT_COMMIT"; then AGENT_VERSION="$AGENT_VERSION/$AGENT_COMMIT" fi if test -n "$AGENT_SUFFIX"; then AGENT_VERSION="$AGENT_VERSION/$AGENT_SUFFIX" fi echo "$AGENT_VERSION" } test_expect_success "checking AgentVersion" ' test_id_compute_agent > expected-agent-version && ipfs id -f "\n" > actual-agent-version && test_cmp expected-agent-version actual-agent-version ' test_expect_success "checking ID of self" ' ipfs config Identity.PeerID > expected-id && ipfs id -f "\n" > actual-id && test_cmp expected-id actual-id ' test_expect_success "checking and converting ID of a random peer while offline" ' # Peer ID taken from `t0140-swarm.sh` test. echo k2k4r8ncs1yoluq95unsd7x2vfhgve0ncjoggwqx9vyh3vl8warrcp15 > expected-id && ipfs id -f "\n" --peerid-base base36 --offline QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N > actual-id && test_cmp expected-id actual-id ' # agent-version-suffix (local, offline) test_launch_ipfs_daemon --agent-version-suffix=test-suffix test_expect_success "checking AgentVersion with suffix (local)" ' test_id_compute_agent test-suffix > expected-agent-version && ipfs id -f "\n" > actual-agent-version && test_cmp expected-agent-version actual-agent-version ' # agent-version-suffix (over libp2p identify protocol) iptb testbed create -type localipfs -count 2 -init startup_cluster 2 --agent-version-suffix=test-suffix-identify test_expect_success "checking AgentVersion with suffix (fetched via libp2p identify protocol)" ' ipfsi 0 id -f "\n" > expected-identify-agent-version && ipfsi 1 id "$(ipfsi 0 config Identity.PeerID)" -f "\n" > actual-libp2p-identify-agent-version && test_cmp expected-identify-agent-version actual-libp2p-identify-agent-version ' iptb stop test_kill_ipfs_daemon # Version.AgentSuffix overrides --agent-version-suffix (local, offline) test_expect_success "setting Version.AgentSuffix in config" ' ipfs config Version.AgentSuffix json-config-suffix ' test_launch_ipfs_daemon --agent-version-suffix=ignored-cli-suffix test_expect_success "checking AgentVersion with suffix set via JSON config" ' test_id_compute_agent json-config-suffix > expected-agent-version && ipfs id -f "\n" > actual-agent-version && test_cmp expected-agent-version actual-agent-version ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0027-rotate.sh ================================================ #!/usr/bin/env bash test_description="Test rotate command" . lib/test-lib.sh test_rotate() { FROM_ALG=$1 TO_ALG=$2 test_expect_success "ipfs init (from $FROM_ALG, to $TO_ALG)" ' export IPFS_PATH="$(pwd)/.ipfs" && case $FROM_ALG in rsa) ipfs init --profile=test -a=rsa > /dev/null ;; ed25519) ipfs init --profile=test -a=ed25519 > /dev/null ;; *) ipfs init --profile=test > /dev/null ;; esac ' test_expect_success "Save first ID and key" ' ipfs id -f="" > first_id && ipfs id -f="" > first_key ' test_launch_ipfs_daemon test_kill_ipfs_daemon test_expect_success "rotating keys" ' case $TO_ALG in rsa) ipfs key rotate -t=rsa -s=2048 --oldkey=oldkey ;; ed25519) ipfs key rotate -t=ed25519 --oldkey=oldkey ;; *) ipfs key rotate --oldkey=oldkey ;; esac ' test_expect_success "'ipfs key rotate -o self' should fail" ' echo "Error: keystore name for back up cannot be named '\''self'\''" >expected-self test_must_fail ipfs key rotate -o self 2>actual-self && test_cmp expected-self actual-self ' test_expect_success "Compare second ID and key to first" ' ipfs id -f="" > second_id && ipfs id -f="" > second_key && ! test_cmp first_id second_id && ! test_cmp first_key second_key ' test_expect_success "checking ID" ' ipfs config Identity.PeerID > expected-id && ipfs id -f "\n" > actual-id && ipfs key list -l --ipns-base=b58mh | grep self | cut -d " " -f1 > keystore-id && ipfs key list -l --ipns-base=b58mh | grep oldkey | cut -d " " -f1 | tr -d "\n" > old-keystore-id && test_cmp expected-id actual-id && test_cmp expected-id keystore-id && test_cmp old-keystore-id first_id ' test_launch_ipfs_daemon test_expect_success "publish name with new and old keys" ' echo "hello world" > msg && ipfs add msg | cut -d " " -f2 | tr -d "\n" > msg_hash && ipfs name publish --offline --allow-offline --key=self $(cat msg_hash) && ipfs name publish --offline --allow-offline --key=oldkey $(cat msg_hash) ' test_kill_ipfs_daemon test_expect_success "clean up ipfs dir" ' rm -rf "$IPFS_PATH" ' } test_rotate 'rsa' '' test_rotate 'ed25519' '' test_rotate '' '' test_rotate 'rsa' 'rsa' test_rotate 'ed25519' 'rsa' test_rotate '' 'rsa' test_rotate 'rsa' 'ed25519' test_rotate 'ed25519' 'ed25519' test_rotate '' 'ed25519' test_done ================================================ FILE: test/sharness/t0030-mount.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Christian Couder # MIT Licensed; see the LICENSE file in this repository. # test_description="Test mount command" . lib/test-lib.sh # if in travis CI, don't test mount (no fuse) if ! test_have_prereq FUSE; then skip_all='skipping mount tests, fuse not available' test_done fi # echo -n "ipfs" > expected && ipfs add --cid-version 1 -Q -w expected export IPFS_NS_MAP="welcome.example.com:/ipfs/bafybeicq7bvn5lz42qlmghaoiwrve74pzi53auqetbantp5kajucsabike" # start iptb + wait for peering NUM_NODES=5 test_expect_success 'init iptb' ' iptb testbed create -type localipfs -count $NUM_NODES -init ' startup_cluster $NUM_NODES # test mount failure before mounting properly. test_expect_success "'ipfs mount' fails when there is no mount dir" ' tmp_ipfs_mount() { ipfsi 0 mount -f=not_ipfs -n=not_ipns -m=not_mfs >output 2>output.err; } && test_must_fail tmp_ipfs_mount ' test_expect_success "'ipfs mount' output looks good" ' test_must_be_empty output && test_should_contain "not_ipns\|not_ipfs\|not_mfs" output.err ' test_expect_success "setup and publish default IPNS value" ' mkdir "$(pwd)/ipfs" "$(pwd)/ipns" "$(pwd)/mfs" && ipfsi 0 name publish QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn ' # make sure stuff is unmounted first # then mount properly test_expect_success FUSE "'ipfs mount' succeeds" ' do_umount "$(pwd)/ipfs" || true && do_umount "$(pwd)/ipns" || true && do_umount "$(pwd)/mfs" || true && ipfsi 0 mount -f "$(pwd)/ipfs" -n "$(pwd)/ipns" -m "$(pwd)/mfs" >actual ' test_expect_success FUSE "'ipfs mount' output looks good" ' echo "IPFS mounted at: $(pwd)/ipfs" >expected && echo "IPNS mounted at: $(pwd)/ipns" >>expected && echo "MFS mounted at: $(pwd)/mfs" >>expected && test_cmp expected actual ' test_expect_success FUSE "local symlink works" ' ipfsi 0 id -f"\n" > expected && basename $(readlink ipns/local) > actual && test_cmp expected actual ' test_expect_success FUSE "can resolve ipns names" ' echo -n "ipfs" > expected && ipfsi 0 add --cid-version 1 -Q -w expected && cat ipns/welcome.example.com/expected > actual && test_cmp expected actual ' test_expect_success FUSE "create mfs file via fuse" ' touch mfs/testfile && ipfsi 0 files ls | grep testfile ' test_expect_success FUSE "create mfs dir via fuse" ' mkdir mfs/testdir && ipfsi 0 files ls | grep testdir ' test_expect_success FUSE "read mfs file from fuse" ' echo content > mfs/testfile && getfattr -n ipfs_cid mfs/testfile ' test_expect_success FUSE "ipfs add file and read it back via fuse" ' echo content3 | ipfsi 0 files write -e /testfile3 && grep content3 mfs/testfile3 ' test_expect_success FUSE "ipfs add file and read it back via fuse" ' echo content > testfile2 && ipfsi 0 add --to-files /testfile2 testfile2 && grep content mfs/testfile2 ' test_expect_success FUSE "test file xattr" ' echo content > mfs/testfile && getfattr -n ipfs_cid mfs/testfile ' test_expect_success FUSE "test file removal" ' touch mfs/testfile && rm mfs/testfile ' test_expect_success FUSE "test nested dirs" ' mkdir -p mfs/foo/bar/baz/qux && echo content > mfs/foo/bar/baz/qux/quux && ipfsi 0 files stat /foo/bar/baz/qux/quux ' test_expect_success "mount directories cannot be removed while active" ' test_must_fail rmdir ipfs ipns mfs 2>/dev/null ' test_expect_success "unmount directories" ' do_umount "$(pwd)/ipfs" && do_umount "$(pwd)/ipns" && do_umount "$(pwd)/mfs" ' test_expect_success "mount directories can be removed after shutdown" ' rmdir ipfs ipns mfs ' test_expect_success 'stop iptb' ' iptb stop ' test_done ================================================ FILE: test/sharness/t0031-mount-publish.sh ================================================ #!/usr/bin/env bash test_description="Test mount command in conjunction with publishing" # imports . lib/test-lib.sh # if in travis CI, don't test mount (no fuse) if ! test_have_prereq FUSE; then skip_all='skipping mount tests, fuse not available' test_done fi test_init_ipfs # start iptb + wait for peering NUM_NODES=3 test_expect_success 'init iptb' ' iptb testbed create -type localipfs -count $NUM_NODES -force -init && startup_cluster $NUM_NODES ' # pre-mount publish HASH=$(echo 'hello warld' | ipfsi 0 add -Q -w --stdin-name "file") test_expect_success "can publish before mounting /ipns" ' ipfsi 0 name publish "$HASH" ' # mount IPFS_MOUNT_DIR="$PWD/ipfs" IPNS_MOUNT_DIR="$PWD/ipns" test_expect_success FUSE "'ipfs mount' succeeds" ' ipfsi 0 mount -f "'"$IPFS_MOUNT_DIR"'" -n "'"$IPNS_MOUNT_DIR"'" >actual ' test_expect_success FUSE "'ipfs mount' output looks good" ' echo "IPFS mounted at: $PWD/ipfs" >expected && echo "IPNS mounted at: $PWD/ipns" >>expected && test_cmp expected actual ' test_expect_success "cannot publish after mounting /ipns" ' echo "Error: cannot manually publish while IPNS is mounted" >expected && test_must_fail ipfsi 0 name publish '$HASH' 2>actual && test_cmp expected actual ' test_expect_success "unmount /ipns out-of-band" ' fusermount -u "'"$IPNS_MOUNT_DIR"'" ' test_expect_success "can publish after unmounting /ipns" ' ipfsi 0 name publish '$HASH' ' # clean-up ipfs test_expect_success "unmount /ipfs" ' fusermount -u "'"$IPFS_MOUNT_DIR"'" ' iptb stop test_done ================================================ FILE: test/sharness/t0032-mount-sharded.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2021 Protocol Labs # MIT Licensed; see the LICENSE file in this repository. # test_description="Test mount command with sharding enabled" . lib/test-lib.sh if ! test_have_prereq FUSE; then skip_all='skipping mount sharded tests, fuse not available' test_done fi test_init_ipfs test_expect_success 'force sharding' ' ipfs config --json Import.UnixFSHAMTDirectorySizeThreshold "\"1B\"" ' test_launch_ipfs_daemon test_mount_ipfs # we're testing nested subdirs which ensures that IPLD ADLs work test_expect_success 'setup test data' ' mkdir testdata && echo a > testdata/a && mkdir testdata/subdir && echo b > testdata/subdir/b ' HASH=QmY59Ufw8zA2BxGPMTcfXg86JVed81Qbxeq5rDkHWSLN1m test_expect_success 'can add the data' ' echo $HASH > expected_hash && ipfs add -r -Q testdata > actual_hash && test_cmp expected_hash actual_hash ' test_expect_success 'can read the data' ' echo a > expected_a && cat "ipfs/$HASH/a" > actual_a && test_cmp expected_a actual_a && echo b > expected_b && cat "ipfs/$HASH/subdir/b" > actual_b && test_cmp expected_b actual_b ' test_expect_success 'can list directories' ' printf "a\nsubdir\n" > expected_ls && ls -1 "ipfs/$HASH" > actual_ls && test_cmp expected_ls actual_ls && printf "b\n" > expected_ls_subdir && ls -1 "ipfs/$HASH/subdir" > actual_ls_subdir && test_cmp expected_ls_subdir actual_ls_subdir ' test_expect_success "unmount" ' do_umount "$(pwd)/ipfs" && do_umount "$(pwd)/ipns" ' test_expect_success 'cleanup' 'rmdir ipfs ipns' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0040-add-and-cat.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Christian Couder # MIT Licensed; see the LICENSE file in this repository. # test_description="Test add and cat commands" . lib/test-lib.sh test_add_cat_file() { test_expect_success "ipfs add --help works" ' ipfs add --help 2> add_help_err1 > /dev/null ' test_expect_success "stdin reading message doesn't show up" ' test_expect_code 1 grep "ipfs: Reading from" add_help_err1 && test_expect_code 1 grep "send Ctrl-d to stop." add_help_err1 ' test_expect_success "ipfs help add works" ' ipfs help add 2> add_help_err2 > /dev/null ' test_expect_success "stdin reading message doesn't show up" ' test_expect_code 1 grep "ipfs: Reading from" add_help_err2 && test_expect_code 1 grep "send Ctrl-d to stop." add_help_err2 ' test_expect_success "ipfs add succeeds" ' echo "Hello Worlds!" >mountdir/hello.txt && ipfs add mountdir/hello.txt >actual ' test_expect_success "ipfs add output looks good" ' HASH="QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH" && echo "added $HASH hello.txt" >expected && test_cmp expected actual ' test_expect_success "ipfs add --only-hash succeeds" ' ipfs add --only-hash mountdir/hello.txt > oh_actual ' test_expect_success "ipfs add --only-hash output looks good" ' test_cmp expected oh_actual ' test_expect_success "ipfs cat succeeds" ' ipfs cat "$HASH" >actual ' test_expect_success "ipfs cat output looks good" ' echo "Hello Worlds!" >expected && test_cmp expected actual ' test_expect_success "ipfs cat with offset succeeds" ' ipfs cat --offset 10 "$HASH" >actual ' test_expect_success "ipfs cat from offset output looks good" ' echo "ds!" >expected && test_cmp expected actual ' test_expect_success "ipfs cat multiple hashes with offset succeeds" ' ipfs cat --offset 10 "$HASH" "$HASH" >actual ' test_expect_success "ipfs cat from offset output looks good" ' echo "ds!" >expected && echo "Hello Worlds!" >>expected && test_cmp expected actual ' test_expect_success "ipfs cat multiple hashes with offset succeeds" ' ipfs cat --offset 16 "$HASH" "$HASH" >actual ' test_expect_success "ipfs cat from offset output looks good" ' echo "llo Worlds!" >expected && test_cmp expected actual ' test_expect_success "ipfs cat from negative offset should fail" ' test_expect_code 1 ipfs cat --offset -102 "$HASH" > actual ' test_expect_success "ipfs cat with length succeeds" ' ipfs cat --length 8 "$HASH" >actual ' test_expect_success "ipfs cat with length output looks good" ' printf "Hello Wo" >expected && test_cmp expected actual ' test_expect_success "ipfs cat multiple hashes with offset and length succeeds" ' ipfs cat --offset 5 --length 15 "$HASH" "$HASH" "$HASH" >actual ' test_expect_success "ipfs cat multiple hashes with offset and length looks good" ' printf " Worlds!\nHello " >expected && test_cmp expected actual ' test_expect_success "ipfs cat with exact length succeeds" ' ipfs cat --length $(ipfs cat "$HASH" | wc -c) "$HASH" >actual ' test_expect_success "ipfs cat with exact length looks good" ' echo "Hello Worlds!" >expected && test_cmp expected actual ' test_expect_success "ipfs cat with 0 length succeeds" ' ipfs cat --length 0 "$HASH" >actual ' test_expect_success "ipfs cat with 0 length looks good" ' : >expected && test_cmp expected actual ' test_expect_success "ipfs cat with oversized length succeeds" ' ipfs cat --length 100 "$HASH" >actual ' test_expect_success "ipfs cat with oversized length looks good" ' echo "Hello Worlds!" >expected && test_cmp expected actual ' test_expect_success "ipfs cat with negative length should fail" ' test_expect_code 1 ipfs cat --length -102 "$HASH" > actual ' test_expect_success "ipfs cat /ipfs/file succeeds" ' ipfs cat /ipfs/$HASH >actual ' test_expect_success "output looks good" ' echo "Hello Worlds!" >expected && test_cmp expected actual ' test_expect_success "ipfs add -t succeeds" ' ipfs add -t mountdir/hello.txt >actual ' test_expect_success "ipfs add -t output looks good" ' HASH="QmUkUQgxXeggyaD5Ckv8ZqfW8wHBX6cYyeiyqvVZYzq5Bi" && echo "added $HASH hello.txt" >expected && test_cmp expected actual ' test_expect_success "ipfs add --chunker size-32 succeeds" ' ipfs add --chunker rabin mountdir/hello.txt >actual ' test_expect_success "ipfs add --chunker size-32 output looks good" ' HASH="QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH" && echo "added $HASH hello.txt" >expected && test_cmp expected actual ' test_expect_success "ipfs add --chunker size-64 succeeds" ' ipfs add --chunker=size-64 mountdir/hello.txt >actual ' test_expect_success "ipfs add --chunker size-64 output looks good" ' HASH="QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH" && echo "added $HASH hello.txt" >expected && test_cmp expected actual ' test_expect_success "ipfs add --chunker=size-0 failed" ' test_expect_code 1 ipfs add -Q --chunker=size-0 mountdir/hello.txt ' test_expect_success "ipfs add --chunker rabin-36-512-1024 succeeds" ' ipfs add --chunker rabin-36-512-1024 mountdir/hello.txt >actual ' test_expect_success "ipfs add --chunker rabin-36-512-1024 output looks good" ' HASH="QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH" && echo "added $HASH hello.txt" >expected && test_cmp expected actual ' test_expect_success "ipfs add --chunker rabin-12-512-1024 failed" ' test_expect_code 1 ipfs add -Q --chunker rabin-12-512-1024 mountdir/hello.txt ' test_expect_success "ipfs add --chunker buzhash succeeds" ' ipfs add --chunker buzhash mountdir/hello.txt >actual ' test_expect_success "ipfs add --chunker buzhash output looks good" ' HASH="QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH" && echo "added $HASH hello.txt" >expected && test_cmp expected actual ' test_expect_success "ipfs add on hidden file succeeds" ' echo "Hello Worlds!" >mountdir/.hello.txt && ipfs add mountdir/.hello.txt >actual ' test_expect_success "ipfs add on hidden file output looks good" ' HASH="QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH" && echo "added $HASH .hello.txt" >expected && test_cmp expected actual ' test_expect_success "add zero length file" ' touch zero-length-file && ZEROHASH=$(ipfs add -q zero-length-file) && echo $ZEROHASH ' test_expect_success "zero length file has correct hash" ' test "$ZEROHASH" = QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH ' test_expect_success "cat zero length file" ' ipfs cat $ZEROHASH > zero-length-file_out ' test_expect_success "make sure it looks good" ' test_cmp zero-length-file zero-length-file_out ' test_expect_success "ipfs add --stdin-name" ' NAMEHASH="QmdFyxZXsFiP4csgfM5uPu99AvFiKH62CSPDw5TP92nr7w" && echo "IPFS" | ipfs add --stdin-name file.txt > actual && echo "added $NAMEHASH file.txt" > expected && test_cmp expected actual ' test_expect_success "ipfs add --stdin-name -w" ' NAMEHASH="QmdFyxZXsFiP4csgfM5uPu99AvFiKH62CSPDw5TP92nr7w" && echo "IPFS" | ipfs add -w --stdin-name file.txt | head -n1> actual && echo "added $NAMEHASH file.txt" > expected && test_cmp expected actual ' test_expect_success "ipfs cat with stdin-name" ' NAMEHASH=$(echo "IPFS" | ipfs add -w --stdin-name file.txt -Q) && ipfs cat /ipfs/$NAMEHASH/file.txt > expected && echo "IPFS" > actual && test_cmp expected actual ' test_expect_success "ipfs add -r ." ' mkdir test_current_dir && echo "Hey" > test_current_dir/hey && mkdir test_current_dir/hello && echo "World" > test_current_dir/hello/world && ( cd test_current_dir && ipfs add -r -Q . > ../actual && cd ../ ) && rm -r test_current_dir ' test_expect_success "ipfs add -r . output looks good" ' echo "QmZQWnfcqJ6hNkkPvrY9Q5X39GP3jUnUbAV4AbmbbR3Cb1" > expected test_cmp expected actual ' test_expect_success "ipfs add -r ./" ' mkdir test_current_dir && echo "Hey" > test_current_dir/hey && mkdir test_current_dir/hello && echo "World" > test_current_dir/hello/world && ( cd test_current_dir && ipfs add -r -Q ./ > ../actual && cd ../ ) && rm -r test_current_dir ' test_expect_success "ipfs add -r ./ output looks good" ' echo "QmZQWnfcqJ6hNkkPvrY9Q5X39GP3jUnUbAV4AbmbbR3Cb1" > expected test_cmp expected actual ' # --cid-base=base32 test_expect_success "ipfs add --cid-base=base32 succeeds" ' echo "base32 test" >mountdir/base32-test.txt && ipfs add --cid-base=base32 mountdir/base32-test.txt >actual ' test_expect_success "ipfs add --cid-base=base32 output looks good" ' HASHb32="bafybeibyosqxljd2eptb4ebbtvk7pb4aoxzqa6ttdsflty6rsslz5y6i34" && echo "added $HASHb32 base32-test.txt" >expected && test_cmp expected actual ' test_expect_success "ipfs add --cid-base=base32 --only-hash succeeds" ' ipfs add --cid-base=base32 --only-hash mountdir/base32-test.txt > oh_actual ' test_expect_success "ipfs add --cid-base=base32 --only-hash output looks good" ' test_cmp expected oh_actual ' test_expect_success "ipfs add --cid-base=base32 --upgrade-cidv0-in-output=false succeeds" ' echo "base32 test" >mountdir/base32-test.txt && ipfs add --cid-base=base32 --upgrade-cidv0-in-output=false mountdir/base32-test.txt >actual ' test_expect_success "ipfs add --cid-base=base32 --upgrade-cidv0-in-output=false output looks good" ' HASHv0=$(cid-fmt -v 0 -b z %s "$HASHb32") && echo "added $HASHv0 base32-test.txt" >expected && test_cmp expected actual ' test_expect_success "ipfs add --cid-base=base32 --upgrade-cidv0-in-output=false --only-hash succeeds" ' ipfs add --cid-base=base32 --upgrade-cidv0-in-output=false --only-hash mountdir/base32-test.txt > oh_actual ' test_expect_success "ipfs add --cid-base=base32 --upgrade-cidv0-in-output=false --only-hash output looks good" ' test_cmp expected oh_actual ' test_expect_success "ipfs cat with base32 hash succeeds" ' ipfs cat "$HASHb32" >actual ' test_expect_success "ipfs cat with base32 hash output looks good" ' echo "base32 test" >expected && test_cmp expected actual ' test_expect_success "ipfs cat using CIDv0 hash succeeds" ' ipfs cat "$HASHv0" >actual ' test_expect_success "ipfs cat using CIDv0 hash looks good" ' echo "base32 test" >expected && test_cmp expected actual ' test_expect_success "ipfs add with multiple files succeeds" ' echo "Helloo Worlds!" >mountdir/hello2.txt && ipfs add mountdir/hello.txt mountdir/hello2.txt >actual ' test_expect_success "ipfs add with multiple files output looks good" ' echo "added QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH hello.txt" >expected && echo "added Qmf35k66MZNW2GijohUmXQEWKZU4cCGTCwK6idfnt152wJ hello2.txt" >> expected && test_cmp expected actual ' test_expect_success "ipfs add with multiple files of same name and import dir succeeds" ' ipfs add mountdir/hello.txt mountdir/hello.txt >actual ' test_expect_success "ipfs add with multiple files of same name output looks good" ' echo "added QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH hello.txt" >expected && test_cmp expected actual ' test_expect_success "ipfs add with multiple files of same name but different dirs fails" ' mkdir -p mountdir/same-file/ && cp mountdir/hello.txt mountdir/same-file/hello.txt && test_expect_code 1 ipfs add mountdir/hello.txt mountdir/same-file/hello.txt >actual && rm mountdir/same-file/hello.txt && rmdir mountdir/same-file ' ## --to-files with single source test_expect_success "ipfs add --to-files /mfspath succeeds" ' mkdir -p mountdir && echo "Hello MFS!" > mountdir/mfs.txt && ipfs add mountdir/mfs.txt --to-files /ipfs-add-to-files >actual ' test_expect_success "ipfs add --to-files output looks good" ' HASH_MFS="QmVT8bL3sGBA2TwvX8JPhrv5CYZL8LLLfW7mxkUjPZsgBr" && echo "added $HASH_MFS mfs.txt" >expected && test_cmp expected actual ' test_expect_success "ipfs files read succeeds" ' ipfs files read /ipfs-add-to-files >actual && ipfs files rm /ipfs-add-to-files ' test_expect_success "ipfs cat output looks good" ' echo "Hello MFS!" >expected && test_cmp expected actual ' test_expect_success "ipfs add --to-files requires argument" ' test_expect_code 1 ipfs add mountdir/mfs.txt --to-files >actual 2>&1 && test_should_contain "Error: missing argument for option \"to-files\"" actual ' test_expect_success "ipfs add --to-files / (MFS root) works" ' echo "Hello MFS!" >expected && ipfs add mountdir/mfs.txt --to-files / && ipfs files read /mfs.txt >actual && test_cmp expected actual && ipfs files rm /mfs.txt && rm mountdir/mfs.txt ' ## --to-files with multiple sources test_expect_success "ipfs add file1 file2 --to-files /mfspath0 (without trailing slash) fails" ' mkdir -p test && echo "file1" > test/mfs1.txt && echo "file2" > test/mfs2.txt && test_expect_code 1 ipfs add test/mfs1.txt test/mfs2.txt --to-files /mfspath0 >actual 2>&1 && test_should_contain "MFS destination is a file: only one entry can be copied to \"/mfspath0\"" actual && ipfs files rm -r --force /mfspath0 ' test_expect_success "ipfs add file1 file2 --to-files /mfsfile1 (without trailing slash + with preexisting file) fails" ' echo test | ipfs files write --create /mfsfile1 && test_expect_code 1 ipfs add test/mfs1.txt test/mfs2.txt --to-files /mfsfile1 >actual 2>&1 && test_should_contain "Error: to-files: cannot put node in path \"/mfsfile1\"" actual && ipfs files rm -r --force /mfsfile1 ' test_expect_success "ipfs add file1 file2 --to-files /mfsdir1 (without trailing slash + with preexisting dir) fails" ' ipfs files mkdir -p /mfsdir1 && test_expect_code 1 ipfs add test/mfs1.txt test/mfs2.txt --to-files /mfsdir1 >actual 2>&1 && test_should_contain "Error: to-files: cannot put node in path \"/mfsdir1\"" actual && ipfs files rm -r --force /mfsdir1 ' test_expect_success "ipfs add file1 file2 --to-files /mfsdir2/ (with trailing slash) succeeds" ' ipfs files mkdir -p /mfsdir2 && test_expect_code 0 ipfs add --cid-version 1 test/mfs1.txt test/mfs2.txt --to-files /mfsdir2/ > actual 2>&1 && test_should_contain "added bafkreihm3rktn5z33luic3youqdsn326toaq3ekesmdvsa53sbrd3f5r3a mfs1.txt" actual && test_should_contain "added bafkreidh5zkhr2vnwa2luwmuj24xo6l3jhfgvkgtk5cyp43oxs7owzpxby mfs2.txt" actual && test_should_not_contain "Error" actual && ipfs files ls /mfsdir2/ > lsout && test_should_contain "mfs1.txt" lsout && test_should_contain "mfs2.txt" lsout && ipfs files rm -r --force /mfsdir2 ' test_expect_success "ipfs add file1 file2 --to-files /mfsfile2/ (with trailing slash + with preexisting file) fails" ' echo test | ipfs files write --create /mfsfile2 && test_expect_code 1 ipfs add test/mfs1.txt test/mfs2.txt --to-files /mfsfile2/ >actual 2>&1 && test_should_contain "Error: to-files: MFS destination \"/mfsfile2/\" is not a directory" actual && ipfs files rm -r --force /mfsfile2 ' ## --to-files with recursive dir # test MFS destination without trailing slash test_expect_success "ipfs add with --to-files /mfs/subdir3 fails because /mfs/subdir3 exists" ' ipfs files mkdir -p /mfs/subdir3 && test_expect_code 1 ipfs add -r test --to-files /mfs/subdir3 >actual 2>&1 && test_should_contain "cannot put node in path \"/mfs/subdir3\": directory already has entry by that name" actual && ipfs files rm -r --force /mfs ' # test recursive import of a dir into MFS subdirectory test_expect_success "ipfs add -r dir --to-files /mfs/subdir4/ succeeds (because of trailing slash)" ' ipfs files mkdir -p /mfs/subdir4 && ipfs add --cid-version 1 -r test --to-files /mfs/subdir4/ >actual 2>&1 && test_should_contain "added bafkreihm3rktn5z33luic3youqdsn326toaq3ekesmdvsa53sbrd3f5r3a test/mfs1.txt" actual && test_should_contain "added bafkreidh5zkhr2vnwa2luwmuj24xo6l3jhfgvkgtk5cyp43oxs7owzpxby test/mfs2.txt" actual && test_should_contain "added bafybeic7xwqwovt4g4bax6d3udp6222i63vj2rblpbim7uy2uw4a5gahha test" actual && test_should_not_contain "Error" actual ipfs files ls /mfs/subdir4/ > lsout && test_should_contain "test" lsout && test_should_not_contain "mfs1.txt" lsout && test_should_not_contain "mfs2.txt" lsout && ipfs files rm -r --force /mfs ' # confirm -w and --to-files are exclusive # context: https://github.com/ipfs/kubo/issues/10611 test_expect_success "ipfs add -r -w dir --to-files /mfs/subdir5/ errors (-w and --to-files are exclusive)" ' ipfs files mkdir -p /mfs/subdir5 && test_expect_code 1 ipfs add -r -w test --to-files /mfs/subdir5/ >actual 2>&1 && test_should_contain "Error" actual && ipfs files rm -r --force /mfs ' } test_add_cat_5MB() { ADD_FLAGS="$1" EXP_HASH="$2" test_expect_success "generate 5MB file using random-data" ' random-data -size=5242880 -seed=41 >mountdir/bigfile ' test_expect_success "sha1 of the file looks ok" ' echo "11145b8c4bc8f87ea2fcfc3d55708b8cac2aadf12862" >sha1_expected && multihash -a=sha1 -e=hex mountdir/bigfile >sha1_actual && test_cmp sha1_expected sha1_actual ' test_expect_success "'ipfs add $ADD_FLAGS bigfile' succeeds" ' ipfs add $ADD_FLAGS mountdir/bigfile >actual || test_fsh cat daemon_err ' test_expect_success "'ipfs add bigfile' output looks good" ' echo "added $EXP_HASH bigfile" >expected && test_cmp expected actual ' test_expect_success "'ipfs cat' succeeds" ' ipfs cat "$EXP_HASH" >actual ' test_expect_success "'ipfs cat' output looks good" ' test_cmp mountdir/bigfile actual ' test_expect_success FUSE "cat ipfs/bigfile succeeds" ' cat "ipfs/$EXP_HASH" >actual ' test_expect_success FUSE "cat ipfs/bigfile looks good" ' test_cmp mountdir/bigfile actual ' test_expect_success "remove hash" ' ipfs pin rm "$EXP_HASH" && ipfs block rm "$EXP_HASH" ' test_expect_success "get base32 version of CID" ' ipfs cid base32 $EXP_HASH > base32_cid && BASE32_HASH=`cat base32_cid` ' test_expect_success "ipfs add --cid-base=base32 bigfile' succeeds" ' ipfs add $ADD_FLAGS --cid-base=base32 mountdir/bigfile >actual || test_fsh cat daemon_err ' test_expect_success "'ipfs add bigfile --cid-base=base32' output looks good" ' echo "added $BASE32_HASH bigfile" >expected && test_cmp expected actual ' test_expect_success "'ipfs cat $BASE32_HASH' succeeds" ' ipfs cat "$BASE32_HASH" >actual ' } test_add_cat_raw() { test_expect_success "add a small file with raw-leaves" ' echo "foobar" > afile && HASH=$(ipfs add -q --raw-leaves afile) ' test_expect_success "cat that small file" ' ipfs cat $HASH > afile_out ' test_expect_success "make sure it looks good" ' test_cmp afile afile_out ' test_expect_success "add zero length file with raw-leaves" ' touch zero-length-file && ZEROHASH=$(ipfs add -q --raw-leaves zero-length-file) && echo $ZEROHASH ' test_expect_success "zero length file has correct hash" ' test "$ZEROHASH" = bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku ' test_expect_success "cat zero length file" ' ipfs cat $ZEROHASH > zero-length-file_out ' test_expect_success "make sure it looks good" ' test_cmp zero-length-file zero-length-file_out ' } test_add_cat_derefargs() { test_expect_success "create and hash zero length file" ' touch zero-length-file && ZEROHASH=$(ipfs add -q -n zero-length-file) ' test_expect_success "create symlink and add with dereferenced arguments" ' ln -s zero-length-file symlink-to-zero && HASH=$(ipfs add -q -n --dereference-args symlink-to-zero) && test $HASH = $ZEROHASH ' } test_add_cat_expensive() { ADD_FLAGS="$1" HASH="$2" test_expect_success EXPENSIVE "generate 100MB file using random-data" ' random-data -size=104857600 -seed=42 >mountdir/bigfile ' test_expect_success EXPENSIVE "sha1 of the file looks ok" ' echo "11141e8c04d7cd019cc0acf0311a8ca6cf2c18413c96" >sha1_expected && multihash -a=sha1 -e=hex mountdir/bigfile >sha1_actual && test_cmp sha1_expected sha1_actual ' test_expect_success EXPENSIVE "ipfs add $ADD_FLAGS bigfile succeeds" ' ipfs add $ADD_FLAGS mountdir/bigfile >actual ' test_expect_success EXPENSIVE "ipfs add bigfile output looks good" ' echo "added $HASH bigfile" >expected && test_cmp expected actual ' test_expect_success EXPENSIVE "ipfs cat succeeds" ' ipfs cat "$HASH" | multihash -a=sha1 -e=hex >sha1_actual ' test_expect_success EXPENSIVE "ipfs cat output looks good" ' ipfs cat "$HASH" >actual && test_cmp mountdir/bigfile actual ' test_expect_success EXPENSIVE "ipfs cat output hashed looks good" ' echo "11141e8c04d7cd019cc0acf0311a8ca6cf2c18413c96" >sha1_expected && test_cmp sha1_expected sha1_actual ' test_expect_success FUSE,EXPENSIVE "cat ipfs/bigfile succeeds" ' cat "ipfs/$HASH" | multihash -a=sha1 -e=hex >sha1_actual ' test_expect_success FUSE,EXPENSIVE "cat ipfs/bigfile looks good" ' test_cmp sha1_expected sha1_actual ' } test_add_named_pipe() { test_expect_success "Adding named pipes explicitly works" ' mkfifo named-pipe1 && ( echo foo > named-pipe1 & echo "added $( echo foo | ipfs add -nq ) named-pipe1" > expected_named_pipes_add ) && mkfifo named-pipe2 && ( echo bar > named-pipe2 & echo "added $( echo bar | ipfs add -nq ) named-pipe2" >> expected_named_pipes_add ) && ipfs add -n named-pipe1 named-pipe2 >actual_pipe_add && rm named-pipe1 && rm named-pipe2 && test_cmp expected_named_pipes_add actual_pipe_add ' test_expect_success "useful error message when recursively adding a named pipe" ' mkdir -p named-pipe-dir && mkfifo named-pipe-dir/named-pipe && STAT=$(generic_stat named-pipe-dir/named-pipe) && test_expect_code 1 ipfs add -r named-pipe-dir 2>actual && printf "Error: unrecognized file type for named-pipe-dir/named-pipe: $STAT\n" >expected && rm named-pipe-dir/named-pipe && rmdir named-pipe-dir && test_cmp expected actual ' } test_add_pwd_is_symlink() { test_expect_success "ipfs add -r adds directory content when ./ is symlink" ' mkdir hellodir && echo "World" > hellodir/world && ln -s hellodir hellolink && ( cd hellolink && ipfs add -r . > ../actual ) && grep "added Qma9CyFdG5ffrZCcYSin2uAETygB25cswVwEYYzwfQuhTe" actual && rm -r hellodir ' } test_launch_ipfs_daemon_and_mount test_expect_success "'ipfs add --help' succeeds" ' ipfs add --help >actual ' test_expect_success "'ipfs add --help' output looks good" ' egrep "ipfs add.*" actual >/dev/null || test_fsh cat actual ' test_expect_success "'ipfs help add' succeeds" ' ipfs help add >actual ' test_expect_success "'ipfs help add' output looks good" ' egrep "ipfs add.*" actual >/dev/null || test_fsh cat actual ' test_expect_success "'ipfs cat --help' succeeds" ' ipfs cat --help >actual ' test_expect_success "'ipfs cat --help' output looks good" ' egrep "ipfs cat.*" actual >/dev/null || test_fsh cat actual ' test_expect_success "'ipfs help cat' succeeds" ' ipfs help cat >actual ' test_expect_success "'ipfs help cat' output looks good" ' egrep "ipfs cat.*" actual >/dev/null || test_fsh cat actual ' test_add_cat_file test_expect_success "ipfs cat succeeds with stdin opened (issue #1141)" ' cat mountdir/hello.txt | while read line; do ipfs cat "$HASH" >actual || exit; done ' test_expect_success "ipfs cat output looks good" ' cat mountdir/hello.txt >expected && test_cmp expected actual ' test_expect_success "ipfs cat accept hash from built input" ' echo "$HASH" | ipfs cat >actual ' test_expect_success "ipfs cat output looks good" ' test_cmp expected actual ' test_expect_success FUSE "cat ipfs/stuff succeeds" ' cat "ipfs/$HASH" >actual ' test_expect_success FUSE "cat ipfs/stuff looks good" ' test_cmp expected actual ' test_expect_success "'ipfs add -q' succeeds" ' echo "Hello Venus!" >mountdir/venus.txt && ipfs add -q mountdir/venus.txt >actual ' test_expect_success "'ipfs add -q' output looks good" ' HASH="QmU5kp3BH3B8tnWUU2Pikdb2maksBNkb92FHRr56hyghh4" && echo "$HASH" >expected && test_cmp expected actual ' test_expect_success "'ipfs add -q' with stdin input succeeds" ' echo "Hello Jupiter!" | ipfs add -q >actual ' test_expect_success "'ipfs add -q' output looks good" ' HASH="QmUnvPcBctVTAcJpigv6KMqDvmDewksPWrNVoy1E1WP5fh" && echo "$HASH" >expected && test_cmp expected actual ' test_expect_success "'ipfs cat' succeeds" ' ipfs cat "$HASH" >actual ' test_expect_success "ipfs cat output looks good" ' echo "Hello Jupiter!" >expected && test_cmp expected actual ' test_expect_success "'ipfs add' with stdin input succeeds" ' printf "Hello Neptune!\nHello Pluton!" | ipfs add >actual ' test_expect_success "'ipfs add' output looks good" ' HASH="QmZDhWpi8NvKrekaYYhxKCdNVGWsFFe1CREnAjP1QbPaB3" && echo "added $HASH $HASH" >expected && test_cmp expected actual ' test_expect_success "'ipfs cat' with built input succeeds" ' echo "$HASH" | ipfs cat >actual ' test_expect_success "ipfs cat with built input output looks good" ' printf "Hello Neptune!\nHello Pluton!" >expected && test_cmp expected actual ' add_directory() { EXTRA_ARGS=$1 test_expect_success "'ipfs add -r $EXTRA_ARGS' succeeds" ' mkdir mountdir/planets && echo "Hello Mars!" >mountdir/planets/mars.txt && echo "Hello Venus!" >mountdir/planets/venus.txt && ipfs add -r $EXTRA_ARGS mountdir/planets >actual ' test_expect_success "'ipfs add -r $EXTRA_ARGS' output looks good" ' echo "added $MARS planets/mars.txt" >expected && echo "added $VENUS planets/venus.txt" >>expected && echo "added $PLANETS planets" >>expected && test_cmp expected actual ' test_expect_success "ipfs cat accept many hashes from built input" ' { echo "$MARS"; echo "$VENUS"; } | ipfs cat >actual ' test_expect_success "ipfs cat output looks good" ' cat mountdir/planets/mars.txt mountdir/planets/venus.txt >expected && test_cmp expected actual ' test_expect_success "ipfs cat accept many hashes as args" ' ipfs cat "$MARS" "$VENUS" >actual ' test_expect_success "ipfs cat output looks good" ' test_cmp expected actual ' test_expect_success "ipfs cat with both arg and stdin" ' echo "$MARS" | ipfs cat "$VENUS" >actual ' test_expect_success "ipfs cat output looks good" ' cat mountdir/planets/venus.txt >expected && test_cmp expected actual ' test_expect_success "ipfs cat with two args and stdin" ' echo "$MARS" | ipfs cat "$VENUS" "$VENUS" >actual ' test_expect_success "ipfs cat output looks good" ' cat mountdir/planets/venus.txt mountdir/planets/venus.txt >expected && test_cmp expected actual ' test_expect_success "ipfs add --quieter succeeds" ' ipfs add -r -Q $EXTRA_ARGS mountdir/planets >actual ' test_expect_success "ipfs add --quieter returns only one correct hash" ' echo "$PLANETS" > expected && test_cmp expected actual ' test_expect_success "cleanup" ' rm -r mountdir/planets ' } PLANETS="QmWSgS32xQEcXMeqd3YPJLrNBLSdsfYCep2U7CFkyrjXwY" MARS="QmPrrHqJzto9m7SyiRzarwkqPcCSsKR2EB1AyqJfe8L8tN" VENUS="QmU5kp3BH3B8tnWUU2Pikdb2maksBNkb92FHRr56hyghh4" add_directory PLANETS="QmfWfQfKCY5Ukv9peBbxM5vqWM9BzmqUSXvdCgjT2wsiBT" MARS="bafkreibmlvvgdyihetgocpof6xk64kjjzdeq2e4c7hqs3krdheosk4tgj4" VENUS="bafkreihfsphazrk2ilejpekyltjeh5k4yvwgjuwg26ueafohqioeo3sdca" add_directory '--raw-leaves' PLANETS="bafybeih7e5dmkyk25up5vxug4q3hrg2fxbzf23dfrac2fns5h7z4aa7ioi" MARS="bafkreibmlvvgdyihetgocpof6xk64kjjzdeq2e4c7hqs3krdheosk4tgj4" VENUS="bafkreihfsphazrk2ilejpekyltjeh5k4yvwgjuwg26ueafohqioeo3sdca" add_directory '--cid-version=1' PLANETS="bafybeif5tuep5ap2d7zyhbktucey75aoacxufgt6i3v4gebmixyipnyp7y" MARS="bafybeiawta2ntdmsy24aro35w3homzl4ak7svr3si7l7gesvq4erglyye4" VENUS="bafybeicvkvhs2fr75ynebtdjqpgm4g2fc63abqbmysupwpmcjl4gx7mzrm" add_directory '--cid-version=1 --raw-leaves=false' PLANETS="bafykbzaceaptbcs7ik5mdfpot3b4ackvxlwh7loc5jcrtkayf64ukl7zyk46e" MARS="bafk2bzaceaqcxw46uzkyd2jmczoogof6pnkqt4dpiv3pwkunsv4g5rkkmecie" VENUS="bafk2bzacebxnke2fb5mgzxyjuuavvcfht4fd3gvn4klkujz6k72wboynhuvfw" add_directory '--hash=blake2b-256' test_expect_success "'ipfs add -rn' succeeds" ' mkdir -p mountdir/moons/jupiter && mkdir -p mountdir/moons/saturn && echo "Hello Europa!" >mountdir/moons/jupiter/europa.txt && echo "Hello Titan!" >mountdir/moons/saturn/titan.txt && echo "hey you are no moon!" >mountdir/moons/mercury.txt && ipfs add -rn mountdir/moons >actual ' test_expect_success "'ipfs add -rn' output looks good" ' MOONS="QmbGoaQZm8kjYfCiN1aBsgwhqfUBGDYTrDb91Mz7Dvq81B" && EUROPA="Qmbjg7zWdqdMaK2BucPncJQDxiALExph5k3NkQv5RHpccu" && JUPITER="QmS5mZddhFPLWFX3w6FzAy9QxyYkaxvUpsWCtZ3r7jub9J" && SATURN="QmaMagZT4rTE7Nonw8KGSK4oe1bh533yhZrCo1HihSG8FK" && TITAN="QmZzppb9WHn552rmRqpPfgU5FEiHH6gDwi3MrB9cTdPwdb" && MERCURY="QmRsTB5CpEUvDUpDgHCzb3VftZ139zrk9zs5ZcgYh9TMPJ" && echo "added $EUROPA moons/jupiter/europa.txt" >expected && echo "added $MERCURY moons/mercury.txt" >>expected && echo "added $TITAN moons/saturn/titan.txt" >>expected && echo "added $JUPITER moons/jupiter" >>expected && echo "added $SATURN moons/saturn" >>expected && echo "added $MOONS moons" >>expected && test_cmp expected actual ' test_expect_success "random-data is installed" ' type random-data ' test_add_cat_5MB "" "QmapAfmzmeWYTNztMQEhUXFcSGrsax22WRG7YN9xLdMeQq" test_add_cat_5MB --raw-leaves "QmabWSFaPusmiZaaVZLhEUtHcj8CCvVeUfkBpKqAkKVMiS" # note: the specified hash implies that internal nodes are stored # using CidV1 and leaves are stored using raw blocks test_add_cat_5MB --cid-version=1 "bafybeifwdkm32fmukqwh3jofm6ma76bcqvn6opxstsnzmya7utboi4cb2m" # note: the specified hash implies that internal nodes are stored # using CidV1 and leaves are stored using CidV1 but using the legacy # format (i.e. not raw) test_add_cat_5MB '--cid-version=1 --raw-leaves=false' "bafybeifq4unep5w4agr3nlynxidj2rymf6dzu6bf4ieqqildkboe5mdmne" # note: --hash=blake2b-256 implies --cid-version=1 which implies --raw-leaves=true # the specified hash represents the leaf nodes stored as raw leaves and # encoded with the blake2b-256 hash function test_add_cat_5MB '--hash=blake2b-256' "bafykbzacebxcnlql4oc3mtscqn32aumqkqxxv3wt7dkyrphgh6lc2gckiq6bw" # the specified hash represents the leaf nodes stored as protoful nodes and # encoded with the blake2b-256 hash function test_add_cat_5MB '--hash=blake2b-256 --raw-leaves=false' "bafykbzacearibnoamkfmcagpfgk2sbgx65qftnsrh4ttd3g7ghooasfnyavme" test_add_cat_expensive "" "Qma1WZKC3jad7e3F7GEDvkFdhPLyMEhKszBF4nBUCBGh6c" # note: the specified hash implies that internal nodes are stored # using CidV1 and leaves are stored using raw blocks test_add_cat_expensive "--cid-version=1" "bafybeibdfw7nsmb3erhej2k6v4eopaswsf5yfv2ikweqa3qsc5no4jywqu" # note: --hash=blake2b-256 implies --cid-version=1 which implies --raw-leaves=true # the specified hash represents the leaf nodes stored as raw leaves and # encoded with the blake2b-256 hash function test_add_cat_expensive '--hash=blake2b-256' "bafykbzaceduy3thhmcf6ptfqzxberlvj7sgo4uokrvd6qwrhim6r3rgcb26qi" test_add_named_pipe test_add_pwd_is_symlink test_add_cat_raw test_expect_success "ipfs add --cid-version=9 fails" ' echo "context" > afile.txt && test_must_fail ipfs add --cid-version=9 afile.txt 2>&1 | tee add_out && grep -q "unknown CID version" add_out ' test_kill_ipfs_daemon # should work offline test_add_cat_file test_add_cat_raw test_expect_success "ipfs add --only-hash succeeds" ' echo "unknown content for only-hash" | ipfs add --only-hash -q > oh_hash ' test_add_cat_derefargs #TODO: this doesn't work when online hence separated out from test_add_cat_file test_expect_success "ipfs cat file fails" ' test_must_fail ipfs cat $(cat oh_hash) ' test_add_named_pipe test_add_pwd_is_symlink # Test daemon in offline mode test_launch_ipfs_daemon_without_network test_add_cat_file test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0042-add-skip.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Christian Couder # MIT Licensed; see the LICENSE file in this repository. # test_description="Test add and cat commands" . lib/test-lib.sh test_add_skip() { test_expect_success "'ipfs add -r' with hidden file succeeds" ' mkdir -p mountdir/planets/.asteroids && echo "mars.txt" >mountdir/planets/.gitignore && echo "Hello Mars" >mountdir/planets/mars.txt && echo "Hello Venus" >mountdir/planets/venus.txt && echo "Hello Pluto" >mountdir/planets/.pluto.txt && echo "Hello Charon" >mountdir/planets/.charon.txt && echo "Hello Ceres" >mountdir/planets/.asteroids/ceres.txt && echo "Hello Pallas" >mountdir/planets/.asteroids/pallas.txt && ipfs add -r mountdir/planets >actual ' test_expect_success "'ipfs add -r' did not include . files" ' cat >expected <<-\EOF && added QmZy3khu7qf696i5HtkgL2NotsCZ8wzvNZJ1eUdA5n8KaV planets/mars.txt added QmQnv4m3Q5512zgVtpbJ9z85osQrzZzGRn934AGh6iVEXz planets/venus.txt added QmR8nD1Vzk5twWVC6oShTHvv7mMYkVh6dApCByBJyV2oj3 planets EOF test_cmp expected actual ' test_expect_success "'ipfs add -r --hidden' succeeds" ' ipfs add -r --hidden mountdir/planets >actual ' test_expect_success "'ipfs add -r --hidden' did include . files" ' cat >expected <<-\EOF && added QmcAREBcjgnUpKfyFmUGnfajA1NQS5ydqRp7WfqZ6JF8Dx planets/.asteroids/ceres.txt added QmZ5eaLybJ5GUZBNwy24AA9EEDTDpA4B8qXnuN3cGxu2uF planets/.asteroids/pallas.txt added QmaowqjedBkUrMUXgzt9c2ZnAJncM9jpJtkFfgdFstGr5a planets/.charon.txt added QmPHrRjTH8FskN3C2iv6BLekDT94o23KSL2u5qLqQqGhVH planets/.gitignore added QmU4zFD5eJtRBsWC63AvpozM9Atiadg9kPVTuTrnCYJiNF planets/.pluto.txt added QmZy3khu7qf696i5HtkgL2NotsCZ8wzvNZJ1eUdA5n8KaV planets/mars.txt added QmQnv4m3Q5512zgVtpbJ9z85osQrzZzGRn934AGh6iVEXz planets/venus.txt added Qmf6rbs5GF85anDuoxpSAdtuZPM9D2Yt3HngzjUVSQ7kDV planets/.asteroids added QmczhHaXyb3bc9APMxe4MXbr87V5YDLKLaw3DZX3fK7HrK planets EOF test_cmp expected actual ' test_expect_success "'ipfs add -r --ignore-rules-path=.gitignore --hidden' succeeds" ' (cd mountdir/planets && ipfs add -r --ignore-rules-path=.gitignore --hidden .) > actual ' test_expect_success "'ipfs add -r --ignore-rules-path=.gitignore --hidden' did not include mars.txt file" ' cat >expected <<-\EOF && added QmcAREBcjgnUpKfyFmUGnfajA1NQS5ydqRp7WfqZ6JF8Dx planets/.asteroids/ceres.txt added QmZ5eaLybJ5GUZBNwy24AA9EEDTDpA4B8qXnuN3cGxu2uF planets/.asteroids/pallas.txt added QmaowqjedBkUrMUXgzt9c2ZnAJncM9jpJtkFfgdFstGr5a planets/.charon.txt added QmPHrRjTH8FskN3C2iv6BLekDT94o23KSL2u5qLqQqGhVH planets/.gitignore added QmU4zFD5eJtRBsWC63AvpozM9Atiadg9kPVTuTrnCYJiNF planets/.pluto.txt added QmQnv4m3Q5512zgVtpbJ9z85osQrzZzGRn934AGh6iVEXz planets/venus.txt added Qmf6rbs5GF85anDuoxpSAdtuZPM9D2Yt3HngzjUVSQ7kDV planets/.asteroids added QmaRsiaCYvc65RqHVAcv2tqyjZgQYgvaNqW1tQGsjfy4N5 planets EOF test_cmp expected actual ' test_expect_success "'ipfs add -r --ignore-rules-path=.gitignore --ignore .asteroids --ignore venus.txt --hidden' succeeds" ' (cd mountdir/planets && ipfs add -r --ignore-rules-path=.gitignore --ignore .asteroids --ignore venus.txt --hidden .) > actual ' test_expect_success "'ipfs add -r --ignore-rules-path=.gitignore --ignore .asteroids --ignore venus.txt --hidden' did not include ignored files" ' cat >expected <<-\EOF && added QmaowqjedBkUrMUXgzt9c2ZnAJncM9jpJtkFfgdFstGr5a planets/.charon.txt added QmPHrRjTH8FskN3C2iv6BLekDT94o23KSL2u5qLqQqGhVH planets/.gitignore added QmU4zFD5eJtRBsWC63AvpozM9Atiadg9kPVTuTrnCYJiNF planets/.pluto.txt added QmemuMahjSh7eYLY3hbz2q8sqMPnbQzBQeUdosqNiWChE6 planets EOF test_cmp expected actual ' test_expect_success "'ipfs add' includes hidden files given explicitly even without --hidden" ' mkdir -p mountdir/dotfiles && echo "set nocompatible" > mountdir/dotfiles/.vimrc cat >expected <<-\EOF && added QmT4uMRDCN7EMpFeqwvKkboszbqeW1kWVGrBxBuCGqZcQc .vimrc EOF ipfs add mountdir/dotfiles/.vimrc >actual cat actual test_cmp expected actual ' test_expect_success "'ipfs add' with an unregistered hash and wrapped leaves fails without crashing" ' test_expect_code 1 ipfs add --hash poseidon-bls12_381-a2-fc1 --raw-leaves=false -r mountdir/planets ' } # should work offline test_init_ipfs test_add_skip # should work online test_launch_ipfs_daemon test_add_skip test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0043-add-w.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Christian Couder # MIT Licensed; see the LICENSE file in this repository. # test_description="Test add -w" add_w_m='QmbDfuW3tZ5PmAucyLBAMzVeETHCHM7Ho9CWdBvWxRGd3i' add_w_1='added QmP9WCV5SjQRoxoCkgywzw4q5X23rhHJJXzPQt4VbNa9M5 0h0r91 added Qmave82G8vLbtx6JCokrrhLPpFNfWj5pbXobddiUASfpe3 ' add_w_12='added QmP9WCV5SjQRoxoCkgywzw4q5X23rhHJJXzPQt4VbNa9M5 0h0r91 added QmNUiT9caQy5zXvw942UYXkjLseQLWBkf7ZJD6RCfk8JgP 951op added QmWXoq9vUtdNxmM16kvJRgyQdi4S4gfYSjd2MsRprBXWmG ' add_w_d1='added QmQKZCZKKL71zcMNpFFVcWzoh5dimX45mKgUu3LhvdaCRn 3s78oa/cb5v5v added QmPng2maSno8o659Lu2QtKg2d2L53RMahoyK6wNkifYaxY 3s78oa/cnd062l-rh added QmX3s7jJjFQhKRuGpDA3W4BYHdCWAyL3oB6U3iSoaYxVxs 3s78oa/es3gm9ck7b added QmSUZXb48DoNjUPpX9Jue1mUpyCghEDZY62iif1JhdofoG 3s78oa/kfo77-6i_hp0ttz added QmdC215Wp2sH47aw6R9CLBVa5uxJB4zEag1gtsKqjYGDb5 3s78oa/p91vs5t added QmSEGJRYb5wrJRBxNsse91YJSpmgf5ikKRtCwvGZ1V1Nc2 3s78oa added QmS2ML7DPVisc4gQtSrwMi3qwS9eyzGR7zVdwqwRPU9rGz ' add_w_d1_v1='added bafkreibpfapmbmf55elpipnoofmda7xbs5spthba2srrovnchttzplmrnm fvmq97/0vz12t0yf added bafkreihc5hdzpjwbqy6b5r2h2oxbm6mp4sx4eqll253k6f5yijsismvoxy fvmq97/2hpfk8slf0 added bafkreihlmwk6pkk7klsmypmk2wfkgijbk7wavhtrcvgrfxvug7x5ndawge fvmq97/nda000755cd76 added bafkreigpntro6bt4m6c5pcnmvk24qyiq3lwffhwry7k2hqtretqhfsfvqa fvmq97/nsz0wsonz added bafkreieeznfvzr6742npktcn4ajzxujst6j2uztwfninhvic4bbvm356u4 fvmq97/pq3f6t0 added bafybeiatm3oos62mm5hu4cmq234wipw2fjaqflq2cdqgc6i6dcgzamxwrm fvmq97 added bafybeifp4ioszjk2377psexdhk7thcxnpaj2wls4yifsntbgxzti7ds4uy ' add_w_d2='added QmP9WCV5SjQRoxoCkgywzw4q5X23rhHJJXzPQt4VbNa9M5 0h0r91 added QmPpv7rFgkBqMYKJok6kVixqJgAGkyPiX3Jrr7n9rU1gcv 1o8ef-25onywi added QmW7zDxGpaJTRpte7uCvMA9eXJ5L274FfsFPK9pE5RShq9 2ju9tn-b09/-qw1d8j9 added QmNNm9D3pn8NXbuYSde614qbb9xE67g9TNV6zXePgSZvHj 2ju9tn-b09/03rfc61t4qq_m added QmUYefaFAWka9LWarDeetQFe8CCSHaAtj4JR7YToYPSJyi 2ju9tn-b09/57dl-1lbjvu added QmcMLvVinwJsHtYxTUXEoPd8XkbuyvJNffZ85PT11cWDc2 2ju9tn-b09/t8h1_w added QmUTZE57VoF7xqWmrrcDNtDXrEs6znTQaRwmwkawGDs1GA 2ju9tn-b09/ugqi0nmv-1 added QmfX5q9CMquL4JnAuG4H13RXjTb9DncMfu9pvpEsWkECJk fvmq97/0vz12t0yf added Qmdr3jR1UATLFeuoieBTHLNNwhCUJbgN5oat7U9X8TtfdZ fvmq97/2hpfk8slf0 added QmfUKgXSiE1wCQuX3Pws9FftthJuAMXrDWhG5EhhnmA6gQ fvmq97/nda000755cd76 added QmYM35pgHvLdKH8ssw9kJeiUY5kcjhb5h3BTiDhAgbsYYh fvmq97/nsz0wsonz added QmNarBSVwzYjLeEjGMJqTNtRCYGCLGo6TJqd21hPi7WXFT fvmq97/pq3f6t0 added QmUNhQpFBZvfH4JyNxiE8QY31bZDpQHMmjSRRnbRZYZ3be 2ju9tn-b09 added QmWtZu8dv4XRK8zPmwbNjS6biqe4bGEF9J5zb51sBJCMro fvmq97 added QmYp7QoL8wRacLn9pJftJSkiiSmNGdWb7qT5ENDW2HXBcu ' add_w_r='QmUerh2irM8cngqJHLGKCn4AGBSyHYAUi8i8zyVzXKNYyb' . lib/test-lib.sh test_add_w() { test_expect_success "random-files is installed" ' type random-files ' test_expect_success "random-files generates test files" ' random-files --seed 7547632 --files 5 --dirs 2 --depth 3 m && echo "$add_w_m" >expected && ipfs add -Q -r m >actual && test_sort_cmp expected actual ' # test single file test_expect_success "ipfs add -w (single file) succeeds" ' ipfs add -w m/0h0r91 >actual ' test_expect_success "ipfs add -w (single file) is correct" ' echo "$add_w_1" >expected && test_sort_cmp expected actual ' # test two files together test_expect_success "ipfs add -w (multiple) succeeds" ' ipfs add -w m/0h0r91 m/951op >actual ' test_expect_success "ipfs add -w (multiple) is correct" ' echo "$add_w_12" >expected && test_sort_cmp expected actual ' test_expect_success "ipfs add -w (multiple) succeeds" ' ipfs add -w m/951op m/0h0r91 >actual ' test_expect_success "ipfs add -w (multiple) orders" ' echo "$add_w_12" >expected && test_sort_cmp expected actual ' # test a directory test_expect_success "ipfs add -w -r (dir) succeeds" ' ipfs add -r -w m/9m7mh3u51z3b/3s78oa >actual ' test_expect_success "ipfs add -w -r (dir) is correct" ' echo "$add_w_d1" >expected && test_sort_cmp expected actual ' # test files and directory test_expect_success "ipfs add -w -r succeeds" ' ipfs add -w -r m/9m7mh3u51z3b/1o8ef-25onywi \ m/vck_-2/2ju9tn-b09 m/9m7mh3u51z3b/fvmq97 m/0h0r91 >actual ' test_expect_success "ipfs add -w -r is correct" ' echo "$add_w_d2" >expected && test_sort_cmp expected actual ' # test -w -r m/* == -r m test_expect_success "ipfs add -w -r m/* == add -r m succeeds" ' ipfs add -Q -w -r m/* >actual ' test_expect_success "ipfs add -w -r m/* == add -r m is correct" ' echo "$add_w_m" >expected && test_sort_cmp expected actual ' # test repeats together test_expect_success "ipfs add -w (repeats) succeeds" ' ipfs add -Q -w -r m/9m7mh3u51z3b/1o8ef-25onywi m/vck_-2/2ju9tn-b09 \ m/9m7mh3u51z3b/fvmq97 m/0h0r91 m/9m7mh3u51z3b m/9m7mh3u51z3b m/0h0r91 \ m/0h0r91 m/vck_-2/0dl083je2 \ m/vck_-2/2ju9tn-b09/-qw1d8j9 >actual ' test_expect_success "ipfs add -w (repeats) is correct" ' echo "$add_w_r" >expected && test_sort_cmp expected actual ' test_expect_success "ipfs add -w -r (dir) --cid-version=1 succeeds" ' ipfs add -r -w --cid-version=1 m/9m7mh3u51z3b/fvmq97 >actual ' test_expect_success "ipfs add -w -r (dir) --cid-version=1 is correct" ' echo "$add_w_d1_v1" >expected && test_sort_cmp expected actual ' test_expect_success "ipfs add -w -r -n (dir) --cid-version=1 succeeds" ' ipfs add -r -w -n --cid-version=1 m/9m7mh3u51z3b/fvmq97 >actual ' test_expect_success "ipfs add -w -r -n (dir) --cid-version=1 is correct" ' echo "$add_w_d1_v1" > expected && test_sort_cmp expected actual ' } test_init_ipfs test_add_w test_launch_ipfs_daemon test_add_w test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0044-add-symlink.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Christian Couder # MIT Licensed; see the LICENSE file in this repository. # test_description="Test add -w" . lib/test-lib.sh test_expect_success "creating files succeeds" ' mkdir -p files/foo && mkdir -p files/bar && echo "some text" > files/foo/baz && ln -s files/foo/baz files/bar/baz && ln -s files/does/not/exist files/bad ' test_add_symlinks() { test_expect_success "ipfs add files succeeds" ' ipfs add -Q -r files >filehash_out ' test_expect_success "output looks good" ' echo QmWdiHKoeSW8G1u7ATCgpx4yMoUhYaJBQGkyPLkS9goYZ8 > filehash_exp && test_cmp filehash_exp filehash_out ' test_expect_success "ipfs add --cid-version=1 files succeeds" ' ipfs add -Q -r --cid-version=1 files >filehash_out ' test_expect_success "output looks good" ' # note this hash implies all internal nodes are stored using CidV1 echo bafybeibyhlx64cklod6isy3h7tsmr4qvam3ae3b74n3hfes5bythjrwyua > filehash_exp && test_cmp filehash_exp filehash_out ' test_expect_success "adding a symlink adds the link itself" ' ipfs add -q files/bar/baz > goodlink_out ' test_expect_success "output looks good" ' echo "QmdocmZeF7qwPT9Z8SiVhMSyKA2KKoA2J7jToW6z6WBmxR" > goodlink_exp && test_cmp goodlink_exp goodlink_out ' test_expect_success "adding a broken symlink works" ' ipfs add -q files/bad > badlink_out ' test_expect_success "output looks good" ' echo "QmWYN8SEXCgNT2PSjB6BnxAx6NJQtazWoBkTRH9GRfPFFQ" > badlink_exp && test_cmp badlink_exp badlink_out ' test_expect_success "adding with symlink in middle of path is same as\ adding with no symlink" ' mkdir -p files2/a/b/c && echo "some other text" > files2/a/b/c/foo && ln -s b files2/a/d ipfs add -rq files2/a/b/c > no_sym && ipfs add -rq files2/a/d/c > sym && test_cmp no_sym sym ' } test_init_ipfs test_add_symlinks test_launch_ipfs_daemon test_add_symlinks test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0045-ls.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Christian Couder # MIT Licensed; see the LICENSE file in this repository. # test_description="Test ls command" . lib/test-lib.sh test_init_ipfs test_ls_cmd() { test_expect_success "'ipfs add -r testData' succeeds" ' mkdir -p testData testData/d1 testData/d2 && echo "test" >testData/f1 && echo "data" >testData/f2 && echo "hello" >testData/d1/a && random-data -size=128 -seed=42 >testData/d1/128 && echo "world" >testData/d2/a && random-data -size=1024 -seed=42 >testData/d2/1024 && echo "badname" >testData/d2/`echo -e "bad\x7fname.txt"` && ipfs add -r testData >actual_add ' test_expect_success "'ipfs add' output looks good" ' cat <<-\EOF >expected_add && added QmWUixdcx1VJtpuAgXAy4e3JPAbEoHE6VEDut5KcYcpuGN testData/d1/128 added QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN testData/d1/a added QmZHVTX2epinyx5baTFV2L2ap9VtgbmfeFdhgntAypT5N3 testData/d2/1024 added QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL testData/d2/a added QmQSLRRd1Lxn6NMsWmmj2g9W3LtSRfmVAVqU3ShneLUrbn testData/d2/bad\x7fname.txt added QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH testData/f1 added QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M testData/f2 added QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j testData/d1 added Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW testData/d2 added QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc testData EOF test_cmp expected_add actual_add ' test_expect_success "'ipfs ls ' succeeds" ' ipfs ls QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j >actual_ls ' test_expect_success "'ipfs ls ' output looks good" ' cat <<-\EOF >expected_ls && QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc: QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j - d1/ Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW - d2/ QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH 5 f1 QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M 5 f2 Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW: QmZHVTX2epinyx5baTFV2L2ap9VtgbmfeFdhgntAypT5N3 1024 1024 QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 6 a QmQSLRRd1Lxn6NMsWmmj2g9W3LtSRfmVAVqU3ShneLUrbn 8 bad\x7fname.txt QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j: QmWUixdcx1VJtpuAgXAy4e3JPAbEoHE6VEDut5KcYcpuGN 128 128 QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 6 a EOF test_cmp expected_ls actual_ls ' test_expect_success "'ipfs ls --size=false ' succeeds" ' ipfs ls --size=false QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j >actual_ls ' test_expect_success "'ipfs ls ' output looks good" ' cat <<-\EOF >expected_ls && QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc: QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j d1/ Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW d2/ QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH f1 QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M f2 Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW: QmZHVTX2epinyx5baTFV2L2ap9VtgbmfeFdhgntAypT5N3 1024 QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL a QmQSLRRd1Lxn6NMsWmmj2g9W3LtSRfmVAVqU3ShneLUrbn bad\x7fname.txt QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j: QmWUixdcx1VJtpuAgXAy4e3JPAbEoHE6VEDut5KcYcpuGN 128 QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN a EOF test_cmp expected_ls actual_ls ' test_expect_success "'ipfs ls --headers ' succeeds" ' ipfs ls --headers QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j >actual_ls_headers ' test_expect_success "'ipfs ls --headers ' output looks good" ' cat <<-\EOF >expected_ls_headers && QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc: Hash Size Name QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j - d1/ Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW - d2/ QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH 5 f1 QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M 5 f2 Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW: Hash Size Name QmZHVTX2epinyx5baTFV2L2ap9VtgbmfeFdhgntAypT5N3 1024 1024 QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 6 a QmQSLRRd1Lxn6NMsWmmj2g9W3LtSRfmVAVqU3ShneLUrbn 8 bad\x7fname.txt QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j: Hash Size Name QmWUixdcx1VJtpuAgXAy4e3JPAbEoHE6VEDut5KcYcpuGN 128 128 QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 6 a EOF test_cmp expected_ls_headers actual_ls_headers ' test_expect_success "'ipfs ls --size=false --cid-base=base32 ' succeeds" ' ipfs ls --size=false --cid-base=base32 $(cid-fmt -v 1 -b base32 %s QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j) >actual_ls_base32 ' test_expect_success "'ipfs ls --size=false --cid-base=base32 ' output looks good" ' cid-fmt -b base32 -v 1 --filter %s < expected_ls > expected_ls_base32 test_cmp expected_ls_base32 actual_ls_base32 ' } test_ls_cmd_streaming() { test_expect_success "'ipfs add -r testData' succeeds" ' mkdir -p testData testData/d1 testData/d2 && echo "test" >testData/f1 && echo "data" >testData/f2 && echo "hello" >testData/d1/a && random-data -size=128 -seed=42 >testData/d1/128 && echo "world" >testData/d2/a && random-data -size=1024 -seed=42 >testData/d2/1024 && echo "badname" >testData/d2/`echo -e "bad\x7fname.txt"` && ipfs add -r testData >actual_add ' test_expect_success "'ipfs add' output looks good" ' cat <<-\EOF >expected_add && added QmWUixdcx1VJtpuAgXAy4e3JPAbEoHE6VEDut5KcYcpuGN testData/d1/128 added QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN testData/d1/a added QmZHVTX2epinyx5baTFV2L2ap9VtgbmfeFdhgntAypT5N3 testData/d2/1024 added QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL testData/d2/a added QmQSLRRd1Lxn6NMsWmmj2g9W3LtSRfmVAVqU3ShneLUrbn testData/d2/bad\x7fname.txt added QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH testData/f1 added QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M testData/f2 added QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j testData/d1 added Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW testData/d2 added QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc testData EOF test_cmp expected_add actual_add ' test_expect_success "'ipfs ls --stream ' succeeds" ' ipfs ls --stream QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j >actual_ls_stream ' test_expect_success "'ipfs ls --stream ' output looks good" ' cat <<-\EOF >expected_ls_stream && QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc: QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j - d1/ Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW - d2/ QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH 5 f1 QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M 5 f2 Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW: QmZHVTX2epinyx5baTFV2L2ap9VtgbmfeFdhgntAypT5N3 1024 1024 QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 6 a QmQSLRRd1Lxn6NMsWmmj2g9W3LtSRfmVAVqU3ShneLUrbn 8 bad\x7fname.txt QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j: QmWUixdcx1VJtpuAgXAy4e3JPAbEoHE6VEDut5KcYcpuGN 128 128 QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 6 a EOF test_cmp expected_ls_stream actual_ls_stream ' test_expect_success "'ipfs ls --size=false --stream ' succeeds" ' ipfs ls --size=false --stream QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j >actual_ls_stream ' test_expect_success "'ipfs ls --size=false --stream ' output looks good" ' cat <<-\EOF >expected_ls_stream && QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc: QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j d1/ Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW d2/ QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH f1 QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M f2 Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW: QmZHVTX2epinyx5baTFV2L2ap9VtgbmfeFdhgntAypT5N3 1024 QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL a QmQSLRRd1Lxn6NMsWmmj2g9W3LtSRfmVAVqU3ShneLUrbn bad\x7fname.txt QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j: QmWUixdcx1VJtpuAgXAy4e3JPAbEoHE6VEDut5KcYcpuGN 128 QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN a EOF test_cmp expected_ls_stream actual_ls_stream ' test_expect_success "'ipfs ls --stream --headers ' succeeds" ' ipfs ls --stream --headers QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j >actual_ls_stream_headers ' test_expect_success "'ipfs ls --stream --headers ' output looks good" ' cat <<-\EOF >expected_ls_stream_headers && QmR5UuxvF2ALd2GRGMCNg1GDiuuvcAyEkQaCV9fNkevWuc: Hash Size Name QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j - d1/ Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW - d2/ QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH 5 f1 QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M 5 f2 Qmapxr4zxxUjoUFzyggydRZDkcJknjbtahYFKokbBAVghW: Hash Size Name QmZHVTX2epinyx5baTFV2L2ap9VtgbmfeFdhgntAypT5N3 1024 1024 QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 6 a QmQSLRRd1Lxn6NMsWmmj2g9W3LtSRfmVAVqU3ShneLUrbn 8 bad\x7fname.txt QmWWEQhcLufF3qPmmbUjqH7WVWBT9JrGJwPiVTryCoBs2j: Hash Size Name QmWUixdcx1VJtpuAgXAy4e3JPAbEoHE6VEDut5KcYcpuGN 128 128 QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 6 a EOF test_cmp expected_ls_stream_headers actual_ls_stream_headers ' } test_ls_cmd_raw_leaves() { test_expect_success "'ipfs add -r --raw-leaves' then 'ipfs ls' works as expected" ' mkdir -p somedir && echo bar > somedir/foo && ipfs add --raw-leaves -r somedir/ > /dev/null && ipfs ls '$1' QmThNTdtKaVoCVrYmM5EBS6U3S5vfKFue2TxbxxAxRcKKE > ls-actual echo "bafkreid5qzpjlgzem2iyzgddv7fjilipxcoxzgwazgn27q3usucn5wlxga 4 foo" > ls-expect test_cmp ls-actual ls-expect ' } test_ls_object() { test_expect_success "ipfs add medium size file then 'ipfs ls --size=false' works as expected" ' random-data -size=500000 -seed=2 > somefile && HASH=$(ipfs add somefile -q) && echo "QmWJuiG6dhfwo3KXxCc9gkdizoMoXbLMCDiTTZgEhSmyyo " > ls-expect && echo "QmNPxtpjhoXMRVKm4oSwcJaS4fck5FR4LufPd5KJr4jYhm " >> ls-expect && ipfs ls --size=false $HASH > ls-actual && test_cmp ls-actual ls-expect ' test_expect_success "ipfs add medium size file then 'ipfs ls' works as expected" ' random-data -size=500000 -seed=2 > somefile && HASH=$(ipfs add somefile -q) && echo "QmWJuiG6dhfwo3KXxCc9gkdizoMoXbLMCDiTTZgEhSmyyo 262144 " > ls-expect && echo "QmNPxtpjhoXMRVKm4oSwcJaS4fck5FR4LufPd5KJr4jYhm 237856 " >> ls-expect && ipfs ls $HASH > ls-actual && test_cmp ls-actual ls-expect ' } # should work offline test_ls_cmd test_ls_cmd_streaming test_ls_cmd_raw_leaves test_ls_cmd_raw_leaves --size test_ls_object # should work online test_launch_ipfs_daemon test_ls_cmd test_ls_cmd_streaming test_ls_cmd_raw_leaves test_ls_cmd_raw_leaves --size test_kill_ipfs_daemon test_ls_object # # test for ls --resolve-type=false # test_expect_success "'ipfs add -r' succeeds" ' mkdir adir && # note: not using a seed as the files need to have truly random content random-data -size=1000 > adir/file1 && random-data -size=1000 > adir/file2 && ipfs add --pin=false -q -r adir > adir-hashes ' test_expect_success "get hashes from add output" ' FILE=`head -1 adir-hashes` && DIR=`tail -1 adir-hashes` && test "$FILE" -a "$DIR" ' test_expect_success "remove a file in dir" ' ipfs block rm $FILE ' test_expect_success "'ipfs ls --resolve-type=false ' fails" ' test_must_fail ipfs ls --resolve-type=false $DIR > /dev/null ' test_expect_success "'ipfs ls' fails" ' test_must_fail ipfs ls $DIR ' test_expect_success "'ipfs ls --resolve-type=true --size=false' fails" ' test_must_fail ipfs ls --resolve-type=true --size=false $DIR ' test_launch_ipfs_daemon_without_network test_expect_success "'ipfs ls --resolve-type=false --size=false' ok" ' ipfs ls --resolve-type=false --size=false $DIR > /dev/null ' test_expect_success "'ipfs ls' fails" ' test_must_fail ipfs ls $DIR ' test_expect_success "'ipfs ls --resolve-type=false --size=true' fails" ' test_must_fail ipfs ls --resolve-type=false --size=true $DIR ' test_kill_ipfs_daemon test_launch_ipfs_daemon # now we try `ipfs ls --resolve-type=false` with the daemon online It # should not even attempt to retrieve the file from the network. If # it does it should eventually fail as the content is random and # should not exist on the network, but we don't want to wait for a # timeout so we will kill the request after a few seconds test_expect_success "'ipfs ls --resolve-type=false --size=false' ok and does not hang" ' go-timeout 2 ipfs ls --resolve-type=false --size=false $DIR ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0046-id-hash.sh ================================================ #!/usr/bin/env bash test_description="Test basic operations with identity hash" . lib/test-lib.sh test_init_ipfs ID_HASH0=bafkqaedknncdsodknncdsnzvnbvuioak ID_HASH0_CONTENTS=jkD98jkD975hkD8 test_expect_success "can fetch random identity hash" ' ipfs cat $ID_HASH0 > expected && echo $ID_HASH0_CONTENTS > actual && test_cmp expected actual ' test_expect_success "can pin random identity hash" ' ipfs pin add $ID_HASH0 ' test_expect_success "ipfs add succeeds with identity hash" ' echo "djkd7jdkd7jkHHG" > junk.txt && HASH=$(ipfs add -q --hash=identity junk.txt) ' test_expect_success "content not actually added" ' ipfs refs local > locals && test_should_not_contain $HASH locals ' test_expect_success "but can fetch it anyway" ' ipfs cat $HASH > actual && test_cmp junk.txt actual ' test_expect_success "block rm does nothing" ' ipfs pin rm $HASH && ipfs block rm $HASH ' test_expect_success "can still fetch it" ' ipfs cat $HASH > actual test_cmp junk.txt actual ' test_expect_success "ipfs add --inline works as expected" ' echo $ID_HASH0_CONTENTS > afile && HASH=$(ipfs add -q --inline afile) ' test_expect_success "ipfs add --inline uses identity multihash" ' MHTYPE=`cid-fmt %h $HASH` echo "mhtype is $MHTYPE" test "$MHTYPE" = identity ' test_expect_success "ipfs add --inline --raw-leaves works as expected" ' echo $ID_HASH0_CONTENTS > afile && HASH=$(ipfs add -q --inline --raw-leaves afile) ' test_expect_success "ipfs add --inline --raw-leaves outputs the correct hash" ' echo "$ID_HASH0" = "$HASH" && test "$ID_HASH0" = "$HASH" ' test_expect_success "create 1000 bytes file and get its hash" ' random-data -size=1000 -seed=2 > 1000bytes && HASH0=$(ipfs add -q --raw-leaves --only-hash 1000bytes) ' test_expect_success "ipfs add --inline --raw-leaves works as expected on large file" ' HASH=$(ipfs add -q --inline --raw-leaves 1000bytes) ' test_expect_success "ipfs add --inline --raw-leaves outputs the correct hash on large file" ' echo "$HASH0" = "$HASH" && test "$HASH0" = "$HASH" ' test_expect_success "enable filestore" ' ipfs config --json Experimental.FilestoreEnabled true ' test_expect_success "can fetch random identity hash (filestore enabled)" ' ipfs cat $ID_HASH0 > expected && echo $ID_HASH0_CONTENTS > actual && test_cmp expected actual ' test_expect_success "can pin random identity hash (filestore enabled)" ' ipfs pin add $ID_HASH0 ' test_expect_success "ipfs add succeeds with identity hash and --nocopy" ' echo "djkd7jdkd7jkHHG" > junk.txt && HASH=$(ipfs add -q --hash=identity --nocopy junk.txt) ' test_expect_success "content not actually added (filestore enabled)" ' ipfs refs local > locals && test_should_not_contain $HASH locals ' test_expect_success "but can fetch it anyway (filestore enabled)" ' ipfs cat $HASH > actual && test_cmp junk.txt actual ' test_done ================================================ FILE: test/sharness/t0047-add-mode-mtime.sh ================================================ #!/usr/bin/env bash test_description="Test storing and retrieving mode and mtime" . lib/test-lib.sh test_init_ipfs test_expect_success "set Import defaults to ensure deterministic cids for mod and mtime tests" ' ipfs config --json Import.CidVersion 0 && ipfs config Import.HashFunction sha2-256 && ipfs config Import.UnixFSChunker size-262144 ' HASH_NO_PRESERVE=QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH PRESERVE_MTIME=1604320482 PRESERVE_MODE="0640" HASH_PRESERVE_MODE=QmQLgxypSNGNFTuUPGCecq6dDEjb6hNB5xSyVmP3cEuNtq HASH_PRESERVE_MTIME=QmQ6kErEW8kztQFV8vbwNU8E4dmtGsYpRiboiLxUEwibvj HASH_PRESERVE_LINK_MTIME=QmbJwotgtr84JxcnjpwJ86uZiyMoxbZuNH4YrdJMypkYaB HASH_PRESERVE_MODE_AND_MTIME=QmYkvboLsvLFcSYmqVJRxvBdYRQLroLv9kELf3LRiCqBri CUSTOM_MTIME=1603539720 CUSTOM_MTIME_NSECS=54321 CUSTOM_MODE="0764" HASH_CUSTOM_MODE=QmchD3BN8TQ3RW6jPLxSaNkqvfuj7syKhzTRmL4EpyY1Nz HASH_CUSTOM_MTIME=QmT3aY4avDcYXCWpU8CJzqUkW7YEuEsx36S8cTNoLcuK1B HASH_CUSTOM_MTIME_NSECS=QmaKH8H5rXBUBCX4vdxi7ktGQEL7wejV7L9rX2qpZjwncz HASH_CUSTOM_MODE_AND_MTIME=QmUkxrtBA8tPjwCYz1HrsoRfDz6NgKut3asVeHVQNH4C8L HASH_CUSTOM_LINK_MTIME=QmV1Uot2gy4bhY9yvYiZxhhchhyYC6MKKoGV1XtWNmpCLe HASH_CUSTOM_LINK_MTIME_NSECS=QmPHYCxYvvHj6VxiPNJ3kXxcPsnJLDYUJqsDJWjvytmrmY mk_name() { tr -dc '[:alnum:]'expected_in && ipfs block put expected_out && test_cmp expected_out actual_out ' test_expect_success "'ipfs block put' with 2 files succeeds" ' echo "Hello Mars!" > a && echo "Hello Venus!" > b && ipfs block put a b | tee actual_out ' test_expect_success "'ipfs block put' output looks good" ' echo "$HASH" >expected_out && echo "$HASHB" >>expected_out && test_cmp expected_out actual_out ' test_expect_success "can set cid codec on block put" ' CODEC_HASH=$(ipfs block put --cid-codec=dag-pb ../t0050-block-data/testPut.pb) ' test_expect_success "block get output looks right" ' ipfs block get $CODEC_HASH > pb_block_out && test_cmp pb_block_out ../t0050-block-data/testPut.pb ' # # "block get" tests # test_expect_success "'ipfs block get' succeeds" ' ipfs block get $HASH >actual_in ' test_expect_success "'ipfs block get' output looks good" ' test_cmp expected_in actual_in ' # # "block stat" tests # test_expect_success "'ipfs block stat' succeeds" ' ipfs block stat $HASH >actual_stat ' test_expect_success "'ipfs block stat' output looks good" ' echo "Key: $HASH" >expected_stat && echo "Size: 12" >>expected_stat && test_cmp expected_stat actual_stat ' # # "block rm" tests # test_expect_success "'ipfs block rm' succeeds" ' ipfs block rm $HASH >actual_rm ' test_expect_success "'ipfs block rm' output looks good" ' echo "removed $HASH" > expected_rm && test_cmp expected_rm actual_rm ' test_expect_success "'ipfs block rm' block actually removed" ' test_must_fail ipfs block stat $HASH ' RANDOMHASH=QmRKqGMAM6EbngbZjSqrvYzq5Qd8b1bSWymjSUY9zQSNDq DIRHASH=QmdWmVmM6W2abTgkEfpbtA1CJyTWS2rhuUB9uP1xV8Uwtf FILE1HASH=Qmae3RedM7SNkWGsdzYzsr6svmsFdsva4WoTvYYsWhUSVz FILE2HASH=QmUtkGLvPf63NwVzLPKPUYgwhn8ZYPWF6vKWN3fZ2amfJF FILE3HASH=Qmesmmf1EEG1orJb6XdK6DabxexsseJnCfw8pqWgonbkoj TESTHASH=QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH test_expect_success "add and pin directory" ' echo "test" | ipfs add --pin=false && mkdir adir && echo "file1" > adir/file1 && echo "file2" > adir/file2 && echo "file3" > adir/file3 && ipfs add -r adir ipfs pin add -r $DIRHASH ' test_expect_success "can't remove pinned block" ' test_must_fail ipfs block rm $DIRHASH 2> block_rm_err ' test_expect_success "can't remove pinned block: output looks good" ' grep -q "$DIRHASH: pinned: recursive" block_rm_err ' test_expect_success "can't remove indirectly pinned block" ' test_must_fail ipfs block rm $FILE1HASH 2> block_rm_err ' test_expect_success "can't remove indirectly pinned block: output looks good" ' grep -q "$FILE1HASH: pinned via $DIRHASH" block_rm_err ' test_expect_success "remove pin" ' ipfs pin rm -r $DIRHASH ' test_expect_success "multi-block 'ipfs block rm' succeeds" ' ipfs block rm $FILE1HASH $FILE2HASH $FILE3HASH > actual_rm ' test_expect_success "multi-block 'ipfs block rm' output looks good" ' grep -F -q "removed $FILE1HASH" actual_rm && grep -F -q "removed $FILE2HASH" actual_rm && grep -F -q "removed $FILE3HASH" actual_rm ' test_expect_success "multi-block 'ipfs block rm '" ' test_must_fail ipfs block rm $RANDOMHASH $TESTHASH $RANDOMHASH &> actual_mixed_rm ' test_expect_success "multi-block 'ipfs block rm ' output looks good" ' echo "cannot remove $RANDOMHASH: ipld: could not find $RANDOMHASH" >> expect_mixed_rm && echo "removed $TESTHASH" >> expect_mixed_rm && echo "cannot remove $RANDOMHASH: ipld: could not find $RANDOMHASH" >> expect_mixed_rm && echo "Error: some blocks not removed" >> expect_mixed_rm test_cmp actual_mixed_rm expect_mixed_rm ' test_expect_success "'add some blocks' succeeds" ' echo "Hello Mars!" | ipfs block put && echo "Hello Venus!" | ipfs block put ' test_expect_success "add and pin directory" ' ipfs add -r adir ipfs pin add -r $DIRHASH ' HASH=QmRKqGMAM6EZngbpjSqrvYzq5Qd8b1bSWymjSUY9zQSNDk HASH2=QmdnpnsaEj69isdw5sNzp3h3HkaDz7xKq7BmvFFBzNr5e7 test_expect_success "multi-block 'ipfs block rm' mixed" ' test_must_fail ipfs block rm $FILE1HASH $DIRHASH $HASH $FILE3HASH $RANDOMHASH $HASH2 2> block_rm_err ' test_expect_success "pinned block not removed" ' ipfs block stat $FILE1HASH && ipfs block stat $FILE3HASH ' test_expect_success "non-pinned blocks removed" ' test_must_fail ipfs block stat $HASH && test_must_fail ipfs block stat $HASH2 ' test_expect_success "error reported on removing non-existent block" ' grep -q "cannot remove $RANDOMHASH" block_rm_err ' test_expect_success "'add some blocks' succeeds" ' echo "Hello Mars!" | ipfs block put && echo "Hello Venus!" | ipfs block put ' test_expect_success "multi-block 'ipfs block rm -f' with non existent blocks succeed" ' ipfs block rm -f $HASH $RANDOMHASH $HASH2 ' test_expect_success "existent blocks removed" ' test_must_fail ipfs block stat $HASH && test_must_fail ipfs block stat $HASH2 ' test_expect_success "'add some blocks' succeeds" ' echo "Hello Mars!" | ipfs block put && echo "Hello Venus!" | ipfs block put ' test_expect_success "multi-block 'ipfs block rm -q' produces no output" ' ipfs block rm -q $HASH $HASH2 > block_rm_out && test ! -s block_rm_out ' # --format used 'protobuf' for 'dag-pb' which was invalid, but we keep # for backward-compatibility test_expect_success "can set deprecated --format=protobuf on block put" ' HASH=$(ipfs block put --format=protobuf ../t0050-block-data/testPut.pb) ' test_expect_success "created an object correctly!" ' ipfs dag get $HASH > obj_out && echo -n "{\"Data\":{\"/\":{\"bytes\":\"dGVzdCBqc29uIGZvciBzaGFybmVzcyB0ZXN0\"}},\"Links\":[]}" > obj_exp && test_cmp obj_out obj_exp ' test_expect_success "block get output looks right" ' ipfs block get $HASH > pb_block_out && test_cmp pb_block_out ../t0050-block-data/testPut.pb ' test_expect_success "can set --cid-codec=dag-pb on block put" ' HASH=$(ipfs block put --cid-codec=dag-pb ../t0050-block-data/testPut.pb) ' test_expect_success "created an object correctly!" ' ipfs dag get $HASH > obj_out && echo -n "{\"Data\":{\"/\":{\"bytes\":\"dGVzdCBqc29uIGZvciBzaGFybmVzcyB0ZXN0\"}},\"Links\":[]}" > obj_exp && test_cmp obj_out obj_exp ' test_expect_success "block get output looks right" ' ipfs block get $HASH > pb_block_out && test_cmp pb_block_out ../t0050-block-data/testPut.pb ' test_expect_success "can set multihash type and length on block put with --format=raw (deprecated)" ' HASH=$(echo "foooo" | ipfs block put --format=raw --mhtype=sha3 --mhlen=20) ' test_expect_success "output looks good" ' test "bafkrifctrq4xazzixy2v4ezymjcvzpskqdwlxra" = "$HASH" ' test_expect_success "can't use both legacy format and custom cid-codec at the same time" ' test_expect_code 1 ipfs block put --format=dag-cbor --cid-codec=dag-json < ../t0050-block-data/testPut.pb 2> output && test_should_contain "unable to use \"format\" (deprecated) and a custom \"cid-codec\" at the same time" output ' test_expect_success "can read block with different hash" ' ipfs block get $HASH > blk_get_out && echo "foooo" > blk_get_exp && test_cmp blk_get_exp blk_get_out ' # # Misc tests # test_expect_success "'ipfs block stat' with nothing from stdin doesn't crash" ' test_expect_code 1 ipfs block stat < /dev/null 2> stat_out ' # lol test_expect_success "no panic in output" ' test_expect_code 1 grep "panic" stat_out ' test_expect_success "can set multihash type and length on block put without format or cid-codec" ' HASH=$(echo "foooo" | ipfs block put --mhtype=sha3 --mhlen=20) ' test_expect_success "output looks good" ' test "bafkrifctrq4xazzixy2v4ezymjcvzpskqdwlxra" = "$HASH" ' test_expect_success "can set multihash type and length on block put with cid-codec=dag-pb" ' HASH=$(echo "foooo" | ipfs block put --mhtype=sha3 --mhlen=20 --cid-codec=dag-pb) ' test_expect_success "output looks good" ' test "bafybifctrq4xazzixy2v4ezymjcvzpskqdwlxra" = "$HASH" ' test_expect_success "put with sha3 and cidv0 fails" ' echo "foooo" | test_must_fail ipfs block put --mhtype=sha3 --mhlen=20 --format=v0 ' test_expect_success "'ipfs block put' check block size" ' dd if=/dev/zero bs=2097153 count=1 > over-2MiB-file && test_expect_code 1 ipfs block put over-2MiB-file >block_put_out 2>&1 ' test_expect_success "ipfs block put output has the correct error" ' grep "produced block is over 2MiB" block_put_out ' test_expect_success "ipfs block put --allow-big-block=true works" ' test_expect_code 0 ipfs block put over-2MiB-file --allow-big-block=true && rm over-2MiB-file ' test_done ================================================ FILE: test/sharness/t0051-object.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Henry Bubert # MIT Licensed; see the LICENSE file in this repository. # test_description="Test object command" . lib/test-lib.sh test_init_ipfs test_patch_create_path() { root=$1 name=$2 target=$3 test_expect_success "object patch --create works" ' PCOUT=$(ipfs object patch $root add-link --create $name $target) ' test_expect_success "output looks good" ' ipfs cat "$PCOUT/$name" >tpcp_out && ipfs cat "$target" >tpcp_exp && test_cmp tpcp_exp tpcp_out ' } test_object_cmd() { EMPTY_DIR=$(echo '{"Links":[]}' | ipfs dag put --store-codec dag-pb) EMPTY_UNIXFS_DIR=$(echo '{"Data":{"/":{"bytes":"CAE"}},"Links":[]}' | ipfs dag put --store-codec dag-pb) test_expect_success "'ipfs object patch' should work (no unixfs-dir)" ' OUTPUT=$(ipfs object patch $EMPTY_DIR add-link foo $EMPTY_DIR) && ipfs dag stat $OUTPUT ' test_expect_success "'ipfs object patch' should work" ' OUTPUT=$(ipfs object patch $EMPTY_UNIXFS_DIR add-link foo $EMPTY_UNIXFS_DIR) && ipfs dag stat $OUTPUT ' test_expect_success "'ipfs object patch' check output block size" ' DIR=$EMPTY_UNIXFS_DIR for i in {1..14} do DIR=$(ipfs object patch "$DIR" add-link "$DIR.jpg" "$DIR") done # Fail when new block goes over the BS limit of 2MiB, but allow manual override test_expect_code 1 ipfs object patch "$DIR" add-link "$DIR.jpg" "$DIR" >patch_out 2>&1 ' test_expect_success "ipfs object patch add-link output has the correct error" ' grep "produced block is over 2MiB" patch_out ' test_expect_success "ipfs object patch --allow-big-block=true add-link works" ' test_expect_code 0 ipfs object patch --allow-big-block=true "$DIR" add-link "$DIR.jpg" "$DIR" ' test_expect_success "'ipfs object patch add-link' should work with paths" ' N1=$(ipfs object patch $EMPTY_UNIXFS_DIR add-link baz $EMPTY_UNIXFS_DIR) && N2=$(ipfs object patch $EMPTY_UNIXFS_DIR add-link bar $N1) && N3=$(ipfs object patch $EMPTY_UNIXFS_DIR add-link foo /ipfs/$N2/bar) && ipfs dag stat /ipfs/$N3 > /dev/null && ipfs dag stat $N3/foo > /dev/null && ipfs dag stat /ipfs/$N3/foo/baz > /dev/null ' test_expect_success "'ipfs object patch add-link' allow linking IPLD objects" ' OBJ=$(echo "123" | ipfs dag put) && N1=$(ipfs object patch $EMPTY_UNIXFS_DIR add-link foo $OBJ) && ipfs dag stat /ipfs/$N1 > /dev/null && ipfs resolve /ipfs/$N1/foo > actual && echo /ipfs/$OBJ > expected && test_cmp expected actual ' test_expect_success "object patch creation looks right" ' echo "bafybeiakusqwohnt7bs75kx6jhmt4oi47l634bmudxfv4qxhpco6xuvgna" > hash_exp && echo $N3 > hash_actual && test_cmp hash_exp hash_actual ' test_expect_success "multilayer ipfs patch works" ' echo "hello world" > hwfile && FILE=$(ipfs add -q hwfile) && EMPTY=$EMPTY_UNIXFS_DIR && ONE=$(ipfs object patch $EMPTY add-link b $EMPTY) && TWO=$(ipfs object patch $EMPTY add-link a $ONE) && ipfs object patch $TWO add-link a/b/c $FILE > multi_patch ' test_expect_success "output looks good" ' ipfs cat $(cat multi_patch)/a/b/c > hwfile_out && test_cmp hwfile hwfile_out ' test_expect_success "can remove the directory" ' ipfs object patch $OUTPUT rm-link foo > rmlink_output ' test_expect_success "output should be empty" ' echo bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354 > rmlink_exp && test_cmp rmlink_exp rmlink_output ' test_expect_success "multilayer rm-link should work" ' ipfs object patch $(cat multi_patch) rm-link a/b/c > multi_link_rm_out ' test_expect_success "output looks good" ' echo "bafybeicourxysmtbe5hacxqico4d5hyvh7gqkrwlmqa4ew7zufn3pj3juu" > multi_link_rm_exp && test_cmp multi_link_rm_exp multi_link_rm_out ' test_patch_create_path $EMPTY a/b/c $FILE test_patch_create_path $EMPTY a $FILE test_patch_create_path $EMPTY a/b/b/b/b $FILE test_expect_success "can create blank object" ' BLANK=$EMPTY_DIR ' test_patch_create_path $BLANK a $FILE test_expect_success "create bad path fails" ' test_must_fail ipfs object patch $EMPTY add-link --create / $FILE ' } # should work offline test_object_cmd # should work online test_launch_ipfs_daemon test_object_cmd test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0052-object-diff.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2016 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test object diff command" . lib/test-lib.sh test_init_ipfs test_expect_success "create some objects for testing diffs" ' mkdir foo && echo "stuff" > foo/bar && mkdir foo/baz && A=$(ipfs add -r -Q foo) && AR=$(ipfs add --raw-leaves -r -Q foo) && echo "more things" > foo/cat && B=$(ipfs add -r -Q foo) && BR=$(ipfs add --raw-leaves -r -Q foo) && echo "nested" > foo/baz/dog && C=$(ipfs add -r -Q foo) CR=$(ipfs add --raw-leaves -r -Q foo) echo "changed" > foo/bar && D=$(ipfs add -r -Q foo) && DR=$(ipfs add --raw-leaves -r -Q foo) && echo "" > single_file && SINGLE_FILE=$(ipfs add -r -Q single_file) && SINGLE_FILE_RAW=$(ipfs add --raw-leaves -r -Q single_file) && mkdir empty_dir EMPTY_DIR=$(ipfs add -r -Q empty_dir) EMPTY_DIR_RAW=$(ipfs add --raw-leaves -r -Q empty_dir) ' test_expect_success "diff against self is empty" ' ipfs object diff $A $A > diff_out ' test_expect_success "identity diff output looks good" ' printf "" > diff_exp && test_cmp diff_exp diff_out ' test_expect_success "diff (raw-leaves) against self is empty" ' ipfs object diff $AR $AR > diff_raw_out ' test_expect_success "identity diff (raw-leaves) output looks good" ' printf "" > diff_raw_exp && test_cmp diff_raw_exp diff_raw_out ' test_expect_success "diff against self (single file) is empty" ' ipfs object diff $SINGLE_FILE $SINGLE_FILE > diff_out && printf "" > diff_exp && test_cmp diff_exp diff_out ' test_expect_success "diff (raw-leaves) against self (single file) is empty" ' ipfs object diff $SINGLE_FILE_RAW $SINGLE_FILE_RAW > diff_raw_out && printf "" > diff_raw_exp && test_cmp diff_raw_exp diff_raw_out ' test_expect_success "diff against self (empty dir) is empty" ' ipfs object diff $EMPTY_DIR $EMPTY_DIR > diff_out && printf "" > diff_exp && test_cmp diff_exp diff_out ' test_expect_success "diff (raw-leaves) against self (empty dir) is empty" ' ipfs object diff $EMPTY_DIR_RAW $EMPTY_DIR_RAW > diff_raw_out && printf "" > diff_raw_exp && test_cmp diff_raw_exp diff_raw_out ' test_expect_success "diff added link works" ' ipfs object diff $A $B > diff_out ' test_expect_success "diff added link looks right" ' echo + QmUSvcqzhdfYM1KLDbM76eLPdS9ANFtkJvFuPYeZt73d7A \"cat\" > diff_exp && test_cmp diff_exp diff_out ' test_expect_success "diff (raw-leaves) added link works" ' ipfs object diff $AR $BR > diff_raw_out ' test_expect_success "diff (raw-leaves) added link looks right" ' echo + bafkreig43bpnc6sjo6izaiqzzq5esapazosa3f3wt6jsflwiu3x7ydhq2u \"cat\" > diff_raw_exp && test_cmp diff_raw_exp diff_raw_out ' test_expect_success "verbose diff added link works" ' ipfs object diff -v $A $B > diff_out ' test_expect_success "verbose diff added link looks right" ' echo Added new link \"cat\" pointing to QmUSvcqzhdfYM1KLDbM76eLPdS9ANFtkJvFuPYeZt73d7A. > diff_exp && test_cmp diff_exp diff_out ' test_expect_success "verbose diff (raw-leaves) added link works" ' ipfs object diff -v $AR $BR > diff_raw_out ' test_expect_success "verbose diff (raw-leaves) added link looks right" ' echo Added new link \"cat\" pointing to bafkreig43bpnc6sjo6izaiqzzq5esapazosa3f3wt6jsflwiu3x7ydhq2u. > diff_raw_exp && test_cmp diff_raw_exp diff_raw_out ' test_expect_success "diff removed link works" ' ipfs object diff -v $B $A > diff_out ' test_expect_success "diff removed link looks right" ' echo Removed link \"cat\" \(was QmUSvcqzhdfYM1KLDbM76eLPdS9ANFtkJvFuPYeZt73d7A\). > diff_exp && test_cmp diff_exp diff_out ' test_expect_success "diff (raw-leaves) removed link works" ' ipfs object diff -v $BR $AR > diff_raw_out ' test_expect_success "diff (raw-leaves) removed link looks right" ' echo Removed link \"cat\" \(was bafkreig43bpnc6sjo6izaiqzzq5esapazosa3f3wt6jsflwiu3x7ydhq2u\). > diff_raw_exp && test_cmp diff_raw_exp diff_raw_out ' test_expect_success "diff nested add works" ' ipfs object diff -v $B $C > diff_out ' test_expect_success "diff looks right" ' echo Added new link \"baz/dog\" pointing to QmdNJQUTZuDpsUcec7YDuCfRfvw1w4J13DCm7YcU4VMZdS. > diff_exp && test_cmp diff_exp diff_out ' test_expect_success "diff (raw-leaves) nested add works" ' ipfs object diff -v $BR $CR > diff_raw_out ' test_expect_success "diff (raw-leaves) looks right" ' echo Added new link \"baz/dog\" pointing to bafkreibxbkgajofglo2esqtv53bcp4nwstnqjr3nu2ylrlui5unldf4qum. > diff_raw_exp && test_cmp diff_raw_exp diff_raw_out ' test_expect_success "diff changed link works" ' ipfs object diff -v $C $D > diff_out ' test_expect_success "diff looks right" ' echo Changed \"bar\" from QmNgd5cz2jNftnAHBhcRUGdtiaMzb5Rhjqd4etondHHST8 to QmRfFVsjSXkhFxrfWnLpMae2M4GBVsry6VAuYYcji5MiZb. > diff_exp && test_cmp diff_exp diff_out ' test_expect_success "diff (raw-leaves) changed link works" ' ipfs object diff -v $CR $DR > diff_raw_out ' test_expect_success "diff（raw-leaves) looks right" ' echo Changed \"bar\" from bafkreidfn2oemjv5ns2fnc4ukgbjwt6bq5gdd4ciz4mpnehqi2dvwxfbde to bafkreid7rmo7yrtlmje7a3f6kxerotpsk6hhovg2pe755use55olukry6e. > diff_raw_exp && test_cmp diff_raw_exp diff_raw_out ' test_done ================================================ FILE: test/sharness/t0053-dag.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2016 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test dag command" . lib/test-lib.sh test_init_ipfs test_expect_success "make a few test files" ' echo "foo" > file1 && echo "bar" > file2 && echo "baz" > file3 && echo "qux" > file4 && HASH1=$(ipfs add --pin=false -q file1) && HASH2=$(ipfs add --pin=false -q file2) && HASH3=$(ipfs add --pin=false -q file3) && HASH4=$(ipfs add --pin=false -q file4) ' test_expect_success "make an ipld object in dag-json" ' printf "{\"hello\":\"world\",\"cats\":[{\"/\":\"%s\"},{\"water\":{\"/\":\"%s\"}}],\"magic\":{\"/\":\"%s\"},\"sub\":{\"dict\":\"ionary\",\"beep\":[0,\"bop\"]}}" $HASH1 $HASH2 $HASH3 > ipld_object ' # This data is in https://github.com/ipld/codec-fixtures/tree/master/fixtures/dagpb_Data_some test_expect_success "make the same ipld object in dag-cbor, dag-json and dag-pb" ' echo "omREYXRhRQABAgMEZUxpbmtzgA==" | base64 -d > ipld_object_dagcbor echo "CgUAAQIDBA==" | base64 -d > ipld_object_dagpb echo "{\"Data\":{\"/\":{\"bytes\":\"AAECAwQ\"}},\"Links\":[]}" > ipld_object_dagjson ' test_dag_cmd() { # Working with a plain IPLD hello-world object that's dag-json and dag-cbor compatible test_expect_success "can add an ipld object using defaults (dag-json to dag-cbor)" ' IPLDHASH=$(cat ipld_object | ipfs dag put) ' test_expect_success "CID looks correct" ' EXPHASH="bafyreiblwimnjbqcdoeafiobk6q27jcw64ew7n2fmmhdpldd63edmjecde" test $EXPHASH = $IPLDHASH ' test_expect_success "'ipfs dag put' check block size" ' dd if=/dev/zero bs=2097153 count=1 > over-2MiB-file && test_expect_code 1 ipfs dag put --input-codec=raw --store-codec=raw over-2MiB-file >dag_put_out 2>&1 ' test_expect_success "ipfs dag put output has the correct error" ' grep "produced block is over 2MiB" dag_put_out ' test_expect_success "ipfs dag put --allow-big-block=true works" ' test_expect_code 0 ipfs dag put --input-codec=raw --store-codec=raw over-2MiB-file --allow-big-block=true && rm over-2MiB-file ' test_expect_success "can add an ipld object using dag-json to dag-json" ' IPLDHASH=$(cat ipld_object | ipfs dag put --input-codec dag-json --store-codec dag-json) ' test_expect_success "CID looks correct" ' EXPHASH="baguqeera6gviseelmbzn2ugoddo5vulxlshqs3kw5ymgsb6w4cabnoh4ldpa" test $EXPHASH = $IPLDHASH ' test_expect_success "can add an ipld object using dag-json to dag-cbor" ' IPLDHASH=$(cat ipld_object | ipfs dag put --input-codec dag-json --store-codec dag-cbor) ' test_expect_success "CID looks correct" ' EXPHASH="bafyreiblwimnjbqcdoeafiobk6q27jcw64ew7n2fmmhdpldd63edmjecde" test $EXPHASH = $IPLDHASH ' test_expect_success "can add an ipld object using cid-base=base58btc" ' IPLDb58HASH=$(cat ipld_object | ipfs dag put -cid-base=base58btc) ' test_expect_success "CID looks correct" ' EXPHASH="zdpuAoN1XJ3GsrxEzMuCbRKZzRUVJekJUCbPVgCgE4D9yYqVi" test $EXPHASH = $IPLDb58HASH ' # Same object, different forms # (1) dag-cbor input test_expect_success "can add a dag-cbor input block stored as dag-cbor" ' IPLDCBORHASH=$(cat ipld_object_dagcbor | ipfs dag put --input-codec dag-cbor --store-codec dag-cbor) ' test_expect_success "dag-cbor CID looks correct" ' EXPHASH="bafyreieculsmrexh3ty5jentbvuku452o27mst4h2tq2rb2zntqhgcstji" test $EXPHASH = $IPLDCBORHASH ' test_expect_success "can add a dag-cbor input block stored as dag-pb" ' IPLDPBHASH=$(cat ipld_object_dagcbor | ipfs dag put --input-codec dag-cbor --store-codec dag-pb) ' test_expect_success "dag-pb CID looks correct" ' EXPHASH="bafybeibazl2z4vqp2tmwcfag6wirmtpnomxknqcgrauj7m2yisrz3qjbom" test $EXPHASH = $IPLDPBHASH ' test_expect_success "can add a dag-cbor input block stored as dag-json" ' IPLDJSONHASH=$(cat ipld_object_dagcbor | ipfs dag put --input-codec dag-cbor --store-codec dag-json) ' test_expect_success "dag-json CID looks correct" ' EXPHASH="baguqeerajwksxu3lxpomdwxvosl542zl3xknhjgxtq3277gafrhl6vdw5tcq" test $EXPHASH = $IPLDJSONHASH ' # (2) dag-json input test_expect_success "can add a dag-json input block stored as dag-cbor" ' IPLDCBORHASH=$(cat ipld_object_dagjson | ipfs dag put --input-codec dag-json --store-codec dag-cbor) ' test_expect_success "dag-cbor CID looks correct" ' EXPHASH="bafyreieculsmrexh3ty5jentbvuku452o27mst4h2tq2rb2zntqhgcstji" test $EXPHASH = $IPLDCBORHASH ' test_expect_success "can add a dag-json input block stored as dag-pb" ' IPLDPBHASH=$(cat ipld_object_dagjson | ipfs dag put --input-codec dag-json --store-codec dag-pb) ' test_expect_success "dag-pb CID looks correct" ' EXPHASH="bafybeibazl2z4vqp2tmwcfag6wirmtpnomxknqcgrauj7m2yisrz3qjbom" test $EXPHASH = $IPLDPBHASH ' test_expect_success "can add a dag-json input block stored as dag-json" ' IPLDJSONHASH=$(cat ipld_object_dagjson | ipfs dag put --input-codec dag-json --store-codec dag-json) ' test_expect_success "dag-json CID looks correct" ' EXPHASH="baguqeerajwksxu3lxpomdwxvosl542zl3xknhjgxtq3277gafrhl6vdw5tcq" test $EXPHASH = $IPLDJSONHASH ' # (3) dag-pb input test_expect_success "can add a dag-pb input block stored as dag-cbor" ' IPLDCBORHASH=$(cat ipld_object_dagpb | ipfs dag put --input-codec dag-pb --store-codec dag-cbor) ' test_expect_success "dag-cbor CID looks correct" ' EXPHASH="bafyreieculsmrexh3ty5jentbvuku452o27mst4h2tq2rb2zntqhgcstji" test $EXPHASH = $IPLDCBORHASH ' test_expect_success "can add a dag-pb input block stored as dag-pb" ' IPLDPBHASH=$(cat ipld_object_dagpb | ipfs dag put --input-codec dag-pb --store-codec dag-pb) ' test_expect_success "dag-pb CID looks correct" ' EXPHASH="bafybeibazl2z4vqp2tmwcfag6wirmtpnomxknqcgrauj7m2yisrz3qjbom" test $EXPHASH = $IPLDPBHASH ' test_expect_success "can add a dag-pb input block stored as dag-json" ' IPLDJSONHASH=$(cat ipld_object_dagpb | ipfs dag put --input-codec dag-pb --store-codec dag-json) ' test_expect_success "dag-json CID looks correct" ' EXPHASH="baguqeerajwksxu3lxpomdwxvosl542zl3xknhjgxtq3277gafrhl6vdw5tcq" test $EXPHASH = $IPLDJSONHASH ' test_expect_success "can get dag-cbor, dag-json, dag-pb blocks as dag-json" ' ipfs dag get $IPLDCBORHASH >& dag-get-cbor && ipfs dag get $IPLDJSONHASH >& dag-get-json && ipfs dag get $IPLDPBHASH >& dag-get-pb ' test_expect_success "can get dag-pb block transcoded as dag-cbor" ' ipfs dag get --output-codec=dag-cbor $IPLDPBHASH >& dag-get-dagpb-transcoded-to-dagcbor && echo "122082a2e4c892e7dcf1d491b30d68aa73ba76bec94f87d4e1a887596ce0730a534a" >sha2_dagpb_to_dagcbor_expected && multihash -a=sha2-256 -e=hex dag-get-dagpb-transcoded-to-dagcbor >sha2_dagpb_to_dagcbor_actual && test_cmp sha2_dagpb_to_dagcbor_expected sha2_dagpb_to_dagcbor_actual ' test_expect_success "dag put and dag get transcodings match" ' ROUNDTRIPDAGCBOR=$(ipfs dag put --input-codec=dag-cbor --store-codec=dag-cbor dag-get-dagpb-transcoded-to-dagcbor) && test $ROUNDTRIPDAGCBOR = $IPLDCBORHASH ' # this doesn't tell us if they are correct, we test that better below test_expect_success "outputs are the same" ' test_cmp dag-get-cbor dag-get-json && test_cmp dag-get-cbor dag-get-pb ' # Traversals using the original hello-world object test_expect_success "various path traversals work" ' ipfs cat $IPLDHASH/cats/0 > out1 && ipfs cat $IPLDHASH/cats/1/water > out2 && ipfs cat $IPLDHASH/magic > out3 ' test_expect_success "outputs look correct" ' test_cmp file1 out1 && test_cmp file2 out2 && test_cmp file3 out3 ' test_expect_success "resolving sub-objects works" ' ipfs dag get $IPLDHASH/hello > sub1 && ipfs dag get $IPLDHASH/sub > sub2 && ipfs dag get $IPLDHASH/sub/beep > sub3 && ipfs dag get $IPLDHASH/sub/beep/0 > sub4 && ipfs dag get $IPLDHASH/sub/beep/1 > sub5 ' test_expect_success "sub-objects look right" ' echo -n "\"world\"" > sub1_exp && test_cmp sub1_exp sub1 && echo -n "{\"beep\":[0,\"bop\"],\"dict\":\"ionary\"}" > sub2_exp && test_cmp sub2_exp sub2 && echo -n "[0,\"bop\"]" > sub3_exp && test_cmp sub3_exp sub3 && echo -n "0" > sub4_exp && test_cmp sub4_exp sub4 && echo -n "\"bop\"" > sub5_exp && test_cmp sub5_exp sub5 ' test_expect_success "traversals using /ipld/ work" ' ipfs dag get /ipld/$IPLDPBHASH/Data > ipld_path_Data_actual ' test_expect_success "retrieved node looks right" ' echo -n "{\"/\":{\"bytes\":\"AAECAwQ\"}}" > ipld_path_Data_expected test_cmp ipld_path_Data_actual ipld_path_Data_expected ' test_expect_success "can pin ipld object" ' ipfs pin add $IPLDHASH ' test_expect_success "can pin dag-pb object" ' ipfs pin add $IPLDPBHASH ' test_expect_success "can pin dag-cbor object" ' ipfs pin add $IPLDCBORHASH ' test_expect_success "can pin dag-json object" ' ipfs pin add $IPLDJSONHASH ' test_expect_success "after gc, objects still accessible" ' ipfs repo gc > /dev/null && ipfs refs -r --timeout=2s $IPLDJSONHASH > /dev/null ' test_expect_success "can get object" ' ipfs dag get $IPLDHASH > ipld_obj_out ' test_expect_success "object links look right" ' grep "{\"/\":\"" ipld_obj_out > /dev/null ' test_expect_success "retrieved object hashes back correctly" ' IPLDHASH2=$(cat ipld_obj_out | ipfs dag put --input-codec dag-json --store-codec dag-cbor) && test "$IPLDHASH" = "$IPLDHASH2" ' test_expect_success "add a normal file" ' HASH=$(echo "foobar" | ipfs add -q) ' test_expect_success "can view protobuf object with dag get" ' ipfs dag get $HASH > dag_get_pb_out ' test_expect_success "output looks correct" ' echo -n "{\"Data\":{\"/\":{\"bytes\":\"CAISB2Zvb2JhcgoYBw\"}},\"Links\":[]}" > dag_get_pb_exp && test_cmp dag_get_pb_exp dag_get_pb_out ' test_expect_success "can call dag get with a path" ' ipfs dag get $IPLDHASH/cats/0 > cat_out ' test_expect_success "output looks correct" ' echo -n "{\"Data\":{\"/\":{\"bytes\":\"CAISBGZvbwoYBA\"}},\"Links\":[]}" > cat_exp && test_cmp cat_exp cat_out ' test_expect_success "non-canonical dag-cbor input is normalized" ' HASH=$(cat ../t0053-dag-data/non-canon.cbor | ipfs dag put --store-codec dag-cbor --input-codec dag-cbor) && test $HASH = "bafyreiawx7ona7oa2ptcoh6vwq4q6bmd7x2ibtkykld327bgb7t73ayrqm" || test_fsh echo $HASH ' test_expect_success "cbor input can be fetched" ' EXPARR=$(ipfs dag get $HASH/arr) test $EXPARR = "[]" ' test_expect_success "add an ipld with pin" ' PINHASH=$(printf {\"foo\":\"bar\"} | ipfs dag put --input-codec dag-json --pin=true) ' test_expect_success "after gc, objects still accessible" ' ipfs repo gc > /dev/null && ipfs refs -r --timeout=2s $PINHASH > /dev/null ' test_expect_success "can add an ipld object with sha3-512 hash" ' IPLDHASH=$(cat ipld_object | ipfs dag put --hash sha3-512) ' test_expect_success "output looks correct" ' EXPHASH="bafyriqforjma7y7akqz7nhuu73r6liggj5zhkbfiqgicywe3fgkna2ijlhod2af3ue7doj56tlzt5hh6iu5esafc4msr3sd53jol5m2o25ucy" test $EXPHASH = $IPLDHASH ' test_expect_success "prepare dag-pb object" ' echo foo > test_file && HASH=$(ipfs add -wQ test_file | ipfs cid base32) ' test_expect_success "dag put with json dag-pb works" ' ipfs dag get $HASH > pbjson && cat pbjson | ipfs dag put --store-codec=dag-pb --input-codec=dag-json > dag_put_out ' test_expect_success "dag put with dag-pb works output looks good" ' echo $HASH > dag_put_exp && test_cmp dag_put_exp dag_put_out ' test_expect_success "dag put with raw dag-pb works" ' ipfs block get $HASH > pbraw && cat pbraw | ipfs dag put --store-codec=dag-pb --input-codec=dag-pb > dag_put_out ' test_expect_success "dag put with dag-pb works output looks good" ' echo $HASH > dag_put_exp && test_cmp dag_put_exp dag_put_out ' test_expect_success "dag put with raw node works" ' echo "foo bar" > raw_node_in && HASH=$(ipfs dag put --store-codec=raw --input-codec=raw -- raw_node_in) && ipfs block get "$HASH" > raw_node_out && test_cmp raw_node_in raw_node_out' test_expect_success "dag put multiple files" ' printf {\"foo\":\"bar\"} > a.json && printf {\"foo\":\"baz\"} > b.json && ipfs dag put a.json b.json > dag_put_out ' test_expect_success "dag put multiple files output looks good" ' echo bafyreiblaotetvwobe7cu2uqvnddr6ew2q3cu75qsoweulzku2egca4dxq > dag_put_exp && echo bafyreibqp7zvp6dvrqhtkbwuzzk7jhtmfmngtiqjajqpm6gtw55o7kqzfi >> dag_put_exp && test_cmp dag_put_exp dag_put_out ' test_expect_success "prepare data for dag resolve" ' NESTED_HASH=$(echo "{\"data\":123}" | ipfs dag put) && HASH=$(echo "{\"obj\":{\"/\":\"${NESTED_HASH}\"}}" | ipfs dag put) ' test_expect_success "dag resolve some things" ' ipfs dag resolve $HASH > resolve_hash && ipfs dag resolve ${HASH}/obj > resolve_obj && ipfs dag resolve ${HASH}/obj/data > resolve_data ' test_expect_success "dag resolve output looks good" ' printf $HASH > resolve_hash_exp && printf $NESTED_HASH > resolve_obj_exp && printf $NESTED_HASH/data > resolve_data_exp && test_cmp resolve_hash_exp resolve_hash && test_cmp resolve_obj_exp resolve_obj && test_cmp resolve_data_exp resolve_data ' test_expect_success "get base32 version of hashes for testing" ' HASHb32=$(ipfs cid base32 $HASH) && NESTED_HASHb32=$(ipfs cid base32 $NESTED_HASH) ' test_expect_success "dag resolve some things with --cid-base=base32" ' ipfs dag resolve $HASH --cid-base=base32 > resolve_hash && ipfs dag resolve ${HASH}/obj --cid-base=base32 > resolve_obj && ipfs dag resolve ${HASH}/obj/data --cid-base=base32 > resolve_data ' test_expect_success "dag resolve output looks good with --cid-base=base32" ' printf $HASHb32 > resolve_hash_exp && printf $NESTED_HASHb32 > resolve_obj_exp && printf $NESTED_HASHb32/data > resolve_data_exp && test_cmp resolve_hash_exp resolve_hash && test_cmp resolve_obj_exp resolve_obj && test_cmp resolve_data_exp resolve_data ' test_expect_success "dag resolve some things with base32 hash" ' ipfs dag resolve $HASHb32 > resolve_hash && ipfs dag resolve ${HASHb32}/obj > resolve_obj && ipfs dag resolve ${HASHb32}/obj/data > resolve_data ' test_expect_success "dag resolve output looks good with base32 hash" ' printf $HASHb32 > resolve_hash_exp && printf $NESTED_HASHb32 > resolve_obj_exp && printf $NESTED_HASHb32/data > resolve_data_exp && test_cmp resolve_hash_exp resolve_hash && test_cmp resolve_obj_exp resolve_obj && test_cmp resolve_data_exp resolve_data ' } # should work offline test_dag_cmd # should work online test_launch_ipfs_daemon test_dag_cmd test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0054-dag-car-import-export-data/README.md ================================================ # Dataset description/sources - lotus_testnet_export_256_multiroot.car - Export of the first 256 block of the testnet chain, with 3 tipset roots. Exported from Lotus by @traviperson on 2019-03-18 - lotus_devnet_genesis.car - Source: https://github.com/filecoin-project/lotus/blob/v0.2.10/build/genesis/devnet.car - lotus_testnet_export_128.car - Export of the first 128 block of the testnet chain, exported from Lotus by @traviperson on 2019-03-24 - lotus_devnet_genesis_shuffled_noroots.car - lotus_testnet_export_128_shuffled_noroots.car - versions of the above with an **empty** root array, and having all blocks shuffled - lotus_devnet_genesis_shuffled_nulroot.car - lotus_testnet_export_128_shuffled_nulroot.car - versions identical to the above, but with a single "empty-block" root each ( in order to work around go-car not following the current "roots can be empty" spec ) - combined_naked_roots_genesis_and_128.car - only the roots of `lotus_devnet_genesis.car` and `lotus_testnet_export_128.car`,to be used in combination with the root-less parts to validate "transactional" pinning - lotus_testnet_export_128_v2.car - lotus_devnet_genesis_v2.car - generated with `car index lotus_testnet_export_128.car > lotus_testnet_export_128_v2.car` - install `go-car` CLI from https://github.com/ipld/go-car - partial-dag-scope-entity.car - unixfs directory entity exported from gateway via `?format=car&dag-scope=entity` ([IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/)) - CAR roots includes directory CID, but only the root block is included in the CAR, making the DAG incomplete ================================================ FILE: test/sharness/t0054-dag-car-import-export.sh ================================================ #!/usr/bin/env bash # test_description="Test car file import/export functionality" . lib/test-lib.sh export -f ipfsi set -o pipefail tar -C ../t0054-dag-car-import-export-data/ --strip-components=1 -Jxf ../t0054-dag-car-import-export-data/test_dataset_car.tar.xz tab=$'\t' test_cmp_sorted() { # use test_cmp to dump out the unsorted file contents as a diff [[ "$( sort "$1" | sha256sum )" == "$( sort "$2" | sha256sum )" ]] \ || test_cmp "$1" "$2" } export -f test_cmp_sorted reset_blockstore() { node=$1 ipfsi "$node" pin ls --quiet --type=recursive | ipfsi "$node" pin rm &>/dev/null ipfsi "$node" repo gc &>/dev/null test_expect_success "pinlist empty" ' [[ -z "$( ipfsi $node pin ls )" ]] ' test_expect_success "nothing left to gc" ' [[ -z "$( ipfsi $node repo gc )" ]] ' } # hammer with concurrent gc to ensure nothing clashes do_import() { node="$1"; shift ( touch spin.gc while [[ -e spin.gc ]]; do ipfsi "$node" repo gc &>/dev/null; done & while [[ -e spin.gc ]]; do ipfsi "$node" repo gc &>/dev/null; done & ipfsi "$node" dag import "$@" 2>&1 && ipfsi "$node" repo verify &>/dev/null result=$? rm -f spin.gc &>/dev/null wait || true # work around possible trigger of a bash bug on overloaded circleci exit $result ) } run_online_imp_exp_tests() { reset_blockstore 0 reset_blockstore 1 cat > basic_import_stats_expected < basic_import_expected # Explainer: # naked_root_import_json_expected output is produced by dag import of combined_naked_roots_genesis_and_128.car # executed when roots are already present in the repo - thus the BlockCount=0 # (if blocks were not present in the repo, ipld: could not find would be returned) cat >naked_root_import_json_expected < basic_import_actual ' test_expect_success "basic import output as expected" ' test_cmp_sorted basic_import_expected basic_import_actual ' test_expect_success "basic import with --stats" ' do_import 0 --stats \ ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car \ ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car \ > basic_import_actual ' test_expect_success "basic import output with --stats as expected" ' test_cmp_sorted basic_import_stats_expected basic_import_actual ' test_expect_success "basic fetch+export 1" ' ipfsi 1 dag export bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy > reexported_testnet_128.car ' test_expect_success "export of shuffled testnet export identical to canonical original" ' test_cmp reexported_testnet_128.car ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car ' test_expect_success "basic fetch+export 2" ' ipfsi 1 dag export bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u > reexported_devnet_genesis.car ' test_expect_success "export of shuffled devnet export identical to canonical original" ' test_cmp reexported_devnet_genesis.car ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car ' test_expect_success "pinlist on node1 still empty" ' [[ -z "$( ipfsi 1 pin ls )" ]] ' test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" ' ipfsi 1 dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ > naked_import_result_json_actual ' test_expect_success "naked import output as expected" ' test_cmp_sorted naked_root_import_json_expected naked_import_result_json_actual ' reset_blockstore 0 reset_blockstore 1 mkfifo pipe_testnet mkfifo pipe_devnet test_expect_success "fifo import" ' ( cat ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car > pipe_testnet & cat ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car > pipe_devnet & do_import 0 --stats \ pipe_testnet \ pipe_devnet \ ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ > basic_fifo_import_actual result=$? wait || true # work around possible trigger of a bash bug on overloaded circleci exit "$result" ) ' test_expect_success "remove fifos" ' rm pipe_testnet pipe_devnet ' test_expect_success "fifo-import output as expected" ' test_cmp_sorted basic_import_stats_expected basic_fifo_import_actual ' } test_expect_success "set up testbed" ' iptb testbed create -type localipfs -count 2 -force -init ' startup_cluster 2 run_online_imp_exp_tests test_expect_success "shut down nodes" ' iptb stop && iptb_wait_stop ' # We want to just init the repo, without using a daemon for stuff below test_init_ipfs --empty-repo=false test_expect_success "basic offline export of 'getting started' dag works" ' ipfs dag export "$HASH_WELCOME_DOCS" >/dev/null ' test_expect_success "basic offline export of nonexistent cid" ' ! ipfs dag export QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 2> offline_fetch_error_actual >/dev/null ' test_expect_success "correct error" ' test_should_contain "Error: block was not found locally (offline): ipld: could not find QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" offline_fetch_error_actual ' cat >multiroot_import_json_stats_expected < multiroot_import_json_expected test_expect_success "multiroot import works (--enc=json)" ' ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual ' test_expect_success "multiroot import expected output" ' test_cmp_sorted multiroot_import_json_expected multiroot_import_json_actual ' test_expect_success "multiroot import works with --stats" ' ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual ' test_expect_success "multiroot import expected output" ' test_cmp_sorted multiroot_import_json_stats_expected multiroot_import_json_actual ' cat >pin_import_expected << EOE {"Stats":{"BlockCount":1198,"BlockBytesCount":468513}} EOE test_expect_success "pin-less import works" ' ipfs dag import --stats --enc=json --pin-roots=false \ ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \ ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \ > no-pin_import_actual ' test_expect_success "expected no pins on --pin-roots=false" ' test_cmp pin_import_expected no-pin_import_actual ' test_expect_success "naked root import works" ' ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ > naked_root_import_json_actual ' test_expect_success "naked root import expected output" ' test_cmp_sorted naked_root_import_json_expected naked_root_import_json_actual ' test_expect_success "'ipfs dag import' check block size" ' BIG_CID=$(dd if=/dev/zero bs=2097153 count=1 | ipfs dag put --input-codec=raw --store-codec=raw --allow-big-block) && ipfs dag export $BIG_CID > over-2MiB-block.car && test_expect_code 1 ipfs dag import over-2MiB-block.car >dag_import_out 2>&1 ' test_expect_success "ipfs dag import output has the correct error" ' grep "block is over 2MiB" dag_import_out ' test_expect_success "ipfs dag import --allow-big-block works" ' test_expect_code 0 ipfs dag import --allow-big-block over-2MiB-block.car ' cat > version_2_import_expected << EOE {"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PinErrorMsg":""}} {"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PinErrorMsg":""}} {"Stats":{"BlockCount":1198,"BlockBytesCount":468513}} EOE test_expect_success "version 2 import" ' ipfs dag import --stats --enc=json \ ../t0054-dag-car-import-export-data/lotus_testnet_export_128_v2.car \ ../t0054-dag-car-import-export-data/lotus_devnet_genesis_v2.car \ > version_2_import_actual ' test_expect_success "version 2 import output as expected" ' test_cmp_sorted version_2_import_expected version_2_import_actual ' test_expect_success "'ipfs dag import' decode IPLD 'dag-json' codec works" ' NEW_HASH=$(echo "{ \"test\": \"dag-json\" }" | ipfs dag put --store-codec dag-json) && ipfs dag export $NEW_HASH > dag-json.car && ipfs dag import dag-json.car && rm dag-json.car ' test_expect_success "'ipfs dag import' decode IPLD 'dag-cbor' codec works" ' NEW_HASH=$(echo "{ \"test\": \"dag-cbor\" }" | ipfs dag put --store-codec dag-cbor) && ipfs dag export $NEW_HASH > dag-cbor.car && ipfs dag import dag-cbor.car && rm dag-cbor.car ' test_expect_success "'ipfs dag import' decode IPLD 'json' codec works" ' NEW_HASH=$(echo "{ \"test\": \"json\" }" | ipfs dag put --store-codec json) && ipfs dag export $NEW_HASH > json.car && ipfs dag import json.car && rm json.car ' test_expect_success "'ipfs dag import' decode IPLD 'cbor' codec works" ' NEW_HASH=$(echo "{ \"test\": \"cbor\" }" | ipfs dag put --store-codec cbor) && ipfs dag export $NEW_HASH > cbor.car && ipfs dag import cbor.car && rm cbor.car ' # IPIP-402 cat > partial_nopin_import_expected << EOE {"Stats":{"BlockCount":1,"BlockBytesCount":1618}} EOE test_expect_success "'ipfs dag import' without pinning works fine with incomplete DAG (unixfs dir exported as dag-scope=entity from IPIP-402)" ' ipfs dag import --stats --enc=json --pin-roots=false ../t0054-dag-car-import-export-data/partial-dag-scope-entity.car >partial_nopin_import_out 2>&1 && test_cmp partial_nopin_import_expected partial_nopin_import_out ' test_expect_success "'ipfs dag import' with pinning errors due to incomplete DAG (unixfs dir exported as dag-scope=entity from IPIP-402)" ' ipfs dag import --stats --enc=json --pin-roots=true ../t0054-dag-car-import-export-data/partial-dag-scope-entity.car >partial_pin_import_out 2>&1 && test_should_contain "\"PinErrorMsg\":\"block was not found locally" partial_pin_import_out ' test_expect_success "'ipfs dag import' pin error in default CLI mode produces exit code 1 (unixfs dir exported as dag-scope=entity from IPIP-402)" ' test_expect_code 1 ipfs dag import ../t0054-dag-car-import-export-data/partial-dag-scope-entity.car >partial_pin_import_out 2>&1 && test_should_contain "Error: pinning root \"QmPDC11yLAbVw3dX5jMeEuSdk4BiVjSd9X87zaYRdVjzW3\" FAILED: block was not found locally" partial_pin_import_out ' test_done ================================================ FILE: test/sharness/t0055-dag-put-json-new-line.sh ================================================ #!/usr/bin/env bash test_description='Test retrieval of JSON put as CBOR does not end with new-line' . lib/test-lib.sh test_init_ipfs test_expect_success 'create test JSON files' ' WANT_JSON="{\"data\":1234}" WANT_HASH="bafyreidbm2zncsc3j25zn7lofgd4woeh6eygdy73thfosuni2rwr3bhcvu" printf "${WANT_JSON}\n" > with_newline.json && printf "${WANT_JSON}" > without_newline.json ' test_expect_success 'puts as CBOR work' ' GOT_HASH_WITHOUT_NEWLINE="$(cat without_newline.json | ipfs dag put --store-codec dag-cbor)" GOT_HASH_WITH_NEWLINE="$(cat with_newline.json | ipfs dag put --store-codec dag-cbor)" ' test_expect_success 'put hashes with or without newline are equal' ' test "${GOT_HASH_WITH_NEWLINE}" = "${GOT_HASH_WITHOUT_NEWLINE}" ' test_expect_success 'hashes are of expected value' ' test "${WANT_HASH}" = "${GOT_HASH_WITH_NEWLINE}" test "${WANT_HASH}" = "${GOT_HASH_WITHOUT_NEWLINE}" ' test_expect_success "retrieval by hash does not have new line" ' ipfs dag get "${WANT_HASH}" > got.json test_cmp without_newline.json got.json ' test_done ================================================ FILE: test/sharness/t0060-daemon.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Juan Batiz-Benet # MIT Licensed; see the LICENSE file in this repository. # test_description="Test daemon command" . lib/test-lib.sh test_expect_success "create pebble config" ' ipfs init --profile=pebbleds,test > /dev/null && cp "$IPFS_PATH/config" init-config ' test_expect_success "cleanup repo" ' rm -rf "$IPFS_PATH" ' test_launch_ipfs_daemon --init --init-config="$(pwd)/init-config" --init-profile=test test_kill_ipfs_daemon test_expect_success "daemon initialization with existing config works" ' ipfs config "Datastore.Spec.path" >actual && test $(cat actual) = "pebbleds" && ipfs config Addresses > orig_addrs ' test_expect_success "cleanup repo" ' rm -rf "$IPFS_PATH" ' test_launch_ipfs_daemon --init --init-config="$(pwd)/init-config" --init-profile=test,randomports test_kill_ipfs_daemon test_expect_success "daemon initialization with existing config + profiles works" ' ipfs config Addresses >new_addrs && test_expect_code 1 diff -q new_addrs orig_addrs ' test_expect_success "cleanup repo" ' rm -rf "$IPFS_PATH" ' test_init_ipfs test_expect_success "set Resource Manager variables showed at startup" ' ipfs config --json Swarm.ResourceMgr.MaxFileDescriptors 1024 && ipfs config Swarm.ResourceMgr.MaxMemory 4GB ' test_launch_ipfs_daemon # this errors if we didn't --init $IPFS_PATH correctly test_expect_success "'ipfs config Identity.PeerID' works" ' PEERID=$(ipfs config Identity.PeerID) ' test_expect_success "'ipfs swarm addrs local' works" ' ipfs swarm addrs local >local_addrs ' test_expect_success "ipfs swarm addrs listen; works" ' ipfs swarm addrs listen >listen_addrs ' test_expect_success "ipfs peer id looks good" ' test_check_peerid "$PEERID" ' # this is for checking SetAllowedOrigins race condition for the api and gateway # See https://github.com/ipfs/go-ipfs/pull/1966 test_expect_success "ipfs API works with the correct allowed origin port" ' curl -s -X POST -H "Origin:http://localhost:$API_PORT" -I "http://$API_ADDR/api/v0/version" ' test_expect_success "ipfs gateway works with the correct allowed origin port" ' curl -s -X POST -H "Origin:http://localhost:$GWAY_PORT" -I "http://$GWAY_ADDR/api/v0/version" ' test_expect_success "ipfs daemon output includes looks good" ' test_should_contain "Initializing daemon..." actual_daemon && test_should_contain "$(ipfs version --all)" actual_daemon && test_should_contain "PeerID: $(ipfs config Identity.PeerID)" actual_daemon && test_should_contain "Swarm listening on 127.0.0.1:" actual_daemon && test_should_contain "RPC API server listening on '$API_MADDR'" actual_daemon && test_should_contain "WebUI: http://'$API_ADDR'/webui" actual_daemon && test_should_contain "Gateway server listening on '$GWAY_MADDR'" actual_daemon && test_should_contain "Daemon is ready" actual_daemon && cat actual_daemon ' test_expect_success ".ipfs/ has been created" ' test -d ".ipfs" && test -f ".ipfs/config" && test -d ".ipfs/datastore" && test -d ".ipfs/blocks" || test_fsh ls -al .ipfs ' # begin same as in t0010 test_expect_success "ipfs version succeeds" ' ipfs version >version.txt ' test_expect_success "ipfs version output looks good" ' egrep "^ipfs version [0-9]+\.[0-9]+\.[0-9]" version.txt >/dev/null || test_fsh cat version.txt ' test_expect_success "ipfs version deps succeeds" ' ipfs version deps >deps.txt ' test_expect_success "ipfs version deps output looks good ( set \$GOIPFSTEST_SKIP_LOCAL_DEVTREE_DEPS_CHECK to skip this test )" ' head -1 deps.txt | grep "go-ipfs@(devel)" && [[ "$GOIPFSTEST_SKIP_LOCAL_DEVTREE_DEPS_CHECK" == "1" ]] || [[ $(tail -n +2 deps.txt | egrep -v -c "^[^ @]+@v[^ @]+( => [^ @]+@v[^ @]+)?$") -eq 0 ]] || test_fsh cat deps.txt ' test_expect_success "ipfs help succeeds" ' ipfs help >help.txt ' test_expect_success "ipfs help output looks good" ' egrep -i "^Usage" help.txt >/dev/null && egrep "ipfs .* " help.txt >/dev/null || test_fsh cat help.txt ' # check transport is encrypted by default and no plaintext is allowed test_expect_success SOCAT "default transport should support encryption (TLS, needs socat )" ' socat -s - tcp:localhost:$SWARM_PORT,connect-timeout=1 > swarmnc < ../t0060-data/mss-tls && grep -q "/tls" swarmnc && test_must_fail grep -q "na" swarmnc || test_fsh cat swarmnc ' test_expect_success SOCAT "default transport should support encryption (Noise, needs socat )" ' socat -s - tcp:localhost:$SWARM_PORT,connect-timeout=1 > swarmnc < ../t0060-data/mss-noise && grep -q "/noise" swarmnc && test_must_fail grep -q "na" swarmnc || test_fsh cat swarmnc ' test_expect_success SOCAT "default transport should not support plaintext (needs socat )" ' socat -s - tcp:localhost:$SWARM_PORT,connect-timeout=1 > swarmnc < ../t0060-data/mss-plaintext && grep -q "na" swarmnc && test_must_fail grep -q "/plaintext" swarmnc || test_fsh cat swarmnc ' test_expect_success "output from streaming commands works" ' test_expect_code 28 curl -X POST -m 5 http://localhost:$API_PORT/api/v0/stats/bw\?poll=true > statsout ' test_expect_success "output looks good" ' grep TotalIn statsout > /dev/null && grep TotalOut statsout > /dev/null && grep RateIn statsout > /dev/null && grep RateOut statsout >/dev/null ' # end same as in t0010 test_expect_success "daemon is still running" ' kill -0 $IPFS_PID ' test_expect_success "'ipfs daemon' can be killed" ' test_kill_repeat_10_sec $IPFS_PID ' test_expect_success "'ipfs daemon' should be able to run with a pipe attached to stdin (issue #861)" ' yes | ipfs daemon >stdin_daemon_out 2>stdin_daemon_err & DAEMON_PID=$! test_wait_for_file 20 100ms "$IPFS_PATH/api" && test_set_address_vars stdin_daemon_out ' test_expect_success "daemon with pipe eventually becomes live" ' pollEndpoint -host='$API_MADDR' -v -tout=1s -tries=10 >stdin_poll_apiout 2>stdin_poll_apierr && test_kill_repeat_10_sec $DAEMON_PID || test_fsh cat stdin_daemon_out || test_fsh cat stdin_daemon_err || test_fsh cat stdin_poll_apiout || test_fsh cat stdin_poll_apierr ' test_expect_success "'ipfs daemon' cleans up when it fails to start" ' test_must_fail ipfs daemon --routing=foobar && test ! -e "$IPFS_PATH/repo.lock" ' ulimit -S -n 512 TEST_ULIMIT_PRESET=1 test_launch_ipfs_daemon test_expect_success "daemon raised its fd limit" ' test_should_not_contain "setting file descriptor limit" actual_daemon ' test_expect_success "daemon actually can handle 2048 file descriptors" ' hang-fds -hold=2s 2000 '$API_MADDR' > /dev/null ' test_expect_success "daemon didn't throw any errors" ' test_expect_code 1 grep "too many open files" daemon_err ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0060-data/mss-noise ================================================ /multistream/1.0.0 /noise ================================================ FILE: test/sharness/t0060-data/mss-plaintext ================================================ /multistream/1.0.0 /plaintext/2.0.0 ================================================ FILE: test/sharness/t0060-data/mss-tls ================================================ /multistream/1.0.0 /tls/1.0.0 ================================================ FILE: test/sharness/t0061-daemon-opts.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Juan Batiz-Benet # MIT Licensed; see the LICENSE file in this repository. # test_description="Test daemon command" . lib/test-lib.sh test_init_ipfs test_launch_ipfs_daemon --disable-transport-encryption gwyaddr=$GWAY_ADDR apiaddr=$API_ADDR # Odd. this fails here, but the inverse works on t0060-daemon. test_expect_success SOCAT 'transport should be unencrypted ( needs socat )' ' socat -s - tcp:localhost:$SWARM_PORT,connect-timeout=1 > swarmnc < ../t0060-data/mss-plaintext && grep -q "/plaintext" swarmnc && test_must_fail grep -q "na" swarmnc || test_fsh cat swarmnc ' test_kill_ipfs_daemon test_launch_ipfs_daemon_without_network gwyaddr=$GWAY_ADDR apiaddr=$API_ADDR test_expect_success 'gateway should work in offline mode' ' echo "hello mars :$gwyaddr :$apiaddr" >expected && HASH=$(ipfs add -q expected) && curl -sfo actual1 "http://$gwyaddr/ipfs/$HASH" && test_cmp expected actual1 ' test_kill_ipfs_daemon test_expect_success 'daemon should not start with bad dht opt' ' test_must_fail ipfs daemon --routing=fdsfdsfds > daemon_output 2>&1 ' test_expect_success 'output contains info about dht option' ' grep "unrecognized routing option:" daemon_output || test_fsh cat daemon_output ' test_expect_success 'daemon should not start with supernode dht opt' ' test_must_fail ipfs daemon --routing=supernode > daemon_output2 2>&1 ' test_expect_success 'output contains info about supernode dht option' ' grep "supernode routing was never fully implemented" daemon_output2 || test_fsh cat daemon_output2 ' test_done ================================================ FILE: test/sharness/t0062-daemon-api.sh ================================================ #!/usr/bin/env bash # # MIT Licensed; see the LICENSE file in this repository. # test_description="Test daemon command" . lib/test-lib.sh test_init_ipfs differentport=$((API_PORT + 1)) api_other="/ip4/127.0.0.1/tcp/$differentport" api_unreachable="/ip4/127.0.0.1/tcp/1" test_expect_success "config setup" ' peerid=$(ipfs config Identity.PeerID) && test_check_peerid "$peerid" ' test_client() { opts="$@" echo "OPTS = " $opts test_expect_success "client must work properly $state" ' printf "$peerid" >expected && ipfs id -f="" $opts >actual && test_cmp expected actual ' } test_client_must_fail() { opts="$@" echo "OPTS = " $opts test_expect_success "client should fail $state" ' echo "Error: cannot connect to the api. Is the daemon running? To run as a standalone CLI command remove the api file in \`\$IPFS_PATH/api\`" >expected_err && test_must_fail ipfs id -f="" $opts >actual 2>actual_err && test_cmp expected_err actual_err ' } test_client_suite() { state="$1" cfg_success="$2" diff_success="$3" api_fromcfg="$4" api_different="$5" # must always work test_client # must always err test_client_must_fail --api "$api_unreachable" if [ "$cfg_success" = true ]; then test_client --api "$api_fromcfg" else test_client_must_fail --api "$api_fromcfg" fi if [ "$diff_success" = true ]; then test_client --api "$api_different" else test_client_must_fail --api "$api_different" fi } # first, test things without daemon, without /api file # with no daemon, everything should fail # (using unreachable because API_MADDR doesn't get set until daemon start) test_client_suite "(daemon off, no --api, no /api file)" false false "$api_unreachable" "$api_other" # then, test things with daemon, with /api file test_launch_ipfs_daemon test_expect_success "'ipfs daemon' creates api file" ' test -f ".ipfs/api" ' test_client_suite "(daemon on, no --api, /api file from cfg)" true false "$API_MADDR" "$api_other" # then, test things without daemon, with /api file test_kill_ipfs_daemon # again, both should fail test_client_suite "(daemon off, no --api, /api file from cfg)" false false "$API_MADDR" "$api_other" test_done ================================================ FILE: test/sharness/t0063-daemon-init.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Juan Batiz-Benet # MIT Licensed; see the LICENSE file in this repository. # test_description="Test daemon --init command" . lib/test-lib.sh # We don't want the normal test_init_ipfs but we need to make sure the # IPFS_PATH is set correctly. export IPFS_PATH="$(pwd)/.ipfs" # safety check since we will be removing the directory if [ -e "$IPFS_PATH" ]; then echo "$IPFS_PATH exists" exit 1 fi test_ipfs_daemon_init() { # Doing it manually since we want to launch the daemon with an # empty or non-existent repo; the normal # test_launch_ipfs_daemon does not work since it assumes the # repo was created a particular way with regard to the API # server. test_expect_success "'ipfs daemon --init' succeeds" ' ipfs daemon --init --init-profile=test >actual_daemon 2>daemon_err & IPFS_PID=$! sleep 2 && if ! kill -0 $IPFS_PID; then cat daemon_err; return 1; fi ' test_expect_success "'ipfs daemon' can be killed" ' test_kill_repeat_10_sec $IPFS_PID ' } test_expect_success "remove \$IPFS_PATH dir" ' rm -rf "$IPFS_PATH" ' test_ipfs_daemon_init test_expect_success "create empty \$IPFS_PATH dir" ' rm -rf "$IPFS_PATH" && mkdir "$IPFS_PATH" ' test_ipfs_daemon_init test_done ================================================ FILE: test/sharness/t0063-external.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="test external command functionality" . lib/test-lib.sh # set here so daemon launches with it PATH=`pwd`/bin:$PATH test_init_ipfs test_expect_success "create fake ipfs-update bin" ' mkdir bin && echo "#!/bin/sh" > bin/ipfs-update && echo "pwd" >> bin/ipfs-update && echo "test -e \"$IPFS_PATH/repo.lock\" || echo \"repo not locked\" " >> bin/ipfs-update && chmod +x bin/ipfs-update && mkdir just_for_test ' test_expect_success "external command runs from current user directory and doesn't lock repo" ' (cd just_for_test && ipfs update) > actual ' test_expect_success "output looks good" ' echo `pwd`/just_for_test > exp && echo "repo not locked" >> exp && test_cmp exp actual ' test_launch_ipfs_daemon test_expect_success "external command runs from current user directory when daemon is running" ' (cd just_for_test && ipfs update) > actual ' test_expect_success "output looks good" ' echo `pwd`/just_for_test > exp && test_cmp exp actual ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0064-api-file.sh ================================================ #!/usr/bin/env bash # # MIT Licensed; see the LICENSE file in this repository. # test_description="Test api file" . lib/test-lib.sh test_init_ipfs test_launch_ipfs_daemon test_kill_ipfs_daemon test_expect_success "version always works" ' ipfs version >/dev/null ' test_expect_success "swarm peers fails when offline" ' test_must_fail ipfs swarm peers >/dev/null ' test_expect_success "swarm peers fails when offline and API specified" ' test_must_fail ipfs swarm peers --api="$API_MADDR" >/dev/null ' test_expect_success "pin ls succeeds when offline" ' ipfs pin ls >/dev/null ' test_expect_success "pin ls fails when offline and API specified" ' test_must_fail ipfs pin ls --api="$API_MADDR" >/dev/null ' test_expect_success "id succeeds when offline" ' ipfs id >/dev/null ' test_expect_success "id fails when offline API specified" ' test_must_fail ipfs id --api="$API_MADDR" >/dev/null ' test_expect_success "create API file" ' echo "$API_MADDR" > "$IPFS_PATH/api" ' test_expect_success "version always works" ' ipfs version >/dev/null ' test_expect_success "id succeeds when offline and API file exists" ' ipfs id >/dev/null ' test_expect_success "pin ls succeeds when offline and API file exists" ' ipfs pin ls >/dev/null ' test_launch_ipfs_daemon test_expect_success "version always works" ' ipfs version >/dev/null ' test_expect_success "id succeeds when online" ' ipfs id >/dev/null ' test_expect_success "swarm peers succeeds when online" ' ipfs swarm peers >/dev/null ' test_expect_success "pin ls succeeds when online" ' ipfs pin ls >/dev/null ' test_expect_success "remove API file when daemon is running" ' rm "$IPFS_PATH/api" ' test_expect_success "version always works" ' ipfs version >/dev/null ' test_expect_success "swarm peers fails when the API file is missing" ' test_must_fail ipfs swarm peers >/dev/null ' test_expect_success "id fails when daemon is running but API file is missing (locks repo)" ' test_must_fail ipfs pin ls >/dev/null ' test_expect_success "pin ls fails when daemon is running but API file is missing (locks repo)" ' test_must_fail ipfs pin ls >/dev/null ' test_kill_ipfs_daemon APIPORT=32563 test_expect_success "Verify gateway file diallable while on unspecified" ' ipfs config Addresses.API /ip4/0.0.0.0/tcp/$APIPORT && test_launch_ipfs_daemon && cat "$IPFS_PATH/api" > api_file_actual && echo -n "/ip4/127.0.0.1/tcp/$APIPORT" > api_file_expected && test_cmp api_file_expected api_file_actual ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0065-active-requests.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2016 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test active request commands" . lib/test-lib.sh test_init_ipfs test_launch_ipfs_daemon test_expect_success "command works" ' ipfs diag cmds > cmd_out ' test_expect_success "invoc shows up in output" ' grep "diag/cmds" cmd_out > /dev/null ' test_expect_success "start longer running command" ' ipfs log tail & LOGPID=$! go-sleep 100ms ' test_expect_success "long running command shows up" ' ipfs diag cmds > cmd_out2 ' test_expect_success "output looks good" ' grep "log/tail" cmd_out2 | grep "true" > /dev/null ' test_expect_success "kill log cmd" ' kill $LOGPID go-sleep 0.5s kill $LOGPID wait $LOGPID || true ' test_expect_success "long running command inactive" ' ipfs diag cmds > cmd_out3 ' test_expect_success "command shows up as inactive" ' grep "log/tail" cmd_out3 | grep "false" ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0066-migration.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2016 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test migrations auto update prompt" . lib/test-lib.sh test_init_ipfs # Remove explicit AutoConf.Enabled=false from test profile to use implicit default # This allows daemon to work with 'auto' values added by v16-to-17 migration ipfs config --json AutoConf.Enabled null >/dev/null 2>&1 MIGRATION_START=7 IPFS_REPO_VER=$(<.ipfs/version) # Generate mock migration binaries gen_mock_migrations() { mkdir bin i=$((MIGRATION_START)) until [ $i -ge $IPFS_REPO_VER ] do j=$((i+1)) echo "#!/bin/bash" > bin/fs-repo-${i}-to-${j} echo "echo fake applying ${i}-to-${j} repo migration" >> bin/fs-repo-${i}-to-${j} # Update version file to the target version for hybrid migration system echo "if [ \"\$1\" = \"-path\" ] && [ -n \"\$2\" ]; then" >> bin/fs-repo-${i}-to-${j} echo " echo $j > \"\$2/version\"" >> bin/fs-repo-${i}-to-${j} echo "elif [ -n \"\$IPFS_PATH\" ]; then" >> bin/fs-repo-${i}-to-${j} echo " echo $j > \"\$IPFS_PATH/version\"" >> bin/fs-repo-${i}-to-${j} echo "fi" >> bin/fs-repo-${i}-to-${j} chmod +x bin/fs-repo-${i}-to-${j} ((i++)) done } # Check for expected output from each migration check_migration_output() { out_file="$1" i=$((MIGRATION_START)) until [ $i -ge $IPFS_REPO_VER ] do j=$((i+1)) grep "applying ${i}-to-${j} repo migration" "$out_file" > /dev/null ((i++)) done } # Create fake migration binaries instead of letting ipfs download from network # To test downloading and running actual binaries, comment out this test. test_expect_success "setup mock migrations" ' gen_mock_migrations && find bin -name "fs-repo-*-to-*" | wc -l > mock_count && echo $((IPFS_REPO_VER-MIGRATION_START)) > expect_mock_count && export PATH="$(pwd)/bin":$PATH && test_cmp mock_count expect_mock_count ' test_expect_success "manually reset repo version to $MIGRATION_START" ' echo "$MIGRATION_START" > "$IPFS_PATH"/version ' test_expect_success "ipfs daemon --migrate=false fails" ' test_expect_code 1 ipfs daemon --migrate=false > false_out 2>&1 ' test_expect_success "output looks good" ' grep "Kubo repository at .* has version .* and needs to be migrated to version" false_out && grep "Error: fs-repo requires migration" false_out ' # The migrations will succeed and the daemon will continue running # since the mock migrations now properly update the repo version number. test_expect_success "ipfs daemon --migrate=true runs migration" ' ipfs daemon --migrate=true > true_out 2>&1 & DAEMON_PID=$! # Wait for daemon to be ready then shutdown gracefully sleep 3 && ipfs shutdown 2>/dev/null || kill $DAEMON_PID 2>/dev/null || true wait $DAEMON_PID 2>/dev/null || true ' test_expect_success "output looks good" ' check_migration_output true_out && (grep "Success: fs-repo migrated to version $IPFS_REPO_VER" true_out > /dev/null || grep "Hybrid migration completed successfully: v$MIGRATION_START → v$IPFS_REPO_VER" true_out > /dev/null) ' test_expect_success "reset repo version for auto-migration test" ' echo "$MIGRATION_START" > "$IPFS_PATH"/version ' test_expect_success "'ipfs daemon' prompts to auto migrate" ' test_expect_code 1 ipfs daemon > daemon_out 2>&1 ' test_expect_success "output looks good" ' grep "Kubo repository at .* has version .* and needs to be migrated to version" daemon_out > /dev/null && grep "Run migrations now?" daemon_out > /dev/null && grep "Error: fs-repo requires migration" daemon_out > /dev/null ' test_expect_success "ipfs repo migrate succeed" ' test_expect_code 0 ipfs repo migrate > migrate_out ' test_expect_success "output looks good" ' grep "Migrating repository from version" migrate_out > /dev/null && (grep "Success: fs-repo migrated to version $IPFS_REPO_VER" migrate_out > /dev/null || grep "Hybrid migration completed successfully: v$MIGRATION_START → v$IPFS_REPO_VER" migrate_out > /dev/null) ' test_expect_success "manually reset repo version to latest" ' echo "$IPFS_REPO_VER" > "$IPFS_PATH"/version ' test_expect_success "detect repo does not need migration" ' test_expect_code 0 ipfs repo migrate > migrate_out ' test_expect_success "output looks good" ' grep "Repository is already at version" migrate_out > /dev/null ' # ensure that we get a lock error if we need to migrate and the daemon is running test_launch_ipfs_daemon test_expect_success "manually reset repo version to $MIGRATION_START" ' echo "$MIGRATION_START" > "$IPFS_PATH"/version ' test_expect_success "ipfs repo migrate fails" ' test_expect_code 1 ipfs repo migrate 2> migrate_out ' test_expect_success "output looks good" ' grep "repo.lock" migrate_out > /dev/null ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0067-unix-api.sh ================================================ #!/usr/bin/env bash # # MIT Licensed; see the LICENSE file in this repository. # test_description="Test unix API transport" . lib/test-lib.sh test_init_ipfs # We can't use the trash dir as the full name must be longer less than 108 bytes # long (because that's the max unix domain socket path length). SOCKDIR="$(mktemp -d "${TMPDIR:-/tmp}/unix-api-sharness.XXXXXX")" test_expect_success "configure" ' peerid=$(ipfs config Identity.PeerID) && ipfs config Addresses.API "/unix/$SOCKDIR/sock" ' test_launch_ipfs_daemon test_expect_success "client works" ' printf "$peerid" >expected && ipfs --api="/unix/$SOCKDIR/sock" id -f="" >actual && test_cmp expected actual ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0070-user-config.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Brian Holder-Chow Lin On # MIT Licensed; see the LICENSE file in this repository. # test_description="Test user-provided config values" . lib/test-lib.sh test_init_ipfs test_expect_success "bootstrap doesn't overwrite user-provided config keys (top-level)" ' ipfs config Identity.PeerID >previous && ipfs config Identity.PeerID foo && ipfs bootstrap rm --all && echo "foo" >expected && ipfs config Identity.PeerID >actual && ipfs config Identity.PeerID $(cat previous) && test_cmp expected actual ' test_done ================================================ FILE: test/sharness/t0080-repo.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test ipfs repo operations" . lib/test-lib.sh test_init_ipfs --empty-repo=false test_launch_ipfs_daemon_without_network test_expect_success "'ipfs repo gc' succeeds" ' ipfs repo gc >gc_out_actual ' test_expect_success "'ipfs add afile' succeeds" ' echo "some text" >afile && HASH=`ipfs add -q afile` ' test_expect_success "added file was pinned" ' ipfs pin ls --type=recursive >actual && grep "$HASH" actual ' test_expect_success "'ipfs repo gc' succeeds" ' ipfs repo gc >gc_out_actual ' test_expect_success "'ipfs repo gc' looks good (patch root)" ' test_should_not_contain "removed $HASH" gc_out_actual ' test_expect_success "'ipfs repo gc' doesn't remove file" ' ipfs cat "$HASH" >out && test_cmp out afile ' test_expect_success "'ipfs pin rm' succeeds" ' ipfs pin rm -r "$HASH" >actual1 ' test_expect_success "'ipfs pin rm' output looks good" ' echo "unpinned $HASH" >expected1 && test_cmp expected1 actual1 ' test_expect_success "ipfs repo gc fully reverse ipfs add (part 1)" ' ipfs repo gc && random-data -size=100000 -seed=41 >gcfile && find "$IPFS_PATH/blocks" -type f -name "*.data" | sort -u > expected_blocks && hash=$(ipfs add -q gcfile) && ipfs pin rm -r $hash && ipfs repo gc ' test_expect_success "'ipfs repo gc --silent' succeeds (no output)" ' echo "should be empty" >bfile && HASH2=`ipfs add -q bfile` && ipfs cat "$HASH2" >expected11 && test_cmp expected11 bfile && ipfs pin rm -r "$HASH2" && ipfs repo gc --silent >gc_out_empty && test_cmp /dev/null gc_out_empty && test_must_fail ipfs cat "$HASH2" 2>err_expected1 && grep "Error: block was not found locally (offline): ipld: could not find $HASH2" err_expected1 ' test_kill_ipfs_daemon test_expect_success "ipfs repo gc fully reverse ipfs add (part 2)" ' find "$IPFS_PATH/blocks" -type f -name "*.data" | sort -u > actual_blocks && test_cmp expected_blocks actual_blocks ' test_launch_ipfs_daemon_without_network test_expect_success "file no longer pinned" ' ipfs pin ls --type=recursive --quiet >actual2 && test_expect_code 1 grep $HASH actual2 ' test_expect_success "recursively pin afile(default action)" ' HASH=`ipfs add -q afile` && ipfs pin add "$HASH" ' test_expect_success "recursively pin rm afile (default action)" ' ipfs pin rm "$HASH" ' test_expect_success "recursively pin afile" ' ipfs pin add -r "$HASH" ' test_expect_success "pinning directly should fail now" ' echo "Error: pin: $HASH already pinned recursively" >expected3 && test_must_fail ipfs pin add -r=false "$HASH" 2>actual3 && test_cmp expected3 actual3 ' test_expect_success "'ipfs pin rm -r=false ' should fail" ' echo "Error: $HASH is pinned recursively" >expected4 test_must_fail ipfs pin rm -r=false "$HASH" 2>actual4 && test_cmp expected4 actual4 ' test_expect_success "remove recursive pin, add direct" ' echo "unpinned $HASH" >expected5 && ipfs pin rm -r "$HASH" >actual5 && test_cmp expected5 actual5 && ipfs pin add -r=false "$HASH" ' test_expect_success "remove direct pin" ' echo "unpinned $HASH" >expected6 && ipfs pin rm "$HASH" >actual6 && test_cmp expected6 actual6 ' test_expect_success "'ipfs repo gc' removes file" ' ipfs block stat $HASH && ipfs repo gc && test_must_fail ipfs block stat $HASH ' # Convert all to a base32-multihash as refs local outputs cidv1 raw # Technically converting refs local output would suffice, but this is more # future proof if we ever switch to adding the files with cid-version 1. test_expect_success "'ipfs refs local' no longer shows file" ' EMPTY_DIR=QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn && HASH_MH=`cid-fmt -b base32 "%M" "$HASH"` && HARDCODED_HASH_MH=`cid-fmt -b base32 "%M" "QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y"` && EMPTY_DIR_MH=`cid-fmt -b base32 "%M" "$EMPTY_DIR"` && HASH_WELCOME_DOCS_MH=`cid-fmt -b base32 "%M" "$HASH_WELCOME_DOCS"` && ipfs refs local | cid-fmt -b base32 --filter "%M" >actual8 && grep "$HARDCODED_HASH_MH" actual8 && grep "$EMPTY_DIR_MH" actual8 && grep "$HASH_WELCOME_DOCS_MH" actual8 && test_must_fail grep "$HASH_MH" actual8 ' test_expect_success "adding multiblock random file succeeds" ' random-data -size=1000000 >multiblock && MBLOCKHASH=`ipfs add -q multiblock` ' test_expect_success "'ipfs pin ls --type=indirect' is correct" ' ipfs refs "$MBLOCKHASH" >refsout && ipfs refs -r "$HASH_WELCOME_DOCS" >>refsout && sed -i"~" "s/\(.*\)/\1 indirect/g" refsout && ipfs pin ls --type=indirect >indirectpins && test_sort_cmp refsout indirectpins ' test_expect_success "pin something directly" ' echo "ipfs is so awesome" >awesome && DIRECTPIN=`ipfs add -q awesome` && echo "unpinned $DIRECTPIN" >expected9 && ipfs pin rm -r "$DIRECTPIN" >actual9 && test_cmp expected9 actual9 && echo "pinned $DIRECTPIN directly" >expected10 && ipfs pin add -r=false "$DIRECTPIN" >actual10 && test_cmp expected10 actual10 ' test_expect_success "'ipfs pin ls --type=direct' is correct" ' echo "$DIRECTPIN direct" >directpinexpected && ipfs pin ls --type=direct >directpinout && test_sort_cmp directpinexpected directpinout ' test_expect_success "'ipfs pin ls --type=recursive' is correct" ' echo "$MBLOCKHASH" >rp_expected && echo "$HASH_WELCOME_DOCS" >>rp_expected && echo "$EMPTY_DIR" >>rp_expected && sed -i"~" "s/\(.*\)/\1 recursive/g" rp_expected && ipfs pin ls --type=recursive >rp_actual && test_sort_cmp rp_expected rp_actual ' test_expect_success "'ipfs pin ls --type=all --quiet' is correct" ' cat directpinout >allpins && cat rp_actual >>allpins && cat indirectpins >>allpins && cut -f1 -d " " allpins | sort | uniq >> allpins_uniq_hashes && ipfs pin ls --type=all --quiet >actual_allpins && test_sort_cmp allpins_uniq_hashes actual_allpins ' test_expect_success "'ipfs refs --unique' is correct" ' mkdir -p uniques && echo "content1" > uniques/file1 && echo "content1" > uniques/file2 && ROOT=$(ipfs add -r -Q uniques) && ipfs refs --unique $ROOT >expected && ipfs add -q uniques/file1 >unique_hash && test_cmp expected unique_hash ' test_expect_success "'ipfs refs --unique --recursive' is correct" ' mkdir -p a/b/c && echo "c1" > a/f1 && echo "c1" > a/b/f1 && echo "c1" > a/b/c/f1 && echo "c2" > a/b/c/f2 && ROOT=$(ipfs add -r -Q a) && ipfs refs --unique --recursive $ROOT >refs_output && wc -l refs_output | sed "s/^ *//g" >line_count && echo "4 refs_output" >expected && test_cmp expected line_count || test_fsh cat refs_output ' test_expect_success "'ipfs refs --recursive (bigger)'" ' mkdir -p b/c/d/e && echo "content1" >b/f && echo "content1" >b/c/f1 && echo "content1" >b/c/d/f2 && echo "content2" >b/c/f2 && echo "content2" >b/c/d/f1 && echo "content2" >b/c/d/e/f && cp -r b b2 && mv b2 b/b2 && cp -r b b3 && mv b3 b/b3 && cp -r b b4 && mv b4 b/b4 && hash=$(ipfs add -r -Q b) && ipfs refs -r "$hash" >refs_output && wc -l refs_output | sed "s/^ *//g" >actual && echo "79 refs_output" >expected && test_cmp expected actual || test_fsh cat refs_output ' test_expect_success "'ipfs refs --unique --recursive (bigger)'" ' ipfs refs -r "$hash" >refs_output && sort refs_output | uniq >expected && ipfs refs -r -u "$hash" >actual && test_sort_cmp expected actual || test_fsh cat refs_output ' get_field_num() { field=$1 file=$2 num=$(grep "$field" "$file" | awk '{ print $2 }') echo $num } test_expect_success "'ipfs repo stat' succeeds" ' ipfs repo stat > repo-stats ' test_expect_success "repo stats came out correct" ' grep "RepoPath" repo-stats && grep "RepoSize" repo-stats && grep "NumObjects" repo-stats && grep "Version" repo-stats && grep "StorageMax" repo-stats ' test_expect_success "'ipfs repo stat --human' succeeds" ' ipfs repo stat --human > repo-stats-human ' test_expect_success "repo stats --human came out correct" ' grep "RepoPath" repo-stats-human && grep -E "RepoSize:\s*([0-9]*[.])?[0-9]+\s+?(B|kB|MB|GB|TB|PB|EB)" repo-stats-human && grep "NumObjects" repo-stats-human && grep "Version" repo-stats-human && grep -E "StorageMax:\s*([0-9]*[.])?[0-9]+\s+?(B|kB|MB|GB|TB|PB|EB)" repo-stats-human || test_fsh cat repo-stats-human ' test_expect_success "'ipfs repo stat' after adding a file" ' ipfs add repo-stats && ipfs repo stat > repo-stats-2 ' test_expect_success "repo stats are updated correctly" ' test $(get_field_num "RepoSize" repo-stats-2) -ge $(get_field_num "RepoSize" repo-stats) ' test_expect_success "'ipfs repo stat --size-only' succeeds" ' ipfs repo stat --size-only > repo-stats-size-only ' test_expect_success "repo stats came out correct for --size-only" ' test_should_contain "RepoSize" repo-stats-size-only && test_should_contain "StorageMax" repo-stats-size-only && test_should_not_contain "RepoPath" repo-stats-size-only && test_should_not_contain "NumObjects" repo-stats-size-only && test_should_not_contain "Version" repo-stats-size-only ' test_expect_success "'ipfs repo version' succeeds" ' ipfs repo version > repo-version ' test_expect_success "repo version came out correct" ' egrep "^ipfs repo version fs-repo@[0-9]+" repo-version >/dev/null ' test_expect_success "'ipfs repo version -q' succeeds" ' ipfs repo version -q > repo-version-q ' test_expect_success "repo version came out correct" ' egrep "^fs-repo@[0-9]+" repo-version-q >/dev/null ' test_kill_ipfs_daemon test_expect_success "remove Datastore.StorageMax from config" ' ipfs config Datastore.StorageMax "" ' test_expect_success "'ipfs repo stat' still succeeds" ' ipfs repo stat > repo-stats ' test_done ================================================ FILE: test/sharness/t0081-repo-pinning.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test ipfs repo pinning" . lib/test-lib.sh test_pin_flag() { object=$1 ptype=$2 expect=$3 echo "test_pin_flag" "$@" if ipfs pin ls --type="$ptype" "$object" >actual then test "$expect" = "true" && return test_fsh cat actual return else test "$expect" = "false" && return test_fsh cat actual return fi } test_pin() { object=$1 shift test_str_contains "recursive" $@ [ "$?" = "0" ] && r="true" || r="false" test_str_contains "indirect" $@ [ "$?" = "0" ] && i="true" || i="false" test_str_contains "direct" $@ [ "$?" = "0" ] && d="true" || d="false" test_pin_flag "$object" "recursive" $r || return 1 test_pin_flag "$object" "indirect" $i || return 1 test_pin_flag "$object" "direct" $d || return 1 return 0 } test_init_ipfs # test runs much faster without daemon. # TODO: turn this back on after: # https://github.com/ipfs/go-ipfs/issues/1075 # test_launch_ipfs_daemon HASH_FILE6="QmRsBC3Y2G6VRPYGAVpZczx1W7Xw54MtM1NcLKTkn6rx3U" HASH_FILE5="QmaN3PtyP8DcVGHi3Q2Fcp7CfAFVcVXKddWbHoNvaA41zf" HASH_FILE4="QmV1aiVgpDknKQugrK59uBUbMrPnsQM1F9FXbFcfgEvUvH" HASH_FILE3="QmZrr4Pzqp3NnMzMfbMhNe7LghfoUFHVx7c9Po9GZrhKZ7" HASH_FILE2="QmSkjTornLY72QhmK9NvAz26815pTaoAL42rF8Qi3w2WBP" HASH_FILE1="QmbgX4aXhSSY88GHmPQ4roizD8wFwPX8jzTLjc8VAp89x4" HASH_DIR4="QmW98gV71Ns4bX7QbgWAqLiGF3SDC1JpveZSgBh4ExaSAd" HASH_DIR3="QmRsCaNBMkweZ9vHT5PJRd2TT9rtNKEKyuognCEVxZxF1H" HASH_DIR2="QmTUTQAgeVfughDSFukMZLbfGvetDJY7Ef5cDXkKK4abKC" HASH_DIR1="QmNyZVFbgvmzguS2jVMRb8PQMNcCMJrn9E3doDhBbcPNTY" HASH_NOPINDIR="QmWHjrRJYSfYKz5V9dWWSKu47GdY7NewyRhyTiroXgWcDU" HASH_NOPIN_FILE1="QmUJT3GQi1dxQyTZbkaWeer9GkCn1d3W3HHRLSDr6PTcpx" HASH_NOPIN_FILE2="QmarR7m9JT7qHEGhuFNZUEMAnoZ8E9QAfsthHCQ9Y2GfoT" DIR1="dir1" DIR2="dir1/dir2" DIR4="dir1/dir2/dir4" DIR3="dir1/dir3" FILE1="dir1/file1" FILE2="dir1/file2" FILE3="dir1/file3" FILE4="dir1/dir2/file4" FILE6="dir1/dir2/dir4/file6" FILE5="dir1/dir3/file5" test_expect_success "'ipfs add dir' succeeds" ' mkdir dir1 && mkdir dir1/dir2 && mkdir dir1/dir2/dir4 && mkdir dir1/dir3 && echo "some text 1" >dir1/file1 && echo "some text 2" >dir1/file2 && echo "some text 3" >dir1/file3 && echo "some text 1" >dir1/dir2/file1 && echo "some text 4" >dir1/dir2/file4 && echo "some text 1" >dir1/dir2/dir4/file1 && echo "some text 2" >dir1/dir2/dir4/file2 && echo "some text 6" >dir1/dir2/dir4/file6 && echo "some text 2" >dir1/dir3/file2 && echo "some text 5" >dir1/dir3/file5 && ipfs add -Q -r dir1 >actual && echo "$HASH_DIR1" >expected && ipfs repo gc && # remove the patch chaff test_cmp expected actual ' test_expect_success "objects are there" ' ipfs cat "$HASH_FILE6" >FILE6_a && ipfs cat "$HASH_FILE5" >FILE5_a && ipfs cat "$HASH_FILE4" >FILE4_a && ipfs cat "$HASH_FILE3" >FILE3_a && ipfs cat "$HASH_FILE2" >FILE2_a && ipfs cat "$HASH_FILE1" >FILE1_a && ipfs ls "$HASH_DIR3" >DIR3_a && ipfs ls "$HASH_DIR4" >DIR4_a && ipfs ls "$HASH_DIR2" >DIR2_a && ipfs ls "$HASH_DIR1" >DIR1_a ' # saving this output for later test_expect_success "ipfs dag get $HASH_DIR1 works" ' ipfs dag get $HASH_DIR1 | jq -r ".Links[] | .Hash | .[\"/\"]" > DIR1_objlink ' test_expect_success "added dir was pinned recursively" ' test_pin_flag $HASH_DIR1 recursive true ' test_expect_success "rest were pinned indirectly" ' test_pin_flag "$HASH_FILE6" indirect true test_pin_flag "$HASH_FILE5" indirect true test_pin_flag "$HASH_FILE4" indirect true test_pin_flag "$HASH_FILE3" indirect true test_pin_flag "$HASH_FILE2" indirect true test_pin_flag "$HASH_FILE1" indirect true test_pin_flag "$HASH_DIR3" indirect true test_pin_flag "$HASH_DIR4" indirect true test_pin_flag "$HASH_DIR2" indirect true ' test_expect_success "added dir was NOT pinned indirectly" ' test_pin_flag "$HASH_DIR1" indirect false ' test_expect_success "nothing is pinned directly" ' ipfs pin ls --type=direct >actual4 && test_must_be_empty actual4 ' test_expect_success "'ipfs repo gc' succeeds" ' ipfs repo gc >gc_out_actual ' test_expect_success "objects are still there" ' cat FILE6_a FILE5_a FILE4_a FILE3_a FILE2_a FILE1_a >expected45 && cat DIR3_a DIR4_a DIR2_a DIR1_a >>expected45 && ipfs cat "$HASH_FILE6" >actual45 && ipfs cat "$HASH_FILE5" >>actual45 && ipfs cat "$HASH_FILE4" >>actual45 && ipfs cat "$HASH_FILE3" >>actual45 && ipfs cat "$HASH_FILE2" >>actual45 && ipfs cat "$HASH_FILE1" >>actual45 && ipfs ls "$HASH_DIR3" >>actual45 && ipfs ls "$HASH_DIR4" >>actual45 && ipfs ls "$HASH_DIR2" >>actual45 && ipfs ls "$HASH_DIR1" >>actual45 && test_cmp expected45 actual45 ' test_expect_success "remove dir recursive pin succeeds" ' echo "unpinned $HASH_DIR1" >expected5 && ipfs pin rm -r "$HASH_DIR1" >actual5 && test_cmp expected5 actual5 ' test_expect_success "none are pinned any more" ' test_pin "$HASH_FILE6" && test_pin "$HASH_FILE5" && test_pin "$HASH_FILE4" && test_pin "$HASH_FILE3" && test_pin "$HASH_FILE2" && test_pin "$HASH_FILE1" && test_pin "$HASH_DIR3" && test_pin "$HASH_DIR4" && test_pin "$HASH_DIR2" && test_pin "$HASH_DIR1" ' test_expect_success "pin some directly and indirectly" ' ipfs pin add -r=false "$HASH_DIR1" >actual7 && ipfs pin add -r=true "$HASH_DIR2" >>actual7 && ipfs pin add -r=false "$HASH_FILE1" >>actual7 && echo "pinned $HASH_DIR1 directly" >expected7 && echo "pinned $HASH_DIR2 recursively" >>expected7 && echo "pinned $HASH_FILE1 directly" >>expected7 && test_cmp expected7 actual7 ' test_expect_success "pin lists look good" ' test_pin $HASH_DIR1 direct && test_pin $HASH_DIR2 recursive && test_pin $HASH_DIR3 && test_pin $HASH_DIR4 indirect && test_pin $HASH_FILE1 indirect direct && test_pin $HASH_FILE2 indirect && test_pin $HASH_FILE3 && test_pin $HASH_FILE4 indirect && test_pin $HASH_FILE5 && test_pin $HASH_FILE6 indirect ' test_expect_success "'ipfs repo gc' succeeds" ' ipfs repo gc && test_must_fail ipfs block stat $HASH_FILE3 && test_must_fail ipfs block stat $HASH_FILE5 && test_must_fail ipfs block stat $HASH_DIR3 ' # use object links for HASH_DIR1 here because its children # no longer exist test_expect_success "some objects are still there" ' cat FILE6_a FILE4_a FILE2_a FILE1_a >expected8 && cat DIR4_a DIR2_a DIR1_objlink >>expected8 && ipfs cat "$HASH_FILE6" >actual8 && ipfs cat "$HASH_FILE4" >>actual8 && ipfs cat "$HASH_FILE2" >>actual8 && ipfs cat "$HASH_FILE1" >>actual8 && ipfs ls "$HASH_DIR4" >>actual8 && ipfs ls "$HASH_DIR2" >>actual8 && ipfs dag get "$HASH_DIR1" | jq -r ".Links[] | .Hash | .[\"/\"]" >>actual8 && test_cmp expected8 actual8 ' # todo: make this faster somehow. test_expect_success "some are no longer there" ' test_must_fail ipfs cat "$HASH_FILE5" && test_must_fail ipfs cat "$HASH_FILE3" && test_must_fail ipfs ls "$HASH_DIR3" ' test_launch_ipfs_daemon_without_network test_expect_success "recursive pin fails without objects" ' test_must_fail ipfs pin add -r "$HASH_DIR1" 2>err_expected8 && grep "ipld: could not find" err_expected8 || test_fsh cat err_expected8 ' # Regression test for https://github.com/ipfs/go-ipfs/issues/4650 # This test requires the daemon. Otherwise, the pin changes are reverted when # the pin fails in the previous test. test_expect_success "failed recursive pin does not remove direct pin" ' test_pin_flag "$HASH_DIR1" direct true ' test_kill_ipfs_daemon test_expect_success "test add nopin file" ' echo "test nopin data" > test_nopin_data && NOPINHASH=$(ipfs add -q --pin=false test_nopin_data) && test_pin_flag "$NOPINHASH" direct false && test_pin_flag "$NOPINHASH" indirect false && test_pin_flag "$NOPINHASH" recursive false ' test_expect_success "test add nopin dir" ' mkdir nopin_dir1 && echo "some nopin text 1" >nopin_dir1/file1 && echo "some nopin text 2" >nopin_dir1/file2 && ipfs add -Q -r --pin=false nopin_dir1 >actual && echo "$HASH_NOPINDIR" >expected && test_cmp actual expected && test_pin_flag "$HASH_NOPINDIR" direct false && test_pin_flag "$HASH_NOPINDIR" indirect false && test_pin_flag "$HASH_NOPINDIR" recursive false && test_pin_flag "$HASH_NOPIN_FILE1" direct false && test_pin_flag "$HASH_NOPIN_FILE1" indirect false && test_pin_flag "$HASH_NOPIN_FILE1" recursive false && test_pin_flag "$HASH_NOPIN_FILE2" direct false && test_pin_flag "$HASH_NOPIN_FILE2" indirect false && test_pin_flag "$HASH_NOPIN_FILE2" recursive false ' FICTIONAL_HASH="QmXV4f9v8a56MxWKBhP3ETsz4EaafudU1cKfPaaJnenc48" test_launch_ipfs_daemon test_expect_success "test unpinning a hash that's not pinned" " test_expect_code 1 ipfs pin rm $FICTIONAL_HASH --timeout=2s test_expect_code 1 ipfs pin rm $FICTIONAL_HASH/a --timeout=2s test_expect_code 1 ipfs pin rm $FICTIONAL_HASH/a/b --timeout=2s " test_kill_ipfs_daemon # test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0082-repo-gc-auto.sh ================================================ #!/usr/bin/env bash # # MIT Licensed; see the LICENSE file in this repository. # test_description="Test ipfs repo auto gc" . lib/test-lib.sh skip_all="skipping auto repo gc tests until they can be fixed" test_done check_ipfs_storage() { ipfs config Datastore.StorageMax } test_init_ipfs test_expect_success "generate 2 600 kB files and 2 MB file using random-data" ' random-data -size=614400 -seed=41 >600k1 && random-data -size=614400 -seed=42 >600k2 && random-data -size=2097152 -seed=43 >2M ' test_expect_success "set ipfs gc watermark, storage max, and gc timeout" ' test_config_set Datastore.StorageMax "2MB" && test_config_set --json Datastore.StorageGCWatermark 60 && test_config_set Datastore.GCPeriod "20ms" ' test_launch_ipfs_daemon --enable-gc test_gc() { test_expect_success "adding data below watermark doesn't trigger auto gc" ' ipfs add 600k1 >/dev/null && disk_usage "$IPFS_PATH/blocks" >expected && go-sleep 40ms && disk_usage "$IPFS_PATH/blocks" >actual && test_cmp expected actual ' test_expect_success "adding data beyond watermark triggers auto gc" ' HASH=`ipfs add -q 600k2` && ipfs pin rm -r $HASH && go-sleep 40ms && DU=$(disk_usage "$IPFS_PATH/blocks") && if test $(uname -s) = "Darwin"; then test "$DU" -lt 1400 # 60% of 2MB else test "$DU" -lt 1000000 fi ' } #TODO: conditional GC test is disabled due to files size bug in ipfs add #test_expect_success "adding data beyond storageMax fails" ' # test_must_fail ipfs add 2M 2>add_fail_out #' #test_expect_success "ipfs add not enough space message looks good" ' # echo "Error: file size exceeds slack space allowed by storageMax. Maybe unpin some files?" >add_fail_exp && # test_cmp add_fail_exp add_fail_out #' test_expect_success "periodic auto gc stress test" ' for i in $(test_seq 1 20) do test_gc || return 1 done ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0084-repo-read-rehash.sh ================================================ #!/usr/bin/env bash # # Copyright (c) Jakub Sztandera # MIT Licensed; see the LICENSE file in this repository. # test_description="Test ipfs blockstore repo read check." . lib/test-lib.sh rm -rf "$IPF_PATH/*" test_init_ipfs H_BLOCK1=$(echo "Block 1" | ipfs add -q) H_BLOCK2=$(echo "Block 2" | ipfs add -q) BS_BLOCK1="XZ/CIQPDDQH5PDJTF4QSNMPFC45FQZH5MBSWCX2W254P7L7HGNHW5MQXZA.data" BS_BLOCK2="CK/CIQNYWBOKHY7TCY7FUOBXKVJ66YRMARDT3KC7PPY6UWWPZR4YA67CKQ.data" test_expect_success 'blocks are swapped' ' ipfs cat $H_BLOCK2 > noswap && cp -f "$IPFS_PATH/blocks/$BS_BLOCK1" "$IPFS_PATH/blocks/$BS_BLOCK2" && ipfs cat $H_BLOCK2 > swap && test_must_fail test_cmp noswap swap ' ipfs config --bool Datastore.HashOnRead true test_check_bad_blocks() { test_expect_success 'getting modified block fails' ' (test_must_fail ipfs cat $H_BLOCK2 2> err_msg) && grep "block in storage has different hash than requested" err_msg ' test_expect_success "block shows up in repo verify" ' test_expect_code 1 ipfs repo verify | cid-fmt --filter -b base32 "%M" > verify_out && H_BLOCK2_MH=`cid-fmt -b base32 "%M" $H_BLOCK2` && grep "$H_BLOCK2_MH" verify_out ' } test_check_bad_blocks test_expect_success "can add and cat a raw-leaf file" ' HASH=$(echo "stuff" | ipfs add -q --raw-leaves) && ipfs cat $HASH > /dev/null ' test_launch_ipfs_daemon test_check_bad_blocks test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0086-repo-verify.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2016 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # # NOTE: This is a legacy sharness test kept for compatibility. # New tests for 'ipfs repo verify' should be added to test/cli/repo_verify_test.go # test_description="Test ipfs repo fsck" . lib/test-lib.sh test_init_ipfs sort_rand() { case `uname` in Linux|FreeBSD) sort -R ;; Darwin) ruby -e 'puts STDIN.readlines.shuffle' ;; *) echo "unsupported system: $(uname)" esac } check_random_corruption() { # Exclude well-known blocks from corruption as they cause test flakiness: # - CIQL7TG2PB52XIZLLHDYIUFMHUQLMMZWBNBZSLDXFCPZ5VDNQQ2WDZQ.data: empty file block # - CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data: empty directory block (has special handling, served from memory even when corrupted on disk) to_break=$(find "$IPFS_PATH/blocks" -type f -name '*.data' | grep -v -E "CIQL7TG2PB52XIZLLHDYIUFMHUQLMMZWBNBZSLDXFCPZ5VDNQQ2WDZQ.data|CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data" | sort_rand | head -n 1) test_expect_success "back up file and overwrite it" ' cp "$to_break" backup_file && echo "this is super broken" > "$to_break" ' test_expect_success "repo verify detects failure" ' test_expect_code 1 ipfs repo verify ' test_expect_success "replace the object" ' cp backup_file "$to_break" ' test_expect_success "ipfs repo verify passes just fine now" ' ipfs repo verify ' } test_expect_success "create some files" ' random-files -depth=3 -dirs=4 -files=10 foobar > /dev/null ' test_expect_success "add them all" ' ipfs add -r -q foobar > /dev/null ' for i in `seq 20` do check_random_corruption done test_done ================================================ FILE: test/sharness/t0087-repo-robust-gc.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2016 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test robustness of garbage collector" . lib/test-lib.sh set -e to_raw_cid() { ipfs cid format -b b --mc raw -v 1 "$1" } test_gc_robust_part1() { test_expect_success "add a 1MB file with --raw-leaves" ' random-data -size=1048576 -seed=56 > afile && HASH1=`ipfs add --raw-leaves -q --cid-version 1 afile` && REFS=`ipfs refs -r $HASH1` && read LEAF1 LEAF2 LEAF3 LEAF4 < <(echo $REFS) ' test_expect_success "find data blocks for added file" ' HASH1MH=`cid-fmt -b base32 "%M" $HASH1` && LEAF1MH=`cid-fmt -b base32 "%M" $LEAF1` && LEAF2MH=`cid-fmt -b base32 "%M" $LEAF2` && HASH1FILE=`find .ipfs/blocks -type f | grep -i $HASH1MH` && LEAF1FILE=`find .ipfs/blocks -type f | grep -i $LEAF1MH` && LEAF2FILE=`find .ipfs/blocks -type f | grep -i $LEAF2MH` ' test_expect_success "remove a leaf node from the repo manually" ' rm "$LEAF1FILE" ' test_expect_success "check that the node is removed" ' test_must_fail ipfs cat $HASH1 ' test_expect_success "'ipfs repo gc' should still be fine" ' ipfs repo gc ' test_expect_success "corrupt the root node of 1MB file" ' test -e "$HASH1FILE" && dd if=/dev/zero of="$HASH1FILE" count=1 bs=100 conv=notrunc ' test_expect_success "'ipfs repo gc' should abort without removing anything" ' test_must_fail ipfs repo gc 2>&1 | tee gc_err && grep -q "could not retrieve links for $HASH1" gc_err && grep -q "aborted" gc_err ' test_expect_success "leaf nodes were not removed after gc" ' ipfs cat $LEAF3 > /dev/null && ipfs cat $LEAF4 > /dev/null ' test_expect_success "unpin the 1MB file" ' ipfs pin rm $HASH1 ' # make sure the permission problem is fixed on exit, otherwise cleanup # will fail trap "chmod 700 `dirname "$LEAF2FILE"` 2> /dev/null || true" 0 test_expect_success "create a permission problem" ' chmod 500 `dirname "$LEAF2FILE"` && test_must_fail ipfs block rm $LEAF2 2>&1 | tee block_rm_err && grep -q "permission denied" block_rm_err ' # repo gc outputs raw multihashes. We check HASH1 with block stat rather than # grepping the output since it's not a raw multihash test_expect_success "'ipfs repo gc' should still run and remove as much as possible" ' test_must_fail ipfs repo gc 2>&1 | tee repo_gc_out && grep -q "could not remove $LEAF2" repo_gc_out && grep -q "removed $(to_raw_cid $LEAF3)" repo_gc_out && grep -q "removed $(to_raw_cid $LEAF4)" repo_gc_out && test_must_fail ipfs block stat $HASH1 ' test_expect_success "fix the permission problem" ' chmod 700 `dirname "$LEAF2FILE"` ' test_expect_success "'ipfs repo gc' should be ok now" ' ipfs repo gc | tee repo_gc_out grep -q "removed $(to_raw_cid $LEAF2)" repo_gc_out ' } test_gc_robust_part2() { test_expect_success "add 1MB file normally (i.e., without raw leaves)" ' random-data -size=1048576 -seed=56 > afile && HASH2=`ipfs add -q afile` ' LEAF1=QmcNNR6JSCUhJ9nyoVQgBhABPgcgdsuYJgdSB1f2g6BF5c LEAF1FILE=.ipfs/blocks/RA/CIQNA5C3BLRUX3LZ7X6UTOV3KSHLARNXVDK3W5KUO6GVHNRP4SGLRAY.data LEAF2=QmPvtiBLgwuwF2wyf9VL8PaYgSt1XwGJ2Yu4AscRGEQvqR LEAF2FILE=.ipfs/blocks/RN/CIQBPIKEATBI7TIHVYRQJZAKEWF2H22PXW3A7LCEPB6MFFL7IA2CRNA.data test_expect_success "add some additional unpinned content" ' random-data -size=1000 -seed=3 > junk1 && random-data -size=1000 -seed=4 > junk2 && JUNK1=`ipfs add --pin=false -q junk1` && JUNK2=`ipfs add --pin=false -q junk2` ' test_expect_success "remove a leaf node from the repo manually" ' rm "$LEAF1FILE" ' test_expect_success "'ipfs repo gc' should abort" ' test_must_fail ipfs repo gc 2>&1 | tee repo_gc_out && grep -q "could not retrieve links for $LEAF1" repo_gc_out && grep -q "aborted" repo_gc_out ' test_expect_success "test that garbage collector really aborted" ' ipfs cat $JUNK1 > /dev/null && ipfs cat $JUNK2 > /dev/null ' test_expect_success "corrupt a key" ' test -e "$LEAF2FILE" && dd if=/dev/zero of="$LEAF2FILE" count=1 bs=100 conv=notrunc ' test_expect_success "'ipfs repo gc' should abort with two errors" ' test_must_fail ipfs repo gc 2>&1 | tee repo_gc_out && grep -q "could not retrieve links for $LEAF1" repo_gc_out && grep -q "could not retrieve links for $LEAF2" repo_gc_out && grep -q "aborted" repo_gc_out ' test_expect_success "'ipfs repo gc --stream-errors' should abort and report each error separately" ' test_must_fail ipfs repo gc --stream-errors 2>&1 | tee repo_gc_out && grep -q "Error: could not retrieve links for $LEAF1" repo_gc_out && grep -q "Error: could not retrieve links for $LEAF2" repo_gc_out && grep -q "Error: garbage collection aborted" repo_gc_out ' test_expect_success "unpin 1MB file" ' ipfs pin rm $HASH2 ' test_expect_success "'ipfs repo gc' should be fine now" ' ipfs repo gc | tee repo_gc_out && grep -q "removed $(to_raw_cid $HASH2)" repo_gc_out && grep -q "removed $(to_raw_cid $LEAF2)" repo_gc_out ' } test_init_ipfs test_gc_robust_part1 test_gc_robust_part2 test_launch_ipfs_daemon_without_network test_gc_robust_part1 test_gc_robust_part2 test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0088-repo-stat-symlink.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2017 John Reed # MIT Licensed; see the LICENSE file in this repository. # test_description="Test 'ipfs repo stat' where IPFS_PATH is a symbolic link" . lib/test-lib.sh test_expect_success "create symbolic link for IPFS_PATH" ' mkdir sym_link_target && ln -s sym_link_target .ipfs ' test_init_ipfs # ensure that the RepoSize is reasonable when checked via a symlink. test_expect_success "'ipfs repo stat' RepoSize is correct with sym link" ' reposize_symlink=$(ipfs repo stat | grep RepoSize | awk '\''{ print $2 }'\'') && symlink_size=$(file_size .ipfs) && test "${reposize_symlink}" -gt "${symlink_size}" ' test_done ================================================ FILE: test/sharness/t0090-get.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Matt Bell # MIT Licensed; see the LICENSE file in this repository. # test_description="Test get command" . lib/test-lib.sh test_init_ipfs test_ipfs_get_flag() { ext="$1"; shift tar_flag="$1"; shift flag="$@" test_expect_success "ipfs get $flag succeeds" ' ipfs get "$HASH" '"$flag"' >actual ' test_expect_success "ipfs get $flag output looks good" ' printf "%s\n" "Saving archive to $HASH$ext" >expected && test_cmp expected actual ' test_expect_success "ipfs get $flag archive output is valid" ' tar "$tar_flag" "$HASH$ext" && test_cmp "$HASH" data && rm "$HASH$ext" && rm "$HASH" ' } # we use a function so that we can run it both offline + online test_get_cmd() { test_expect_success "'ipfs get --help' succeeds" ' ipfs get --help >actual ' test_expect_success "'ipfs get --help' output looks good" ' egrep "ipfs get.*" actual >/dev/null || test_fsh cat actual ' test_expect_success "ipfs get succeeds" ' echo "Hello Worlds!" >data && HASH=`ipfs add -q data` && ipfs get "$HASH" >actual ' test_expect_success "ipfs get output looks good" ' printf "%s\n" "Saving file(s) to $HASH" >expected && test_cmp expected actual ' test_expect_success "ipfs get file output looks good" ' test_cmp "$HASH" data ' test_expect_success "ipfs get DOES NOT error when trying to overwrite a file" ' ipfs get "$HASH" >actual && rm "$HASH" ' test_expect_success "ipfs get works with raw leaves" ' HASH2=$(ipfs add --raw-leaves -q data) && ipfs get "$HASH2" >actual2 ' test_expect_success "ipfs get output looks good" ' printf "%s\n" "Saving file(s) to $HASH2" >expected2 && test_cmp expected2 actual2 ' test_expect_success "ipfs get file output looks good" ' test_cmp "$HASH2" data ' test_ipfs_get_flag ".tar" "-xf" -a test_ipfs_get_flag ".tar.gz" "-zxf" -a -C test_ipfs_get_flag ".tar.gz" "-zxf" -a -C -l 9 test_expect_success "ipfs get succeeds (directory)" ' mkdir -p dir && touch dir/a && mkdir -p dir/b && echo "Hello, Worlds!" >dir/b/c && HASH2=`ipfs add -r -Q dir` && ipfs get "$HASH2" >actual ' test_expect_success "ipfs get output looks good (directory)" ' printf "%s\n" "Saving file(s) to $HASH2" >expected && test_cmp expected actual ' test_expect_success "ipfs get output is valid (directory)" ' test_cmp dir/a "$HASH2"/a && test_cmp dir/b/c "$HASH2"/b/c && rm -r "$HASH2" ' # Test issue #4720: problems when path contains a trailing slash. test_expect_success "ipfs get with slash (directory)" ' ipfs get "$HASH2/" && test_cmp dir/a "$HASH2"/a && test_cmp dir/b/c "$HASH2"/b/c && rm -r "$HASH2" ' test_expect_success "ipfs get -a -C succeeds (directory)" ' ipfs get "$HASH2" -a -C >actual ' test_expect_success "ipfs get -a -C output looks good (directory)" ' printf "%s\n" "Saving archive to $HASH2.tar.gz" >expected && test_cmp expected actual ' test_expect_success "gzipped tar archive output is valid (directory)" ' tar -zxf "$HASH2".tar.gz && test_cmp dir/a "$HASH2"/a && test_cmp dir/b/c "$HASH2"/b/c && rm -r "$HASH2" ' test_expect_success "ipfs get ../.. should fail" ' test_must_fail ipfs get ../.. 2>actual && test_should_contain "Error: invalid path \"../..\"" actual ' test_expect_success "create small file" ' echo "foo" > small && ipfs add -q small > hash_small ' test_expect_success "get small file" ' ipfs get -o out_small $(cat hash_small) && test_cmp small out_small ' test_expect_success "create medium file" ' head -c 16000 > medium && ipfs add -q medium > hash_medium ' test_expect_success "get medium file" ' ipfs get -o out_medium $(cat hash_medium) && test_cmp medium out_medium ' } test_get_fail() { test_expect_success "create an object that has unresolvable links" ' cat <<-\EOF >bad_object && {"Data":{"/":{"bytes":"CAE"}},"Links":[{"Hash":{"/":"Qmd4mG6pDFDmDTn6p3hX1srP8qTbkyXKj5yjpEsiHDX3u8"},"Name":"bar","Tsize":56},{"Hash":{"/":"QmUTjwRnG28dSrFFVTYgbr6LiDLsBmRr2SaUSTGheK2YqG"},"Name":"baz","Tsize":24266},{"Hash":{"/":"QmZzaC6ydNXiR65W8VjGA73ET9MZ6VFAqUT1ngYMXcpihn"},"Name":"foo","Tsize":1897}]} EOF cat bad_object | ipfs dag put --store-codec dag-pb > put_out ' test_expect_success "output looks good" ' echo "bafybeifrjjol3gixedca6etdwccnvwfvhurc4wb3i5mnk2rvwvyfcgwxd4" > put_exp && test_cmp put_exp put_out ' test_expect_success "ipfs get fails" ' test_expect_code 1 ipfs get QmaGidyrnX8FMbWJoxp8HVwZ1uRKwCyxBJzABnR1S2FVUr ' } # should work offline test_get_cmd # only really works offline, will try and search network when online test_get_fail # should work online test_launch_ipfs_daemon test_get_cmd test_expect_success "empty request to get doesn't panic and returns error" ' curl -X POST "http://$API_ADDR/api/v0/get" > curl_out || true && grep "argument \"ipfs-path\" is required" curl_out ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0095-refs.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2018 Protocol Labs, Inc # MIT Licensed; see the LICENSE file in this repository. # test_description="Test 'ipfs refs' command" . lib/test-lib.sh test_init_ipfs test_launch_ipfs_daemon_without_network # This file performs tests with the following directory # structure. # # L0- _______ A_________ # / | \ \ # L1- B C D 1.txt # / \ | | # L2- D 1.txt B 2.txt # | / \ # L3- 2.txt D 1.txt # | # L4- 2.txt # # 'ipfs add -r A' output: # # added QmdytmR4wULMd3SLo6ePF4s3WcRHWcpnJZ7bHhoj3QB13v A/1.txt # added QmdytmR4wULMd3SLo6ePF4s3WcRHWcpnJZ7bHhoj3QB13v A/B/1.txt # added QmSFxnK675wQ9Kc1uqWKyJUaNxvSc2BP5DbXCD3x93oq61 A/B/D/2.txt # added QmdytmR4wULMd3SLo6ePF4s3WcRHWcpnJZ7bHhoj3QB13v A/C/B/1.txt # added QmSFxnK675wQ9Kc1uqWKyJUaNxvSc2BP5DbXCD3x93oq61 A/C/B/D/2.txt # added QmSFxnK675wQ9Kc1uqWKyJUaNxvSc2BP5DbXCD3x93oq61 A/D/2.txt # added QmSanP5DpxpqfDdS4yekHY1MqrVge47gtxQcp2e2yZ4UwS A/B/D # added QmNkQvpiyAEtbeLviC7kqfifYoK1GXPcsSxTpP1yS3ykLa A/B # added QmSanP5DpxpqfDdS4yekHY1MqrVge47gtxQcp2e2yZ4UwS A/C/B/D # added QmNkQvpiyAEtbeLviC7kqfifYoK1GXPcsSxTpP1yS3ykLa A/C/B # added QmXXazTjeNCKFnpW1D65vTKsTs8fbgkCWTv8Em4pdK2coH A/C # added QmSanP5DpxpqfDdS4yekHY1MqrVge47gtxQcp2e2yZ4UwS A/D # added QmU6xujRsYzcrkocuR3fhfnkZBB8eyUFFq4WKRGw2aS15h A # # 'ipfs refs -r QmU6xujRsYzcrkocuR3fhfnkZBB8eyUFFq4WKRGw2aS15h' sample output # that shows visit order in a stable go-ipfs version: # # QmdytmR4wULMd3SLo6ePF4s3WcRHWcpnJZ7bHhoj3QB13v - 1.txt # QmNkQvpiyAEtbeLviC7kqfifYoK1GXPcsSxTpP1yS3ykLa - B (A/B) # QmdytmR4wULMd3SLo6ePF4s3WcRHWcpnJZ7bHhoj3QB13v - 1.txt (A/B/1.txt) # QmSanP5DpxpqfDdS4yekHY1MqrVge47gtxQcp2e2yZ4UwS - D (A/B/D) # QmSFxnK675wQ9Kc1uqWKyJUaNxvSc2BP5DbXCD3x93oq61 - 2.txt (A/B/D/2.txt) # QmXXazTjeNCKFnpW1D65vTKsTs8fbgkCWTv8Em4pdK2coH - C (A/C) # QmNkQvpiyAEtbeLviC7kqfifYoK1GXPcsSxTpP1yS3ykLa - B (A/C/B) # QmdytmR4wULMd3SLo6ePF4s3WcRHWcpnJZ7bHhoj3QB13v - 1.txt (A/C/B/1.txt) # QmSanP5DpxpqfDdS4yekHY1MqrVge47gtxQcp2e2yZ4UwS - D (A/C/B/D) # QmSFxnK675wQ9Kc1uqWKyJUaNxvSc2BP5DbXCD3x93oq61 - 2.txt (A/C/B/D/2.txt) # QmSanP5DpxpqfDdS4yekHY1MqrVge47gtxQcp2e2yZ4UwS - D (A/D) # QmSFxnK675wQ9Kc1uqWKyJUaNxvSc2BP5DbXCD3x93oq61 - 2.txt (A/D/2.txt) refsroot=QmU6xujRsYzcrkocuR3fhfnkZBB8eyUFFq4WKRGw2aS15h test_expect_success "create and add folders for refs" ' mkdir -p A/B/D A/C/B/D A/D echo "1" > A/1.txt echo "1" > A/B/1.txt echo "1" > A/C/B/1.txt echo "2" > A/B/D/2.txt echo "2" > A/C/B/D/2.txt echo "2" > A/D/2.txt root=$(ipfs add -r -Q A) [[ "$root" == "$refsroot" ]] ' test_refs_output() { ARGS=$1 FILTER=$2 test_expect_success "ipfs refs $ARGS -r" ' cat < expected.txt QmdytmR4wULMd3SLo6ePF4s3WcRHWcpnJZ7bHhoj3QB13v QmNkQvpiyAEtbeLviC7kqfifYoK1GXPcsSxTpP1yS3ykLa QmdytmR4wULMd3SLo6ePF4s3WcRHWcpnJZ7bHhoj3QB13v QmSanP5DpxpqfDdS4yekHY1MqrVge47gtxQcp2e2yZ4UwS QmSFxnK675wQ9Kc1uqWKyJUaNxvSc2BP5DbXCD3x93oq61 QmXXazTjeNCKFnpW1D65vTKsTs8fbgkCWTv8Em4pdK2coH QmNkQvpiyAEtbeLviC7kqfifYoK1GXPcsSxTpP1yS3ykLa QmdytmR4wULMd3SLo6ePF4s3WcRHWcpnJZ7bHhoj3QB13v QmSanP5DpxpqfDdS4yekHY1MqrVge47gtxQcp2e2yZ4UwS QmSFxnK675wQ9Kc1uqWKyJUaNxvSc2BP5DbXCD3x93oq61 QmSanP5DpxpqfDdS4yekHY1MqrVge47gtxQcp2e2yZ4UwS QmSFxnK675wQ9Kc1uqWKyJUaNxvSc2BP5DbXCD3x93oq61 EOF ipfs refs $ARGS -r $refsroot > refsr.txt test_cmp expected.txt refsr.txt ' # Unique is like above but removing duplicates test_expect_success "ipfs refs $ARGS -r --unique" ' cat < expected.txt QmdytmR4wULMd3SLo6ePF4s3WcRHWcpnJZ7bHhoj3QB13v QmNkQvpiyAEtbeLviC7kqfifYoK1GXPcsSxTpP1yS3ykLa QmSanP5DpxpqfDdS4yekHY1MqrVge47gtxQcp2e2yZ4UwS QmSFxnK675wQ9Kc1uqWKyJUaNxvSc2BP5DbXCD3x93oq61 QmXXazTjeNCKFnpW1D65vTKsTs8fbgkCWTv8Em4pdK2coH EOF ipfs refs $ARGS -r --unique $refsroot > refsr.txt test_cmp expected.txt refsr.txt ' # First level is 1.txt, B, C, D test_expect_success "ipfs refs $ARGS" ' cat < expected.txt QmdytmR4wULMd3SLo6ePF4s3WcRHWcpnJZ7bHhoj3QB13v QmNkQvpiyAEtbeLviC7kqfifYoK1GXPcsSxTpP1yS3ykLa QmXXazTjeNCKFnpW1D65vTKsTs8fbgkCWTv8Em4pdK2coH QmSanP5DpxpqfDdS4yekHY1MqrVge47gtxQcp2e2yZ4UwS EOF ipfs refs $ARGS $refsroot > refs.txt test_cmp expected.txt refs.txt ' # max-depth=0 should return an empty list test_expect_success "ipfs refs $ARGS -r --max-depth=0" ' cat < expected.txt EOF ipfs refs $ARGS -r --max-depth=0 $refsroot > refs.txt test_cmp expected.txt refs.txt ' # max-depth=1 should be equivalent to running without -r test_expect_success "ipfs refs $ARGS -r --max-depth=1" ' ipfs refs $ARGS -r --max-depth=1 $refsroot > refsr.txt ipfs refs $ARGS $refsroot > refs.txt test_cmp refsr.txt refs.txt ' # We should see the depth limit engage at level 2 test_expect_success "ipfs refs $ARGS -r --max-depth=2" ' cat < expected.txt QmdytmR4wULMd3SLo6ePF4s3WcRHWcpnJZ7bHhoj3QB13v QmNkQvpiyAEtbeLviC7kqfifYoK1GXPcsSxTpP1yS3ykLa QmdytmR4wULMd3SLo6ePF4s3WcRHWcpnJZ7bHhoj3QB13v QmSanP5DpxpqfDdS4yekHY1MqrVge47gtxQcp2e2yZ4UwS QmXXazTjeNCKFnpW1D65vTKsTs8fbgkCWTv8Em4pdK2coH QmNkQvpiyAEtbeLviC7kqfifYoK1GXPcsSxTpP1yS3ykLa QmSanP5DpxpqfDdS4yekHY1MqrVge47gtxQcp2e2yZ4UwS QmSFxnK675wQ9Kc1uqWKyJUaNxvSc2BP5DbXCD3x93oq61 EOF ipfs refs $ARGS -r --max-depth=2 $refsroot > refsr.txt test_cmp refsr.txt expected.txt ' # Here branch pruning and re-exploration come into place # At first it should see D at level 2 and don't go deeper. # But then after doing C it will see D at level 1 and go deeper # so that it outputs the hash for 2.txt (-q61). # We also see that C/B is pruned as it's been shown before. # # Excerpt from diagram above: # # L0- _______ A_________ # / | \ \ # L1- B C D 1.txt # / \ | | # L2- D 1.txt B 2.txt test_expect_success "ipfs refs $ARGS -r --unique --max-depth=2" ' cat < expected.txt QmdytmR4wULMd3SLo6ePF4s3WcRHWcpnJZ7bHhoj3QB13v QmNkQvpiyAEtbeLviC7kqfifYoK1GXPcsSxTpP1yS3ykLa QmSanP5DpxpqfDdS4yekHY1MqrVge47gtxQcp2e2yZ4UwS QmXXazTjeNCKFnpW1D65vTKsTs8fbgkCWTv8Em4pdK2coH QmSFxnK675wQ9Kc1uqWKyJUaNxvSc2BP5DbXCD3x93oq61 EOF ipfs refs $ARGS -r --unique --max-depth=2 $refsroot > refsr.txt test_cmp refsr.txt expected.txt ' } test_refs_output '' 'cat' test_refs_output '--cid-base=base32' 'ipfs cid base32' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0101-iptb-name.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test ipfs repo operations" . lib/test-lib.sh num_nodes=4 test_expect_success "set up an iptb cluster" ' iptb testbed create -type localipfs -count $num_nodes -force -init ' startup_cluster $num_nodes test_expect_success "add an object on one node" ' echo "ipns is super fun" > file && HASH_FILE=$(ipfsi 1 add -q file) ' test_expect_success "publish that object as an ipns entry" ' ipfsi 1 name publish $HASH_FILE ' test_expect_success "add an entry on another node pointing to that one" ' NODE1_ID=$(iptb attr get 1 id) && ipfsi 2 name publish /ipns/$NODE1_ID ' test_expect_success "cat that entry on a third node" ' NODE2_ID=$(iptb attr get 2 id) && ipfsi 3 cat /ipns/$NODE2_ID > output ' test_expect_success "ensure output was the same" ' test_cmp file output ' test_expect_success "shut down iptb" ' iptb stop ' test_done ================================================ FILE: test/sharness/t0109-gateway-web-_redirects.sh ================================================ #!/usr/bin/env bash test_description="Test HTTP Gateway _redirects support" . lib/test-lib.sh test_init_ipfs test_launch_ipfs_daemon ## ============================================================================ ## Test _redirects file support ## ============================================================================ # Import test case # Run `ipfs cat /ipfs/$REDIRECTS_DIR_CID/_redirects` to see sample _redirects file test_expect_success "Add the _redirects file test directory" ' ipfs dag import --pin-roots ../t0109-gateway-web-_redirects-data/redirects.car ' CAR_ROOT_CID=QmQyqMY5vUBSbSxyitJqthgwZunCQjDVtNd8ggVCxzuPQ4 REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples | cut -d "/" -f3) REDIRECTS_DIR_HOSTNAME="${REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT" test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file" ' curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/redirect-one" > response && test_should_contain "301 Moved Permanently" response && test_should_contain "Location: /one.html" response ' test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/301-redirect-one redirects with 301, per _redirects file" ' curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/301-redirect-one" > response && test_should_contain "301 Moved Permanently" response && test_should_contain "Location: /one.html" response ' test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/302-redirect-two redirects with 302, per _redirects file" ' curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/302-redirect-two" > response && test_should_contain "302 Found" response && test_should_contain "Location: /two.html" response ' test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/200-index returns 200, per _redirects file" ' curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/200-index" > response && test_should_contain "my index" response && test_should_contain "200 OK" response ' test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/posts/:year/:month/:day/:title redirects with 301 and placeholders, per _redirects file" ' curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/posts/2022/01/01/hello-world" > response && test_should_contain "301 Moved Permanently" response && test_should_contain "Location: /articles/2022/01/01/hello-world" response ' test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/splat/one.html redirects with 301 and splat placeholder, per _redirects file" ' curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/splat/one.html" > response && test_should_contain "301 Moved Permanently" response && test_should_contain "Location: /redirected-splat/one.html" response ' # ensure custom 4xx works and has the same cache headers as regular /ipfs/ path CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/404.html | cut -d "/" -f3) test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/not-found/has-no-redirects-entry returns custom 404, per _redirects file" ' curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/not-found/has-no-redirects-entry" > response && test_should_contain "404 Not Found" response && test_should_contain "Cache-Control: public, max-age=29030400, immutable" response && test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response && test_should_contain "my 404" response ' CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/410.html | cut -d "/" -f3) test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry returns custom 410, per _redirects file" ' curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/gone/has-no-redirects-entry" > response && test_should_contain "410 Gone" response && test_should_contain "Cache-Control: public, max-age=29030400, immutable" response && test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response && test_should_contain "my 410" response ' CUSTOM_4XX_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples/451.html | cut -d "/" -f3) test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry returns custom 451, per _redirects file" ' curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/unavail/has-no-redirects-entry" > response && test_should_contain "451 Unavailable For Legal Reasons" response && test_should_contain "Cache-Control: public, max-age=29030400, immutable" response && test_should_contain "Etag: \"$CUSTOM_4XX_CID\"" response && test_should_contain "my 451" response ' test_expect_success "request for $REDIRECTS_DIR_HOSTNAME/catch-all returns 200, per _redirects file" ' curl -sD - --resolve $REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$REDIRECTS_DIR_HOSTNAME/catch-all" > response && test_should_contain "200 OK" response && test_should_contain "my index" response ' # This test ensures _redirects is supported only on Web Gateways that use Host header (DNSLink, Subdomain) test_expect_success "request for http://127.0.0.1:$GWAY_PORT/ipfs/$REDIRECTS_DIR_CID/301-redirect-one returns generic 404 (no custom 404 from _redirects since no origin isolation)" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$REDIRECTS_DIR_CID/301-redirect-one" > response && test_should_contain "404 Not Found" response && test_should_not_contain "my 404" response ' # With CRLF line terminator NEWLINE_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/newlines | cut -d "/" -f3) NEWLINE_REDIRECTS_DIR_HOSTNAME="${NEWLINE_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT" test_expect_success "newline: _redirects has CRLF line terminators" ' ipfs cat /ipfs/$NEWLINE_REDIRECTS_DIR_CID/_redirects | file - > response && test_should_contain "with CRLF line terminators" response ' test_expect_success "newline: request for $NEWLINE_REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file" ' curl -sD - --resolve $NEWLINE_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$NEWLINE_REDIRECTS_DIR_HOSTNAME/redirect-one" > response && test_should_contain "301 Moved Permanently" response && test_should_contain "Location: /one.html" response ' # Good codes GOOD_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/good-codes | cut -d "/" -f3) GOOD_REDIRECTS_DIR_HOSTNAME="${GOOD_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT" test_expect_success "good codes: request for $GOOD_REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file" ' curl -sD - --resolve $GOOD_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$GOOD_REDIRECTS_DIR_HOSTNAME/a301" > response && test_should_contain "301 Moved Permanently" response && test_should_contain "Location: /b301" response ' # Bad codes BAD_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/bad-codes | cut -d "/" -f3) BAD_REDIRECTS_DIR_HOSTNAME="${BAD_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT" # if accessing a path that doesn't exist, read _redirects and fail parsing, and return error test_expect_success "bad codes: request for $BAD_REDIRECTS_DIR_HOSTNAME/not-found returns error about bad code" ' curl -sD - --resolve $BAD_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$BAD_REDIRECTS_DIR_HOSTNAME/not-found" > response && test_should_contain "500" response && test_should_contain "status code 999 is not supported" response ' # if accessing a path that does exist, don't read _redirects and therefore don't fail parsing test_expect_success "bad codes: request for $BAD_REDIRECTS_DIR_HOSTNAME/found.html doesn't return error about bad code" ' curl -sD - --resolve $BAD_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$BAD_REDIRECTS_DIR_HOSTNAME/found.html" > response && test_should_contain "200" response && test_should_contain "my found" response && test_should_not_contain "unsupported redirect status" response ' # Invalid file, containing "hello" INVALID_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/invalid | cut -d "/" -f3) INVALID_REDIRECTS_DIR_HOSTNAME="${INVALID_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT" # if accessing a path that doesn't exist, read _redirects and fail parsing, and return error test_expect_success "invalid file: request for $INVALID_REDIRECTS_DIR_HOSTNAME/not-found returns error about invalid redirects file" ' curl -sD - --resolve $INVALID_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$INVALID_REDIRECTS_DIR_HOSTNAME/not-found" > response && test_should_contain "500" response && test_should_contain "could not parse _redirects:" response ' # Invalid file, containing forced redirect INVALID_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/forced | cut -d "/" -f3) INVALID_REDIRECTS_DIR_HOSTNAME="${INVALID_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT" # if accessing a path that doesn't exist, read _redirects and fail parsing, and return error test_expect_success "invalid file: request for $INVALID_REDIRECTS_DIR_HOSTNAME/not-found returns error about invalid redirects file" ' curl -sD - --resolve $INVALID_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$INVALID_REDIRECTS_DIR_HOSTNAME/not-found" > response && test_should_contain "500" response && test_should_contain "could not parse _redirects:" response && test_should_contain "forced redirects (or \"shadowing\") are not supported" response ' # if accessing a path that doesn't exist and _redirects file is too large, return error TOO_LARGE_REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/too-large | cut -d "/" -f3) TOO_LARGE_REDIRECTS_DIR_HOSTNAME="${TOO_LARGE_REDIRECTS_DIR_CID}.ipfs.localhost:$GWAY_PORT" test_expect_success "invalid file: request for $TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found returns error about too large redirects file" ' curl -sD - --resolve $TOO_LARGE_REDIRECTS_DIR_HOSTNAME:127.0.0.1 "http://$TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found" > response && test_should_contain "500" response && test_should_contain "could not parse _redirects:" response && test_should_contain "redirects file size cannot exceed" response ' test_kill_ipfs_daemon # disable wildcard DNSLink gateway # and enable it on specific DNSLink hostname ipfs config --json Gateway.NoDNSLink true && \ ipfs config --json Gateway.PublicGateways '{ "dnslink-enabled-on-fqdn.example.org": { "NoDNSLink": false, "UseSubdomains": false, "Paths": ["/ipfs"] }, "dnslink-disabled-on-fqdn.example.com": { "NoDNSLink": true, "UseSubdomains": false, "Paths": [] } }' || exit 1 # DNSLink test requires a daemon in online mode with precached /ipns/ mapping # REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples | cut -d "/" -f3) DNSLINK_FQDN="dnslink-enabled-on-fqdn.example.org" NO_DNSLINK_FQDN="dnslink-disabled-on-fqdn.example.com" export IPFS_NS_MAP="$DNSLINK_FQDN:/ipfs/$REDIRECTS_DIR_CID" # restart daemon to apply config changes test_launch_ipfs_daemon # make sure test setup is valid (fail if CoreAPI is unable to resolve) test_expect_success "spoofed DNSLink record resolves in cli" " ipfs resolve /ipns/$DNSLINK_FQDN > result && test_should_contain \"$REDIRECTS_DIR_CID\" result && ipfs cat /ipns/$DNSLINK_FQDN/_redirects > result && test_should_contain \"index.html\" result " test_expect_success "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file" ' curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response && test_should_contain "301 Moved Permanently" response && test_should_contain "Location: /one.html" response ' # ensure custom 404 works and has the same cache headers as regular /ipns/ paths test_expect_success "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file" ' curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/not-found/has-no-redirects-entry" > response && test_should_contain "404 Not Found" response && test_should_contain "Etag: \"Qmd9GD7Bauh6N2ZLfNnYS3b7QVAijbud83b8GE8LPMNBBP\"" response && test_should_not_contain "Cache-Control: public, max-age=29030400, immutable" response && test_should_not_contain "immutable" response && test_should_contain "Date: " response && test_should_contain "my 404" response ' test_expect_success "request for $NO_DNSLINK_FQDN/redirect-one does not redirect, since DNSLink is disabled" ' curl -sD - --resolve $NO_DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$NO_DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response && test_should_not_contain "one.html" response && test_should_not_contain "301 Moved Permanently" response && test_should_not_contain "Location:" response ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0112-gateway-cors.sh ================================================ #!/usr/bin/env bash test_description="Test CORS behavior on Gateway port" . lib/test-lib.sh test_init_ipfs # Default config test_expect_success "Default Gateway.HTTPHeaders is empty (implicit CORS values from boxo/gateway)" ' cat < expected {} EOF ipfs config --json Gateway.HTTPHeaders > actual && test_cmp expected actual ' test_launch_ipfs_daemon thash='bafkqabtimvwgy3yk' # hello # Gateway # HTTP GET Request test_expect_success "GET to Gateway succeeds" ' curl -svX GET -H "Origin: https://example.com" "http://127.0.0.1:$GWAY_PORT/ipfs/$thash" >/dev/null 2>curl_output && cat curl_output ' # GET Response from Gateway should contain CORS headers test_expect_success "GET response for Gateway resource looks good" ' test_should_contain "< Access-Control-Allow-Origin: \*" curl_output && test_should_contain "< Access-Control-Allow-Methods: GET" curl_output && test_should_contain "< Access-Control-Allow-Methods: HEAD" curl_output && test_should_contain "< Access-Control-Allow-Methods: OPTIONS" curl_output && test_should_contain "< Access-Control-Allow-Headers: Content-Type" curl_output && test_should_contain "< Access-Control-Allow-Headers: Range" curl_output && test_should_contain "< Access-Control-Allow-Headers: User-Agent" curl_output && test_should_contain "< Access-Control-Allow-Headers: X-Requested-With" curl_output && test_should_contain "< Access-Control-Expose-Headers: Content-Range" curl_output && test_should_contain "< Access-Control-Expose-Headers: Content-Length" curl_output && test_should_contain "< Access-Control-Expose-Headers: X-Chunked-Output" curl_output && test_should_contain "< Access-Control-Expose-Headers: X-Stream-Output" curl_output && test_should_contain "< Access-Control-Expose-Headers: X-Ipfs-Path" curl_output && test_should_contain "< Access-Control-Expose-Headers: X-Ipfs-Roots" curl_output ' # HTTP OPTIONS Request test_expect_success "OPTIONS to Gateway succeeds" ' curl -svX OPTIONS -H "Origin: https://example.com" "http://127.0.0.1:$GWAY_PORT/ipfs/$thash" 2>curl_output && cat curl_output ' # OPTION Response from Gateway should contain CORS headers test_expect_success "OPTIONS response for Gateway resource looks good" ' test_should_contain "< Access-Control-Allow-Origin: \*" curl_output && test_should_contain "< Access-Control-Allow-Methods: GET" curl_output && test_should_contain "< Access-Control-Allow-Methods: HEAD" curl_output && test_should_contain "< Access-Control-Allow-Methods: OPTIONS" curl_output && test_should_contain "< Access-Control-Allow-Headers: Content-Type" curl_output && test_should_contain "< Access-Control-Allow-Headers: Range" curl_output && test_should_contain "< Access-Control-Allow-Headers: User-Agent" curl_output && test_should_contain "< Access-Control-Allow-Headers: X-Requested-With" curl_output && test_should_contain "< Access-Control-Expose-Headers: Content-Range" curl_output && test_should_contain "< Access-Control-Expose-Headers: Content-Length" curl_output && test_should_contain "< Access-Control-Expose-Headers: X-Chunked-Output" curl_output && test_should_contain "< Access-Control-Expose-Headers: X-Stream-Output" curl_output && test_should_contain "< Access-Control-Expose-Headers: X-Ipfs-Path" curl_output && test_should_contain "< Access-Control-Expose-Headers: X-Ipfs-Roots" curl_output ' # HTTP OPTIONS Request on path → subdomain HTTP 301 redirect # (regression test for https://github.com/ipfs/kubo/issues/9983#issuecomment-1599673976) test_expect_success "OPTIONS to Gateway succeeds" ' curl -svX OPTIONS -H "Origin: https://example.com" "http://localhost:$GWAY_PORT/ipfs/$thash" 2>curl_output && cat curl_output ' # OPTION Response from Gateway should contain CORS headers test_expect_success "OPTIONS response for subdomain redirect looks good" ' test_should_contain "HTTP/1.1 301 Moved Permanently" curl_output && test_should_contain "Location" curl_output && test_should_contain "< Access-Control-Allow-Origin: \*" curl_output && test_should_contain "< Access-Control-Allow-Methods: GET" curl_output ' test_kill_ipfs_daemon # Test CORS safelisting of custom headers test_expect_success "Can configure gateway headers" ' ipfs config --json Gateway.HTTPHeaders.Access-Control-Allow-Headers "[\"X-Custom1\"]" && ipfs config --json Gateway.HTTPHeaders.Access-Control-Expose-Headers "[\"X-Custom2\"]" && ipfs config --json Gateway.HTTPHeaders.Access-Control-Allow-Origin "[\"localhost\"]" ' test_launch_ipfs_daemon test_expect_success "OPTIONS to Gateway without custom headers succeeds" ' curl -svX OPTIONS -H "Origin: https://example.com" "http://127.0.0.1:$GWAY_PORT/ipfs/$thash" 2>curl_output && cat curl_output ' # Range and Content-Range are safelisted by default, and keeping them makes better devexp # because it does not cause regressions in range requests made by JS test_expect_success "Access-Control-Allow-Headers extends the implicit list" ' test_should_contain "< Access-Control-Allow-Headers: Range" curl_output && test_should_contain "< Access-Control-Allow-Headers: X-Custom1" curl_output && test_should_contain "< Access-Control-Expose-Headers: Content-Range" curl_output && test_should_contain "< Access-Control-Expose-Headers: Content-Length" curl_output && test_should_contain "< Access-Control-Expose-Headers: X-Ipfs-Path" curl_output && test_should_contain "< Access-Control-Expose-Headers: X-Ipfs-Roots" curl_output && test_should_contain "< Access-Control-Expose-Headers: X-Custom2" curl_output ' test_expect_success "OPTIONS to Gateway with a custom header succeeds" ' curl -svX OPTIONS -H "Origin: https://example.com" -H "Access-Control-Request-Headers: X-Unexpected-Custom" "http://127.0.0.1:$GWAY_PORT/ipfs/$thash" 2>curl_output && cat curl_output ' test_expect_success "Access-Control-Allow-Headers extends the implicit list" ' test_should_not_contain "< Access-Control-Allow-Headers: X-Unexpected-Custom" curl_output && test_should_contain "< Access-Control-Allow-Headers: Range" curl_output && test_should_contain "< Access-Control-Allow-Headers: X-Custom1" curl_output && test_should_contain "< Access-Control-Expose-Headers: Content-Range" curl_output && test_should_contain "< Access-Control-Expose-Headers: X-Custom2" curl_output ' # Origin is sensitive security perimeter, and we assume override should remove # any implicit records test_expect_success "Access-Control-Allow-Origin replaces the implicit list" ' test_should_contain "< Access-Control-Allow-Origin: localhost" curl_output ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0114-gateway-subdomains/README.md ================================================ # Dataset description/sources - fixtures.car - raw CARv1 - QmUKd....ipns-record - ipns record, encoded with protocol buffer - 12D3K....ipns-record - ipns record, encoded with protocol buffer Generated with: ```sh # using ipfs version 0.21.0-dev (03a98280e3e642774776cd3d0435ab53e5dfa867) # CIDv0to1 is necessary because raw-leaves are enabled by default during # "ipfs add" with CIDv1 and disabled with CIDv0 CID_VAL="hello" CIDv1=$(echo $CID_VAL | ipfs add --cid-version 1 -Q) CIDv0=$(echo $CID_VAL | ipfs add --cid-version 0 -Q) CIDv0to1=$(echo "$CIDv0" | ipfs cid base32) # sha512 will be over 63char limit, even when represented in Base36 CIDv1_TOO_LONG=$(echo $CID_VAL | ipfs add --cid-version 1 --hash sha2-512 -Q) echo CID_VAL=${CID_VAL} echo CIDv1=${CIDv1} echo CIDv0=${CIDv0} echo CIDv0to1=${CIDv0to1} echo CIDv1_TOO_LONG=${CIDv1_TOO_LONG} # Directory tree crafted to test for edge cases like "/ipfs/ipfs/ipns/bar" mkdir -p testdirlisting/ipfs/ipns && echo "hello" > testdirlisting/hello && echo "text-file-content" > testdirlisting/ipfs/ipns/bar && mkdir -p testdirlisting/api && mkdir -p testdirlisting/ipfs && echo "I am a txt file" > testdirlisting/api/file.txt && echo "I am a txt file" > testdirlisting/ipfs/file.txt && DIR_CID=$(ipfs add -Qr --cid-version 1 testdirlisting) echo DIR_CID=${DIR_CID} # ./testdirlisting ipfs files mkdir /t0114/ ipfs files cp /ipfs/${CIDv1} /t0114/ ipfs files cp /ipfs/${CIDv0} /t0114/ ipfs files cp /ipfs/${CIDv0to1} /t0114/ ipfs files cp /ipfs/${DIR_CID} /t0114/ ipfs files cp /ipfs/${CIDv1_TOO_LONG} /t0114/ ROOT=`ipfs files stat /t0114/ --hash` ipfs dag export ${ROOT} > ./fixtures.car # Then the keys KEY_NAME=test_key_rsa_$RANDOM RSA_KEY=$(ipfs key gen --ipns-base=b58mh --type=rsa --size=2048 ${KEY_NAME} | head -n1 | tr -d "\n") RSA_IPNS_IDv0=$(echo "$RSA_KEY" | ipfs cid format -v 0) RSA_IPNS_IDv1=$(echo "$RSA_KEY" | ipfs cid format -v 1 --mc libp2p-key -b base36) RSA_IPNS_IDv1_DAGPB=$(echo "$RSA_IPNS_IDv0" | ipfs cid format -v 1 -b base36) # publish a record valid for a 100 years ipfs name publish --key ${KEY_NAME} --allow-offline -Q --ttl=876600h --lifetime=876600h "/ipfs/$CIDv1" ipfs routing get /ipns/${RSA_KEY} > ${RSA_KEY}.ipns-record echo RSA_KEY=${RSA_KEY} echo RSA_IPNS_IDv0=${RSA_IPNS_IDv0} echo RSA_IPNS_IDv1=${RSA_IPNS_IDv1} echo RSA_IPNS_IDv1_DAGPB=${RSA_IPNS_IDv1_DAGPB} KEY_NAME=test_key_ed25519_$RANDOM ED25519_KEY=$(ipfs key gen --ipns-base=b58mh --type=ed25519 ${KEY_NAME} | head -n1 | tr -d "\n") ED25519_IPNS_IDv0=$ED25519_KEY ED25519_IPNS_IDv1=$(ipfs key list -l --ipns-base=base36 | grep ${KEY_NAME} | cut -d " " -f1 | tr -d "\n") ED25519_IPNS_IDv1_DAGPB=$(echo "$ED25519_IPNS_IDv1" | ipfs cid format -v 1 -b base36 --mc dag-pb) # ed25519 fits under 63 char limit when represented in base36 IPNS_ED25519_B58MH=$(ipfs key list -l --ipns-base b58mh | grep $KEY_NAME | cut -d" " -f1 | tr -d "\n") IPNS_ED25519_B36CID=$(ipfs key list -l --ipns-base base36 | grep $KEY_NAME | cut -d" " -f1 | tr -d "\n") # publish a record valid for a 100 years ipfs name publish --key ${KEY_NAME} --allow-offline -Q --ttl=876600h --lifetime=876600h "/ipfs/$CIDv1" ipfs routing get /ipns/${ED25519_KEY} > ${ED25519_KEY}.ipns-record echo ED25519_KEY=${ED25519_KEY} echo ED25519_IPNS_IDv0=${ED25519_IPNS_IDv0} echo ED25519_IPNS_IDv1=${ED25519_IPNS_IDv1} echo ED25519_IPNS_IDv1_DAGPB=${ED25519_IPNS_IDv1_DAGPB} echo IPNS_ED25519_B58MH=${IPNS_ED25519_B58MH} echo IPNS_ED25519_B36CID=${IPNS_ED25519_B36CID} # CID_VAL=hello # CIDv1=bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am # CIDv0=QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN # CIDv0to1=bafybeiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4 # CIDv1_TOO_LONG=bafkrgqhhyivzstcz3hhswshfjgy6ertgmnqeleynhwt4dlfsthi4hn7zgh4uvlsb5xncykzapi3ocd4lzogukir6ksdy6wzrnz6ohnv4aglcs # DIR_CID=bafybeiht6dtwk3les7vqm6ibpvz6qpohidvlshsfyr7l5mpysdw2vmbbhe # ./testdirlisting # RSA_KEY=QmVujd5Vb7moysJj8itnGufN7MEtPRCNHkKpNuA4onsRa3 # RSA_IPNS_IDv0=QmVujd5Vb7moysJj8itnGufN7MEtPRCNHkKpNuA4onsRa3 # RSA_IPNS_IDv1=k2k4r8m7xvggw5pxxk3abrkwyer625hg01hfyggrai7lk1m63fuihi7w # RSA_IPNS_IDv1_DAGPB=k2jmtxu61bnhrtj301lw7zizknztocdbeqhxgv76l2q9t36fn9jbzipo # ED25519_KEY=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d # ED25519_IPNS_IDv0=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d # ED25519_IPNS_IDv1=k51qzi5uqu5dk3v4rmjber23h16xnr23bsggmqqil9z2gduiis5se8dht36dam # ED25519_IPNS_IDv1_DAGPB=k50rm9yjlt0jey4fqg6wafvqprktgbkpgkqdg27tpqje6iimzxewnhvtin9hhq # IPNS_ED25519_B58MH=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d # IPNS_ED25519_B36CID=k51qzi5uqu5dk3v4rmjber23h16xnr23bsggmqqil9z2gduiis5se8dht36dam ``` ================================================ FILE: test/sharness/t0114-gateway-subdomains.sh ================================================ #!/usr/bin/env bash # # Copyright (c) Protocol Labs test_description="Test subdomain support on the HTTP gateway" . lib/test-lib.sh ## ============================================================================ ## Helpers specific to subdomain tests ## ============================================================================ # Helper that tests gateway response over direct HTTP # and in all supported HTTP proxy modes test_localhost_gateway_response_should_contain() { local label="$1" local expected="$3" # explicit "Host: $hostname" header to match browser behavior # and also make tests independent from DNS local host=$(echo $2 | cut -d'/' -f3 | cut -d':' -f1) local hostname=$(echo $2 | cut -d'/' -f3 | cut -d':' -f1,2) # Proxy is the same as HTTP Gateway, we use raw IP and port to be sure local proxy="http://127.0.0.1:$GWAY_PORT" # Create a raw URL version with IP to ensure hostname from Host header is used # (removes false-positives, Host header is used for passing hostname already) local url="$2" local rawurl=$(echo "$url" | sed "s/$hostname/127.0.0.1:$GWAY_PORT/") #echo "hostname: $hostname" #echo "url before: $url" #echo "url after: $rawurl" # regular HTTP request # (hostname in Host header, raw IP in URL) test_expect_success "$label (direct HTTP)" " curl -H \"Host: $hostname\" -sD - \"$rawurl\" > response && test_should_contain \"$expected\" response " # HTTP proxy # (hostname is passed via URL) # Note: proxy client should not care, but curl does DNS lookup # for some reason anyway, so we pass static DNS mapping test_expect_success "$label (HTTP proxy)" " curl -x $proxy --resolve $hostname:127.0.0.1 -sD - \"$url\" > response && test_should_contain \"$expected\" response " # HTTP proxy 1.0 # (repeating proxy test with older spec, just to be sure) test_expect_success "$label (HTTP proxy 1.0)" " curl --proxy1.0 $proxy --resolve $hostname:127.0.0.1 -sD - \"$url\" > response && test_should_contain \"$expected\" response " # HTTP proxy tunneling (CONNECT) # https://tools.ietf.org/html/rfc7231#section-4.3.6 # In HTTP/1.x, the pseudo-method CONNECT # can be used to convert an HTTP connection into a tunnel to a remote host test_expect_success "$label (HTTP proxy tunneling)" " curl --proxytunnel -x $proxy -H \"Host: $hostname\" -sD - \"$rawurl\" > response && test_should_contain \"$expected\" response " } # Helper that checks gateway response for specific hostname in Host header test_hostname_gateway_response_should_contain() { local label="$1" local hostname="$2" local url="$3" local rawurl=$(echo "$url" | sed "s/$hostname/127.0.0.1:$GWAY_PORT/") local expected="$4" test_expect_success "$label" " curl -H \"Host: $hostname\" -sD - \"$rawurl\" > response && test_should_contain \"$expected\" response " } ## ============================================================================ ## Start IPFS Node and prepare test CIDs ## ============================================================================ test_expect_success "ipfs init" ' export IPFS_PATH="$(pwd)/.ipfs" && ipfs init --profile=test > /dev/null ' test_launch_ipfs_daemon_without_network # Import test case # See the static fixtures in ./t0114-gateway-subdomains/ CID_VAL=hello CIDv1=bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am CIDv0=QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN CIDv0to1=bafybeiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4 CIDv1_TOO_LONG=bafkrgqhhyivzstcz3hhswshfjgy6ertgmnqeleynhwt4dlfsthi4hn7zgh4uvlsb5xncykzapi3ocd4lzogukir6ksdy6wzrnz6ohnv4aglcs DIR_CID=bafybeiht6dtwk3les7vqm6ibpvz6qpohidvlshsfyr7l5mpysdw2vmbbhe RSA_KEY=QmVujd5Vb7moysJj8itnGufN7MEtPRCNHkKpNuA4onsRa3 RSA_IPNS_IDv0=QmVujd5Vb7moysJj8itnGufN7MEtPRCNHkKpNuA4onsRa3 RSA_IPNS_IDv1=k2k4r8m7xvggw5pxxk3abrkwyer625hg01hfyggrai7lk1m63fuihi7w RSA_IPNS_IDv1_DAGPB=k2jmtxu61bnhrtj301lw7zizknztocdbeqhxgv76l2q9t36fn9jbzipo ED25519_KEY=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d ED25519_IPNS_IDv0=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d ED25519_IPNS_IDv1=k51qzi5uqu5dk3v4rmjber23h16xnr23bsggmqqil9z2gduiis5se8dht36dam ED25519_IPNS_IDv1_DAGPB=k50rm9yjlt0jey4fqg6wafvqprktgbkpgkqdg27tpqje6iimzxewnhvtin9hhq IPNS_ED25519_B58MH=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d IPNS_ED25519_B36CID=k51qzi5uqu5dk3v4rmjber23h16xnr23bsggmqqil9z2gduiis5se8dht36dam test_expect_success "Add the test fixtures" ' ipfs dag import --pin-roots ../t0114-gateway-subdomains/fixtures.car && ipfs routing put --allow-offline /ipns/${RSA_KEY} ../t0114-gateway-subdomains/${RSA_KEY}.ipns-record && ipfs routing put --allow-offline /ipns/${ED25519_KEY} ../t0114-gateway-subdomains/${ED25519_KEY}.ipns-record ' # ensure we start with empty Gateway.PublicGateways test_expect_success 'start daemon with empty config for Gateway.PublicGateways' ' test_kill_ipfs_daemon && ipfs config --json Gateway.PublicGateways "{}" && test_launch_ipfs_daemon_without_network ' ## ============================================================================ ## Test path-based requests to a local gateway with default config ## (forced redirects to http://*.localhost) ## ============================================================================ # /ipfs/ # IP remains old school path-based gateway test_localhost_gateway_response_should_contain \ "request for 127.0.0.1/ipfs/{CID} stays on path" \ "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ "$CID_VAL" # 'localhost' hostname is used for subdomains, and should not return # payload directly, but redirect to URL with proper origin isolation test_localhost_gateway_response_should_contain \ "request for localhost/ipfs/{CIDv1} returns HTTP 301 Moved Permanently" \ "http://localhost:$GWAY_PORT/ipfs/$CIDv1" \ "301 Moved Permanently" test_localhost_gateway_response_should_contain \ "request for localhost/ipfs/{CIDv1} returns Location HTTP header for subdomain redirect in browsers" \ "http://localhost:$GWAY_PORT/ipfs/$CIDv1" \ "Location: http://$CIDv1.ipfs.localhost:$GWAY_PORT/" test_localhost_gateway_response_should_contain \ "request for localhost/ipfs/{DIR_CID} returns HTTP 301 Moved Permanently" \ "http://localhost:$GWAY_PORT/ipfs/$DIR_CID" \ "301 Moved Permanently" test_localhost_gateway_response_should_contain \ "request for localhost/ipfs/{DIR_CID} returns Location HTTP header for subdomain redirect in browsers" \ "http://localhost:$GWAY_PORT/ipfs/$DIR_CID/" \ "Location: http://$DIR_CID.ipfs.localhost:$GWAY_PORT/" # Kubo specific end-to-end test # (independent of gateway-conformance) # We return human-readable body with HTTP 301 so existing cli scripts that use path-based # gateway are informed to enable following HTTP redirects test_localhost_gateway_response_should_contain \ "request for localhost/ipfs/{CIDv1} includes human-readable link and redirect info in HTTP 301 body" \ "http://localhost:$GWAY_PORT/ipfs/$CIDv1" \ ">Moved Permanently" # end Kubo specific end-to-end test test_localhost_gateway_response_should_contain \ "request for localhost/ipfs/{CIDv0} redirects to CIDv1 representation in subdomain" \ "http://localhost:$GWAY_PORT/ipfs/$CIDv0" \ "Location: http://${CIDv0to1}.ipfs.localhost:$GWAY_PORT/" # /ipns/ test_localhost_gateway_response_should_contain \ "request for localhost/ipns/{CIDv0} redirects to CIDv1 with libp2p-key multicodec in subdomain" \ "http://localhost:$GWAY_PORT/ipns/$RSA_IPNS_IDv0" \ "Location: http://${RSA_IPNS_IDv1}.ipns.localhost:$GWAY_PORT/" test_localhost_gateway_response_should_contain \ "request for localhost/ipns/{CIDv0} redirects to CIDv1 with libp2p-key multicodec in subdomain" \ "http://localhost:$GWAY_PORT/ipns/$ED25519_IPNS_IDv0" \ "Location: http://${ED25519_IPNS_IDv1}.ipns.localhost:$GWAY_PORT/" # /ipns/ # Kubo specific end-to-end test # (independent of gateway-conformance) test_localhost_gateway_response_should_contain \ "request for localhost/ipns/{fqdn} redirects to DNSLink in subdomain" \ "http://localhost:$GWAY_PORT/ipns/en.wikipedia-on-ipfs.org/wiki" \ "Location: http://en.wikipedia-on-ipfs.org.ipns.localhost:$GWAY_PORT/wiki" # end Kubo specific end-to-end test ## ============================================================================ ## Test subdomain-based requests to a local gateway with default config ## (origin per content root at http://*.localhost) ## ============================================================================ # {CID}.ipfs.localhost test_localhost_gateway_response_should_contain \ "request for {CID}.ipfs.localhost should return expected payload" \ "http://${CIDv1}.ipfs.localhost:$GWAY_PORT" \ "$CID_VAL" # ensure /ipfs/ namespace is not mounted on subdomain test_localhost_gateway_response_should_contain \ "request for {CID}.ipfs.localhost/ipfs/{CID} should return HTTP 404" \ "http://${CIDv1}.ipfs.localhost:$GWAY_PORT/ipfs/$CIDv1" \ "404 Not Found" # ensure requests to /ipfs/* are not blocked, if content root has such subdirectory test_localhost_gateway_response_should_contain \ "request for {CID}.ipfs.localhost/ipfs/file.txt should return data from a file in CID content root" \ "http://${DIR_CID}.ipfs.localhost:$GWAY_PORT/ipfs/file.txt" \ "I am a txt file" # Kubo specific end-to-end test # (independent of gateway-conformance) # This tests link to parent specific to boxo + relative pathing end-to-end tests specific to Kubo. # {CID}.ipfs.localhost/sub/dir (Directory Listing) DIR_HOSTNAME="${DIR_CID}.ipfs.localhost:$GWAY_PORT" test_expect_success "valid file and subdirectory paths in directory listing at {cid}.ipfs.localhost" ' curl -s --resolve $DIR_HOSTNAME:127.0.0.1 "http://$DIR_HOSTNAME" > list_response && test_should_contain "hello" list_response && test_should_contain "ipfs" list_response ' test_expect_success "valid parent directory path in directory listing at {cid}.ipfs.localhost/sub/dir" ' curl -s --resolve $DIR_HOSTNAME:127.0.0.1 "http://$DIR_HOSTNAME/ipfs/ipns/" > list_response && test_should_contain ".." list_response && test_should_contain "bar" list_response ' test_expect_success "request for deep path resource at {cid}.ipfs.localhost/sub/dir/file" ' curl -s --resolve $DIR_HOSTNAME:127.0.0.1 "http://$DIR_HOSTNAME/ipfs/ipns/bar" > list_response && test_should_contain "text-file-content" list_response ' # end Kubo specific end-to-end test # *.ipns.localhost # .ipns.localhost test_localhost_gateway_response_should_contain \ "request for {CIDv1-libp2p-key}.ipns.localhost returns expected payload" \ "http://${RSA_IPNS_IDv1}.ipns.localhost:$GWAY_PORT" \ "$CID_VAL" test_localhost_gateway_response_should_contain \ "request for {CIDv1-libp2p-key}.ipns.localhost returns expected payload" \ "http://${ED25519_IPNS_IDv1}.ipns.localhost:$GWAY_PORT" \ "$CID_VAL" test_localhost_gateway_response_should_contain \ "localhost request for {CIDv1-dag-pb}.ipns.localhost redirects to CID with libp2p-key multicodec" \ "http://${RSA_IPNS_IDv1_DAGPB}.ipns.localhost:$GWAY_PORT" \ "Location: http://${RSA_IPNS_IDv1}.ipns.localhost:$GWAY_PORT/" test_localhost_gateway_response_should_contain \ "localhost request for {CIDv1-dag-pb}.ipns.localhost redirects to CID with libp2p-key multicodec" \ "http://${ED25519_IPNS_IDv1_DAGPB}.ipns.localhost:$GWAY_PORT" \ "Location: http://${ED25519_IPNS_IDv1}.ipns.localhost:$GWAY_PORT/" # .ipns.localhost # DNSLink test requires a daemon in online mode with precached /ipns/ mapping test_kill_ipfs_daemon DNSLINK_FQDN="dnslink-test.example.com" export IPFS_NS_MAP="$DNSLINK_FQDN:/ipfs/$CIDv1" test_launch_ipfs_daemon test_localhost_gateway_response_should_contain \ "request for {dnslink}.ipns.localhost returns expected payload" \ "http://$DNSLINK_FQDN.ipns.localhost:$GWAY_PORT" \ "$CID_VAL" ## ============================================================================ ## Test DNSLink inlining on HTTP gateways ## ============================================================================ # set explicit subdomain gateway config for the hostname ipfs config --json Gateway.PublicGateways '{ "localhost": { "UseSubdomains": true, "InlineDNSLink": true, "Paths": ["/ipfs", "/ipns", "/api"] }, "example.com": { "UseSubdomains": true, "InlineDNSLink": true, "Paths": ["/ipfs", "/ipns", "/api"] } }' || exit 1 # restart daemon to apply config changes test_kill_ipfs_daemon test_launch_ipfs_daemon_without_network test_localhost_gateway_response_should_contain \ "request for localhost/ipns/{fqdn} redirects to DNSLink in subdomain with DNS inlining" \ "http://localhost:$GWAY_PORT/ipns/en.wikipedia-on-ipfs.org/wiki" \ "Location: http://en-wikipedia--on--ipfs-org.ipns.localhost:$GWAY_PORT/wiki" test_hostname_gateway_response_should_contain \ "request for example.com/ipns/{fqdn} redirects to DNSLink in subdomain with DNS inlining" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipns/en.wikipedia-on-ipfs.org/wiki" \ "Location: http://en-wikipedia--on--ipfs-org.ipns.example.com/wiki" ## ============================================================================ ## Test subdomain-based requests with a custom hostname config ## (origin per content root at http://*.example.com) ## ============================================================================ # set explicit subdomain gateway config for the hostname ipfs config --json Gateway.PublicGateways '{ "example.com": { "UseSubdomains": true, "Paths": ["/ipfs", "/ipns", "/api"] } }' || exit 1 # restart daemon to apply config changes test_kill_ipfs_daemon test_launch_ipfs_daemon_without_network # example.com/ip(f|n)s/* # ============================================================================= # path requests to the root hostname should redirect # to a subdomain URL with proper origin isolation test_hostname_gateway_response_should_contain \ "request for example.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.example.com" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ "Location: http://$CIDv1.ipfs.example.com/" # error message should include original CID # (and it should be case-sensitive, as we can't assume everyone uses base32) test_hostname_gateway_response_should_contain \ "request for example.com/ipfs/{InvalidCID} produces useful error before redirect" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipfs/QmInvalidCID" \ 'invalid path \"/ipfs/QmInvalidCID\"' test_hostname_gateway_response_should_contain \ "request for example.com/ipfs/{CIDv0} produces redirect to {CIDv1}.ipfs.example.com" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv0" \ "Location: http://${CIDv0to1}.ipfs.example.com/" # Support X-Forwarded-Proto test_expect_success "request for http://example.com/ipfs/{CID} with X-Forwarded-Proto: https produces redirect to HTTPS URL" " curl -H \"X-Forwarded-Proto: https\" -H \"Host: example.com\" -sD - \"http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1\" > response && test_should_contain \"Location: https://$CIDv1.ipfs.example.com/\" response " # Support ipfs:// in https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler test_hostname_gateway_response_should_contain \ "request for example.com/ipfs/?uri=ipfs%3A%2F%2F.. produces redirect to /ipfs/.. content path" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipfs/?uri=ipfs%3A%2F%2FQmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco%2Fwiki%2FDiego_Maradona.html" \ "Location: /ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco/wiki/Diego_Maradona.html" # example.com/ipns/ test_hostname_gateway_response_should_contain \ "request for example.com/ipns/{CIDv0} redirects to CIDv1 with libp2p-key multicodec in subdomain" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipns/$RSA_IPNS_IDv0" \ "Location: http://${RSA_IPNS_IDv1}.ipns.example.com/" test_hostname_gateway_response_should_contain \ "request for example.com/ipns/{CIDv0} redirects to CIDv1 with libp2p-key multicodec in subdomain" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipns/$ED25519_IPNS_IDv0" \ "Location: http://${ED25519_IPNS_IDv1}.ipns.example.com/" # example.com/ipns/ test_hostname_gateway_response_should_contain \ "request for example.com/ipns/{fqdn} redirects to DNSLink in subdomain" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipns/en.wikipedia-on-ipfs.org/wiki" \ "Location: http://en.wikipedia-on-ipfs.org.ipns.example.com/wiki" # DNSLink on Public gateway with a single-level wildcard TLS cert # "Option C" from https://github.com/ipfs/in-web-browsers/issues/169 test_expect_success \ "request for example.com/ipns/{fqdn} with X-Forwarded-Proto redirects to TLS-safe label in subdomain" " curl -H \"Host: example.com\" -H \"X-Forwarded-Proto: https\" -sD - \"http://127.0.0.1:$GWAY_PORT/ipns/en.wikipedia-on-ipfs.org/wiki\" > response && test_should_contain \"Location: https://en-wikipedia--on--ipfs-org.ipns.example.com/wiki\" response " # Support ipns:// in https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler test_hostname_gateway_response_should_contain \ "request for example.com/ipns/?uri=ipns%3A%2F%2F.. produces redirect to /ipns/.. content path" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipns/?uri=ipns%3A%2F%2Fen.wikipedia-on-ipfs.org" \ "Location: /ipns/en.wikipedia-on-ipfs.org" # *.ipfs.example.com: subdomain requests made with custom FQDN in Host header test_hostname_gateway_response_should_contain \ "request for {CID}.ipfs.example.com should return expected payload" \ "${CIDv1}.ipfs.example.com" \ "http://127.0.0.1:$GWAY_PORT/" \ "$CID_VAL" test_hostname_gateway_response_should_contain \ "request for {CID}.ipfs.example.com/ipfs/{CID} should return HTTP 404" \ "${CIDv1}.ipfs.example.com" \ "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ "404 Not Found" # Kubo specific end-to-end test # (independent of gateway-conformance) # HTML specific to Boxo/Kubo, and relative pathing specific to code in Kubo # {CID}.ipfs.example.com/sub/dir (Directory Listing) DIR_FQDN="${DIR_CID}.ipfs.example.com" test_expect_success "valid file and directory paths in directory listing at {cid}.ipfs.example.com" ' curl -s -H "Host: $DIR_FQDN" http://127.0.0.1:$GWAY_PORT > list_response && test_should_contain "hello" list_response && test_should_contain "ipfs" list_response ' test_expect_success "valid parent directory path in directory listing at {cid}.ipfs.example.com/sub/dir" ' curl -s -H "Host: $DIR_FQDN" http://127.0.0.1:$GWAY_PORT/ipfs/ipns/ > list_response && test_should_contain ".." list_response && test_should_contain "bar" list_response ' # Note 1: we test for sneaky subdir names {cid}.ipfs.example.com/ipfs/ipns/ :^) # Note 2: example.com/ipfs/.. present in HTML will be redirected to subdomain, so this is expected behavior test_expect_success "valid breadcrumb links in the header of directory listing at {cid}.ipfs.example.com/sub/dir" ' curl -s -H "Host: $DIR_FQDN" http://127.0.0.1:$GWAY_PORT/ipfs/ipns/ > list_response && test_should_contain "Index of" list_response && test_should_contain "/ipfs/${DIR_CID}/ipfs/ipns" list_response ' # end Kubo specific end-to-end test test_expect_success "request for deep path resource {cid}.ipfs.example.com/sub/dir/file" ' curl -s -H "Host: $DIR_FQDN" http://127.0.0.1:$GWAY_PORT/ipfs/ipns/bar > list_response && test_should_contain "text-file-content" list_response ' # *.ipns.example.com # ============================================================================ # .ipns.example.com test_hostname_gateway_response_should_contain \ "request for {CIDv1-libp2p-key}.ipns.example.com returns expected payload" \ "${RSA_IPNS_IDv1}.ipns.example.com" \ "http://127.0.0.1:$GWAY_PORT" \ "$CID_VAL" test_hostname_gateway_response_should_contain \ "request for {CIDv1-libp2p-key}.ipns.example.com returns expected payload" \ "${ED25519_IPNS_IDv1}.ipns.example.com" \ "http://127.0.0.1:$GWAY_PORT" \ "$CID_VAL" test_hostname_gateway_response_should_contain \ "hostname request for {CIDv1-dag-pb}.ipns.localhost redirects to CID with libp2p-key multicodec" \ "${RSA_IPNS_IDv1_DAGPB}.ipns.example.com" \ "http://127.0.0.1:$GWAY_PORT" \ "Location: http://${RSA_IPNS_IDv1}.ipns.example.com/" test_hostname_gateway_response_should_contain \ "hostname request for {CIDv1-dag-pb}.ipns.localhost redirects to CID with libp2p-key multicodec" \ "${ED25519_IPNS_IDv1_DAGPB}.ipns.example.com" \ "http://127.0.0.1:$GWAY_PORT" \ "Location: http://${ED25519_IPNS_IDv1}.ipns.example.com/" # DNSLink: .ipns.example.com # (not really useful outside of localhost, as setting TLS for more than one # level of wildcard is a pain, but we support it if someone really wants it) # ============================================================================ # DNSLink test requires a daemon in online mode with precached /ipns/ mapping test_kill_ipfs_daemon DNSLINK_FQDN="dnslink-subdomain-gw-test.example.org" export IPFS_NS_MAP="$DNSLINK_FQDN:/ipfs/$CIDv1" test_launch_ipfs_daemon test_hostname_gateway_response_should_contain \ "request for {dnslink}.ipns.example.com returns expected payload" \ "$DNSLINK_FQDN.ipns.example.com" \ "http://127.0.0.1:$GWAY_PORT" \ "$CID_VAL" # DNSLink on Public gateway with a single-level wildcard TLS cert # "Option C" from https://github.com/ipfs/in-web-browsers/issues/169 test_expect_success \ "request for {single-label-dnslink}.ipns.example.com with X-Forwarded-Proto returns expected payload" " curl -H \"Host: dnslink--subdomain--gw--test-example-org.ipns.example.com\" -H \"X-Forwarded-Proto: https\" -sD - \"http://127.0.0.1:$GWAY_PORT\" > response && test_should_contain \"$CID_VAL\" response " ## Test subdomain handling of CIDs that do not fit in a single DNS Label (>63chars) ## https://github.com/ipfs/go-ipfs/issues/7318 ## ============================================================================ # local: *.localhost test_localhost_gateway_response_should_contain \ "request for a ED25519 libp2p-key at localhost/ipns/{b58mh} returns Location HTTP header for DNS-safe subdomain redirect in browsers" \ "http://localhost:$GWAY_PORT/ipns/$IPNS_ED25519_B58MH" \ "Location: http://${IPNS_ED25519_B36CID}.ipns.localhost:$GWAY_PORT/" # router should not redirect to hostnames that could fail due to DNS limits test_localhost_gateway_response_should_contain \ "request for a too long CID at localhost/ipfs/{CIDv1} returns human readable error" \ "http://localhost:$GWAY_PORT/ipfs/$CIDv1_TOO_LONG" \ "CID incompatible with DNS label length limit of 63" test_localhost_gateway_response_should_contain \ "request for a too long CID at localhost/ipfs/{CIDv1} returns HTTP Error 400 Bad Request" \ "http://localhost:$GWAY_PORT/ipfs/$CIDv1_TOO_LONG" \ "400 Bad Request" # direct request should also fail (provides the same UX as router and avoids confusion) test_localhost_gateway_response_should_contain \ "request for a too long CID at {CIDv1}.ipfs.localhost returns expected payload" \ "http://$CIDv1_TOO_LONG.ipfs.localhost:$GWAY_PORT" \ "400 Bad Request" # public subdomain gateway: *.example.com test_hostname_gateway_response_should_contain \ "request for a ED25519 libp2p-key at example.com/ipns/{b58mh} returns Location HTTP header for DNS-safe subdomain redirect in browsers" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ED25519_B58MH" \ "Location: http://${IPNS_ED25519_B36CID}.ipns.example.com" test_hostname_gateway_response_should_contain \ "request for a too long CID at example.com/ipfs/{CIDv1} returns human readable error" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1_TOO_LONG" \ "CID incompatible with DNS label length limit of 63" test_hostname_gateway_response_should_contain \ "request for a too long CID at example.com/ipfs/{CIDv1} returns HTTP Error 400 Bad Request" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1_TOO_LONG" \ "400 Bad Request" test_hostname_gateway_response_should_contain \ "request for a too long CID at {CIDv1}.ipfs.example.com returns HTTP Error 400 Bad Request" \ "$CIDv1_TOO_LONG.ipfs.example.com" \ "http://127.0.0.1:$GWAY_PORT/" \ "400 Bad Request" # Disable selected Paths for the subdomain gateway hostname # ============================================================================= # disable /ipns for the hostname by not whitelisting it ipfs config --json Gateway.PublicGateways '{ "example.com": { "UseSubdomains": true, "Paths": ["/ipfs"] } }' || exit 1 # restart daemon to apply config changes test_kill_ipfs_daemon test_launch_ipfs_daemon_without_network # refuse requests to Paths that were not explicitly whitelisted for the hostname test_hostname_gateway_response_should_contain \ "request for *.ipns.example.com returns HTTP 404 Not Found when /ipns is not on Paths whitelist" \ "${RSA_IPNS_IDv1}.ipns.example.com" \ "http://127.0.0.1:$GWAY_PORT" \ "404 Not Found" test_hostname_gateway_response_should_contain \ "request for *.ipns.example.com returns HTTP 404 Not Found when /ipns is not on Paths whitelist" \ "${ED25519_IPNS_IDv1}.ipns.example.com" \ "http://127.0.0.1:$GWAY_PORT" \ "404 Not Found" ## ============================================================================ ## Test path-based requests with a custom hostname config ## ============================================================================ # set explicit no-subdomain gateway config for the hostname ipfs config --json Gateway.PublicGateways '{ "example.com": { "UseSubdomains": false, "Paths": ["/ipfs"] } }' || exit 1 # restart daemon to apply config changes test_kill_ipfs_daemon test_launch_ipfs_daemon_without_network # example.com/ip(f|n)s/* smoke-tests # ============================================================================= # confirm path gateway works for /ipfs test_hostname_gateway_response_should_contain \ "request for example.com/ipfs/{CIDv1} returns expected payload" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ "$CID_VAL" # refuse subdomain requests on path gateway # (we don't want false sense of security) test_hostname_gateway_response_should_contain \ "request for {CID}.ipfs.example.com/ipfs/{CID} should return HTTP 404 Not Found" \ "${CIDv1}.ipfs.example.com" \ "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ "404 Not Found" # refuse requests to Paths that were not explicitly whitelisted for the hostname test_hostname_gateway_response_should_contain \ "request for example.com/ipns/ returns HTTP 404 Not Found when /ipns is not on Paths whitelist" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipns/$RSA_IPNS_IDv1" \ "404 Not Found" test_hostname_gateway_response_should_contain \ "request for example.com/ipns/ returns HTTP 404 Not Found when /ipns is not on Paths whitelist" \ "example.com" \ "http://127.0.0.1:$GWAY_PORT/ipns/$ED25519_IPNS_IDv1" \ "404 Not Found" ## ============================================================================ ## Test DNSLink requests with a custom PublicGateway (hostname config) ## (DNSLink site at http://dnslink-test.example.com) ## ============================================================================ test_kill_ipfs_daemon # disable wildcard DNSLink gateway # and enable it on specific NSLink hostname ipfs config --json Gateway.NoDNSLink true && \ ipfs config --json Gateway.PublicGateways '{ "dnslink-enabled-on-fqdn.example.org": { "NoDNSLink": false, "UseSubdomains": false, "Paths": ["/ipfs"] }, "only-dnslink-enabled-on-fqdn.example.org": { "NoDNSLink": false, "UseSubdomains": false, "Paths": [] }, "dnslink-disabled-on-fqdn.example.com": { "NoDNSLink": true, "UseSubdomains": false, "Paths": [] } }' || exit 1 # DNSLink test requires a daemon in online mode with precached /ipns/ mapping DNSLINK_FQDN="dnslink-enabled-on-fqdn.example.org" ONLY_DNSLINK_FQDN="only-dnslink-enabled-on-fqdn.example.org" NO_DNSLINK_FQDN="dnslink-disabled-on-fqdn.example.com" export IPFS_NS_MAP="$DNSLINK_FQDN:/ipfs/$CIDv1,$ONLY_DNSLINK_FQDN:/ipfs/$DIR_CID" # restart daemon to apply config changes test_launch_ipfs_daemon # make sure test setup is valid (fail if CoreAPI is unable to resolve) test_expect_success "spoofed DNSLink record resolves in cli" " ipfs resolve /ipns/$DNSLINK_FQDN > result && test_should_contain \"$CIDv1\" result && ipfs cat /ipns/$DNSLINK_FQDN > result && test_should_contain \"$CID_VAL\" result " # DNSLink enabled test_hostname_gateway_response_should_contain \ "request for http://{dnslink-fqdn}/ PublicGateway returns expected payload" \ "$DNSLINK_FQDN" \ "http://127.0.0.1:$GWAY_PORT/" \ "$CID_VAL" test_hostname_gateway_response_should_contain \ "request for {dnslink-fqdn}/ipfs/{cid} returns expected payload when /ipfs is on Paths whitelist" \ "$DNSLINK_FQDN" \ "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ "$CID_VAL" # Test for a fun edge case: DNSLink-only gateway without /ipfs/ namespace # mounted, and with subdirectory named "ipfs" ¯\_(ツ)_/¯ test_hostname_gateway_response_should_contain \ "request for {dnslink-fqdn}/ipfs/file.txt returns data from content root when /ipfs in not on Paths whitelist" \ "$ONLY_DNSLINK_FQDN" \ "http://127.0.0.1:$GWAY_PORT/ipfs/file.txt" \ "I am a txt file" test_hostname_gateway_response_should_contain \ "request for {dnslink-fqdn}/ipns/{peerid} returns 404 when path is not whitelisted" \ "$DNSLINK_FQDN" \ "http://127.0.0.1:$GWAY_PORT/ipns/$RSA_IPNS_IDv0" \ "404 Not Found" test_hostname_gateway_response_should_contain \ "request for {dnslink-fqdn}/ipns/{peerid} returns 404 when path is not whitelisted" \ "$DNSLINK_FQDN" \ "http://127.0.0.1:$GWAY_PORT/ipns/$ED25519_IPNS_IDv0" \ "404 Not Found" # DNSLink disabled test_hostname_gateway_response_should_contain \ "request for http://{dnslink-fqdn}/ returns 404 when NoDNSLink=true" \ "$NO_DNSLINK_FQDN" \ "http://127.0.0.1:$GWAY_PORT/" \ "404 Not Found" test_hostname_gateway_response_should_contain \ "request for {dnslink-fqdn}/ipfs/{cid} returns 404 when path is not whitelisted" \ "$NO_DNSLINK_FQDN" \ "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv0" \ "404 Not Found" ## ============================================================================ ## Test wildcard DNSLink (any hostname, with default config) ## ============================================================================ test_kill_ipfs_daemon # enable wildcard DNSLink gateway (any value in Host header) # and remove custom PublicGateways ipfs config --json Gateway.NoDNSLink false && \ ipfs config --json Gateway.PublicGateways '{}' || exit 1 # DNSLink test requires a daemon in online mode with precached /ipns/ mapping DNSLINK_FQDN="wildcard-dnslink-not-in-config.example.com" export IPFS_NS_MAP="$DNSLINK_FQDN:/ipfs/$CIDv1" # restart daemon to apply config changes test_launch_ipfs_daemon # make sure test setup is valid (fail if CoreAPI is unable to resolve) test_expect_success "spoofed DNSLink record resolves in cli" " ipfs resolve /ipns/$DNSLINK_FQDN > result && test_should_contain \"$CIDv1\" result && ipfs cat /ipns/$DNSLINK_FQDN > result && test_should_contain \"$CID_VAL\" result " # gateway test test_hostname_gateway_response_should_contain \ "request for http://{dnslink-fqdn}/ (wildcard) returns expected payload" \ "$DNSLINK_FQDN" \ "http://127.0.0.1:$GWAY_PORT/" \ "$CID_VAL" ## ============================================================================ ## Test support for X-Forwarded-Host ## ============================================================================ # set explicit subdomain gateway config for the hostname ipfs config --json Gateway.PublicGateways '{ "example.com": { "UseSubdomains": true, "Paths": ["/ipfs", "/ipns", "/api"] } }' || exit 1 # restart daemon to apply config changes test_kill_ipfs_daemon test_launch_ipfs_daemon_without_network test_expect_success "request for http://fake.domain.com/ipfs/{CID} doesn't match the example.com gateway" " curl -H \"Host: fake.domain.com\" -sD - \"http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1\" > response && test_should_contain \"200 OK\" response " test_expect_success "request for http://fake.domain.com/ipfs/{CID} with X-Forwarded-Host: example.com match the example.com gateway" " curl -H \"Host: fake.domain.com\" -H \"X-Forwarded-Host: example.com\" -sD - \"http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1\" > response && test_should_contain \"Location: http://$CIDv1.ipfs.example.com/\" response " test_expect_success "request for http://fake.domain.com/ipfs/{CID} with X-Forwarded-Host: example.com and X-Forwarded-Proto: https match the example.com gateway, redirect with https" " curl -H \"Host: fake.domain.com\" -H \"X-Forwarded-Host: example.com\" -H \"X-Forwarded-Proto: https\" -sD - \"http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1\" > response && test_should_contain \"Location: https://$CIDv1.ipfs.example.com/\" response " # Kubo specific end-to-end test # (independent of gateway-conformance) # test configuration being wired up correctly end-to-end ## ============================================================================ ## Test support for wildcards in gateway config ## ============================================================================ # set explicit subdomain gateway config for the hostnames ipfs config --json Gateway.PublicGateways '{ "*.example1.com": { "UseSubdomains": true, "Paths": ["/ipfs"] }, "*.*.example2.com": { "UseSubdomains": true, "Paths": ["/ipfs"] }, "foo.*.example3.com": { "UseSubdomains": true, "Paths": ["/ipfs"] }, "foo.bar-*-boo.example4.com": { "UseSubdomains": true, "Paths": ["/ipfs"] } }' || exit 1 # restart daemon to apply config changes test_kill_ipfs_daemon test_launch_ipfs_daemon_without_network # *.example1.com test_hostname_gateway_response_should_contain \ "request for foo.example1.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.foo.example1.com" \ "foo.example1.com" \ "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ "Location: http://$CIDv1.ipfs.foo.example1.com/" test_hostname_gateway_response_should_contain \ "request for {CID}.ipfs.foo.example1.com should return expected payload" \ "${CIDv1}.ipfs.foo.example1.com" \ "http://127.0.0.1:$GWAY_PORT/" \ "$CID_VAL" # *.*.example2.com test_hostname_gateway_response_should_contain \ "request for foo.bar.example2.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.foo.bar.example2.com" \ "foo.bar.example2.com" \ "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ "Location: http://$CIDv1.ipfs.foo.bar.example2.com/" test_hostname_gateway_response_should_contain \ "request for {CID}.ipfs.foo.bar.example2.com should return expected payload" \ "${CIDv1}.ipfs.foo.bar.example2.com" \ "http://127.0.0.1:$GWAY_PORT/" \ "$CID_VAL" # foo.*.example3.com test_hostname_gateway_response_should_contain \ "request for foo.bar.example3.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.foo.bar.example3.com" \ "foo.bar.example3.com" \ "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ "Location: http://$CIDv1.ipfs.foo.bar.example3.com/" test_hostname_gateway_response_should_contain \ "request for {CID}.ipfs.foo.bar.example3.com should return expected payload" \ "${CIDv1}.ipfs.foo.bar.example3.com" \ "http://127.0.0.1:$GWAY_PORT/" \ "$CID_VAL" # foo.bar-*-boo.example4.com test_hostname_gateway_response_should_contain \ "request for foo.bar-dev-boo.example4.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.foo.bar-dev-boo.example4.com" \ "foo.bar-dev-boo.example4.com" \ "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ "Location: http://$CIDv1.ipfs.foo.bar-dev-boo.example4.com/" test_hostname_gateway_response_should_contain \ "request for {CID}.ipfs.foo.bar-dev-boo.example4.com should return expected payload" \ "${CIDv1}.ipfs.foo.bar-dev-boo.example4.com" \ "http://127.0.0.1:$GWAY_PORT/" \ "$CID_VAL" ## ============================================================================ ## Test support for overriding implicit defaults ## ============================================================================ # disable subdomain gateway at localhost by removing implicit config ipfs config --json Gateway.PublicGateways '{ "localhost": null }' || exit 1 # restart daemon to apply config changes test_kill_ipfs_daemon test_launch_ipfs_daemon_without_network test_localhost_gateway_response_should_contain \ "request for localhost/ipfs/{CID} stays on path when subdomain gw is explicitly disabled" \ "http://localhost:$GWAY_PORT/ipfs/$CIDv1" \ "$CID_VAL" # ============================================================================= # ensure we end with empty Gateway.PublicGateways ipfs config --json Gateway.PublicGateways '{}' test_kill_ipfs_daemon test_expect_success "clean up ipfs dir" ' rm -rf "$IPFS_PATH" ' test_done # end Kubo specific end-to-end test ================================================ FILE: test/sharness/t0115-gateway-dir-listing/README.md ================================================ # Dataset description/sources - fixtures.car - raw CARv1 generated with: ```sh # using ipfs version 0.18.1 mkdir -p rootDir/ipfs && mkdir -p rootDir/ipns && mkdir -p rootDir/api && mkdir -p rootDir/ą/ę && echo "I am a txt file on path with utf8" > rootDir/ą/ę/file-źł.txt && echo "I am a txt file in confusing /api dir" > rootDir/api/file.txt && echo "I am a txt file in confusing /ipfs dir" > rootDir/ipfs/file.txt && echo "I am a txt file in confusing /ipns dir" > rootDir/ipns/file.txt && DIR_CID=$(ipfs add -Qr --cid-version 1 rootDir) && FILE_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Hash) && FILE_SIZE=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Size) echo "$FILE_CID / $FILE_SIZE" echo DIR_CID=${DIR_CID} echo FILE_CID=${FILE_CID} echo FILE_SIZE=${FILE_SIZE} ipfs dag export ${DIR_CID} > ./fixtures.car # DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir/ # FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt # FILE_SIZE=34 ``` ================================================ FILE: test/sharness/t0115-gateway-dir-listing.sh ================================================ #!/usr/bin/env bash # # Copyright (c) Protocol Labs test_description="Test directory listing (dir-index-html) on the HTTP gateway" . lib/test-lib.sh ## ============================================================================ ## Start IPFS Node and prepare test CIDs ## ============================================================================ test_expect_success "ipfs init" ' export IPFS_PATH="$(pwd)/.ipfs" && ipfs init --profile=test > /dev/null ' test_launch_ipfs_daemon_without_network # Import test case # See the static fixtures in ./t0115-gateway-dir-listing/ test_expect_success "Add the test directory" ' ipfs dag import --pin-roots ../t0115-gateway-dir-listing/fixtures.car ' DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir/ FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt FILE_SIZE=34 ## ============================================================================ ## Test dir listing on path gateway (eg. 127.0.0.1:8080/ipfs/) ## ============================================================================ test_expect_success "path gw: backlink on root CID should be hidden" ' curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ > list_response && test_should_contain "Index of" list_response && test_should_not_contain ".." list_response ' test_expect_success "path gw: redirect dir listing to URL with trailing slash" ' curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ą/ę > list_response && test_should_contain "HTTP/1.1 301 Moved Permanently" list_response && test_should_contain "Location: /ipfs/${DIR_CID}/%C4%85/%C4%99/" list_response ' test_expect_success "path gw: Etag should be present" ' curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ą/ę/ > list_response && test_should_contain "Index of" list_response && test_should_contain "Etag: \"DirIndex-" list_response ' test_expect_success "path gw: breadcrumbs should point at /ipfs namespace mounted at Origin root" ' test_should_contain "/ipfs/$DIR_CID/ą/ę" list_response ' test_expect_success "path gw: backlink on subdirectory should point at parent directory" ' test_should_contain ".." list_response ' test_expect_success "path gw: name column should be a link to its content path" ' test_should_contain "file-źł.txt" list_response ' test_expect_success "path gw: hash column should be a CID link with filename param" ' test_should_contain "" list_response ' ## ============================================================================ ## Test dir listing on subdomain gateway (eg. .ipfs.localhost:8080) ## ============================================================================ DIR_HOSTNAME="${DIR_CID}.ipfs.localhost" # note: we skip DNS lookup by running curl with --resolve $DIR_HOSTNAME:127.0.0.1 test_expect_success "subdomain gw: backlink on root CID should be hidden" ' curl -sD - --resolve $DIR_HOSTNAME:$GWAY_PORT:127.0.0.1 http://$DIR_HOSTNAME:$GWAY_PORT/ > list_response && test_should_contain "Index of" list_response && test_should_not_contain ".." list_response ' test_expect_success "subdomain gw: redirect dir listing to URL with trailing slash" ' curl -sD - --resolve $DIR_HOSTNAME:$GWAY_PORT:127.0.0.1 http://$DIR_HOSTNAME:$GWAY_PORT/ą/ę > list_response && test_should_contain "HTTP/1.1 301 Moved Permanently" list_response && test_should_contain "Location: /%C4%85/%C4%99/" list_response ' test_expect_success "subdomain gw: Etag should be present" ' curl -sD - --resolve $DIR_HOSTNAME:$GWAY_PORT:127.0.0.1 http://$DIR_HOSTNAME:$GWAY_PORT/ą/ę/ > list_response && test_should_contain "Index of" list_response && test_should_contain "Etag: \"DirIndex-" list_response ' test_expect_success "subdomain gw: backlink on subdirectory should point at parent directory" ' test_should_contain ".." list_response ' test_expect_success "subdomain gw: breadcrumbs should leverage path-based router mounted on the parent domain" ' test_should_contain "/ipfs/$DIR_CID/ą/ę" list_response ' test_expect_success "subdomain gw: name column should be a link to content root mounted at subdomain origin" ' test_should_contain "file-źł.txt" list_response ' test_expect_success "subdomain gw: hash column should be a CID link to path router with filename param" ' test_should_contain "" list_response ' ## ============================================================================ ## Test dir listing on DNSLink gateway (eg. example.com) ## ============================================================================ # DNSLink test requires a daemon in online mode with precached /ipns/ mapping test_kill_ipfs_daemon DNSLINK_HOSTNAME="website.example.com" export IPFS_NS_MAP="$DNSLINK_HOSTNAME:/ipfs/$DIR_CID" test_launch_ipfs_daemon # Note that: # - this type of gateway is also tested in gateway_test.go#TestIPNSHostnameBacklinks # (go tests and sharness tests should be kept in sync) # - we skip DNS lookup by running curl with --resolve $DNSLINK_HOSTNAME:127.0.0.1 test_expect_success "dnslink gw: backlink on root CID should be hidden" ' curl -v -sD - --resolve $DNSLINK_HOSTNAME:$GWAY_PORT:127.0.0.1 http://$DNSLINK_HOSTNAME:$GWAY_PORT/ > list_response && test_should_contain "Index of" list_response && test_should_not_contain ".." list_response ' test_expect_success "dnslink gw: redirect dir listing to URL with trailing slash" ' curl -sD - --resolve $DNSLINK_HOSTNAME:$GWAY_PORT:127.0.0.1 http://$DNSLINK_HOSTNAME:$GWAY_PORT/ą/ę > list_response && test_should_contain "HTTP/1.1 301 Moved Permanently" list_response && test_should_contain "Location: /%C4%85/%C4%99/" list_response ' test_expect_success "dnslink gw: Etag should be present" ' curl -sD - --resolve $DNSLINK_HOSTNAME:$GWAY_PORT:127.0.0.1 http://$DNSLINK_HOSTNAME:$GWAY_PORT/ą/ę/ > list_response && test_should_contain "Index of" list_response && test_should_contain "Etag: \"DirIndex-" list_response ' test_expect_success "dnslink gw: backlink on subdirectory should point at parent directory" ' test_should_contain ".." list_response ' test_expect_success "dnslink gw: breadcrumbs should point at content root mounted at dnslink origin" ' test_should_contain "/ipns/website.example.com/ą/ę" list_response ' test_expect_success "dnslink gw: name column should be a link to content root mounted at dnslink origin" ' test_should_contain "file-źł.txt" list_response ' # DNSLink websites don't have public gateway mounted by default # See: https://github.com/ipfs/dir-index-html/issues/42 test_expect_success "dnslink gw: hash column should be a CID link to cid.ipfs.tech" ' test_should_contain "" list_response ' ## ============================================================================ ## End of tests, cleanup ## ============================================================================ test_kill_ipfs_daemon test_expect_success "clean up ipfs dir" ' rm -rf "$IPFS_PATH" ' test_done ================================================ FILE: test/sharness/t0116-gateway-cache/README.md ================================================ # Dataset description/sources - fixtures.car - raw CARv1 generated with: ```sh # using ipfs version 0.21.0-dev (03a98280e3e642774776cd3d0435ab53e5dfa867) mkdir -p root2/root3/root4 && echo "hello" > root2/root3/root4/index.html && ROOT1_CID=$(ipfs add -Qrw --cid-version 1 root2) ROOT2_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2 | cut -d "/" -f3) ROOT3_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2/root3 | cut -d "/" -f3) ROOT4_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2/root3/root4 | cut -d "/" -f3) FILE_CID=$(ipfs resolve -r /ipfs/$ROOT1_CID/root2/root3/root4/index.html | cut -d "/" -f3) TEST_IPNS_ID=$(ipfs key gen --ipns-base=base36 --type=ed25519 cache_test_key | head -n1 | tr -d "\n") # publish a record valid for a 100 years ipfs name publish --key cache_test_key --allow-offline -Q --ttl=876600h --lifetime=876600h "/ipfs/$ROOT1_CID" ipfs routing get /ipns/${TEST_IPNS_ID} > ${TEST_IPNS_ID}.ipns-record echo ROOT1_CID=${ROOT1_CID} # ./ echo ROOT2_CID=${ROOT2_CID} # ./root2 echo ROOT3_CID=${ROOT3_CID} # ./root2/root3 echo ROOT4_CID=${ROOT4_CID} # ./root2/root3/root4 echo FILE_CID=${FILE_CID} # ./root2/root3/root4/index.html echo TEST_IPNS_ID=${TEST_IPNS_ID} ipfs dag export ${ROOT1_CID} > ./fixtures.car # ROOT1_CID=bafybeib3ffl2teiqdncv3mkz4r23b5ctrwkzrrhctdbne6iboayxuxk5ui # ./ # ROOT2_CID=bafybeih2w7hjocxjg6g2ku25hvmd53zj7og4txpby3vsusfefw5rrg5sii # ./root2 # ROOT3_CID=bafybeiawdvhmjcz65x5egzx4iukxc72hg4woks6v6fvgyupiyt3oczk5ja # ./root2/root3 # ROOT4_CID=bafybeifq2rzpqnqrsdupncmkmhs3ckxxjhuvdcbvydkgvch3ms24k5lo7q # ./root2/root3/root4 # FILE_CID=bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am # ./root2/root3/root4/index.html # TEST_IPNS_ID=k51qzi5uqu5dlxdsdu5fpuu7h69wu4ohp32iwm9pdt9nq3y5rpn3ln9j12zfhe ``` ================================================ FILE: test/sharness/t0116-gateway-cache.sh ================================================ #!/usr/bin/env bash test_description="Test HTTP Gateway Cache Control Support" . lib/test-lib.sh test_init_ipfs test_launch_ipfs_daemon_without_network # Cache control support is based on logical roots (each path segment == one logical root). # To maximize the test surface, we want to test: # - /ipfs/ content path # - /ipns/ content path # - at least 3 levels # - separate tests for a directory listing and a file # - have implicit index.html for a good measure # /ipns/root1/root2/root3/ (/ipns/root1/root2/root3/index.html) # Note: we cover important UnixFS-focused edge case here: # # ROOT3_CID - dir listing (dir-index-html response) # ROOT4_CID - index.html returned as a root response (dir/), instead of generated dir-index-html # FILE_CID - index.html returned directly, as a file # # Caching of things like raw blocks, CARs, dag-json and dag-cbor # is tested in their respective suites. ROOT1_CID=bafybeib3ffl2teiqdncv3mkz4r23b5ctrwkzrrhctdbne6iboayxuxk5ui # ./ ROOT2_CID=bafybeih2w7hjocxjg6g2ku25hvmd53zj7og4txpby3vsusfefw5rrg5sii # ./root2 ROOT3_CID=bafybeiawdvhmjcz65x5egzx4iukxc72hg4woks6v6fvgyupiyt3oczk5ja # ./root2/root3 ROOT4_CID=bafybeifq2rzpqnqrsdupncmkmhs3ckxxjhuvdcbvydkgvch3ms24k5lo7q # ./root2/root3/root4 FILE_CID=bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am # ./root2/root3/root4/index.html TEST_IPNS_ID=k51qzi5uqu5dlxdsdu5fpuu7h69wu4ohp32iwm9pdt9nq3y5rpn3ln9j12zfhe # Import test case # See the static fixtures in ./t0116-gateway-cache/ test_expect_success "Add the test directory" ' ipfs dag import --pin-roots ../t0116-gateway-cache/fixtures.car ipfs routing put --allow-offline /ipns/${TEST_IPNS_ID} ../t0116-gateway-cache/${TEST_IPNS_ID}.ipns-record ' # Etag test_expect_success "GET for /ipfs/ unixfs dir listing succeeds" ' curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/" >/dev/null 2>curl_ipfs_dir_listing_output ' test_expect_success "GET for /ipns/ unixfs dir listing succeeds" ' curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/" >/dev/null 2>curl_ipns_dir_listing_output ' ## dir generated listing test_expect_success "GET /ipfs/ dir response has special Etag for generated dir listing" ' test_should_contain "< Etag: \"DirIndex" curl_ipfs_dir_listing_output && grep -E "< Etag: \"DirIndex-.+_CID-${ROOT3_CID}\"" curl_ipfs_dir_listing_output ' test_expect_success "GET /ipns/ dir response has special Etag for generated dir listing" ' test_should_contain "< Etag: \"DirIndex" curl_ipns_dir_listing_output && grep -E "< Etag: \"DirIndex-.+_CID-${ROOT3_CID}\"" curl_ipns_dir_listing_output ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0119-prometheus-data/prometheus_metrics ================================================ exchange_bitswap_requests_in_flight exchange_bitswap_response_bytes_bucket exchange_bitswap_response_bytes_count exchange_bitswap_response_bytes_sum exchange_bitswap_wantlists_items_total exchange_bitswap_wantlists_seconds_bucket exchange_bitswap_wantlists_seconds_count exchange_bitswap_wantlists_seconds_sum exchange_bitswap_wantlists_total exchange_httpnet_request_duration_seconds_bucket exchange_httpnet_request_duration_seconds_count exchange_httpnet_request_duration_seconds_sum exchange_httpnet_request_sent_bytes exchange_httpnet_requests_body_failure exchange_httpnet_requests_failure exchange_httpnet_requests_in_flight exchange_httpnet_requests_total exchange_httpnet_response_bytes_bucket exchange_httpnet_response_bytes_count exchange_httpnet_response_bytes_sum exchange_httpnet_wantlists_items_total exchange_httpnet_wantlists_seconds_bucket exchange_httpnet_wantlists_seconds_count exchange_httpnet_wantlists_seconds_sum exchange_httpnet_wantlists_total go_gc_duration_seconds go_gc_duration_seconds_count go_gc_duration_seconds_sum go_gc_gogc_percent go_gc_gomemlimit_bytes go_goroutines go_info go_memstats_alloc_bytes go_memstats_alloc_bytes_total go_memstats_buck_hash_sys_bytes go_memstats_frees_total go_memstats_gc_sys_bytes go_memstats_heap_alloc_bytes go_memstats_heap_idle_bytes go_memstats_heap_inuse_bytes go_memstats_heap_objects go_memstats_heap_released_bytes go_memstats_heap_sys_bytes go_memstats_last_gc_time_seconds go_memstats_mallocs_total go_memstats_mcache_inuse_bytes go_memstats_mcache_sys_bytes go_memstats_mspan_inuse_bytes go_memstats_mspan_sys_bytes go_memstats_next_gc_bytes go_memstats_other_sys_bytes go_memstats_stack_inuse_bytes go_memstats_stack_sys_bytes go_memstats_sys_bytes go_sched_gomaxprocs_threads go_threads http_server_request_body_size_bytes_bucket http_server_request_body_size_bytes_count http_server_request_body_size_bytes_sum http_server_request_duration_seconds_bucket http_server_request_duration_seconds_count http_server_request_duration_seconds_sum http_server_response_body_size_bytes_bucket http_server_response_body_size_bytes_count http_server_response_body_size_bytes_sum ipfs_bitswap_active_block_tasks ipfs_bitswap_active_tasks ipfs_bitswap_bcast_skips_total ipfs_bitswap_blocks_received ipfs_bitswap_haves_received ipfs_bitswap_pending_block_tasks ipfs_bitswap_pending_tasks ipfs_bitswap_recv_all_blocks_bytes_bucket ipfs_bitswap_recv_all_blocks_bytes_count ipfs_bitswap_recv_all_blocks_bytes_sum ipfs_bitswap_recv_dup_blocks_bytes_bucket ipfs_bitswap_recv_dup_blocks_bytes_count ipfs_bitswap_recv_dup_blocks_bytes_sum ipfs_bitswap_send_times_bucket ipfs_bitswap_send_times_count ipfs_bitswap_send_times_sum ipfs_bitswap_sent_all_blocks_bytes_bucket ipfs_bitswap_sent_all_blocks_bytes_count ipfs_bitswap_sent_all_blocks_bytes_sum ipfs_bitswap_want_blocks_total ipfs_bitswap_wanthaves_broadcast ipfs_bitswap_wantlist_total ipfs_bs_cache_boxo_blockstore_cache_hits ipfs_bs_cache_boxo_blockstore_cache_total ipfs_fsrepo_datastore_batchcommit_errors_total ipfs_fsrepo_datastore_batchcommit_latency_seconds_bucket ipfs_fsrepo_datastore_batchcommit_latency_seconds_count ipfs_fsrepo_datastore_batchcommit_latency_seconds_sum ipfs_fsrepo_datastore_batchcommit_total ipfs_fsrepo_datastore_batchdelete_errors_total ipfs_fsrepo_datastore_batchdelete_latency_seconds_bucket ipfs_fsrepo_datastore_batchdelete_latency_seconds_count ipfs_fsrepo_datastore_batchdelete_latency_seconds_sum ipfs_fsrepo_datastore_batchdelete_total ipfs_fsrepo_datastore_batchput_errors_total ipfs_fsrepo_datastore_batchput_latency_seconds_bucket ipfs_fsrepo_datastore_batchput_latency_seconds_count ipfs_fsrepo_datastore_batchput_latency_seconds_sum ipfs_fsrepo_datastore_batchput_size_bytes_bucket ipfs_fsrepo_datastore_batchput_size_bytes_count ipfs_fsrepo_datastore_batchput_size_bytes_sum ipfs_fsrepo_datastore_batchput_total ipfs_fsrepo_datastore_check_errors_total ipfs_fsrepo_datastore_check_latency_seconds_bucket ipfs_fsrepo_datastore_check_latency_seconds_count ipfs_fsrepo_datastore_check_latency_seconds_sum ipfs_fsrepo_datastore_check_total ipfs_fsrepo_datastore_delete_errors_total ipfs_fsrepo_datastore_delete_latency_seconds_bucket ipfs_fsrepo_datastore_delete_latency_seconds_count ipfs_fsrepo_datastore_delete_latency_seconds_sum ipfs_fsrepo_datastore_delete_total ipfs_fsrepo_datastore_du_errors_total ipfs_fsrepo_datastore_du_latency_seconds_bucket ipfs_fsrepo_datastore_du_latency_seconds_count ipfs_fsrepo_datastore_du_latency_seconds_sum ipfs_fsrepo_datastore_du_total ipfs_fsrepo_datastore_gc_errors_total ipfs_fsrepo_datastore_gc_latency_seconds_bucket ipfs_fsrepo_datastore_gc_latency_seconds_count ipfs_fsrepo_datastore_gc_latency_seconds_sum ipfs_fsrepo_datastore_gc_total ipfs_fsrepo_datastore_get_errors_total ipfs_fsrepo_datastore_get_latency_seconds_bucket ipfs_fsrepo_datastore_get_latency_seconds_count ipfs_fsrepo_datastore_get_latency_seconds_sum ipfs_fsrepo_datastore_get_size_bytes_bucket ipfs_fsrepo_datastore_get_size_bytes_count ipfs_fsrepo_datastore_get_size_bytes_sum ipfs_fsrepo_datastore_get_total ipfs_fsrepo_datastore_getsize_errors_total ipfs_fsrepo_datastore_getsize_latency_seconds_bucket ipfs_fsrepo_datastore_getsize_latency_seconds_count ipfs_fsrepo_datastore_getsize_latency_seconds_sum ipfs_fsrepo_datastore_getsize_total ipfs_fsrepo_datastore_has_errors_total ipfs_fsrepo_datastore_has_latency_seconds_bucket ipfs_fsrepo_datastore_has_latency_seconds_count ipfs_fsrepo_datastore_has_latency_seconds_sum ipfs_fsrepo_datastore_has_total ipfs_fsrepo_datastore_put_errors_total ipfs_fsrepo_datastore_put_latency_seconds_bucket ipfs_fsrepo_datastore_put_latency_seconds_count ipfs_fsrepo_datastore_put_latency_seconds_sum ipfs_fsrepo_datastore_put_size_bytes_bucket ipfs_fsrepo_datastore_put_size_bytes_count ipfs_fsrepo_datastore_put_size_bytes_sum ipfs_fsrepo_datastore_put_total ipfs_fsrepo_datastore_query_errors_total ipfs_fsrepo_datastore_query_latency_seconds_bucket ipfs_fsrepo_datastore_query_latency_seconds_count ipfs_fsrepo_datastore_query_latency_seconds_sum ipfs_fsrepo_datastore_query_total ipfs_fsrepo_datastore_scrub_errors_total ipfs_fsrepo_datastore_scrub_latency_seconds_bucket ipfs_fsrepo_datastore_scrub_latency_seconds_count ipfs_fsrepo_datastore_scrub_latency_seconds_sum ipfs_fsrepo_datastore_scrub_total ipfs_fsrepo_datastore_sync_errors_total ipfs_fsrepo_datastore_sync_latency_seconds_bucket ipfs_fsrepo_datastore_sync_latency_seconds_count ipfs_fsrepo_datastore_sync_latency_seconds_sum ipfs_fsrepo_datastore_sync_total ipfs_http_gw_concurrent_requests ipfs_http_request_duration_seconds ipfs_http_request_duration_seconds_count ipfs_http_request_duration_seconds_sum ipfs_http_request_size_bytes ipfs_http_request_size_bytes_count ipfs_http_request_size_bytes_sum ipfs_http_requests_total ipfs_http_response_size_bytes ipfs_http_response_size_bytes_count ipfs_http_response_size_bytes_sum ipfs_info libp2p_autonat_next_probe_timestamp libp2p_autonat_reachability_status libp2p_autonat_reachability_status_confidence libp2p_autorelay_candidate_loop_state libp2p_autorelay_candidates_circuit_v2_support_total libp2p_autorelay_desired_reservations libp2p_autorelay_relay_addresses_count libp2p_autorelay_relay_addresses_updated_total libp2p_autorelay_reservation_requests_outcome_total libp2p_autorelay_reservations_closed_total libp2p_autorelay_reservations_opened_total libp2p_autorelay_status libp2p_eventbus_events_emitted_total libp2p_eventbus_subscriber_event_queued libp2p_eventbus_subscriber_queue_full libp2p_eventbus_subscriber_queue_length libp2p_eventbus_subscribers_total libp2p_holepunch_address_outcomes_total libp2p_holepunch_outcomes_total libp2p_identify_addrs_count libp2p_identify_addrs_received_bucket libp2p_identify_addrs_received_count libp2p_identify_addrs_received_sum libp2p_identify_identify_pushes_triggered_total libp2p_identify_protocols_count libp2p_identify_protocols_received_bucket libp2p_identify_protocols_received_count libp2p_identify_protocols_received_sum libp2p_rcmgr_conn_memory_bucket libp2p_rcmgr_conn_memory_count libp2p_rcmgr_conn_memory_sum libp2p_rcmgr_connections libp2p_rcmgr_fds libp2p_rcmgr_peer_connections_bucket libp2p_rcmgr_peer_connections_count libp2p_rcmgr_peer_connections_sum libp2p_rcmgr_peer_memory_bucket libp2p_rcmgr_peer_memory_count libp2p_rcmgr_peer_memory_sum libp2p_rcmgr_peer_streams_bucket libp2p_rcmgr_peer_streams_count libp2p_rcmgr_peer_streams_sum libp2p_rcmgr_previous_conn_memory_bucket libp2p_rcmgr_previous_conn_memory_count libp2p_rcmgr_previous_conn_memory_sum libp2p_rcmgr_previous_peer_connections_bucket libp2p_rcmgr_previous_peer_connections_count libp2p_rcmgr_previous_peer_connections_sum libp2p_rcmgr_previous_peer_memory_bucket libp2p_rcmgr_previous_peer_memory_count libp2p_rcmgr_previous_peer_memory_sum libp2p_rcmgr_previous_peer_streams_bucket libp2p_rcmgr_previous_peer_streams_count libp2p_rcmgr_previous_peer_streams_sum libp2p_relaysvc_connection_duration_seconds_bucket libp2p_relaysvc_connection_duration_seconds_count libp2p_relaysvc_connection_duration_seconds_sum libp2p_relaysvc_data_transferred_bytes_total libp2p_relaysvc_status libp2p_swarm_dial_ranking_delay_seconds_bucket libp2p_swarm_dial_ranking_delay_seconds_count libp2p_swarm_dial_ranking_delay_seconds_sum otel_scope_info process_cpu_seconds_total process_max_fds process_network_receive_bytes_total process_network_transmit_bytes_total process_open_fds process_resident_memory_bytes process_start_time_seconds process_virtual_memory_bytes process_virtual_memory_max_bytes provider_provides_total target_info ================================================ FILE: test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_enabling_rcmgr ================================================ libp2p_rcmgr_limit ================================================ FILE: test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_measure_profile ================================================ flatfs_datastore_batchcommit_errors_total flatfs_datastore_batchcommit_latency_seconds_bucket flatfs_datastore_batchcommit_latency_seconds_count flatfs_datastore_batchcommit_latency_seconds_sum flatfs_datastore_batchcommit_total flatfs_datastore_batchdelete_errors_total flatfs_datastore_batchdelete_latency_seconds_bucket flatfs_datastore_batchdelete_latency_seconds_count flatfs_datastore_batchdelete_latency_seconds_sum flatfs_datastore_batchdelete_total flatfs_datastore_batchput_errors_total flatfs_datastore_batchput_latency_seconds_bucket flatfs_datastore_batchput_latency_seconds_count flatfs_datastore_batchput_latency_seconds_sum flatfs_datastore_batchput_size_bytes_bucket flatfs_datastore_batchput_size_bytes_count flatfs_datastore_batchput_size_bytes_sum flatfs_datastore_batchput_total flatfs_datastore_check_errors_total flatfs_datastore_check_latency_seconds_bucket flatfs_datastore_check_latency_seconds_count flatfs_datastore_check_latency_seconds_sum flatfs_datastore_check_total flatfs_datastore_delete_errors_total flatfs_datastore_delete_latency_seconds_bucket flatfs_datastore_delete_latency_seconds_count flatfs_datastore_delete_latency_seconds_sum flatfs_datastore_delete_total flatfs_datastore_du_errors_total flatfs_datastore_du_latency_seconds_bucket flatfs_datastore_du_latency_seconds_count flatfs_datastore_du_latency_seconds_sum flatfs_datastore_du_total flatfs_datastore_gc_errors_total flatfs_datastore_gc_latency_seconds_bucket flatfs_datastore_gc_latency_seconds_count flatfs_datastore_gc_latency_seconds_sum flatfs_datastore_gc_total flatfs_datastore_get_errors_total flatfs_datastore_get_latency_seconds_bucket flatfs_datastore_get_latency_seconds_count flatfs_datastore_get_latency_seconds_sum flatfs_datastore_get_size_bytes_bucket flatfs_datastore_get_size_bytes_count flatfs_datastore_get_size_bytes_sum flatfs_datastore_get_total flatfs_datastore_getsize_errors_total flatfs_datastore_getsize_latency_seconds_bucket flatfs_datastore_getsize_latency_seconds_count flatfs_datastore_getsize_latency_seconds_sum flatfs_datastore_getsize_total flatfs_datastore_has_errors_total flatfs_datastore_has_latency_seconds_bucket flatfs_datastore_has_latency_seconds_count flatfs_datastore_has_latency_seconds_sum flatfs_datastore_has_total flatfs_datastore_put_errors_total flatfs_datastore_put_latency_seconds_bucket flatfs_datastore_put_latency_seconds_count flatfs_datastore_put_latency_seconds_sum flatfs_datastore_put_size_bytes_bucket flatfs_datastore_put_size_bytes_count flatfs_datastore_put_size_bytes_sum flatfs_datastore_put_total flatfs_datastore_query_errors_total flatfs_datastore_query_latency_seconds_bucket flatfs_datastore_query_latency_seconds_count flatfs_datastore_query_latency_seconds_sum flatfs_datastore_query_total flatfs_datastore_scrub_errors_total flatfs_datastore_scrub_latency_seconds_bucket flatfs_datastore_scrub_latency_seconds_count flatfs_datastore_scrub_latency_seconds_sum flatfs_datastore_scrub_total flatfs_datastore_sync_errors_total flatfs_datastore_sync_latency_seconds_bucket flatfs_datastore_sync_latency_seconds_count flatfs_datastore_sync_latency_seconds_sum flatfs_datastore_sync_total leveldb_datastore_batchcommit_errors_total leveldb_datastore_batchcommit_latency_seconds_bucket leveldb_datastore_batchcommit_latency_seconds_count leveldb_datastore_batchcommit_latency_seconds_sum leveldb_datastore_batchcommit_total leveldb_datastore_batchdelete_errors_total leveldb_datastore_batchdelete_latency_seconds_bucket leveldb_datastore_batchdelete_latency_seconds_count leveldb_datastore_batchdelete_latency_seconds_sum leveldb_datastore_batchdelete_total leveldb_datastore_batchput_errors_total leveldb_datastore_batchput_latency_seconds_bucket leveldb_datastore_batchput_latency_seconds_count leveldb_datastore_batchput_latency_seconds_sum leveldb_datastore_batchput_size_bytes_bucket leveldb_datastore_batchput_size_bytes_count leveldb_datastore_batchput_size_bytes_sum leveldb_datastore_batchput_total leveldb_datastore_check_errors_total leveldb_datastore_check_latency_seconds_bucket leveldb_datastore_check_latency_seconds_count leveldb_datastore_check_latency_seconds_sum leveldb_datastore_check_total leveldb_datastore_delete_errors_total leveldb_datastore_delete_latency_seconds_bucket leveldb_datastore_delete_latency_seconds_count leveldb_datastore_delete_latency_seconds_sum leveldb_datastore_delete_total leveldb_datastore_du_errors_total leveldb_datastore_du_latency_seconds_bucket leveldb_datastore_du_latency_seconds_count leveldb_datastore_du_latency_seconds_sum leveldb_datastore_du_total leveldb_datastore_gc_errors_total leveldb_datastore_gc_latency_seconds_bucket leveldb_datastore_gc_latency_seconds_count leveldb_datastore_gc_latency_seconds_sum leveldb_datastore_gc_total leveldb_datastore_get_errors_total leveldb_datastore_get_latency_seconds_bucket leveldb_datastore_get_latency_seconds_count leveldb_datastore_get_latency_seconds_sum leveldb_datastore_get_size_bytes_bucket leveldb_datastore_get_size_bytes_count leveldb_datastore_get_size_bytes_sum leveldb_datastore_get_total leveldb_datastore_getsize_errors_total leveldb_datastore_getsize_latency_seconds_bucket leveldb_datastore_getsize_latency_seconds_count leveldb_datastore_getsize_latency_seconds_sum leveldb_datastore_getsize_total leveldb_datastore_has_errors_total leveldb_datastore_has_latency_seconds_bucket leveldb_datastore_has_latency_seconds_count leveldb_datastore_has_latency_seconds_sum leveldb_datastore_has_total leveldb_datastore_put_errors_total leveldb_datastore_put_latency_seconds_bucket leveldb_datastore_put_latency_seconds_count leveldb_datastore_put_latency_seconds_sum leveldb_datastore_put_size_bytes_bucket leveldb_datastore_put_size_bytes_count leveldb_datastore_put_size_bytes_sum leveldb_datastore_put_total leveldb_datastore_query_errors_total leveldb_datastore_query_latency_seconds_bucket leveldb_datastore_query_latency_seconds_count leveldb_datastore_query_latency_seconds_sum leveldb_datastore_query_total leveldb_datastore_scrub_errors_total leveldb_datastore_scrub_latency_seconds_bucket leveldb_datastore_scrub_latency_seconds_count leveldb_datastore_scrub_latency_seconds_sum leveldb_datastore_scrub_total leveldb_datastore_sync_errors_total leveldb_datastore_sync_latency_seconds_bucket leveldb_datastore_sync_latency_seconds_count leveldb_datastore_sync_latency_seconds_sum leveldb_datastore_sync_total libp2p_rcmgr_limit ================================================ FILE: test/sharness/t0119-prometheus.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2020 Protocol Labs # MIT/Apache-2.0 Licensed; see the LICENSE file in this repository. # test_description="Test prometheus metrics are exposed correctly" . lib/test-lib.sh test_init_ipfs test_expect_success "enable ResourceMgr in the config" ' ipfs config --json Swarm.ResourceMgr.Enabled false ' test_launch_ipfs_daemon test_expect_success "collect metrics" ' curl "$API_ADDR/debug/metrics/prometheus" > raw_metrics ' test_kill_ipfs_daemon test_expect_success "filter metrics" ' sed -ne "s/^\([a-z0-9_]\+\).*/\1/p" raw_metrics | LC_ALL=C sort | uniq > filtered_metrics ' test_expect_success "make sure metrics haven't changed" ' diff -u ../t0119-prometheus-data/prometheus_metrics filtered_metrics ' # Check what was added by enabling ResourceMgr.Enabled # # NOTE: we won't see all the dynamic ones, but that is ok: the point of the # test here is to detect regression when rcmgr metrics disappear due to # refactor/human error. test_expect_success "enable ResourceMgr in the config" ' ipfs config --json Swarm.ResourceMgr.Enabled true ' test_launch_ipfs_daemon test_expect_success "collect metrics" ' curl "$API_ADDR/debug/metrics/prometheus" > raw_metrics ' test_kill_ipfs_daemon test_expect_success "filter metrics and find ones added by enabling ResourceMgr" ' sed -ne "s/^\([a-z0-9_]\+\).*/\1/p" raw_metrics | LC_ALL=C sort > filtered_metrics && grep -v -x -f ../t0119-prometheus-data/prometheus_metrics filtered_metrics | LC_ALL=C sort | uniq > rcmgr_metrics ' test_expect_success "make sure initial metrics added by setting ResourceMgr.Enabled haven't changed" ' diff -u ../t0119-prometheus-data/prometheus_metrics_added_by_enabling_rcmgr rcmgr_metrics ' # Reinitialize ipfs with --profile=flatfs-measure and check metrics. test_expect_success "remove ipfs directory" ' rm -rf .ipfs mountdir ipfs ipns ' test_init_ipfs_measure test_launch_ipfs_daemon test_expect_success "collect metrics" ' curl "$API_ADDR/debug/metrics/prometheus" > raw_metrics ' test_kill_ipfs_daemon test_expect_success "filter metrics and find ones added by enabling flatfs-measure profile" ' sed -ne "s/^\([a-z0-9_]\+\).*/\1/p" raw_metrics | LC_ALL=C sort > filtered_metrics && grep -v -x -f ../t0119-prometheus-data/prometheus_metrics filtered_metrics | LC_ALL=C sort | uniq > measure_metrics ' test_expect_success "make sure initial metrics added by initializing with flatfs-measure profile haven't changed" ' diff -u ../t0119-prometheus-data/prometheus_metrics_added_by_measure_profile measure_metrics ' test_done ================================================ FILE: test/sharness/t0120-bootstrap.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # # changing the bootstrap peers will require changing it in two places :) BP1="/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN" BP2="/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa" BP3="/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb" BP4="/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt" BP5="/dnsaddr/va1.bootstrap.libp2p.io/p2p/12D3KooWKnDdG3iXw9eTFijk3EWSunZcFi54Zka4wmtqtt6rPxc8" BP6="/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" BP7="/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" test_description="Test ipfs bootstrap operations" # NOTE: For AutoConf bootstrap functionality (add default, --expand-auto, etc.) # see test/cli/bootstrap_auto_test.go and test/cli/autoconf/expand_test.go . lib/test-lib.sh test_init_ipfs # we use a function so that we can run it both offline + online test_bootstrap_list_cmd() { printf "" >list_expected for BP in "$@" do echo "$BP" >>list_expected done test_expect_success "'ipfs bootstrap' succeeds" ' ipfs bootstrap >list_actual ' test_expect_success "'ipfs bootstrap' output looks good" ' test_cmp list_expected list_actual ' test_expect_success "'ipfs bootstrap list' succeeds" ' ipfs bootstrap list >list2_actual ' test_expect_success "'ipfs bootstrap list' output looks good" ' test_cmp list_expected list2_actual ' } # we use a function so that we can run it both offline + online test_bootstrap_cmd() { # remove all peers just in case. # if this fails, the first listing may not be empty ipfs bootstrap rm --all test_bootstrap_list_cmd test_expect_success "'ipfs bootstrap add' succeeds" ' ipfs bootstrap add "$BP1" "$BP2" "$BP3" >add_actual ' test_expect_success "'ipfs bootstrap add' output looks good" ' echo "added $BP1" >add_expected && echo "added $BP2" >>add_expected && echo "added $BP3" >>add_expected && test_cmp add_expected add_actual ' test_bootstrap_list_cmd $BP1 $BP2 $BP3 test_expect_success "'ipfs bootstrap rm' succeeds" ' ipfs bootstrap rm "$BP1" "$BP3" >rm_actual ' test_expect_success "'ipfs bootstrap rm' output looks good" ' echo "removed $BP1" >rm_expected && echo "removed $BP3" >>rm_expected && test_cmp rm_expected rm_actual ' test_expect_success "'ipfs bootstrap rm' fails on bad peers" ' test_expect_code 1 ipfs bootstrap rm "foo/bar" ' test_bootstrap_list_cmd $BP2 test_expect_success "'ipfs bootstrap rm --all' succeeds" ' ipfs bootstrap rm --all >rm2_actual ' test_expect_success "'ipfs bootstrap rm' output looks good" ' echo "removed $BP2" >rm2_expected && test_cmp rm2_expected rm2_actual ' test_bootstrap_list_cmd test_expect_success "'ipfs bootstrap add' accepts args from stdin" ' echo $BP1 > bpeers && echo $BP2 >> bpeers && echo $BP3 >> bpeers && echo $BP4 >> bpeers && cat bpeers | ipfs bootstrap add > add_stdin_actual ' test_expect_success "output looks good" ' echo "added $BP1" > bpeers_add_exp && echo "added $BP2" >> bpeers_add_exp && echo "added $BP3" >> bpeers_add_exp && echo "added $BP4" >> bpeers_add_exp && test_cmp add_stdin_actual bpeers_add_exp ' test_bootstrap_list_cmd $BP1 $BP2 $BP3 $BP4 test_expect_success "'ipfs bootstrap rm' accepts args from stdin" ' cat bpeers | ipfs bootstrap rm > rm_stdin_actual ' test_expect_success "output looks good" ' echo "removed $BP1" > bpeers_rm_exp && echo "removed $BP2" >> bpeers_rm_exp && echo "removed $BP3" >> bpeers_rm_exp && echo "removed $BP4" >> bpeers_rm_exp && test_cmp rm_stdin_actual bpeers_rm_exp ' test_bootstrap_list_cmd } # should work offline test_bootstrap_cmd # should work online test_launch_ipfs_daemon test_bootstrap_cmd test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0121-bootstrap-iptb.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2016 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # # changing the bootstrap peers will require changing it in two places :) test_description="test node bootstrapping" . lib/test-lib.sh test_init_ipfs test_expect_success "disable mdns" ' ipfs config Discovery.MDNS.Enabled false --json ' test_launch_ipfs_daemon test_expect_success "setup iptb nodes" ' iptb testbed create -type localipfs -count 5 -force -init ' test_expect_success "start up iptb nodes" ' iptb start -wait ' test_expect_success "check peers works" ' ipfs swarm peers >peers_out ' test_expect_success "correct number of peers" ' test -z "`cat peers_out`" ' betterwait() { while kill -0 $1; do true; done } test_expect_success "bring down iptb nodes" ' PID0=$(cat "$IPTB_ROOT/testbeds/default/0/daemon.pid") && PID1=$(cat "$IPTB_ROOT/testbeds/default/1/daemon.pid") && PID2=$(cat "$IPTB_ROOT/testbeds/default/2/daemon.pid") && PID3=$(cat "$IPTB_ROOT/testbeds/default/3/daemon.pid") && PID4=$(cat "$IPTB_ROOT/testbeds/default/4/daemon.pid") && iptb stop && # TODO: add --wait flag to iptb stop betterwait $PID0 betterwait $PID1 betterwait $PID2 betterwait $PID3 betterwait $PID4 ' test_expect_success "reset iptb nodes" ' # the api does not seem to get cleaned up in sharness tests for some reason iptb testbed create -type localipfs -count 5 -force -init ' test_expect_success "set bootstrap addrs" ' bsn_peer_id=$(ipfs id -f "") && BADDR="/ip4/127.0.0.1/tcp/$SWARM_PORT/p2p/$bsn_peer_id" && ipfsi 0 bootstrap add $BADDR && ipfsi 1 bootstrap add $BADDR && ipfsi 2 bootstrap add $BADDR && ipfsi 3 bootstrap add $BADDR && ipfsi 4 bootstrap add $BADDR ' test_expect_success "start up iptb nodes" ' iptb start -wait ' test_expect_success "check peers works" ' ipfs swarm peers > peers_out ' test_expect_success "correct number of peers" ' test `cat peers_out | wc -l` = 5 ' test_kill_ipfs_daemon test_expect_success "bring down iptb nodes" ' iptb stop ' test_done ================================================ FILE: test/sharness/t0131-multinode-client-routing.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test client mode dht" . lib/test-lib.sh check_file_fetch() { node=$1 fhash=$2 fname=$3 test_expect_success "can fetch file" ' ipfsi $node cat $fhash > fetch_out ' test_expect_success "file looks good" ' test_cmp $fname fetch_out ' } run_single_file_test() { test_expect_success "add a file on node1" ' random-data -size=1000000 > filea && FILEA_HASH=$(ipfsi 1 add -q filea) ' check_file_fetch 9 $FILEA_HASH filea check_file_fetch 8 $FILEA_HASH filea check_file_fetch 7 $FILEA_HASH filea check_file_fetch 6 $FILEA_HASH filea check_file_fetch 5 $FILEA_HASH filea check_file_fetch 4 $FILEA_HASH filea check_file_fetch 3 $FILEA_HASH filea check_file_fetch 2 $FILEA_HASH filea check_file_fetch 1 $FILEA_HASH filea check_file_fetch 0 $FILEA_HASH filea } NNODES=10 test_expect_success "set up testbed" ' iptb testbed create -type localipfs -count $NNODES -force -init && iptb run -- ipfs config --json "Routing.LoopbackAddressesOnLanDHT" true ' test_expect_success "start up nodes" ' iptb start -wait [0-7] && iptb start -wait [8-9] -- --routing=dhtclient ' test_expect_success "connect up nodes" ' iptb connect [1-9] 0 ' test_expect_success "add a file on a node in client mode" ' random-data -size=1000000 > filea && FILE_HASH=$(ipfsi 8 add -q filea) ' test_expect_success "retrieve that file on a node in client mode" ' check_file_fetch 9 $FILE_HASH filea ' run_single_file_test test_expect_success "shut down nodes" ' iptb stop ' test_done ================================================ FILE: test/sharness/t0140-swarm.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test ipfs swarm command" . lib/test-lib.sh test_init_ipfs test_launch_ipfs_daemon test_expect_success 'disconnected: peers is empty' ' ipfs swarm peers >actual && test_must_be_empty actual ' test_expect_success 'disconnected: addrs local has localhost' ' ipfs swarm addrs local >actual && grep "/ip4/127.0.0.1" actual ' test_expect_success 'disconnected: addrs local matches ipfs id' ' ipfs id -f="\\n" | sort >expected && ipfs swarm addrs local --id | sort >actual && test_cmp expected actual ' test_expect_success "ipfs id self works" ' myid=$(ipfs id -f="") && ipfs id --timeout=1s $myid > output ' test_expect_success "output looks good" ' grep $myid output && grep PublicKey output ' addr="/ip4/127.0.0.1/tcp/9898/p2p/QmUWKoHbjsqsSMesRC2Zoscs8edyFz6F77auBB1YBBhgpX" test_expect_success "can't trigger a dial backoff with swarm connect" ' test_expect_code 1 ipfs swarm connect $addr 2> connect_out test_expect_code 1 ipfs swarm connect $addr 2>> connect_out test_expect_code 1 ipfs swarm connect $addr 2>> connect_out test_expect_code 1 grep "backoff" connect_out ' test_kill_ipfs_daemon announceCfg='["/ip4/127.0.0.1/tcp/4001", "/ip4/1.2.3.4/tcp/1234"]' test_expect_success "test_config_set succeeds" " ipfs config --json Addresses.Announce '$announceCfg' " test_launch_ipfs_daemon test_expect_success 'Addresses.Announce affects addresses' ' ipfs swarm addrs local >actual && test_should_contain "/ip4/1.2.3.4/tcp/1234" actual && ipfs id -f"" | xargs -n1 echo >actual && test_should_contain "/ip4/1.2.3.4/tcp/1234" actual ' test_kill_ipfs_daemon announceCfg='["/ip4/127.0.0.1/tcp/4001", "/ip4/1.2.3.4/tcp/1234"]' test_expect_success "test_config_set succeeds" " ipfs config --json Addresses.Announce '$announceCfg' " # Include "/ip4/1.2.3.4/tcp/1234" to ensure we deduplicate addrs already present in Swarm.Announce appendAnnounceCfg='["/dnsaddr/dynamic.example.com", "/ip4/10.20.30.40/tcp/4321", "/ip4/1.2.3.4/tcp/1234"]' test_expect_success "test_config_set Announce and AppendAnnounce succeeds" " ipfs config --json Addresses.Announce '$announceCfg' && ipfs config --json Addresses.AppendAnnounce '$appendAnnounceCfg' " test_launch_ipfs_daemon test_expect_success 'Addresses.AppendAnnounce is applied on top of Announce' ' ipfs swarm addrs local >actual && test_should_contain "/ip4/1.2.3.4/tcp/1234" actual && test_should_contain "/dnsaddr/dynamic.example.com" actual && test_should_contain "/ip4/10.20.30.40/tcp/4321" actual && ipfs id -f"" | xargs -n1 echo | tee actual && test_should_contain "/ip4/1.2.3.4/tcp/1234/p2p" actual && test_should_contain "/dnsaddr/dynamic.example.com/p2p/" actual && test_should_contain "/ip4/10.20.30.40/tcp/4321/p2p/" actual ' test_kill_ipfs_daemon noAnnounceCfg='["/ip4/1.2.3.4/tcp/1234", "/ip4/10.20.30.40/tcp/4321"]' test_expect_success "test_config_set succeeds" " ipfs config --json Addresses.NoAnnounce '$noAnnounceCfg' " test_launch_ipfs_daemon test_expect_success "Addresses.NoAnnounce affects addresses from Announce and AppendAnnounce" ' ipfs swarm addrs local >actual && test_should_not_contain "/ip4/1.2.3.4/tcp/1234" actual && test_should_not_contain "/ip4/10.20.30.40/tcp/4321" actual && ipfs id -f"" | xargs -n1 echo >actual && test_should_not_contain "/ip4/1.2.3.4/tcp/1234" actual && test_should_not_contain "/ip4/10.20.30.40/tcp/4321" actual ' test_kill_ipfs_daemon noAnnounceCfg='["/ip4/1.2.3.4/ipcidr/16"]' test_expect_success "test_config_set succeeds" " ipfs config --json Addresses.NoAnnounce '$noAnnounceCfg' " test_launch_ipfs_daemon test_expect_success "Addresses.NoAnnounce with /ipcidr affects addresses" ' ipfs swarm addrs local >actual && test_should_not_contain "/ip4/1.2.3.4/tcp/1234" actual && ipfs id -f"" | xargs -n1 echo >actual && test_should_not_contain "/ip4/1.2.3.4/tcp/1234" actual ' test_kill_ipfs_daemon test_launch_ipfs_daemon test_expect_success "'ipfs swarm peering ls' lists peerings" ' ipfs swarm peering ls ' peeringID='QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N' peeringID2='QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5K' peeringAddr='/ip4/1.2.3.4/tcp/1234/p2p/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N' peeringAddr2='/ip4/1.2.3.4/tcp/1234/p2p/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5K' test_expect_success "'ipfs swarm peering add' adds a peering" ' ipfs swarm peering ls > peeringls && ! test_should_contain ${peeringID} peeringls && ! test_should_contain ${peeringID2} peeringls && ipfs swarm peering add ${peeringAddr} ${peeringAddr2} ' test_expect_success 'a peering is added' ' ipfs swarm peering ls > peeringadd && test_should_contain ${peeringID} peeringadd && test_should_contain ${peeringID2} peeringadd ' test_expect_success "'swarm peering rm' removes a peering" ' ipfs swarm peering rm ${peeringID} ' test_expect_success 'peering is removed' ' ipfs swarm peering ls > peeringrm && ! test_should_contain ${peeringID} peeringrm ' test_kill_ipfs_daemon test_expect_success "set up tcp testbed" ' iptb testbed create -type localipfs -count 2 -force -init ' startup_cluster 2 test_expect_success "disconnect work without specifying a transport address" ' [ $(ipfsi 0 swarm peers | wc -l) -eq 1 ] && ipfsi 0 swarm disconnect "/p2p/$(iptb attr get 1 id)" && [ $(ipfsi 0 swarm peers | wc -l) -eq 0 ] ' test_expect_success "connect work without specifying a transport address" ' [ $(ipfsi 0 swarm peers | wc -l) -eq 0 ] && ipfsi 0 swarm connect "/p2p/$(iptb attr get 1 id)" && [ $(ipfsi 0 swarm peers | wc -l) -eq 1 ] ' test_expect_success "/p2p addresses work" ' [ $(ipfsi 0 swarm peers | wc -l) -eq 1 ] && ipfsi 0 swarm disconnect "/p2p/$(iptb attr get 1 id)" && [ $(ipfsi 0 swarm peers | wc -l) -eq 0 ] && ipfsi 0 swarm connect "/p2p/$(iptb attr get 1 id)" && [ $(ipfsi 0 swarm peers | wc -l) -eq 1 ] ' test_expect_success "ipfs id is consistent for node 0" ' ipfsi 1 id "$(iptb attr get 0 id)" > 1see0 && ipfsi 0 id > 0see0 && test_cmp 1see0 0see0 ' test_expect_success "ipfs id is consistent for node 1" ' ipfsi 0 id "$(iptb attr get 1 id)" > 0see1 && ipfsi 1 id > 1see1 && test_cmp 0see1 1see1 ' test_expect_success "addresses contain /p2p/..." ' test_should_contain "/p2p/$(iptb attr get 1 id)\"" 0see1 && test_should_contain "/p2p/$(iptb attr get 1 id)\"" 1see1 && test_should_contain "/p2p/$(iptb attr get 0 id)\"" 1see0 && test_should_contain "/p2p/$(iptb attr get 0 id)\"" 0see0 ' test_expect_success "stopping cluster" ' iptb stop ' test_done ================================================ FILE: test/sharness/t0141-addfilter.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test ipfs swarm command" AF1="/ip4/192.168.0.0/ipcidr/16" AF2="/ip4/127.0.0.0/ipcidr/8" AF3="/ip6/2008:bcd::/ipcidr/32" AF4="/ip4/172.16.0.0/ipcidr/12" . lib/test-lib.sh test_init_ipfs test_swarm_filter_cmd() { printf "" > list_expected for AF in "$@" do echo "$AF" >>list_expected done test_expect_success "'ipfs swarm filters' succeeds" ' ipfs swarm filters > list_actual ' test_expect_success "'ipfs swarm filters' output looks good" ' test_sort_cmp list_expected list_actual ' } test_config_swarm_addrfilters_cmd() { printf "" > list_expected for AF in "$@" do echo "$AF" >>list_expected done test_expect_success "'ipfs config Swarm.AddrFilters' succeeds" ' ipfs config Swarm.AddrFilters > list_actual ' printf "" > list_actual_cleaned if [ "$( cat list_actual )" != "[]" -a "$( cat list_actual )" != "null" ]; then grep -v "^\]" list_actual | grep -v "^\[" | tr -d '" ,' > list_actual_cleaned fi test_expect_success "'ipfs config Swarm.AddrFilters' output looks good" ' test_sort_cmp list_expected list_actual_cleaned ' } test_swarm_filters() { # expect first address from config test_swarm_filter_cmd $AF1 $AF4 test_config_swarm_addrfilters_cmd $AF1 $AF4 ipfs swarm filters rm all test_swarm_filter_cmd test_config_swarm_addrfilters_cmd test_expect_success "'ipfs swarm filter add' succeeds" ' ipfs swarm filters add $AF1 $AF2 $AF3 ' test_swarm_filter_cmd $AF1 $AF2 $AF3 test_config_swarm_addrfilters_cmd $AF1 $AF2 $AF3 test_expect_success "'ipfs swarm filter rm' succeeds" ' ipfs swarm filters rm $AF2 $AF3 ' test_swarm_filter_cmd $AF1 test_config_swarm_addrfilters_cmd $AF1 test_expect_success "'ipfs swarm filter add' succeeds" ' ipfs swarm filters add $AF4 $AF2 ' test_swarm_filter_cmd $AF1 $AF2 $AF4 test_config_swarm_addrfilters_cmd $AF1 $AF2 $AF4 test_expect_success "'ipfs swarm filter rm' succeeds" ' ipfs swarm filters rm $AF1 $AF2 $AF4 ' test_swarm_filter_cmd test_config_swarm_addrfilters_cmd } test_expect_success "init without any filters" ' echo "null" >expected && ipfs config Swarm.AddrFilters >actual && test_cmp expected actual ' test_expect_success "adding addresses to the config to filter succeeds" ' ipfs config --json Swarm.AddrFilters "[\"$AF1\", \"$AF4\"]" ' test_launch_ipfs_daemon test_swarm_filters test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0142-testfilter.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2020 Protocol Labs # MIT/Apache-2.0 Licensed; see the LICENSE file in this repository. # test_description="Test swarm filters are effective" AF="/ip4/127.0.0.0/ipcidr/24" . lib/test-lib.sh NUM_NODES=3 test_expect_success "set up testbed" ' iptb testbed create -type localipfs -count $NUM_NODES -force -init && iptb run -- ipfs config --json "Routing.LoopbackAddressesOnLanDHT" true ' test_expect_success 'filter 127.0.0.0/24 on node 1' ' ipfsi 1 config --json Swarm.AddrFilters "[\"$AF\"]" ' for i in $(seq 0 $(( NUM_NODES - 1 ))); do test_expect_success "change IP for node $i" ' ipfsi $i config --json "Addresses.Swarm" \ "[\"/ip4/127.0.$i.1/tcp/0\",\"/ip4/127.0.$i.1/udp/0/quic\",\"/ip4/127.0.$i.1/tcp/0/ws\"]" ' done test_expect_success 'start cluster' ' iptb start --wait ' test_expect_success 'connecting 1 to 0 fails' ' test_must_fail iptb connect 1 0 ' test_expect_success 'connecting 0 to 1 fails' ' test_must_fail iptb connect 1 0 ' test_expect_success 'connecting 2 to 0 succeeds' ' iptb connect 2 0 ' test_expect_success 'connecting 1 to 0 with dns addrs fails' ' ipfsi 0 id -f "" | sed "s|^/ip4/127.0.0.1/|/dns4/localhost/|" > addrs && test_must_fail ipfsi 1 swarm connect $(cat addrs) ' test_expect_success 'stopping cluster' ' iptb stop ' test_done ================================================ FILE: test/sharness/t0150-clisuggest.sh ================================================ #!/usr/bin/env bash test_description="Test ipfs cli cmd suggest" . lib/test-lib.sh test_suggest() { test_expect_success "test command fails" ' test_must_fail ipfs kog 2>actual ' test_expect_success "test one command is suggested" ' grep "Did you mean this?" actual && grep "log" actual || test_fsh cat actual ' test_expect_success "test command fails" ' test_must_fail ipfs li 2>actual ' test_expect_success "test multiple commands are suggested" ' grep "Did you mean any of these?" actual && grep "ls" actual && grep "log" actual || test_fsh cat actual ' } test_init_ipfs test_suggest test_launch_ipfs_daemon test_suggest test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0151-sysdiag.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="test output of sysdiag command" . lib/test-lib.sh test_init_ipfs test_expect_success "ipfs diag sys succeeds" ' ipfs diag sys > output ' test_expect_success "output contains some expected keys" ' grep "virt" output && grep "interface_addresses" output && grep "arch" output && grep "online" output ' test_expect_success "uname succeeds" ' UOUT=$(uname) ' test_expect_success "output is similar to uname" ' case $UOUT in Linux) grep linux output > /dev/null ;; Darwin) grep darwin output > /dev/null ;; FreeBSD) grep freebsd output > /dev/null ;; CYGWIN*) grep windows output > /dev/null ;; *) test_fsh echo system check for $UOUT failed, unsupported system? ;; esac ' test_done ================================================ FILE: test/sharness/t0152-profile.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2016 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test profile collection" . lib/test-lib.sh test_init_ipfs test_expect_success "profiling requires a running daemon" ' test_must_fail ipfs diag profile ' test_launch_ipfs_daemon test_expect_success "test profiling (without sampling)" ' ipfs diag profile --profile-time=0 > cmd_out ' test_expect_success "filename shows up in output" ' grep -q "ipfs-profile" cmd_out > /dev/null ' test_expect_success "profile file created" ' test -e "$(sed -n -e "s/.*\(ipfs-profile.*\.zip\)/\1/p" cmd_out)" ' test_expect_success "test profiling with -o" ' ipfs diag profile --profile-time=1s -o test-profile.zip ' test_expect_success "test that test-profile.zip exists" ' test -e test-profile.zip ' test_expect_success "test profiling with specific collectors" ' ipfs diag profile --collectors version,goroutines-stack -o test-profile-small.zip ' test_kill_ipfs_daemon if ! test_have_prereq UNZIP; then test_done fi test_expect_success "unpack profiles" ' unzip -d profiles test-profile.zip && unzip -d profiles-small test-profile-small.zip ' test_expect_success "cpu profile is valid" ' go tool pprof -top profiles/ipfs "profiles/cpu.pprof" | grep -q "Type: cpu" ' test_expect_success "heap profile is valid" ' go tool pprof -top profiles/ipfs "profiles/heap.pprof" | grep -q "Type: inuse_space" ' test_expect_success "goroutines profile is valid" ' go tool pprof -top profiles/ipfs "profiles/goroutines.pprof" | grep -q "Type: goroutine" ' test_expect_success "mutex profile is valid" ' go tool pprof -top profiles/ipfs "profiles/mutex.pprof" | grep -q "Type: delay" ' test_expect_success "block profile is valid" ' go tool pprof -top profiles/ipfs "profiles/block.pprof" | grep -q "Type: delay" ' test_expect_success "goroutines stacktrace is valid" ' grep -q "goroutine" "profiles/goroutines.stacks" ' test_expect_success "the small profile only contains the requested data" ' find profiles-small -type f | sort > actual && echo -e "profiles-small/goroutines.stacks\nprofiles-small/version.json" > expected && test_cmp expected actual ' test_done ================================================ FILE: test/sharness/t0160-resolve.sh ================================================ #!/usr/bin/env bash test_description="Test resolve command" . lib/test-lib.sh test_init_ipfs test_expect_success "resolve: prepare files" ' mkdir -p a/b && echo "a/b/c" >a/b/c && a_hash=$(ipfs add -Q -r a) && b_hash=$(ipfs add -Q -r a/b) && c_hash=$(ipfs add -Q -r a/b/c) && a_hash_b32=$(cid-fmt -v 1 -b b %s $a_hash) && b_hash_b32=$(cid-fmt -v 1 -b b %s $b_hash) && c_hash_b32=$(cid-fmt -v 1 -b b %s $c_hash) ' test_expect_success "resolve: prepare dag" ' dag_hash=$(ipfs dag put <<<"{\"i\": {\"j\": {\"k\": \"asdfasdfasdf\"}}}") ' test_expect_success "resolve: prepare keys" ' self_hash=$(ipfs key list --ipns-base=base36 -l | grep self | cut -d " " -f1) && alt_hash=$(ipfs key gen --ipns-base=base36 -t rsa alt) ' test_resolve_setup_name() { local key="$1" local ref="$2" # we pass here --ttl=0s to ensure that it does not get cached by namesys. # the alternative would be to wait between tests to ensure that the namesys # cache gets purged in time, but that adds runtime time for the tests. test_expect_success "resolve: prepare $key" ' ipfs name publish --key="$key" --ttl=0s --allow-offline "$ref" ' } test_resolve() { src=$1 dst=$2 extra=$3 test_expect_success "resolve succeeds: $src" ' ipfs resolve $extra "$src" >actual ' test_expect_success "resolved correctly: $src -> $dst" ' printf "$dst\n" >expected && test_cmp expected actual ' } test_resolve_cmd() { echo '-- starting test_resolve_cmd' test_resolve "/ipfs/$a_hash" "/ipfs/$a_hash" test_resolve "/ipfs/$a_hash/b" "/ipfs/$b_hash" test_resolve "/ipfs/$a_hash/b/c" "/ipfs/$c_hash" test_resolve "/ipfs/$b_hash/c" "/ipfs/$c_hash" test_resolve "/ipld/$dag_hash/i/j/k" "/ipld/$dag_hash/i/j/k" test_resolve "/ipld/$dag_hash/i/j" "/ipld/$dag_hash/i/j" test_resolve "/ipld/$dag_hash/i" "/ipld/$dag_hash/i" test_resolve_setup_name "self" "/ipfs/$a_hash" test_resolve "/ipns/$self_hash" "/ipfs/$a_hash" test_resolve "/ipns/$self_hash/b" "/ipfs/$b_hash" test_resolve "/ipns/$self_hash/b/c" "/ipfs/$c_hash" test_resolve_setup_name "self" "/ipfs/$b_hash" test_resolve "/ipns/$self_hash" "/ipfs/$b_hash" test_resolve "/ipns/$self_hash/c" "/ipfs/$c_hash" test_resolve_setup_name "self" "/ipfs/$c_hash" test_resolve "/ipns/$self_hash" "/ipfs/$c_hash" # simple recursion succeeds test_resolve_setup_name "alt" "/ipns/$self_hash" test_resolve "/ipns/$alt_hash" "/ipfs/$c_hash" # partial resolve succeeds test_resolve "/ipns/$alt_hash" "/ipns/$self_hash" -r=false # infinite recursion fails test_resolve_setup_name "self" "/ipns/$self_hash" test_expect_success "recursive resolve terminates" ' test_expect_code 1 ipfs resolve /ipns/$self_hash 2>recursion_error && grep "recursion limit exceeded" recursion_error ' } test_resolve_cmd_b32() { echo '-- starting test_resolve_cmd_b32' # no flags needed, base should be preserved test_resolve "/ipfs/$a_hash_b32" "/ipfs/$a_hash_b32" test_resolve "/ipfs/$a_hash_b32/b" "/ipfs/$b_hash_b32" test_resolve "/ipfs/$a_hash_b32/b/c" "/ipfs/$c_hash_b32" test_resolve "/ipfs/$b_hash_b32/c" "/ipfs/$c_hash_b32" # flags needed passed in path does not contain cid to derive base test_resolve_setup_name "self" "/ipfs/$a_hash_b32" test_resolve "/ipns/$self_hash" "/ipfs/$a_hash_b32" --cid-base=base32 test_resolve "/ipns/$self_hash/b" "/ipfs/$b_hash_b32" --cid-base=base32 test_resolve "/ipns/$self_hash/b/c" "/ipfs/$c_hash_b32" --cid-base=base32 test_resolve_setup_name "self" "/ipfs/$b_hash_b32" --cid-base=base32 test_resolve "/ipns/$self_hash" "/ipfs/$b_hash_b32" --cid-base=base32 test_resolve "/ipns/$self_hash/c" "/ipfs/$c_hash_b32" --cid-base=base32 test_resolve_setup_name "self" "/ipfs/$c_hash_b32" test_resolve "/ipns/$self_hash" "/ipfs/$c_hash_b32" --cid-base=base32 # peer ID represented as CIDv1 require libp2p-key multicodec # https://github.com/libp2p/specs/blob/master/RFC/0001-text-peerid-cid.md local self_hash_b32protobuf=$(echo $self_hash | ipfs cid format -v 1 -b b --mc dag-pb) local self_hash_b32libp2pkey=$(echo $self_hash | ipfs cid format -v 1 -b b --mc libp2p-key) test_expect_success "resolve of /ipns/{cidv1} with multicodec other than libp2p-key returns a meaningful error" ' test_expect_code 1 ipfs resolve /ipns/$self_hash_b32protobuf 2>cidcodec_error && test_should_contain "Error: peer ID represented as CIDv1 require libp2p-key multicodec: retry with /ipns/$self_hash_b32libp2pkey" cidcodec_error ' } test_resolve_cmd_success() { test_resolve "/ipfs/$a_hash" "/ipfs/$a_hash" test_resolve "/ipfs/$a_hash/b" "/ipfs/$b_hash" test_resolve "/ipfs/$a_hash/b/c" "/ipfs/$c_hash" test_resolve "/ipfs/$b_hash/c" "/ipfs/$c_hash" test_resolve "/ipld/$dag_hash" "/ipld/$dag_hash" test_resolve "/ipld/$dag_hash/i/j/k" "/ipld/$dag_hash/i/j/k" test_resolve "/ipld/$dag_hash/i/j" "/ipld/$dag_hash/i/j" test_resolve "/ipld/$dag_hash/i" "/ipld/$dag_hash/i" test_resolve_setup_name "self" "/ipfs/$a_hash" test_resolve "/ipns/$self_hash" "/ipfs/$a_hash" test_resolve "/ipns/$self_hash/b" "/ipfs/$b_hash" test_resolve "/ipns/$self_hash/b/c" "/ipfs/$c_hash" test_resolve_setup_name "self" "/ipfs/$b_hash" test_resolve "/ipns/$self_hash" "/ipfs/$b_hash" test_resolve "/ipns/$self_hash/c" "/ipfs/$c_hash" test_resolve_setup_name "self" "/ipfs/$c_hash" test_resolve "/ipns/$self_hash" "/ipfs/$c_hash" } # should work offline test_resolve_cmd test_resolve_cmd_b32 # should work online test_launch_ipfs_daemon test_resolve_cmd_success test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0165-keystore-data/README.md ================================================ # OpenSSL generated keys for import/export tests Created with commands: ```bash openssl genpkey -algorithm ED25519 > openssl_ed25519.pem openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 > openssl_rsa.pem ``` secp key used in the 'restrict import key' test. From: https://docs.openssl.org/1.1.1/man1/genpkey/ ```bash openssl genpkey -genparam -algorithm EC -out ecp.pem \ -pkeyopt ec_paramgen_curve:secp384r1 \ -pkeyopt ec_param_enc:named_curve openssl genpkey -paramfile ecp.pem -out openssl_secp384r1.pem rm ecp.pem ``` Note: The Bitcoin `secp256k1` curve which is what `go-libp2p-core/crypto` actually generates and would be of interest to test against is not recognized by the Go library: ``` Error: parsing PKCS8 format: x509: failed to parse EC private key embedded in PKCS#8: x509: unknown elliptic curve ``` We keep the `secp384r1` type instead from the original openssl example. ================================================ FILE: test/sharness/t0165-keystore-data/openssl_ed25519.pem ================================================ -----BEGIN PRIVATE KEY----- MC4CAQAwBQYDK2VwBCIEIJ2M1na2f3dRm4b1FcAQvsn7q08+XfBZcr4MgH4yiBdz -----END PRIVATE KEY----- ================================================ FILE: test/sharness/t0165-keystore-data/openssl_rsa.pem ================================================ -----BEGIN PRIVATE KEY----- MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDSaJB9EKnShOs6 sbGkB40crn72yNKXj5OBPS2wBDTHWwxyhTB0qJirOT2QYW2DmR/4lPfVk5/f4CJ7 xIHUBJRoC+NTwqHit24DQBd00tNG4EnKn2Dad/arZ/nEVshkKiGXn0qXxiHHsaCn X/pnVPU4+O7fdfUlz2EKf3Og/ocRCFrdMsULR2QwDc0YWsY8ngrcKegyFCbKjXjo zvfbGevCDPlhKaZLxRy0PHnON00YC4KO6d77XpbECFvsE1aG1RxYQX0Zjr+i8UvD UJp/YCoRNEX54/wKpGebMUrFse5K9hBsFen/wCsPnOsYPSb9g8qyoYRDBnr9sIe1 9MxFTMy/AgMBAAECggEAKXu2KQI1CS1tlzfbdySJ/MKmg49afckv4sYmENLzeO6J iLabtBRdbTyu151t0wlIlWEBb9lYJvJwuggnNJ7mh5D4c9YmxqU1imyDc2PxhcLI qas8lDYcqvSn+L7HaYAo+VTNhxjoJg/uRbGVk/PbGS1zIxmFiLvXPROdv3sPNBsf EYMDH9q7/8DI6dNBQPxtTKlTDLDsTezbkNFQ74znlXgQYcfY1mXljcRtbJqhQJT3 uppktESPwLRmqtT9H+v9nCtQR6OLmAmLWNgMrSdGKBsSsgJwv2xfpNMffwd84dtT uGrS2K+BY0TH2q+Xx04r18GLCst3U5MBSklyHQ/mwQKBgQDqnxNOnK41/n/Q8X4a /TUnZBx/JHiCoQoa06AsMxFgOvV3ycR+Z9lwb5I5BsicH1GUcHIxSY3mCyd4fLwE FC0QIyNhPJ5oFKh0Oynjm+79VE8v7kK2qqRL4zUpaCXEsSOrhRsCY0/WQdMUPVsh okXDUIv37G9KUcjdrhNVpGK3oQKBgQDllK7augIhmlQZTdSLTgmuzhYsXdSGDML/ Bx48q7OvPhvZIIOsygLGhtcBk2xG6PN1yP44cx9dvcTnzxU6TEblO5P8TWY0BSNj ZuC5wdxLwc3KUdLd9JLR7qcbjqndDruE01rQFVQ3MDbyB1+VrJgiVHIEomJJrKGm FQ+314moXwKBgQDL90sDlnZk/kED1k15DRN+kSus5HnXpkRwmfWvNx4t+FOZtdCa y5Fei8Akz17rStbTIwZDDtzLVnsT5exV52xdkQ6a4+YaOYtQsHZ0JwWXOgo1cv6Q ary2NGns+1uKKS0HWYnng4rOix8Dg2uMS9Q2PfnQqLz/cSYcgc7RLz2awQKBgQDd HSaLYztKQeldtahPwwlwYuzYLkbSFNh559EnfffBgIAxzy8C7E1gB95sliBi61oQ x1SR6c776hoLaVd4np5picgt6B3XXFuJETy/rAcQr8gUZFpDi5sctk4cLHtNfTL9 6tI8N061GKrS0GcvMNwVtF9cN0mSy8GkxAQvfFgI4QKBgQC4NVimIPptfFckulAL /t0vkdLhCRr1+UFNhgsQJhCZpfWZK4x8If6Jru/eiU7ywEsL6fHE2ENvyoTjV33g b9yJ7SV4zkz4VhBxc3p26SIvBgLqtHwH8IkIonlbfQFoEAg1iOneLvimPy0YGHsG +bTwwlAJJhctILkFtAbooeAQVQ== -----END PRIVATE KEY----- ================================================ FILE: test/sharness/t0165-keystore-data/openssl_secp384r1.pem ================================================ -----BEGIN PRIVATE KEY----- MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDC8DksLZwPKGQS8tuWI w+dNYiSHUyw30NkrK9YGmjgp84sVVa5NGrv0QniAnNWG1DqhZANiAATq0d5KV1MF IIpF4beNX+YsmFdqB2oDLhznO/4xNsFCFKE39oGQmuMwnQhDNZZ2CQA8csfgZmuF OSooe/Ru6ubyeGVKafcHJrOMBvl8hIg0tVWgIAhuXiTHq0UL0QTv9Vk= -----END PRIVATE KEY----- ================================================ FILE: test/sharness/t0165-keystore.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2017 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test keystore commands" . lib/test-lib.sh test_init_ipfs test_key_cmd() { # test key output format test_expect_success "create an RSA key and test B58MH/B36CID output formats" ' PEERID=$(ipfs key gen --ipns-base=b58mh --type=rsa --size=2048 key_rsa) && test_check_rsa2048_b58mh_peerid $PEERID && ipfs key rm key_rsa && PEERID=$(ipfs key gen --ipns-base=base36 --type=rsa --size=2048 key_rsa) && test_check_rsa2048_base36_peerid $PEERID ' test_expect_success "test RSA key sk export format" ' ipfs key export key_rsa && test_check_rsa2048_sk key_rsa.key && rm key_rsa.key ' test_expect_success "test RSA key B58MH/B36CID multihash format" ' PEERID=$(ipfs key list --ipns-base=b58mh -l | grep key_rsa | head -n 1 | cut -d " " -f1) && test_check_rsa2048_b58mh_peerid $PEERID && PEERID=$(ipfs key list --ipns-base=base36 -l | grep key_rsa | head -n 1 | cut -d " " -f1) && test_check_rsa2048_base36_peerid $PEERID && ipfs key rm key_rsa ' test_expect_success "create an ED25519 key and test B58MH/B36CID output formats" ' PEERID=$(ipfs key gen --ipns-base=b58mh --type=ed25519 key_ed25519) && test_check_ed25519_b58mh_peerid $PEERID && ipfs key rm key_ed25519 && PEERID=$(ipfs key gen --ipns-base=base36 --type=ed25519 key_ed25519) && test_check_ed25519_base36_peerid $PEERID ' test_expect_success "test ED25519 key sk export format" ' ipfs key export key_ed25519 && test_check_ed25519_sk key_ed25519.key && rm key_ed25519.key ' test_expect_success "test ED25519 key B58MH/B36CID multihash format" ' PEERID=$(ipfs key list --ipns-base=b58mh -l | grep key_ed25519 | head -n 1 | cut -d " " -f1) && test_check_ed25519_b58mh_peerid $PEERID && PEERID=$(ipfs key list --ipns-base=base36 -l | grep key_ed25519 | head -n 1 | cut -d " " -f1) && test_check_ed25519_base36_peerid $PEERID && ipfs key rm key_ed25519 ' # end of format test test_expect_success "create a new rsa key" ' rsahash=$(ipfs key gen generated_rsa_key --type=rsa --size=2048) echo $rsahash > rsa_key_id ' test_key_import_export_all_formats rsa_key test_expect_success "create a new ed25519 key" ' edhash=$(ipfs key gen generated_ed25519_key --type=ed25519) echo $edhash > ed25519_key_id ' test_key_import_export_all_formats ed25519_key test_openssl_compatibility_all_types INVALID_KEY=../t0165-keystore-data/openssl_secp384r1.pem test_expect_success "import key type we don't generate fails" ' test_must_fail ipfs key import restricted-type -f pem-pkcs8-cleartext $INVALID_KEY 2>&1 | tee key_exp_out && grep -q "Error: key type \*crypto.ECDSAPrivateKey is not allowed to be imported" key_exp_out && rm key_exp_out ' test_expect_success "import key type we don't generate succeeds with flag" ' ipfs key import restricted-type --allow-any-key-type -f pem-pkcs8-cleartext $INVALID_KEY > /dev/null && ipfs key rm restricted-type ' test_expect_success "test export file option" ' ipfs key export generated_rsa_key -o=named_rsa_export_file && test_cmp generated_rsa_key.key named_rsa_export_file && ipfs key export generated_ed25519_key -o=named_ed25519_export_file && test_cmp generated_ed25519_key.key named_ed25519_export_file ' test_expect_success "key export can't export self" ' test_must_fail ipfs key export self 2>&1 | tee key_exp_out && grep -q "Error: cannot export key with name" key_exp_out && test_must_fail ipfs key export self -o=selfexport 2>&1 | tee key_exp_out && grep -q "Error: cannot export key with name" key_exp_out ' test_expect_success "key import can't import self" ' ipfs key gen overwrite_self_import && ipfs key export overwrite_self_import && test_must_fail ipfs key import self overwrite_self_import.key 2>&1 | tee key_imp_out && grep -q "Error: cannot import key with name" key_imp_out && ipfs key rm overwrite_self_import && rm overwrite_self_import.key ' test_expect_success "add a default key" ' ipfs key gen quxel ' test_expect_success "all keys show up in list output" ' echo generated_ed25519_key > list_exp && echo generated_rsa_key >> list_exp && echo quxel >> list_exp && echo self >> list_exp ipfs key list > list_out && test_sort_cmp list_exp list_out ' test_expect_success "key hashes show up in long list output" ' ipfs key list -l | grep $edhash > /dev/null && ipfs key list -l | grep $rsahash > /dev/null ' test_expect_success "key list -l contains self key with peerID" ' PeerID="$(ipfs config Identity.PeerID)" ipfs key list -l --ipns-base=b58mh | grep "$PeerID\s\+self" ' test_expect_success "key rm remove a key" ' ipfs key rm generated_rsa_key echo generated_ed25519_key > list_exp && echo quxel >> list_exp && echo self >> list_exp ipfs key list > list_out && test_sort_cmp list_exp list_out ' test_expect_success "key rm can't remove self" ' test_must_fail ipfs key rm self 2>&1 | tee key_rm_out && grep -q "Error: cannot remove key with name" key_rm_out ' test_expect_success "key rename rename a key" ' ipfs key rename generated_ed25519_key fooed echo fooed > list_exp && echo quxel >> list_exp && echo self >> list_exp ipfs key list > list_out && test_sort_cmp list_exp list_out ' test_expect_success "key rename rename key output succeeds" ' key_content=$(ipfs key gen key1 --type=rsa --size=2048) && ipfs key rename key1 key2 >rs && echo "Key $key_content renamed to key2" >expect && test_cmp rs expect ' test_expect_success "key rename can't rename self" ' test_must_fail ipfs key rename self bar 2>&1 | tee key_rename_out && grep -q "Error: cannot rename key with name" key_rename_out ' test_expect_success "key rename can't overwrite self, even with force" ' test_must_fail ipfs key rename -f fooed self 2>&1 | tee key_rename_out && grep -q "Error: cannot overwrite key with name" key_rename_out ' test_launch_ipfs_daemon test_expect_success "online import rsa key" ' ipfs key import generated_rsa_key generated_rsa_key.key > roundtrip_rsa_key_id && test_cmp rsa_key_id roundtrip_rsa_key_id ' # export works directly on the keystore present in IPFS_PATH test_expect_success "prepare ed25519 key while daemon is running" ' edhash=$(ipfs key gen generated_ed25519_key --type=ed25519) echo $edhash > ed25519_key_id ' test_key_import_export_all_formats ed25519_key test_openssl_compatibility_all_types test_expect_success "key export over HTTP /api/v0/key/export is not possible" ' ipfs key gen nohttpexporttest_key --type=ed25519 && curl -X POST -sI "http://$API_ADDR/api/v0/key/export&arg=nohttpexporttest_key" | grep -q "^HTTP/1.1 404 Not Found" ' test_expect_success "online rotate rsa key" ' test_must_fail ipfs key rotate ' test_kill_ipfs_daemon } test_check_rsa2048_sk() { sklen=$(ls -l $1 | awk '{print $5}') && test "$sklen" -lt "1600" && test "$sklen" -gt "1000" || { echo "Bad RSA2048 sk '$1' with len '$sklen'" return 1 } } test_check_ed25519_sk() { sklen=$(ls -l $1 | awk '{print $5}') && test "$sklen" -lt "100" && test "$sklen" -gt "30" || { echo "Bad ED25519 sk '$1' with len '$sklen'" return 1 } } test_key_import_export_all_formats() { KEY_NAME=$1 test_key_import_export $KEY_NAME pem-pkcs8-cleartext test_key_import_export $KEY_NAME libp2p-protobuf-cleartext } test_key_import_export() { local KEY_NAME FORMAT KEY_NAME=$1 FORMAT=$2 ORIG_KEY="generated_$KEY_NAME" if [ $FORMAT == "pem-pkcs8-cleartext" ]; then FILE_EXT="pem" else FILE_EXT="key" fi test_expect_success "export and import $KEY_NAME with format $FORMAT" ' ipfs key export $ORIG_KEY --format=$FORMAT && ipfs key rm $ORIG_KEY && ipfs key import $ORIG_KEY $ORIG_KEY.$FILE_EXT --format=$FORMAT > imported_key_id && test_cmp ${KEY_NAME}_id imported_key_id ' } # Test the entire import/export cycle with a openssl-generated key. # 1. Import openssl key with PEM format. # 2. Export key with libp2p format. # 3. Reimport key. # 4. Now exported with PEM format. # 5. Compare with original openssl key. # 6. Clean up. test_openssl_compatibility() { local KEY_NAME FORMAT KEY_NAME=$1 test_expect_success "import and export $KEY_NAME with all formats" ' ipfs key import test-openssl -f pem-pkcs8-cleartext $KEY_NAME > /dev/null && ipfs key export test-openssl -f libp2p-protobuf-cleartext -o $KEY_NAME.libp2p.key && ipfs key rm test-openssl && ipfs key import test-openssl -f libp2p-protobuf-cleartext $KEY_NAME.libp2p.key > /dev/null && ipfs key export test-openssl -f pem-pkcs8-cleartext -o $KEY_NAME.ipfs-exported.pem && ipfs key rm test-openssl && test_cmp $KEY_NAME $KEY_NAME.ipfs-exported.pem && rm $KEY_NAME.libp2p.key && rm $KEY_NAME.ipfs-exported.pem ' } test_openssl_compatibility_all_types() { test_openssl_compatibility ../t0165-keystore-data/openssl_ed25519.pem test_openssl_compatibility ../t0165-keystore-data/openssl_rsa.pem } test_key_cmd test_done ================================================ FILE: test/sharness/t0180-p2p.sh ================================================ #!/usr/bin/env bash test_description="Test experimental p2p commands" . lib/test-lib.sh # start iptb + wait for peering test_expect_success 'init iptb' ' iptb testbed create -type localipfs --count 3 --init ' test_expect_success 'generate test data' ' echo "ABCDEF" > test0.bin && echo "012345" > test1.bin ' startup_cluster 3 test_expect_success 'peer ids' ' PEERID_0=$(iptb attr get 0 id) && PEERID_1=$(iptb attr get 1 id) ' check_test_ports() { test_expect_success "test ports are closed" ' (! (netstat -aln | grep "LISTEN" | grep -E "[.:]10101 ")) && (! (netstat -aln | grep "LISTEN" | grep -E "[.:]10102 ")) && (! (netstat -aln | grep "LISTEN" | grep -E "[.:]10103 ")) && (! (netstat -aln | grep "LISTEN" | grep -E "[.:]10104 ")) ' } check_test_ports test_expect_success 'fail without config option being enabled' ' test_must_fail ipfsi 0 p2p stream ls ' test_expect_success "enable filestore config setting" ' ipfsi 0 config --json Experimental.Libp2pStreamMounting true ipfsi 1 config --json Experimental.Libp2pStreamMounting true ipfsi 2 config --json Experimental.Libp2pStreamMounting true ' test_expect_success 'start p2p listener' ' ipfsi 0 p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log ' test_expect_success 'cannot re-register p2p listener' ' test_must_fail ipfsi 0 p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10103 2>&1 > listener-stdouterr.log ' # Server to client communications spawn_sending_server() { test_expect_success 'S->C Spawn sending server' ' ma-pipe-unidir --listen --pidFile=listener.pid send /ip4/127.0.0.1/tcp/10101 < test0.bin & test_wait_for_file 30 100ms listener.pid && kill -0 $(cat listener.pid) ' } test_server_to_client() { test_expect_success 'S->C Connect and receive data' ' ma-pipe-unidir recv /ip4/127.0.0.1/tcp/10102 > client.out ' test_expect_success 'S->C Ensure server finished' ' test ! -f listener.pid ' test_expect_success 'S->C Output looks good' ' test_cmp client.out test0.bin ' } spawn_sending_server test_expect_success 'S->C(/p2p/peerID) Setup client side' ' ipfsi 1 p2p forward /x/p2p-test /ip4/127.0.0.1/tcp/10102 /p2p/${PEERID_0} 2>&1 > dialer-stdouterr.log ' test_expect_success 'S->C Setup(dnsaddr/addr/p2p/peerID) client side' ' ipfsi 1 p2p forward /x/p2p-test /ip4/127.0.0.1/tcp/10103 /dnsaddr/bootstrap.libp2p.io/p2p/${PEERID_0} 2>&1 > dialer-stdouterr.log ' test_expect_success 'S->C Setup(dnsaddr/addr) client side' ' ipfsi 1 p2p forward /x/p2p-test /ip4/127.0.0.1/tcp/10104 /dnsaddr/example-dnsaddr.multiformats.io 2>&1 > dialer-stdouterr.log ' test_expect_success 'S->C Output is empty' ' test_must_be_empty dialer-stdouterr.log ' test_expect_success "'ipfs p2p ls | grep' succeeds" ' ipfsi 1 p2p ls | grep "/x/p2p-test /ip4/127.0.0.1/tcp/10104" ' test_server_to_client test_expect_success 'S->C Connect with dead server' ' ma-pipe-unidir recv /ip4/127.0.0.1/tcp/10102 > client.out ' test_expect_success 'S->C Output is empty' ' test_must_be_empty client.out ' spawn_sending_server test_server_to_client test_expect_success 'S->C Close local listener' ' ipfsi 1 p2p close -p /x/p2p-test ' check_test_ports # Client to server communications test_expect_success 'C->S Spawn receiving server' ' ma-pipe-unidir --listen --pidFile=listener.pid recv /ip4/127.0.0.1/tcp/10101 > server.out & test_wait_for_file 30 100ms listener.pid && kill -0 $(cat listener.pid) ' test_expect_success 'C->S Setup client side' ' ipfsi 1 p2p forward /x/p2p-test /ip4/127.0.0.1/tcp/10102 /p2p/${PEERID_0} 2>&1 > dialer-stdouterr.log ' test_expect_success 'C->S Connect and receive data' ' ma-pipe-unidir send /ip4/127.0.0.1/tcp/10102 < test1.bin ' test_expect_success 'C->S Ensure server finished' ' go-sleep 250ms && test ! -f listener.pid ' test_expect_success 'C->S Output looks good' ' test_cmp server.out test1.bin ' test_expect_success 'C->S Close local listener' ' ipfsi 1 p2p close -p /x/p2p-test ' check_test_ports # Checking port test_expect_success "cannot accept 0 port in 'ipfs p2p listen'" ' test_must_fail ipfsi 2 p2p listen /x/p2p-test/0 /ip4/127.0.0.1/tcp/0 ' test_expect_success "'ipfs p2p forward' accept 0 port" ' ipfsi 2 p2p forward /x/p2p-test/0 /ip4/127.0.0.1/tcp/0 /p2p/$PEERID_0 ' test_expect_success "'ipfs p2p ls' output looks good" ' echo "true" > forward_0_expected && ipfsi 2 p2p ls | awk '\''{print $2}'\'' | sed "s/.*\///" | awk -F: '\''{if($1>0)print"true"}'\'' > forward_0_actual && ipfsi 2 p2p close -p /x/p2p-test/0 && test_cmp forward_0_expected forward_0_actual ' # Listing streams test_expect_success "'ipfs p2p ls' succeeds" ' echo "/x/p2p-test /p2p/$PEERID_0 /ip4/127.0.0.1/tcp/10101" > expected && ipfsi 0 p2p ls > actual ' test_expect_success "'ipfs p2p ls' output looks good" ' test_cmp expected actual ' test_expect_success "Cannot re-register app handler" ' test_must_fail ipfsi 0 p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10101 ' test_expect_success "'ipfs p2p stream ls' output is empty" ' ipfsi 0 p2p stream ls > actual && test_must_be_empty actual ' check_test_ports test_expect_success "Setup: Idle stream" ' ma-pipe-unidir --listen --pidFile=listener.pid recv /ip4/127.0.0.1/tcp/10101 & ipfsi 1 p2p forward /x/p2p-test /ip4/127.0.0.1/tcp/10102 /p2p/$PEERID_0 && ma-pipe-unidir --pidFile=client.pid recv /ip4/127.0.0.1/tcp/10102 & test_wait_for_file 30 100ms listener.pid && test_wait_for_file 30 100ms client.pid && kill -0 $(cat listener.pid) && kill -0 $(cat client.pid) ' test_expect_success "'ipfs p2p stream ls' succeeds" ' echo "3 /x/p2p-test /p2p/$PEERID_1 /ip4/127.0.0.1/tcp/10101" > expected ipfsi 0 p2p stream ls > actual ' test_expect_success "'ipfs p2p stream ls' output looks good" ' test_cmp expected actual ' test_expect_success "'ipfs p2p stream close' closes stream" ' ipfsi 0 p2p stream close 3 && ipfsi 0 p2p stream ls > actual && [ ! -f listener.pid ] && [ ! -f client.pid ] && test_must_be_empty actual ' test_expect_success "'ipfs p2p close' closes remote handler" ' ipfsi 0 p2p close -p /x/p2p-test && ipfsi 0 p2p ls > actual && test_must_be_empty actual ' test_expect_success "'ipfs p2p close' closes local handler" ' ipfsi 1 p2p close -p /x/p2p-test && ipfsi 1 p2p ls > actual && test_must_be_empty actual ' check_test_ports test_expect_success "Setup: Idle stream(2)" ' ma-pipe-unidir --listen --pidFile=listener.pid recv /ip4/127.0.0.1/tcp/10101 & ipfsi 0 p2p listen /x/p2p-test2 /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log && ipfsi 1 p2p forward /x/p2p-test2 /ip4/127.0.0.1/tcp/10102 /p2p/$PEERID_0 2>&1 > dialer-stdouterr.log && ma-pipe-unidir --pidFile=client.pid recv /ip4/127.0.0.1/tcp/10102 & test_wait_for_file 30 100ms listener.pid && test_wait_for_file 30 100ms client.pid && kill -0 $(cat listener.pid) && kill -0 $(cat client.pid) ' test_expect_success "'ipfs p2p stream ls' succeeds(2)" ' echo "4 /x/p2p-test2 /p2p/$PEERID_1 /ip4/127.0.0.1/tcp/10101" > expected ipfsi 0 p2p stream ls > actual test_cmp expected actual ' test_expect_success "'ipfs p2p close -a' closes remote app handlers" ' ipfsi 0 p2p close -a && ipfsi 0 p2p ls > actual && test_must_be_empty actual ' test_expect_success "'ipfs p2p close -a' closes local app handlers" ' ipfsi 1 p2p close -a && ipfsi 1 p2p ls > actual && test_must_be_empty actual ' test_expect_success "'ipfs p2p stream close -a' closes streams" ' ipfsi 0 p2p stream close -a && ipfsi 0 p2p stream ls > actual && [ ! -f listener.pid ] && [ ! -f client.pid ] && test_must_be_empty actual ' check_test_ports test_expect_success "'ipfs p2p close' closes app numeric handlers" ' ipfsi 0 p2p listen /x/1234 /ip4/127.0.0.1/tcp/10101 && ipfsi 0 p2p close -p /x/1234 && ipfsi 0 p2p ls > actual && test_must_be_empty actual ' test_expect_success "'ipfs p2p close' closes by target addr" ' ipfsi 0 p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10101 && ipfsi 0 p2p close -t /ip4/127.0.0.1/tcp/10101 && ipfsi 0 p2p ls > actual && test_must_be_empty actual ' test_expect_success "'ipfs p2p close' closes right listeners" ' ipfsi 0 p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10101 && ipfsi 0 p2p forward /x/p2p-test /ip4/127.0.0.1/tcp/10101 /p2p/$PEERID_1 && echo "/x/p2p-test /p2p/$PEERID_0 /ip4/127.0.0.1/tcp/10101" > expected && ipfsi 0 p2p close -l /ip4/127.0.0.1/tcp/10101 && ipfsi 0 p2p ls > actual && test_cmp expected actual ' check_test_ports test_expect_success "'ipfs p2p close' closes by listen addr" ' ipfsi 0 p2p close -l /p2p/$PEERID_0 && ipfsi 0 p2p ls > actual && test_must_be_empty actual ' # Peer reporting test_expect_success 'start p2p listener reporting peer' ' ipfsi 0 p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10101 --report-peer-id 2>&1 > listener-stdouterr.log ' test_expect_success 'C->S Spawn receiving server' ' ma-pipe-unidir --listen --pidFile=listener.pid recv /ip4/127.0.0.1/tcp/10101 > server.out & test_wait_for_file 30 100ms listener.pid && kill -0 $(cat listener.pid) ' test_expect_success 'C->S Setup client side' ' ipfsi 1 p2p forward /x/p2p-test /ip4/127.0.0.1/tcp/10102 /p2p/${PEERID_0} 2>&1 > dialer-stdouterr.log ' test_expect_success 'C->S Connect and receive data' ' ma-pipe-unidir send /ip4/127.0.0.1/tcp/10102 < test1.bin ' test_expect_success 'C->S Ensure server finished' ' go-sleep 250ms && test ! -f listener.pid ' test_expect_success 'C->S Output looks good' ' echo ${PEERID_1} > expected && cat test1.bin >> expected && test_cmp server.out expected ' test_expect_success 'C->S Close listeners' ' ipfsi 1 p2p close -p /x/p2p-test && ipfsi 0 p2p close -p /x/p2p-test && ipfsi 0 p2p ls > actual && test_must_be_empty actual && ipfsi 1 p2p ls > actual && test_must_be_empty actual ' test_expect_success "non /x/ scoped protocols are not allowed" ' test_must_fail ipfsi 0 p2p listen /its/not/a/x/path /ip4/127.0.0.1/tcp/10101 2> actual && echo "Error: protocol name must be within '"'"'/x/'"'"' namespace" > expected test_cmp expected actual ' check_test_ports test_expect_success 'start p2p listener on custom proto' ' ipfsi 0 p2p listen --allow-custom-protocol /p2p-test /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log && test_must_be_empty listener-stdouterr.log ' spawn_sending_server test_expect_success 'S->C Setup client side (custom proto)' ' ipfsi 1 p2p forward --allow-custom-protocol /p2p-test /ip4/127.0.0.1/tcp/10102 /p2p/${PEERID_0} 2>&1 > dialer-stdouterr.log ' test_server_to_client test_expect_success 'C->S Close local listener' ' ipfsi 1 p2p close -p /p2p-test ipfsi 1 p2p ls > actual && test_must_be_empty actual ' check_test_ports test_expect_success 'stop iptb' ' iptb stop ' check_test_ports test_done ================================================ FILE: test/sharness/t0181-private-network.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test private network feature" . lib/test-lib.sh test_init_ipfs test_expect_success "disable AutoConf for private network tests" ' ipfs config --json AutoConf.Enabled false ' export LIBP2P_FORCE_PNET=1 test_expect_success "daemon won't start with force pnet env but with no key" ' test_must_fail go-timeout 5 ipfs daemon > stdout 2>&1 ' unset LIBP2P_FORCE_PNET test_expect_success "daemon output includes info about the reason" ' grep "private network was not configured but is enforced by the environment" stdout || test_fsh cat stdout ' pnet_key() { echo '/key/swarm/psk/1.0.0/' echo '/bin/' random-data -size=32 } pnet_key > "${IPFS_PATH}/swarm.key" LIBP2P_FORCE_PNET=1 test_launch_ipfs_daemon test_expect_success "set up iptb testbed" ' iptb testbed create -type localipfs -count 5 -force -init && iptb run -- ipfs config --json "Routing.LoopbackAddressesOnLanDHT" true && iptb run -- ipfs config --json "Swarm.Transports.Network.Websocket" false && iptb run -- ipfs config --json Addresses.Swarm '"'"'["/ip4/127.0.0.1/tcp/0"]'"'"' && iptb run -- ipfs config --json AutoConf.Enabled false ' set_key() { node="$1" keyfile="$2" cp "$keyfile" "${IPTB_ROOT}/testbeds/default/${node}/swarm.key" } pnet_key > key1 pnet_key > key2 set_key 1 key1 set_key 2 key1 set_key 3 key2 set_key 4 key2 unset LIBP2P_FORCE_PNET test_expect_success "start nodes" ' iptb start -wait [0-4] ' test_expect_success "try connecting node in public network with priv networks" ' test_must_fail iptb connect --timeout=2s [1-4] 0 ' test_expect_success "node 0 (public network) swarm is empty" ' ipfsi 0 swarm peers && [ $(ipfsi 0 swarm peers | wc -l) -eq 0 ] ' test_expect_success "try connecting nodes in different private networks" ' test_must_fail iptb connect 2 3 ' test_expect_success "node 3 (pnet 2) swarm is empty" ' ipfsi 3 swarm peers && [ $(ipfsi 3 swarm peers | wc -l) -eq 0 ] ' test_expect_success "connect nodes in the same pnet" ' iptb connect 1 2 && iptb connect 3 4 ' test_expect_success "nodes 1 and 2 have connected" ' ipfsi 2 swarm peers && [ $(ipfsi 2 swarm peers | wc -l) -eq 1 ] ' test_expect_success "nodes 3 and 4 have connected" ' ipfsi 4 swarm peers && [ $(ipfsi 4 swarm peers | wc -l) -eq 1 ] ' run_single_file_test() { node1=$1 node2=$2 test_expect_success "add a file on node$node1" ' random-data -size=1000000 > filea && FILEA_HASH=$(ipfsi $node1 add -q filea) ' check_file_fetch $node1 $FILEA_HASH filea check_file_fetch $node2 $FILEA_HASH filea } check_file_fetch() { node="$1" fhash="$2" fname="$3" test_expect_success "can fetch file" ' ipfsi $node cat $fhash > fetch_out ' test_expect_success "file looks good" ' test_cmp $fname fetch_out ' } run_single_file_test 1 2 run_single_file_test 2 1 run_single_file_test 3 4 run_single_file_test 4 3 test_expect_success "stop testbed" ' iptb stop ' test_kill_ipfs_daemon # Test that AutoConf with default mainnet URL fails on private networks test_expect_success "setup test repo with AutoConf enabled and private network" ' export IPFS_PATH="$(pwd)/.ipfs-autoconf-test" && ipfs init --profile=test > /dev/null && ipfs config --json AutoConf.Enabled true && pnet_key > "${IPFS_PATH}/swarm.key" ' test_expect_success "daemon fails with AutoConf + private network error" ' export IPFS_PATH="$(pwd)/.ipfs-autoconf-test" && test_expect_code 1 ipfs daemon > autoconf_stdout 2> autoconf_stderr ' test_expect_success "error message mentions AutoConf and private network conflict" ' grep "AutoConf cannot use the default mainnet URL" autoconf_stderr > /dev/null && grep "private network.*swarm.key" autoconf_stderr > /dev/null && grep "AutoConf.Enabled=false" autoconf_stderr > /dev/null ' test_done ================================================ FILE: test/sharness/t0182-circuit-relay.sh ================================================ #!/usr/bin/env bash test_description="Test circuit relay" . lib/test-lib.sh # start iptb + wait for peering NUM_NODES=3 test_expect_success 'init iptb' ' iptb testbed create -type localipfs -count $NUM_NODES -init && iptb run -- ipfs config --json "Routing.LoopbackAddressesOnLanDHT" true ' # Network topology: A <-> Relay <-> B test_expect_success 'start up nodes for configuration' ' iptb start -wait -- --routing=none ' test_expect_success 'peer ids' ' PEERID_0=$(iptb attr get 0 id) && PEERID_1=$(iptb attr get 1 id) && PEERID_2=$(iptb attr get 2 id) ' relayaddrs=$(ipfsi 1 swarm addrs local | jq --raw-input . | jq --slurp .) staticrelay=$(ipfsi 1 swarm addrs local | sed -e "s|$|/p2p/$PEERID_1|g" | jq --raw-input . | jq --slurp .) test_expect_success 'configure the relay node as a static relay for node A' ' ipfsi 0 config Internal.Libp2pForceReachability private && ipfsi 0 config --json Swarm.RelayClient.Enabled true && ipfsi 0 config --json Swarm.RelayClient.StaticRelays "$staticrelay" ' test_expect_success 'configure the relay node' ' ipfsi 1 config Internal.Libp2pForceReachability public && ipfsi 1 config --json Swarm.RelayService.Enabled true && ipfsi 1 config --json Addresses.Swarm "$relayaddrs" ' test_expect_success 'configure the node B' ' ipfsi 2 config Internal.Libp2pForceReachability private && ipfsi 2 config --json Swarm.RelayClient.Enabled true ' test_expect_success 'restart nodes' ' iptb stop && iptb_wait_stop && iptb start -wait -- --routing=none ' test_expect_success 'connect A <-> Relay' ' iptb connect 0 1 ' test_expect_success 'connect B <-> Relay' ' iptb connect 2 1 ' test_expect_success 'wait until relay is ready to do work' ' while ! ipfsi 2 swarm connect /p2p/$PEERID_1/p2p-circuit/p2p/$PEERID_0; do iptb stop && iptb_wait_stop && iptb start -wait -- --routing=none && iptb connect 0 1 && iptb connect 2 1 && sleep 5 done ' test_expect_success 'connect A <-Relay-> B' ' ipfsi 2 swarm connect /p2p/$PEERID_1/p2p-circuit/p2p/$PEERID_0 > peers_out ' test_expect_success 'output looks good' ' echo "connect $PEERID_0 success" > peers_exp && test_cmp peers_exp peers_out ' test_expect_success 'peers for A look good' ' ipfsi 0 swarm peers > peers_out && test_should_contain "/p2p/$PEERID_1/p2p-circuit/p2p/$PEERID_2$" peers_out ' test_expect_success 'peers for B look good' ' ipfsi 2 swarm peers > peers_out && test_should_contain "/p2p/$PEERID_1/p2p-circuit/p2p/$PEERID_0$" peers_out ' test_expect_success 'stop iptb' ' iptb stop ' test_done ================================================ FILE: test/sharness/t0183-namesys-pubsub.sh ================================================ #!/usr/bin/env bash test_description="Test IPNS pubsub" . lib/test-lib.sh # start iptb + wait for peering NUM_NODES=5 test_expect_success 'init iptb' ' iptb testbed create -type localipfs -count $NUM_NODES -init ' run_ipnspubsub_tests() { test_expect_success 'peer ids' ' PEERID_0_BASE36=$(ipfsi 0 key list --ipns-base=base36 -l | grep self | head -n 1 | cut -d " " -f1) && PEERID_0_B58MH=$(ipfsi 0 key list --ipns-base=b58mh -l | grep self | head -n 1 | cut -d " " -f1) ' test_expect_success 'check namesys pubsub state' ' echo enabled > expected && ipfsi 0 name pubsub state > state0 && ipfsi 1 name pubsub state > state1 && ipfsi 2 name pubsub state > state2 && test_cmp expected state0 && test_cmp expected state1 && test_cmp expected state2 ' # These commands are *expected* to fail. We haven't published anything yet. test_expect_success 'subscribe nodes to the publisher topic' ' ipfsi 1 name resolve /ipns/$PEERID_0_BASE36 --timeout=1s; ipfsi 2 name resolve /ipns/$PEERID_0_BASE36 --timeout=1s; true ' test_expect_success 'check subscriptions' ' echo /ipns/$PEERID_0_BASE36 > expected_base36 && echo /ipns/$PEERID_0_B58MH > expected_b58mh && ipfsi 1 name pubsub subs > subs1 && ipfsi 2 name pubsub subs > subs2 && ipfsi 1 name pubsub subs --ipns-base=b58mh > subs1_b58mh && ipfsi 2 name pubsub subs --ipns-base=b58mh > subs2_b58mh && test_cmp expected_base36 subs1 && test_cmp expected_base36 subs2 && test_cmp expected_b58mh subs1_b58mh && test_cmp expected_b58mh subs2_b58mh ' test_expect_success 'add an object on publisher node' ' echo "ipns is super fun" > file && HASH_FILE=$(ipfsi 0 add -q file) ' test_expect_success 'publish that object as an ipns entry' ' ipfsi 0 name publish $HASH_FILE ' test_expect_success 'wait for the flood' ' sleep 1 ' test_expect_success 'resolve name in subscriber nodes' ' echo "/ipfs/$HASH_FILE" > expected && ipfsi 1 name resolve /ipns/$PEERID_0_BASE36 > name1 && ipfsi 2 name resolve /ipns/$PEERID_0_BASE36 > name2 && test_cmp expected name1 && test_cmp expected name2 ' test_expect_success 'cancel subscriptions to the publisher topic' ' ipfsi 1 name pubsub cancel /ipns/$PEERID_0_BASE36 && ipfsi 2 name pubsub cancel /ipns/$PEERID_0_BASE36 ' test_expect_success 'check subscriptions' ' rm -f expected && touch expected && ipfsi 1 name pubsub subs > subs1 && ipfsi 2 name pubsub subs > subs2 && test_cmp expected subs1 && test_cmp expected subs2 ' test_expect_success "shut down iptb" ' iptb stop ' } # Test everything with ipns-pubsub enabled via config test_expect_success 'enable ipns over pubsub' ' iptb run -- ipfs config --json Ipns.UsePubsub true ' startup_cluster $NUM_NODES run_ipnspubsub_tests # Test again, this time CLI parameter override the config test_expect_success 'enable ipns over pubsub' ' iptb run -- ipfs config --json Ipns.UsePubsub false ' startup_cluster $NUM_NODES --enable-namesys-pubsub run_ipnspubsub_tests # Confirm negative CLI flag takes precedence over positive config test_expect_success 'enable the pubsub-ipns via config' ' iptb run -- ipfs config --json Ipns.UsePubsub true ' startup_cluster $NUM_NODES --enable-namesys-pubsub=false test_expect_success 'ipns pubsub cmd fails because it was disabled via cli flag' ' test_expect_code 1 ipfsi 1 name pubsub subs 2> pubsubipns_cmd_out ' test_expect_success "ipns pubsub cmd produces error" " echo -e \"Error: IPNS pubsub subsystem is not enabled\nUse 'ipfs name pubsub subs --help' for information about this command\" > expected && test_cmp expected pubsubipns_cmd_out " test_expect_success 'stop iptb' ' iptb stop ' test_done ================================================ FILE: test/sharness/t0184-http-proxy-over-p2p.sh ================================================ #!/usr/bin/env bash test_description="Test http proxy over p2p" . lib/test-lib.sh if ! test_have_prereq SOCAT; then skip_all="skipping '$test_description': socat is not available" test_done fi WEB_SERVE_PORT=5099 IPFS_GATEWAY_PORT=5199 SENDER_GATEWAY="http://127.0.0.1:$IPFS_GATEWAY_PORT" function show_logs() { echo "*****************" echo " RECEIVER LOG " echo "*****************" iptb logs 1 echo "*****************" echo " SENDER LOG " echo "*****************" iptb logs 0 echo "*****************" echo "REMOTE_SERVER LOG" echo $REMOTE_SERVER_LOG echo "*****************" cat $REMOTE_SERVER_LOG } function start_http_server() { REMOTE_SERVER_LOG="server.log" rm -f $REMOTE_SERVER_LOG touch response socat tcp-listen:$WEB_SERVE_PORT,fork,bind=127.0.0.1,reuseaddr 'SYSTEM:cat response'!!CREATE:$REMOTE_SERVER_LOG & REMOTE_SERVER_PID=$! socat /dev/null tcp:127.0.01:$WEB_SERVE_PORT,retry=10 return $? } function teardown_remote_server() { exec 7<&- kill $REMOTE_SERVER_PID > /dev/null 2>&1 wait $REMOTE_SERVER_PID || true } function serve_content() { local body=$1 local status_code=${2:-"200 OK"} local length=$((1 + ${#body})) echo -e "HTTP/1.1 $status_code\nContent-length: $length\n\n$body" > response } function curl_check_response_code() { local expected_status_code=$1 local path_stub=${2:-p2p/$RECEIVER_ID/http/index.txt} local status_code=$(curl -s --write-out %{http_code} --output /dev/null $SENDER_GATEWAY/$path_stub) if [[ "$status_code" -ne "$expected_status_code" ]]; then echo "Found status-code "$status_code", expected "$expected_status_code return 1 fi return 0 } function curl_send_proxy_request_and_check_response() { local expected_status_code=$1 local expected_content=$2 # # make a request to SENDER_IPFS via the proxy endpoint # CONTENT_PATH="retrieved-file" STATUS_CODE="$(curl -s -o $CONTENT_PATH --write-out %{http_code} $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt)" # # check status code # if [[ "$STATUS_CODE" -ne "$expected_status_code" ]]; then echo -e "Found status-code "$STATUS_CODE", expected "$expected_status_code show_logs return 1 fi # # check content # RESPONSE_CONTENT="$(tail -n 1 $CONTENT_PATH)" if [[ "$RESPONSE_CONTENT" == "$expected_content" ]]; then return 0 else echo -e "Found response content:\n'"$RESPONSE_CONTENT"'\nthat differs from expected content:\n'"$expected_content"'" return 1 fi } function curl_send_multipart_form_request() { local expected_status_code=$1 local FILE_PATH="uploaded-file" FILE_CONTENT="curl will send a multipart-form POST request when sending a file which is handy" echo $FILE_CONTENT > $FILE_PATH # # send multipart form request # STATUS_CODE="$(curl -o /dev/null -s -F file=@$FILE_PATH --write-out %{http_code} $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt)" # # check status code # if [[ "$STATUS_CODE" -ne "$expected_status_code" ]]; then echo -e "Found status-code "$STATUS_CODE", expected "$expected_status_code return 1 fi # # check request method # if ! grep "POST /index.txt" $REMOTE_SERVER_LOG > /dev/null; then echo "Remote server request method/resource path was incorrect" show_logs return 1 fi # # check request is multipart-form # if ! grep "Content-Type: multipart/form-data;" $REMOTE_SERVER_LOG > /dev/null; then echo "Request content-type was not multipart/form-data" show_logs return 1 fi return 0 } test_expect_success 'configure nodes' ' iptb testbed create -type localipfs -count 2 -force -init && iptb run -- ipfs config --json "Routing.LoopbackAddressesOnLanDHT" true && ipfsi 0 config --json Experimental.Libp2pStreamMounting true && ipfsi 1 config --json Experimental.Libp2pStreamMounting true && ipfsi 0 config --json Experimental.P2pHttpProxy true && ipfsi 0 config --json Addresses.Gateway "[\"/ip4/127.0.0.1/tcp/$IPFS_GATEWAY_PORT\"]" ' test_expect_success 'configure a subdomain gateway with /p2p/ path whitelisted' " ipfsi 0 config --json Gateway.PublicGateways '{ \"example.com\": { \"UseSubdomains\": true, \"Paths\": [\"/p2p/\"] } }' " test_expect_success 'start and connect nodes' ' iptb start -wait && iptb connect 0 1 ' test_expect_success 'setup p2p listener on the receiver' ' ipfsi 1 p2p listen --allow-custom-protocol /http /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT && ipfsi 1 p2p listen /x/custom/http /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT ' test_expect_success 'setup environment' ' RECEIVER_ID=$(ipfsi 1 id -f="" --peerid-base=b58mh) RECEIVER_ID_CIDv1=$(ipfsi 1 id -f="" --peerid-base=base36) ' test_expect_success 'handle proxy http request sends bad-gateway when remote server not available ' ' curl_send_proxy_request_and_check_response 502 "" ' test_expect_success 'start http server' ' start_http_server ' test_expect_success 'handle proxy http request propagates error response from remote' ' serve_content "SORRY GUYS, I LOST IT" "404 Not Found" && curl_send_proxy_request_and_check_response 404 "SORRY GUYS, I LOST IT" ' test_expect_success 'handle proxy http request ' ' serve_content "THE WOODS ARE LOVELY DARK AND DEEP" && curl_send_proxy_request_and_check_response 200 "THE WOODS ARE LOVELY DARK AND DEEP" ' test_expect_success 'handle proxy http request invalid request' ' curl_check_response_code 400 p2p/DERPDERPDERP ' test_expect_success 'handle proxy http request unknown proxy peer ' ' UNKNOWN_PEER="k51qzi5uqu5dlmbel1sd8rs4emr3bfosk9bm4eb42514r4lakt4oxw3a3fa2tm" && curl_check_response_code 502 p2p/$UNKNOWN_PEER/http/index.txt ' test_expect_success 'handle proxy http request to invalid proxy peer ' ' curl_check_response_code 400 p2p/invalid_peer/http/index.txt ' test_expect_success 'handle proxy http request to custom protocol' ' serve_content "THE WOODS ARE LOVELY DARK AND DEEP" && curl_check_response_code 200 p2p/$RECEIVER_ID/x/custom/http/index.txt ' test_expect_success 'handle proxy http request to missing protocol' ' serve_content "THE WOODS ARE LOVELY DARK AND DEEP" && curl_check_response_code 502 p2p/$RECEIVER_ID/x/missing/http/index.txt ' test_expect_success 'handle proxy http request missing the /http' ' curl_check_response_code 400 p2p/$RECEIVER_ID/x/custom/index.txt ' test_expect_success 'handle multipart/form-data http request' ' serve_content "OK" && curl_send_multipart_form_request 200 ' # OK: $peerid.p2p.example.com/http/index.txt test_expect_success "handle http request to a subdomain gateway" ' serve_content "SUBDOMAIN PROVIDES ORIGIN ISOLATION PER RECEIVER_ID" && curl -H "Host: $RECEIVER_ID_CIDv1.p2p.example.com" -sD - $SENDER_GATEWAY/http/index.txt > p2p_response && test_should_contain "SUBDOMAIN PROVIDES ORIGIN ISOLATION PER RECEIVER_ID" p2p_response ' # FAIL: $peerid.p2p.example.com/p2p/$peerid/http/index.txt test_expect_success "handle invalid http request to a subdomain gateway" ' serve_content "SUBDOMAIN DOES NOT SUPPORT FULL /p2p/ PATH" && curl -H "Host: $RECEIVER_ID_CIDv1.p2p.example.com" -sD - $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt > p2p_response && test_should_contain "400 Bad Request" p2p_response ' # REDIRECT: example.com/p2p/$peerid/http/index.txt → $peerid.p2p.example.com/http/index.txt test_expect_success "redirect http path request to subdomain gateway" ' serve_content "SUBDOMAIN ROOT REDIRECTS /p2p/ PATH TO SUBDOMAIN" && curl -H "Host: example.com" -sD - $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt > p2p_response && test_should_contain "Location: http://$RECEIVER_ID_CIDv1.p2p.example.com/http/index.txt" p2p_response ' test_expect_success 'stop http server' ' teardown_remote_server ' test_expect_success 'stop nodes' ' iptb stop ' test_done ================================================ FILE: test/sharness/t0185-autonat.sh ================================================ #!/usr/bin/env bash test_description="Test autonat" . lib/test-lib.sh # NOTE: This is currently a really dumb test just to make sure this service # starts. We need better tests but testing AutoNAT without public IP addresses # is tricky. test_init_ipfs test_expect_success "enable autonat" ' ipfs config AutoNAT.ServiceMode enabled ' test_launch_ipfs_daemon test_kill_ipfs_daemon test_expect_success "enable autonat" ' ipfs config AutoNAT.ServiceMode disabled ' test_launch_ipfs_daemon test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0190-quic-ping.sh ================================================ #!/usr/bin/env bash test_description="Test ping over QUIC command" . lib/test-lib.sh test_init_ipfs # start iptb + wait for peering test_expect_success 'init iptb' ' iptb testbed create -type localipfs -count 2 -init ' addr1='"[\"/ip4/127.0.0.1/udp/0/quic-v1\"]"' addr2='"[\"/ip4/127.0.0.1/udp/0/quic-v1\"]"' test_expect_success "add QUIC swarm addresses" ' ipfsi 0 config --json Addresses.Swarm '$addr1' && ipfsi 1 config --json Addresses.Swarm '$addr2' ' startup_cluster 2 test_expect_success 'peer ids' ' PEERID_0=$(iptb attr get 0 id) && PEERID_1=$(iptb attr get 1 id) ' test_expect_success "test ping other" ' ipfsi 0 ping -n2 -- "$PEERID_1" && ipfsi 1 ping -n2 -- "$PEERID_0" ' test_expect_success "test ping self" ' test_must_fail ipfsi 0 ping -n2 -- "$PEERID_0" && test_must_fail ipfsi 1 ping -n2 -- "$PEERID_1" ' test_expect_success "test ping 0" ' test_must_fail ipfsi 0 ping -n0 -- "$PEERID_1" && test_must_fail ipfsi 1 ping -n0 -- "$PEERID_0" ' test_expect_success 'stop iptb' ' iptb stop ' test_done ================================================ FILE: test/sharness/t0191-webtransport-ping.sh ================================================ #!/usr/bin/env bash test_description="Test ping over WebTransport command" . lib/test-lib.sh test_init_ipfs # start iptb + wait for peering test_expect_success 'init iptb' ' iptb testbed create -type localipfs -count 2 -init ' addr1='"[\"/ip4/127.0.0.1/udp/0/quic-v1/webtransport\"]"' addr2='"[\"/ip4/127.0.0.1/udp/0/quic-v1/webtransport\"]"' test_expect_success "add WebTransport swarm addresses" ' ipfsi 0 config --json Addresses.Swarm '$addr1' && ipfsi 0 config --json Swarm.Transports.Network.WebTransport true && ipfsi 1 config --json Addresses.Swarm '$addr2' && ipfsi 1 config --json Swarm.Transports.Network.WebTransport true ' startup_cluster 2 test_expect_success 'peer ids' ' PEERID_0=$(iptb attr get 0 id) && PEERID_1=$(iptb attr get 1 id) ' test_expect_success "test ping other" ' ipfsi 0 ping -n2 -- "$PEERID_1" && ipfsi 1 ping -n2 -- "$PEERID_0" ' test_expect_success "test ping self" ' test_must_fail ipfsi 0 ping -n2 -- "$PEERID_0" && test_must_fail ipfsi 1 ping -n2 -- "$PEERID_1" ' test_expect_success "test ping 0" ' test_must_fail ipfsi 0 ping -n0 -- "$PEERID_1" && test_must_fail ipfsi 1 ping -n0 -- "$PEERID_0" ' test_expect_success 'stop iptb' ' iptb stop ' test_done ================================================ FILE: test/sharness/t0195-noise.sh ================================================ #!/usr/bin/env bash test_description="Test ping over NOISE command" . lib/test-lib.sh test_init_ipfs # start iptb + wait for peering test_expect_success 'init iptb' ' iptb testbed create -type localipfs -count 4 -init ' tcp_addr='"[\"/ip4/127.0.0.1/tcp/0\"]"' test_expect_success "configure security transports" ' iptb run < connect_error 2>&1 && test_should_contain "failed to negotiate security protocol" connect_error || test_fsh cat connect_error ' test_expect_success 'stop iptb' ' iptb stop ' test_done ================================================ FILE: test/sharness/t0220-bitswap.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="test bitswap commands" . lib/test-lib.sh test_init_ipfs test_launch_ipfs_daemon test_expect_success "'ipfs bitswap stat' succeeds" ' ipfs bitswap stat >stat_out ' test_expect_success "'ipfs bitswap stat' output looks good" ' cat <expected && bitswap status blocks received: 0 blocks sent: 0 data received: 0 data sent: 0 dup blocks received: 0 dup data received: 0 wantlist [0 keys] partners [0] EOF test_cmp expected stat_out ' test_expect_success "ipfs peer id looks good" ' PEERID=$(ipfs config Identity.PeerID) && test_check_peerid "$PEERID" ' test_expect_success "'ipfs bitswap wantlist -p' works" ' ipfs bitswap wantlist -p "$PEERID" >wantlist_p_out ' test_expect_success "'ipfs bitswap wantlist -p' output looks good" ' test_must_be_empty wantlist_p_out ' test_expect_success "hash was removed from wantlist" ' ipfs bitswap wantlist > wantlist_out && test_must_be_empty wantlist_out ' test_expect_success "'ipfs bitswap stat' succeeds" ' ipfs bitswap stat >stat_out ' test_expect_success "'ipfs bitswap stat' output looks good" ' cat <expected && bitswap status blocks received: 0 blocks sent: 0 data received: 0 data sent: 0 dup blocks received: 0 dup data received: 0 wantlist [0 keys] partners [0] EOF test_cmp expected stat_out ' test_expect_success "'ipfs bitswap wantlist -p' works" ' ipfs bitswap wantlist -p "$PEERID" >wantlist_p_out ' test_expect_success "'ipfs bitswap wantlist -p' output looks good" ' test_cmp wantlist_out wantlist_p_out ' test_expect_success "'ipfs bitswap stat --human' succeeds" ' ipfs bitswap stat --human >stat_out_human ' test_expect_success "'ipfs bitswap stat --human' output looks good" ' cat <expected && bitswap status blocks received: 0 blocks sent: 0 data received: 0 B data sent: 0 B dup blocks received: 0 dup data received: 0 B wantlist [0 keys] partners [0] EOF test_cmp expected stat_out_human ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0230-channel-streaming-http-content-type.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Cayman Nava # MIT Licensed; see the LICENSE file in this repository. # test_description="Test Content-Type for channel-streaming commands" . lib/test-lib.sh test_init_ipfs test_ls_cmd() { test_expect_success "Text encoded channel-streaming command succeeds" ' mkdir -p testdir && echo "hello test" >testdir/test.txt && ipfs add -r testdir && curl -X POST -i "http://$API_ADDR/api/v0/refs?arg=QmTcJAn3JP8ZMAKS6WS75q8sbTyojWKbxcUHgLYGWur4Ym&stream-channels=true&encoding=text" >actual_output ' test_expect_success "Text encoded channel-streaming command output looks good" ' printf "HTTP/1.1 200 OK\r\n" >expected_output && printf "Access-Control-Allow-Headers: X-Stream-Output, X-Chunked-Output, X-Content-Length\r\n" >>expected_output && printf "Access-Control-Expose-Headers: X-Stream-Output, X-Chunked-Output, X-Content-Length\r\n" >>expected_output && printf "Content-Type: text/plain\r\n" >>expected_output && printf "Server: kubo/%s\r\n" $(ipfs version -n) >>expected_output && printf "Trailer: X-Stream-Error\r\n" >>expected_output && printf "Vary: Origin\r\n" >>expected_output && printf "X-Chunked-Output: 1\r\n" >>expected_output && printf "Transfer-Encoding: chunked\r\n" >>expected_output && printf "\r\n" >>expected_output && echo QmRmPLc1FsPAn8F8F9DQDEYADNX5ER2sgqiokEvqnYknVW >>expected_output && cat actual_output | grep -vE Date > cleaned_output && test_cmp expected_output cleaned_output ' test_expect_success "JSON encoded channel-streaming command succeeds" ' mkdir -p testdir && echo "hello test" >testdir/test.txt && ipfs add -r testdir && curl -X POST -i "http://$API_ADDR/api/v0/refs?arg=QmTcJAn3JP8ZMAKS6WS75q8sbTyojWKbxcUHgLYGWur4Ym&stream-channels=true&encoding=json" >actual_output ' test_expect_success "JSON encoded channel-streaming command output looks good" ' printf "HTTP/1.1 200 OK\r\n" >expected_output && printf "Access-Control-Allow-Headers: X-Stream-Output, X-Chunked-Output, X-Content-Length\r\n" >>expected_output && printf "Access-Control-Expose-Headers: X-Stream-Output, X-Chunked-Output, X-Content-Length\r\n" >>expected_output && printf "Content-Type: application/json\r\n" >>expected_output && printf "Server: kubo/%s\r\n" $(ipfs version -n) >>expected_output && printf "Trailer: X-Stream-Error\r\n" >>expected_output && printf "Vary: Origin\r\n" >>expected_output && printf "X-Chunked-Output: 1\r\n" >>expected_output && printf "Transfer-Encoding: chunked\r\n" >>expected_output && printf "\r\n" >>expected_output && cat <<-\EOF >>expected_output && {"Ref":"QmRmPLc1FsPAn8F8F9DQDEYADNX5ER2sgqiokEvqnYknVW","Err":""} EOF printf "\n" >> expected_output && perl -pi -e '"'"'chomp if eof'"'"' expected_output && cat actual_output | grep -vE Date > cleaned_output && test_cmp expected_output cleaned_output ' } # should work online (only) test_launch_ipfs_daemon test_ls_cmd test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0231-channel-streaming.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test output of streaming json commands" . lib/test-lib.sh test_init_ipfs get_api_port() { cat "$IPFS_PATH/api" | awk -F/ '{ print $5 }' } test_ls_cmd() { test_expect_success "make a file with multiple refs" ' HASH=$(random-data -size=1000000 | ipfs add -q) ' test_expect_success "can get refs through curl" ' PORT=$(get_api_port) && curl http://localhost:$PORT/api/v0/refs/$HASH > output ' # make sure newlines are printed between each object test_expect_success "output looks good" ' test_expect_code 1 grep "}{" output > /dev/null ' } # should work online (only) test_launch_ipfs_daemon test_ls_cmd test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0235-cli-request.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="test http requests made by cli" . lib/test-lib.sh if ! test_have_prereq SOCAT; then skip_all="skipping '$test_description': socat is not available" test_done fi test_init_ipfs test_expect_success "start nc" ' rm -f nc_out nc_outp nc_inp && mkfifo nc_inp nc_outp socat PIPE:nc_inp!!PIPE:nc_outp tcp-listen:5005,fork,max-children=1,bind=127.0.0.1 & NCPID=$! exec 6>nc_inp 7nc_out && echo -e "HTTP/1.1 200 OK\r" >&6 && echo -e "Content-Type: application/json\r" >&6 && echo -e "Content-Length: 21\r" >&6 && echo -e "\r" >&6 && echo -e "{\"Version\":\"0.23.0\"}\r" >&6 && echo -e "\r" >&6 && # handle request for /api/v0/cat while read line; do if [[ "$line" == "$(echo -e "\r")" ]]; then break fi echo "$line" done <&7 >nc_out && echo -e "HTTP/1.1 200 OK\r" >&6 && echo -e "Content-Type: text/plain\r" >&6 && echo -e "Content-Length: 0\r" >&6 && echo -e "\r" >&6 && exec 6<&- && # Wait for IPFS wait $IPFSPID ' test_expect_success "stop nc" ' kill "$NCPID" && wait "$NCPID" || true ' test_expect_success "output does not contain multipart info" ' test_expect_code 1 grep multipart nc_out ' test_expect_success "request looks good" ' grep "POST /api/v0/cat" nc_out ' test_expect_success "api flag does not appear in request" ' test_expect_code 1 grep "api=/ip4" nc_out ' test_expect_success "host has dns name not ip address" ' grep "Host: localhost:5005" nc_out ' test_done ================================================ FILE: test/sharness/t0236-cli-api-dns-resolve.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="test dns resolution of api endpoint by cli" . lib/test-lib.sh if ! test_have_prereq SOCAT; then skip_all="skipping '$test_description': socat is not available" test_done fi test_init_ipfs test_expect_success "start nc" ' rm -f nc_out nc_outp nc_inp && mkfifo nc_inp nc_outp socat PIPE:nc_inp!!PIPE:nc_outp tcp-listen:5006,fork,max-children=1,bind=127.0.0.1 & NCPID=$! exec 6>nc_inp 7nc_out && echo -e "HTTP/1.1 200 OK\r" >&6 && echo -e "Content-Type: application/json\r" >&6 && echo -e "Content-Length: 21\r" >&6 && echo -e "\r" >&6 && echo -e "{\"Version\":\"0.23.0\"}\r" >&6 && echo -e "\r" >&6 && # handle request for /api/v0/cat while read line; do if [[ "$line" == "$(echo -e "\r")" ]]; then break fi echo "$line" done <&7 >nc_out && echo -e "HTTP/1.1 200 OK\r" >&6 && echo -e "Content-Type: text/plain\r" >&6 && echo -e "Content-Length: 0\r" >&6 && echo -e "\r" >&6 && exec 6<&- && # Wait for IPFS wait $IPFSPID ' test_expect_success "stop nc" ' kill "$NCPID" && wait "$NCPID" || true ' test_expect_success "request was received by local nc server" ' grep "POST /api/v0/cat" nc_out ' test_done ================================================ FILE: test/sharness/t0240-republisher.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test ipfs repo operations" . lib/test-lib.sh export DEBUG=true setup_iptb() { num_nodes="$1" bound=$(expr "$num_nodes" - 1) test_expect_success "iptb init" ' iptb testbed create -type localipfs -count $num_nodes -init ' for i in $(test_seq 0 "$bound") do test_expect_success "set configs up for node $i" ' ipfsi "$i" config Ipns.RepublishPeriod 40s && ipfsi "$i" config --json Ipns.ResolveCacheSize 0 ' done startup_cluster "$num_nodes" } teardown_iptb() { test_expect_success "shut down nodes" ' iptb stop ' } verify_can_resolve() { num_nodes="$1" bound=$(expr "$num_nodes" - 1) name="$2" expected="$3" msg="$4" for node in $(test_seq 0 "$bound") do test_expect_success "$msg: node $node can resolve entry" ' ipfsi "$node" name resolve "$name" > resolve ' test_expect_success "$msg: output for node $node looks right" ' printf "/ipfs/$expected\n" > expected && test_cmp expected resolve ' done } verify_cannot_resolve() { num_nodes="$1" bound=$(expr "$num_nodes" - 1) name="$2" msg="$3" for node in $(test_seq 0 "$bound") do test_expect_success "$msg: resolution fails on node $node" ' test_expect_code 1 ipfsi "$node" name resolve "$name" ' done } num_test_nodes=4 setup_iptb "$num_test_nodes" test_expect_success "publish succeeds" ' HASH=$(date +"%FT%T.%N%z" | ipfsi 1 add -q) && ipfsi 1 name publish -t 10s $HASH ' test_expect_success "get id succeeds" ' id=$(ipfsi 1 id -f "") ' verify_can_resolve "$num_test_nodes" "$id" "$HASH" "just after publishing" go-sleep 10s verify_cannot_resolve "$num_test_nodes" "$id" "after 10 seconds, records are invalid" go-sleep 30s verify_can_resolve "$num_test_nodes" "$id" "$HASH" "republisher fires after 30 seconds" # test_expect_success "generate new key" ' KEY2=`ipfsi 1 key gen beepboop --type ed25519` ' test_expect_success "publish with new key succeeds" ' HASH=$(date +"%FT%T.%N%z" | ipfsi 1 add -q) && ipfsi 1 name publish -t 10s -k "$KEY2" $HASH ' verify_can_resolve "$num_test_nodes" "$KEY2" "$HASH" "new key just after publishing" go-sleep 10s verify_cannot_resolve "$num_test_nodes" "$KEY2" "new key cannot resolve after 10 seconds" go-sleep 30s verify_can_resolve "$num_test_nodes" "$KEY2" "$HASH" "new key can resolve again after republish" # teardown_iptb test_done ================================================ FILE: test/sharness/t0250-files-api.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2015 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="test the unix files api" . lib/test-lib.sh test_init_ipfs create_files() { FILE1=$(echo foo | ipfs add "$@" -q) && FILE2=$(echo bar | ipfs add "$@" -q) && FILE3=$(echo baz | ipfs add "$@" -q) && mkdir -p stuff_test && echo cats > stuff_test/a && echo dogs > stuff_test/b && echo giraffes > stuff_test/c && DIR1=$(ipfs add -r "$@" -Q stuff_test) } verify_path_exists() { # simply running ls on a file should be a good 'check' ipfs files ls $1 } verify_dir_contents() { dir=$1 shift rm -f expected touch expected for e in $@ do echo $e >> expected done test_expect_success "can list dir" ' ipfs files ls $dir > output ' test_expect_success "dir entries look good" ' test_sort_cmp output expected ' } test_sharding() { local EXTRA ARGS EXTRA=$1 ARGS=$2 # only applied to the initial directory test_expect_success "make a directory $EXTRA" ' ipfs files mkdir $ARGS /foo ' test_expect_success "can make 100 files in a directory $EXTRA" ' printf "" > list_exp_raw for i in `seq 100 -1 1` do echo $i | ipfs files write --create /foo/file$i || return 1 echo file$i >> list_exp_raw done ' # Create the files in reverse (unsorted) order (`seq 100 -1 1`) # to check the sort in the `ipfs files ls` command. `ProtoNode` # links are always sorted at the DAG layer so the sorting feature # is tested with sharded directories. test_expect_success "sorted listing works $EXTRA" ' ipfs files ls /foo > list_out && sort list_exp_raw > list_exp && test_cmp list_exp list_out ' test_expect_success "unsorted listing works $EXTRA" ' ipfs files ls -U /foo > list_out && sort list_exp_raw > sort_list_not_exp && ! test_cmp sort_list_not_exp list_out ' test_expect_success "can read a file from sharded directory $EXTRA" ' ipfs files read /foo/file65 > file_out && echo "65" > file_exp && test_cmp file_out file_exp ' test_expect_success "can pin a file from sharded directory $EXTRA" ' ipfs files stat --hash /foo/file42 > pin_file_hash && ipfs pin add < pin_file_hash > pin_hash ' test_expect_success "can unpin a file from sharded directory $EXTRA" ' read -r _ HASH _ < pin_hash && ipfs pin rm $HASH ' test_expect_success "output object was really sharded and has correct hash $EXTRA" ' ipfs files stat --hash /foo > expected_foo_hash && echo $SHARD_HASH > actual_foo_hash && test_cmp expected_foo_hash actual_foo_hash ' test_expect_success "clean up $EXTRA" ' ipfs files rm -r /foo ' } test_files_api() { local EXTRA ARGS RAW_LEAVES EXTRA=$1 ARGS=$2 RAW_LEAVES=$3 test_expect_success "can mkdir in root $EXTRA" ' ipfs files mkdir $ARGS /cats ' test_expect_success "'files ls' lists root by default $EXTRA" ' ipfs files ls >actual && echo "cats" >expected && test_cmp expected actual ' test_expect_success "directory was created $EXTRA" ' verify_path_exists /cats ' test_expect_success "directory is empty $EXTRA" ' verify_dir_contents /cats ' # we do verification of stat formatting now as we depend on it test_expect_success "stat works $EXTRA" ' ipfs files stat / >stat ' test_expect_success "hash is first line of stat $EXTRA" ' ipfs ls $(head -1 stat) | grep "cats" ' test_expect_success "stat --hash gives only hash $EXTRA" ' ipfs files stat --hash / >actual && head -n1 stat >expected && test_cmp expected actual ' test_expect_success "stat with multiple format options should fail $EXTRA" ' test_must_fail ipfs files stat --hash --size / ' test_expect_success "compare hash option with format $EXTRA" ' ipfs files stat --hash / >expected && ipfs files stat --format='"'"''"'"' / >actual && test_cmp expected actual ' test_expect_success "compare size option with format $EXTRA" ' ipfs files stat --size / >expected && ipfs files stat --format='"'"''"'"' / >actual && test_cmp expected actual ' test_expect_success "check root hash $EXTRA" ' ipfs files stat --hash / > roothash ' test_expect_success "stat works outside of MFS" ' ipfs files stat /ipfs/$DIR1 ' test_expect_success "stat compute the locality of a dag" ' ipfs files stat --with-local /ipfs/$DIR1 > output grep -q "(100.00%)" output ' test_expect_success "cannot mkdir / $EXTRA" ' test_expect_code 1 ipfs files mkdir $ARGS / ' test_expect_success "check root hash was not changed $EXTRA" ' ipfs files stat --hash / > roothashafter && test_cmp roothash roothashafter ' test_expect_success "can put files into directory $EXTRA" ' ipfs files cp /ipfs/$FILE1 /cats/file1 ' test_expect_success "file shows up in directory $EXTRA" ' verify_dir_contents /cats file1 ' test_expect_success "file has correct hash and size in directory $EXTRA" ' echo "file1 $FILE1 4" > ls_l_expected && ipfs files ls -l /cats > ls_l_actual && test_cmp ls_l_expected ls_l_actual ' test_expect_success "file has correct hash and size listed with -l" ' echo "file1 $FILE1 4" > ls_l_expected && ipfs files ls -l /cats/file1 > ls_l_actual && test_cmp ls_l_expected ls_l_actual ' test_expect_success "file has correct hash and size listed with --long" ' echo "file1 $FILE1 4" > ls_l_expected && ipfs files ls --long /cats/file1 > ls_l_actual && test_cmp ls_l_expected ls_l_actual ' test_expect_success "file has correct hash and size listed with -l --cid-base=base32" ' echo "file1 `cid-fmt -v 1 -b base32 %s $FILE1` 4" > ls_l_expected && ipfs files ls --cid-base=base32 -l /cats/file1 > ls_l_actual && test_cmp ls_l_expected ls_l_actual ' test_expect_success "file shows up with the correct name" ' echo "file1" > ls_l_expected && ipfs files ls /cats/file1 > ls_l_actual && test_cmp ls_l_expected ls_l_actual ' test_expect_success "can stat file $EXTRA" ' ipfs files stat /cats/file1 > file1stat_orig ' test_expect_success "stat output looks good" ' grep -v CumulativeSize: file1stat_orig > file1stat_actual && echo "$FILE1" > file1stat_expect && echo "Size: 4" >> file1stat_expect && echo "ChildBlocks: 0" >> file1stat_expect && echo "Type: file" >> file1stat_expect && echo "Mode: not set (not set)" >> file1stat_expect && echo "Mtime: not set" >> file1stat_expect && test_cmp file1stat_expect file1stat_actual ' test_expect_success "can stat file with --cid-base=base32 $EXTRA" ' ipfs files stat --cid-base=base32 /cats/file1 > file1stat_orig ' test_expect_success "stat output looks good with --cid-base=base32" ' grep -v CumulativeSize: file1stat_orig > file1stat_actual && echo `cid-fmt -v 1 -b base32 %s $FILE1` > file1stat_expect && echo "Size: 4" >> file1stat_expect && echo "ChildBlocks: 0" >> file1stat_expect && echo "Type: file" >> file1stat_expect && echo "Mode: not set (not set)" >> file1stat_expect && echo "Mtime: not set" >> file1stat_expect && test_cmp file1stat_expect file1stat_actual ' test_expect_success "can read file $EXTRA" ' ipfs files read /cats/file1 > file1out ' test_expect_success "output looks good $EXTRA" ' echo foo > expected && test_cmp expected file1out ' test_expect_success "can put another file into root $EXTRA" ' ipfs files cp /ipfs/$FILE2 /file2 ' test_expect_success "file shows up in root $EXTRA" ' verify_dir_contents / file2 cats ' test_expect_success "can read file $EXTRA" ' ipfs files read /file2 > file2out ' test_expect_success "output looks good $EXTRA" ' echo bar > expected && test_cmp expected file2out ' test_expect_success "can make deep directory $EXTRA" ' ipfs files mkdir $ARGS -p /cats/this/is/a/dir ' test_expect_success "directory was created correctly $EXTRA" ' verify_path_exists /cats/this/is/a/dir && verify_dir_contents /cats this file1 && verify_dir_contents /cats/this is && verify_dir_contents /cats/this/is a && verify_dir_contents /cats/this/is/a dir && verify_dir_contents /cats/this/is/a/dir ' test_expect_success "dir has correct name" ' DIR_HASH=$(ipfs files stat /cats/this --hash) && echo "this/ $DIR_HASH 0" > ls_dir_expected && ipfs files ls -l /cats | grep this/ > ls_dir_actual && test_cmp ls_dir_expected ls_dir_actual ' test_expect_success "can copy file into new dir $EXTRA" ' ipfs files cp /ipfs/$FILE3 /cats/this/is/a/dir/file3 ' test_expect_success "can copy file into deep dir using -p flag $EXTRA" ' ipfs files cp -p /ipfs/$FILE3 /cats/some/other/dir/file3 ' test_expect_success "file copied into deep dir exists $EXTRA" ' ipfs files read /cats/some/other/dir/file3 > file_out && echo "baz" > file_exp && test_cmp file_out file_exp ' test_expect_success "cleanup deep cp -p test $EXTRA" ' ipfs files rm -r /cats/some ' test_expect_success "can read file $EXTRA" ' ipfs files read /cats/this/is/a/dir/file3 > output ' test_expect_success "output looks good $EXTRA" ' echo baz > expected && test_cmp expected output ' test_expect_success "file shows up in dir $EXTRA" ' verify_dir_contents /cats/this/is/a/dir file3 ' test_expect_success "can remove file $EXTRA" ' ipfs files rm /cats/this/is/a/dir/file3 ' test_expect_success "file no longer appears $EXTRA" ' verify_dir_contents /cats/this/is/a/dir ' test_expect_success "can remove dir $EXTRA" ' ipfs files rm -r /cats/this/is/a/dir ' test_expect_success "dir no longer appears $EXTRA" ' verify_dir_contents /cats/this/is/a ' test_expect_success "can remove file from root $EXTRA" ' ipfs files rm /file2 ' test_expect_success "file no longer appears $EXTRA" ' verify_dir_contents / cats ' test_expect_success "check root hash $EXTRA" ' ipfs files stat --hash / > roothash ' test_expect_success "cannot remove root $EXTRA" ' test_expect_code 1 ipfs files rm -r / ' test_expect_success "check root hash was not changed $EXTRA" ' ipfs files stat --hash / > roothashafter && test_cmp roothash roothashafter ' # test read options test_expect_success "read from offset works $EXTRA" ' ipfs files read -o 1 /cats/file1 > output ' test_expect_success "output looks good $EXTRA" ' echo oo > expected && test_cmp expected output ' test_expect_success "read with size works $EXTRA" ' ipfs files read -n 2 /cats/file1 > output ' test_expect_success "output looks good $EXTRA" ' printf fo > expected && test_cmp expected output ' test_expect_success "cannot read from negative offset $EXTRA" ' test_expect_code 1 ipfs files read --offset -3 /cats/file1 ' test_expect_success "read from offset 0 works $EXTRA" ' ipfs files read --offset 0 /cats/file1 > output ' test_expect_success "output looks good $EXTRA" ' echo foo > expected && test_cmp expected output ' test_expect_success "read last byte works $EXTRA" ' ipfs files read --offset 2 /cats/file1 > output ' test_expect_success "output looks good $EXTRA" ' echo o > expected && test_cmp expected output ' test_expect_success "offset past end of file fails $EXTRA" ' test_expect_code 1 ipfs files read --offset 5 /cats/file1 ' test_expect_success "cannot read negative count bytes $EXTRA" ' test_expect_code 1 ipfs read --count -1 /cats/file1 ' test_expect_success "reading zero bytes prints nothing $EXTRA" ' ipfs files read --count 0 /cats/file1 > output ' test_expect_success "output looks good $EXTRA" ' printf "" > expected && test_cmp expected output ' test_expect_success "count > len(file) prints entire file $EXTRA" ' ipfs files read --count 200 /cats/file1 > output ' test_expect_success "output looks good $EXTRA" ' echo foo > expected && test_cmp expected output ' # test write test_expect_success "can write file $EXTRA" ' echo "ipfs rocks" > tmpfile && cat tmpfile | ipfs files write $ARGS $RAW_LEAVES --create /cats/ipfs ' test_expect_success "file was created $EXTRA" ' verify_dir_contents /cats ipfs file1 this ' test_expect_success "can read file we just wrote $EXTRA" ' ipfs files read /cats/ipfs > output ' test_expect_success "can write to offset $EXTRA" ' echo "is super cool" | ipfs files write $ARGS $RAW_LEAVES -o 5 /cats/ipfs ' test_expect_success "file looks correct $EXTRA" ' echo "ipfs is super cool" > expected && ipfs files read /cats/ipfs > output && test_cmp expected output ' test_expect_success "file hash correct $EXTRA" ' echo $FILE_HASH > filehash_expected && ipfs files stat --hash /cats/ipfs > filehash && test_cmp filehash_expected filehash ' test_expect_success "can't write to negative offset $EXTRA" ' test_expect_code 1 ipfs files write $ARGS $RAW_LEAVES --offset -1 /cats/ipfs < output ' test_expect_success "verify file was not changed $EXTRA" ' ipfs files stat --hash /cats/ipfs > afterhash && test_cmp filehash afterhash ' test_expect_success "write new file for testing $EXTRA" ' echo foobar | ipfs files write $ARGS $RAW_LEAVES --create /fun ' test_expect_success "write to offset past end works $EXTRA" ' echo blah | ipfs files write $ARGS $RAW_LEAVES --offset 50 /fun ' test_expect_success "can read file $EXTRA" ' ipfs files read /fun > sparse_output ' test_expect_success "output looks good $EXTRA" ' echo foobar > sparse_expected && echo blah | dd of=sparse_expected bs=50 seek=1 && test_cmp sparse_expected sparse_output ' test_expect_success "cleanup $EXTRA" ' ipfs files rm /fun ' test_expect_success "cannot write to directory $EXTRA" ' ipfs files stat --hash /cats > dirhash && test_expect_code 1 ipfs files write $ARGS $RAW_LEAVES /cats < output ' test_expect_success "verify dir was not changed $EXTRA" ' ipfs files stat --hash /cats > afterdirhash && test_cmp dirhash afterdirhash ' test_expect_success "cannot write to nonexistent path $EXTRA" ' test_expect_code 1 ipfs files write $ARGS $RAW_LEAVES /cats/bar/ < output ' test_expect_success "no new paths were created $EXTRA" ' verify_dir_contents /cats file1 ipfs this ' # Temporary check to uncover source of flaky test fail (see # https://github.com/ipfs/go-ipfs/issues/8131 for more details). # We suspect that sometimes the daemon isn't running when in fact we need # it to for the `--flush=false` flag to take effect. To try to spot the # specific error before it manifests itself in the failed test we explicitly # poll the damon API when it should be running ($WITH_DAEMON set). # Test taken from `test/sharness/lib/test-lib.sh` (but with less retries # as the daemon is either running or not but there is no 'bootstrap' time # needed in this case). test_expect_success "'ipfs daemon' is running when WITH_DAEMON is set" ' test -z "$WITH_DAEMON" || pollEndpoint -host=$API_MADDR -v -tout=1s -tries=3 2>poll_apierr > poll_apiout || test_fsh cat actual_daemon || test_fsh cat daemon_err || test_fsh cat poll_apierr || test_fsh cat poll_apiout ' test_expect_success "write 'no-flush' succeeds $EXTRA" ' echo "testing" | ipfs files write $ARGS $RAW_LEAVES -f=false -e /cats/walrus ' # Skip this test if the commands are not being run through the daemon # ($WITH_DAEMON not set) as standalone commands will *always* flush # after being done and the 'no-flush' call from the previous test will # not be enforced. test_expect_success "root hash not bubbled up yet $EXTRA" ' test -z "$WITH_DAEMON" || (ipfs refs local > refsout && test_expect_code 1 grep $ROOT_HASH refsout) ' test_expect_success "changes bubbled up to root on inspection $EXTRA" ' ipfs files stat --hash / > root_hash ' test_expect_success "root hash looks good $EXTRA" ' export EXP_ROOT_HASH="$ROOT_HASH" && echo $EXP_ROOT_HASH > root_hash_exp && test_cmp root_hash_exp root_hash ' test_expect_success "/cats hash looks good $EXTRA" ' export EXP_CATS_HASH="$CATS_HASH" && echo $EXP_CATS_HASH > cats_hash_exp && ipfs files stat --hash /cats > cats_hash test_cmp cats_hash_exp cats_hash ' test_expect_success "flush root succeeds $EXTRA" ' ipfs files flush / ' # test mv test_expect_success "can mv dir $EXTRA" ' ipfs files mv /cats/this/is /cats/ ' test_expect_success "can mv dir and dest dir is / $EXTRA" ' ipfs files mv /cats/is / ' test_expect_success "can mv dir and dest dir path has no trailing slash $EXTRA" ' ipfs files mv /is /cats ' test_expect_success "mv worked $EXTRA" ' verify_dir_contents /cats file1 ipfs this is walrus && verify_dir_contents /cats/this ' test_expect_success "cleanup, remove 'cats' $EXTRA" ' ipfs files rm -r /cats ' test_expect_success "cleanup looks good $EXTRA" ' verify_dir_contents / ' # test truncating test_expect_success "create a new file $EXTRA" ' echo "some content" | ipfs files write $ARGS $RAW_LEAVES --create /cats ' test_expect_success "truncate and write over that file $EXTRA" ' echo "fish" | ipfs files write $ARGS $RAW_LEAVES --truncate /cats ' test_expect_success "output looks good $EXTRA" ' ipfs files read /cats > file_out && echo "fish" > file_exp && test_cmp file_out file_exp ' test_expect_success "file hash correct $EXTRA" ' echo $TRUNC_HASH > filehash_expected && ipfs files stat --hash /cats > filehash && test_cmp filehash_expected filehash ' test_expect_success "cleanup $EXTRA" ' ipfs files rm /cats ' # test flush flags test_expect_success "mkdir --flush works $EXTRA" ' ipfs files mkdir $ARGS --flush --parents /flushed/deep ' test_expect_success "mkdir --flush works a second time $EXTRA" ' ipfs files mkdir $ARGS --flush --parents /flushed/deep ' test_expect_success "dir looks right $EXTRA" ' verify_dir_contents / flushed ' test_expect_success "child dir looks right $EXTRA" ' verify_dir_contents /flushed deep ' test_expect_success "cleanup $EXTRA" ' ipfs files rm -r /flushed ' test_expect_success "child dir looks right $EXTRA" ' verify_dir_contents / ' # test for https://github.com/ipfs/go-ipfs/issues/2654 test_expect_success "create and remove dir $EXTRA" ' ipfs files mkdir $ARGS /test_dir && ipfs files rm -r "/test_dir" ' test_expect_success "create test file $EXTRA" ' echo "content" | ipfs files write $ARGS $RAW_LEAVES -e "/test_file" ' test_expect_success "copy test file onto test dir $EXTRA" ' ipfs files cp "/test_file" "/test_dir" ' test_expect_success "test /test_dir $EXTRA" ' ipfs files stat "/test_dir" | grep -q "^Type: file" ' test_expect_success "clean up /test_dir and /test_file $EXTRA" ' ipfs files rm -r /test_dir && ipfs files rm -r /test_file ' test_expect_success "make a directory and a file $EXTRA" ' ipfs files mkdir $ARGS /adir && echo "blah" | ipfs files write $ARGS $RAW_LEAVES --create /foobar ' test_expect_success "copy a file into a directory $EXTRA" ' ipfs files cp /foobar /adir/ ' test_expect_success "file made it into directory $EXTRA" ' ipfs files ls /adir | grep foobar ' test_expect_success "test copy --force overwrites files" ' ipfs files cp /ipfs/$FILE1 /file1 && ipfs files cp /ipfs/$FILE2 /file2 && ipfs files cp --force /file1 /file2 && test "`ipfs files read /file1`" = "`ipfs files read /file2`" ' test_expect_success "clean up" ' ipfs files rm /file1 && ipfs files rm /file2 ' test_expect_success "should fail to write file and create intermediate directories with no --parents flag set $EXTRA" ' echo "ipfs rocks" | test_must_fail ipfs files write --create /parents/foo/ipfs.txt ' test_expect_success "can write file and create intermediate directories $EXTRA" ' echo "ipfs rocks" | ipfs files write --create --parents /parents/foo/bar/baz/ipfs.txt && ipfs files stat "/parents/foo/bar/baz/ipfs.txt" | grep -q "^Type: file" ' test_expect_success "can write file and create intermediate directories with short flags $EXTRA" ' echo "ipfs rocks" | ipfs files write -e -p /parents/foo/bar/baz/qux/quux/garply/ipfs.txt && ipfs files stat "/parents/foo/bar/baz/qux/quux/garply/ipfs.txt" | grep -q "^Type: file" ' test_expect_success "can write another file in the same directory with -e -p $EXTRA" ' echo "ipfs rocks" | ipfs files write -e -p /parents/foo/bar/baz/qux/quux/garply/ipfs2.txt && ipfs files stat "/parents/foo/bar/baz/qux/quux/garply/ipfs2.txt" | grep -q "^Type: file" ' test_expect_success "clean up $EXTRA" ' ipfs files rm -r /foobar /adir /parents ' test_expect_success "root mfs entry is empty $EXTRA" ' verify_dir_contents / ' test_expect_success "repo gc $EXTRA" ' ipfs repo gc ' # test rm test_expect_success "remove file forcibly" ' echo "hello world" | ipfs files write --create /forcibly && ipfs files rm --force /forcibly && verify_dir_contents / ' test_expect_success "remove multiple files forcibly" ' echo "hello world" | ipfs files write --create /forcibly_one && echo "hello world" | ipfs files write --create /forcibly_two && ipfs files rm --force /forcibly_one /forcibly_two && verify_dir_contents / ' test_expect_success "remove directory forcibly" ' ipfs files mkdir /forcibly-dir && ipfs files rm --force /forcibly-dir && verify_dir_contents / ' test_expect_success "remove multiple directories forcibly" ' ipfs files mkdir /forcibly-dir-one && ipfs files mkdir /forcibly-dir-two && ipfs files rm --force /forcibly-dir-one /forcibly-dir-two && verify_dir_contents / ' test_expect_success "remove multiple files" ' echo "hello world" | ipfs files write --create /file_one && echo "hello world" | ipfs files write --create /file_two && ipfs files rm /file_one /file_two ' test_expect_success "remove multiple directories" ' ipfs files mkdir /forcibly-dir-one && ipfs files mkdir /forcibly-dir-two && ipfs files rm -r /forcibly-dir-one /forcibly-dir-two && verify_dir_contents / ' test_expect_success "remove nonexistent path forcibly" ' ipfs files rm --force /nonexistent ' test_expect_success "remove deeply nonexistent path forcibly" ' ipfs files rm --force /deeply/nonexistent ' # This one should return code 1 but still remove the rest of the valid files. test_expect_success "remove multiple files (with nonexistent one)" ' echo "hello world" | ipfs files write --create /file_one && echo "hello world" | ipfs files write --create /file_two && test_expect_code 1 ipfs files rm /file_one /nonexistent /file_two verify_dir_contents / ' } # test with and without the daemon (EXTRA="with-daemon" and EXTRA="no-daemon" # respectively). # FIXME: Check if we are correctly using the "no-daemon" flag in these test # combinations. tests_for_files_api() { local EXTRA EXTRA=$1 test_expect_success "can create some files for testing ($EXTRA)" ' create_files ' # default: CIDv0, dag-pb for all files (no raw-leaves) ROOT_HASH=QmcwKfTMCT7AaeiD92hWjnZn9b6eh9NxnhfSzN5x2vnDpt CATS_HASH=Qma88m8ErTGkZHbBWGqy1C7VmEmX8wwNDWNpGyCaNmEgwC FILE_HASH=QmQdQt9qooenjeaNhiKHF3hBvmNteB4MQBtgu3jxgf9c7i TRUNC_HASH=QmPVnT9gocPbqzN4G6SMp8vAPyzcjDbUJrNdKgzQquuDg4 test_files_api "($EXTRA)" test_expect_success "can create some files for testing with raw-leaves ($EXTRA)" ' create_files --raw-leaves ' # partial raw-leaves: initial files created with --raw-leaves, test ops without if [ "$EXTRA" = "with-daemon" ]; then ROOT_HASH=QmTpKiKcAj4sbeesN6vrs5w3QeVmd4QmGpxRL81hHut4dZ CATS_HASH=QmPhPkmtUGGi8ySPHoPu1qbfryLJKKq1GYxpgLyyCruvGe test_files_api "($EXTRA, partial raw-leaves)" fi # raw-leaves: single-block files become RawNode (CIDv1), dirs stay CIDv0 ROOT_HASH=QmTHzLiSouBHVTssS8xRzmfWGAvTGhPEjtPdB6pWMQdxJX CATS_HASH=QmPJkzbCoBuL379TbHgwF1YbVHnKgiDa5bjqYhe6Lovdms FILE_HASH=bafybeibkrazpbejqh3qun7xfnsl7yofl74o4jwhxebpmtrcpavebokuqtm TRUNC_HASH=bafybeigwhb3q36yrm37jv5fo2ap6r6eyohckqrxmlejrenex4xlnuxiy3e test_files_api "($EXTRA, raw-leaves)" '' --raw-leaves # cidv1 for mkdir: different from raw-leaves since mkdir forces CIDv1 dirs ROOT_HASH=QmTLdTaZNj8Mvq1cgYup59ZFJFv1KxptouFSZUZKeq7X3z CATS_HASH=bafybeihsqinttigpskqqj63wgalrny3lifvqv5ml7igrirdhlcf73l3wvm FILE_HASH=bafybeibkrazpbejqh3qun7xfnsl7yofl74o4jwhxebpmtrcpavebokuqtm TRUNC_HASH=bafybeigwhb3q36yrm37jv5fo2ap6r6eyohckqrxmlejrenex4xlnuxiy3e if [ "$EXTRA" = "with-daemon" ]; then test_files_api "($EXTRA, cidv1)" --cid-version=1 fi test_expect_success "can update root hash to cidv1" ' ipfs files chcid --cid-version=1 / && echo bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354 > hash_expect && ipfs files stat --hash / > hash_actual && test_cmp hash_expect hash_actual ' # cidv1 root: root upgraded to CIDv1 via chcid, all new dirs/files also CIDv1 ROOT_HASH=bafybeickjecu37qv6ue54ofk3n4rpm4g4abuofz7yc4qn4skffy263kkou CATS_HASH=bafybeihsqinttigpskqqj63wgalrny3lifvqv5ml7igrirdhlcf73l3wvm test_files_api "($EXTRA, cidv1 root)" if [ "$EXTRA" = "with-daemon" ]; then test_expect_success "can update root hash to blake2b-256" ' ipfs files chcid --hash=blake2b-256 / && echo bafykbzacebugfutjir6qie7apo5shpry32ruwfi762uytd5g3u2gk7tpscndq > hash_expect && ipfs files stat --hash / > hash_actual && test_cmp hash_expect hash_actual ' # blake2b-256 root: using blake2b-256 hash instead of sha2-256 ROOT_HASH=bafykbzaceaebvwrjdw5rfhqqh5miaq3g42yybnrw3kxxxx43ggyttm6xn2zek CATS_HASH=bafykbzaceaqvpxs3dfl7su6744jgyvifbusow2tfixdy646chasdwyz2boagc FILE_HASH=bafykbzaceca45w2i3o3q3ctqsezdv5koakz7sxsw37ygqjg4w54m2bshzevxy TRUNC_HASH=bafykbzaceadeu7onzmlq7v33ytjpmo37rsqk2q6mzeqf5at55j32zxbcdbwig test_files_api "($EXTRA, blake2b-256 root)" fi test_expect_success "can update root hash back to cidv0" ' ipfs files chcid / --cid-version=0 && echo QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn > hash_expect && ipfs files stat --hash / > hash_actual && test_cmp hash_expect hash_actual ' } tests_for_files_api "no-daemon" test_launch_ipfs_daemon_without_network WITH_DAEMON=1 # FIXME: Used only on a specific test inside `test_files_api` but we should instead # propagate the `"with-daemon"` argument in its caller `tests_for_files_api`. tests_for_files_api "with-daemon" test_kill_ipfs_daemon test_expect_success "enable sharding in config" ' ipfs config --json Import.UnixFSHAMTDirectorySizeThreshold "\"1B\"" ' test_launch_ipfs_daemon_without_network # sharding cidv0: HAMT-sharded directory with 100 files, CIDv0 SHARD_HASH=QmPkwLJTYZRGPJ8Lazr9qPdrLmswPtUjaDbEpmR9jEh1se test_sharding "(cidv0)" # sharding cidv1: HAMT-sharded directory with 100 files, CIDv1 SHARD_HASH=bafybeiaulcf7c46pqg3tkud6dsvbgvlnlhjuswcwtfhxts5c2kuvmh5keu test_sharding "(cidv1 root)" "--cid-version=1" test_kill_ipfs_daemon # Test automatic sharding and unsharding # We shard based on size with a threshold of 256 KiB (see config file docs) # above which directories are sharded. # # The directory size is estimated as the size of each link. Links are roughly # the entry name + the CID byte length (e.g. 34 bytes for a CIDv0). So for # entries of length 10 we need 256 KiB / (34 + 10) ~ 6000 entries in the # directory to trigger sharding. test_expect_success "set up automatic sharding/unsharding data" ' mkdir big_dir for i in `seq 5960` # Just above the number of entries that trigger sharding for 256KiB do echo $i > big_dir/`printf "file%06d" $i` # fixed length of 10 chars done ' test_expect_success "reset automatic sharding" ' ipfs config --json Import.UnixFSHAMTDirectorySizeThreshold null ' test_launch_ipfs_daemon_without_network LARGE_SHARDED="QmWfjnRWRvdvYezQWnfbvrvY7JjrpevsE9cato1x76UqGr" LARGE_MINUS_5_UNSHARDED="QmbVxi5zDdzytrjdufUejM92JsWj8wGVmukk6tiPce3p1m" test_add_large_sharded_dir() { exphash="$1" test_expect_success "ipfs add on directory succeeds" ' ipfs add -r -Q big_dir > shardbigdir_out && echo "$exphash" > shardbigdir_exp && test_cmp shardbigdir_exp shardbigdir_out ' test_expect_success "can access a path under the dir" ' ipfs cat "$exphash/file000030" > file30_out && test_cmp big_dir/file000030 file30_out ' } test_add_large_sharded_dir "$LARGE_SHARDED" test_expect_success "remove a few entries from big_dir/ to trigger unsharding" ' ipfs files cp /ipfs/"$LARGE_SHARDED" /big_dir && for i in `seq 5` do ipfs files rm /big_dir/`printf "file%06d" $i` done && ipfs files stat --hash /big_dir > unshard_dir_hash && echo "$LARGE_MINUS_5_UNSHARDED" > unshard_exp && test_cmp unshard_exp unshard_dir_hash ' test_expect_success "add a few entries to big_dir/ to retrigger sharding" ' for i in `seq 5` do ipfs files cp /ipfs/"$LARGE_SHARDED"/`printf "file%06d" $i` /big_dir/`printf "file%06d" $i` done && ipfs files stat --hash /big_dir > shard_dir_hash && echo "$LARGE_SHARDED" > shard_exp && test_cmp shard_exp shard_dir_hash ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0251-files-flushing.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2016 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="test the unix files api flushing" . lib/test-lib.sh test_init_ipfs verify_path_exists() { # simply running ls on a file should be a good 'check' ipfs files ls $1 } verify_dir_contents() { dir=$1 shift rm -f expected touch expected for e in $@ do echo $e >> expected done test_expect_success "can list dir" ' ipfs files ls $dir > output ' test_expect_success "dir entries look good" ' test_sort_cmp output expected ' } test_launch_ipfs_daemon test_expect_success "can copy a file in" ' HASH=$(echo "foo" | ipfs add -q) && ipfs files cp /ipfs/$HASH /file ' test_kill_ipfs_daemon test_launch_ipfs_daemon test_expect_success "file is still there" ' verify_path_exists /file ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0252-files-gc.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2016 Kevin Atkinson # MIT Licensed; see the LICENSE file in this repository. # test_description="test how the unix files api interacts with the gc" . lib/test-lib.sh test_init_ipfs test_expect_success "object not removed after gc" ' echo "hello world" > hello.txt && cat hello.txt | ipfs files write --create /hello.txt && ipfs repo gc && ipfs cat QmVib14uvPnCP73XaCDpwugRuwfTsVbGyWbatHAmLSdZUS ' test_expect_success "/hello.txt still accessible after gc" ' ipfs files read /hello.txt > hello-actual && test_cmp hello.txt hello-actual ' ADIR_HASH=QmbCgoMYVuZq8m1vK31JQx9DorwQdLMF1M3sJ7kygLLqnW FILE1_HASH=QmX4eaSJz39mNhdu5ACUwTDpyA6y24HmrQNnAape6u3buS test_expect_success "gc okay after adding incomplete node -- prep" ' ipfs files mkdir /adir && echo "file1" | ipfs files write --create /adir/file1 && echo "file2" | ipfs files write --create /adir/file2 && ipfs pin add --recursive=false $ADIR_HASH && ipfs files rm -r /adir && ipfs repo gc && # will remove /adir/file1 and /adir/file2 but not /adir test_must_fail ipfs cat $FILE1_HASH && ipfs files cp /ipfs/$ADIR_HASH /adir && ipfs pin rm $ADIR_HASH ' test_expect_success "gc okay after adding incomplete node" ' ipfs dag get $ADIR_HASH && ipfs repo gc && ipfs dag get $ADIR_HASH ' test_expect_success "add directory with direct pin" ' mkdir mydir/ && echo "hello world!" > mydir/hello.txt && FILE_UNPINNED=$(ipfs add --pin=false -q -r mydir/hello.txt) && DIR_PINNED=$(ipfs add --pin=false -Q -r mydir) && ipfs add --pin=false -r mydir && ipfs pin add --recursive=false $DIR_PINNED && ipfs cat $FILE_UNPINNED ' test_expect_success "run gc and make sure directory contents are removed" ' ipfs repo gc && test_must_fail ipfs cat $FILE_UNPINNED ' test_expect_success "add incomplete directory and make sure gc is okay" ' ipfs files cp /ipfs/$DIR_PINNED /mydir && ipfs repo gc && test_must_fail ipfs cat $FILE_UNPINNED ' test_expect_success "add back directory contents and run gc" ' ipfs add --pin=false mydir/hello.txt && ipfs repo gc ' test_expect_success "make sure directory contents are not removed" ' ipfs cat $FILE_UNPINNED ' test_done ================================================ FILE: test/sharness/t0260-sharding.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2014 Christian Couder # MIT Licensed; see the LICENSE file in this repository. # test_description="Test directory sharding" . lib/test-lib.sh test_expect_success "set up test data" ' mkdir testdata for i in `seq 2000` do echo $i > testdata/file$i done ' test_add_dir() { exphash="$1" test_expect_success "ipfs add on directory succeeds" ' ipfs add -r -Q testdata > sharddir_out && echo "$exphash" > sharddir_exp && test_cmp sharddir_exp sharddir_out ' test_expect_success "ipfs get on directory succeeds" ' ipfs get -o testdata-out "$exphash" && test_cmp testdata testdata-out ' } test_init_ipfs UNSHARDED="QmavrTrQG4VhoJmantURAYuw3bowq3E2WcvP36NRQDAC1N" test_expect_success "force sharding off" ' ipfs config --json Import.UnixFSHAMTDirectorySizeThreshold "\"1G\"" ' test_add_dir "$UNSHARDED" test_launch_ipfs_daemon test_add_dir "$UNSHARDED" test_kill_ipfs_daemon test_expect_success "force sharding on" ' ipfs config --json Import.UnixFSHAMTDirectorySizeThreshold "\"1B\"" ' SHARDED="QmSCJD1KYLhVVHqBK3YyXuoEqHt7vggyJhzoFYbT8v1XYL" test_add_dir "$SHARDED" test_launch_ipfs_daemon test_add_dir "$SHARDED" test_kill_ipfs_daemon test_expect_success "sharded and unsharded output look the same" ' ipfs ls "$SHARDED" | sort > sharded_out && ipfs ls "$UNSHARDED" | sort > unsharded_out && test_cmp sharded_out unsharded_out ' test_expect_success "ipfs cat error output the same" ' test_expect_code 1 ipfs cat "$SHARDED" 2> sharded_err && test_expect_code 1 ipfs cat "$UNSHARDED" 2> unsharded_err && test_cmp sharded_err unsharded_err ' test_expect_success "'ipfs ls --resolve-type=false --size=false' admits missing block" ' ipfs ls "$SHARDED" | head -1 > first_file && ipfs ls --size=false "$SHARDED" | sort > sharded_out_nosize && read -r HASH _ NAME missing_out && test_cmp sharded_out_nosize missing_out ' test_launch_ipfs_daemon test_expect_success "gateway can resolve sharded dirs" ' echo 100 > expected && curl -sfo actual "http://127.0.0.1:$GWAY_PORT/ipfs/$SHARDED/file100" && test_cmp expected actual ' test_expect_success "'ipfs resolve' can resolve sharded dirs" ' echo /ipfs/QmZ3RfWk1u5LEGYLHA633B5TNJy3Du27K6Fny9wcxpowGS > expected && ipfs resolve "/ipfs/$SHARDED/file100" > actual && test_cmp expected actual ' test_kill_ipfs_daemon test_add_dir_v1() { exphash="$1" test_expect_success "ipfs add (CIDv1) on directory succeeds" ' ipfs add -r -Q --cid-version=1 testdata > sharddir_out && echo "$exphash" > sharddir_exp && test_cmp sharddir_exp sharddir_out ' test_expect_success "can access a path under the dir" ' ipfs cat "$exphash/file20" > file20_out && test_cmp testdata/file20 file20_out ' } # this hash implies the directory is CIDv1 and leaf entries are CIDv1 and raw SHARDEDV1="bafybeibiemewfzzdyhq2l74wrd6qj2oz42usjlktgnlqv4yfawgouaqn4u" test_add_dir_v1 "$SHARDEDV1" test_launch_ipfs_daemon test_add_dir_v1 "$SHARDEDV1" test_kill_ipfs_daemon test_list_incomplete_dir() { test_expect_success "ipfs add (CIDv1) on very large directory with sha3 succeeds" ' ipfs add -r -Q --cid-version=1 --hash=sha3-256 --pin=false testdata > sharddir_out && largeSHA3dir=$(cat sharddir_out) ' test_expect_success "delete intermediate node from DAG" ' ipfs block rm "/ipld/$largeSHA3dir/Links/0/Hash" ' test_expect_success "can list part of the directory" ' ipfs ls "$largeSHA3dir" 2> ls_err_out echo "Error: failed to fetch all nodes" > exp_err_out && cat ls_err_out && test_cmp exp_err_out ls_err_out ' } test_list_incomplete_dir test_done ================================================ FILE: test/sharness/t0270-filestore.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2017 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test out the filestore nocopy functionality" . lib/test-lib.sh test_expect_success "create a dataset" ' random-files -seed=483 -depth=3 -dirs=4 -files=6 -filesize=1000000 somedir > /dev/null ' EXPHASH="QmXKtATsEt42CF5JoSsmzJstrvwEB5P89YQtdX4mdf9E3M" get_repo_size() { disk_usage "$IPFS_PATH" } assert_repo_size_less_than() { expval="$1" test_expect_success "check repo size" ' test "$(get_repo_size)" -lt "$expval" || { echo should be below "$expval" && test_fsh get_repo_size; } ' } assert_repo_size_greater_than() { expval="$1" test_expect_success "check repo size" ' test "$(get_repo_size)" -gt "$expval" || { echo should be above "$expval" && test_fsh get_repo_size; } ' } test_filestore_adds() { test_expect_success "nocopy add succeeds" ' HASH=$(ipfs add --raw-leaves --nocopy -r -Q somedir) ' test_expect_success "nocopy add has right hash" ' test "$HASH" = "$EXPHASH" ' assert_repo_size_less_than 1000000 test_expect_success "normal add with fscache doesn't duplicate data" ' ipfs add --raw-leaves --fscache -r -q somedir > /dev/null ' assert_repo_size_less_than 1000000 test_expect_success "normal add without fscache duplicates data" ' ipfs add --raw-leaves -r -q somedir > /dev/null ' assert_repo_size_greater_than 1000000 } init_ipfs_filestore() { test_expect_success "clean up old node" ' rm -rf "$IPFS_PATH" mountdir ipfs ipns mfs ' test_init_ipfs # Check the _early_ error message test_expect_success "nocopy add errors and has right message" ' test_must_fail ipfs add --nocopy -r somedir 2> add_out && grep "either the filestore or the urlstore must be enabled" add_out ' assert_repo_size_less_than 1000000 test_expect_success "enable urlstore config setting" ' ipfs config --json Experimental.UrlstoreEnabled true ' # Check the _late_ error message test_expect_success "nocopy add errors and has right message when the urlstore is enabled" ' test_must_fail ipfs add --nocopy -r somedir 2> add_out && grep "filestore is not enabled" add_out ' assert_repo_size_less_than 1000000 test_expect_success "enable filestore config setting" ' ipfs config --json Experimental.UrlstoreEnabled true && ipfs config --json Experimental.FilestoreEnabled true ' } init_ipfs_filestore test_filestore_adds test_debug ' echo "pwd=$(pwd)"; echo "IPFS_PATH=$IPFS_PATH" ' init_ipfs_filestore test_launch_ipfs_daemon test_filestore_adds test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0271-filestore-utils.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2017 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test out the filestore nocopy functionality" . lib/test-lib.sh test_init_filestore() { test_expect_success "clean up old node" ' rm -rf "$IPFS_PATH" mountdir ipfs ipns mfs ' test_init_ipfs test_expect_success "enable filestore config setting" ' ipfs config --json Experimental.FilestoreEnabled true ' } test_init_dataset() { test_expect_success "create a dataset" ' rm -r somedir mkdir somedir && random-data -size=1000 -seed=1 > somedir/file1 && random-data -size=10000 -seed=2 > somedir/file2 && random-data -size=1000000 -seed=3 > somedir/file3 ' } test_init() { test_init_filestore test_init_dataset } EXPHASH="QmXqfraAT3U8ct14PPPXcFkWyvmqUZazLdo29GXTKSHkP4" cat < ls_expect_file_order bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu 1000 somedir/file1 0 bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 10000 somedir/file2 0 bafkreiemzfmzws23c2po4m6deiueknqfty7r3voes3e3zujmobrooc2ngm 262144 somedir/file3 0 bafkreihgm53yhxn427lnfdwhqgpawc62qejog7gega5kqb6uwbyhjm47hu 262144 somedir/file3 262144 bafkreigl2pjptgxz6cexcnua56zc5dwsyrc4ph2eulmcb634oes6gzvmuy 262144 somedir/file3 524288 bafkreifjcthslybjizk36xffcsb32fsbguxz3ptkl7723wz4u3qikttmam 213568 somedir/file3 786432 EOF sort < ls_expect_file_order > ls_expect_key_order FILE1_HASH=bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu FILE2_HASH=bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 FILE3_HASH=QmYEZtRGGk8rgM8MetegLLRHMKskPCg7zWpmQQAo3cQiN5 cat < verify_expect_file_order ok bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu 1000 somedir/file1 0 ok bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 10000 somedir/file2 0 ok bafkreiemzfmzws23c2po4m6deiueknqfty7r3voes3e3zujmobrooc2ngm 262144 somedir/file3 0 ok bafkreihgm53yhxn427lnfdwhqgpawc62qejog7gega5kqb6uwbyhjm47hu 262144 somedir/file3 262144 ok bafkreigl2pjptgxz6cexcnua56zc5dwsyrc4ph2eulmcb634oes6gzvmuy 262144 somedir/file3 524288 ok bafkreifjcthslybjizk36xffcsb32fsbguxz3ptkl7723wz4u3qikttmam 213568 somedir/file3 786432 EOF sort < verify_expect_file_order > verify_expect_key_order cat < verify_rm_expect ok bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 10000 somedir/file2 0 keep ok bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu 1000 somedir/file1 0 keep changed bafkreiemzfmzws23c2po4m6deiueknqfty7r3voes3e3zujmobrooc2ngm 262144 somedir/file3 0 remove changed bafkreifjcthslybjizk36xffcsb32fsbguxz3ptkl7723wz4u3qikttmam 213568 somedir/file3 786432 remove changed bafkreigl2pjptgxz6cexcnua56zc5dwsyrc4ph2eulmcb634oes6gzvmuy 262144 somedir/file3 524288 remove changed bafkreihgm53yhxn427lnfdwhqgpawc62qejog7gega5kqb6uwbyhjm47hu 262144 somedir/file3 262144 remove EOF cat < verify_after_rm_expect ok bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 10000 somedir/file2 0 ok bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu 1000 somedir/file1 0 EOF IPFS_CMD="ipfs" test_filestore_adds() { test_expect_success "$IPFS_CMD add nocopy add succeeds" ' HASH=$($IPFS_CMD add --raw-leaves --nocopy -r -Q somedir) ' test_expect_success "nocopy add has right hash" ' test "$HASH" = "$EXPHASH" ' test_expect_success "'$IPFS_CMD filestore ls' output looks good'" ' $IPFS_CMD filestore ls | sort > ls_actual && test_cmp ls_expect_key_order ls_actual ' test_expect_success "'$IPFS_CMD filestore ls --file-order' output looks good'" ' $IPFS_CMD filestore ls --file-order > ls_actual && test_cmp ls_expect_file_order ls_actual ' test_expect_success "'$IPFS_CMD filestore ls HASH' works" ' $IPFS_CMD filestore ls $FILE1_HASH > ls_actual && grep -q somedir/file1 ls_actual ' test_expect_success "can retrieve multi-block file" ' $IPFS_CMD cat $FILE3_HASH > file3.data && test_cmp somedir/file3 file3.data ' } # check that the filestore is in a clean state test_filestore_state() { test_expect_success "$IPFS_CMD filestore verify' output looks good'" ' $IPFS_CMD filestore verify | LC_ALL=C sort > verify_actual test_cmp verify_expect_key_order verify_actual ' } test_filestore_verify() { test_filestore_state test_expect_success "$IPFS_CMD filestore verify --file-order' output looks good'" ' $IPFS_CMD filestore verify --file-order > verify_actual test_cmp verify_expect_file_order verify_actual ' test_expect_success "'$IPFS_CMD filestore verify HASH' works" ' $IPFS_CMD filestore verify $FILE1_HASH > verify_actual && grep -q somedir/file1 verify_actual ' test_expect_success "rename a file" ' mv somedir/file1 somedir/file1.bk ' test_expect_success "can not retrieve block after backing file moved" ' test_must_fail $IPFS_CMD cat $FILE1_HASH ' test_expect_success "'$IPFS_CMD filestore verify' shows file as missing" ' $IPFS_CMD filestore verify > verify_actual && grep no-file verify_actual | grep -q somedir/file1 ' test_expect_success "move file back" ' mv somedir/file1.bk somedir/file1 ' test_expect_success "block okay now" ' $IPFS_CMD cat $FILE1_HASH > file1.data && test_cmp somedir/file1 file1.data ' test_expect_success "change first bit of file" ' dd if=/dev/zero of=somedir/file3 bs=1024 count=1 ' test_expect_success "can not retrieve block after backing file changed" ' test_must_fail $IPFS_CMD cat $FILE3_HASH ' test_expect_success "'$IPFS_CMD filestore verify' shows file as changed" ' $IPFS_CMD filestore verify > verify_actual && grep changed verify_actual | grep -q somedir/file3 ' # reset the state for the next test test_init_dataset } test_filestore_rm_bad_blocks() { test_filestore_state test_expect_success "change first bit of file" ' dd if=/dev/zero of=somedir/file3 bs=1024 count=1 ' test_expect_success "'$IPFS_CMD filestore verify --remove-bad-blocks' shows changed file removed" ' $IPFS_CMD filestore verify --remove-bad-blocks > verify_rm_actual && test_cmp verify_rm_expect verify_rm_actual ' test_expect_success "'$IPFS_CMD filestore verify' shows only files that were not removed" ' $IPFS_CMD filestore verify > verify_after && test_cmp verify_after_rm_expect verify_after ' # reset the state for the next test test_init_dataset } test_filestore_dups() { # make sure the filestore is in a clean state test_filestore_state test_expect_success "'$IPFS_CMD filestore dups'" ' $IPFS_CMD add --raw-leaves somedir/file1 && $IPFS_CMD filestore dups > dups_actual && echo "$FILE1_HASH" > dups_expect test_cmp dups_expect dups_actual ' } # # No daemon # test_init test_filestore_adds test_filestore_verify test_filestore_dups test_filestore_rm_bad_blocks # # With daemon # test_init # must be in offline mode so tests that retrieve non-existent blocks # doesn't hang test_launch_ipfs_daemon_without_network test_filestore_adds test_filestore_verify test_filestore_dups test_kill_ipfs_daemon test_filestore_rm_bad_blocks ## ## base32 ## EXPHASH="bafybeienfbjfbywu5y44i5qm4wxajblgy5a6xuc4eepjaw5fq223wwsy3m" cat < ls_expect_file_order bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu 1000 somedir/file1 0 bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 10000 somedir/file2 0 bafkreiemzfmzws23c2po4m6deiueknqfty7r3voes3e3zujmobrooc2ngm 262144 somedir/file3 0 bafkreihgm53yhxn427lnfdwhqgpawc62qejog7gega5kqb6uwbyhjm47hu 262144 somedir/file3 262144 bafkreigl2pjptgxz6cexcnua56zc5dwsyrc4ph2eulmcb634oes6gzvmuy 262144 somedir/file3 524288 bafkreifjcthslybjizk36xffcsb32fsbguxz3ptkl7723wz4u3qikttmam 213568 somedir/file3 786432 EOF sort < ls_expect_file_order > ls_expect_key_order FILE1_HASH=bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu FILE2_HASH=bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 FILE3_HASH=bafybeietaxxjghilcjhc2m4zcmicm7yjvkjdfkamc3ct2hq4gmsb3shqsi cat < verify_expect_file_order ok bafkreidx7ivgllulfkzyoo4oa7dfrg4mjmudg2qgdivoooj4s7lh3m5nqu 1000 somedir/file1 0 ok bafkreic2wqrsyr3y3qgzbvufen2w25r3p3zljckqyxkpcagsxz3zdcosd4 10000 somedir/file2 0 ok bafkreiemzfmzws23c2po4m6deiueknqfty7r3voes3e3zujmobrooc2ngm 262144 somedir/file3 0 ok bafkreihgm53yhxn427lnfdwhqgpawc62qejog7gega5kqb6uwbyhjm47hu 262144 somedir/file3 262144 ok bafkreigl2pjptgxz6cexcnua56zc5dwsyrc4ph2eulmcb634oes6gzvmuy 262144 somedir/file3 524288 ok bafkreifjcthslybjizk36xffcsb32fsbguxz3ptkl7723wz4u3qikttmam 213568 somedir/file3 786432 EOF sort < verify_expect_file_order > verify_expect_key_order IPFS_CMD="ipfs --cid-base=base32" # # No daemon # test_init test_filestore_adds test_filestore_verify test_filestore_dups test_filestore_rm_bad_blocks # # With daemon # test_init # must be in offline mode so tests that retrieve non-existent blocks # doesn't hang test_launch_ipfs_daemon_without_network test_filestore_adds test_filestore_verify test_filestore_dups test_kill_ipfs_daemon test_done test_filestore_rm_bad_blocks ## test_done ================================================ FILE: test/sharness/t0272-urlstore.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2017 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test out the urlstore functionality" . lib/test-lib.sh test_expect_success "create some random files" ' random-data -size=2222 -seed=7 > file1 && random-data -size=500000 -seed=7 > file2 && random-data -size=50000000 -seed=7 > file3 ' test_urlstore() { ADD_CMD="${@}" test_init_ipfs test_expect_success "add files using trickle dag format without raw leaves" ' HASH1a=$(ipfs add -q --trickle --raw-leaves=false file1) && HASH2a=$(ipfs add -q --trickle --raw-leaves=false file2) && HASH3a=$(ipfs add -q --trickle --raw-leaves=false file3) ' test_launch_ipfs_daemon_without_network test_expect_success "make sure files can be retrieved via the gateway" ' curl http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a -o file1.actual && test_cmp file1 file1.actual && curl http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a -o file2.actual && test_cmp file2 file2.actual && curl http://127.0.0.1:$GWAY_PORT/ipfs/$HASH3a -o file3.actual && test_cmp file3 file3.actual ' test_expect_success "add files without enabling url store using $ADD_CMD" ' test_must_fail ipfs $ADD_CMD http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a && test_must_fail ipfs $ADD_CMD http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a ' test_kill_ipfs_daemon test_expect_success "enable urlstore" ' ipfs config --json Experimental.UrlstoreEnabled true ' test_launch_ipfs_daemon_without_network test_expect_success "add files using gateway address via url store using $ADD_CMD" ' HASH1=$(ipfs $ADD_CMD --pin=false http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a) && HASH2=$(ipfs $ADD_CMD http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a) ' test_expect_success "make sure hashes are different" ' test $HASH1a != $HASH1 && test $HASH2a != $HASH2 ' test_expect_success "get files via urlstore" ' rm -f file1.actual file2.actual && ipfs get $HASH1 -o file1.actual && test_cmp file1 file1.actual && ipfs get $HASH2 -o file2.actual && test_cmp file2 file2.actual ' cat < ls_expect bafkreiconmdoujderxi757nf4wjpo4ukbhlo6mmxs6pg3yl53ln3ykldvi 2222 http://127.0.0.1:$GWAY_PORT/ipfs/QmUNEBSK2uPLSZU3Dj6XbSHjdGze4huWxESx2R4Ef1cKRW 0 bafkreifybqsfcheqkxzlhuuvoi3u6wz42kic4yqohvkia2i5fg3mpkqt3i 262144 http://127.0.0.1:$GWAY_PORT/ipfs/QmTgZc5bhTHUcqGN8rRP9oTJBv1UeJVWufPMPiUfbP9Ghs 0 bafkreigxuuyoickqhwxu4kjckmgfqb7ygd426qiakryvvstixy523imym4 237856 http://127.0.0.1:$GWAY_PORT/ipfs/QmTgZc5bhTHUcqGN8rRP9oTJBv1UeJVWufPMPiUfbP9Ghs 262144 EOF test_expect_success "ipfs filestore ls works with urls" ' ipfs filestore ls | sort > ls_actual && test_cmp ls_expect ls_actual ' cat < verify_expect ok bafkreifybqsfcheqkxzlhuuvoi3u6wz42kic4yqohvkia2i5fg3mpkqt3i 262144 http://127.0.0.1:$GWAY_PORT/ipfs/QmTgZc5bhTHUcqGN8rRP9oTJBv1UeJVWufPMPiUfbP9Ghs 0 ok bafkreigxuuyoickqhwxu4kjckmgfqb7ygd426qiakryvvstixy523imym4 237856 http://127.0.0.1:$GWAY_PORT/ipfs/QmTgZc5bhTHUcqGN8rRP9oTJBv1UeJVWufPMPiUfbP9Ghs 262144 ok bafkreiconmdoujderxi757nf4wjpo4ukbhlo6mmxs6pg3yl53ln3ykldvi 2222 http://127.0.0.1:$GWAY_PORT/ipfs/QmUNEBSK2uPLSZU3Dj6XbSHjdGze4huWxESx2R4Ef1cKRW 0 EOF test_expect_success "ipfs filestore verify works with urls" ' ipfs filestore verify | sort > verify_actual && test_cmp verify_expect verify_actual ' test_expect_success "garbage collect file1 from the urlstore" ' ipfs repo gc > /dev/null ' test_expect_success "can no longer retrieve file1 from urlstore" ' rm -f file1.actual && test_must_fail ipfs get $HASH1 -o file1.actual ' test_expect_success "can still retrieve file2 from urlstore" ' rm -f file2.actual && ipfs get $HASH2 -o file2.actual && test_cmp file2 file2.actual ' test_expect_success "remove original hashes from local gateway" ' ipfs pin rm $HASH1a $HASH2a && ipfs repo gc > /dev/null ' test_expect_success "gateway no longer has files" ' test_must_fail curl -f http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a -o file1.actual test_must_fail curl -f http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a -o file2.actual ' cat < verify_expect_2 error bafkreifybqsfcheqkxzlhuuvoi3u6wz42kic4yqohvkia2i5fg3mpkqt3i 262144 http://127.0.0.1:$GWAY_PORT/ipfs/QmTgZc5bhTHUcqGN8rRP9oTJBv1UeJVWufPMPiUfbP9Ghs 0 error bafkreigxuuyoickqhwxu4kjckmgfqb7ygd426qiakryvvstixy523imym4 237856 http://127.0.0.1:$GWAY_PORT/ipfs/QmTgZc5bhTHUcqGN8rRP9oTJBv1UeJVWufPMPiUfbP9Ghs 262144 EOF test_expect_success "ipfs filestore verify is correct" ' ipfs filestore verify | sort > verify_actual_2 && test_cmp verify_expect_2 verify_actual_2 ' test_expect_success "files cannot be retrieved via the urlstore" ' test_must_fail ipfs cat $HASH1 > /dev/null && test_must_fail ipfs cat $HASH2 > /dev/null ' test_expect_success "remove broken files" ' ipfs pin rm $HASH2 && ipfs repo gc > /dev/null ' test_expect_success "add large file using gateway address via url store" ' HASH3=$(ipfs ${ADD_CMD[@]} http://127.0.0.1:$GWAY_PORT/ipfs/$HASH3a) ' test_expect_success "make sure hashes are different" ' test $HASH3a != $HASH3 ' test_expect_success "get large file via urlstore" ' rm -f file3.actual && ipfs get $HASH3 -o file3.actual && test_cmp file3 file3.actual ' test_expect_success "check that the trickle option works" ' HASHat=$(ipfs add -q --cid-version=1 --raw-leaves=true -n --trickle file3) && HASHut=$(ipfs $ADD_CMD --trickle http://127.0.0.1:$GWAY_PORT/ipfs/$HASH3a) && test $HASHat = $HASHut ' test_expect_success "add files using gateway address via url store using --cid-base=base32" ' HASH1a=$(ipfs add -q --trickle --raw-leaves=false file1) && HASH2a=$(ipfs add -q --trickle --raw-leaves=false file2) && HASH1b32=$(ipfs --cid-base=base32 $ADD_CMD http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a) && HASH2b32=$(ipfs --cid-base=base32 $ADD_CMD http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a) ' test_kill_ipfs_daemon test_expect_success "files cannot be retrieved via the urlstore" ' test_must_fail ipfs cat $HASH1 > /dev/null && test_must_fail ipfs cat $HASH2 > /dev/null && test_must_fail ipfs cat $HASH3 > /dev/null ' test_expect_success "check that the hashes were correct" ' HASH1e=$(ipfs add -q -n --cid-version=1 --raw-leaves=true file1) && HASH2e=$(ipfs add -q -n --cid-version=1 --raw-leaves=true file2) && HASH3e=$(ipfs add -q -n --cid-version=1 --raw-leaves=true file3) && test $HASH1e = $HASH1 && test $HASH2e = $HASH2 && test $HASH3e = $HASH3 ' test_expect_success "check that the base32 hashes were correct" ' HASH1e32=$(ipfs cid base32 $HASH1e) HASH2e32=$(ipfs cid base32 $HASH2e) test $HASH1e32 = $HASH1b32 && test $HASH2e32 = $HASH2b32 ' test_expect_success "ipfs cleanup" ' rm -rf "$IPFS_PATH" && rmdir ipfs ipns mountdir ' } test_urlstore add -q --nocopy --cid-version=1 test_done ================================================ FILE: test/sharness/t0275-cid-security-data/CIQG6PGTD2VV34S33BE4MNCQITBRFYUPYQLDXYARR3DQW37MOT7K5XI.data ================================================  Um ================================================ FILE: test/sharness/t0275-cid-security-data/EICEM7ITSI.data ================================================ testing ================================================ FILE: test/sharness/t0275-cid-security.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2017 Jakub Sztandera # MIT Licensed; see the LICENSE file in this repository. # test_description="Cid Security" . lib/test-lib.sh test_init_ipfs test_expect_success "adding using unsafe function fails with error" ' echo foo | test_must_fail ipfs add --hash shake-128 2>add_out ' test_expect_success "error reason is pointed out" ' grep "potentially insecure hash functions not allowed" add_out || test_fsh cat add_out ' test_expect_success "adding using too short of a hash function gives out an error" ' echo foo | test_must_fail ipfs block put -f protobuf --mhlen 19 2>block_out ' test_expect_success "error reason is pointed out" ' grep "digest too small" block_out ' test_cat_get() { test_expect_success "ipfs cat fails with unsafe hash function" ' test_must_fail ipfs cat bafksebhh7d53e 2>ipfs_cat ' test_expect_success "error reason is pointed out" ' grep "potentially insecure hash functions not allowed" ipfs_cat ' test_expect_success "ipfs get fails with too short function" ' test_must_fail ipfs get bafkreez3itiri7ghbbf6lzej7paxyxy2qznpw 2>ipfs_get ' test_expect_success "error reason is pointed out" ' grep "digest too small" ipfs_get ' } test_gc() { test_expect_success "injecting insecure block" ' mkdir -p "$IPFS_PATH/blocks/TS" && cp -f ../t0275-cid-security-data/EICEM7ITSI.data "$IPFS_PATH/blocks/TS" ' test_expect_success "gc works" 'ipfs repo gc > gc_out' test_expect_success "gc removed bad block" ' grep bafksebcgpujze gc_out ' } # should work offline test_cat_get test_gc # should work online test_launch_ipfs_daemon test_cat_get test_gc test_expect_success "add block linking to insecure" ' mkdir -p "$IPFS_PATH/blocks/5X" && cp -f "../t0275-cid-security-data/CIQG6PGTD2VV34S33BE4MNCQITBRFYUPYQLDXYARR3DQW37MOT7K5XI.data" "$IPFS_PATH/blocks/5X" ' test_expect_success "ipfs cat fails with code 1 and not timeout" ' test_expect_code 1 go-timeout 1 ipfs cat QmVpsktzNeJdfWEpyeix93QJdQaBSgRNxebSbYSo9SQPGx ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0276-cidv0v1.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2017 Jakub Sztandera # MIT Licensed; see the LICENSE file in this repository. # test_description="CID Version 0/1 Duality" . lib/test-lib.sh test_init_ipfs # # # test_expect_success "create two small files" ' random-data -size=1000 -seed=7 > afile random-data -size=1000 -seed=9 > bfile ' test_expect_success "add file using CIDv1 but don't pin" ' AHASHv1=$(ipfs add -q --cid-version=1 --raw-leaves=false --pin=false afile) ' test_expect_success "add file using CIDv0" ' AHASHv0=$(ipfs add -q --cid-version=0 afile) ' test_expect_success "check hashes" ' test "$(cid-fmt %v-%c $AHASHv0)" = "cidv0-dag-pb" && test "$(cid-fmt %v-%c $AHASHv1)" = "cidv1-dag-pb" && test "$(cid-fmt -b z -v 0 %s $AHASHv1)" = "$AHASHv0" ' test_expect_success "make sure CIDv1 hash really is in the repo" ' ipfs block stat $AHASHv1 ' test_expect_success "make sure CIDv0 hash really is in the repo" ' ipfs block stat $AHASHv0 ' test_expect_success "run gc" ' ipfs repo gc ' test_expect_success "make sure the CIDv0 hash is in the repo" ' ipfs block stat $AHASHv0 ' test_expect_success "make sure we can get CIDv0 added file" ' ipfs cat $AHASHv0 > thefile && test_cmp afile thefile ' test_expect_success "make sure the CIDv1 hash is not in the repo" ' ! ipfs refs local | grep -q $AHASHv1 ' test_expect_success "clean up" ' ipfs pin rm $AHASHv0 && ipfs repo gc && ! ipfs refs local | grep -q $AHASHv0 ' # # # test_expect_success "add file using CIDv1 but don't pin" ' ipfs add -q --cid-version=1 --raw-leaves=false --pin=false afile ' test_expect_success "check that we can access the file when converted to CIDv0" ' ipfs cat $AHASHv0 > thefile && test_cmp afile thefile ' test_expect_success "clean up" ' ipfs repo gc ' test_expect_success "add file using CIDv0 but don't pin" ' ipfs add -q --cid-version=0 --raw-leaves=false --pin=false afile ' test_expect_success "check that we can access the file when converted to CIDv1" ' ipfs cat $AHASHv1 > thefile && test_cmp afile thefile ' # # # test_expect_success "set up iptb testbed" ' iptb testbed create -type localipfs -count 2 -init && iptb run -- ipfs config --json "Routing.LoopbackAddressesOnLanDHT" true ' test_expect_success "start nodes" ' iptb start -wait && iptb connect 0 1 ' test_expect_success "add afile using CIDv0 to node 0" ' iptb run 0 -- ipfs add -q --cid-version=0 afile ' test_expect_success "get afile using CIDv1 via node 1" ' iptb -quiet run 1 -- ipfs --timeout=2s cat $AHASHv1 > thefile && test_cmp afile thefile ' test_expect_success "add bfile using CIDv1 to node 0" ' BHASHv1=$(iptb -quiet run 0 -- ipfs add -q --cid-version=1 --raw-leaves=false bfile) ' test_expect_success "get bfile using CIDv0 via node 1" ' BHASHv0=$(cid-fmt -b z -v 0 %s $BHASHv1) echo $BHASHv1 && iptb -quiet run 1 -- ipfs --timeout=2s cat $BHASHv0 > thefile && test_cmp bfile thefile ' test_expect_success "stop testbed" ' iptb stop ' test_done ================================================ FILE: test/sharness/t0280-plugin-dag-jose-data/dag-cbor/bafyreicxyzuqbx5yb7ytkgkuofwksbal3ygtswxuri25crxdxms55m5fki ================================================ bivpPSWIuAyO8CpevzCLctagvWZAMBblhzDCsQWOAKdlkSAiprotectedx'eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4R0NNIn0jciphertextx 3XqLW28NHP-raqW8vMfIHOzko4N3IRaR ================================================ FILE: test/sharness/t0280-plugin-dag-jose-data/dag-cbor/bafyreihkt4u6euddfhofkutfzxwet7w7zm5qrjpop655yhnb5dnzqw26lm ================================================ bivpQ8xpPt_zZrfvHgR-ctagvjxHjcVusu0yrOBzw-Ex5zAiprotectedx3eyJhbGciOiJSU0EtT0FFUC0yNTYiLCJlbmMiOiJBMjU2R0NNIn0jciphertextx WTaw6WeqhaZDDhedzGYnsty4SMR-RzOwjrecipientsmencrypted_keyyVEqYaN4dFEH0vX4jU3d768hwOYSgZhElvVDzqdIKA6PFHsL4PPwJ7EIuebLrxwABJqXWBNG0kUBRjHuCv51VlxvX9WoH9ik7Qew0yROCGBj_AJef15PiZzUVUQwteHVDuSQs4OcsMfj18zc_ObskHvUMaN0PdCBA-G__7rGR2tcYSJOywbvxqqTENsCZNvasKxHSOuA_bjVsRmWloUMwLJkrbQxPAsVcwoPjAYF2agQ8D40AGFVEzGmhQDLI-OpXI-AfZYBurE7f_fU_NsYtqmFj5vZ9lvVCV1QsZa_HRhQlBBHxjTKyCBufY-0G4omt2nzYhyO-TaH44eUh81HFzww ================================================ FILE: test/sharness/t0280-plugin-dag-jose-data/dag-jose/bagcqcerakjv2mmdlbai3urym22bw5kaw7nqov73yaxf6xjnp7e56sclsrooa ================================================ bivX;J:k5 Yi3J*ctagP2B]Ak $(basename {}); \ diff {} $(basename {}) \' ' test_expect_success "retrieve dag-jose in non-dag-jose encodings" $' find ../t0280-plugin-dag-jose-data -type f | xargs -I {} sh -c \' \ codec=$(basename $(dirname {})); \ joseHash=$(ipfs dag put --store-codec dag-jose --input-codec=$codec {}); \ ipfs dag get --output-codec dag-cbor $joseHash > /dev/null; \ ipfs dag get --output-codec dag-json $joseHash > /dev/null \' ' } # should work offline test_dag_jose # should work online test_launch_ipfs_daemon test_dag_jose test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0280-plugin-data/example.go ================================================ package main import ( "fmt" "os" "github.com/ipfs/kubo/plugin" ) var Plugins = []plugin.Plugin{ &testPlugin{}, } var _ = Plugins // used type testPlugin struct{} func (*testPlugin) Name() string { return "test-plugin" } func (*testPlugin) Version() string { return "0.1.0" } func (*testPlugin) Init(env *plugin.Environment) error { fmt.Fprintf(os.Stderr, "testplugin %s\n", env.Repo) fmt.Fprintf(os.Stderr, "testplugin %v\n", env.Config) return nil } ================================================ FILE: test/sharness/t0280-plugin-fx.sh ================================================ #!/usr/bin/env bash test_description="Test fx plugin" . lib/test-lib.sh test_init_ipfs export GOLOG_LOG_LEVEL="fxtestplugin=debug" export TEST_FX_PLUGIN=1 test_launch_ipfs_daemon test_expect_success "expected log entry should be present" ' fgrep "invoked test fx function" daemon_err >/dev/null ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0280-plugin-git.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2017 Jakub Sztandera # MIT Licensed; see the LICENSE file in this repository. # test_description="Test git plugin" . lib/test-lib.sh test_init_ipfs # from https://github.com/ipfs/go-ipld-git/blob/master/make-test-repo.sh test_expect_success "prepare test data" ' tar xzf ../t0280-plugin-git-data/git.tar.gz ' test_dag_git() { test_expect_success "add objects via dag put" ' find objects -type f -exec ipfs dag put --store-codec=git-raw --input-codec=0x300078 --hash=sha1 {} \; -exec echo -n \; > hashes ' test_expect_success "successfully get added objects" ' cat hashes | xargs -I {} ipfs dag get -- {} > /dev/null ' test_expect_success "dag get works" ' echo -n "{\"message\":\"Some version\n\",\"object\":{\"/\":\"baf4bcfeq6c2mspupcvftgevza56h7rmozose6wi\"},\"tag\":\"v1\",\"tagger\":{\"date\":\"1497302532\",\"email\":\"johndoe@example.com\",\"name\":\"John Doe\",\"timezone\":\"+0200\"},\"type\":\"commit\"}" > tag_expected && ipfs dag get baf4bcfhzi72pcj5cc4ocz7igcduubuu7aa3cddi > tag_actual ' test_expect_success "outputs look correct" ' test_cmp tag_expected tag_actual ' test_expect_success "path traversals work" ' echo -n "{\"date\":\"1497302532\",\"email\":\"johndoe@example.com\",\"name\":\"John Doe\",\"timezone\":\"+0200\"}" > author_expected && echo -n "{\"/\":{\"bytes\":\"YmxvYiAxMgBIZWxsbyB3b3JsZAo\"}}" > file1_expected && echo -n "{\"/\":{\"bytes\":\"YmxvYiA3ACcsLnB5Zgo\"}}" > file2_expected && ipfs dag get baf4bcfhzi72pcj5cc4ocz7igcduubuu7aa3cddi/object/author > author_actual && ipfs dag get baf4bcfhzi72pcj5cc4ocz7igcduubuu7aa3cddi/object/tree/file/hash > file1_actual && ipfs dag get baf4bcfhzi72pcj5cc4ocz7igcduubuu7aa3cddi/object/parents/0/tree/dir2/hash/f3/hash > file2_actual ' test_expect_success "outputs look correct" ' test_cmp author_expected author_actual && test_cmp file1_expected file1_actual && test_cmp file2_expected file2_actual ' } # should work offline test_dag_git # should work online test_launch_ipfs_daemon test_dag_git test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0280-plugin-peerlog.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2017 Jakub Sztandera # MIT Licensed; see the LICENSE file in this repository. # test_description="Test peerlog plugin" . lib/test-lib.sh test_expect_success "setup testbed" ' iptb testbed create -type localipfs -count 2 -force -init ' startup_cluster 2 test_expect_success "peerlog is disabled by default" ' go-sleep 100ms iptb logs 0 >node0logs test_expect_code 1 grep peerlog node0logs ' test_expect_success 'stop iptb' 'iptb stop' test_expect_success "setup testbed" ' iptb testbed create -type localipfs -count 2 -force -init ' test_expect_success "enable peerlog config setting" ' iptb run -- ipfs config --json Plugins.Plugins.peerlog.Config.Enabled true ' startup_cluster 2 test_expect_success "peerlog plugin is logged" ' go-sleep 100ms iptb logs 0 >node0logs grep peerlog node0logs ' test_expect_success 'peer id' ' PEERID_1=$(iptb attr get 1 id) ' test_expect_success "peer id is logged" ' iptb logs 0 | grep -q "$PEERID_1" ' test_expect_success 'stop iptb' 'iptb stop' test_done ================================================ FILE: test/sharness/t0280-plugin.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2019 Protocol Labs # MIT Licensed; see the LICENSE file in this repository. # test_description="Test plugin loading" . lib/test-lib.sh if ! test_have_prereq PLUGIN; then skip_all='skipping plugin tests, plugins not available' test_done fi test_init_ipfs test_expect_success "ipfs id succeeds" ' ipfs id ' test_expect_success "make a bad plugin" ' mkdir -p "$IPFS_PATH/plugins" && echo foobar > "$IPFS_PATH/plugins/foo.so" && chmod +x "$IPFS_PATH/plugins/foo.so" ' test_expect_success "ipfs id fails due to a bad plugin" ' test_expect_code 1 ipfs id ' test_expect_success "cleanup bad plugin" ' rm "$IPFS_PATH/plugins/foo.so" ' test_expect_success "install test plugin" ' go build \ -asmflags=all="-trimpath=${GOPATH}" -gcflags=all="-trimpath=${GOPATH}" \ -buildmode=plugin -o "$IPFS_PATH/plugins/example.so" ../t0280-plugin-data/example.go && chmod +x "$IPFS_PATH/plugins/example.so" ' test_plugin() { local loads="$1" local repo="$2" local config="$3" rm -f id_raw_output id_output id_output_expected test_expect_success "id runs" ' ipfs id 2>id_raw_output >/dev/null ' test_expect_success "filter test plugin output" ' sed -ne "s/^testplugin //p" id_raw_output >id_output ' if [ "$loads" != "true" ]; then test_expect_success "plugin doesn't load" ' test_must_be_empty id_output ' else test_expect_success "plugin produces the correct output" ' echo "$repo" >id_output_expected && echo "$config" >>id_output_expected && test_cmp id_output id_output_expected ' fi } test_plugin true "$IPFS_PATH" "" test_expect_success "disable the plugin" ' ipfs config --json Plugins.Plugins.test-plugin.Disabled true ' test_plugin false test_expect_success "re-enable the plugin" ' ipfs config --json Plugins.Plugins.test-plugin.Disabled false ' test_plugin true "$IPFS_PATH" "" test_expect_success "configure the plugin" ' ipfs config Plugins.Plugins.test-plugin.Config foobar ' test_plugin true "$IPFS_PATH" "foobar" test_expect_success "noplugin flag works" ' test_must_fail go run -tags=noplugin github.com/ipfs/go-ipfs/cmd/ipfs id > output 2>&1 test_should_contain "not built with plugin support" output ' test_expect_success "noplugin flag works" ' CGO_ENABLED=0 test_must_fail go run github.com/ipfs/go-ipfs/cmd/ipfs id > output 2>&1 test_should_contain "not built with cgo support" output ' test_done ================================================ FILE: test/sharness/t0290-cid.sh ================================================ #!/usr/bin/env bash test_description="Test cid commands" . lib/test-lib.sh # NOTE: Primary tests for "ipfs cid" commands are in test/cli/cid_test.go # These sharness tests are kept for backward compatibility but new tests # should be added to test/cli/cid_test.go instead. If any of these tests # break, consider removing them and updating only the test/cli version. # note: all "ipfs cid" commands should work without requiring a repo CIDv0="QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv" CIDv1="zdj7WZAAFKPvYPPzyJLso2hhxo8a7ZACFQ4DvvfrNXTHidofr" CIDb32="bafybeibxm2nsadl3fnxv2sxcxmxaco2jl53wpeorjdzidjwf5aqdg7wa6u" CIDbase="QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6" CIDb32pb="bafybeievd6mwe6vcwnkwo3eizs3h7w3a34opszbyfxziqdxguhjw7imdve" CIDb32raw="bafkreievd6mwe6vcwnkwo3eizs3h7w3a34opszbyfxziqdxguhjw7imdve" CIDb32dagcbor="bafyreievd6mwe6vcwnkwo3eizs3h7w3a34opszbyfxziqdxguhjw7imdve" test_expect_success "cid base32 works" ' echo $CIDb32 > expected && ipfs cid base32 $CIDv0 > actual1 && test_cmp actual1 expected && ipfs cid base32 $CIDv1 > actual2 && test_cmp expected actual2 ' test_expect_success "cid format -v 1 -b base58btc" ' echo $CIDv1 > expected && ipfs cid format -v 1 -b base58btc $CIDv0 > actual1 && test_cmp actual1 expected && ipfs cid format -v 1 -b base58btc $CIDb32 > actual2 && test_cmp expected actual2 ' test_expect_success "cid format -v 0" ' echo $CIDv0 > expected && ipfs cid format -v 0 $CIDb32 > actual && test_cmp expected actual ' cat < various_cids QmZZRTyhDpL5Jgift1cHbAhexeE1m2Hw8x8g7rTcPahDvo QmPhk6cJkRcFfZCdYam4c9MKYjFG9V29LswUnbrFNhtk2S bafybeihtwdtifv43rn5cyilnmkwofdcxi2suqimmo62vn3etf45gjoiuwy bafybeiek4tfxkc4ov6jsmb63fzbirrsalnjw24zd5xawo2fgxisd4jmpyq zdj7WgYfT2gfsgiUxzPYboaRbP9H9CxZE5jVMK9pDDwCcKDCR zdj7WbTaiJT1fgatdet9Ei9iDB5hdCxkbVyhyh8YTUnXMiwYi uAXASIDsp4T3Wnd6kXFOQaljH3GFK_ixkjMtVhB9VOBrPK3bp uAXASIDdmmyANeytvXUriuy4BO0lfd2eR0UjygabF6CAzfsD1 EOF cat < various_cids_base32 bafybeifgwyq5gs4l2mru5klgwjfmftjvkmbyyjurbupuz2bst7mhmg2hwa bafybeiauil46g3lb32jemjbl7yspca3twdcg4wwkbsgdgvgdj5fpfv2f64 bafybeihtwdtifv43rn5cyilnmkwofdcxi2suqimmo62vn3etf45gjoiuwy bafybeiek4tfxkc4ov6jsmb63fzbirrsalnjw24zd5xawo2fgxisd4jmpyq bafybeifffq3aeaymxejo37sn5fyaf7nn7hkfmzwdxyjculx3lw4tyhk7uy bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354 bafybeib3fhqt3vu532sfyu4qnjmmpxdbjl7cyzemznkyih2vhanm6k3w5e bafybeibxm2nsadl3fnxv2sxcxmxaco2jl53wpeorjdzidjwf5aqdg7wa6u EOF cat < various_cids_v1 zdj7WgefqQm5HogBQ2bckZuTYYDarRTUZi51GYCnerHD2G86j zdj7WWnzU3Nbu5rYGWZHKigUXBtAwShs2SHDCM1TQEvC9TeCN zdj7WmqAbpsfXgiRBtZP1oAP9QWuuY3mqbc5JhpxJkfT3vYCu zdj7Wen5gtfr7AivXip3zYd1peuq2QfKrqAn4FGiciVWb96YB zdj7WgYfT2gfsgiUxzPYboaRbP9H9CxZE5jVMK9pDDwCcKDCR zdj7WbTaiJT1fgatdet9Ei9iDB5hdCxkbVyhyh8YTUnXMiwYi zdj7WZQrAvnY5ge3FNg5cmCsNwsvpYjdtu2yEmnWYQ4ES7Nzk zdj7WZAAFKPvYPPzyJLso2hhxo8a7ZACFQ4DvvfrNXTHidofr EOF test_expect_success "cid base32 works from stdin" ' cat various_cids | ipfs cid base32 > actual && test_cmp various_cids_base32 actual ' test_expect_success "cid format -v 1 -b base58btc works from stdin" ' cat various_cids | ipfs cid format -v 1 -b base58btc > actual && test_cmp various_cids_v1 actual ' cat < bases_expect 0 identity 0 48 base2 b 98 base32 B 66 base32upper c 99 base32pad C 67 base32padupper f 102 base16 F 70 base16upper k 107 base36 K 75 base36upper m 109 base64 M 77 base64pad t 116 base32hexpad T 84 base32hexpadupper u 117 base64url U 85 base64urlpad v 118 base32hex V 86 base32hexupper z 122 base58btc Z 90 base58flickr 🚀 128640 base256emoji EOF cat < codecs_expect 81 cbor 85 raw 112 dag-pb 113 dag-cbor 114 libp2p-key 120 git-raw 123 torrent-info 124 torrent-file 128 blake3-hashseq 129 leofcoin-block 130 leofcoin-tx 131 leofcoin-pr 133 dag-jose 134 dag-cose 144 eth-block 145 eth-block-list 146 eth-tx-trie 147 eth-tx 148 eth-tx-receipt-trie 149 eth-tx-receipt 150 eth-state-trie 151 eth-account-snapshot 152 eth-storage-trie 153 eth-receipt-log-trie 154 eth-receipt-log 176 bitcoin-block 177 bitcoin-tx 178 bitcoin-witness-commitment 192 zcash-block 193 zcash-tx 208 stellar-block 209 stellar-tx 224 decred-block 225 decred-tx 240 dash-block 241 dash-tx 250 swarm-manifest 251 swarm-feed 252 beeson 297 dag-json 496 swhid-1-snp 512 json 46083 rdfc-1 46593 json-jcs EOF cat < supported_codecs_expect 81 cbor 85 raw 112 dag-pb 113 dag-cbor 114 libp2p-key 120 git-raw 133 dag-jose 297 dag-json 512 json EOF cat < hashes_expect 0 identity 17 sha1 18 sha2-256 19 sha2-512 20 sha3-512 21 sha3-384 22 sha3-256 23 sha3-224 25 shake-256 26 keccak-224 27 keccak-256 28 keccak-384 29 keccak-512 30 blake3 86 dbl-sha2-256 45588 blake2b-160 45589 blake2b-168 45590 blake2b-176 45591 blake2b-184 45592 blake2b-192 45593 blake2b-200 45594 blake2b-208 45595 blake2b-216 45596 blake2b-224 45597 blake2b-232 45598 blake2b-240 45599 blake2b-248 45600 blake2b-256 45601 blake2b-264 45602 blake2b-272 45603 blake2b-280 45604 blake2b-288 45605 blake2b-296 45606 blake2b-304 45607 blake2b-312 45608 blake2b-320 45609 blake2b-328 45610 blake2b-336 45611 blake2b-344 45612 blake2b-352 45613 blake2b-360 45614 blake2b-368 45615 blake2b-376 45616 blake2b-384 45617 blake2b-392 45618 blake2b-400 45619 blake2b-408 45620 blake2b-416 45621 blake2b-424 45622 blake2b-432 45623 blake2b-440 45624 blake2b-448 45625 blake2b-456 45626 blake2b-464 45627 blake2b-472 45628 blake2b-480 45629 blake2b-488 45630 blake2b-496 45631 blake2b-504 45632 blake2b-512 45652 blake2s-160 45653 blake2s-168 45654 blake2s-176 45655 blake2s-184 45656 blake2s-192 45657 blake2s-200 45658 blake2s-208 45659 blake2s-216 45660 blake2s-224 45661 blake2s-232 45662 blake2s-240 45663 blake2s-248 45664 blake2s-256 EOF test_expect_success "cid bases" ' cat <<-EOF > expect identity base2 base32 base32upper base32pad base32padupper base16 base16upper base36 base36upper base64 base64pad base32hexpad base32hexpadupper base64url base64urlpad base32hex base32hexupper base58btc base58flickr base256emoji EOF ipfs cid bases > actual && test_cmp expect actual ' test_expect_success "cid bases --prefix" ' cat <<-EOF > expect identity 0 base2 b base32 B base32upper c base32pad C base32padupper f base16 F base16upper k base36 K base36upper m base64 M base64pad t base32hexpad T base32hexpadupper u base64url U base64urlpad v base32hex V base32hexupper z base58btc Z base58flickr 🚀 base256emoji EOF ipfs cid bases --prefix > actual && test_cmp expect actual ' test_expect_success "cid bases --prefix --numeric" ' ipfs cid bases --prefix --numeric > actual && test_cmp bases_expect actual ' test_expect_success "cid codecs" ' cut -c 8- codecs_expect > expect && ipfs cid codecs > actual test_cmp expect actual ' test_expect_success "cid codecs --numeric" ' ipfs cid codecs --numeric > actual && test_cmp codecs_expect actual ' test_expect_success "cid codecs --supported" ' cut -c 8- supported_codecs_expect > expect && ipfs cid codecs --supported > actual test_cmp expect actual ' test_expect_success "cid codecs --supported --numeric" ' ipfs cid codecs --supported --numeric > actual && test_cmp supported_codecs_expect actual ' test_expect_success "cid hashes" ' cut -c 8- hashes_expect > expect && ipfs cid hashes > actual test_cmp expect actual ' test_expect_success "cid hashes --numeric" ' ipfs cid hashes --numeric > actual && test_cmp hashes_expect actual ' test_expect_success "cid format -c raw" ' echo $CIDb32raw > expected && ipfs cid format --mc raw -b base32 $CIDb32pb > actual && test_cmp actual expected ' test_expect_success "cid format --mc dag-pb -v 0" ' echo $CIDbase > expected && ipfs cid format --mc dag-pb -v 0 $CIDb32raw > actual && test_cmp actual expected ' test_expect_success "cid format --mc dag-cbor" ' echo $CIDb32dagcbor > expected && ipfs cid format --mc dag-cbor $CIDb32pb > actual && test_cmp actual expected ' # this was an old flag that we removed, explicitly to force an error # so the user would read about the new multicodec names introduced # by https://github.com/ipfs/go-cid/commit/b2064d74a8b098193b316689a715cdf4e4934805 test_expect_success "cid format --codec fails" ' echo "Error: unknown option \"codec\"" > expected && test_expect_code 1 ipfs cid format --codec protobuf 2> actual && test_cmp actual expected ' test_expect_success "cid format -b base256emoji " ' echo "🚀🪐⭐💻😅❓💎🌈🌸🌚💰💍🌒😵🐶💁🤐🌎👼🙃🙅☺🌚😞🤤⭐🚀😃✈🌕😚🍻💜🐷⚽✌😊" > expected && ipfs cid format -b base256emoji bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi > actual && test_cmp actual expected ' test_expect_success "cid format -b base32 " ' echo "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi" > expected && ipfs cid format -b base32 🚀🪐⭐💻😅❓💎🌈🌸🌚💰💍🌒😵🐶💁🤐🌎👼🙃🙅☺🌚😞🤤⭐🚀😃✈🌕😚🍻💜🐷⚽✌😊 > actual && test_cmp actual expected ' test_done ================================================ FILE: test/sharness/t0295-multibase.sh ================================================ #!/usr/bin/env bash test_description="Test multibase commands" . lib/test-lib.sh # note: all "ipfs multibase" commands should work without requiring a repo cat < bases_expect 0 identity 0 48 base2 b 98 base32 B 66 base32upper c 99 base32pad C 67 base32padupper f 102 base16 F 70 base16upper k 107 base36 K 75 base36upper m 109 base64 M 77 base64pad t 116 base32hexpad T 84 base32hexpadupper u 117 base64url U 85 base64urlpad v 118 base32hex V 86 base32hexupper z 122 base58btc Z 90 base58flickr 128640 base256emoji EOF # TODO: expose same cmd under multibase? test_expect_success "multibase list" ' cut -c 12- bases_expect > expect && ipfs multibase list > actual && test_cmp expect actual ' test_expect_success "multibase encode works (stdin)" ' echo -n uaGVsbG8 > expected && echo -n hello | ipfs multibase encode > actual && test_cmp actual expected ' test_expect_success "multibase encode works (file)" ' echo -n hello > file && echo -n uaGVsbG8 > expected && ipfs multibase encode ./file > actual && test_cmp actual expected ' test_expect_success "multibase encode -b (custom base)" ' echo -n f68656c6c6f > expected && echo -n hello | ipfs multibase encode -b base16 > actual && test_cmp actual expected ' test_expect_success "multibase decode works (stdin)" ' echo -n hello > expected && echo -n uaGVsbG8 | ipfs multibase decode > actual && test_cmp actual expected ' test_expect_success "multibase decode works (file)" ' echo -n uaGVsbG8 > file && echo -n hello > expected && ipfs multibase decode ./file > actual && test_cmp actual expected ' test_expect_success "multibase encode+decode roundtrip" ' echo -n hello > expected && cat expected | ipfs multibase encode -b base64 | ipfs multibase decode > actual && test_cmp actual expected ' test_expect_success "mutlibase transcode works (stdin)" ' echo -n f68656c6c6f > expected && echo -n uaGVsbG8 | ipfs multibase transcode -b base16 > actual && test_cmp actual expected ' test_expect_success "multibase transcode works (file)" ' echo -n uaGVsbG8 > file && echo -n f68656c6c6f > expected && ipfs multibase transcode ./file -b base16> actual && test_cmp actual expected ' test_expect_success "multibase error on unknown multibase prefix" ' echo "Error: failed to decode multibase: selected encoding not supported" > expected && echo -n ę-that-should-do-the-trick | ipfs multibase decode 2> actual ; test_cmp actual expected ' test_expect_success "multibase error on a character outside of the base" " echo \"Error: failed to decode multibase: encoding/hex: invalid byte: U+007A 'z'\" > expected && echo -n f6c6f6cz | ipfs multibase decode 2> actual ; test_cmp actual expected " test_done ================================================ FILE: test/sharness/t0320-pubsub.sh ================================================ #!/usr/bin/env bash test_description="Test pubsub command" . lib/test-lib.sh # start iptb + wait for peering NUM_NODES=5 test_expect_success 'init iptb' ' iptb testbed create -type localipfs -count $NUM_NODES -init ' test_expect_success 'disable the DHT' ' iptb run -- ipfs config Routing.Type none ' run_pubsub_tests() { test_expect_success 'peer ids' ' PEERID_0=$(iptb attr get 0 id) && PEERID_2=$(iptb attr get 2 id) ' # ipfs pubsub sub test_expect_success 'pubsub' ' echo -n -e "test\nOK" | ipfs multibase encode -b base64url > expected && touch empty && mkfifo wait || test_fsh echo init fail # ipfs pubsub sub is long-running so we need to start it in the background and # wait put its output somewhere where we can access it ( ipfsi 0 pubsub sub --enc=json testTopic | if read line; then echo $line | jq -j .data > actual && echo > wait fi ) & ' test_expect_success "wait until ipfs pubsub sub is ready to do work" ' go-sleep 500ms ' test_expect_success "can see peer subscribed to testTopic" ' ipfsi 1 pubsub peers testTopic > peers_out ' test_expect_success "output looks good" ' echo $PEERID_0 > peers_exp && test_cmp peers_exp peers_out ' test_expect_success "publish something from file" ' echo -n -e "test\nOK" > payload-file && ipfsi 1 pubsub pub testTopic payload-file &> pubErr ' test_expect_success "wait until echo > wait executed" ' cat wait && test_cmp pubErr empty && test_cmp expected actual ' test_expect_success "wait for another pubsub message" ' echo -n -e "test\nOK\r\n2" | ipfs multibase encode -b base64url > expected && mkfifo wait2 || test_fsh echo init fail # ipfs pubsub sub is long-running so we need to start it in the background and # wait put its output somewhere where we can access it ( ipfsi 2 pubsub sub --enc=json testTopic | if read line; then echo $line | jq -j .data > actual && echo > wait2 fi ) & ' test_expect_success "wait until ipfs pubsub sub is ready to do work" ' go-sleep 500ms ' test_expect_success "publish something from stdin" ' echo -n -e "test\nOK\r\n2" | ipfsi 3 pubsub pub testTopic &> pubErr ' test_expect_success "wait until echo > wait executed" ' cat wait2 && test_cmp pubErr empty && test_cmp expected actual ' test_expect_success 'cleanup fifos' ' rm -f wait wait2 ' } # Normal tests - enabled via config test_expect_success 'enable the pubsub' ' iptb run -- ipfs config --json Pubsub.Enabled true ' startup_cluster $NUM_NODES run_pubsub_tests test_expect_success 'stop iptb' ' iptb stop ' test_expect_success 'disable the pubsub' ' iptb run -- ipfs config --json Pubsub.Enabled false ' # Normal tests - enabled via daemon option flag startup_cluster $NUM_NODES --enable-pubsub-experiment run_pubsub_tests test_expect_success 'stop iptb' ' iptb stop ' # Test with some nodes not signing messages. test_expect_success 'disable signing on nodes 1-3' ' iptb run [0-3] -- ipfs config --json Pubsub.DisableSigning true ' startup_cluster $NUM_NODES --enable-pubsub-experiment test_expect_success 'set node 4 to listen on testTopic' ' rm -f node4_actual && ipfsi 4 pubsub sub --enc=json testTopic > node4_actual & ' run_pubsub_tests test_expect_success 'stop iptb' ' iptb stop ' test_expect_success 'node 4 got no unsigned messages' ' test_must_be_empty node4_actual ' # Confirm negative CLI flag takes precedence over positive config # --enable-pubsub-experiment=false + Pubsub.Enabled:true test_expect_success 'enable the pubsub via config' ' iptb run -- ipfs config --json Pubsub.Enabled true ' startup_cluster $NUM_NODES --enable-pubsub-experiment=false test_expect_success 'pubsub cmd fails because it was disabled via cli flag' ' test_expect_code 1 ipfsi 4 pubsub ls 2> pubsub_cmd_out ' test_expect_success "pubsub cmd produces error" ' echo "Error: experimental pubsub feature not enabled, run daemon with --enable-pubsub-experiment to use" > expected && test_cmp expected pubsub_cmd_out ' test_expect_success 'stop iptb' ' iptb stop ' test_done ================================================ FILE: test/sharness/t0321-pubsub-gossipsub.sh ================================================ #!/usr/bin/env bash test_description="Test pubsub with gossipsub" . lib/test-lib.sh # start iptb + wait for peering NUM_NODES=5 test_expect_success 'init iptb' ' iptb testbed create -type localipfs -count $NUM_NODES -init ' test_expect_success "enable gossipsub" ' for x in $(seq 0 4); do ipfsi $x config Pubsub.Router gossipsub done ' # this is just a copy of t0180-pubsub; smell. startup_cluster $NUM_NODES --enable-pubsub-experiment test_expect_success 'peer ids' ' PEERID_0=$(iptb attr get 0 id) && PEERID_2=$(iptb attr get 2 id) ' test_expect_success 'pubsub' ' echo -n -e "test\nOK" | ipfs multibase encode -b base64url > expected && touch empty && mkfifo wait || test_fsh echo init fail # ipfs pubsub sub is long-running so we need to start it in the background and # wait put its output somewhere where we can access it ( ipfsi 0 pubsub sub --enc=json testTopic | if read line; then echo $line | jq -j .data > actual && echo > wait fi ) & ' test_expect_success "wait until ipfs pubsub sub is ready to do work" ' go-sleep 500ms ' test_expect_success "can see peer subscribed to testTopic" ' ipfsi 1 pubsub peers testTopic > peers_out ' test_expect_success "output looks good" ' echo $PEERID_0 > peers_exp && test_cmp peers_exp peers_out ' test_expect_success "publish something from a file" ' echo -n -e "test\nOK" > payload-file && ipfsi 1 pubsub pub testTopic payload-file &> pubErr ' test_expect_success "wait until echo > wait executed" ' cat wait && test_cmp pubErr empty && test_cmp expected actual ' test_expect_success "wait for another pubsub message" ' echo -n -e "test\nOK2" | ipfs multibase encode -b base64url > expected && mkfifo wait2 || test_fsh echo init fail # ipfs pubsub sub is long-running so we need to start it in the background and # wait put its output somewhere where we can access it ( ipfsi 2 pubsub sub --enc=json testTopic | if read line; then echo $line | jq -j .data > actual && echo > wait2 fi ) & ' test_expect_success "wait until ipfs pubsub sub is ready to do work" ' go-sleep 500ms ' test_expect_success "publish something" ' echo -n -e "test\nOK2" | ipfsi 1 pubsub pub testTopic &> pubErr ' test_expect_success "wait until echo > wait executed" ' cat wait2 && test_cmp pubErr empty && test_cmp expected actual ' test_expect_success 'stop iptb' ' iptb stop ' test_done ================================================ FILE: test/sharness/t0322-pubsub-http-rpc.sh ================================================ #!/usr/bin/env bash test_description="Test pubsub command behavior over HTTP RPC API" . lib/test-lib.sh test_init_ipfs test_launch_ipfs_daemon --enable-pubsub-experiment # Require topic as multibase # https://github.com/ipfs/go-ipfs/pull/8183 test_expect_success "/api/v0/pubsub/pub URL arg must be multibase encoded" ' echo test > data.txt && curl -s -X POST -F "data=@data.txt" "$API_ADDR/api/v0/pubsub/pub?arg=foobar" > result && test_should_contain "error" result && test_should_contain "URL arg must be multibase encoded" result ' # Use URL-safe multibase # base64 should produce error when used in URL args, base64url should be used test_expect_success "/api/v0/pubsub/pub URL arg must be in URL-safe multibase" ' echo test > data.txt && curl -s -X POST -F "data=@data.txt" "$API_ADDR/api/v0/pubsub/pub?arg=mZm9vYmFyCg" > result && test_should_contain "error" result && test_should_contain "URL arg must be base64url encoded" result ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0400-api-no-gateway/README.md ================================================ # Dataset description/sources - fixtures.car - raw CARv1 generated with: ```sh # using ipfs version 0.18.1 HASH=$(echo "testing" | ipfs add -q) ipfs dag export $HASH > fixtures.car echo HASH=${HASH} # a file containing the string "testing" # HASH=QmNYERzV2LfD2kkfahtfv44ocHzEFK1sLBaE7zdcYT2GAZ # a file containing the string "testing" ``` ================================================ FILE: test/sharness/t0400-api-no-gateway.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2016 Lars Gierth # MIT Licensed; see the LICENSE file in this repository. # test_description="Test API security" . lib/test-lib.sh test_init_ipfs # Import test case # See the static fixtures in ./t0400-api-no-gateway/ test_expect_success "Add the test directory" ' ipfs dag import ../t0400-api-no-gateway/fixtures.car ' HASH=QmNYERzV2LfD2kkfahtfv44ocHzEFK1sLBaE7zdcYT2GAZ # a file containing the string "testing" # by default, we don't let you load arbitrary ipfs objects through the api, # because this would open up the api to scripting vulnerabilities. # only the webui objects are allowed. # if you know what you're doing, go ahead and pass --unrestricted-api. test_launch_ipfs_daemon test_expect_success "Gateway on API unavailable" ' test_curl_resp_http_code "http://127.0.0.1:$API_PORT/ipfs/$HASH" "HTTP/1.1 404 Not Found" ' test_kill_ipfs_daemon test_launch_ipfs_daemon --unrestricted-api test_expect_success "Gateway on --unrestricted-api API available" ' test_curl_resp_http_code "http://127.0.0.1:$API_PORT/ipfs/$HASH" "HTTP/1.1 200 OK" ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0401-api-browser-security.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2020 Protocol Labs # MIT Licensed; see the LICENSE file in this repository. # test_description="Test API browser security" . lib/test-lib.sh test_init_ipfs PEERID=$(ipfs config Identity.PeerID) test_launch_ipfs_daemon test_expect_success "browser is unable to access API without Origin" ' curl -sD - -X POST -A "Mozilla" "http://127.0.0.1:$API_PORT/api/v0/id" >curl_output && grep "HTTP/1.1 403 Forbidden" curl_output ' test_expect_success "browser is unable to access API with invalid Origin" ' curl -sD - -X POST -A "Mozilla" -H "Origin: https://invalid.example.com" "http://127.0.0.1:$API_PORT/api/v0/id" >curl_output && grep "HTTP/1.1 403 Forbidden" curl_output ' test_expect_success "browser is able to access API if Origin is the API port on localhost (ipv4)" ' curl -sD - -X POST -A "Mozilla" -H "Origin: http://127.0.0.1:$API_PORT" "http://127.0.0.1:$API_PORT/api/v0/id" >curl_output && grep "HTTP/1.1 200 OK" curl_output && grep "$PEERID" curl_output ' test_expect_success "browser is able to access API if Origin is the API port on localhost (ipv6)" ' curl -sD - -X POST -A "Mozilla" -H "Origin: http://[::1]:$API_PORT" "http://127.0.0.1:$API_PORT/api/v0/id" >curl_output && grep "HTTP/1.1 200 OK" curl_output && grep "$PEERID" curl_output ' test_expect_success "browser is able to access API if Origin is the API port on localhost (localhost name)" ' curl -sD - -X POST -A "Mozilla" -H "Origin: http://localhost:$API_PORT" "http://127.0.0.1:$API_PORT/api/v0/id" >curl_output && grep "HTTP/1.1 200 OK" curl_output && grep "$PEERID" curl_output ' test_expect_success "Random browser extension is unable to access RPC API due to invalid Origin" ' curl -sD - -X POST -A "Mozilla" -H "Origin: chrome-extension://invalidextensionid" "http://127.0.0.1:$API_PORT/api/v0/id" >curl_output && grep "HTTP/1.1 403 Forbidden" curl_output ' test_expect_success "Companion extension is able to access RPC API on localhost" ' curl -sD - -X POST -A "Mozilla" -H "Origin: chrome-extension://nibjojkomfdiaoajekhjakgkdhaomnch" "http://127.0.0.1:$API_PORT/api/v0/id" >curl_output && cat curl_output && grep "HTTP/1.1 200 OK" curl_output && grep "$PEERID" curl_output ' test_expect_success "Companion beta extension is able to access API on localhost" ' curl -sD - -X POST -A "Mozilla" -H "Origin: chrome-extension://hjoieblefckbooibpepigmacodalfndh" "http://127.0.0.1:$API_PORT/api/v0/id" >curl_output && grep "HTTP/1.1 200 OK" curl_output && grep "$PEERID" curl_output ' test_kill_ipfs_daemon test_expect_success "setting CORS in API.HTTPHeaders works via CLI" " ipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin '[\"https://valid.example.com\"]' && ipfs config --json API.HTTPHeaders.Access-Control-Allow-Methods '[\"POST\"]' && ipfs config --json API.HTTPHeaders.Access-Control-Allow-Headers '[\"X-Requested-With\"]' " test_launch_ipfs_daemon test_expect_success "Companion extension is able to access RPC API even when custom Access-Control-Allow-Origin is set" ' ipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin | grep -q valid.example.com && curl -sD - -X POST -A "Mozilla" -H "Origin: chrome-extension://nibjojkomfdiaoajekhjakgkdhaomnch" "http://127.0.0.1:$API_PORT/api/v0/id" >curl_output && cat curl_output && grep "HTTP/1.1 200 OK" curl_output && grep "$PEERID" curl_output ' # https://developer.mozilla.org/en-US/docs/Glossary/Preflight_request test_expect_success "OPTIONS with preflight request to API with CORS allowlist succeeds" ' curl -svX OPTIONS -A "Mozilla" -H "Origin: https://valid.example.com" -H "Access-Control-Request-Method: POST" -H "Access-Control-Request-Headers: origin, x-requested-with" "http://127.0.0.1:$API_PORT/api/v0/id" 2>curl_output && cat curl_output ' # OPTION Response from Gateway should contain CORS headers, otherwise JS won't work test_expect_success "OPTIONS response for API with CORS allowslist looks good" ' grep "< Access-Control-Allow-Origin: https://valid.example.com" curl_output ' test_expect_success "browser is able to access API with valid Origin matching CORS allowlist" ' curl -sD - -X POST -A "Mozilla" -H "Origin: https://valid.example.com" "http://127.0.0.1:$API_PORT/api/v0/id" >curl_output && grep "HTTP/1.1 200 OK" curl_output && grep "$PEERID" curl_output ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0410-api-add.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2016 Tom O'Donnell # MIT Licensed; see the LICENSE file in this repository. # test_description="Test API add command" . lib/test-lib.sh test_init_ipfs # Verify that the API add command returns size test_launch_ipfs_daemon test_expect_success "API Add response includes size field" ' echo "hi" | curl -s -F file=@- "http://localhost:$API_PORT/api/v0/add" | grep "\"Size\": *\"11\"" ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0500-issues-and-regressions-offline.sh ================================================ #!/usr/bin/env bash test_description="Tests for various fixed issues and regressions." . lib/test-lib.sh # Tests go here test_expect_success "ipfs init with occupied input works - #2748" ' export IPFS_PATH="ipfs_path" echo "" | go-timeout 10 ipfs init && rm -rf ipfs_path ' test_init_ipfs test_expect_success "ipfs cat --help succeeds when input remains open" ' yes | go-timeout 1 ipfs cat --help ' test_expect_success "ipfs pin ls --help succeeds when input remains open" ' yes | go-timeout 1 ipfs pin ls --help ' test_expect_success "ipfs add on 1MB from stdin woks" ' random-data -size=1048576 -seed=42 | ipfs add -q > 1MB.hash ' test_expect_success "'ipfs refs -r -e \$(cat 1MB.hash)' succeeds" ' ipfs refs -r -e $(cat 1MB.hash) > refs-e.out ' test_expect_success "output of 'ipfs refs -e' links to separate blocks" ' grep "$(cat 1MB.hash) ->" refs-e.out ' test_expect_success "output of 'ipfs refs -e' contains all first level links" ' grep "$(cat 1MB.hash) ->" refs-e.out | sed -e '\''s/.* -> //'\'' | sort > refs-s.out && ipfs refs "$(cat 1MB.hash)" | sort > refs-one.out && test_cmp refs-s.out refs-one.out ' test_done ================================================ FILE: test/sharness/t0600-issues-and-regressions-online.sh ================================================ #!/usr/bin/env bash test_description="Tests for various fixed issues and regressions." . lib/test-lib.sh test_init_ipfs --empty-repo=false test_launch_ipfs_daemon # Tests go here test_expect_success "commands command with flag flags works via HTTP API - #2301" ' curl -X POST "http://$API_ADDR/api/v0/commands?flags" | grep "verbose" ' test_expect_success "ipfs refs local over HTTP API returns NDJOSN not flat - #2803" ' echo "Hello World" | ipfs add && curl -X POST "http://$API_ADDR/api/v0/refs/local" | grep "Ref" | grep "Err" ' test_expect_success "args expecting stdin don't crash when not given" ' curl -X POST "$API_ADDR/api/v0/bootstrap/add" > result ' test_expect_success "no panic traces on daemon" ' test_must_fail grep "nil pointer dereference" daemon_err ' test_expect_success "metrics work" ' curl -X POST "$API_ADDR/debug/metrics/prometheus" > pro_data && grep "ipfs_bs_cache_boxo_blockstore_cache_total" < pro_data || test_fsh cat pro_data ' test_expect_success "pin add api looks right - #3753" ' HASH=$(date +"%FT%T.%N%z" | ipfs add -q) && curl -X POST "http://$API_ADDR/api/v0/pin/add/$HASH" > pinadd_out && echo "{\"Pins\":[\"$HASH\"]}" > pinadd_exp && test_cmp pinadd_out pinadd_exp ' test_expect_success "pin add api looks right - #3753" ' curl -X POST "http://$API_ADDR/api/v0/pin/rm/$HASH" > pinrm_out && echo "{\"Pins\":[\"$HASH\"]}" > pinrm_exp && test_cmp pinrm_out pinrm_exp ' test_expect_success SOCAT "no daemon crash on improper file argument - #4003 ( test needs socat )" ' FNC=$(echo $API_ADDR | awk -F: '\''{ printf "%s:%s", $1, $2 }'\'') && printf "POST /api/v0/add?pin=true HTTP/1.1\r\nHost: $API_ADDR\r\nContent-Type: multipart/form-data; boundary=Pyw9xQLtiLPE6XcI\r\nContent-Length: 22\r\n\r\n\r\n--Pyw9xQLtiLPE6XcI\r\n" | socat STDIO tcp-connect:$FNC | grep -m1 "500 Internal Server Error" ' test_kill_ipfs_daemon test_expect_success "ipfs daemon --offline --mount fails - #2995" ' test_expect_code 1 ipfs daemon --offline --mount 2>daemon_err && grep "mount is not currently supported in offline mode" daemon_err || test_fsh cat daemon_err ' test_launch_ipfs_daemon_without_network test_expect_success "'ipfs name resolve' succeeds after ipfs id when daemon offline" ' PEERID=`ipfs key list --ipns-base=base36 -l | grep self | cut -d " " -f1` && test_check_peerid "${PEERID}" && ipfs name publish --allow-offline -Q "/ipfs/$HASH_WELCOME_DOCS" >publish_out ' test_expect_success "pubrmlish --quieter output looks good" ' echo "${PEERID}" >expected1 && test_cmp expected1 publish_out ' test_expect_success "'ipfs name resolve' succeeds" ' ipfs name resolve "$PEERID" >output ' test_expect_success "resolve output looks good" ' printf "/ipfs/%s\n" "$HASH_WELCOME_DOCS" >expected2 && test_cmp expected2 output ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness/t0701-delegated-routing-reframe/FindProvidersRequest ================================================ {"FindProvidersRequest":{"Key":{"/":"bafybeigvgzoolc3drupxhlevdp2ugqcrbcsqfmcek2zxiw5wctk3xjpjwy"}}} ================================================ FILE: test/sharness/t0701-delegated-routing-reframe/FindProvidersResponse ================================================ {"FindProvidersResponse":{"Providers":[{"Node":{"peer":{"ID":{"/":{"bytes":"EiAngCqwSSL46hQ5+DWaJsZ1SPV2RwrqwID/OEuj5Rdgqw"}},"Multiaddresses":[{"/":{"bytes":"NiJwZWVyLmlwZnMtZWxhc3RpYy1wcm92aWRlci1hd3MuY29tBgu43QM"}}]}},"Proto":[{"2304":{}}]}]}} ================================================ FILE: test/sharness/t0702-delegated-routing-http/FindProvidersResponse ================================================ {"Providers":[{"Protocol":"transport-bitswap","Schema":"bitswap","ID":"12D3KooWARYacCc6eoCqvsS9RW9MA2vo51CV75deoiqssx3YgyYJ","Addrs":["/ip4/0.0.0.0/tcp/4001","/ip4/0.0.0.0/tcp/4002"]}]} ================================================ FILE: test/sharness/t0800-blake3.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2020 Claudia Richoux # MIT Licensed; see the LICENSE file in this repository. # test_description="Test blake3 mhash support" . lib/test-lib.sh test_init_ipfs # the blake3 hash of "foo\n" in UTF8 (which is what comes out of echo when you pipe into `ipfs`) starts with "49dc870df1de7fd60794cebce449f5ccdae575affaa67a24b62acb03e039db92" # without the newline it's "04e0bb39f30b1a3feb89f536c93be15055482df748674b00d26e5a75777702e9". so if you start seeing these values that's your problem BLAKE3RAWCID32BYTE="bafkr4icj3sdq34o6p7lapfgoxtset5om3lsxll72uz5cjnrkzmb6aoo3si" BLAKE3RAWCID64BYTE="bafkr4qcj3sdq34o6p7lapfgoxtset5om3lsxll72uz5cjnrkzmb6aoo3sknmbprpe27pbfrb67tonydgfot5ixuq4skiva76ppbgpjlzc4ua4" BLAKE3RAWCID128BYTE="bafkr5aabjhoiodpr3z75mb4uz26oispvztnok5np7kthujfwflfqhybz3ojjvqf6f4tl54eweh36nzxamyv2pvc6sdsjjcud7z54ez5fpelsqdtax2k3rvuq3wdl5a4blxv3gvsroa3nakfzzknamhu2apf3vvytyiobabrn2bfnfajq66ikjy5lewsp5jyddsg5l7u3emr2ancimryay" ### block tests, including for various sizes of hash ### test_expect_success "putting a block with an mhash blake3 succeeds (default 32 bytes)" ' HASH=$(echo "foo" | ipfs block put --mhtype=blake3 --cid-codec=raw | tee actual_out) && test $BLAKE3RAWCID32BYTE = "$HASH" ' test_expect_success "block get output looks right" ' ipfs block get $BLAKE3RAWCID32BYTE > blk_get_out && echo "foo" > blk_get_exp && test_cmp blk_get_exp blk_get_out ' test_expect_success "putting a block with an mhash blake3 succeeds: 64 bytes" ' HASH=$(echo "foo" | ipfs block put --mhtype=blake3 --mhlen=64 --cid-codec=raw | tee actual_out) && test $BLAKE3RAWCID64BYTE = "$HASH" ' test_expect_success "64B block get output looks right" ' ipfs block get $BLAKE3RAWCID64BYTE > blk_get_out && echo "foo" > blk_get_exp && test_cmp blk_get_exp blk_get_out ' test_expect_success "putting a block with an mhash blake3 succeeds: 128 bytes" ' HASH=$(echo "foo" | ipfs block put --mhtype=blake3 --mhlen=128 --cid-codec=raw | tee actual_out) && test $BLAKE3RAWCID128BYTE = "$HASH" ' test_expect_success "32B block get output looks right" ' ipfs block get $BLAKE3RAWCID128BYTE > blk_get_out && echo "foo" > blk_get_exp && test_cmp blk_get_exp blk_get_out ' ### dag tests ### test_expect_success "dag put works with blake3" ' HASH=$(echo "foo" | ipfs dag put --input-codec=raw --store-codec=raw --hash=blake3 | tee actual_out) && test $BLAKE3RAWCID32BYTE = "$HASH" ' test_expect_success "dag get output looks right" ' ipfs dag get --output-codec=raw $BLAKE3RAWCID32BYTE > dag_get_out && echo "foo" > dag_get_exp && test_cmp dag_get_exp dag_get_out ' ### add and cat tests ### test_expect_success "adding a file with just foo in it to ipfs" ' echo "foo" > afile && HASH=$(ipfs add -q --hash=blake3 --raw-leaves afile | tee actual_out) && test $BLAKE3RAWCID32BYTE = "$HASH" ' test_expect_success "catting it" ' ipfs cat $BLAKE3RAWCID32BYTE > cat_out && echo "foo" > cat_exp && test_cmp cat_exp cat_out ' test_done ================================================ FILE: test/sharness/x0601-pin-fail-test.sh ================================================ #!/usr/bin/env bash # # Copyright (c) 2016 Jeromy Johnson # MIT Licensed; see the LICENSE file in this repository. # test_description="Test very large number of pins" . lib/test-lib.sh test_init_ipfs test_launch_ipfs_daemon test_expect_success "pre-test setup" ' printf "" > pins && ipfs pin ls --type=recursive -q > rec_pins_before ' for i in `seq 9000` do test_expect_success "ipfs add (and pin) a file" ' echo $i | ipfs add -q >> pins ' done test_expect_success "get pinset afterwards" ' ipfs pin ls --type=recursive -q | sort > rec_pins_after && cat pins rec_pins_before | sort | uniq > exp_pins_after && test_cmp rec_pins_after exp_pins_after ' test_kill_ipfs_daemon test_done ================================================ FILE: test/sharness_test_coverage_helper.sh ================================================ #!/bin/sh USAGE="$0 [-h] [-v]" usage() { echo "$USAGE" echo " Print sharness test coverage" echo " Options:" echo " -h|--help: print this usage message and exit" echo " -v|--verbose: print logs of what happens" exit 0 } log() { test -z "$VERBOSE" || echo "->" "$@" } die() { printf >&2 "fatal: %s\n" "$@" exit 1 } # get user options while [ "$#" -gt "0" ]; do # get options arg="$1" shift case "$arg" in -h|--help) usage ;; -v|--verbose) VERBOSE=1 ;; -*) die "unrecognised option: '$arg'\n$USAGE" ;; *) die "too many arguments\n$USAGE" ;; esac done log "Create temporary directory" DATE=$(date +"%Y-%m-%dT%H:%M:%SZ") TMPDIR=$(mktemp -d "/tmp/coverage_helper.$DATE.XXXXXX") || die "could not 'mktemp -d /tmp/coverage_helper.$DATE.XXXXXX'" log "Grep the sharness tests for ipfs commands" CMD_RAW="$TMPDIR/ipfs_cmd_raw.txt" git grep -n -E '\Wipfs\W' -- sharness/t*-*.sh >"$CMD_RAW" || die "Could not grep ipfs in the sharness tests" grep_out() { pattern="$1" src="$TMPDIR/ipfs_cmd_${2}.txt" dst="$TMPDIR/ipfs_cmd_${3}.txt" desc="$4" log "Remove $desc" egrep -v "$pattern" "$src" >"$dst" || die "Could not remove $desc" } grep_out 'test_expect_.*ipfs' raw expect "test_expect_{success,failure} lines" grep_out '^[^:]+:[^:]+:\s*#' expect comment "comments" grep_out 'test_description=' comment desc "test_description lines" grep_out '^[^:]+:[^:]+:\s*\w+="[^"]*"\s*(\&\&)?\s*$' desc def "variable definition lines" grep_out '^[^:]+:[^:]+:\s*e?grep\W[^|]*\Wipfs' def grep "grep lines" grep_out '^[^:]+:[^:]+:\s*cat\W[^|]*\Wipfs' grep cat "cat lines" grep_out '^[^:]+:[^:]+:\s*rmdir\W[^|]*\Wipfs' cat rmdir "rmdir lines" grep_out '^[^:]+:[^:]+:\s*echo\W[^|]*\Wipfs' cat echo "echo lines" grep_in() { pattern="$1" src="$TMPDIR/ipfs_cmd_${2}.txt" dst="$TMPDIR/ipfs_cmd_${3}.txt" desc="$4" log "Keep $desc" egrep "$pattern" "$src" >"$dst" } grep_in '\Wipfs\W.*/ipfs/' echo slash_in1 "ipfs.*/ipfs/" grep_in '/ipfs/.*\Wipfs\W' echo slash_in2 "/ipfs/.*ipfs" grep_out '/ipfs/' echo slash "/ipfs/" grep_in '\Wipfs\W.*\.ipfs' slash dot_in1 "ipfs.*\.ipfs" grep_in '\.ipfs.*\Wipfs\W' slash dot_in2 "\.ipfs.*ipfs" grep_out '\.ipfs' slash dot ".ipfs" log "Print result" CMD_RES="$TMPDIR/ipfs_cmd_result.txt" for f in dot slash_in1 slash_in2 dot_in1 dot_in2 do fname="$TMPDIR/ipfs_cmd_${f}.txt" cat "$fname" || die "Could not cat '$fname'" done | sort | uniq >"$CMD_RES" || die "Could not write '$CMD_RES'" log "Get all the ipfs commands from 'ipfs commands'" CMD_CMDS="$TMPDIR/commands.txt" ipfs commands --flags >"$CMD_CMDS" || die "'ipfs commands' failed" # Portable function to reverse lines in a file reverse() { if type tac >/dev/null then tac "$@" else tail -r "$@" fi } log "Match the test line commands with the commands they use" GLOBAL_REV="$TMPDIR/global_results_reversed.txt" process_command() { ipfs="$1" cmd="$2" sub1="$3" sub2="$4" sub3="$5" if test -n "$cmd" then CMD_OUT="$TMPDIR/res_${ipfs}_${cmd}" PATTERN="$ipfs(\W.*)*\W$cmd" NAME="$ipfs $cmd" if test -n "$sub1" then CMD_OUT="${CMD_OUT}_${sub1}" PATTERN="$PATTERN(\W.*)*\W$sub1" NAME="$NAME $sub1" if test -n "$sub2" then CMD_OUT="${CMD_OUT}_${sub2}" PATTERN="$PATTERN(\W.*)*\W$sub2" NAME="$NAME $sub2" if test -n "$sub3" then CMD_OUT="${CMD_OUT}_${sub3}" PATTERN="$PATTERN(\W.*)*\W$sub3" NAME="$NAME $sub3" fi fi fi egrep "$PATTERN" "$CMD_RES" >"$CMD_OUT.txt" reverse "$CMD_OUT.txt" | sed -e 's/^sharness\///' | cut -d- -f1 | uniq -c >>"$GLOBAL_REV" fi } reverse "$CMD_CMDS" | while read -r line do LONG_CMD=$(echo "$line" | cut -d/ -f1) SHORT_CMD=$(expr "$line" : "[^/]*/*\(.*\)") log "Processing $LONG_CMD" process_command $LONG_CMD LONG_NAME="$NAME" log "Processing $SHORT_CMD" process_command $SHORT_CMD SHORT_NAME="$NAME" test -n "$SHORT_CMD" && echo "$SHORT_NAME" >>"$GLOBAL_REV" test "$LONG_CMD" != "ipfs" && echo "$LONG_NAME" >>"$GLOBAL_REV" echo >>"$GLOBAL_REV" done # The following will allow us to check that # we are properly excluding enough stuff using: # diff -u ipfs_cmd_result.txt cmd_found.txt log "Get all the line commands that matched" CMD_FOUND="$TMPDIR/cmd_found.txt" cat $TMPDIR/res_*.txt | sort -n | uniq >"$CMD_FOUND" log "Print results" reverse "$GLOBAL_REV" # Remove temp directory... ================================================ FILE: test/unit/.gitignore ================================================ gotest.json gotest.junit.xml ================================================ FILE: test/unit/Rules.mk ================================================ include mk/header.mk CLEAN += $(d)/gotest.json $(d)/gotest.junit.xml # Convert gotest.json (produced by test_unit) to JUnit XML format $(d)/gotest.junit.xml: test/bin/gotestsum $(d)/gotest.json gotestsum --no-color --junitfile $@ --raw-command cat $(@D)/gotest.json include mk/footer.mk ================================================ FILE: thirdparty/README.md ================================================ packages under this directory _must not_ import packages under `ipfs/kubo` that are not also under `thirdparty`. ================================================ FILE: thirdparty/unit/unit.go ================================================ package unit import "fmt" type Information int64 const ( _ Information = iota // ignore first value by assigning to blank identifier KB = 1 << (10 * iota) MB GB TB PB EB ) func (i Information) String() string { tmp := int64(i) // default d := tmp symbol := "B" switch { case i > EB: d = tmp / EB symbol = "EB" case i > PB: d = tmp / PB symbol = "PB" case i > TB: d = tmp / TB symbol = "TB" case i > GB: d = tmp / GB symbol = "GB" case i > MB: d = tmp / MB symbol = "MB" case i > KB: d = tmp / KB symbol = "KB" } return fmt.Sprintf("%d %s", d, symbol) } ================================================ FILE: thirdparty/unit/unit_test.go ================================================ package unit import "testing" // and the award for most meta goes to... func TestByteSizeUnit(t *testing.T) { if 1*KB != 1*1024 { t.Fatal(1 * KB) } if 1*MB != 1*1024*1024 { t.Fail() } if 1*GB != 1*1024*1024*1024 { t.Fail() } if 1*TB != 1*1024*1024*1024*1024 { t.Fail() } if 1*PB != 1*1024*1024*1024*1024*1024 { t.Fail() } if 1*EB != 1*1024*1024*1024*1024*1024*1024 { t.Fail() } } ================================================ FILE: thirdparty/verifbs/verifbs.go ================================================ package verifbs import ( "context" bstore "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/verifcid" blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" ) type VerifBSGC struct { bstore.GCBlockstore } func (bs *VerifBSGC) Put(ctx context.Context, b blocks.Block) error { if err := verifcid.ValidateCid(verifcid.DefaultAllowlist, b.Cid()); err != nil { return err } return bs.GCBlockstore.Put(ctx, b) } func (bs *VerifBSGC) PutMany(ctx context.Context, blks []blocks.Block) error { for _, b := range blks { if err := verifcid.ValidateCid(verifcid.DefaultAllowlist, b.Cid()); err != nil { return err } } return bs.GCBlockstore.PutMany(ctx, blks) } func (bs *VerifBSGC) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) { if err := verifcid.ValidateCid(verifcid.DefaultAllowlist, c); err != nil { return nil, err } return bs.GCBlockstore.Get(ctx, c) } type VerifBS struct { bstore.Blockstore } func (bs *VerifBS) Put(ctx context.Context, b blocks.Block) error { if err := verifcid.ValidateCid(verifcid.DefaultAllowlist, b.Cid()); err != nil { return err } return bs.Blockstore.Put(ctx, b) } func (bs *VerifBS) PutMany(ctx context.Context, blks []blocks.Block) error { for _, b := range blks { if err := verifcid.ValidateCid(verifcid.DefaultAllowlist, b.Cid()); err != nil { return err } } return bs.Blockstore.PutMany(ctx, blks) } func (bs *VerifBS) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) { if err := verifcid.ValidateCid(verifcid.DefaultAllowlist, c); err != nil { return nil, err } return bs.Blockstore.Get(ctx, c) } ================================================ FILE: tracing/doc.go ================================================ // Package tracing contains the tracing logic for go-ipfs, including configuring the tracer and // helping keep consistent naming conventions across the stack. // // NOTE: Tracing is currently experimental. Span names may change unexpectedly, spans may be removed, // and backwards-incompatible changes may be made to tracing configuration, options, and defaults. // // Tracing is configured through environment variables, as consistent with the OpenTelemetry spec as possible: // // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/sdk-environment-variables.md // // OTEL_TRACES_EXPORTER: a comma-separated list of exporters: // - otlp // - zipkin // - file // // Different exporters have their own set of environment variables, depending on the exporter. These are typically // standard environment variables. Some common ones: // // OTLP HTTP/gRPC: // // - OTEL_EXPORTER_OTLP_PROTOCOL // one of [grpc, http/protobuf] // default: grpc // - OTEL_EXPORTER_OTLP_ENDPOINT // - OTEL_EXPORTER_OTLP_CERTIFICATE // - OTEL_EXPORTER_OTLP_HEADERS // - OTEL_EXPORTER_OTLP_COMPRESSION // - OTEL_EXPORTER_OTLP_TIMEOUT // // Zipkin: // // - OTEL_EXPORTER_ZIPKIN_ENDPOINT // // File: // // - OTEL_EXPORTER_FILE_PATH // file path to write JSON traces // default: `$PWD/traces.json` // // For example, if you run a local IPFS daemon, you can use the jaegertracing/all-in-one Docker image to run // a full Jaeger stack and configure Kubo to publish traces to it: // // docker run -d --rm --name jaeger \ // -e COLLECTOR_OTLP_ENABLED=true \ // -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \ // -p 5775:5775/udp \ // -p 6831:6831/udp \ // -p 6832:6832/udp \ // -p 5778:5778 \ // -p 16686:16686 \ // -p 14250:14250 \ // -p 14268:14268 \ // -p 14269:14269 \ // -p 4317:4317 \ // -p 4318:4318 \ // -p 9411:9411 \ // jaegertracing/all-in-one // OTEL_EXPORTER_OTLP_INSECURE=true OTEL_TRACES_EXPORTER=otlp ipfs daemon --init // // # In this example the Jaeger UI is available at http://localhost:16686. // // Span names follow a convention of ., some examples: // // - component=Gateway + span=Request -> Gateway.Request // - component=CoreAPI.PinAPI + span=Verify.CheckPin -> CoreAPI.PinAPI.Verify.CheckPin // // We follow the OpenTelemetry convention of using whatever TracerProvider is registered globally. package tracing ================================================ FILE: tracing/tracing.go ================================================ package tracing import ( "context" "fmt" "github.com/ipfs/boxo/tracing" version "github.com/ipfs/kubo" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.4.0" traceapi "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace/noop" ) // shutdownTracerProvider adds a shutdown method for tracer providers. // // Note that this doesn't directly use the provided TracerProvider interface // to avoid build breaking go-ipfs if new methods are added to it. type shutdownTracerProvider interface { traceapi.TracerProvider Tracer(instrumentationName string, opts ...traceapi.TracerOption) traceapi.Tracer Shutdown(ctx context.Context) error } // noopShutdownTracerProvider adds a no-op Shutdown method to a TracerProvider. type noopShutdownTracerProvider struct{ traceapi.TracerProvider } func (n *noopShutdownTracerProvider) Shutdown(ctx context.Context) error { return nil } // NewTracerProvider creates and configures a TracerProvider. func NewTracerProvider(ctx context.Context) (shutdownTracerProvider, error) { exporters, err := tracing.NewSpanExporters(ctx) if err != nil { return nil, err } if len(exporters) == 0 { return &noopShutdownTracerProvider{TracerProvider: noop.NewTracerProvider()}, nil } options := []trace.TracerProviderOption{} for _, exporter := range exporters { options = append(options, trace.WithBatcher(exporter)) } r, err := resource.Merge( resource.Default(), resource.NewSchemaless( semconv.ServiceNameKey.String("Kubo"), semconv.ServiceVersionKey.String(version.CurrentVersionNumber), ), ) if err != nil { return nil, err } options = append(options, trace.WithResource(r)) return trace.NewTracerProvider(options...), nil } // Span starts a new span using the standard IPFS tracing conventions. func Span(ctx context.Context, componentName string, spanName string, opts ...traceapi.SpanStartOption) (context.Context, traceapi.Span) { return otel.Tracer("Kubo").Start(ctx, fmt.Sprintf("%s.%s", componentName, spanName), opts...) } ================================================ FILE: version.go ================================================ package ipfs import ( "fmt" "runtime" "github.com/ipfs/kubo/core/commands/cmdutils" ) // CurrentCommit is the current git commit, this is set as a ldflag in the Makefile. var CurrentCommit string // taggedRelease is set via ldflag when building from a version-tagged commit // with a clean tree. When set, the commit hash is omitted from the libp2p // identify agent version and the HTTP user agent, since the version number // already identifies the exact source. var taggedRelease string // CurrentVersionNumber is the current application's version literal. const CurrentVersionNumber = "0.41.0-dev" const ApiVersion = "/kubo/" + CurrentVersionNumber + "/" //nolint // RepoVersion is the version number that we are currently expecting to see. const RepoVersion = 18 // GetUserAgentVersion is the libp2p user agent used by go-ipfs. func GetUserAgentVersion() string { // For tagged release builds with a clean tree, the commit hash is // redundant since the version number identifies the exact source. commit := CurrentCommit if taggedRelease != "" { commit = "" } userAgent := "kubo/" + CurrentVersionNumber if commit != "" { userAgent += "/" + commit } if userAgentSuffix != "" { userAgent += "/" + userAgentSuffix } return cmdutils.CleanAndTrim(userAgent) } var userAgentSuffix string func SetUserAgentSuffix(suffix string) { userAgentSuffix = cmdutils.CleanAndTrim(suffix) } type VersionInfo struct { Version string Commit string Repo string System string Golang string } func GetVersionInfo() *VersionInfo { return &VersionInfo{ Version: CurrentVersionNumber, Commit: CurrentCommit, Repo: fmt.Sprint(RepoVersion), System: runtime.GOARCH + "/" + runtime.GOOS, // TODO: Precise version here Golang: runtime.Version(), } } ================================================ FILE: version_test.go ================================================ package ipfs import ( "testing" "github.com/stretchr/testify/assert" ) // TestGetUserAgentVersion verifies the user agent string used in libp2p // identify and HTTP requests. Tagged release builds (where the commit matches // the tag) skip the commit hash from the agent version, since the version // number already identifies the exact source. func TestGetUserAgentVersion(t *testing.T) { origCommit := CurrentCommit origTagged := taggedRelease origSuffix := userAgentSuffix t.Cleanup(func() { CurrentCommit = origCommit taggedRelease = origTagged userAgentSuffix = origSuffix }) tests := []struct { name string commit string tagged string suffix string expected string }{ // dev builds without ldflags { name: "no commit, no suffix", expected: "kubo/" + CurrentVersionNumber, }, // dev builds with commit set via ldflags { name: "with commit", commit: "abc1234", expected: "kubo/" + CurrentVersionNumber + "/abc1234", }, { name: "with suffix, no commit", suffix: "test-suffix", expected: "kubo/" + CurrentVersionNumber + "/test-suffix", }, { name: "with commit and suffix", commit: "abc1234", suffix: "test-suffix", expected: "kubo/" + CurrentVersionNumber + "/abc1234/test-suffix", }, // tagged release builds: commit is redundant because the version // number already maps to an exact git tag, so it is omitted to // save bytes in identify and HTTP user-agent headers. { name: "tagged release ignores commit", commit: "abc1234", tagged: "1", expected: "kubo/" + CurrentVersionNumber, }, { name: "tagged release with suffix ignores commit", commit: "abc1234", tagged: "1", suffix: "test-suffix", expected: "kubo/" + CurrentVersionNumber + "/test-suffix", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { CurrentCommit = tt.commit taggedRelease = tt.tagged SetUserAgentSuffix(tt.suffix) assert.Equal(t, tt.expected, GetUserAgentVersion()) }) } }